diff --git a/Makefile b/Makefile index d7659622..ae7b48e6 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ SANDBOXD_VERSION ?= $(shell git describe --tags --match 'v*' --always --dirty) EXTRACT_IMAGE ?= $(BOOT_IMAGE) -# The parent workspace's go.work excludes these modules; GOWORK=off keeps -# local invocations identical to CI. +# The parent workspace's go.work includes these modules; GOWORK=off keeps +# local invocations identical to CI, which has no workspace. GO_MODULES := protocol/wire sandboxd sdk/go e2e mcp GO_OSES := linux darwin diff --git a/README.md b/README.md index f6651865..eb23013a 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ performance) — source in writable), signed preview URLs, the HTTP-upgrade byte relay to silkd, usage + audit journals, /metrics, reap + restart reconcile, memberlist mesh with redirect placement -- `sdk/go/` — Go SDK (stdlib-only): `Connect/New/Lookup`, `Exec/Run`, files, +- `sdk/go/` — Go SDK: `Connect/New/Lookup`, `Exec/Run`, files, `Push/Pull`, sessions, `Find/Replace`, `Watch`, git verbs, `OpenPty`, `Fork/Hibernate/Promote/Checkpoint`, `DialPort/ProxyPort/PreviewURL`, `StartLsp`, `Spawn/Ps/Kill/Logs/Attach`; `protocol/wire` carries the frame @@ -87,8 +87,9 @@ make silkd-image # silkd release binary in a scratch carrier image make images # base + python images against the local boot + silkd images ``` -The parent workspace's `go.work` excludes these modules; the Makefile forces -`GOWORK=off` so local runs match CI. silkd's integration tests spawn real +The parent workspace's `go.work` includes these modules, which is why the +Makefile forces `GOWORK=off`: CI has no workspace, and a workspace build +resolves sibling checkouts instead of the pinned versions. silkd's integration tests spawn real processes — run them in a Linux container too (`docker run rust:1 … cargo test`) before touching platform-sensitive paths; macOS green alone has hidden Linux-only breakage before. diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index bae1325a..22eb62b0 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -43,6 +43,7 @@ impl Marks { } pub fn run() -> ! { + let mut marks = Marks::new(); // Best-effort: if devtmpfs fails there is no console either; later // failures then power off silently, which is still the right end state. let _ = sys::mount( @@ -65,7 +66,7 @@ pub fn run() -> ! { Ok(cfg) => cfg, Err(err) => sys::fatal(&err, cfg::debug_requested(&cmdline)), }; - let mut marks = Marks::new(); + marks.mark("early"); if let Err(err) = assemble(&cfg, &mut marks) { sys::fatal(&err, cfg.debug); } @@ -158,8 +159,9 @@ fn persist_network(cfg: &BootCfg) { eprintln!("sandbox-init: WARN: mkdir {dir}: {err}"); return; } - for ip in &cfg.ips { - let Some(mac) = wait_nic_mac(&ip.device, NIC_TIMEOUT) else { + let devices: Vec<&str> = cfg.ips.iter().map(|ip| ip.device.as_str()).collect(); + for (ip, mac) in cfg.ips.iter().zip(wait_nic_macs(&devices, NIC_TIMEOUT)) { + let Some(mac) = mac else { eprintln!( "sandbox-init: WARN: NIC {} not found, static config skipped", ip.device @@ -173,23 +175,30 @@ fn persist_network(cfg: &BootCfg) { } } -fn wait_nic_mac(device: &str, timeout: Duration) -> Option { - let path = format!("/sys/class/net/{device}/address"); +/// Resolves every NIC's MAC in one sysfs sweep per poll iteration, against one +/// shared deadline — a missing NIC must cost the timeout once, not once each. +fn wait_nic_macs(devices: &[&str], timeout: Duration) -> Vec> { let deadline = Instant::now() + timeout; + let mut found: Vec> = vec![None; devices.len()]; loop { - if let Ok(mac) = fs::read_to_string(&path) { - let mac = mac.trim_end().to_string(); - if !mac.is_empty() && mac != "00:00:00:00:00:00" { - return Some(mac); + for (slot, device) in found.iter_mut().zip(devices) { + if slot.is_none() { + *slot = read_nic_mac(device); } } - if Instant::now() >= deadline { - return None; + if found.iter().all(Option::is_some) || Instant::now() >= deadline { + return found; } std::thread::sleep(POLL_INTERVAL); } } +fn read_nic_mac(device: &str) -> Option { + let mac = fs::read_to_string(format!("/sys/class/net/{device}/address")).ok()?; + let mac = mac.trim_end(); + (!mac.is_empty() && mac != "00:00:00:00:00:00").then(|| mac.to_string()) +} + fn mkdir_all(path: &str) -> Result<(), String> { fs::create_dir_all(path).map_err(|err| format!("mkdir {path}: {err}")) } diff --git a/boot/init/src/cfg.rs b/boot/init/src/cfg.rs index 91c021a7..aa24d1ef 100644 --- a/boot/init/src/cfg.rs +++ b/boot/init/src/cfg.rs @@ -243,7 +243,6 @@ mod tests { assert!(!debug_requested("a sandbox.debug=0 b")); assert!(!debug_requested("a sandbox.debugger b")); assert!(!debug_requested("")); - // Last occurrence wins, like parse(). assert!(!debug_requested("sandbox.debug=1 x sandbox.debug=0")); assert!(debug_requested("sandbox.debug=0 x sandbox.debug")); } @@ -298,7 +297,6 @@ mod tests { #[test] fn parse_cocoon_ip_param() { - // Exact shape BuildIPParams emits, with both DNS servers. let cfg = parse( "cocoon.layers=l0 cocoon.cow=cow \ ip=172.20.100.5::172.20.100.1:255.255.255.0:vm1:eth0:off:8.8.8.8:8.8.4.4", diff --git a/docs/langchain.md b/docs/langchain.md index 04481b5d..aafdb362 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -19,7 +19,7 @@ schemas, sync-native with `asyncio.to_thread` async bridges): | tool | what it does | |---|---| -| `sandbox_exec` | run a shell command; stdout/stderr/exit code; disk state persists across calls | +| `sandbox_exec` | run a shell command, cut off after 5 minutes; stdout/stderr/exit code; disk state persists across calls | | `sandbox_write_file` | write a text file (atomic on the guest) | | `sandbox_read_file` | read a text file | | `sandbox_list_dir` | list a directory as JSON | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index dbe2ade2..542f2aa6 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -500,8 +500,9 @@ silently. Folding rules: billable compute seconds per sandbox = Σ(hibernate→wake) minus the archived span; archived storage seconds = Σ(archive→unarchive/archive_delete), a cheaper store tier than a hibernated VM's RAM. An interval left open by a crash clamps to the claim's -deadline, and the next reconcile emits `reap` for claims it drops. The `vm` -name joins cocoon's machine-level metering ledger for audit cross-checks. +deadline; reconcile records nothing for the claims it drops, so a restart +leaves those intervals without a terminal event. The `vm` name joins +cocoon's machine-level metering ledger for audit cross-checks. ## GET /v1/sandboxes/{id}/agent diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 8848d0e4..99e3c75e 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -237,6 +237,13 @@ records, while `client.checkpoint(id)` creates an entry-node-bound handle for an already known id without listing. `new()` follows an owner redirect and may heal a missing record locally; `delete()` acts on the handle's bound node. +`delete()` also asks every peer that node currently sees to drop any replica a +heal pulled — best-effort eventual cleanup, not a fleet-wide revocation. A peer +that misses the broadcast (offline, partitioned, or joined later) keeps serving +branches from its replica until the node's `checkpoint_ttl_hours` ages it out; +with that TTL at its default of 0 (keep forever), an unreachable peer's replica +has no cleanup bound at all. + ## Language servers (LSP) ```python @@ -244,6 +251,7 @@ lsp = sb.start_lsp("python", "/work") # flavor image provides the server stream = lsp.request() # JSON-RPC byte stream (frame it yourself) # ... speak Content-Length-framed JSON-RPC over stream.send()/recv() ... stream.close() # ends the session and reaps the server +lsp.close() # same as lsp.stop(); both are context-managers ``` `start_lsp` spawns the language server the flavor image ships for the @@ -319,7 +327,7 @@ A session is a real persistent shell: cwd, env and shell state survive across calls. ```python -sess = sb.session(cwd="/work", env={"PATH": "..."}) +sess = sb.session(cwd="/work", env={"PATH": "..."}) # context-manager sess.exec("export", "MARK=1") # persists sess.exec("sh", "-c", "echo $MARK") # "1\n" sess.close() @@ -435,7 +443,8 @@ zero. - `SilkdError(kind, message)` — typed guest failure; `kind` is `bad_request` / `not_found` / `unimplemented` / `internal` - `ExitError(code, stderr, stdout)` — non-zero exit from `exec` -- `ProtocolError` — broken stream +- `ProtocolError` — broken stream (EOF, oversized or undecodable frame) +- `StreamTimeout` — a socket operation hit the client `timeout`; retry the call ```python try: diff --git a/docs/sdk.md b/docs/sdk.md index ed0963b0..577aa31d 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -4,9 +4,10 @@ import sandbox "github.com/cocoonstack/sandbox/sdk/go" ``` -The SDK is stdlib-only. One `Client` talks to one entry node; sandbox -handles dial their owning node directly, so a client works unchanged against -a single node or a cluster. +The SDK has no third-party dependencies (only the sibling `protocol/wire` +module). One `Client` talks to one entry node; sandbox handles dial their +owning node directly, so a client works unchanged against a single node or a +cluster. ## A complete example @@ -529,5 +530,7 @@ if errors.As(err, &e) && e.Kind == wire.KindUnimplemented { } ``` -Context cancellation is honored on every call: canceling the ctx closes the -underlying connection and the call returns `ctx.Err()`. +Context cancellation is honored on every call that takes a ctx: canceling it +closes the underlying connection. Data-plane calls return `ctx.Err()` +directly; control-plane verbs return a `*url.Error` wrapping it, so test with +`errors.Is`, never `==`. `Close` is the exception: it takes no ctx. diff --git a/docs/silkd.md b/docs/silkd.md index befa4be9..4f096a58 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -35,12 +35,12 @@ a frame only one side can parse fails CI. | sessions | `session_create {id?, cwd?, env?}` / `session_list` / `session_rm {id}` | a session is a real persistent bash; `exec` with `session` runs inside it. Idle sessions are reaped after 30 minutes | | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | streaming both directions; write commits atomically via temp+rename and inherits an overwritten file's mode; `fs_list` streams 4096-entry batches | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar | -| search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files | +| search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count | | watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | | pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | a shell under a pseudo-terminal; output as `stdout` frames, input as `stdin` frames, exit terminal. PTYs register in the proc table like any exec | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | structured results (porcelain-v2 status, commit hash, branch list). `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways (`data_end` half-closes the guest socket); the server closing ends the stream with `done`. Works on both lanes — the no-network lane's only way in | -| lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early | +| lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early, and a server nothing attaches within 5 minutes is reaped | | misc | `info` | `{version, proto, uptime_secs, procs, sessions}` — the in-guest readiness probe (sandboxd consumes it; no SDK surface). Distinct from the control plane's `GET /v1/info`, which reports node pools and claims | ## Error kinds diff --git a/e2e/cmd/androidsmoke/main.go b/e2e/cmd/androidsmoke/main.go index 63448bd3..0456384e 100644 --- a/e2e/cmd/androidsmoke/main.go +++ b/e2e/cmd/androidsmoke/main.go @@ -96,7 +96,7 @@ func initTree(ctx context.Context, sb *sandbox.Sandbox) error { return nil } if time.Now().After(deadline) { - return fmt.Errorf("framework never came up:\n%s", tail(out)) + return fmt.Errorf("framework never came up:\n%s", head(out)) } time.Sleep(3 * time.Second) } @@ -151,7 +151,7 @@ func dialAdb(ctx context.Context, sb *sandbox.Sandbox) (string, error) { return reply, nil } -func tail(s string) string { +func head(s string) string { lines := strings.Split(strings.TrimSpace(s), "\n") if len(lines) > 12 { lines = lines[:12] diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 3a96f6ba..b4ae082a 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -49,8 +49,6 @@ func TestEndToEnd(t *testing.T) { }) t.Run("refill then warm claim", func(t *testing.T) { - // Warm-hit = zero VM ops is proven by the pool unit tests; asserting - // create counts here would race the background refill ticker. waitFor(t, func() bool { infos, _ := stack.mgr.Info() return len(infos) == 1 && infos[0].Warm >= 1 @@ -102,8 +100,7 @@ func TestForkEndToEnd(t *testing.T) { t.Fatalf("child %d exec: %q, %v", i, out, err) } } - // Children are independent claims: releasing one leaves the sibling and - // the parent alive. + if err := children[0].Close(); err != nil { t.Fatalf("close child 0: %v", err) } @@ -131,8 +128,7 @@ func TestPromoteEndToEnd(t *testing.T) { if tpl.ContentDigest == "" { t.Fatal("Promote returned an empty content digest") } - // Both claim surfaces must work: the owner-bound handle and the - // name-based Client call on the (single) node that holds the template. + child, err := tpl.New(t.Context()) if err != nil { t.Fatalf("claim via template handle: %v", err) @@ -206,9 +202,6 @@ func TestCheckpointEndToEnd(t *testing.T) { } } -// TestTwoTenantFlow drives two tenants through the real stack: each claims -// under its own token, quotas bind per tenant, checkpoint listings are -// isolated, and operator surfaces refuse tenant tokens with 403. func TestTwoTenantFlow(t *testing.T) { tenants := []config.TenantSpec{ {Name: "acme", Token: "acme-tok", MaxClaims: 1}, @@ -354,9 +347,6 @@ func TestVolumesEndToEnd(t *testing.T) { } } -// TestWritableVolumeEndToEnd drives one writable claim through the whole -// stack: the SDK's mode reaches the engine as a writable attach, a live writer -// refuses every other claim on the name, and release unmounts before removal. func TestWritableVolumeEndToEnd(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -389,9 +379,8 @@ func TestWritableVolumeEndToEnd(t *testing.T) { } } -// TestVolumeModeWireShape pins the claim reply's volume bytes independently of -// the SDK mirror. The read-only leg runs second on purpose: it is admitted -// only because the writer's release cleared the dirty marker. +// The read-only leg runs second: it is admitted only once the writer's +// release has cleared the dirty marker. func TestVolumeModeWireShape(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -448,10 +437,6 @@ func TestClaimRefRoundTrip(t *testing.T) { } } -// TestAttachOnlyVolumeEndToEnd drives one attach-only writable claim through -// the whole stack: the device is attached writable and nothing else happens — -// no mount, no marker, no unmount at release — while admission still excludes -// every other claim on the name, which is what protects third parties. func TestAttachOnlyVolumeEndToEnd(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -487,9 +472,6 @@ func TestAttachOnlyVolumeEndToEnd(t *testing.T) { assertNoDirtyMarker(t, scratch, "release") } -// TestAttachOnlyVolumeWireShape pins both claim replies' volume bytes: an -// attach-only entry echoes without a mount, the eager entry is unchanged, and -// the request flag never rides back in either. func TestAttachOnlyVolumeWireShape(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -570,7 +552,7 @@ func startStack(t *testing.T, apiToken string, pools ...config.PoolSpec) *stack func startTenantStack(t *testing.T, apiToken string, tenants []config.TenantSpec, volumes []config.VolumeSpec, pools ...config.PoolSpec) *stack { t.Helper() - // Short prefix: the sockets under it must fit darwin's 104-byte sun_path. + dir, err := os.MkdirTemp("", "sbx") if err != nil { t.Fatalf("setup: %v", err) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 7e612422..4ebb5cc8 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -16,9 +16,8 @@ import ( "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" ) -// fakeEngine replaces only the cocoon CLI: every "VM" it creates is a -// silkdtest daemon behind a real hybrid-vsock UDS, so probing and the data -// plane run the production code paths end to end. +// fakeEngine replaces only the cocoon CLI: every "VM" is a silkdtest daemon +// behind a real hybrid-vsock UDS, so the data plane runs production code. type fakeEngine struct { real *engine.Engine dir string @@ -82,9 +81,8 @@ func (f *fakeEngine) SnapshotRemove(_ context.Context, _ string) error { return func (f *fakeEngine) SnapshotList(_ context.Context) ([]string, error) { return nil, nil } -// Hibernate closes the VM's silkd listener (the "VMM died") and Restore -// brings a fresh one up, mirroring the real stop/resume observable from the -// control plane. +// Hibernate closes the VM's silkd listener and Restore brings a fresh one up, +// mirroring the stop/resume the control plane observes. func (f *fakeEngine) Hibernate(ctx context.Context, name, _ string) error { return f.Remove(ctx, name) } diff --git a/mcp/server_test.go b/mcp/server_test.go index b0495448..4860bef7 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -81,7 +81,7 @@ func TestServeSpeaksMCP(t *testing.T) { if got := toolText(t, replies[5]); got != "released" { t.Errorf("release: %q", got) } - // exec against the released sandbox: an in-band tool error, not a crash. + execReply := replies[6]["result"].(map[string]any) if execReply["isError"] != true { t.Errorf("exec after release: %+v, want isError", execReply) diff --git a/protocol/wire/frame.go b/protocol/wire/frame.go index dbcd7a5e..24dcd9e7 100644 --- a/protocol/wire/frame.go +++ b/protocol/wire/frame.go @@ -131,6 +131,17 @@ type Response interface{ RespType() string } type respDecoder func([]byte) (Response, error) +// reqPtr and respPtr turn a mistyped decoder table entry into a compile error. +type reqPtr[T any] interface { + *T + Request +} + +type respPtr[T any] interface { + *T + Response +} + // B64 carries request payload bytes. It exists because silkd's deserializer // requires a base64 string and rejects null — which is exactly what // encoding/json emits for a nil []byte. Decoding needs no counterpart: @@ -654,7 +665,7 @@ func DecodeRequest(line []byte) (Request, error) { if err != nil { return nil, fmt.Errorf("parse request frame: %w", err) } - dec, ok := requestDecoders[op] + dec, ok := requestDecoders[string(op)] if !ok { return nil, fmt.Errorf("unknown op %q", op) } @@ -676,7 +687,7 @@ func DecodeResponse(line []byte) (Response, error) { if err != nil { return nil, fmt.Errorf("parse response frame: %w", err) } - dec, ok := responseDecoders[typ] + dec, ok := responseDecoders[string(typ)] if !ok { return nil, fmt.Errorf("unknown response type %q", typ) } @@ -737,13 +748,14 @@ func encodeTagged(head string, v any) ([]byte, error) { // frameTag returns the frame's dispatch tag: a canonical head (both producers // emit the tag first, unescaped) yields it with a prefix cut; anything else // takes the full token walk. -func frameTag(line, head []byte, key string) (string, error) { +func frameTag(line, head []byte, key string) ([]byte, error) { if rest, ok := bytes.CutPrefix(line, head); ok { if end := bytes.IndexAny(rest, `"\`); end >= 0 && rest[end] == '"' { - return string(rest[:end]), nil + return rest[:end], nil } } - return scanTag(line, key) + tag, err := scanTag(line, key) + return []byte(tag), err } // scanTag extracts the string value of a top-level key without decoding the @@ -805,20 +817,18 @@ func decodeAs[T any](line []byte) (*T, error) { } // decodeReq / decodeResp adapt decodeAs to the interface-typed decoder maps. -// A table entry pairing an op with a non-Request type is caught by the shared -// fixture round-trip test, not the compiler. -func decodeReq[T any](line []byte) (Request, error) { +func decodeReq[T any, PT reqPtr[T]](line []byte) (Request, error) { v, err := decodeAs[T](line) if err != nil { return nil, err } - return any(v).(Request), nil + return PT(v), nil } -func decodeResp[T any](line []byte) (Response, error) { +func decodeResp[T any, PT respPtr[T]](line []byte) (Response, error) { v, err := decodeAs[T](line) if err != nil { return nil, err } - return any(v).(Response), nil + return PT(v), nil } diff --git a/protocol/wire/frame_test.go b/protocol/wire/frame_test.go index 8c89f8b3..50eefd1e 100644 --- a/protocol/wire/frame_test.go +++ b/protocol/wire/frame_test.go @@ -12,9 +12,6 @@ import ( const fixtureDir = "fixtures/v1" -// TestFixtureCorpusRoundTrips is the protocol/README contract: every golden -// frame must decode and re-encode to canonical-JSON equality, both request -// and response sides — a frame only one implementation round-trips is drift. func TestFixtureCorpusRoundTrips(t *testing.T) { entries, err := os.ReadDir(fixtureDir) if err != nil { @@ -60,17 +57,12 @@ func TestFixtureCorpusRoundTrips(t *testing.T) { } seen++ } - // Exact, deliberately: adding a verb means adding its fixture and - // bumping this — a new frame type outside the corpus is drift waiting - // to happen. + if seen != 60 { t.Fatalf("fixture corpus: %d frames, want exactly 60", seen) } } -// TestEveryVerbHasAFixture pins corpus completeness: every request and -// response type the SDK knows must appear in at least one golden frame, so -// no verb can exist outside the three-language drift guard. func TestEveryVerbHasAFixture(t *testing.T) { allRequests := []Request{ Exec{}, @@ -166,10 +158,7 @@ func TestEveryVerbHasAFixture(t *testing.T) { } } -// TestEnumValueSetsMatchFixture pins the full enum value lists -// (order-sensitive): frame fixtures carry one representative value per enum, -// so a variant renamed or added on only one side would otherwise drift -// silently. Twin of silkd's enum_value_sets_match_fixture. +// Twin of silkd's enum_value_sets_match_fixture; the lists are order-sensitive. func TestEnumValueSetsMatchFixture(t *testing.T) { raw, err := os.ReadFile(filepath.Join(fixtureDir, "enums.json")) if err != nil { @@ -209,8 +198,6 @@ func TestDataFieldsAreBase64(t *testing.T) { } func TestRequestDataNeverNull(t *testing.T) { - // silkd's deserializer wants a base64 string; a nil payload must encode - // as "" — encoding/json's null would abort the RPC as a protocol error. for _, req := range []Request{&Stdin{}, &Data{}} { frame, err := EncodeRequest(req) if err != nil { @@ -232,9 +219,6 @@ func TestUnknownFieldsIgnored(t *testing.T) { } } -// TestBulkDecodeStrictShape guards the fast bulk slicer: only the exact -// canonical frame takes the fast path, so a "data" match inside another -// field or any malformed JSON must never yield a silently wrong payload. func TestBulkDecodeStrictShape(t *testing.T) { resp, err := DecodeResponse([]byte(`{"type":"stdout","meta":{"data":"WFhY"},"data":"aGk="}`)) if err != nil { diff --git a/sandboxd/ca.go b/sandboxd/ca.go index b2c806e2..840ee6cf 100644 --- a/sandboxd/ca.go +++ b/sandboxd/ca.go @@ -10,8 +10,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/egress" ) -// runCA is the operator PKI tool: mint the cluster root once, then issue a -// per-node intermediate from it. The root key stays with the operator. +// runCA is the operator PKI tool: mint the cluster root, then per-node intermediates. func runCA(args []string) error { if len(args) == 0 { return fmt.Errorf("usage: sandboxd ca {init|issue-intermediate}") @@ -103,8 +102,7 @@ func writeCAFiles(dir, name string, certPEM, keyPEM []byte, force bool) error { return nil } -// writeKeyMaterial writes path with O_EXCL unless force; the Stat precheck -// above is a courtesy. +// writeKeyMaterial writes path with O_EXCL unless force. func writeKeyMaterial(path string, data []byte, perm os.FileMode, force bool) error { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC if !force { diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index ec4973c7..2d4cd98d 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -39,33 +39,23 @@ type PoolSpec struct { types.PoolKey Warm int `json:"warm"` - // WarmMax, when >0, turns on the demand-adaptive watermark: the warm - // target may rise from Warm (the floor) toward WarmMax while claims - // arrive faster than the pool provisions, and decays back in silence. + // WarmMax, when >0, lets the warm target rise from Warm toward it under demand. WarmMax int `json:"warm_max,omitempty"` - // Egress is this pool's allow-list, intersected with the tenant's for the - // effective policy (a request must pass both); when both matched rules - // name a secret, the pool's is injected. Nil denies all egress. + // Egress is this pool's allow-list, intersected with the tenant's; nil denies all egress. Egress *egress.Policy `json:"egress,omitempty"` - // IdleHibernateSeconds, when >0, hibernates this none-lane pool's idle claims - // after that many seconds without a data-plane connection; the next - // call wakes them transparently. Zero disables. + // IdleHibernateSeconds, when >0, hibernates idle claims after that many seconds. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` - // ArchiveAfterSeconds, when >0, checkpoints a hibernated claim to the - // store after that many idle seconds and drops its local VM; a later call - // restores it. Requires IdleHibernateSeconds>0 and must exceed it. + // ArchiveAfterSeconds, when >0, archives a hibernated claim; must exceed IdleHibernateSeconds. ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` - // ArchiveDeleteAfterSeconds, when >0, purges an archived claim's store - // checkpoint that many seconds after archiving. Zero keeps it forever. + // ArchiveDeleteAfterSeconds, when >0, purges the checkpoint that long after archiving. ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` } -// ValidateLimits checks the warm/watermark/idle bounds — shared by the -// config file path and the PUT /v1/pools path so the two cannot drift. +// ValidateLimits checks the warm/watermark/idle bounds shared by config and PUT /v1/pools. func (s PoolSpec) ValidateLimits() error { if s.Warm < 0 { return fmt.Errorf("warm must not be negative") @@ -88,9 +78,7 @@ type StoreConfig struct { S3 *s3.Config `json:"s3,omitempty"` } -// EgressCAConfig provisions HTTPS interception: RootCert is the cluster root -// baked into guests; IntermediateCert/Key are this node's signing CA. The root -// private key never appears here. +// EgressCAConfig provisions HTTPS interception; the root private key never appears here. type EgressCAConfig struct { RootCert string `json:"root_cert"` IntermediateCert string `json:"intermediate_cert"` @@ -103,9 +91,6 @@ func (e *EgressCAConfig) Set() bool { } // TenantSpec declares one tenant: its bearer token and its live-claim quota. -// APIToken stays the operator (root) credential with full access; tenant -// tokens reach the resource-creating verbs only, and everything a tenant -// creates is stamped with its name. type TenantSpec struct { Name string `json:"name"` Token string `json:"token"` //nolint:gosec // config field, not a hardcoded credential @@ -115,9 +100,7 @@ type TenantSpec struct { Egress *egress.Policy `json:"egress,omitempty"` } -// VolumeSpec declares one operator-managed dataset disk. The entry — name, -// access list, mode — is declared fleet-wide; Writable admits rw claims, and -// the image at Path is held by exactly one node. +// VolumeSpec declares one operator-managed dataset disk, held by exactly one node. type VolumeSpec struct { Name string `json:"name"` Path string `json:"path"` @@ -126,10 +109,7 @@ type VolumeSpec struct { Tenants []string `json:"tenants,omitempty"` } -// MeshConfig configures cluster membership. Two v1 constraints: every node -// shares one APIToken and Tenants set (a redirect replays the claim's token, -// so a peer missing that tenant answers 401), and only a node with its own -// egress attachment can take a redirected egress claim. +// MeshConfig configures cluster membership; every node shares one APIToken and tenant set. type MeshConfig struct { NodeID string `json:"node_id"` // unique name; defaults to Bind Bind string `json:"bind"` // memberlist host:port @@ -137,8 +117,7 @@ type MeshConfig struct { ClusterKey string `json:"cluster_key,omitempty"` // base64 gossip-encryption key } -// ParsedBind splits Bind into host and port, rejecting a wildcard host — -// memberlist would advertise an unroutable address. +// ParsedBind splits Bind into host and port, rejecting a wildcard host. func (mc *MeshConfig) ParsedBind() (string, int, error) { host, portStr, err := net.SplitHostPort(mc.Bind) if err != nil { @@ -177,50 +156,34 @@ type Config struct { DataDir string `json:"data_dir"` CocoonBin string `json:"cocoon_bin"` - // AdvertiseAddr is the host:port the data plane reaches this node at; it - // is returned as a claim's owner address (and, at M2c, gossiped). Defaults - // to Listen, which is correct when Listen is a routable host:port. + // AdvertiseAddr is the host:port the data plane reaches this node at; defaults to Listen. AdvertiseAddr string `json:"advertise_addr,omitempty"` - // Bridges shards egress-lane VMs over several host bridge devices with - // the raw TAP-on-bridge attachment: taps stay in the root netns, so - // guarded egress can lock them. Mutually exclusive with Networks; with - // neither set the node serves only the no-network lane. + // Bridges shards egress-lane VMs over host bridges; their taps stay lockable in the root netns. Bridges []string `json:"bridges,omitempty"` - // Networks shards egress-lane VMs over several CNI conflists, one Linux - // bridge each: a bridge holds at most 1024 ports (BR_MAX_PORTS, no - // sysctl), so N conflists raise the node's egress ceiling to N×1024. + // Networks shards egress-lane VMs over CNI conflists; a bridge holds at most 1024 ports. Networks []string `json:"networks,omitempty"` - // RestoreMode rides clones and wake restores as cocoon's --restore-mode. Opt-in: mmap - // needs every node's cloud-hypervisor to carry CoW restore — an older CH - // silently eager-copies while reporting success. + // RestoreMode is cocoon's --restore-mode; an older CH silently eager-copies an mmap restore. RestoreMode types.RestoreMode `json:"restore_mode,omitempty"` // NoDirectIO enables buffered writable disks for cold boots and clones. NoDirectIO bool `json:"no_direct_io,omitempty"` - // APIToken, when set, guards claim and info; per-sandbox tokens guard - // sandbox-scoped calls regardless. + // APIToken, when set, guards claim and info. APIToken string `json:"api_token,omitempty"` //nolint:gosec // config field, not a hardcoded credential - // Tenants adds per-tenant bearer tokens next to APIToken; empty keeps - // the single-token behavior. + // Tenants adds per-tenant bearer tokens next to APIToken. Tenants []TenantSpec `json:"tenants,omitempty"` - // Secrets registers node-side credentials the egress proxy injects by - // name; values come from the environment (value_env), never this file. + // Secrets registers node-side credentials; values come from value_env, never this file. Secrets []egress.SecretSpec `json:"secrets,omitempty"` - // IdleHibernateSeconds is the idle policy for unpooled none-lane claims - // (template and checkpoint claims); per-pool settings override it for - // pooled keys. Zero disables. + // IdleHibernateSeconds is the idle policy for unpooled claims; per-pool settings override. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` - // ArchiveAfterSeconds / ArchiveDeleteAfterSeconds are the archive policy - // for unpooled keys; per-pool settings override them. Same rules as - // PoolSpec (archive_after must exceed idle_hibernate). + // ArchiveAfterSeconds and ArchiveDeleteAfterSeconds are the archive policy for unpooled keys. ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` @@ -229,58 +192,40 @@ type Config struct { // PreviewAdvertise is the browser-facing base, shareable fleet-wide behind one proxy. PreviewAdvertise string `json:"preview_advertise,omitempty"` - // CheckpointDir is where checkpoints live; defaults to - // /checkpoints. Point it at a shared FUSE mount (JuiceFS over - // object storage, NFS) and every node sharing the mount resolves every - // checkpoint — the path's filesystem is the operator's choice. + // CheckpointDir is where checkpoints live; defaults to /checkpoints. CheckpointDir string `json:"checkpoint_dir,omitempty"` - // CheckpointStore selects the checkpoint backend; absent means the - // dir backend at CheckpointDir. Kind "s3" stores checkpoints in object - // storage (credentials from the AWS chain, never this file). + // CheckpointStore selects the checkpoint backend; absent means the dir backend. CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` - // CheckpointPeerHeal lets a node pull a checkpoint it lacks from a peer - // rather than failing the branch. Requires cluster_key (the pull presents - // the fleet token to a probed address) and a nonzero checkpoint TTL (the - // revocation bound for a replica a delete broadcast missed). Off by default. + // CheckpointPeerHeal lets a node pull a checkpoint it lacks from a peer; off by default. CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` - // EgressInternalAllow re-admits CIDRs through the proxy's SSRF guard, - // node-wide (every pool and tenant). Prefixes, not a permit-private - // switch: the guest bridges are themselves ULA/RFC1918. + // EgressInternalAllow re-admits CIDRs through the proxy's SSRF guard, node-wide. EgressInternalAllow []string `json:"egress_internal_allow,omitempty"` - // CheckpointTTLHours ages out checkpoints (0 = keep forever); the - // sweep runs hourly and on startup. + // CheckpointTTLHours ages out checkpoints; 0 keeps them forever. CheckpointTTLHours int `json:"checkpoint_ttl_hours,omitempty"` - // MaxClaims caps live claims node-wide; 0 means unlimited. Claim, - // fork, and checkpoint-branch requests beyond it answer 429. + // MaxClaims caps live claims node-wide; 0 means unlimited. MaxClaims int `json:"max_claims,omitempty"` - // AuditLog, when true, appends every relayed request frame's op and - // addressing fields (never payloads) to /audit.jsonl. + // AuditLog, when true, appends relayed request ops, never payloads, to audit.jsonl. AuditLog bool `json:"audit_log,omitempty"` - // MaxForkCount caps children per fork call — each child is a full-RAM VM. - // Defaults to 16. + // MaxForkCount caps children per fork call; each child is a full-RAM VM. MaxForkCount int `json:"max_fork_count,omitempty"` // Volumes is the node-local catalog of operator-managed dataset images. Volumes []VolumeSpec `json:"volumes,omitempty"` - // RefillConcurrency caps concurrent VM provisioning node-wide — warm-pool - // refills, fork clones, and reap/hibernate/reconcile batches share one - // budget; 0 auto-scales with the node's CPU count. + // RefillConcurrency caps concurrent VM provisioning node-wide; 0 auto-scales with CPUs. RefillConcurrency int `json:"refill_concurrency,omitempty"` - // Mesh, when set, joins this node to a memberlist cluster for redirect - // placement; nil is a single node (mesh of one, no gossip). + // Mesh, when set, joins this node to a memberlist cluster; nil is a mesh of one. Mesh *MeshConfig `json:"mesh,omitempty"` - // EgressCA provisions HTTPS interception; required when any pool has an - // intercept rule. + // EgressCA provisions HTTPS interception; required for any intercept rule. EgressCA *EgressCAConfig `json:"egress_ca,omitempty"` Pools []PoolSpec `json:"pools"` @@ -291,10 +236,7 @@ func (c *Config) HasEgress() bool { return len(c.Bridges) > 0 || len(c.Networks) > 0 } -// ClusterDigest fingerprints the must-match config so a divergent node is -// caught early, not at a later 401 or an outlived replica. With cluster_key it -// HMACs token material too; without one only tenant names, the CA root and the -// checkpoint TTL ride the (maybe cleartext) wire. +// ClusterDigest fingerprints the must-match config; without cluster_key it omits tokens. func (c *Config) ClusterDigest(caFingerprint string) string { names := make([]string, len(c.Tenants)) for i, t := range c.Tenants { @@ -320,8 +262,7 @@ func (c *Config) ClusterDigest(caFingerprint string) string { return hex.EncodeToString(sum[:]) } -// guardsEgressLane: any tenant policy counts (claims mint egress keys without a -// pool), a none-lane pool policy does not (it locks no tap). +// guardsEgressLane counts any tenant policy, but only an egress-lane pool policy. func (c *Config) guardsEgressLane() bool { return slices.ContainsFunc(c.Tenants, func(t TenantSpec) bool { return t.Egress != nil }) || slices.ContainsFunc(c.Pools, func(p PoolSpec) bool { return p.Net == types.NetEgress && p.Egress != nil }) @@ -350,8 +291,7 @@ func (c *Config) validate() error { if err := c.validateAttachment(); err != nil { return err } - // A CNI network's tap lives in the VM netns, unreachable from the root-netns - // nft lock; guarded egress needs a bridge. + // a CNI network's tap lives in the VM netns, unreachable from the root-netns nft lock. if len(c.Networks) > 0 && c.guardsEgressLane() { return fmt.Errorf("guarded egress needs a bridge lane, not a CNI network: the tap lives in the VM netns and cannot be locked") } @@ -502,8 +442,7 @@ func (c *Config) validateSecrets() (map[string]struct{}, error) { return names, nil } -// validateAttachment checks the egress-lane attachment; a repeated shard -// would report N shards while filling one bridge. +// validateAttachment checks the egress-lane attachment; a repeated shard fills one bridge. func (c *Config) validateAttachment() error { if len(c.Bridges) > 0 && len(c.Networks) > 0 { return fmt.Errorf("bridges and networks are mutually exclusive") @@ -514,8 +453,7 @@ func (c *Config) validateAttachment() error { return validateShards(c.Networks, "networks", "conflist") } -// validateEgressAllow rejects a malformed prefix at load: a typo would -// otherwise surface as a refused dial long after startup. +// validateEgressAllow rejects a malformed prefix at load, not at a later refused dial. func (c *Config) validateEgressAllow() error { for _, cidr := range c.EgressInternalAllow { if _, err := netip.ParsePrefix(cidr); err != nil { @@ -574,8 +512,7 @@ func Load(path string) (*Config, error) { return cfg, nil } -// validatePolicy checks the rules and that each secret ref is registered; a -// nil policy is valid (deny-all). +// validatePolicy checks the rules and secret refs; a nil policy is deny-all. func validatePolicy(p *egress.Policy, secrets map[string]struct{}) error { if p == nil { return nil @@ -594,8 +531,7 @@ func validatePolicy(p *egress.Policy, secrets map[string]struct{}) error { return nil } -// validateArchiveWindow checks the archive thresholds shared by PoolSpec and -// the node Config: non-negative, and archive_after must sit past idle_hibernate. +// validateArchiveWindow requires archive_after past idle_hibernate, both non-negative. func validateArchiveWindow(idle, after, del int) error { if after < 0 || del < 0 { return fmt.Errorf("archive seconds must not be negative") diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index ac37e344..aebb4345 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -23,7 +23,7 @@ func TestClusterDigest(t *testing.T) { if base.ClusterDigest("other-fp") == d { t.Error("an egress CA root change is not reflected") } - // Keyless: token material must stay off the (maybe cleartext) wire. + if (&Config{APIToken: "other", PreviewSecret: "ps", Tenants: base.Tenants}).ClusterDigest("ca-fp") != d { t.Error("api_token leaked into the keyless digest") } @@ -207,7 +207,6 @@ func TestLoadAcceptsEgressPolicy(t *testing.T) { } func TestLoadAcceptsUnguardedCNINetwork(t *testing.T) { - // Only guarded egress needs a bridge; an unguarded CNI network lane is fine. path := writeConfig(t, `{"networks":["cni"],"pools":[{"template":"rt:24.04","net":"egress","size":"small"}]}`) if _, err := Load(path); err != nil { t.Fatalf("Load: %v", err) @@ -215,8 +214,6 @@ func TestLoadAcceptsUnguardedCNINetwork(t *testing.T) { } func TestLoadAcceptsNoneLanePolicyOnCNI(t *testing.T) { - // A none-lane policy rides the vsock proxy and locks no tap, so it is valid on - // a CNI network — the bridge requirement is only for a guarded egress lane. for _, body := range []string{ `{"networks":["cni"],"pools":[{"template":"rt:24.04","net":"none","size":"small","egress":{"allow":[{"host":"x"}]}}]}`, `{"networks":["cni"],"pools":[{"template":"rt:24.04","net":"egress","size":"small"},{"template":"rt:24.04","net":"none","size":"medium","egress":{"allow":[{"host":"x"}]}}]}`, diff --git a/sandboxd/egress/bench_test.go b/sandboxd/egress/bench_test.go index 6ac3a146..54f3bf9a 100644 --- a/sandboxd/egress/bench_test.go +++ b/sandboxd/egress/bench_test.go @@ -73,9 +73,6 @@ func benchCA(b *testing.B) *CA { return ca } -// benchFront serves an HTTPS echo behind the proxy; intercept toggles the rule, -// and guest roots match: the cluster root when intercepted, the origin's own -// cert when spliced. func benchFront(b *testing.B, intercept bool) (proxyAddr string, roots *x509.CertPool) { b.Helper() upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/sandboxd/egress/ca.go b/sandboxd/egress/ca.go index 8fe93fe1..0faa2259 100644 --- a/sandboxd/egress/ca.go +++ b/sandboxd/egress/ca.go @@ -24,8 +24,7 @@ const ( pemTypeCertificate = "CERTIFICATE" ) -// CA signs per-host interception leaves with the node's intermediate. The -// shared cluster root is baked into guests; the root key never reaches a node. +// CA signs per-host interception leaves with the node's intermediate. type CA struct { rootPEM []byte rootFP string @@ -35,8 +34,7 @@ type CA struct { leafKey *ecdsa.PrivateKey } -// LoadCA builds the node CA; the intermediate must be signed by the bundle's -// first root cert (a bundle carries old+new roots during a rotation). +// LoadCA builds the node CA; the intermediate must be signed by the bundle's first cert. func LoadCA(rootCertPEM, interCertPEM, interKeyPEM []byte) (*CA, error) { root, err := parseRootBundle(rootCertPEM) if err != nil { @@ -59,8 +57,7 @@ func LoadCA(rootCertPEM, interCertPEM, interKeyPEM []byte) (*CA, error) { if err = inter.CheckSignatureFrom(root); err != nil { return nil, fmt.Errorf("intermediate not signed by root: %w", err) } - // Reject a self-signed intermediate: the root must not double as one, which - // would put the root key on the node. + // reject a self-signed intermediate: it would put the root key on the node. if inter.CheckSignatureFrom(inter) == nil { return nil, fmt.Errorf("intermediate must not be a self-signed root") } @@ -86,12 +83,10 @@ func LoadCA(rootCertPEM, interCertPEM, interKeyPEM []byte) (*CA, error) { // CertPEM is the cluster root to bake into an intercepted guest's trust store. func (c *CA) CertPEM() []byte { return c.rootPEM } -// Fingerprint is the SHA-256 of the baked root PEM; changed bytes rebuild the -// golden, a rotated intermediate does not. +// Fingerprint is the SHA-256 of the baked root PEM, so a rotated intermediate keeps it. func (c *CA) Fingerprint() string { return c.rootFP } -// SignLeaf issues a fresh interception leaf for host, chained [leaf, -// intermediate] for a guest that trusts the root. +// SignLeaf issues a fresh interception leaf for host, chained [leaf, intermediate]. func (c *CA) SignLeaf(host string) (*tls.Certificate, error) { serial, err := serialNumber() if err != nil { @@ -146,8 +141,7 @@ func parseCert(pemBytes []byte, what string) (*x509.Certificate, error) { return cert, nil } -// parseRootBundle returns the first cert (the anchor) and rejects any non-CA -// block: the whole file lands in guest trust stores. +// parseRootBundle returns the anchor and rejects a non-CA block: guests trust the whole file. func parseRootBundle(pemBytes []byte) (*x509.Certificate, error) { var anchor *x509.Certificate for rest := pemBytes; ; { diff --git a/sandboxd/egress/ca_issue.go b/sandboxd/egress/ca_issue.go index 6d2677c7..696aee02 100644 --- a/sandboxd/egress/ca_issue.go +++ b/sandboxd/egress/ca_issue.go @@ -16,8 +16,7 @@ const ( interValidity = 5 * 365 * 24 * time.Hour ) -// GenerateRoot mints a cluster root CA (cert + key PEM). Run once per cluster; -// the key stays with the operator and never reaches a node. +// GenerateRoot mints a cluster root CA; the key stays with the operator, never a node. func GenerateRoot(commonName string) (certPEM, keyPEM []byte, err error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { @@ -40,8 +39,7 @@ func GenerateRoot(commonName string) (certPEM, keyPEM []byte, err error) { return marshalCA(tmpl, tmpl, &key.PublicKey, key, key) } -// IssueIntermediate signs a per-node intermediate CA (leaves only, path length -// zero) from the cluster root, to provision to nodeID. +// IssueIntermediate signs a per-node intermediate CA from the cluster root. func IssueIntermediate(rootCertPEM, rootKeyPEM []byte, nodeID string) (certPEM, keyPEM []byte, err error) { root, err := parseCert(rootCertPEM, "root") if err != nil { diff --git a/sandboxd/egress/intercept.go b/sandboxd/egress/intercept.go index 93f68184..21b223c8 100644 --- a/sandboxd/egress/intercept.go +++ b/sandboxd/egress/intercept.go @@ -15,14 +15,11 @@ const ( interceptTimeout = 30 * time.Second leafRenewBefore = 24 * time.Hour - // maxLeaves bounds one sandbox's leaf cache against a wildcard rule minting - // unbounded per-host leaves. + // maxLeaves bounds one sandbox's leaf cache against a wildcard rule. maxLeaves = 1024 ) -// serveIntercept terminates a matched CONNECT's TLS with a node-signed leaf the -// guest trusts, then serves the decrypted HTTP/1.1 through interceptHandler. -// Upstream is re-originated with real-root verification (mitmTr), never trusted. +// serveIntercept terminates a matched CONNECT's TLS with a node-signed leaf. func (p *Proxy) serveIntercept(w http.ResponseWriter, r *http.Request, host string) { leaf, err := p.leafFor(host) if err != nil { @@ -47,8 +44,7 @@ func (p *Proxy) serveIntercept(w http.ResponseWriter, r *http.Request, host stri if _, err := io.WriteString(client, "HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { return } - // A guest that never sends a ClientHello would pin this goroutine until the - // sandbox dies; cleared once the handshake lands. + // a guest that never sends a ClientHello would pin this goroutine until the sandbox dies. _ = client.SetDeadline(time.Now().Add(interceptTimeout)) tlsConn := tls.Server(client, cfg) if err := tlsConn.HandshakeContext(r.Context()); err != nil { @@ -63,8 +59,7 @@ func (p *Proxy) serveIntercept(w http.ResponseWriter, r *http.Request, host stri _ = srv.Serve(ln) } -// leafFor returns this sandbox's cached leaf for host, signing a fresh one on -// miss or near-expiry; the cache and its flush are per-sandbox. +// leafFor returns the cached leaf for host, signing a fresh one on miss or near-expiry. func (p *Proxy) leafFor(host string) (*tls.Certificate, error) { p.leafMu.Lock() defer p.leafMu.Unlock() @@ -82,8 +77,7 @@ func (p *Proxy) leafFor(host string) (*tls.Certificate, error) { return crt, nil } -// interceptHandler forwards one decrypted request per call: host is the CONNECT -// authority's hostname (policy + leaf key), authority its host:port (upstream). +// interceptHandler forwards one decrypted request per call; host is bare, authority host:port. type interceptHandler struct { proxy *Proxy host string @@ -96,17 +90,14 @@ func (h *interceptHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.relay(w, r, h.host, rule, decision, p.mitmTr, func(out *http.Request) { out.URL.Scheme = "https" out.URL.Host = h.authority - // Keep the guest's Host when it names the CONNECT host (SigV4-style - // signing breaks on the ":443" rewrite); a foreign Host snaps to the - // authority. + // keep the guest's Host when it names the CONNECT host: SigV4 signing breaks on a rewrite. if !strings.EqualFold(hostOnly(r.Host), h.host) { out.Host = h.authority } }) } -// singleConnListener feeds one already-accepted conn to an http.Server and then -// blocks until that conn closes, so Serve exits once the tunnel ends. +// singleConnListener feeds one accepted conn to an http.Server, then blocks until it closes. type singleConnListener struct { conn net.Conn handed atomic.Bool diff --git a/sandboxd/egress/intercept_test.go b/sandboxd/egress/intercept_test.go index 79058635..109b2f10 100644 --- a/sandboxd/egress/intercept_test.go +++ b/sandboxd/egress/intercept_test.go @@ -70,9 +70,6 @@ func TestInterceptInjectsSecretIntoHTTPS(t *testing.T) { } func TestInterceptBindsInnerRequestToInterceptRule(t *testing.T) { - // A broad plain rule precedes the specific intercept rule for the same - // host: the inner request must still bind to the intercept rule (inject its - // secret), not first-match the shadowing plain rule. policy := Policy{Allow: []Rule{ {Host: "*.example.com"}, {Host: "api.example.com", Secret: "gh", Intercept: true}, @@ -93,9 +90,6 @@ func TestInterceptBindsInnerRequestToInterceptRule(t *testing.T) { } func TestInterceptCaptureEnforcesRuleMethod(t *testing.T) { - // The intercept rule is GET-only; a broad plain rule would allow POST. Once - // a host is captured for interception, the intercept rule governs its - // decrypted traffic — POST must be denied, not routed via the plain rule. policy := Policy{Allow: []Rule{ {Host: "*.example.com"}, {Host: "api.example.com", Methods: []string{"GET"}, Intercept: true}, @@ -116,9 +110,6 @@ func TestInterceptCaptureEnforcesRuleMethod(t *testing.T) { } func TestInterceptReachesLaterInterceptRuleByMethod(t *testing.T) { - // A wildcard intercept rule (GET) precedes a specific intercept rule (POST) - // for the same host. The inner POST must reach the second rule by host+method - // — binding the whole tunnel to the first intercept rule makes it unreachable. policy := Policy{Allow: []Rule{ {Host: "*.example.com", Methods: []string{"GET"}, Intercept: true}, {Host: "api.example.com", Methods: []string{"POST"}, Intercept: true}, @@ -257,8 +248,6 @@ func trustUpstream(upstream *httptest.Server) *x509.CertPool { return pool } -// connectTLS opens a CONNECT tunnel through the proxy and completes the guest -// TLS handshake against the leaf the proxy presents. func connectTLS(t *testing.T, proxyAddr, target, serverName string, roots *x509.CertPool) *tls.Conn { t.Helper() conn, err := net.Dial("tcp", proxyAddr) @@ -318,8 +307,6 @@ func roundTripTLSHost(t *testing.T, tc *tls.Conn, method, host string) *http.Res return resp } -// readPreamble reads the CONNECT reply byte-wise up to the blank line, so no -// TLS bytes are consumed; the proxy sends nothing more until the ClientHello. func readPreamble(t *testing.T, conn net.Conn) string { t.Helper() var sb strings.Builder diff --git a/sandboxd/egress/policy.go b/sandboxd/egress/policy.go index da3b914e..49f175f7 100644 --- a/sandboxd/egress/policy.go +++ b/sandboxd/egress/policy.go @@ -18,13 +18,7 @@ const ( // Decision is the policy verdict for one request. type Decision int -// Rule allows requests whose host matches Host — an exact name, a -// "*."-prefixed suffix wildcard ("*.example.com" matches sub.example.com but -// not the apex), or "*" for any host — and whose method is in Methods (empty -// means any). Secret names a node-side credential the proxy injects into a -// matched request; empty injects nothing. Intercept terminates a matched -// HTTPS CONNECT so the request is filtered by method and the secret injected; -// only a pool rule may set it. Host matching is case-insensitive. +// Rule allows requests matching Host (exact, "*." suffix, or "*") and Methods; empty means any. type Rule struct { Host string `json:"host"` Methods []string `json:"methods,omitempty"` @@ -53,20 +47,17 @@ func (r Rule) matchMethod(method string) bool { slices.ContainsFunc(r.Methods, func(m string) bool { return strings.EqualFold(m, method) }) } -// Policy is one sandbox's egress allow-list. Default-deny is implicit: a -// request matching no Allow rule is denied. +// Policy is one sandbox's egress allow-list; a request matching no rule is denied. type Policy struct { Allow []Rule `json:"allow"` } -// Intercepts reports whether any rule terminates HTTPS; nil is false. It gates -// baking the cluster root cert into a pool's golden. +// Intercepts reports whether any rule terminates HTTPS; nil is false. func (p *Policy) Intercepts() bool { return p != nil && slices.ContainsFunc(p.Allow, func(r Rule) bool { return r.Intercept }) } -// Validate rejects a rule with an empty or bare-wildcard host; an empty -// allow-list is valid (it denies everything). +// Validate rejects an empty or bare-wildcard host; an empty allow-list denies everything. func (p Policy) Validate() error { for i, r := range p.Allow { switch r.Host { @@ -79,10 +70,7 @@ func (p Policy) Validate() error { return nil } -// Eval returns the first matching non-intercept allow rule with DecisionAllow, -// or a zero rule with DecisionDeny. host is a bare hostname (no port). Intercept -// rules answer only the CONNECT paths (EvalHost/EvalInner) — matching one here -// would inject its secret over cleartext. +// Eval returns the first matching non-intercept rule; matching one here would leak its secret. func (p Policy) Eval(host, method string) (Rule, Decision) { host = strings.ToLower(host) for _, r := range p.Allow { @@ -93,8 +81,7 @@ func (p Policy) Eval(host, method string) (Rule, Decision) { return Rule{}, DecisionDeny } -// EvalHost matches by host only (a CONNECT tunnel carries no method), preferring -// an intercept rule over an earlier plain match: interception is a host property. +// EvalHost matches by host only, preferring an intercept rule over an earlier plain match. func (p Policy) EvalHost(host string) (Rule, Decision) { host = strings.ToLower(host) first := -1 @@ -113,9 +100,7 @@ func (p Policy) EvalHost(host string) (Rule, Decision) { return Rule{}, DecisionDeny } -// EvalInner decides a decrypted request over an intercepted tunnel: it matches -// only intercept rules, by host+method, so a plain rule can neither shadow the -// injection nor rescue a method the intercept rules reject. +// EvalInner matches only intercept rules, so a plain rule cannot shadow or rescue one. func (p Policy) EvalInner(host, method string) (Rule, Decision) { host = strings.ToLower(host) for _, r := range p.Allow { @@ -126,16 +111,14 @@ func (p Policy) EvalInner(host, method string) (Rule, Decision) { return Rule{}, DecisionDeny } -// Evaluator is what the proxy consults per request; Policy is one, Compose -// intersects two. +// Evaluator is what the proxy consults per request. type Evaluator interface { Eval(host, method string) (Rule, Decision) EvalHost(host string) (Rule, Decision) EvalInner(host, method string) (Rule, Decision) } -// Compose intersects a pool and a tenant policy: a request must pass both, and -// the pool rule (with its secret) wins on a double allow. +// Compose intersects a pool and a tenant policy; the pool rule wins on a double allow. func Compose(pool, tenant Policy) Evaluator { return composite{pool: pool, tenant: tenant} } diff --git a/sandboxd/egress/policy_test.go b/sandboxd/egress/policy_test.go index d2f1ae6f..6f9dd9ff 100644 --- a/sandboxd/egress/policy_test.go +++ b/sandboxd/egress/policy_test.go @@ -19,7 +19,7 @@ func TestPolicyEval(t *testing.T) { {"exact host case-insensitive", "API.GitHub.com", "POST", DecisionAllow, "gh"}, {"exact host wrong method falls through to catch-all deny", "api.github.com", "DELETE", DecisionDeny, ""}, {"wildcard subdomain", "storage.googleapis.com", "PUT", DecisionAllow, ""}, - {"wildcard does not match apex", "googleapis.com", "GET", DecisionAllow, ""}, // caught by "*"/GET + {"wildcard does not match apex", "googleapis.com", "GET", DecisionAllow, ""}, {"catch-all get", "example.com", "GET", DecisionAllow, ""}, {"catch-all denies non-get", "example.com", "POST", DecisionDeny, ""}, } @@ -41,8 +41,7 @@ func TestEvalSkipsInterceptRules(t *testing.T) { {Host: "api.github.com", Secret: "gh", Intercept: true}, {Host: "plain.github.com"}, }} - // An intercept-only host must deny the plaintext forward path outright — - // matching would inject the rule's secret over cleartext. + if rule, d := p.Eval("api.github.com", "GET"); d != DecisionDeny || rule.Secret != "" { t.Errorf("Eval on an intercept-only host = %+v/%v, want deny", rule, d) } @@ -70,7 +69,7 @@ func TestEvalHostPrefersInterceptRule(t *testing.T) { func TestEvalInnerMatchesOnlyInterceptRulesByMethod(t *testing.T) { p := Policy{Allow: []Rule{ - {Host: "*.example.com"}, // plain: must not shadow or rescue + {Host: "*.example.com"}, {Host: "api.example.com", Methods: []string{"GET"}, Secret: "gh", Intercept: true}, {Host: "api.example.com", Methods: []string{"POST"}, Intercept: true}, }} diff --git a/sandboxd/egress/proxy.go b/sandboxd/egress/proxy.go index 65b0a2e6..1bdda05f 100644 --- a/sandboxd/egress/proxy.go +++ b/sandboxd/egress/proxy.go @@ -19,27 +19,21 @@ import ( const idleConnTimeout = 90 * time.Second -// hopHeaders are hop-by-hop and proxy-scoped headers this hop owns: stripped -// from forwarded requests and from relayed responses rather than passed on. +// hopHeaders are hop-by-hop and proxy-scoped headers this hop owns, never passed on. var hopHeaders = []string{ "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Proxy-Connection", "TE", "Trailer", "Transfer-Encoding", "Upgrade", } -// DialFunc opens the upstream connection for a permitted request. The proxy -// stays transport-agnostic: the node wires the real egress path (a bridge -// dial, or a checked resolver) behind it. +// DialFunc opens the upstream connection for a permitted request. type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) -// Secrets resolves a rule's Secret name to the header it injects. The value -// lives node-side and the proxy is its only holder, so a matched request -// carries the credential's effect without the guest ever seeing it. +// Secrets resolves a rule's Secret name to the header it injects. type Secrets interface { Header(name string) (header, value string, ok bool) } -// Event is one audited egress attempt, keyed by sandbox and tenant. Injected -// names the credential added to a permitted request, never its value. +// Event is one audited egress attempt; Injected names the credential, never its value. type Event struct { Sandbox string Tenant string @@ -49,10 +43,7 @@ type Event struct { Injected string } -// Proxy is one sandbox's forward proxy: an http.Handler serving CONNECT -// tunnels and absolute-form HTTP, gated by Policy and audited per request. -// One Proxy carries one sandbox's identity, so the connection a listener -// hands it needs no in-band token. +// Proxy is one sandbox's forward proxy, gated by Policy and audited per request. type Proxy struct { sandbox string tenant string @@ -64,8 +55,7 @@ type Proxy struct { tr *http.Transport mitmTr *http.Transport - // leaves caches this sandbox's interception leaves; per-proxy, so one - // sandbox's host churn never evicts or blocks another's. + // leaves caches this sandbox's interception leaves; per-proxy, never shared. leafMu sync.Mutex leaves map[string]*tls.Certificate @@ -75,8 +65,7 @@ type Proxy struct { closed bool } -// New builds a Proxy for one sandbox. secrets, ca, and audit may be nil (no -// injection, no HTTPS interception, no audit). dial must be set. +// New builds a Proxy for one sandbox; secrets, ca, and audit may be nil, dial must not. func New(sandbox, tenant string, policy Evaluator, secrets Secrets, ca *CA, dial DialFunc, audit func(Event)) *Proxy { p := &Proxy{ sandbox: sandbox, @@ -86,8 +75,7 @@ func New(sandbox, tenant string, policy Evaluator, secrets Secrets, ca *CA, dial ca: ca, audit: audit, dial: dial, - // The stdlib default of 2 idle conns per host re-dials bursty - // same-host plaintext traffic. + // the stdlib default of 2 idle conns per host re-dials bursty same-host traffic. tr: &http.Transport{DialContext: dial, MaxIdleConnsPerHost: 8, IdleConnTimeout: idleConnTimeout}, conns: map[net.Conn]struct{}{}, } @@ -107,8 +95,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.serveForward(w, r) } -// Close ends every hijacked tunnel, which outlives http.Server.Close, so a -// released claim's guest cannot keep an authorized session. Idempotent. +// Close ends every hijacked tunnel, which outlives http.Server.Close; idempotent. func (p *Proxy) Close() { p.connMu.Lock() conns := slices.Collect(maps.Keys(p.conns)) @@ -145,9 +132,7 @@ func (p *Proxy) untrack(conn net.Conn) { // serveConnect gates an HTTPS/opaque tunnel by host. func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request) { host := hostOnly(r.Host) - // Host-gate the interception decision: the tunnel's CONNECT verb is not the - // request method, so a matched intercept rule enforces its methods on the - // decrypted inner request. The plain splice keeps the host+method gate. + // host-gate interception: the tunnel's CONNECT verb is not the request method. if rule, d := p.policy.EvalHost(host); d == DecisionAllow && rule.Intercept && p.ca != nil { p.serveIntercept(w, r, host) return @@ -190,8 +175,7 @@ func (p *Proxy) serveForward(w http.ResponseWriter, r *http.Request) { p.relay(w, r, host, rule, decision, p.tr, nil) } -// relay forwards one policy-evaluated request upstream and mirrors the -// response; prepare, when set, rewrites the clone before it leaves. +// relay forwards one policy-evaluated request upstream and mirrors the response. func (p *Proxy) relay(w http.ResponseWriter, r *http.Request, host string, rule Rule, decision Decision, tr *http.Transport, prepare func(*http.Request)) { if decision == DecisionDeny { p.record(Event{Method: r.Method, Host: host, Decision: decision}) @@ -220,9 +204,7 @@ func (p *Proxy) relay(w http.ResponseWriter, r *http.Request, host string, rule _, _ = io.Copy(w, resp.Body) } -// inject sets the header the rule's Secret resolves to, overwriting any -// guest-supplied value, and returns the secret name it applied (empty when -// the rule names none or the store cannot resolve it). +// inject sets the rule's secret header, overwriting any guest-supplied value. func (p *Proxy) inject(rule Rule, h http.Header) string { if rule.Secret == "" || p.secrets == nil { return "" @@ -257,8 +239,7 @@ func stripHop(h http.Header) { } } -// splice copies bytes both ways until either side ends, then returns; each -// direction's end half-closes the peer so its EOF propagates. +// splice copies both ways until either side ends, half-closing the peer so EOF propagates. func splice(a, b net.Conn) { done := make(chan struct{}) go func() { @@ -271,14 +252,12 @@ func splice(a, b net.Conn) { <-done } -// denied answers a policy rejection with a typed 403 the guest's client -// surfaces as a distinct failure rather than a hang. +// denied answers a policy rejection with a 403 rather than a hang. func denied(w http.ResponseWriter, host string) { http.Error(w, fmt.Sprintf("egress denied: %s", host), http.StatusForbidden) } -// hostOnly strips a port from an authority, tolerating a bare host and IPv6 -// literals. +// hostOnly strips a port from an authority, tolerating a bare host and IPv6 literals. func hostOnly(authority string) string { if host, _, err := net.SplitHostPort(authority); err == nil { return host diff --git a/sandboxd/egress/proxy_test.go b/sandboxd/egress/proxy_test.go index 2e9c3a8a..d568cc53 100644 --- a/sandboxd/egress/proxy_test.go +++ b/sandboxd/egress/proxy_test.go @@ -115,8 +115,6 @@ func TestForwardNeverInjectsInterceptSecret(t *testing.T) { } } -// TestForwardStripsHopHeaders: fixed and Connection-named hop headers die at -// this hop, in both directions. func TestForwardStripsHopHeaders(t *testing.T) { var gotHop, gotConn string upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -250,8 +248,6 @@ func (f fakeSecrets) Header(name string) (string, string, bool) { return hv[0], hv[1], ok } -// fixedDial ignores the requested address and connects to target, so a test -// policy can name any host while the bytes land on a local backend. func fixedDial(target string) DialFunc { return func(ctx context.Context, network, _ string) (net.Conn, error) { var d net.Dialer @@ -298,7 +294,6 @@ func readStatus(t *testing.T, br *bufio.Reader) string { return line } -// echoServer starts a one-shot TCP echo backend and returns its address. func echoServer(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/sandboxd/egress/secrets.go b/sandboxd/egress/secrets.go index deb43157..a26486ae 100644 --- a/sandboxd/egress/secrets.go +++ b/sandboxd/egress/secrets.go @@ -8,9 +8,7 @@ import ( "golang.org/x/net/http/httpguts" ) -// nonInjectable rejects secret headers the transport owns or strips — -// injecting one would audit as done while the request carries something -// else. Derived from the proxy's hop set so the two lists cannot drift. +// nonInjectable rejects secret headers the transport owns or strips. var nonInjectable = func() map[string]struct{} { m := map[string]struct{}{"host": {}, "content-length": {}} for _, h := range hopHeaders { @@ -19,9 +17,7 @@ var nonInjectable = func() map[string]struct{} { return m }() -// SecretSpec declares a node-side credential the proxy injects: Header is the -// request header it sets, valued from the ValueEnv env var — the literal -// never sits in the config file. +// SecretSpec declares a node-side credential the proxy injects, valued from ValueEnv. type SecretSpec struct { Name string `json:"name"` Header string `json:"header"` @@ -51,16 +47,12 @@ type resolvedSecret struct { value string } -// SecretStore is the resolved node-side credential registry, implementing the -// Proxy's Secrets interface. Values live only here — never in a policy or a -// gossiped struct. +// SecretStore is the resolved node-side credential registry; values live only here. type SecretStore struct { byName map[string]resolvedSecret } -// NewSecretStore resolves each spec's value; an unset or empty ValueEnv is an -// error, never a silently-empty credential, and a value a header cannot carry -// (CTLs would split the injected request) is rejected before it can be sent. +// NewSecretStore resolves each spec's value; an unset or invalid one is an error. func NewSecretStore(specs []SecretSpec) (*SecretStore, error) { byName := make(map[string]resolvedSecret, len(specs)) for _, s := range specs { diff --git a/sandboxd/engine/capacity_test.go b/sandboxd/engine/capacity_test.go index 5c1be760..5595fabe 100644 --- a/sandboxd/engine/capacity_test.go +++ b/sandboxd/engine/capacity_test.go @@ -5,8 +5,6 @@ import ( "testing" ) -// TestOnlyNodeWideFailuresCarryACapacitySignature guards the classifier. -// Parking on an ordinary clone failure would stall a healthy node over one bad VM. func TestOnlyNodeWideFailuresCarryACapacitySignature(t *testing.T) { for _, tc := range []struct { name string diff --git a/sandboxd/engine/cloneargs_test.go b/sandboxd/engine/cloneargs_test.go index ffa2540e..fe333429 100644 --- a/sandboxd/engine/cloneargs_test.go +++ b/sandboxd/engine/cloneargs_test.go @@ -64,8 +64,6 @@ func TestLifecycleArgsApplyDirectIOPolicy(t *testing.T) { } } -// TestEgressVMsSpreadOverEveryConfiguredShard pins the point of both shard -// lists: a shard the hash never picks is bridge capacity that does not exist. func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { shards := []string{"sbx0", "sbx1", "sbx2", "sbx3"} key := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} @@ -87,7 +85,6 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { counts[args[j+1]]++ } for _, s := range shards { - // The bound catches a starved shard, not non-uniformity. if counts[s] < 4000/len(shards)/2 { t.Errorf("shard %s got %d of 4000, want a fair share: %v", s, counts[s], counts) } @@ -96,8 +93,6 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { } } -// TestNetworkChoiceIsStableForAName guards restore and teardown: the record -// persists the network a VM was built on, so the choice must not move. func TestNetworkChoiceIsStableForAName(t *testing.T) { shards := []string{"a", "b", "c"} first := shardOf(shards, "sbx-pool1-42") @@ -108,9 +103,6 @@ func TestNetworkChoiceIsStableForAName(t *testing.T) { } } -// TestNetArgsHonorsTheLaneAndTheAttachment keeps the three attachment shapes -// distinct: no NIC on the none lane, a device on the bridge lane, and a -// single-entry list byte-for-byte the old scalar. func TestNetArgsHonorsTheLaneAndTheAttachment(t *testing.T) { none := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeMedium} egress := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 06c3e9b8..330264d2 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -28,7 +28,6 @@ const ( // RequiredCocoon carries the snapshot/store performance baseline. RequiredCocoon = "v0.5.2" - // Stale-create verdicts, mirroring cocoon's reconcile-stale-create verb. StaleCreateCollected StaleCreateOutcome = "collected" StaleCreateBusy StaleCreateOutcome = "busy" StaleCreateNotCreating StaleCreateOutcome = "not-creating" @@ -47,15 +46,11 @@ const ( infoMax = 4096 // info response frame cap outputTail = 400 - // portForwardMax caps the port_forward handshake reply line; the byte - // stream follows on the same conn. + // portForwardMax caps the port_forward handshake reply line. portForwardMax = 4096 ) -// capacitySignatures mean "this node cannot attach another VM", not "this VM -// failed"; matched as text because cocoon is a subprocess and the distinction -// survives only in its stderr. "exchange full" is EXFULL from a bridge at -// BR_MAX_PORTS — permanent until VMs are removed. +// capacitySignatures mean the node cannot attach another VM, not that this VM failed. var capacitySignatures = []string{ "exchange full", "no space left on device", @@ -73,15 +68,12 @@ type Engine struct { restoreMode types.RestoreMode } -// New returns a cocoon engine with node-wide network and disk policy. bridges -// and networks are the egress-lane shard lists (host bridge devices vs CNI -// conflists) egress VMs spread over; see shardOf. +// New returns a cocoon engine with node-wide network and disk policy. func New(bin string, bridges, networks []string, noDirectIO bool, restoreMode types.RestoreMode) *Engine { return &Engine{bin: bin, bridges: bridges, networks: networks, noDirectIO: noDirectIO, restoreMode: restoreMode} } -// Version reports cocoon's version string — a "vX.Y.Z" release or a -// "master-" dev build. +// Version reports cocoon's version string: a "vX.Y.Z" release or a "master-" dev build. func (e *Engine) Version(ctx context.Context) (string, error) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -97,9 +89,7 @@ func (e *Engine) Version(ctx context.Context) (string, error) { return "", fmt.Errorf("cocoon version: no Version line in output") } -// VersionWarning probes cocoon and returns a non-empty warning when its -// version is below RequiredCocoon (or unreadable); version is set when known. -// A dev build (no vX.Y.Z) is assumed current. +// VersionWarning returns a non-empty warning when cocoon's version is below RequiredCocoon. func (e *Engine) VersionWarning(ctx context.Context) (version, warning string) { v, err := e.Version(ctx) if err != nil { @@ -120,8 +110,7 @@ func (e *Engine) Clone(ctx context.Context, fromDir, name string, key types.Pool return parseRecord(ctx, out), nil } -// CloneSnap clones from a local-store snapshot by name — fork's fast path, -// which skips the export-to-dir copy that `Clone --from-dir` requires. +// CloneSnap clones from a local-store snapshot by name — fork's fast path. func (e *Engine) CloneSnap(ctx context.Context, snap, name string, key types.PoolKey) (types.VMRecord, error) { out, err := e.run(ctx, e.cloneSnapArgs(snap, name, key)...) if err != nil { @@ -130,8 +119,7 @@ func (e *Engine) CloneSnap(ctx context.Context, snap, name string, key types.Poo return parseRecord(ctx, out), nil } -// RunCold boots a VM from the template image (golden builds and cache-miss -// claims), returning its lifecycle record. +// RunCold boots a VM from the template image, returning its lifecycle record. func (e *Engine) RunCold(ctx context.Context, name string, key types.PoolKey) (types.VMRecord, error) { out, err := e.run(ctx, e.runColdArgs(name, key)...) if err != nil { @@ -146,8 +134,7 @@ func (e *Engine) Remove(ctx context.Context, name string) error { return err } -// ReconcileStaleCreate reclaims a creating-state record via cocoon's -// free-ops-lock predicate — safe against an in-flight clone where rm --force is not. +// ReconcileStaleCreate reclaims a creating-state record via cocoon's free-ops-lock predicate. func (e *Engine) ReconcileStaleCreate(ctx context.Context, name string) (StaleCreateOutcome, error) { out, err := e.run(ctx, "vm", "reconcile-stale-create", name, argOutput, formatJSON) if err != nil { @@ -168,15 +155,13 @@ func (e *Engine) SnapshotSave(ctx context.Context, vmName, snapName string) erro return err } -// Hibernate atomically snapshots a running VM under snapName and stops it, -// freeing its memory; Restore with the snapshot resumes it. +// Hibernate atomically snapshots a running VM under snapName and stops it. func (e *Engine) Hibernate(ctx context.Context, vmName, snapName string) error { _, err := e.run(ctx, "vm", "hibernate", argName, snapName, vmName) return err } -// Restore resumes a VM from a snapshot with its memory state and identity -// intact (cocoon reseeds entropy only on restore), returning its vsock UDS. +// Restore resumes a VM from a snapshot, returning its vsock UDS. func (e *Engine) Restore(ctx context.Context, vmName, snapRef string) (string, error) { out, err := e.run(ctx, e.restoreCmdArgs(vmName, snapRef)...) if err != nil { @@ -185,8 +170,7 @@ func (e *Engine) Restore(ctx context.Context, vmName, snapRef string) (string, e return parseRecord(ctx, out).VsockSocket, nil } -// SnapshotExport exports a snapshot into toDir (cocoon requires it absent or -// empty); the result pairs with `vm clone --from-dir`. +// SnapshotExport exports a snapshot into toDir, which cocoon requires absent or empty. func (e *Engine) SnapshotExport(ctx context.Context, snapName, toDir string) error { _, err := e.run(ctx, "snapshot", "export", snapName, "--to-dir", toDir) return err @@ -238,8 +222,7 @@ func (e *Engine) List(ctx context.Context, filters ...string) ([]types.VMRecord, return vms, nil } -// DialSilkd connects to a VM's silkd through the hybrid-vsock UDS: -// dial, send "CONNECT ", expect an "OK" reply. +// DialSilkd connects to a VM's silkd through the hybrid-vsock UDS. func (e *Engine) DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) { var d net.Dialer conn, err := d.DialContext(ctx, "unix", vsockSocket) @@ -267,8 +250,7 @@ func (e *Engine) DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, e return conn, nil } -// Probe polls until silkd completes an info round-trip — the claim-ready -// signal (probe what the product uses, not cocoon-agent). +// Probe polls until silkd completes an info round-trip — the claim-ready signal. func (e *Engine) Probe(ctx context.Context, vsockSocket string, timeout time.Duration) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -285,9 +267,7 @@ func (e *Engine) Probe(ctx context.Context, vsockSocket string, timeout time.Dur } } -// DialGuestPort opens a raw byte stream to 127.0.0.1:port inside the guest -// by driving silkd's port_forward handshake, then returns the connection -// carrying the relayed TCP bytes — the preview server proxies HTTP over it. +// DialGuestPort opens a raw byte stream to 127.0.0.1:port inside the guest. func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uint16) (net.Conn, error) { conn, err := e.DialSilkd(ctx, vsockSocket) if err != nil { @@ -304,7 +284,9 @@ func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uin _ = conn.Close() return nil, fmt.Errorf("write port_forward: %w", err) } - line, readErr := readLine(conn, portForwardMax) + // the forwarded stream's own reader takes the handshake reply, so an over-read stays buffered. + r := bufio.NewReaderSize(conn, portReadBuf) + line, readErr := readBufLine(r, portForwardMax) if readErr != nil { _ = conn.Close() if ctxErr := ctx.Err(); ctxErr != nil { @@ -321,13 +303,11 @@ func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uin _ = conn.Close() return nil, fmt.Errorf("port_forward %d: %s", port, respFail(resp)) } - return newGuestPortConn(conn), nil + return newGuestPortConn(conn, r), nil } func (e *Engine) cloneArgs(fromDir, name string, key types.PoolKey) []string { - // --pull: a checkpoint/template export carries only COW + memory; on a - // cross-node claim the base image blobs resolve locally or are pulled - // by digest. + // --pull: a checkpoint/template export carries only COW + memory, not the base image. args := []string{"vm", "clone", "--from-dir", fromDir, argName, name, "--pull", argOutput, formatJSON, e.directIOArg()} args = append(args, e.restoreArgs()...) return append(args, e.netArgs(name, key, false)...) @@ -403,8 +383,7 @@ func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string) error { if _, err = conn.Write(append(probe, '\n')); err != nil { return fmt.Errorf("write info: %w", err) } - // Unlike the handshake reader, buffered over-read is safe here: the conn - // is discarded after this one reply. + // buffered over-read is safe here: the conn is discarded after this one reply. reply, err := bufio.NewReader(io.LimitReader(conn, infoMax)).ReadBytes('\n') if err != nil { return fmt.Errorf("read info reply: %w", err) @@ -419,8 +398,7 @@ func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string) error { return nil } -// EgressSocketPath is the host UDS the VMM connects when the guest dials -// CID2:egressPort — sandboxd listens here to serve the egress proxy. +// EgressSocketPath is the host UDS the VMM connects when the guest dials CID2:egressPort. func EgressSocketPath(vsockSocket string) string { return fmt.Sprintf("%s_%d", vsockSocket, egressPort) } @@ -439,9 +417,7 @@ func CapacitySignature(err error) string { return "" } -// shardOf picks a shard by hashing the VM name, not by counter: the record -// persists the attachment a VM was built on, so the choice must be -// reproducible without process state. +// shardOf picks a shard by hashing the VM name, so the choice needs no process state. func shardOf(shards []string, name string) string { if len(shards) == 0 { return "" @@ -452,8 +428,7 @@ func shardOf(shards []string, name string) string { return shards[int(h.Sum32()>>1)%len(shards)] } -// respFail renders a non-success reply: the error frame's own text, or the -// unexpected frame's type. +// respFail renders a non-success reply: the error text, or the unexpected frame's type. func respFail(resp wire.Response) string { if errResp, ok := resp.(*wire.ErrorResp); ok { return errResp.Error() @@ -461,8 +436,7 @@ func respFail(resp wire.Response) string { return "unexpected frame " + resp.RespType() } -// parseRecord reads a lifecycle command's --output json VM record; best-effort, -// so an unparseable record yields the zero value (callers fall back to vm list). +// parseRecord is best-effort: an unparseable record yields the zero value. func parseRecord(ctx context.Context, out []byte) types.VMRecord { var rec types.VMRecord if err := json.Unmarshal(out, &rec); err != nil { @@ -472,8 +446,7 @@ func parseRecord(ctx context.Context, out []byte) types.VMRecord { return rec } -// belowFloor reports whether cocoon version v is below RequiredCocoon; -// comparable is false for a non-release build (no vX.Y.Z), assumed current. +// belowFloor reports whether v is below RequiredCocoon; comparable is false for a dev build. func belowFloor(v string) (below, comparable bool) { cur, ok := parseSemver(v) if !ok { @@ -503,8 +476,19 @@ func parseSemver(s string) ([3]int, bool) { return out, true } -// readLine reads byte-wise so nothing past the newline is consumed — -// the same conn carries the silkd protocol right after the handshake. +// readBufLine takes one line from r, capped at max bytes. +func readBufLine(r *bufio.Reader, max int) (string, error) { + line, err := r.ReadSlice('\n') + if err != nil { + return "", err + } + if len(line) > max { + return "", fmt.Errorf("reply exceeds %d bytes", max) + } + return string(line[:len(line)-1]), nil +} + +// readLine reads byte-wise so nothing past the newline is consumed. func readLine(conn net.Conn, max int) (string, error) { var sb strings.Builder var b [1]byte diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index d16967a4..790a9bfb 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -20,8 +20,7 @@ const ( func TestDialSilkdConsumesOnlyHandshake(t *testing.T) { path := sockPath(t) - // Sentinel rides in the same write as the handshake: if DialSilkd - // over-reads past the newline, the first Read below loses it. + listenMuxer(t, path, "OK 2048\nX") conn, err := New("cocoon", nil, nil, false, "").DialSilkd(t.Context(), path) @@ -112,7 +111,7 @@ func TestDialGuestPortCtxCancel(t *testing.T) { if _, err := io.WriteString(conn, "OK 2048\n"); err != nil { return } - _, _ = r.ReadString('\n') // consume port_forward, never answer + _, _ = r.ReadString('\n') <-hold }() @@ -132,10 +131,6 @@ func TestProbeTimeout(t *testing.T) { } } -// listenMuxer fakes the hybrid-vsock muxer on a UDS: per connection it -// consumes the CONNECT line, replies with handshake, then answers the first -// frame line of connection i with frames[i] (last frame repeats; no frames -// means handshake only). func listenMuxer(t *testing.T, path, handshake string, frames ...string) { t.Helper() ln, err := net.Listen("unix", path) @@ -173,8 +168,6 @@ func serveConn(conn net.Conn, handshake, frame string) { _, _ = io.WriteString(conn, frame) } -// sockPath returns a socket path in a fresh short-prefix temp dir — -// t.TempDir() paths can exceed darwin's 104-byte sun_path limit. func sockPath(t *testing.T) string { t.Helper() dir, err := os.MkdirTemp("", "sbx") diff --git a/sandboxd/engine/installca.go b/sandboxd/engine/installca.go index 9269b8dc..439cebc1 100644 --- a/sandboxd/engine/installca.go +++ b/sandboxd/engine/installca.go @@ -13,15 +13,11 @@ import ( const ( caCertGuestPath = "/usr/local/share/ca-certificates/sandbox-egress.crt" caBundlePath = "/etc/ssl/certs/ca-certificates.crt" - // guestExecPATH is set on the guest command because silkd starts it with an - // empty environment. + // guestExecPATH is set because silkd starts the guest command with an empty environment. guestExecPATH = "/usr/sbin:/usr/bin:/sbin:/bin" ) -// InstallCACert makes the guest trust the cluster root: drop the cert in the -// source dir and append it to the active bundle. It avoids update-ca-certificates -// on purpose — silkd runs before systemd-tmpfiles clears /tmp, where that script -// stages temp files. Off the claim path (golden build / cold provision). +// InstallCACert makes the guest trust the cluster root without update-ca-certificates. func (e *Engine) InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error { ctx, cancel := context.WithTimeout(ctx, cmdTimeout) defer cancel() diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 8fa3cb0c..8c502c43 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -2,10 +2,12 @@ package engine import ( "bufio" + "bytes" "encoding/json" "fmt" "io" "net" + "slices" "strings" "sync" "testing" @@ -117,7 +119,7 @@ func serveFakeSilkd(t *testing.T, path string) *fakeSilkd { func (f *fakeSilkd) serve(conn net.Conn) { defer func() { _ = conn.Close() }() r := bufio.NewReader(conn) - if _, err := r.ReadString('\n'); err != nil { // CONNECT line + if _, err := r.ReadString('\n'); err != nil { return } if _, err := io.WriteString(conn, "OK 2048\n"); err != nil { @@ -202,7 +204,7 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri func (f *fakeSilkd) handleList(conn net.Conn, path string) { f.mu.Lock() f.listCalls++ - entries, reply := append([]wire.DirEntry(nil), f.block...), f.listErr + entries, reply := slices.Clone(f.block), f.listErr f.mu.Unlock() if reply != "" { writeFakeSilkdResponse(conn, &wire.ErrorResp{Kind: wire.KindInternal, Message: reply}) @@ -225,7 +227,7 @@ func (f *fakeSilkd) handleRead(conn net.Conn, path string) { f.readMisses[path]-- ok = false } - data = append([]byte(nil), data...) + data = bytes.Clone(data) f.mu.Unlock() if reply != "" { writeFakeSilkdResponse(conn, &wire.ErrorResp{Kind: wire.KindInternal, Message: reply}) diff --git a/sandboxd/engine/portconn.go b/sandboxd/engine/portconn.go index c0b34b0f..16176a95 100644 --- a/sandboxd/engine/portconn.go +++ b/sandboxd/engine/portconn.go @@ -13,30 +13,26 @@ import ( ) const ( - // portWriteChunk keeps each data frame (base64 ×4/3 + envelope) well - // under silkd's 8MiB frame cap, so a large HTTP body is split across - // frames. + // portWriteChunk keeps each data frame well under silkd's 8MiB frame cap. portWriteChunk = 1 << 20 - // portReadBuf fits silkd's data frames (32KiB chunks, base64-expanded) - // in one buffered read. + // portReadBuf fits silkd's data frames in one buffered read. portReadBuf = 64 << 10 ) var portDataHead = []byte(`{"type":"data","data":"`) -// guestPortConn adapts silkd's newline-JSON data-frame port_forward channel to -// a plain net.Conn (base64 payloads), the server-side twin of the SDK's -// PortConn, so a reverse proxy can splice HTTP straight through it. +// guestPortConn adapts silkd's port_forward data frames to a plain net.Conn. type guestPortConn struct { net.Conn r *bufio.Reader wbuf []byte + rbuf []byte pending []byte } -func newGuestPortConn(conn net.Conn) *guestPortConn { - return &guestPortConn{Conn: conn, r: bufio.NewReaderSize(conn, portReadBuf)} +func newGuestPortConn(conn net.Conn, r *bufio.Reader) *guestPortConn { + return &guestPortConn{Conn: conn, r: r} } func (g *guestPortConn) Read(p []byte) (int, error) { @@ -45,7 +41,7 @@ func (g *guestPortConn) Read(p []byte) (int, error) { if err != nil { return 0, err } - if data, ok := fastPortData(line); ok { + if data, ok := g.fastPortData(line); ok { g.pending = data continue } @@ -74,8 +70,7 @@ func (g *guestPortConn) Write(p []byte) (int, error) { written := 0 for len(p) > 0 { n := min(len(p), portWriteChunk) - // The hot relay path renders into one reused buffer instead of - // allocating two frame-sized slices per chunk. + // the hot relay path reuses one buffer instead of allocating per chunk. g.wbuf = wire.AppendBulkRequest(g.wbuf, "data", p[:n]) if _, err := g.Conn.Write(g.wbuf); err != nil { return written, err @@ -86,10 +81,8 @@ func (g *guestPortConn) Write(p []byte) (int, error) { return written, nil } -// fastPortData slices the canonical data frame's base64 out without a JSON -// parse; the SDK's fastBulk sets the contract: only the exact canonical -// shape takes the slice, anything else falls back to the full parse. -func fastPortData(line []byte) ([]byte, bool) { +// fastPortData slices the canonical data frame's base64 out without a JSON parse. +func (g *guestPortConn) fastPortData(line []byte) ([]byte, bool) { after, ok := bytes.CutPrefix(line, portDataHead) if !ok { return nil, false @@ -98,10 +91,14 @@ func fastPortData(line []byte) ([]byte, bool) { if !ok || !bytes.Equal(rest, []byte("}\n")) { return nil, false } - out := make([]byte, base64.StdEncoding.DecodedLen(len(b64))) - n, err := base64.StdEncoding.Decode(out, b64) + if size := base64.StdEncoding.DecodedLen(len(b64)); cap(g.rbuf) < size { + g.rbuf = make([]byte, size) + } else { + g.rbuf = g.rbuf[:size] + } + n, err := base64.StdEncoding.Decode(g.rbuf, b64) if err != nil || base64.StdEncoding.EncodedLen(n) != len(b64) { return nil, false } - return out[:n], true + return g.rbuf[:n], true } diff --git a/sandboxd/engine/portconn_test.go b/sandboxd/engine/portconn_test.go index 0311e9b8..759488de 100644 --- a/sandboxd/engine/portconn_test.go +++ b/sandboxd/engine/portconn_test.go @@ -1,6 +1,7 @@ package engine import ( + "bufio" "bytes" "encoding/base64" "encoding/json" @@ -14,9 +15,8 @@ import ( func TestGuestPortConnFramesBothWays(t *testing.T) { client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - gc := newGuestPortConn(client) + gc := newGuestPortConn(client, bufio.NewReaderSize(client, portReadBuf)) - // Read: a silkd data frame decodes to raw bytes; done maps to EOF. go func() { frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("HTTP/1.1 200 OK"))}) _, _ = silk.Write(append(frame, '\n')) @@ -35,7 +35,7 @@ func TestGuestPortConnFramesBothWays(t *testing.T) { func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - gc := newGuestPortConn(client) + gc := newGuestPortConn(client, bufio.NewReaderSize(client, portReadBuf)) go func() { _, _ = gc.Write([]byte("GET / HTTP/1.1\r\n")) }() @@ -54,9 +54,6 @@ func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { } } -// TestGuestPortConnRoundTripsFixtures pins the engine's hand-rolled framing -// to the shared protocol corpus: portconn is not built on sdk/go/silkd, so -// without this test it could drift from the frames both real peers agree on. func TestGuestPortConnRoundTripsFixtures(t *testing.T) { const fixtureDir = "../../protocol/wire/fixtures/v1" fixture := func(name string) []byte { @@ -70,9 +67,8 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - gc := newGuestPortConn(client) + gc := newGuestPortConn(client, bufio.NewReaderSize(client, portReadBuf)) - // Read path: the corpus data frame decodes to its payload, done is EOF. go func() { _, _ = silk.Write(fixture("resp_data.json")) _, _ = silk.Write(fixture("resp_done.json")) @@ -86,19 +82,17 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { t.Errorf("fixture done frame: %v, want EOF", err) } - // Error frames surface their type, not a silent close. client2, silk2 := net.Pipe() t.Cleanup(func() { _ = client2.Close(); _ = silk2.Close() }) - gc2 := newGuestPortConn(client2) + gc2 := newGuestPortConn(client2, bufio.NewReaderSize(client2, portReadBuf)) go func() { _, _ = silk2.Write(fixture("resp_error.json")) }() if _, err := gc2.Read(buf); err == nil || err == io.EOF { t.Errorf("fixture error frame: %v, want a typed error", err) } - // Write path: the emitted frame is byte-identical to the corpus request. client3, silk3 := net.Pipe() t.Cleanup(func() { _ = client3.Close(); _ = silk3.Close() }) - gc3 := newGuestPortConn(client3) + gc3 := newGuestPortConn(client3, bufio.NewReaderSize(client3, portReadBuf)) go func() { _, _ = gc3.Write([]byte("hello")) }() line := make([]byte, 256) n, _ = silk3.Read(line) diff --git a/sandboxd/engine/silkd.go b/sandboxd/engine/silkd.go index c0c521f7..c9afd760 100644 --- a/sandboxd/engine/silkd.go +++ b/sandboxd/engine/silkd.go @@ -12,8 +12,7 @@ import ( const silkdChunk = 256 * 1024 -// silkdSession is a dialed silkd conn bound to a ctx, with wire-typed -// request/reply helpers. The hot port-forward relay (portconn.go) does not use it. +// silkdSession is a dialed silkd conn bound to a ctx, with wire-typed request/reply helpers. type silkdSession struct { conn net.Conn sc *bufio.Scanner @@ -58,8 +57,7 @@ func (e *Engine) dialSilkdSession(ctx context.Context, vsockSocket string) (*sil }, nil } -// silkdStream serves one request per dial: silkd handles a single request -// per connection. +// silkdStream serves one request per dial: silkd handles one request per connection. func (e *Engine) silkdStream(ctx context.Context, vsockSocket string, req wire.Request, onFrame func(wire.Response) error) error { s, err := e.dialSilkdSession(ctx, vsockSocket) if err != nil { @@ -119,8 +117,7 @@ func (e *Engine) silkdReadFile(ctx context.Context, vsockSocket, path string) ([ return data, nil } -// silkdStat answers with a single Stat frame and no terminal Done, so it -// cannot ride silkdStream. +// silkdStat answers with a single Stat frame and no Done, so it cannot ride silkdStream. func (e *Engine) silkdStat(ctx context.Context, vsockSocket, path string) error { s, err := e.dialSilkdSession(ctx, vsockSocket) if err != nil { diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index 1255bf04..16fa51d5 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -13,8 +13,7 @@ import ( ) const ( - // VolumeCallTimeout bounds one guest teardown exec; the pool scales its - // whole quiesce budget from it. + // VolumeCallTimeout bounds one guest teardown exec. VolumeCallTimeout = 2 * time.Second volumePollInterval = 10 * time.Millisecond @@ -69,8 +68,7 @@ func (e *Engine) MountVolume(ctx context.Context, vsockSocket, name, mount strin return nil } -// UnmountVolume flushes and detaches a guest mount, so a writable image's -// dirty state reaches the backing file before the VM is removed. +// UnmountVolume flushes and detaches a guest mount before the VM is removed. func (e *Engine) UnmountVolume(ctx context.Context, vsockSocket, mount string) error { ctx, cancel := context.WithTimeout(ctx, VolumeCallTimeout) defer cancel() @@ -80,8 +78,7 @@ func (e *Engine) UnmountVolume(ctx context.Context, vsockSocket, mount string) e return nil } -// SyncGuest flushes the guest page cache — the only flush left for a mount that -// refused to unmount, since a lazy unmount would keep writing behind us. +// SyncGuest flushes the guest page cache, the only flush left for a failed unmount. func (e *Engine) SyncGuest(ctx context.Context, vsockSocket string) error { ctx, cancel := context.WithTimeout(ctx, VolumeCallTimeout) defer cancel() diff --git a/sandboxd/main.go b/sandboxd/main.go index 14a58c92..a866bc9b 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -34,8 +34,7 @@ import ( const ( shutdownGrace = 5 * time.Second gossipInterval = time.Second - // Slowloris protection; ReadTimeout/WriteTimeout must stay zero — cold - // claims block up to the cold probe timeout and relays stream forever. + // Slowloris protection; ReadTimeout/WriteTimeout must stay zero for streaming relays. readHeaderTimeout = 5 * time.Second ) @@ -208,8 +207,7 @@ func startMesh(ctx context.Context, cfg *config.Config, mgr *pool.Manager) (*mes return msh, nil } -// gossipNodeState republishes this node's warm-pool counts, templates, and -// locally available volumes every tick so placement tracks filesystem changes. +// gossipNodeState republishes this node's counts, templates, and volumes every tick. func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { t := time.NewTicker(gossipInterval) defer t.Stop() diff --git a/sandboxd/mesh/epoch.go b/sandboxd/mesh/epoch.go index f9439505..c0ec53cc 100644 --- a/sandboxd/mesh/epoch.go +++ b/sandboxd/mesh/epoch.go @@ -9,9 +9,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// loadEpoch reads the persisted gossip epoch, or 0 when the file is absent, -// unreadable, or above MaxInt64 — a UnixNano-derived counter cannot legitimately -// exceed it, so a corrupt value falls back to the wall-clock seed. +// loadEpoch reads the persisted gossip epoch, or 0 when it is absent, unreadable, or corrupt. func loadEpoch(path string) uint64 { raw, err := os.ReadFile(path) //nolint:gosec // node-local data-dir path if err != nil { diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 83df2e27..d915d02c 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -20,12 +20,15 @@ import ( "github.com/projecteru2/core/log" ) -// leaveTimeout bounds the graceful-leave broadcast on shutdown so a wedged -// network can't hang the exit path. -const leaveTimeout = time.Second +const ( + // leaveTimeout bounds the graceful-leave broadcast so a wedged network cannot hang the exit path. + leaveTimeout = time.Second -// NodeState is one node's gossiped placement view. Epoch resolves merges: the -// higher epoch for a given node wins. + // epochLease keeps the durable floor ahead of the counter, off the gossip tick. + epochLease = 1024 +) + +// NodeState is one node's gossiped placement view; the higher Epoch wins a merge. type NodeState struct { NodeID string `json:"node_id"` Addr string `json:"addr"` // data-plane advertise address @@ -40,13 +43,12 @@ type NodeState struct { type Mesh struct { ml *memberlist.Memberlist epochPath string - // ctx is the daemon's, for logging inside memberlist callbacks — the - // delegate interfaces carry no context of their own. + // ctx is the daemon's, for logging inside memberlist callbacks. ctx context.Context - // updateMu serializes UpdateSelf end-to-end: two updates reading one epoch - // would both bump to E+1 and the loser's payload would be dropped. + // updateMu serializes UpdateSelf and guards leased; one shared epoch read drops a payload. updateMu sync.Mutex + leased uint64 mu sync.Mutex self NodeState @@ -54,18 +56,15 @@ type Mesh struct { live map[string]struct{} // members SWIM reports; gossip about anyone else is ignored } -// New starts a mesh member listening per cfg. selfAddr is the data-plane -// address peers should dial for a redirect; secretKey (16/24/32 bytes) enables -// gossip encryption when non-empty; dataDir holds the persisted epoch. +// New starts a mesh member listening per cfg. func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, secretKey []byte, dataDir string) (*Mesh, error) { epochPath := filepath.Join(dataDir, "mesh-epoch") - // Seed strictly above the persisted floor (the last epoch peers saw): - // seeding at it ties their stale copy and merge's `>` rejects the restart's - // fresh state. loadEpoch caps the floor so the +1 cannot wrap. + // seed strictly above the persisted floor: merge's `>` rejects a tie with a stale copy. epoch := max(uint64(time.Now().UnixNano()), loadEpoch(epochPath)+1) //nolint:gosec // UnixNano is positive for current times m := &Mesh{ ctx: ctx, epochPath: epochPath, + leased: epoch + epochLease, self: NodeState{ NodeID: nodeID, Addr: selfAddr, @@ -75,7 +74,7 @@ func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, s view: map[string]NodeState{}, live: map[string]struct{}{}, } - if err := m.persistEpoch(epoch); err != nil { + if err := m.persistEpoch(m.leased); err != nil { return nil, fmt.Errorf("persist mesh epoch: %w", err) } m.view[nodeID] = m.self @@ -105,9 +104,7 @@ func (m *Mesh) Join(seeds []string) error { return nil } -// UpdateSelf republishes this node's warm-pool counts, promoted-template set, -// and locally available volumes. An unchanged view does not bump the epoch. -// templates and volumes must arrive sorted: the compare is order-sensitive. +// UpdateSelf republishes this node's counts, templates, and volumes; both slices must be sorted. func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, volumes []string) { m.updateMu.Lock() defer m.updateMu.Unlock() @@ -118,12 +115,14 @@ func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, } epoch := m.self.Epoch + 1 m.mu.Unlock() - // Persist the candidate before publishing it: memberlist gossips self the - // instant it enters the view, so a crash before the write would strand peers - // on an epoch a backwards-clock restart can't beat. - if err := m.persistEpoch(epoch); err != nil { - log.WithFunc("mesh.UpdateSelf").Warnf(ctx, "persist epoch: %v", err) - return + // the durable floor must stay above anything gossiped: memberlist publishes self at once. + if epoch > m.leased { + leased := epoch + epochLease + if err := m.persistEpoch(leased); err != nil { + log.WithFunc("mesh.UpdateSelf").Warnf(ctx, "persist epoch: %v", err) + return + } + m.leased = leased } m.mu.Lock() m.self.Epoch = epoch @@ -134,8 +133,7 @@ func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, m.mu.Unlock() } -// SetSelfDigest records this node's cluster-invariant config digest so peers can -// detect divergence; call it once before Join, before any gossip ships. +// SetSelfDigest records this node's config digest; call it once before Join. func (m *Mesh) SetSelfDigest(digest string) { m.mu.Lock() defer m.mu.Unlock() @@ -143,8 +141,7 @@ func (m *Mesh) SetSelfDigest(digest string) { m.view[m.self.NodeID] = m.self } -// ConfigMismatches counts peers whose config digest differs from this node's — -// the gauge for alerting on a divergent cluster, recomputed per read. +// ConfigMismatches counts peers whose config digest differs from this node's. func (m *Mesh) ConfigMismatches() int { m.mu.Lock() defer m.mu.Unlock() @@ -160,9 +157,7 @@ func (m *Mesh) ConfigMismatches() int { return n } -// Candidates returns up to two peer addresses that report warm(keyHash) > 0, -// chosen power-of-two-choices to avoid herding every waiter onto one node. -// Self is never a candidate — the caller has already missed locally. +// Candidates returns up to two peers reporting warm(keyHash) > 0, self excluded. func (m *Mesh) Candidates(keyHash string) []string { return m.warmCandidates(keyHash, func(NodeState) bool { return true }) } @@ -172,22 +167,17 @@ func (m *Mesh) VolumeCandidates(keyHash string, names []string) []string { return m.warmCandidates(keyHash, func(st NodeState) bool { return containsAll(st.Volumes, names) }) } -// TemplateOwners returns up to two peer addresses whose gossiped template -// set contains keyHash — the redirect targets for a name-based claim or -// delete of a template this node does not hold. Self is excluded: the caller -// has already checked its own disk. +// TemplateOwners returns up to two peers whose gossiped template set contains keyHash. func (m *Mesh) TemplateOwners(keyHash string) []string { return m.owners(func(st NodeState) bool { return slices.Contains(st.Templates, keyHash) }) } -// VolumeOwners returns peers that currently advertise every requested volume. -// Self is excluded because the caller checks local availability first. +// VolumeOwners returns peers that advertise every requested volume; self is excluded. func (m *Mesh) VolumeOwners(names []string) []string { return m.owners(func(st NodeState) bool { return containsAll(st.Volumes, names) }) } -// TemplateVolumeOwners returns peers that hold both the promoted template and -// every requested volume, avoiding an incorrect intersection after truncation. +// TemplateVolumeOwners returns peers holding both the template and every requested volume. func (m *Mesh) TemplateVolumeOwners(keyHash string, names []string) []string { return m.owners(func(st NodeState) bool { return slices.Contains(st.Templates, keyHash) && containsAll(st.Volumes, names) @@ -214,8 +204,7 @@ func (m *Mesh) Members() []NodeState { return slices.Collect(maps.Values(m.view)) } -// PeerAddrs returns the data-plane addresses of the other nodes, for a -// client-side Lookup scatter. +// PeerAddrs returns the data-plane addresses of the other nodes. func (m *Mesh) PeerAddrs() []string { m.mu.Lock() defer m.mu.Unlock() @@ -291,8 +280,7 @@ func (m *Mesh) admit(nodeID string) { m.live[nodeID] = struct{}{} } -// forget drops a departed node from the placement view so redirects stop -// targeting a dead peer; SWIM detected the death, the view must follow. +// forget drops a departed node from the placement view so redirects stop targeting it. func (m *Mesh) forget(nodeID string) { m.mu.Lock() defer m.mu.Unlock() @@ -302,8 +290,7 @@ func (m *Mesh) forget(nodeID string) { } } -// merge absorbs a peer's view, keeping the higher epoch per node and never -// letting a peer overwrite this node's own authoritative self entry. +// merge absorbs a peer's view, keeping the higher epoch per node; self is never overwritten. func (m *Mesh) merge(states []NodeState) { m.mu.Lock() defer m.mu.Unlock() @@ -318,10 +305,7 @@ func (m *Mesh) merge(states []NodeState) { if ok && st.Epoch <= cur.Epoch { continue } - // Warn on each distinct divergent digest (not once per lifetime): a - // mismatched api_token/tenants/preview_secret/CA root 401s cross-node - // redirects and fails interception. Warn-only — refusing would partition - // a rolling credential rotation. + // warn-only: refusing a divergent digest would partition a rolling credential rotation. if m.self.Digest != "" && st.Digest != "" && st.Digest != m.self.Digest && (!ok || cur.Digest != st.Digest) { log.WithFunc("mesh.merge").Warnf(m.ctx, "peer %s config digest %s differs from this node's %s: cluster-invariant config diverges (redirects may 401, interception may fail)", @@ -343,8 +327,7 @@ func containsAll(have, need []string) bool { var _ memberlist.Delegate = (*delegate)(nil) -// delegate carries this node's full view on each memberlist push/pull sync, so -// state propagates transitively across the cluster. +// delegate carries this node's full view on each memberlist push/pull sync. type delegate Mesh func (d *delegate) NodeMeta(int) []byte { return nil } diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index b00aae50..9901d426 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -17,7 +17,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { m := newTestMesh(t, "a") mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}}) mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 3, Pools: map[string]int{"k": 5}}}) - mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) // stale + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) got := 0 for _, st := range m.Members() { @@ -33,7 +33,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil, nil) - // A peer claiming to be "a" must not clobber our authoritative self entry. + mergeStates(t, m, []NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) for _, st := range m.Members() { @@ -45,10 +45,10 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate + m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm + {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"other": 4}}, }) @@ -63,7 +63,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate + m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, @@ -87,7 +87,7 @@ func TestForgetPrunesDeadNode(t *testing.T) { if got := m.Candidates("k"); got != nil { t.Errorf("candidates after forget %v, want nil (b pruned)", got) } - // forgetting self is a no-op. + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) m.forget("a") if len(m.Members()) != 1 { @@ -126,7 +126,7 @@ func TestCandidatesPowerOfTwo(t *testing.T) { {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, }) - // With ≥2 warm peers, exactly two distinct candidates come back. + for range 20 { cands := m.Candidates("k") if len(cands) != 2 || cands[0] == cands[1] { @@ -148,8 +148,6 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { } a.mesh.UpdateSelf(t.Context(), map[string]int{"kk": 4}, []string{"tpl-hash"}, []string{"dataset"}) - // Push/pull sync propagates a's warm counts and template set to b within - // a few intervals. deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { cands := b.mesh.Candidates("kk") diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index 43c2b281..17a9dbd4 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -24,7 +24,7 @@ func TestEpochRoundTrip(t *testing.T) { func TestLoadEpochRejectsImplausibleValue(t *testing.T) { p := filepath.Join(t.TempDir(), "e") - // A saturated value must not seed a counter that +1 can't climb past. + if err := os.WriteFile(p, []byte("18446744073709551615"), 0o600); err != nil { t.Fatalf("write: %v", err) } @@ -35,12 +35,12 @@ func TestLoadEpochRejectsImplausibleValue(t *testing.T) { func TestEpochSeededAbovePersistedFloor(t *testing.T) { dir := t.TempDir() - huge := uint64(1) << 62 // above any plausible wall-clock nanos + huge := uint64(1) << 62 if err := storeEpoch(filepath.Join(dir, "mesh-epoch"), huge); err != nil { t.Fatalf("store: %v", err) } m := newBoundMesh(t, dir) - // Strictly above: seeding at the persisted value would tie peers' stale copy. + if m.self.Epoch != huge+1 { t.Errorf("epoch seeded %d, want persisted floor + 1 = %d (a backwards clock must land above peers' last-seen epoch)", m.self.Epoch, huge+1) } @@ -50,8 +50,9 @@ func TestUpdateSelfPersistFailKeepsOldState(t *testing.T) { dir := t.TempDir() m := newBoundMesh(t, dir) before := m.self.Epoch - // A non-existent dir makes the durable write fail: the epoch must not publish. + m.epochPath = filepath.Join(dir, "gone", "mesh-epoch") + m.leased = before m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) if m.self.Epoch != before { t.Errorf("self epoch advanced to %d on a failed persist, want %d held", m.self.Epoch, before) @@ -71,6 +72,29 @@ func TestUpdateSelfPersistsEpoch(t *testing.T) { } } +func TestUpdateSelfLeasesEpochFloor(t *testing.T) { + dir := t.TempDir() + m := newBoundMesh(t, dir) + path := filepath.Join(dir, "mesh-epoch") + seeded := loadEpoch(path) + if seeded <= m.self.Epoch { + t.Fatalf("seeded floor %d, want above the live epoch %d", seeded, m.self.Epoch) + } + for i := range 8 { + m.UpdateSelf(t.Context(), map[string]int{"k": i + 1}, nil, nil) + } + if got := loadEpoch(path); got != seeded { + t.Errorf("floor rewritten to %d during %d leased ticks, want %d held", got, 8, seeded) + } + m.mu.Lock() + m.self.Epoch = seeded + m.mu.Unlock() + m.UpdateSelf(t.Context(), map[string]int{"k": 99}, nil, nil) + if got := loadEpoch(path); got <= m.self.Epoch { + t.Errorf("floor %d not above the published epoch %d after crossing the lease", got, m.self.Epoch) + } +} + func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { m := newBoundMesh(t, t.TempDir()) base := m.self.Epoch @@ -81,8 +105,7 @@ func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { wg.Go(func() { m.UpdateSelf(t.Context(), a, nil, nil) }) wg.Go(func() { m.UpdateSelf(t.Context(), b, nil, nil) }) wg.Wait() - // Serialized updates with distinct payloads must both land: the loser - // of the old TOCTOU race silently dropped its payload and one bump. + want := base + uint64(2*(i+1)) //nolint:gosec // loop index is small and positive if m.self.Epoch != want { t.Fatalf("round %d: epoch %d, want %d (an update was dropped)", i, m.self.Epoch, want) diff --git a/sandboxd/netfilter/nft_linux.go b/sandboxd/netfilter/nft_linux.go index c85a5d40..3c493919 100644 --- a/sandboxd/netfilter/nft_linux.go +++ b/sandboxd/netfilter/nft_linux.go @@ -4,7 +4,6 @@ package netfilter import ( "fmt" - "slices" "strings" "github.com/google/nftables" @@ -14,9 +13,7 @@ import ( const tablePrefix = "sandbox_egress_" -// Lock drops a tap's guest-initiated egress (DHCP renewal excepted) at its -// ingress hook, forcing all egress through the vsock proxy. One netdev table -// per tap in the root netns; idempotent. +// Lock drops a tap's guest egress (DHCP excepted) at its ingress hook; idempotent. func Lock(tap string) error { _ = Unlock(tap) // a prior table for this tap would block a clean re-lock c, err := nftables.New() @@ -34,10 +31,7 @@ func Lock(tap string) error { Policy: &policy, Device: tap, }) - // Accept only IPv4 broadcast DHCP (sport 68 → dport 67, dst 255.255.255.255); - // the nfproto guard blocks an IPv6 datagram carrying 0xffffffff at the - // IPv4-daddr offset. The unchecked payload can only reach the local L2 - // segment, so egress VMs must not share a broadcast domain. + // accept only IPv4 broadcast DHCP; egress VMs must not share a broadcast domain. c.AddRule(&nftables.Rule{Table: t, Chain: ch, Exprs: []expr.Any{ &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.NFPROTO_IPV4}}, @@ -60,21 +54,23 @@ func Lock(tap string) error { return nil } -// EnsureLock applies Lock only if the tap has no table yet, so a restart can -// re-adopt a live VM's surviving lock without a re-lock window. -func EnsureLock(tap string) error { +// LockedTaps reports the taps that already carry a table, so a restart re-adopts them. +func LockedTaps() (map[string]bool, error) { c, err := nftables.New() if err != nil { - return fmt.Errorf("nft conn: %w", err) + return nil, fmt.Errorf("nft conn: %w", err) } tables, err := c.ListTablesOfFamily(nftables.TableFamilyNetdev) if err != nil { - return fmt.Errorf("list netdev tables: %w", err) + return nil, fmt.Errorf("list netdev tables: %w", err) } - if slices.ContainsFunc(tables, func(t *nftables.Table) bool { return t.Name == tablePrefix+tap }) { - return nil + locked := make(map[string]bool, len(tables)) + for _, t := range tables { + if tap, ok := strings.CutPrefix(t.Name, tablePrefix); ok { + locked[tap] = true + } } - return Lock(tap) + return locked, nil } // Unlock removes the tap's table; a no-op when it is already gone. @@ -88,9 +84,7 @@ func Unlock(tap string) error { return nil } -// SweepExcept drops every per-tap egress table whose tap is not in keep, clearing -// locks orphaned by VMs that went away while the daemon was down. It owns the -// whole sandbox_egress_* namespace in the root netns — one sandboxd per host. +// SweepExcept drops every per-tap egress table whose tap is not in keep; one sandboxd per host. func SweepExcept(keep map[string]bool) error { c, err := nftables.New() if err != nil { diff --git a/sandboxd/netfilter/nft_other.go b/sandboxd/netfilter/nft_other.go index e448ad79..92c241e5 100644 --- a/sandboxd/netfilter/nft_other.go +++ b/sandboxd/netfilter/nft_other.go @@ -4,12 +4,11 @@ package netfilter import "errors" -// Lock and Unlock fail off Linux (nftables is Linux-only), so a caller cannot -// mistake a missing lock for an applied one. +// Lock fails off Linux, so a caller cannot mistake a missing lock for an applied one. func Lock(string) error { return errors.ErrUnsupported } func Unlock(string) error { return errors.ErrUnsupported } -func EnsureLock(string) error { return errors.ErrUnsupported } +func LockedTaps() (map[string]bool, error) { return nil, errors.ErrUnsupported } func SweepExcept(keep map[string]bool) error { return nil } diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 168f7953..39a24bc5 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -37,8 +37,7 @@ func (m *Manager) archiveEnabledFor(key types.PoolKey) bool { return m.archiveAfterFor(key) > 0 } -// archiveOnce checkpoints hibernated claims idle past their archive threshold -// to the store and drops their local VM; the next access restores them. +// archiveOnce checkpoints hibernated claims idle past their archive threshold and drops their VM. func (m *Manager) archiveOnce(ctx context.Context) { if !m.archiveEnabled { return @@ -78,9 +77,7 @@ func (m *Manager) archiveOnce(ctx context.Context) { }() } -// archive publishes a hibernated claim's wake image to the store, then swaps -// its compute backing from local VM to store checkpoint (ArchiveCk), preserving -// id/token/tenant so a later access restores it (wakeArchived). +// archive swaps a hibernated claim's backing from local VM to a store checkpoint. func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // Must finish even if the sweep ctx is canceled, or the record diverges from the store. ctx = context.WithoutCancel(ctx) @@ -91,8 +88,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { if !ok { return errWokeMeanwhile } - // Pin the wake image before it exists: from the moment Publish makes it - // listable, a tenant/root delete would strand the claim it will back. + // pin the ck before Publish makes it listable, or a delete would strand the claim ckID := store.CheckpointID(randHex(8)) m.mu.Lock() m.pendingCks[ckID] = struct{}{} @@ -103,9 +99,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { if err != nil { return err } - // Re-validate under Transition+m.mu: the claim must still be hibernated on - // the exact snapshot the export captured — a wake, or a wake+re-hibernate - // landing inside the publish window, means the checkpoint holds stale state. + // the claim must still hold the exact snapshot the export captured, or the ck is stale sb.Transition.Lock() m.mu.Lock() if m.claimed[sb.ID] != sb || sb.HibernateSnap != srcSnap || sb.ArchiveCk != "" { @@ -122,7 +116,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { } else { sb.Deadline = time.Time{} // keep forever: no retention (reap skips a zero deadline) } - js := m.store.snapshot(m.claimed) + js := m.store.set(sb) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { // Roll back so memory matches the still-durable hibernated record; drop the orphan ck. @@ -130,15 +124,14 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { sb.ArchiveCk = "" sb.HibernateSnap, sb.VsockSocket, sb.VMName = snap, sock, vmName sb.Deadline = prevDeadline - rb := m.store.snapshot(m.claimed) + rb := m.store.set(sb) m.mu.Unlock() sb.Transition.Unlock() m.recommit(ctx, rb) m.deleteOrphanArchiveCk(ctx, ck.ID) return fmt.Errorf("archive %s: persist claims: %w", sb.ID, saveErr) } - // Disarm under Transition so a wake that runs the instant we release it - // re-arms cleanly instead of being clobbered by a late disarm. + // disarm under Transition so a wake right after the release is not clobbered by a late disarm m.disarmEgress(sb.ID, true) sb.Transition.Unlock() // Committed: the store ck is authoritative now; reclaim the local footprint. @@ -149,8 +142,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { return nil } -// wakeArchived restores an archived claim from its store checkpoint into a -// fresh local VM, keeping id/token/tenant; the caller holds the Transition lock. +// wakeArchived restores an archived claim into a fresh local VM; caller holds Transition. func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, error) { // Egress never archives; a corrupt archived egress claim must fail closed. if sb.Key.Net == types.NetEgress { @@ -158,10 +150,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, } ctx = context.WithoutCancel(ctx) ck := sb.ArchiveCk - // Exclusive, not shared: this path deletes the consumed ck below. The - // lock releases before eviction is considered, below — evicting while - // still held would let a concurrent recLock for ck split onto a - // different mutex. + // evicting while the lock is held would split a concurrent recLock onto a different mutex l := m.recLock(ck) l.Lock() consumed := false @@ -206,8 +195,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, log.WithFunc("pool.wakeArchived").Warnf(ctx, "clear archive ck %s: %v", ck, clearErr) } } - // Only the none lane reaches here (the egress guard above fails closed), so - // this rebinds the none-lane proxy; there is no NIC to re-lock. + // only the none lane reaches here, so there is no NIC to re-lock if proxyErr := m.armEgressProxy(ctx, sb); proxyErr != nil { log.WithFunc("pool.wakeArchived").Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) } @@ -219,9 +207,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, return built.VsockSocket, nil } -// commitWake swaps an archived record back to a live VM if still claimed -// (Release/reap skip the Transition lock). A failed persist rolls the record -// back to archived so memory matches disk and the ck stays pinned. +// commitWake swaps an archived record back to a live VM if the claim is still live. func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, sock string) (live, saved bool) { m.mu.Lock() live = m.claimed[sb.ID] == sb @@ -232,12 +218,12 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc ck, deadline := sb.ArchiveCk, sb.Deadline sb.VMName, sb.VsockSocket, sb.ArchiveCk = vmName, sock, "" sb.Deadline = time.Now().Add(clampTTL(0)) // a woken sandbox is a fresh lease - js := m.store.snapshot(m.claimed) + js := m.store.set(sb) m.mu.Unlock() if err := m.store.commit(js); err != nil { m.mu.Lock() sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.Deadline = "", "", ck, deadline - rb := m.store.snapshot(m.claimed) + rb := m.store.set(sb) m.mu.Unlock() m.recommit(ctx, rb) log.WithFunc("pool.commitWake").Warnf(ctx, "persist claims: %v", err) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 7d0fdc1d..220a7f05 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -20,10 +20,6 @@ const ( testArchiveReap = "reap" ) -// TestArchivePublishWindowPinsCheckpoint guards the pre-pin: between an -// archive's checkpoint publish and the ArchiveCk commit, the checkpoint must -// be invisible to listings and refuse deletion — a delete landing in that -// window stranded the claim. func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -53,11 +49,6 @@ func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { } } -// TestArchiveWakeRehibernateWindowAborts guards the commit's identity check: a -// wake + re-hibernate landing inside the publish window leaves HibernateSnap -// non-empty but pointing at NEWER state than the exported checkpoint — -// committing would archive stale data and destroy the only copy of the new -// snapshot. func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -105,10 +96,6 @@ func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { } } -// TestArchiveCkHiddenAcrossNodes: on a shared store, another node has no -// in-memory pin for this node's archive wake images — the meta Archive flag -// must hide them from its listings, refuse its deletes, and spare its TTL -// sweep, or a routine cleanup elsewhere strands the claim. func TestArchiveCkHiddenAcrossNodes(t *testing.T) { shared := t.TempDir() mkMgr := func() *Manager { @@ -142,11 +129,6 @@ func TestArchiveCkHiddenAcrossNodes(t *testing.T) { } } -// TestReconcileReclaimsOrphanArchiveCk: a crash between an archive's publish -// and its journal commit leaves a flagged record listings hide and deletes -// refuse; reconcile identifies it by the journal (the sandbox is ours, under -// a different ArchiveCk) and reclaims it. A flagged record of an unknown -// sandbox is left alone. func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { eng := newFakeEngine() dataDir := t.TempDir() @@ -188,8 +170,6 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { } } -// TestClaimCheckpointRefusesArchive: wake images are lifecycle-internal — -// branching from one would race its consumption by the owner's wake. func TestClaimCheckpointRefusesArchive(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -204,9 +184,6 @@ func TestClaimCheckpointRefusesArchive(t *testing.T) { } } -// TestArchiveLifecycleRoundTrip exercises the real sweep wiring: -// claim→idle-hibernate→archive→wake, and asserts the id/token survive the -// store round-trip while the local VM and its wake image are reclaimed. func TestArchiveLifecycleRoundTrip(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -257,12 +234,10 @@ func TestArchiveLifecycleRoundTrip(t *testing.T) { }) } -// TestArchiveTTLSweepSparesLiveCheckpoint guards the fix where the checkpoint -// TTL sweep deleted the store image backing a live archived claim, bricking it. func TestArchiveTTLSweepSparesLiveCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) - m.ckptTTL = time.Nanosecond // every checkpoint is instantly age-expired + m.ckptTTL = time.Nanosecond sb := mustClaim(t, m, testKey) id, token := sb.ID, sb.Token mustArchive(t, m, sb) @@ -277,8 +252,6 @@ func TestArchiveTTLSweepSparesLiveCheckpoint(t *testing.T) { } } -// TestArchiveKeepsForeverWhenDeleteZero guards the fix where a zero -// archive-delete window purged archives immediately instead of keeping them. func TestArchiveKeepsForeverWhenDeleteZero(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(0)) @@ -303,9 +276,6 @@ func TestArchiveKeepsForeverWhenDeleteZero(t *testing.T) { } } -// TestArchiveRetentionPurge covers the reapPurge path: an archive past its -// retention deadline drops from claims, deletes its checkpoint, and releases -// its tenant slot. func TestArchiveRetentionPurge(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -318,14 +288,12 @@ func TestArchiveRetentionPurge(t *testing.T) { ck := sb.ArchiveCk m.mu.Lock() - sb.Deadline = time.Now().Add(-time.Second) // retention window elapsed + sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() m.reapOnce(t.Context()) waitFor(t, func() bool { return archivedCount(m) == 0 }) - // archiveDeletes is the purge goroutine's last side effect, sequenced after - // the checkpoint delete; waiting on it makes both observable (a bare read - // races the async purge, flaking under -race). + waitFor(t, func() bool { return m.Counters().ArchiveDeletes > 0 }) if ckExists(t, m, ck) { t.Error("purge did not delete the archived checkpoint") @@ -340,9 +308,6 @@ func TestArchiveRetentionPurge(t *testing.T) { }) } -// TestReleaseArchivedDeletesCheckpoint guards the fix where releasing an -// archived claim orphaned its store checkpoint and called Remove on an empty -// VM name. func TestReleaseArchivedDeletesCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -525,9 +490,6 @@ func TestArchiveWakeRequiresDeleteMarker(t *testing.T) { } } -// Not a synctest candidate: each subtest holds m.store.mu itself to force the -// release/reap goroutine to block acquiring it — same plain-sync.Mutex shape -// as TestArchiveDeleteRetryRechecksWakeRollback, already verified to deadlock. func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { for _, action := range []string{testArchiveRelease, testArchiveReap} { t.Run(action, func(t *testing.T) { @@ -542,11 +504,11 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { m.mu.Unlock() } - m.store.mu.Lock() + m.store.writeMu.Lock() locked := true t.Cleanup(func() { if locked { - m.store.mu.Unlock() + m.store.writeMu.Unlock() } }) done := make(chan error, 1) @@ -570,7 +532,7 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { t.Error("retry deleted the checkpoint before the claims commit") } m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") - m.store.mu.Unlock() + m.store.writeMu.Unlock() locked = false if err := <-done; action == testArchiveRelease && err == nil { t.Fatal("release succeeded despite a persist failure") @@ -639,10 +601,6 @@ func TestArchiveRemovalRequiresDeleteMarker(t *testing.T) { } } -// Not a synctest candidate: this test holds m.store.mu itself to force the -// wake goroutine to block acquiring it — a plain sync.Mutex, never "durably -// blocked" by synctest's rules, so the waitFor below can't fast-forward past -// it. Verified as a real deadlock (times out), not a flake. func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -650,11 +608,11 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { mustArchive(t, m, sb) id, token, ck := sb.ID, sb.Token, sb.ArchiveCk - m.store.mu.Lock() + m.store.writeMu.Lock() locked := true t.Cleanup(func() { if locked { - m.store.mu.Unlock() + m.store.writeMu.Unlock() } }) woke := make(chan error, 1) @@ -681,7 +639,7 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { return m.recRefs[ck] >= 2 }) m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") - m.store.mu.Unlock() + m.store.writeMu.Unlock() locked = false if err := <-woke; err == nil { t.Fatal("wake succeeded despite a persist failure") @@ -702,8 +660,6 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { } } -// TestIdleOnceSkipsArchived guards the fix where the idle sweep tried to -// hibernate an archived (VM-less) claim, corrupting its record. func TestIdleOnceSkipsArchived(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -724,9 +680,6 @@ func TestIdleOnceSkipsArchived(t *testing.T) { }) } -// TestDeleteCheckpointCannotBrickArchive guards the fix where the archive -// image was listable and deletable through the ordinary checkpoint API, -// letting a tenant permanently strand its own archived sandbox. func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -735,8 +688,6 @@ func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { mustArchive(t, m, sb) ck := sb.ArchiveCk - // The archive image is lifecycle-internal — it must not surface as a user - // checkpoint, or be deletable as one. if ckpts, err := m.Checkpoints(t.Context(), ""); err != nil || len(ckpts) != 0 { t.Fatalf("Checkpoints listed %d records, want the archive image hidden (%v)", len(ckpts), err) } @@ -751,9 +702,6 @@ func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { } } -// TestArchivePersistFailureRollsBack guards the durability fix where archive -// destroyed the local VM+snapshot before its transition reached disk: a failed -// persist must roll the record back to hibernated and keep the local backing. func TestArchivePersistFailureRollsBack(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -784,9 +732,6 @@ func TestArchivePersistFailureRollsBack(t *testing.T) { }) } -// TestReleaseArchivedRollsBackOnPersistFailure guards the durability fix where -// a release whose removal did not persist must roll back — the claim survives -// and its ck stays pinned, so a restart still wakes it. func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -810,9 +755,6 @@ func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { }) } -// TestWakeArchivedRollsBackOnPersistFailure guards the durability fix where a -// wake whose cleared-ArchiveCk record did not persist must roll back to -// archived, keeping the ck pinned. func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -839,8 +781,6 @@ func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { }) } -// TestReapPurgeRollsBackOnPersistFailure guards the same rollback on the reaper: -// a retention purge that does not persist keeps the claim and its pinned ck. func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -850,7 +790,7 @@ func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { mustArchive(t, m, sb) ck := sb.ArchiveCk m.mu.Lock() - sb.Deadline = time.Now().Add(-time.Second) // retention elapsed + sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() breakStore(t, m) @@ -868,8 +808,6 @@ func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { }) } -// TestArchiveOnceSkipsInFlight guards the in-flight dedup: a sandbox already -// being archived (marked by a racing reap) must not be exported twice. func TestArchiveOnceSkipsInFlight(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -881,7 +819,7 @@ func TestArchiveOnceSkipsInFlight(t *testing.T) { backdate(m, sb, time.Hour) m.mu.Lock() - m.archiving[sb.ID] = struct{}{} // a reap-triggered archive is already exporting it + m.archiving[sb.ID] = struct{}{} m.mu.Unlock() exportsBefore := eng.exportCount() @@ -896,10 +834,6 @@ func TestArchiveOnceSkipsInFlight(t *testing.T) { }) } -// TestArchiveWakeEvictsRecLock proves the archive->wake cycle leaves no -// recLocks residue: wakeArchived recLocks the consumed ck, so it must evict it -// after the delete or a repeatedly idle->archive->wake workload leaks a lock -// per cycle. func TestArchiveWakeEvictsRecLock(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -907,25 +841,18 @@ func TestArchiveWakeEvictsRecLock(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } - countLocks := func() int { - n := 0 - m.recLocks.Range(func(_, _ any) bool { n++; return true }) - return n - } - base := countLocks() + base := lockCount(m) if err := m.archive(t.Context(), sb); err != nil { t.Fatalf("archive: %v", err) } if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake archived: %v", err) } - if got := countLocks(); got != base { + if got := lockCount(m); got != base { t.Errorf("recLocks grew %d->%d over an archive+wake cycle, want no growth", base, got) } } -// stallingStore parks Publish after the record becomes visible, exposing the -// window between an archive's publish and its ArchiveCk commit. type stallingStore struct { store.Store published chan string @@ -954,15 +881,11 @@ func (s *failingDeleteStore) Delete(_ context.Context, id string) error { return errors.New("delete failed") } -// breakStore points the claim store at a path whose parent is missing, so the -// next save fails — the fault used to exercise the persist-failure paths. func breakStore(t *testing.T, m *Manager) { t.Helper() m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") } -// healStore repairs breakStore's path and waits for the detached recommit to -// converge, so its retry loop does not outlive the test. func healStore(t *testing.T, m *Manager) { t.Helper() if err := os.MkdirAll(filepath.Dir(m.store.path), 0o750); err != nil { @@ -976,8 +899,6 @@ func archivedCount(m *Manager) int { return g.Archived } -// archivePool is the pool shape the archive tests share: idle→hibernate at 1s, -// archive at 2s, a long retention window (overridden per test). func archivePool(delete int) config.PoolSpec { return config.PoolSpec{ PoolKey: testKey, @@ -988,9 +909,6 @@ func archivePool(delete int) config.PoolSpec { } } -// mustArchive drives a fresh claim to the archived state deterministically -// (hibernate then archive directly), bypassing the sweep thresholds so a test -// can start from a known archived record. func mustArchive(t *testing.T, m *Manager, sb *types.Sandbox) { t.Helper() if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { @@ -1016,8 +934,6 @@ func archiveCkMarked(m *Manager, ck string) bool { return err == nil } -// pinnedHidden reports whether the store holds exactly one checkpoint and it is -// hidden from listings — i.e. a live claim's ArchiveCk still pins it. func pinnedHidden(t *testing.T, m *Manager, ck string) bool { t.Helper() ckpts, err := m.Checkpoints(t.Context(), "") diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index bc294b61..75e66e27 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -16,8 +16,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// DeleteScope selects how far a checkpoint delete reaches. Fleet-wide is the -// zero value, so the narrower local-only delete must be asked for. +// DeleteScope selects how far a checkpoint delete reaches; fleet-wide is the zero value. type DeleteScope int const ( @@ -31,9 +30,7 @@ var ( ErrHealBusy = errors.New("too many checkpoint heals in flight") ) -// Checkpoint captures a claimed sandbox's state under a fresh id; the source -// keeps running (a hibernated one is captured from its wake image). tenant -// attributes the record; empty means the operator (root). +// Checkpoint captures a claimed sandbox's state under a fresh id; the source keeps running. func (m *Manager) Checkpoint(ctx context.Context, id string, cred Cred, name, tenant string) (types.Checkpoint, error) { sb, ok := m.resolve(id, cred) if !ok { @@ -59,15 +56,9 @@ func (m *Manager) Checkpoint(ctx context.Context, id string, cred Cred, name, te return ckpt, nil } -// ClaimCheckpoint provisions a fresh claim cloned from a checkpoint — a -// branch. The checkpoint's recorded key applies (snapshots pin size and -// lane); the checkpoint itself is read-only and reusable. The claim is -// attributed to tenant, not the checkpoint's recorder — the unguessable id -// is the capability to branch. +// ClaimCheckpoint provisions a fresh claim cloned from a checkpoint, attributed to tenant. func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { - // Reject a bad or unknown id before recLock: a rejected id must not leave - // a lock-map entry (only a delete evicts one). It precedes quota: a full - // node must still answer "not here". + // a rejected id must not leave a lock-map entry, so resolve before recLock and quota ckpt, err := m.loadCheckpoint(ctx, ckptID) if err != nil { return nil, err @@ -78,15 +69,12 @@ func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.D return m.claimLoaded(ctx, ckpt, ttl, tenant) } -// ClaimCheckpointHeal claims a checkpoint this node does not hold locally, -// pulling it from a peer first. The server calls it only after the local -// claim missed and a redirect could not answer. +// ClaimCheckpointHeal claims a checkpoint held only by a peer, pulling it first. func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { if m.healer == nil || !store.CheckpointIDRe.MatchString(ckptID) { return nil, ErrUnknownCheckpoint } - // Resolving here means moving a guest memory image, so quota rejects a - // full node before that cost, not after. + // resolving moves a guest memory image, so quota rejects a full node before that cost if err := m.overQuota(1, tenant); err != nil { return nil, err } @@ -97,11 +85,7 @@ func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl ti return m.claimLoaded(ctx, ckpt, ttl, tenant) } -// Checkpoints lists the store's checkpoints, newest first — on a shared -// backend (a FUSE mount, a bucket), that is the cluster's set, not one -// node's. A non-empty tenant filters to that tenant's records; empty (root) -// lists everything. Checkpoints backing a live archived claim are hidden: -// they are lifecycle-internal wake images, not user checkpoints. +// Checkpoints lists tenant's checkpoints, newest first, hiding archive wake images. func (m *Manager) Checkpoints(ctx context.Context, tenant string) ([]types.Checkpoint, error) { metas, err := m.ckpts.Metas(ctx) if err != nil { @@ -123,13 +107,9 @@ func (m *Manager) Checkpoints(ctx context.Context, tenant string) ([]types.Check return ckpts, nil } -// DeleteCheckpoint removes a checkpoint's snapshot and record, then -// broadcasts to peers when fleet-scoped so a healed copy does not outlive it. -// A tenant may delete only its own records — anything else answers -// ErrUnknownCheckpoint, never a hint the id exists; root deletes anything. +// DeleteCheckpoint removes a checkpoint's snapshot and record, broadcasting when fleet-scoped. func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { - // Reject a bad id before recLock: a rejected id must not leave a - // lock-map entry. + // a rejected id must not leave a lock-map entry if !store.CheckpointIDRe.MatchString(ckptID) { return ErrUnknownCheckpoint } @@ -144,8 +124,7 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s if !tenantOwns(tenant, ckpt.Tenant) { return ErrUnknownCheckpoint } - // ckpt.Archive guards wake images across every node sharing the store; - // the pin set guards this node's not-yet-committed ones. + // ckpt.Archive guards wake images store-wide; the pin set guards uncommitted local ones if _, pinned := m.pinnedArchiveCks()[ckptID]; pinned || ckpt.Archive { return ErrUnknownCheckpoint // backs an archived sandbox, not a deletable checkpoint } @@ -159,15 +138,13 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s return nil } -// HasCheckpoint answers the ownership probe: a branchable local record, never -// a fetch. +// HasCheckpoint answers the ownership probe: a branchable local record, never a fetch. func (m *Manager) HasCheckpoint(ctx context.Context, ckptID string) bool { ckpt, err := m.loadCheckpoint(ctx, ckptID) return err == nil && !ckpt.Archive } -// FetchCheckpoint materializes a checkpoint's export for a peer transfer, -// returning the local directory, its meta, and the release to call when done. +// FetchCheckpoint materializes a checkpoint's export for a peer transfer. func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, []byte, func(), error) { if _, err := m.loadCheckpoint(ctx, ckptID); err != nil { return "", nil, nil, err @@ -183,15 +160,11 @@ func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, [ } return "", nil, nil, fmt.Errorf("fetch checkpoint: %w", err) } - // The read lock spans the transfer: a delete must not pull the export out - // from under a stream already writing it to a peer. + // the read lock spans the transfer so a delete cannot pull the export from under it return dir, meta, func() { release(); l.RUnlock(); m.recDone(ckptID) }, nil } -// publishCheckpoint stages the sandbox's exported state, writes the meta, and -// publishes it to the store, returning the record and the source snapshot the -// export captured. Shared by Checkpoint and archive; a hibernated source -// exports its wake image directly (no VM start — refill.sourceSnap). +// publishCheckpoint stages, writes the meta, and publishes, returning the record and source snap. func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID, name, tenant string, archive bool) (types.Checkpoint, string, error) { ckpt := types.Checkpoint{ ID: ckID, @@ -224,8 +197,7 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID return ckpt, srcSnap, nil } -// claimLoaded is the shared body once ckpt's meta is resolved: it re-fetches -// under the record lock, so a delete racing the pre-check cannot slip through. +// claimLoaded re-fetches under the record lock, so a delete racing the pre-check cannot slip in. func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl time.Duration, tenant string) (*types.Sandbox, error) { l := m.recLock(ckpt.ID) l.RLock() @@ -257,10 +229,7 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti return out, err } -// healCheckpoint dedups concurrent heals of ckptID onto one flight (which -// owns its own staging dir — never a caller's) and lets THIS call abandon -// waiting the moment ctx is done; the flight itself runs detached (runHeal), -// so a client hanging up never abandons a transfer already paid for. +// healCheckpoint dedups concurrent heals onto one detached flight; only the wait honors ctx. func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { return ckpt, nil @@ -279,10 +248,7 @@ func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Chec } } -// runHeal is healCheckpoint's flight body: it stages and pulls WITHOUT -// holding ckptID's record lock — a heal budget runs up to 30 minutes, and -// holding the lock across it would pin every same-id operation behind an -// uncancellable wait. +// runHeal stages and pulls without ckptID's record lock: a pull runs up to 30 minutes. func (m *Manager) runHeal(ckptID string) (types.Checkpoint, error) { select { case m.healSem <- struct{}{}: @@ -325,18 +291,14 @@ func (m *Manager) runHeal(ckptID string) (types.Checkpoint, error) { return m.loadCheckpoint(ctx, ckptID) } -// markHealPending records ckptID as staging or pulling, unlocked, so -// vetoIfHealPending knows a concurrent delete must veto rather than ignore -// it. Bounded by healSem: at most maxConcurrentHeals entries ever exist. +// markHealPending records ckptID as staging or pulling, so a delete can veto it. func (m *Manager) markHealPending(ckptID string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() m.healPending[ckptID] = struct{}{} } -// clearHealPending un-marks ckptID and reports whether vetoIfHealPending -// vetoed it in the meantime; call under ckptID's recLock, so the check and -// the eventual publish decide together. +// clearHealPending un-marks ckptID and reports a veto; call under ckptID's recLock. func (m *Manager) clearHealPending(ckptID string) (aborted bool) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -347,10 +309,7 @@ func (m *Manager) clearHealPending(ckptID string) (aborted bool) { return aborted } -// vetoIfHealPending marks ckptID aborted when a heal is pending, so the -// heal's locked decide phase (clearHealPending) abandons its publish instead -// of resurrecting a checkpoint this delete just answered "not here" for. A -// no-op when no heal is pending, so an unrelated miss leaves no residue. +// vetoIfHealPending aborts a pending heal so its publish cannot resurrect a deleted checkpoint. func (m *Manager) vetoIfHealPending(ckptID string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -359,9 +318,7 @@ func (m *Manager) vetoIfHealPending(ckptID string) { } } -// pinnedArchiveCks is the set of checkpoint ids backing a live archived claim -// or an archive transition in flight (pendingCks): wake images the listing hides -// and delete/TTL must spare (deleting one would strand its sandbox). +// pinnedArchiveCks is the set of wake-image ids the listing hides and delete/TTL must spare. func (m *Manager) pinnedArchiveCks() map[string]struct{} { m.mu.Lock() defer m.mu.Unlock() @@ -377,17 +334,14 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { return pinned } -// sweepExpiredCheckpoints ages out checkpoints older than the configured -// TTL; explicit deletes never wait for it. It runs detached from the Run loop. +// sweepExpiredCheckpoints ages out checkpoints older than the configured TTL. func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { if !m.ckptSweeping.CompareAndSwap(false, true) { return } defer m.ckptSweeping.Store(false) logger := log.WithFunc("pool.sweepExpiredCheckpoints") - // Checkpoints already hides archive images backing a live claim, so the - // TTL never reaches one; their retention is the claim's own Deadline - // (reapPurge). + // Checkpoints hides archive images, so the TTL never reaches one; reapPurge retires them ckpts, err := m.Checkpoints(ctx, "") if err != nil { logger.Error(ctx, err, "list for retention") @@ -404,9 +358,7 @@ func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { } } -// deleteCkLocked removes a checkpoint under its record lock, so the delete -// never runs beneath an in-flight branch clone, heal, or archived wake; the -// lock is released before eviction is considered (see recDoneEvict). +// deleteCkLocked removes a checkpoint under its record lock, releasing it before eviction. func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { l := m.recLock(ckID) l.Lock() @@ -434,8 +386,7 @@ func (m *Manager) loadCheckpoint(ctx context.Context, ckptID string) (types.Chec return parseCheckpoint(raw) } -// validateHealedCheckpoint checks a staged pull's shape before publishing it: -// an unreadable or misattributed record would suppress every later heal. +// validateHealedCheckpoint checks a staged pull's shape: a bad record suppresses later heals. func validateHealedCheckpoint(staging, wantID string) error { raw, err := os.ReadFile(filepath.Join(staging, store.MetaFile)) //nolint:gosec // staging dir is this manager's own if err != nil { @@ -455,17 +406,14 @@ func validateHealedCheckpoint(staging, wantID string) error { return fmt.Errorf("healed record %s has an invalid key: %w", wantID, keyErr) } if !ckpt.Key.Capturable() { - // A well-formed egress key is not branchable: publishing it would - // poison the id and suppress a good owner; reject, try the next owner. + // publishing a non-branchable key would poison the id and suppress a good owner return fmt.Errorf("healed record %s has a non-branchable (egress) key", wantID) } export, err := os.ReadDir(filepath.Join(staging, store.ExportDir)) if err != nil { return fmt.Errorf("healed record %s missing export dir: %w", wantID, err) } - // A present export with no regular file clones to nothing (an empty dir, or - // only empty subdirs); the byte content is cocoon's own format and not - // sandboxd's to validate further. + // an export with no regular file clones to nothing; deeper content is cocoon's own format if !hasRegularFile(export) { return fmt.Errorf("healed record %s has no export content", wantID) } @@ -481,8 +429,7 @@ func validateHealedCheckpoint(staging, wantID string) error { return nil } -// hasRegularFile reports whether entries holds at least one regular file, so an -// export of only empty subdirectories is treated as empty. +// hasRegularFile reports whether entries holds at least one regular file. func hasRegularFile(entries []os.DirEntry) bool { return slices.ContainsFunc(entries, func(e os.DirEntry) bool { return e.Type().IsRegular() }) } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index e9e2d8c1..0c2c94e7 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -16,9 +16,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// TestClaimCheckpointNeverPullsOnMiss: ClaimCheckpoint answers from the local -// store alone — a redirect exists precisely so this path never pays a peer -// transfer; only ClaimCheckpointHeal may. func TestClaimCheckpointNeverPullsOnMiss(t *testing.T) { ckpt := types.Checkpoint{ID: "ck_00000000000000bb", Key: testKey, CreatedAt: time.Now()} m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) @@ -31,8 +28,6 @@ func TestClaimCheckpointNeverPullsOnMiss(t *testing.T) { } } -// TestClaimCheckpointHealPullsOnce: a heal pays the transfer once and leaves -// the record served from the local store from then on. func TestClaimCheckpointHealPullsOnce(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -53,8 +48,6 @@ func TestClaimCheckpointHealPullsOnce(t *testing.T) { } } -// TestClaimCheckpointAfterHealStaysLocal: once healed, the record is served -// from the local store — a later branch must not pull again. func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -71,12 +64,6 @@ func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { } } -// TestClaimCheckpointHealDedupsConcurrentPulls: two branches racing to heal -// the same missing checkpoint must share one transfer, not each pay for it — -// healFlights (not the record lock, which no longer spans the transfer) -// dedups them onto one Manager-owned staging dir, and each branch must -// still get a genuinely valid, non-empty checkpoint out of it (the exact -// case a caller-owned-staging-dir singleflight used to get wrong). func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { synctest.Test(t, func(t *testing.T) { id := "ck_00000000000000bb" @@ -93,7 +80,7 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { }) } waitFor(t, func() bool { return puller.count() >= 1 }) - time.Sleep(50 * time.Millisecond) // let the second goroutine join the same flight + time.Sleep(50 * time.Millisecond) close(puller.release) wg.Wait() @@ -112,14 +99,6 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { }) } -// TestDeleteVetoesInFlightHeal covers the Codex r2 blocker: a heal's transfer -// runs unlocked (holding the lock across a 30-minute budget would pin every -// other operation on the id behind it), so a delete arriving while it stages -// must not block on that transfer — but it must also not let the heal -// resurrect the checkpoint moments after the delete answered "not here" for -// it. The delete returns immediately (nothing local to delete yet) and -// vetoes the pending heal, so the heal's own locked decide phase abandons -// its publish instead of racing the delete to a stale conclusion. func TestDeleteVetoesInFlightHeal(t *testing.T) { synctest.Test(t, func(t *testing.T) { id := "ck_00000000000000bb" @@ -132,7 +111,7 @@ func TestDeleteVetoesInFlightHeal(t *testing.T) { _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") healDone <- err }() - waitFor(t, func() bool { return puller.count() >= 1 }) // heal is staging, unpublished + waitFor(t, func() bool { return puller.count() >= 1 }) if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteLocal); !errors.Is(err, ErrUnknownCheckpoint) { t.Fatalf("delete while heal is staging: %v, want ErrUnknownCheckpoint (nothing local yet, and it must not block)", err) @@ -148,11 +127,6 @@ func TestDeleteVetoesInFlightHeal(t *testing.T) { }) } -// TestClaimCheckpointHealCtxCancelReturnsPromptly: a caller that gives up -// waiting must not be stuck behind the whole heal budget — the flight runs -// detached (context.Background() in runHeal), so this call returns -// ctx.Err() as soon as its own ctx is canceled, while the transfer keeps -// running and still lands. func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { synctest.Test(t, func(t *testing.T) { id := "ck_00000000000000bb" @@ -166,7 +140,7 @@ func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { _, err := m.ClaimCheckpointHeal(ctx, id, time.Hour, "") done <- err }() - waitFor(t, func() bool { return puller.count() >= 1 }) // transfer started, unlocked + waitFor(t, func() bool { return puller.count() >= 1 }) cancel() select { @@ -178,72 +152,58 @@ func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { t.Fatal("ClaimCheckpointHeal did not return promptly after ctx cancellation") } - // Detached from the canceled caller, the transfer must still complete - // and publish. close(puller.release) waitFor(t, func() bool { return m.HasCheckpoint(t.Context(), id) }) }) } -// TestRecLockEvictionWaitsForAllHolders covers the lock-identity-split half -// of the same Codex r2 blocker: evicting an id's entry while another holder -// still references it would let that holder's (or a new caller's) next -// recLock for the same id return a different mutex, splitting mutual -// exclusion between them. Eviction must wait until every holder has -// released, not just the one that happened to delete the record. func TestRecLockEvictionWaitsForAllHolders(t *testing.T) { m := newTestManager(t, newFakeEngine()) const id = "ck_00000000000000aa" l1 := m.recLock(id) l1.Lock() - l2 := m.recLock(id) // a second, concurrent holder registers interest first + l2 := m.recLock(id) if l2 != l1 { t.Fatalf("recLock returned a different mutex while the first holder is still outstanding") } l1.Unlock() - m.recDoneEvict(id) // first holder's release must not evict: l2 is still outstanding - if _, ok := m.recLocks.Load(id); !ok { + m.recDoneEvict(id) + if !hasRecLock(m, id) { t.Fatal("entry evicted while a second holder still references it") } - l3 := m.recLock(id) // a third caller arriving now must still find the SAME mutex + l3 := m.recLock(id) if l3 != l1 { t.Fatal("a concurrent recLock diverged onto a different mutex for the same id — the lock-identity split") } - m.recDone(id) // l2's release - m.recDoneEvict(id) // l3's release, last one out - if _, ok := m.recLocks.Load(id); ok { + m.recDone(id) + m.recDoneEvict(id) + if hasRecLock(m, id) { t.Error("entry not evicted once every holder released") } } -// TestRecLockEvictDeferredToLastHolder is the production sequence the test -// above sidesteps: a delete (recDoneEvict) releases while an ordinary holder -// (recDone) still references the id, so the ordinary holder is the last one -// out. The eviction the delete asked for must not be lost. func TestRecLockEvictDeferredToLastHolder(t *testing.T) { m := newTestManager(t, newFakeEngine()) const id = "ck_00000000000000aa" - l1 := m.recLock(id) // the delete - l2 := m.recLock(id) // a concurrent claim/fetch + l1 := m.recLock(id) + l2 := m.recLock(id) if l2 != l1 { t.Fatal("recLock diverged for the same id") } - m.recDoneEvict(id) // delete releases first, wants eviction, but l2 still holds - if _, ok := m.recLocks.Load(id); !ok { + m.recDoneEvict(id) + if !hasRecLock(m, id) { t.Fatal("entry evicted while an ordinary holder still references it") } - m.recDone(id) // the ordinary holder, last one out - if _, ok := m.recLocks.Load(id); ok { + m.recDone(id) + if hasRecLock(m, id) { t.Error("recLocks entry leaked: a delete's deferred eviction was lost when a plain recDone dropped the last reference") } } -// TestFetchCheckpointNeverPulls: the peer-transfer blob endpoint must never -// trigger a recursive pull, even with a healer installed that could serve it. func TestFetchCheckpointNeverPulls(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -257,8 +217,6 @@ func TestFetchCheckpointNeverPulls(t *testing.T) { } } -// TestHasCheckpoint covers the probe answer across a present, missing, and -// archive record: only a live branchable checkpoint reports true. func TestHasCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -282,10 +240,6 @@ func TestHasCheckpoint(t *testing.T) { } } -// TestClaimCheckpointQuotaDoesNotMaskMiss: a full node must still answer -// "not here" for a checkpoint it does not hold, or the handler's -// redirect/heal tiers behind ErrUnknownCheckpoint never run; a full node -// that DOES hold the record still answers the quota error. func TestClaimCheckpointQuotaDoesNotMaskMiss(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -304,9 +258,6 @@ func TestClaimCheckpointQuotaDoesNotMaskMiss(t *testing.T) { } } -// TestClaimCheckpointHealQuotaBeforePull: unlike ClaimCheckpoint's cheap -// local read, resolving a heal means a peer transfer — quota must reject a -// full node before that cost, so the puller must never be called. func TestClaimCheckpointHealQuotaBeforePull(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -321,10 +272,6 @@ func TestClaimCheckpointHealQuotaBeforePull(t *testing.T) { } } -// TestClaimCheckpointHealConcurrencyCapRejectsExtra: heals of DIFFERENT ids -// (so the recLock does not serialize them) must still be capped node-wide — -// each pulls a whole guest memory image, so distinct ids are not free just -// because they don't contend on the same lock. func TestClaimCheckpointHealConcurrencyCapRejectsExtra(t *testing.T) { m := newTestManager(t, newFakeEngine()) m.healSem = make(chan struct{}, 1) @@ -347,8 +294,6 @@ func TestClaimCheckpointHealConcurrencyCapRejectsExtra(t *testing.T) { } } -// TestValidateHealedCheckpointAcceptsValid: a well-formed staged record must -// pass every check. func TestValidateHealedCheckpointAcceptsValid(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa"}) @@ -357,9 +302,6 @@ func TestValidateHealedCheckpointAcceptsValid(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsMismatchedID: a peer's meta.json must -// name the id that was actually requested, or a hostile peer could serve a -// crafted record under someone else's id. func TestValidateHealedCheckpointRejectsMismatchedID(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000bb"}) @@ -368,8 +310,6 @@ func TestValidateHealedCheckpointRejectsMismatchedID(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsMissingMeta: no meta.json means nothing -// to trust. func TestValidateHealedCheckpointRejectsMissingMeta(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { @@ -380,12 +320,9 @@ func TestValidateHealedCheckpointRejectsMissingMeta(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsMissingExport: a meta with no export -// dir cannot back a branch. func TestValidateHealedCheckpointRejectsMissingExport(t *testing.T) { dir := t.TempDir() - // A valid key, so the record reaches the export check rather than failing - // earlier on the key. + meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa", Key: testKey}) if err != nil { t.Fatalf("setup: %v", err) @@ -398,8 +335,6 @@ func TestValidateHealedCheckpointRejectsMissingExport(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsArchive: a wake image must never be -// healed in as a branchable checkpoint. func TestValidateHealedCheckpointRejectsArchive(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Archive: true}) @@ -408,10 +343,6 @@ func TestValidateHealedCheckpointRejectsArchive(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsEmptyExport: an export directory that is -// present but empty clones to nothing — publishing it would suppress a good -// owner and fail every later local claim, so it must be rejected and the next -// owner tried. func TestValidateHealedCheckpointRejectsEmptyExport(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { @@ -429,8 +360,6 @@ func TestValidateHealedCheckpointRejectsEmptyExport(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsInvalidKey: a record whose key does not -// name a branchable pool must not be published. func TestValidateHealedCheckpointRejectsInvalidKey(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Key: types.PoolKey{Net: "bogus"}}) @@ -439,9 +368,6 @@ func TestValidateHealedCheckpointRejectsInvalidKey(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsEgressKey: an egress-lane key is valid but -// not branchable (claimLoaded refuses it), so a healed record carrying one must -// be rejected rather than published to poison the id. func TestValidateHealedCheckpointRejectsEgressKey(t *testing.T) { dir := t.TempDir() egress := testKey @@ -452,8 +378,6 @@ func TestValidateHealedCheckpointRejectsEgressKey(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsContentlessExport: an export directory -// holding only an empty subdirectory clones to nothing and must be rejected. func TestValidateHealedCheckpointRejectsContentlessExport(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, store.ExportDir, "sub"), 0o750); err != nil { @@ -471,8 +395,6 @@ func TestValidateHealedCheckpointRejectsContentlessExport(t *testing.T) { } } -// TestDeleteCheckpointBroadcasts: a local delete calls the peer-delete hook -// once so a copy healed onto another node is removed too. func TestDeleteCheckpointBroadcasts(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -492,9 +414,6 @@ func TestDeleteCheckpointBroadcasts(t *testing.T) { } } -// TestDeleteCheckpointNoForwardSkipsBroadcast: a delete arriving as a -// broadcast (no_forward) must not re-broadcast, or the fleet would loop -// forever. func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -514,9 +433,6 @@ func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { } } -// TestDeleteCheckpointSharedStoreSkipsBroadcast: an s3 (or other shared) -// backend has one copy every node already sees, so broadcasting would be a -// no-op fan-out; DeleteCheckpoint must skip it. func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -537,9 +453,6 @@ func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { } } -// newHealManager builds a Manager with a healer wired to a puller serving -// ckpt at addrs, mirroring WithPeerHeal's construction directly (tests are in -// the same package). func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manager, *healPuller) { t.Helper() m := newTestManager(t, newFakeEngine()) @@ -548,14 +461,11 @@ func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manag return m, puller } -// healPuller fakes peer.Puller: it writes a valid record (meta.json + an -// export/ dir the fakeEngine "clones" from) and can block until -// release, so a test can prove concurrent misses dedup to one pull. type healPuller struct { mu sync.Mutex calls int ckpt types.Checkpoint - release chan struct{} // non-nil: Pull blocks until closed + release chan struct{} } func (p *healPuller) Pull(_ context.Context, _, _, dst string) error { @@ -584,9 +494,6 @@ func (p *healPuller) count() int { return p.calls } -// blockingPuller writes a valid record stamped with the requested id, after -// signaling started and then blocking until release — for proving the -// node-wide heal concurrency cap against distinct, non-contending ids. type blockingPuller struct { started chan struct{} release chan struct{} @@ -609,11 +516,10 @@ func (p *blockingPuller) Pull(_ context.Context, _, id, dst string) error { return os.WriteFile(filepath.Join(dst, store.MetaFile), meta, 0o600) } -// plantHealedRecord writes a valid-shaped staged record for validate tests. func plantHealedRecord(t *testing.T, dir string, ckpt types.Checkpoint) { t.Helper() if ckpt.Key == (types.PoolKey{}) { - ckpt.Key = testKey // a healed record carries a real, branchable key + ckpt.Key = testKey } if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { t.Fatalf("setup: %v", err) diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index a924463a..8ee529aa 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -106,8 +106,8 @@ func TestRetentionSweepsExpiredCheckpoints(t *testing.T) { t.Fatalf("setup: %v", err) } } - plant("ck_000000000000000a", 2*time.Hour) // expired - plant("ck_000000000000000b", time.Minute) // fresh + plant("ck_000000000000000a", 2*time.Hour) + plant("ck_000000000000000b", time.Minute) m.sweepExpiredCheckpoints(t.Context()) @@ -147,7 +147,6 @@ func TestCheckpointTenantIsolation(t *testing.T) { t.Errorf("root listing %+v, %v; want both records", ckpts, err) } - // A tenant cannot delete another's record — and learns nothing beyond 404. if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "beta", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("cross-tenant delete: %v, want ErrUnknownCheckpoint", err) } @@ -159,20 +158,12 @@ func TestCheckpointTenantIsolation(t *testing.T) { } } -// TestCheckpointDeleteEvictsRecLock proves recLocks does not retain a lock per -// checkpoint: ck ids are single-use, so a create+delete cycle must leave the -// map at its prior size, or a long-running node leaks one RWMutex per ck. func TestCheckpointDeleteEvictsRecLock(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) src := mustClaim(t, m, testKey) - countLocks := func() int { - n := 0 - m.recLocks.Range(func(_, _ any) bool { n++; return true }) - return n - } - base := countLocks() // the golden-resolve's template lock: bounded, never evicted + base := lockCount(m) var ids []string for range 5 { @@ -186,26 +177,18 @@ func TestCheckpointDeleteEvictsRecLock(t *testing.T) { if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteFleet); err != nil { t.Fatalf("delete %s: %v", id, err) } - if _, ok := m.recLocks.Load(id); ok { + if hasRecLock(m, id) { t.Errorf("recLocks retained a lock for deleted checkpoint %s", id) } } - if got := countLocks(); got != base { + if got := lockCount(m); got != base { t.Errorf("recLocks grew %d->%d over 5 checkpoint create+delete cycles, want no net growth", base, got) } } -// TestRejectedCheckpointOpsLeakNoRecLock: a rejected claim or delete (bad -// format, unknown, wrong tenant) must not create a lock-map entry — otherwise -// a guess-loop grows recLocks without bound. func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { m := newTestManager(t, newFakeEngine()) - countLocks := func() int { - n := 0 - m.recLocks.Range(func(_, _ any) bool { n++; return true }) - return n - } - base := countLocks() + base := lockCount(m) for _, id := range []string{"../../etc", "ck_zz", "", "ck_00112233445566ff", "ck_deadbeefdeadbeef"} { if _, err := m.ClaimCheckpoint(t.Context(), id, time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { @@ -215,21 +198,20 @@ func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { t.Errorf("delete %q: %v, want ErrUnknownCheckpoint", id, err) } } - if got := countLocks(); got != base { + if got := lockCount(m); got != base { t.Errorf("recLocks grew %d->%d over rejected checkpoint ops, want no growth", base, got) } - // A wrong-tenant delete of a real checkpoint must also leak nothing. src := mustClaim(t, m, testKey) ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "acme") if err != nil { t.Fatalf("checkpoint: %v", err) } - before := countLocks() + before := lockCount(m) if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "other", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("wrong-tenant delete: %v, want ErrUnknownCheckpoint", err) } - if got := countLocks(); got != before { + if got := lockCount(m); got != before { t.Errorf("recLocks grew %d->%d over a wrong-tenant delete, want no growth", before, got) } } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index cd42f0cc..f7d248a6 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -21,10 +21,7 @@ const ( reapPurge // archived past retention: delete the store checkpoint ) -// ClaimWarm transfers ownership of a warm sandbox without provisioning; -// ErrNoWarm means the pool is empty (the caller may redirect or provision). -// tenant attributes the claim; empty means the operator (root). claimRef is an -// opaque caller reference recorded on the claim; empty means none. +// ClaimWarm transfers ownership of a warm sandbox without provisioning; ErrNoWarm means empty. func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { start := time.Now() if err := m.validate(key); err != nil { @@ -71,13 +68,11 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Dur } // ClaimProvision creates a claim-ready sandbox (golden clone or cold boot). -// claimRef is an opaque caller reference recorded on the claim; empty means none. func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { return m.claimProvision(ctx, key, ttl, tenant, claimRef, volumes, false) } -// ClaimProvisionPromoted requires key to resolve from a promoted template and -// never falls through to a cold image boot. +// ClaimProvisionPromoted requires key to resolve from a promoted template, never a cold boot. func (m *Manager) ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { return m.claimProvision(ctx, key, ttl, tenant, claimRef, volumes, true) } @@ -91,8 +86,7 @@ func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { return m.releaseResolved(ctx, id, sb) } -// ClaimDeadline authorizes a sandbox by token and returns its lease -// deadline — the preview mint clamps a URL's life to it. +// ClaimDeadline authorizes a sandbox by token and returns its lease deadline. func (m *Manager) ClaimDeadline(id, token string) (time.Time, error) { m.mu.Lock() defer m.mu.Unlock() @@ -120,8 +114,7 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. return m.eng.DialGuestPort(ctx, sock, port) } -// AgentSocket resolves a claimed sandbox's vsock UDS without waking it (the -// ownership probe must not restore a hibernated VM). +// AgentSocket resolves a claimed sandbox's vsock UDS without waking it. func (m *Manager) AgentSocket(id, token string) (string, error) { m.mu.Lock() defer m.mu.Unlock() @@ -129,15 +122,11 @@ func (m *Manager) AgentSocket(id, token string) (string, error) { if !ok { return "", ErrUnknownSandbox } - // No activity stamp here: owner/Lookup probes use this path, and a - // control-plane poll must not keep an idle sandbox awake. The relay's - // stamp lives in WakeAgentSocket. + // no activity stamp: a control-plane poll must not keep an idle sandbox awake return sb.VsockSocket, nil } -// releaseResolved drops the resolved claim and tears down its VM. resolve -// unlocks before this runs, so it re-checks under m.mu that sb is still the -// live claim: a second release racing in must not tear down twice. +// releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { m.mu.Lock() if m.claimed[id] != sb { @@ -154,14 +143,14 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand } delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) - js := m.store.snapshot(m.claimed) + js := m.store.del(id) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() - m.claimed[id] = sb // roll back so memory matches the still-durable claim; the restored claim re-pins ck + m.claimed[id] = sb // roll back so memory matches the still-durable claim m.tenantDelta(sb.Tenant, 1) delete(m.pendingCks, ck) - rb := m.store.snapshot(m.claimed) + rb := m.store.set(sb) m.mu.Unlock() if ck != "" { if clearErr := m.clearArchiveCk(ck); clearErr != nil { @@ -192,15 +181,14 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand return err } -// overQuota is a cheap advisory precheck; finalizeBatch does the authoritative -// admission check, so this just spares a doomed request the provision cost. +// overQuota is an advisory precheck; finalizeBatch does the authoritative admission check. func (m *Manager) overQuota(extra int, tenant string) error { m.mu.Lock() defer m.mu.Unlock() return m.quotaErr(extra, tenant) } -// admitVolumes takes one claim's holds, then the marker check; a refusal on either leaves nothing held. +// admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) if err := m.admitClaim(tenant, applied); err != nil { @@ -213,9 +201,7 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types return applied, nil } -// admitClaim is the provision path's one admission section: the advisory quota -// precheck plus the authoritative volume reservation, so a busy volume is -// refused before any VM is built. +// admitClaim refuses a busy volume or an over-quota tenant before any VM is built. func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { m.mu.Lock() defer m.mu.Unlock() @@ -225,8 +211,7 @@ func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { return m.reserveVolumes(volumes) } -// quotaErr answers ErrQuota over the node cap, the tenant cap, or a -// draining node; callers hold m.mu. +// quotaErr answers ErrQuota over a cap or a draining node; callers hold m.mu. func (m *Manager) quotaErr(extra int, tenant string) error { if m.draining { return fmt.Errorf("%w: node draining", ErrQuota) @@ -256,8 +241,7 @@ func (m *Manager) tenantDelta(tenant string, delta int) { } } -// finalize stamps identity, persists the claim, and destroys the VM if the -// store write fails so a durable claim always matches a live VM. +// finalize stamps identity and persists the claim; a failed write destroys the VM. func (m *Manager) finalize(ctx context.Context, sb *types.Sandbox, ttl time.Duration) (*types.Sandbox, error) { if err := m.finalizeBatch(ctx, []*types.Sandbox{sb}, ttl); err != nil { return nil, err @@ -265,9 +249,7 @@ func (m *Manager) finalize(ctx context.Context, sb *types.Sandbox, ttl time.Dura return sb, nil } -// finalizeBatch stamps identities and persists the batch as one journal write; -// a failed write destroys every VM in it — a durable claim always matches a -// live VM, all-or-nothing. One tenant per batch (fork children inherit it). +// finalizeBatch persists the batch as one write, all-or-nothing; one tenant per batch. func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl time.Duration) error { now := time.Now() for _, sb := range sbs { @@ -278,8 +260,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t if quotaErr := m.quotaErr(len(sbs), sbs[0].Tenant); quotaErr != nil { m.mu.Unlock() for _, sb := range sbs { - // Never handed out, but the rw mount itself dirtied the journal — the - // clean umount is owed exactly as on the rollback path. + // the rw mount dirtied the journal, so the clean umount is still owed td := m.quiesceVolumes(ctx, sb) m.removeOrRetry(ctx, sb.VMName, "", "", td) } @@ -289,7 +270,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t m.claimed[sb.ID] = sb m.tenantDelta(sb.Tenant, 1) } - js := m.store.snapshot(m.claimed) + js := m.store.set(sbs...) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.rollbackClaim(ctx, sbs) @@ -301,8 +282,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t return fmt.Errorf("arm egress %s: %w", sb.ID, armErr) } } - // Usage lands only after the whole batch armed: a rollback must not leave - // claim events with no terminal release/reap in the billing stream. + // usage lands only after the batch armed, so a rollback leaves no unterminated claim event for _, sb := range sbs { m.recordUsage(ctx, usageEvent{ Event: "claim", //nolint:goconst // event name; other occurrences are test assertions @@ -314,16 +294,16 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t return nil } -// rollbackClaim unwinds a claim batch after a persist or egress-arm failure: -// drop the claims, reconverge the journal, destroy the VMs — a NIC that -// cannot be locked must never be handed out. +// rollbackClaim unwinds a claim batch: a NIC that cannot be locked is never handed out. func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { m.mu.Lock() - for _, sb := range sbs { + ids := make([]string, len(sbs)) + for i, sb := range sbs { delete(m.claimed, sb.ID) m.tenantDelta(sb.Tenant, -1) + ids[i] = sb.ID } - rb := m.store.snapshot(m.claimed) + rb := m.store.del(ids...) m.mu.Unlock() m.recommit(ctx, rb) for _, sb := range sbs { @@ -332,8 +312,7 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { } } -// abortVolumeClaim removes a failed claim's VM, holds riding the teardown; -// clearing reserved stops the deferred unreserve from double-releasing them. +// abortVolumeClaim removes a failed claim's VM and clears reserved against a double release. func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved *[]types.Volume) { m.removeOrRetry(ctx, vmName, "", "", volumeTeardown{holds: *reserved}) *reserved = nil @@ -348,6 +327,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() var expired []victim + var dropped []string for id, sb := range m.claimed { // A zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { @@ -357,8 +337,7 @@ func (m *Manager) reapOnce(ctx context.Context) { case sb.ArchiveCk != "": expired = append(expired, victim{action: reapPurge, id: id, ck: sb.ArchiveCk, tenant: sb.Tenant, sb: sb}) case sb.HibernateSnap != "" && m.archiveEnabledFor(sb.Key): - // End-of-lease hibernated + archive-enabled: archive instead of - // destroy, kept in m.claimed for archive() to re-validate and move. + // archive instead of destroy, kept in m.claimed for archive() to move if _, busy := m.archiving[id]; busy { continue // an archive is already exporting this sandbox } @@ -368,8 +347,12 @@ func (m *Manager) reapOnce(ctx context.Context) { expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb}) delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) + dropped = append(dropped, id) } } + if len(dropped) > 0 { + m.store.del(dropped...) + } m.mu.Unlock() if len(expired) == 0 { return @@ -392,7 +375,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() keep = expired[:0] - var purgeCks []string + var purgeCks, purged []string for _, v := range expired { if v.action == reapPurge { if m.claimed[v.id] != v.sb || v.sb.ArchiveCk != v.ck { @@ -402,6 +385,7 @@ func (m *Manager) reapOnce(ctx context.Context) { delete(m.claimed, v.id) m.tenantDelta(v.tenant, -1) purgeCks = append(purgeCks, v.ck) + purged = append(purged, v.id) } keep = append(keep, v) } @@ -410,10 +394,11 @@ func (m *Manager) reapOnce(ctx context.Context) { m.mu.Unlock() return } - js := m.store.snapshot(m.claimed) + js := m.store.del(purged...) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() + var restored []*types.Sandbox for _, v := range expired { if v.action == reapArchive { delete(m.archiving, v.id) // never removed from m.claimed @@ -421,9 +406,10 @@ func (m *Manager) reapOnce(ctx context.Context) { m.claimed[v.id] = v.sb // roll back so memory matches the still-durable claim m.tenantDelta(v.tenant, 1) delete(m.pendingCks, v.ck) + restored = append(restored, v.sb) } } - rb := m.store.snapshot(m.claimed) + rb := m.store.set(restored...) m.mu.Unlock() for _, ck := range purgeCks { if clearErr := m.clearArchiveCk(ck); clearErr != nil { @@ -434,8 +420,7 @@ func (m *Manager) reapOnce(ctx context.Context) { logger.Error(ctx, saveErr, "persist reap; rolled back") return } - // Engine subprocesses (worst case minutes on a hung stop): fan out bounded - // so a big batch never stalls the ticker loop. + // engine teardown can take minutes, so fan out bounded and never stall the ticker loop m.runBounded(ctx, len(expired), func(ctx context.Context, i int) { v := expired[i] switch v.action { @@ -468,9 +453,7 @@ func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { m.recordUsage(ctx, usageEvent{Event: "archive_delete", ID: id, Reference: ck, Tenant: tenant}) } -// recommit re-persists a snapshot in the background until its own success or -// any newer durable write ends the loop (commit coalesces by sequence); -// detached, because callers may hold the transition lock. +// recommit re-persists in the background until success or a newer durable write; detached. func (m *Manager) recommit(ctx context.Context, snap claimSnapshot) { go func() { backoff := recommitBackoff diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 5542df8f..9b1c2c7d 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -5,26 +5,22 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "path/filepath" "slices" "sync" - "sync/atomic" "time" "github.com/cocoonstack/sandbox/sandboxd/types" ) -// claimSnapshot is a sequenced, detached copy of the claim set awaiting a -// durable write; commit() marshals it off the manager mutex. +// claimSnapshot sequences one persist request; commit skips it once a newer write has landed. type claimSnapshot struct { - claims map[string]claimDTO - seq uint64 + seq uint64 } -// claimDTO is the persisted projection of a Sandbox (its json fields only), -// copied by value so commit's marshal runs off the manager mutex without -// racing the live record's Transition mutex and lastActivity. +// claimDTO is the persisted projection of a Sandbox, copied so commit marshals off m.mu. type claimDTO struct { ID string `json:"id"` VMName string `json:"vm_name"` @@ -42,19 +38,20 @@ type claimDTO struct { FromCheckpoint string `json:"from_checkpoint,omitempty"` } -// claimStore persists claimed sandboxes across daemon restarts. Warm pool -// VMs are deliberately not persisted: they are cheap to rebuild and unsafe -// to trust after an unsupervised gap. +// claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string - seq atomic.Uint64 // sequence source; bumped in snapshot (mutation order) + writeMu sync.Mutex + mu sync.Mutex - written uint64 // highest sequence on disk; guarded by mu + dtos map[string]claimDTO + seq uint64 + written uint64 } func newClaimStore(dataDir string) *claimStore { - return &claimStore{path: filepath.Join(dataDir, "claims.json")} + return &claimStore{path: filepath.Join(dataDir, "claims.json"), dtos: map[string]claimDTO{}} } func (s *claimStore) load() (map[string]*types.Sandbox, error) { @@ -72,33 +69,59 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { return claims, nil } -// snapshot copies the claim set into a detached, sequenced value; the caller -// holds the manager mutex, so sequences are handed out in mutation order. -func (s *claimStore) snapshot(claims map[string]*types.Sandbox) claimSnapshot { - seq := s.seq.Add(1) - dtos := make(map[string]claimDTO, len(claims)) +// set records each sandbox's projection and sequences the write; callers may hold m.mu. +func (s *claimStore) set(sbs ...*types.Sandbox) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + for _, sb := range sbs { + s.dtos[sb.ID] = dtoOf(sb) + } + s.seq++ + return claimSnapshot{seq: s.seq} +} + +// del drops the projections of ids and sequences the write; callers may hold m.mu. +func (s *claimStore) del(ids ...string) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + for _, id := range ids { + delete(s.dtos, id) + } + s.seq++ + return claimSnapshot{seq: s.seq} +} + +// mark sequences a persist of the projection as it already stands. +func (s *claimStore) mark() claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + return claimSnapshot{seq: s.seq} +} + +// reset rebuilds the whole projection; the startup path, before contention exists. +func (s *claimStore) reset(claims map[string]*types.Sandbox) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + s.dtos = make(map[string]claimDTO, len(claims)) for id, sb := range claims { - dtos[id] = claimDTO{ - ID: sb.ID, VMName: sb.VMName, Key: sb.Key, Token: sb.Token, - Deadline: sb.Deadline, Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, - Volumes: slices.Clone(sb.Volumes), VsockSocket: sb.VsockSocket, - TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, - ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, - } + s.dtos[id] = dtoOf(sb) } - return claimSnapshot{claims: dtos, seq: seq} + s.seq++ + return claimSnapshot{seq: s.seq} } -// commit marshals a snapshot and atomically replaces claims.json, off the -// manager mutex; serialized by s.mu, a stale snapshot is a no-op. Unsynced by -// design (#21): an fsync would sit on the sub-ms warm-claim path. +// commit atomically replaces claims.json; unsynced by design (#21) to keep the claim path fast. func (s *claimStore) commit(snap claimSnapshot) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() s.mu.Lock() - defer s.mu.Unlock() if snap.seq <= s.written { + s.mu.Unlock() return nil } - raw, err := json.Marshal(snap.claims) + dtos, seq := maps.Clone(s.dtos), s.seq + s.mu.Unlock() + raw, err := json.Marshal(dtos) if err != nil { return fmt.Errorf("encode claims: %w", err) } @@ -109,19 +132,30 @@ func (s *claimStore) commit(snap claimSnapshot) error { if err := os.Rename(tmp, s.path); err != nil { return fmt.Errorf("commit claims: %w", err) } - s.written = snap.seq + s.mu.Lock() + s.written = seq + s.mu.Unlock() return nil } -// save is the combined form for the startup Reconcile pass, before contention -// exists; hot-path callers split snapshot()/commit() around the manager mutex. +// save is the combined form for the startup Reconcile pass, before contention exists. func (s *claimStore) save(claims map[string]*types.Sandbox) error { - return s.commit(s.snapshot(claims)) + return s.commit(s.reset(claims)) } -// synced reports whether every handed-out snapshot has reached disk. +// synced reports whether every sequenced change has reached disk. func (s *claimStore) synced() bool { s.mu.Lock() defer s.mu.Unlock() - return s.written == s.seq.Load() + return s.written == s.seq +} + +func dtoOf(sb *types.Sandbox) claimDTO { + return claimDTO{ + ID: sb.ID, VMName: sb.VMName, Key: sb.Key, Token: sb.Token, + Deadline: sb.Deadline, Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, + Volumes: slices.Clone(sb.Volumes), VsockSocket: sb.VsockSocket, + TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, + ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, + } } diff --git a/sandboxd/pool/claims_bench_test.go b/sandboxd/pool/claims_bench_test.go index 03fb1df9..c8e8ec2f 100644 --- a/sandboxd/pool/claims_bench_test.go +++ b/sandboxd/pool/claims_bench_test.go @@ -10,8 +10,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// BenchmarkStoreSaveScaling measures the claims-journal persist cost (marshal -// + write + rename) as the live-claim set grows. func BenchmarkStoreSaveScaling(b *testing.B) { for _, n := range []int{10, 100, 1000} { b.Run(fmt.Sprintf("claims=%d", n), func(b *testing.B) { @@ -26,19 +24,14 @@ func BenchmarkStoreSaveScaling(b *testing.B) { } } -// BenchmarkStorePersistContention reports how long a concurrent manager-mutex -// acquire (a data-plane op) waits while the journal is being persisted, under -// the old "write under the lock" path (save) vs the new "marshal under the lock, -// write off it" split (snapshot+commit). The ns/acquire metric is the win. func BenchmarkStorePersistContention(b *testing.B) { for _, n := range []int{100, 1000} { claims := benchClaims(n) - b.Run(fmt.Sprintf("under-lock/n=%d", n), func(b *testing.B) { - benchPersistContention(b, claims, false) - }) - b.Run(fmt.Sprintf("off-lock/n=%d", n), func(b *testing.B) { - benchPersistContention(b, claims, true) - }) + for _, arm := range []string{"under-lock", "rebuild", "incremental"} { + b.Run(fmt.Sprintf("%s/n=%d", arm, n), func(b *testing.B) { + benchPersistContention(b, claims, arm) + }) + } } } @@ -55,8 +48,10 @@ func benchClaims(n int) map[string]*types.Sandbox { return claims } -func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, offLock bool) { +func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, arm string) { s := newClaimStore(b.TempDir()) + s.reset(claims) + one := claims[fmt.Sprintf("sb_%016x", 0)] var mu sync.Mutex var waitNs, waits atomic.Int64 done := make(chan struct{}) @@ -76,15 +71,21 @@ func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, offL }() for b.Loop() { - if offLock { + switch arm { + case "under-lock": + mu.Lock() + _ = s.save(claims) + mu.Unlock() + case "rebuild": mu.Lock() - snap := s.snapshot(claims) // marshal under the lock + snap := s.reset(claims) mu.Unlock() - _ = s.commit(snap) // write off the lock - } else { + _ = s.commit(snap) + default: mu.Lock() - _ = s.save(claims) // marshal + write both under the lock + snap := s.set(one) mu.Unlock() + _ = s.commit(snap) } } close(done) diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index c13d2418..72f3d744 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -34,7 +34,7 @@ func TestStoreRoundTrip(t *testing.T) { func TestClaimStoreSnapshotDetachesVolumes(t *testing.T) { s := newClaimStore(t.TempDir()) sb := &types.Sandbox{ID: "sb_a", Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}} - snap := s.snapshot(map[string]*types.Sandbox{sb.ID: sb}) + snap := s.set(sb) sb.Volumes[0].Mount = "/mutated" if err := s.commit(snap); err != nil { t.Fatalf("commit: %v", err) @@ -73,20 +73,14 @@ func TestStoreLoadCorruptFile(t *testing.T) { } } -// TestClaimStoreCommitCoalesces asserts an older snapshot never overwrites a -// newer one already on disk — the guard that lets the write leave the manager -// mutex without losing the latest state. func TestClaimStoreCommitCoalesces(t *testing.T) { s := newClaimStore(t.TempDir()) - one := map[string]*types.Sandbox{"sb_a": {ID: "sb_a"}} - two := map[string]*types.Sandbox{"sb_a": {ID: "sb_a"}, "sb_b": {ID: "sb_b"}} - - older := s.snapshot(one) // seq 1 - newer := s.snapshot(two) // seq 2 + older := s.set(&types.Sandbox{ID: "sb_a"}) + newer := s.set(&types.Sandbox{ID: "sb_b"}) if err := s.commit(newer); err != nil { t.Fatalf("commit newer: %v", err) } - if err := s.commit(older); err != nil { // must be a no-op, not an overwrite + if err := s.commit(older); err != nil { t.Fatalf("commit older: %v", err) } if got, err := s.load(); err != nil || len(got) != 2 { @@ -94,14 +88,61 @@ func TestClaimStoreCommitCoalesces(t *testing.T) { } } -// TestClaimStoreConcurrentCommits exercises the writer off the manager mutex: -// many snapshot+commit pairs race, and the file must stay parseable (-race). +func TestClaimStoreIncrementalProjection(t *testing.T) { + s := newClaimStore(t.TempDir()) + a := &types.Sandbox{ID: "sb_a", VMName: "sbx-1", Key: testKey} + b := &types.Sandbox{ID: "sb_b", VMName: "sbx-2", Key: testKey} + if err := s.commit(s.set(a, b)); err != nil { + t.Fatalf("commit set: %v", err) + } + a.HibernateSnap = "snap-a" + if err := s.commit(s.set(a)); err != nil { + t.Fatalf("commit update: %v", err) + } + got, err := s.load() + if err != nil { + t.Fatalf("load: %v", err) + } + if len(got) != 2 || got["sb_a"].HibernateSnap != "snap-a" || got["sb_b"].VMName != "sbx-2" { + t.Fatalf("after incremental update: %+v", got) + } + if delErr := s.commit(s.del("sb_a")); delErr != nil { + t.Fatalf("commit del: %v", delErr) + } + if got, err = s.load(); err != nil || len(got) != 1 || got["sb_b"] == nil { + t.Fatalf("after del: %+v, %v", got, err) + } + if resetErr := s.commit(s.reset(map[string]*types.Sandbox{"sb_c": {ID: "sb_c", Key: testKey}})); resetErr != nil { + t.Fatalf("commit reset: %v", resetErr) + } + if got, err = s.load(); err != nil || len(got) != 1 || got["sb_c"] == nil { + t.Fatalf("after reset: %+v, %v", got, err) + } +} + +func TestClaimStoreMarkRepersists(t *testing.T) { + s := newClaimStore(t.TempDir()) + s.set(&types.Sandbox{ID: "sb_a", Key: testKey}) + if s.synced() { + t.Fatal("synced with an unwritten change") + } + if err := s.commit(s.mark()); err != nil { + t.Fatalf("commit mark: %v", err) + } + if !s.synced() { + t.Error("not synced after flushing the mark") + } + if got, err := s.load(); err != nil || len(got) != 1 { + t.Errorf("after mark flush: %+v, %v", got, err) + } +} + func TestClaimStoreConcurrentCommits(t *testing.T) { s := newClaimStore(t.TempDir()) - claims := map[string]*types.Sandbox{"sb_a": {ID: "sb_a"}} + sb := &types.Sandbox{ID: "sb_a"} var wg sync.WaitGroup for range 50 { - wg.Go(func() { _ = s.commit(s.snapshot(claims)) }) + wg.Go(func() { _ = s.commit(s.set(sb)) }) } wg.Wait() if got, err := s.load(); err != nil || len(got) != 1 { diff --git a/sandboxd/pool/drain.go b/sandboxd/pool/drain.go index 5580df10..6b55fce6 100644 --- a/sandboxd/pool/drain.go +++ b/sandboxd/pool/drain.go @@ -2,9 +2,7 @@ package pool import "context" -// Drain cordons the node: claim/fork/branch answer 429, warm VMs are -// destroyed, live claims run to their leases. Not persisted — a restarted -// node serves again. +// Drain cordons the node: claims answer 429 and warm VMs are destroyed. Not persisted. func (m *Manager) Drain(ctx context.Context) { var trim []string m.mu.Lock() diff --git a/sandboxd/pool/egress.go b/sandboxd/pool/egress.go index 706ee4bc..8941d074 100644 --- a/sandboxd/pool/egress.go +++ b/sandboxd/pool/egress.go @@ -20,11 +20,7 @@ import ( var ( nat64Range = netip.MustParsePrefix("64:ff9b::/96") // RFC 6052 NAT64; the embedded v4 is checked instead - // internalRanges is every IANA special-purpose prefix with Globally Reachable = - // false that IsGlobalUnicast/IsPrivate do not already exclude, plus the - // deprecated IPv4-compatible embedding, mirrored from the IPv4/IPv6 - // Special-Purpose registries (snapshot 2026-07-11). 2001::/23 is blocked whole - // (its handful of globally-reachable anycast carve-outs are not egress targets). + // internalRanges lists the IANA special-purpose prefixes IsGlobalUnicast/IsPrivate leave in. internalRanges = []netip.Prefix{ netip.MustParsePrefix("0.0.0.0/8"), // "this network" netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT; some clouds host metadata here @@ -48,9 +44,7 @@ var ( } ) -// egressListener is one sandbox's egress accept point: an http.Server serving -// egress.Proxy over the per-sandbox UDS the VMM connects when the guest dials -// CID2:egressPort. +// egressListener is one sandbox's egress accept point over the per-sandbox UDS. type egressListener struct { srv *http.Server proxy *egress.Proxy @@ -65,9 +59,7 @@ func (e *egressListener) close() { _ = os.Remove(e.path) } -// armEgress locks the egress-lane NIC then binds the proxy: fail-closed (a -// lock error aborts the claim), and no policy still yields a locked NIC with -// no proxy — default-deny, never a free NIC. +// armEgress locks the egress-lane NIC then binds the proxy: fail-closed, never a free NIC. func (m *Manager) armEgress(ctx context.Context, sb *types.Sandbox) error { if err := m.lockEgressNIC(ctx, sb); err != nil { return err @@ -75,8 +67,7 @@ func (m *Manager) armEgress(ctx context.Context, sb *types.Sandbox) error { return m.armEgressProxy(ctx, sb) } -// lockEgressNIC nft-locks the egress-lane NIC and records the tap for unlock; -// the engine lookup is the fallback for claims from pre-tap journals. +// lockEgressNIC nft-locks the egress-lane NIC and records the tap for unlock. func (m *Manager) lockEgressNIC(ctx context.Context, sb *types.Sandbox) error { if !m.lockEgress || sb.Key.Net != types.NetEgress { return nil @@ -111,9 +102,7 @@ func (m *Manager) tapOf(ctx context.Context, vmName string) (string, error) { return "", fmt.Errorf("no tap for %s", vmName) } -// armEgressProxy binds the vsock egress proxy for a claim whose effective policy -// permits something; a no-op otherwise, so no listener means default-deny (the -// egress-lane NIC stays nft-locked, the none lane's proxy dial is refused). +// armEgressProxy binds the egress proxy only when the effective policy permits something. func (m *Manager) armEgressProxy(ctx context.Context, sb *types.Sandbox) error { if !m.guardedEgress || sb.VsockSocket == "" { return nil @@ -140,8 +129,7 @@ func (m *Manager) armEgressProxy(ctx context.Context, sb *types.Sandbox) error { return nil } -// disarmIfReleased tears down a proxy armed on a wake path if Release dropped -// the claim in the arm window (Release skips the Transition lock); reports gone. +// disarmIfReleased tears down a wake-path proxy if Release dropped the claim in the arm window. func (m *Manager) disarmIfReleased(sb *types.Sandbox) bool { m.mu.Lock() live := m.claimed[sb.ID] == sb @@ -152,8 +140,7 @@ func (m *Manager) disarmIfReleased(sb *types.Sandbox) bool { return !live } -// disarmEgress tears down a sandbox's egress proxy listener, and its NIC lock -// only when removed (a failed removal keeps a running guest locked). Idempotent. +// disarmEgress tears down a sandbox's egress listener, and its NIC lock only when removed. func (m *Manager) disarmEgress(id string, removed bool) { if !m.guardedEgress && !m.lockEgress { return @@ -175,17 +162,14 @@ func (m *Manager) disarmEgress(id string, removed bool) { } } -// poolIntercepts reports whether the configured pool for key HTTPS-intercepts, -// so its golden bakes the node CA. +// poolIntercepts reports whether the configured pool for key HTTPS-intercepts. func (m *Manager) poolIntercepts(key types.PoolKey) bool { m.mu.Lock() defer m.mu.Unlock() return m.poolEgress[key].Intercepts() } -// effectivePolicy resolves pool ∩ tenant; root has no tenant layer, and a key -// no pool is configured for (a promoted template, a cold-booted image) has no -// pool layer. +// effectivePolicy resolves pool ∩ tenant; root has no tenant layer, an unpooled key no pool one. func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { m.mu.Lock() defer m.mu.Unlock() @@ -206,9 +190,7 @@ func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { return egress.Compose(*poolPol, *tenantPol), true } -// newEgressDialer builds the proxy's upstream dialer: internal targets are -// blocked (the proxy must not be an SSRF), then exactly the node-named -// prefixes re-admitted. +// newEgressDialer blocks internal targets, then re-admits exactly the node-named prefixes. func newEgressDialer(allow []netip.Prefix) *net.Dialer { return &net.Dialer{Control: func(_, address string, _ syscall.RawConn) error { host, _, err := net.SplitHostPort(address) @@ -224,8 +206,7 @@ func newEgressDialer(allow []netip.Prefix) *net.Dialer { b := ip.As16() ip = netip.AddrFrom4([4]byte{b[12], b[13], b[14], b[15]}) } - // After the NAT64 unwrap (a v4-in-v6 target matches as the v4 it is), - // before the block, so a named prefix wins. + // after the NAT64 unwrap and before the block, so a named prefix wins if slices.ContainsFunc(allow, func(p netip.Prefix) bool { return p.Contains(ip) }) { return nil } @@ -237,8 +218,7 @@ func newEgressDialer(allow []netip.Prefix) *net.Dialer { }} } -// parsePrefixes turns the allow-list into prefixes; config validation already -// rejected malformed entries, so a leftover is dropped rather than failing dials. +// parsePrefixes turns the allow-list into prefixes; config validation already rejected bad ones. func parsePrefixes(cidrs []string) []netip.Prefix { out := make([]netip.Prefix, 0, len(cidrs)) for _, c := range cidrs { diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index dba61fff..b9d6245d 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -35,9 +35,8 @@ func TestEgressProxyInjectsAndGates(t *testing.T) { pol := &egress.Policy{Allow: []egress.Rule{{Host: host, Secret: "gh"}}} m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: pol}) - m.dial = (&net.Dialer{}).DialContext // the test origin is on loopback, which the SSRF guard blocks + m.dial = (&net.Dialer{}).DialContext - // A short socket dir: the UDS path + "_2049" must fit the OS sun_path cap. sockDir, err := os.MkdirTemp("/tmp", "eg") if err != nil { t.Fatalf("sockdir: %v", err) @@ -78,8 +77,7 @@ func TestEgressProxyInjectsAndGates(t *testing.T) { func TestArmEgressFailsClosedWhenNICUnlockable(t *testing.T) { m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - // The engine reports no NIC tap, so the nft lock cannot apply; arming must - // fail rather than hand out an egress-lane NIC that bypasses the proxy. + sb := &types.Sandbox{ID: "sb_eg_no_tap", Key: egKey, VMName: "sbx-no-tap-1"} if armErr := m.armEgress(t.Context(), sb); armErr == nil { t.Fatal("armEgress must fail closed when the egress-lane NIC cannot be locked") @@ -87,12 +85,9 @@ func TestArmEgressFailsClosedWhenNICUnlockable(t *testing.T) { } func TestArmEgressLocksEgressLaneWithoutPolicy(t *testing.T) { - // guardedEgress on (one pool carries a policy) plus a policyless egress-lane - // pool: that lane must still lock the NIC (default-deny), so arming a claim - // whose NIC cannot be locked fails rather than handing out a free NIC. m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}}}}, - config.PoolSpec{PoolKey: egKey}, // egress lane, no egress policy + config.PoolSpec{PoolKey: egKey}, ) sb := &types.Sandbox{ID: "sb_eg_np", Key: egKey, VMName: "sbx-np-1"} if armErr := m.armEgress(t.Context(), sb); armErr == nil { @@ -101,8 +96,6 @@ func TestArmEgressLocksEgressLaneWithoutPolicy(t *testing.T) { } func TestEgressLaneDoesNotHibernate(t *testing.T) { - // cocoon resumes a guest before its fresh tap can be re-locked, so an - // egress-lane sandbox must refuse to hibernate rather than wake unlocked. m := newTestManager(t, newFakeEngine()) sb := &types.Sandbox{ID: "sb_eg_h", Key: types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall}} sb.Transition.Lock() @@ -114,7 +107,6 @@ func TestEgressLaneDoesNotHibernate(t *testing.T) { } func TestEgressLaneWakeFailsClosed(t *testing.T) { - // A hibernated/archived egress claim (corrupt state) must fail closed on wake. m := newTestManager(t, newFakeEngine()) cases := []struct { name string @@ -147,7 +139,7 @@ func TestEffectivePolicyComposition(t *testing.T) { }{ {"root takes the pool policy whole", "", true, both, nil, "a.test", "z.test", true}, {"root without a pool policy", "", true, nil, nil, "", "", false}, - {"tenant intersects", "acme", true, both, tenantOnly, "b.test", "a.test", true}, // a.test allowed by pool, denied by tenant + {"tenant intersects", "acme", true, both, tenantOnly, "b.test", "a.test", true}, {"tenant declaring no policy", "acme", true, both, nil, "", "", false}, {"tenant on a policyless pool", "acme", true, nil, tenantOnly, "", "", false}, {"tenant on a promoted template", "acme", false, nil, tenantOnly, "b.test", "a.test", true}, @@ -158,7 +150,7 @@ func TestEffectivePolicyComposition(t *testing.T) { t.Run(tc.name, func(t *testing.T) { m.pools = map[types.PoolKey]*pool{} if tc.pooled { - m.pools[testKey] = &pool{key: testKey} + m.pools[testKey] = newPool(testKey) } m.poolEgress = map[types.PoolKey]*egress.Policy{} if tc.pool != nil { @@ -204,7 +196,7 @@ func TestLockUsesProvisionedTapWithoutList(t *testing.T) { } switch { case lockErr != nil && !strings.Contains(lockErr.Error(), "tap-fake0"): - t.Errorf("lock failed outside nft: %v", lockErr) // nft itself may fail off-linux/unprivileged + t.Errorf("lock failed outside nft: %v", lockErr) case lockErr == nil: m.mu.Lock() got := m.egressTaps[sb.ID] @@ -222,7 +214,7 @@ func TestLockFallsBackToListForPreTapClaims(t *testing.T) { if _, err := eng.RunCold(t.Context(), "sbx-old", egKey); err != nil { t.Fatalf("seed vm: %v", err) } - // A pre-tap-journal claim has no TAP; the lock resolves it through the engine. + sb := &types.Sandbox{ID: "sb_old", Key: egKey, VMName: "sbx-old"} lockErr := m.lockEgressNIC(t.Context(), sb) if calls := eng.listCalls(); calls != 1 { @@ -236,8 +228,7 @@ func TestLockFallsBackToListForPreTapClaims(t *testing.T) { func TestBatchArmFailureRecordsNoUsage(t *testing.T) { eng := newFakeEngine() m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - // Second member (egress, no VM) fails to arm; the rolled-back batch must - // leave no claim event, else it stays open forever for the collector. + sbs := []*types.Sandbox{ {VMName: "sbx-ok", Key: testKey}, {VMName: "sbx-eg", Key: egKey}, @@ -264,11 +255,11 @@ func TestBatchArmFailureRecordsNoUsage(t *testing.T) { func TestRollbackKeepsLockWhenRemoveFails(t *testing.T) { eng := newFakeEngine() eng.removeErrFor = "sbx-r1" - // Both VMs are running; sbx-r1's remove fails and the engine keeps listing it. + eng.vms["sbx-r1"] = "/run/sbx-r1.sock" eng.vms["sbx-r2"] = "/run/sbx-r2.sock" m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - // Both members armed before a later batch member failed; sbx-r1's remove fails. + sbs := []*types.Sandbox{ {ID: "sb_r1", VMName: "sbx-r1", Key: egKey}, {ID: "sb_r2", VMName: "sbx-r2", Key: egKey}, @@ -300,7 +291,7 @@ func TestRollbackKeepsLockWhenRemoveFails(t *testing.T) { func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { eng := newFakeEngine() eng.removeErrFor = "sbx-q1" - // Running: removeVM reports "still here" only while the engine lists it. + eng.vms["sbx-q1"] = "/run/sbx-q1.sock" m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) sb := &types.Sandbox{ID: "sb_q1", VMName: "sbx-q1", Key: egKey, TAP: "tap-q1"} @@ -328,11 +319,9 @@ func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { } func TestEgressLaneLocksWithNoPolicyAnywhere(t *testing.T) { - // A bridge egress lane with zero policy on the node: guardedEgress is false, - // but the NIC must still be nft-locked (default-deny), never handed out free. eng := newFakeEngine() eng.tap = "tap-nopol" - m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey}) // egress lane, no policy + m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey}) if m.guardedEgress { t.Fatal("no policy configured; guardedEgress should be false") } @@ -342,8 +331,6 @@ func TestEgressLaneLocksWithNoPolicyAnywhere(t *testing.T) { attempted := m.egressTaps[sb.ID] == "tap-nopol" m.mu.Unlock() if lockErr != nil { - // nft is linux-only; off Linux the lock fails but must name the tap it - // tried, proving lockEgressNIC ran rather than short-circuiting as before. attempted = strings.Contains(lockErr.Error(), "tap-nopol") } if !attempted { @@ -352,8 +339,6 @@ func TestEgressLaneLocksWithNoPolicyAnywhere(t *testing.T) { } func TestDisarmKeepsLockWhenRemoveFailed(t *testing.T) { - // A failed VM removal must tear down the proxy listener (stop credential - // injection for the dropped claim) but keep the NIC lock (the guest runs on). m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Egress: egPolicy}) m.dial = (&net.Dialer{}).DialContext sockDir, err := os.MkdirTemp("/tmp", "eg") @@ -366,7 +351,7 @@ func TestDisarmKeepsLockWhenRemoveFailed(t *testing.T) { t.Fatalf("arm proxy: %v", armErr) } m.mu.Lock() - m.egressTaps[sb.ID] = "tap-dz" // as lockEgressNIC would record at claim + m.egressTaps[sb.ID] = "tap-dz" m.mu.Unlock() path := engine.EgressSocketPath(sb.VsockSocket) @@ -383,11 +368,9 @@ func TestDisarmKeepsLockWhenRemoveFailed(t *testing.T) { } func TestSetPoolsPreservesEgressPolicy(t *testing.T) { - // The pool API cannot carry egress (config-only), so no SetPools call — warm - // change or drain-then-re-add — may drop a pool's policy. m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) gd := filepath.Join(m.goldensDir(), egKey.Hash()) - if err := os.MkdirAll(gd, 0o750); err != nil { // on disk so re-add adopts it, sparing an async build + if err := os.MkdirAll(gd, 0o750); err != nil { t.Fatalf("golden dir: %v", err) } m.mu.Lock() @@ -403,7 +386,7 @@ func TestSetPoolsPreservesEgressPolicy(t *testing.T) { if !policyLive() { t.Fatal("a warm change wiped the pool egress policy") } - if err := m.SetPools(t.Context(), nil); err != nil { // drain: the pool object is deleted + if err := m.SetPools(t.Context(), nil); err != nil { t.Fatalf("SetPools drain: %v", err) } if err := m.SetPools(t.Context(), []config.PoolSpec{{PoolKey: egKey}}); err != nil { @@ -416,31 +399,30 @@ func TestSetPoolsPreservesEgressPolicy(t *testing.T) { func TestEgressDialerBlocksInternal(t *testing.T) { blocked := []string{ - // Caught by IsGlobalUnicast/IsPrivate/IsLinkLocal. "127.0.0.1:80", "169.254.169.254:80", "10.0.0.5:443", "192.168.1.1:22", "172.16.0.1:80", "255.255.255.255:80", "[::1]:80", "[::ffff:127.0.0.1]:80", "[fe80::1]:80", "[fc00::1]:80", "[ff02::1]:80", - // IANA IPv4 special-purpose, Globally Reachable = false (one per table prefix). + "0.6.6.6:80", "100.100.100.200:80", "192.0.0.1:80", "192.0.2.1:80", "192.88.99.2:80", "198.18.0.1:80", "198.19.255.255:80", "198.51.100.1:80", "203.0.113.1:80", "240.1.2.3:80", - // IANA IPv6 special-purpose + deprecated v4-in-v6 embeddings. + "[100::1]:80", "[100:0:0:1::1]:80", "[2001:db8::1]:80", "[3fff::1]:80", "[5f00::1]:80", "[::127.0.0.1]:80", - "[2001::a9fe:a9fe]:80", // Teredo, within 2001::/23 - "[2001:2::1]:80", // benchmarking, within 2001::/23 - "[2001:10::1]:80", // ORCHID, within 2001::/23 - "[64:ff9b:1::8.8.8.8]:80", // RFC 8215 local-use NAT64 - "[2002:a9fe:a9fe::]:80", // 6to4-embedded 169.254.169.254 - "[64:ff9b::a9fe:a9fe]:80", // NAT64-embedded 169.254.169.254 - "[64:ff9b::a00:1]:80", // NAT64-embedded 10.0.0.1 + "[2001::a9fe:a9fe]:80", + "[2001:2::1]:80", + "[2001:10::1]:80", + "[64:ff9b:1::8.8.8.8]:80", + "[2002:a9fe:a9fe::]:80", + "[64:ff9b::a9fe:a9fe]:80", + "[64:ff9b::a00:1]:80", } for _, addr := range blocked { if err := newEgressDialer(nil).Control("tcp", addr, nil); err == nil { t.Errorf("dial to internal %s allowed; SSRF not blocked", addr) } } - // public v4 direct, public v6, and a public v4 via the NAT64 well-known prefix + for _, addr := range []string{"93.184.216.34:443", "[2606:4700:4700::1111]:443", "[64:ff9b::5db8:d822]:443"} { if err := newEgressDialer(nil).Control("tcp", addr, nil); err != nil { t.Errorf("dial to public %s blocked: %v", addr, err) @@ -465,9 +447,6 @@ func TestEgressLaneCannotForkOrCheckpoint(t *testing.T) { } } -// TestEgressDialerAdmitsOnlyNamedInternalPrefixes pins the SSRF boundary: -// named prefixes re-admit corporate services without re-admitting the guest -// bridges, which are themselves ULA/RFC1918. func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { d := newEgressDialer(parsePrefixes([]string{"fdc8::/16", "10.8.0.0/16"})) check := func(addr string) error { @@ -481,7 +460,7 @@ func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { "public v6": {"[2606:2800:220:1:248:1893:25c8:1946]:443", true}, "corporate v6": {"[fdc8:17:9:200f::1]:443", true}, "corporate v4": {"10.8.7.149:443", true}, - // Guests and the host gateway share fc00::/7 with the corporate range. + "another sandbox on the bridge": {"[fd00:c0c0:38::5]:443", false}, "the host's bridge gateway": {"[fd00:c0c0:38::1]:443", false}, "other private v4": {"192.168.1.1:443", false}, @@ -501,18 +480,15 @@ func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { } } -// TestEgressDialerWildcardAllowsEverything pins the fully-open form as a -// deliberate choice — loopback and cloud metadata included — for fleets with -// a policy-enforcing forward proxy in front. func TestEgressDialerWildcardAllowsEverything(t *testing.T) { d := newEgressDialer(parsePrefixes([]string{"0.0.0.0/0", "::/0"})) for _, addr := range []string{ - "93.184.216.34:443", // public - "[fdc8:17:9:200f::1]:443", // corporate v6 - "10.8.7.149:443", // corporate v4 - "[fd00:c0c0:38::5]:443", // another sandbox - "127.0.0.1:7777", // the node itself - "169.254.169.254:80", // cloud metadata + "93.184.216.34:443", + "[fdc8:17:9:200f::1]:443", + "10.8.7.149:443", + "[fd00:c0c0:38::5]:443", + "127.0.0.1:7777", + "169.254.169.254:80", } { if err := d.Control("tcp", addr, nil); err != nil { t.Errorf("%s blocked under a wildcard allow-list: %v", addr, err) @@ -520,9 +496,6 @@ func TestEgressDialerWildcardAllowsEverything(t *testing.T) { } } -// egressClient builds an HTTP client that reaches the origin through the -// per-sandbox egress proxy UDS, exactly as the guest's proxy client would over -// vsock. Playing the VMM's role lets the host-side path run without a real VM. func egressClient(path string) *http.Client { return &http.Client{ Timeout: 5 * time.Second, @@ -547,8 +520,6 @@ func egressManager(t *testing.T, eng *fakeEngine, pools ...config.PoolSpec) *Man return m } -// writeTestEgressCA provisions a cluster root + this node's intermediate to temp -// files, as `sandboxd ca` would, and returns the config pointing at them. func writeTestEgressCA(t *testing.T) *config.EgressCAConfig { t.Helper() rootCert, rootKey, err := egress.GenerateRoot("test cluster ca") diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index d7613620..06462f42 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -10,12 +10,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Fork clones a claimed sandbox into count children, each a fresh claim with -// its own lease: memory, disk, and guest state (sessions, processes, tmpfs) -// duplicate at the snapshot point, and cocoon's clone reseed gives every -// child a distinct machine identity. Children inherit the parent's tenant -// and count against its quota. All-or-nothing: any child failing destroys -// the ones already built, so an error means no child survived. +// Fork clones a claimed sandbox into count children, each a fresh claim; all-or-nothing. func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl time.Duration) ([]*types.Sandbox, error) { sb, ok := m.resolve(id, cred) if !ok { @@ -56,9 +51,7 @@ func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl return children, nil } -// forkClones builds count children from a claimed source: a running source is -// cloned straight from a fresh store snapshot (no export copy); a hibernated -// source stays on the export path (its shared wake image needs a private copy). +// forkClones clones a running source from a fresh snapshot, a hibernated one from an export. func (m *Manager) forkClones(ctx context.Context, sb *types.Sandbox, count int) ([]*types.Sandbox, error) { create, cleanup, err := m.forkSource(ctx, sb) if err != nil { @@ -68,9 +61,7 @@ func (m *Manager) forkClones(ctx context.Context, sb *types.Sandbox, count int) return m.cloneBatch(ctx, count, sb.Key, create) } -// forkSource decides and captures the fork source under the transition lock -// (a racing hibernate cannot swap the snapshot out from under the fan-out), -// returning the per-child provisioner and its cleanup. +// forkSource captures the fork source under the transition lock, against a racing hibernate. func (m *Manager) forkSource(ctx context.Context, sb *types.Sandbox) (vmProvisioner, func(), error) { sb.Transition.Lock() defer sb.Transition.Unlock() diff --git a/sandboxd/pool/fork_test.go b/sandboxd/pool/fork_test.go index 0b09e194..ef97e87e 100644 --- a/sandboxd/pool/fork_test.go +++ b/sandboxd/pool/fork_test.go @@ -10,14 +10,6 @@ import ( "time" ) -// TestForkRacingHibernateUsesPrivateSource pins the transition-locked source -// decision: a hibernate landing between Fork's entry and its capture must -// never let the fan-out clone the shared wake snapshot. -// -// Not a synctest candidate: Fork's block on sb.Transition (a plain -// sync.Mutex) is never "durably blocked" by synctest's rules, so the bubble -// can't fast-forward past the 20ms sleep that lets Fork reach it — verified -// as a real deadlock (times out at 120s), not a flake. func TestForkRacingHibernateUsesPrivateSource(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -46,7 +38,7 @@ func TestForkRacingHibernateUsesPrivateSource(t *testing.T) { children = append(children, sb.ID) } }) - time.Sleep(20 * time.Millisecond) // let Fork reach the transition lock + time.Sleep(20 * time.Millisecond) close(eng.hibernateStall) wg.Wait() @@ -78,7 +70,7 @@ func TestForkFromRunning(t *testing.T) { if len(eng.snapSaves) != 1 || !strings.HasPrefix(eng.snapSaves[0], forkPrefix) { t.Errorf("snapSaves %v, want one %s* snapshot", eng.snapSaves, forkPrefix) } - // Fast path: clone the children straight from the store snapshot, no export. + if len(eng.exports) != 0 { t.Errorf("exported %v, want none (fork clones from the snapshot directly)", eng.exports) } @@ -135,7 +127,7 @@ func TestForkFromHibernatedUsesWakeImage(t *testing.T) { if slices.Contains(eng.snapRemoves, hibSnap) { t.Error("hibernate snapshot dropped by fork — the parent could never wake") } - // The parent stays hibernated and must still wake. + if _, err := m.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after fork: %v", err) } @@ -167,17 +159,16 @@ func TestForkUnknownSandbox(t *testing.T) { func TestForkAllOrNothing(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - parent, err := claimAny(t.Context(), m, testKey, 0) // cold boot, no Clone calls yet + parent, err := claimAny(t.Context(), m, testKey, 0) if err != nil { t.Fatalf("Claim: %v", err) } - eng.cloneFailNth = 2 // fail one of the three fork clones + eng.cloneFailNth = 2 if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 3, 0); err == nil { t.Fatal("Fork succeeded despite a failed clone") } - // Every successfully built child is destroyed; only the parent's claim - // survives in memory and on disk. + if _, g := m.Info(); g.Claimed != 1 { t.Errorf("claimed=%d, want only the parent", g.Claimed) } @@ -206,11 +197,10 @@ func TestReconcileSweepsOrphanSnapshots(t *testing.T) { hibernatePrefix + "orphan-1", forkPrefix + "stale-2", goldenPrefix + "stale-3", - "user-snapshot", // not ours: never touched + "user-snapshot", } eng.mu.Unlock() - // A fresh manager over the same journal models the post-crash restart. m2 := newTestManagerAt(t, eng, dataDir) if err := m2.Reconcile(t.Context()); err != nil { t.Fatalf("Reconcile: %v", err) @@ -225,7 +215,7 @@ func TestReconcileSweepsOrphanSnapshots(t *testing.T) { t.Errorf("snapshot %q swept but must be kept", keep) } } - // The adopted hibernated claim still wakes after the restart. + if _, err := m2.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after reconcile: %v", err) } @@ -249,7 +239,7 @@ func TestForkChildrenInheritTenantAndQuota(t *testing.T) { t.Errorf("child %d tenant %q, want acme", i, c.Tenant) } } - // Parent plus two children fill acme's cap of 3; one more child is over. + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 1, 0); !errors.Is(err, ErrQuota) { t.Errorf("fork past the tenant cap: %v, want ErrQuota", err) } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 02c470ee..cae92656 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -13,9 +13,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Hibernate atomically snapshots a claimed sandbox and stops its VM, freeing -// memory; the next agent access wakes it. Idempotent on an already-hibernated -// sandbox. func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { sb, ok := m.resolve(id, cred) if !ok { @@ -26,9 +23,6 @@ func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { return m.hibernateLocked(ctx, sb) } -// WakeAgentSocket resolves the sandbox's vsock UDS for the relay, first -// restoring the VM if it is hibernated; concurrent wakes queue on the -// transition lock and find the fast path. func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string, error) { sb, ok := m.claim(id, token) if !ok { @@ -44,12 +38,10 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return ErrVolumeCapture } if sb.Key.Net == types.NetEgress { - // cocoon resumes the guest before its fresh tap can be re-locked, so a - // woken egress guest would egress unlocked; keep the lane live instead. + // cocoon resumes the guest before its fresh tap can be re-locked return ErrNoEgressHibernate } - // An adopted hibernate was never billed (its confirming list had failed), - // so record it even if this resolve's own persist is still converging. + // an adopted hibernate was never billed, so record it here adopted, resolveErr := m.resolvePendingSnap(ctx, sb) if adopted { m.recordHibernate(ctx, sb) @@ -64,24 +56,20 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error } return nil } - // A started transition must finish even if the caller hangs up (the - // engine bounds every step), or the record would disagree with the VM. + // a started transition must finish, or the record disagrees with the VM ctx = context.WithoutCancel(ctx) - // VMName-based (cocoon rejects sb.ID's "sb_" underscore) plus a random - // suffix so the wake's async snapshot drop can't collide with a re-hibernate. + // cocoon rejects sb.ID's underscore, so the snapshot name is VMName-based snap := hibernatePrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-" + randHex(3) - // Journal the intent before the engine stops anything: an intent that - // cannot be written aborts with the VM untouched, and a committed intent - // lets Reconcile adopt a hibernate whose final commit never landed. + // journal the intent first so Reconcile can adopt a hibernate whose commit never landed if err := m.store.commit(m.setPendingSnap(sb, snap)); err != nil { m.mu.Lock() sb.PendingSnap = "" + m.store.set(sb) m.mu.Unlock() return fmt.Errorf("hibernate %s: persist intent: %w", sb.ID, err) } if hibErr := m.eng.Hibernate(ctx, sb.VMName, snap); hibErr != nil { - // The engine can report failure after the snapshot landed (a CLI - // timeout); resolve the intent against the real snapshot. + // the engine can report failure after the snapshot landed adopted, resolveErr := m.resolvePendingSnap(ctx, sb) if adopted { m.recordHibernate(ctx, sb) @@ -119,8 +107,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, if sb.ArchiveCk != "" { return m.wakeArchived(ctx, sb) } - // Settle a dangling intent before choosing running vs hibernated; bill an - // adopted hibernate before the wake closes its window. + // settle a dangling intent before choosing running vs hibernated adopted, err := m.resolvePendingSnap(ctx, sb) if adopted { m.recordHibernate(ctx, sb) @@ -129,15 +116,13 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, return "", fmt.Errorf("wake %s: %w", sb.ID, err) } if sb.HibernateSnap == "" { - // No journal sync here — data-plane fast path; a lagging journal only - // says hibernated while the VM runs, which restart Reconcile heals. + // no journal sync on this data-plane fast path; restart Reconcile heals the lag if sb.StaleSnap != "" && m.store.synced() { m.dropStale(ctx, sb) } return sb.VsockSocket, nil } - // The egress lane never hibernates (its fresh tap can't be locked before the - // guest resumes); a hibernated one is corrupt state — fail closed. + // the egress lane never hibernates, so a hibernated one is corrupt state if sb.Key.Net == types.NetEgress { return "", fmt.Errorf("wake %s: egress lane cannot resume from hibernation", sb.ID) } @@ -147,9 +132,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, snap := sb.HibernateSnap restoredSock, err := m.eng.Restore(ctx, sb.VMName, snap) if err != nil { - // Restore may have booted the VM before the engine errored (a CLI - // timeout); tear it down, keeping the snapshot, so the next wake - // restores cleanly instead of re-restoring an already-running VM. + // restore may have booted the VM before the engine errored m.destroy(ctx, sb.VMName) return "", fmt.Errorf("wake %s: %w", sb.ID, err) } @@ -174,21 +157,17 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, return "", ErrUnknownSandbox } if err != nil { - // The lagging journal still references the snapshot; park it for - // reclaim after a later write lands, not just the restart sweep. + // the lagging journal still references the snapshot, so park it for later reclaim sb.StaleSnap = snap return "", fmt.Errorf("wake %s: persist claims: %w", sb.ID, err) } m.dropStale(ctx, sb) - // The resume consumed the memory image; reclaim its disk off the wake- - // return path (name reuse guarded by randHex, failed drops by the sweep). + // the resume consumed the memory image; reclaim its disk off the wake-return path (the randHex suffix keeps a re-hibernate from reusing the name) go m.dropSnap(ctx, snap) return sock, nil } -// idleOnce hibernates claims idle past their pool's (or the node's) -// threshold. Best-effort: a connection racing the sweep may see its sandbox -// hibernate right after — the next call wakes it transparently. +// idleOnce hibernates claims idle past their pool's (or the node's) threshold. func (m *Manager) idleOnce(ctx context.Context) { if !m.idleEnabled { return @@ -215,8 +194,7 @@ func (m *Manager) idleOnce(ctx context.Context) { m.idleSweep.Store(false) return } - // Hibernates are seconds-long engine snapshots: fan out off the - // housekeeping loop on the refill budget. + // hibernates are seconds-long engine snapshots, so fan out off the housekeeping loop go func() { defer m.idleSweep.Store(false) logger := log.WithFunc("pool.idleOnce") @@ -227,9 +205,7 @@ func (m *Manager) idleOnce(ctx context.Context) { }() } -// idleHibernate re-validates a sweep victim under the Transition lock: a -// data-plane connection that arrived after the sweep's snapshot refreshes -// LastActivity, and hibernating underneath it would cut a live call. +// idleHibernate re-validates a sweep victim under the Transition lock. func (m *Manager) idleHibernate(ctx context.Context, id, token string, sweepStart time.Time) error { sb, ok := m.claim(id, token) if !ok { @@ -246,10 +222,7 @@ func (m *Manager) idleHibernate(ctx context.Context, id, token string, sweepStar return m.hibernateLocked(ctx, sb) } -// commitTransition publishes a hibernate/wake result and persists the journal -// only if the claim is still live — Release/reap skip the transition lock, and -// publishing after them resurrects state nobody owns. Returns a failed write -// (caller must not report success); recommit converges disk in the background. +// commitTransition publishes a hibernate/wake result only if the claim is still live. func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, sock string) (live bool, err error) { m.mu.Lock() live = m.claimed[sb.ID] == sb @@ -258,7 +231,7 @@ func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, sb.HibernateSnap = snap sb.VsockSocket = sock sb.PendingSnap = "" - js = m.store.snapshot(m.claimed) + js = m.store.set(sb) } m.mu.Unlock() if !live { @@ -271,9 +244,7 @@ func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, return true, nil } -// syncClaims backs the idempotent hibernate fast path: a retry after a -// failed persist must not answer success while the journal still lags the -// transition it reports. The caller holds sb.Transition. +// syncClaims flushes a lagging journal so a hibernate retry cannot report a false success. func (m *Manager) syncClaims(ctx context.Context, sb *types.Sandbox) error { if !m.store.synced() { if err := m.store.commit(m.claimsSnapshot()); err != nil { @@ -295,13 +266,10 @@ func (m *Manager) setPendingSnap(sb *types.Sandbox, snap string) claimSnapshot { m.mu.Lock() defer m.mu.Unlock() sb.PendingSnap = snap - return m.store.snapshot(m.claimed) + return m.store.set(sb) } -// resolvePendingSnap settles a hibernate intent whose engine result was never -// confirmed: the snapshot's presence decides whether to adopt it as -// HibernateSnap or clear the intent. An unusable snapshot list keeps the -// intent and errors. The caller holds sb.Transition. +// resolvePendingSnap settles an unconfirmed hibernate intent against the engine's snapshot list. func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (adopted bool, err error) { if sb.PendingSnap == "" { return false, nil @@ -314,17 +282,17 @@ func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (ad live := m.claimed[sb.ID] == sb pending := sb.PendingSnap exists := slices.Contains(snaps, pending) + var js claimSnapshot if live { if exists { sb.HibernateSnap = pending } sb.PendingSnap = "" + js = m.store.set(sb) } - js := m.store.snapshot(m.claimed) m.mu.Unlock() if !live { - // Released mid-resolve: Release captured an empty HibernateSnap, so a - // completed hibernate's snapshot is an orphan it could not drop. + // Release captured an empty HibernateSnap, so a completed hibernate's snapshot is an orphan if exists { m.dropSnap(ctx, pending) } @@ -332,8 +300,7 @@ func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (ad } if err := m.store.commit(js); err != nil { m.recommit(ctx, js) - // The transition is decided in memory: report the adoption so the - // caller bills it, surfacing the persist error — commitTransition contract. + // the transition is decided in memory, so report the adoption for the caller to bill return exists, fmt.Errorf("resolve hibernate intent %s: persist: %w", sb.ID, err) } return exists, nil @@ -344,9 +311,7 @@ func (m *Manager) recordHibernate(ctx context.Context, sb *types.Sandbox) { m.recordUsage(ctx, usageEvent{Event: "hibernate", ID: sb.ID, VMName: sb.VMName}) } -// skipIdle reports the claims an idle sweep must leave alone: the egress lane -// cannot resume, a mounted volume cannot be captured, and an already -// hibernated or archived claim has nothing left to do. +// skipIdle reports the claims an idle sweep must leave alone. func skipIdle(sb *types.Sandbox, idle time.Duration, now time.Time) bool { return idle <= 0 || sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index bfa27520..ff92f83e 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -13,11 +13,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// TestHibernatePersistFailureSurfacesAndConverges guards the durability fix -// where a hibernate whose journal write failed still reported success — a -// restart then dropped the claim and destroyed the VM and snapshot. With the -// intent write first, an unwritable journal aborts before the engine stops -// anything; a retry after the store heals runs the full transition. func TestHibernatePersistFailureSurfacesAndConverges(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -46,10 +41,6 @@ func TestHibernatePersistFailureSurfacesAndConverges(t *testing.T) { } } -// TestWakePersistFailureSurfaces is the wake side of the same fix: retries -// must surface it, the snapshot the lagging journal references must survive -// (a fast-path retry keeps serving the awake VM — a restart adopts running -// VMs, so the lag is harmless there), and it is reclaimed once a write lands. func TestWakePersistFailureSurfaces(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -77,8 +68,7 @@ func TestWakePersistFailureSurfaces(t *testing.T) { if err := os.MkdirAll(filepath.Dir(broken), 0o750); err != nil { t.Fatalf("heal store dir: %v", err) } - // The background recommit converges the journal; the next fast-path wake - // then reclaims the parked snapshot. + waitFor(t, func() bool { if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("fast-path wake: %v", err) @@ -92,16 +82,12 @@ func TestWakePersistFailureSurfaces(t *testing.T) { }) } -// TestHibernateAmbiguousErrorTreatedAsDone: the engine can report failure -// after the snapshot landed (a CLI timeout). A snapshot the list confirms -// present means the hibernate happened — the call must not clear the intent -// or fail, or a restart would destroy a genuinely hibernated claim. func TestHibernateAmbiguousErrorTreatedAsDone(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) eng.hibernateErr = errors.New("cli timeout") - eng.hibernateErrCompletes = true // the snapshot landed before the timeout + eng.hibernateErrCompletes = true if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v, want nil (the snapshot landed despite the error)", err) @@ -115,14 +101,11 @@ func TestHibernateAmbiguousErrorTreatedAsDone(t *testing.T) { } } -// TestHibernateVerifiedFailureClearsIntent: only a snapshot the list confirms -// absent proves the hibernate did not happen — then the intent clears, the -// claim stays running, and a retry hibernates cleanly. func TestHibernateVerifiedFailureClearsIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - eng.hibernateErr = errors.New("cli failed") // completes=false: no snapshot + eng.hibernateErr = errors.New("cli failed") if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success despite a verified failure") @@ -140,10 +123,6 @@ func TestHibernateVerifiedFailureClearsIntent(t *testing.T) { } } -// TestHibernateUnverifiableErrorKeepsIntent: when the engine errors and the -// snapshot list is also unusable (the same cocoon fault), the outcome is -// unknown — the call must surface the error and keep the intent for Reconcile, -// never report success on an unconfirmed transition. func TestHibernateUnverifiableErrorKeepsIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -163,10 +142,6 @@ func TestHibernateUnverifiableErrorKeepsIntent(t *testing.T) { } } -// TestWakeRestoreErrorDestroysAndRetries: Restore can boot the VM before the -// engine errors (a CLI timeout); the failed wake must tear it down (keeping -// the snapshot) so the next wake restores cleanly instead of re-restoring an -// already-running VM. func TestWakeRestoreErrorDestroysAndRetries(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -196,15 +171,11 @@ func TestWakeRestoreErrorDestroysAndRetries(t *testing.T) { } } -// TestRetryResolvesDanglingIntent: a hibernate that actually completed but -// left a dangling intent (CLI and list both timed out) must be resolved by a -// later retry against the real snapshot — adopted, not overwritten with a -// fresh name that would strand the real snapshot and drop the claim on restart. func TestRetryResolvesDanglingIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - // First hibernate completes, but the CLI and the list both time out. + eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") @@ -216,7 +187,6 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { t.Fatalf("intent %q, want the real snapshot %q kept", got[sb.ID].PendingSnap, realSnap) } - // cocoon recovers; the retry must adopt the real snapshot, not re-hibernate. eng.hibernateErr = nil eng.snapListErr = nil if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { @@ -240,9 +210,6 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { } } -// TestWakeResolvesDanglingIntent: a Wake facing a dangling intent whose -// snapshot exists must resolve it and restore, not return the stopped VM's -// stale socket as if it were live. func TestWakeResolvesDanglingIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -271,16 +238,12 @@ func TestWakeResolvesDanglingIntent(t *testing.T) { } } -// TestResolvePendingSnapReleaseRaceDropsOrphan: a Release landing while a wake -// is mid-resolve (Release captured an empty HibernateSnap, so it drops -// nothing) must not strand the intent's real snapshot — resolvePendingSnap -// sees the claim gone and drops the orphan itself. func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - // Drive to a dangling intent whose snapshot really exists. + eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") @@ -289,7 +252,6 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { } realSnap := eng.hibernates[0] - // A wake resolves, but stalls inside SnapshotList; Release lands first. eng.snapListErr = nil eng.snapListStall = make(chan struct{}) callsBefore := eng.snapListCalls() @@ -311,9 +273,6 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { }) } -// TestResolveAdoptBillsWhenPersistFails: a real transition must be billed even -// when the adopting resolve's own journal write fails — memory is authoritative -// and recommit converges disk, so it must not silently escape metering. func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -325,7 +284,7 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } - // The list recovers, but the claims journal is now unwritable. + eng.snapListErr = nil broken := filepath.Join(t.TempDir(), "gone", "claims.json") m.store.path = broken @@ -335,7 +294,7 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { if n := m.Counters().Hibernates; n != 1 { t.Errorf("hibernates billed=%d, want 1 (billed even while the persist lags)", n) } - // Heal so the background recommit converges (and its goroutine exits). + if err := os.MkdirAll(filepath.Dir(broken), 0o750); err != nil { t.Fatalf("heal store dir: %v", err) } @@ -346,9 +305,6 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { }) } -// TestWakeProbeFailureDestroysAndRetries: a readiness-probe timeout after a -// successful Restore must tear the booted VM down (keeping the snapshot) so -// the next wake restores cleanly, instead of re-restoring a running VM. func TestWakeProbeFailureDestroysAndRetries(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -378,8 +334,6 @@ func TestWakeProbeFailureDestroysAndRetries(t *testing.T) { } } -// TestFlushClaimsClosesShutdownWindow: the shutdown hook persists what the -// background recommit has not converged yet (here, a wake whose commit failed). func TestFlushClaimsClosesShutdownWindow(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -441,12 +395,11 @@ func TestHibernateWakeCycle(t *testing.T) { if len(eng.restores) != 1 { t.Errorf("engine restored %d times, want 1", len(eng.restores)) } - waitFor(t, func() bool { return eng.snapRemoved(hibSnap) }) // wake drops it off the return path + waitFor(t, func() bool { return eng.snapRemoved(hibSnap) }) if _, g := m.Info(); g.Hibernated != 0 { t.Errorf("hibernated count %d after wake, want 0", g.Hibernated) } - // Awake sandbox: the fast path must not touch the engine again. if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("fast-path WakeAgentSocket: %v", err) } @@ -456,8 +409,6 @@ func TestHibernateWakeCycle(t *testing.T) { }) } -// TestWakeDropsSnapshotOffReturnPath: wake returns the socket without waiting -// for the consumed snapshot's (subprocess) removal. func TestWakeDropsSnapshotOffReturnPath(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -537,8 +488,6 @@ func TestReleaseMidHibernateDropsOrphanSnapshot(t *testing.T) { return len(eng.hibernates) == 1 }) - // Release lands while the engine transition is in flight: the claim is - // gone before Hibernate can commit, so its snapshot must not leak. if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Release: %v", err) } @@ -555,10 +504,6 @@ func TestReleaseMidHibernateDropsOrphanSnapshot(t *testing.T) { }) } -// TestReconcileAdoptsJournaledIntent: a hibernate that stopped the VM but -// whose final commit never landed leaves a journaled intent (PendingSnap) -// over a stopped VM; reconcile adopts it as hibernated — no heuristics, the -// intent names the wake image. func TestReconcileAdoptsJournaledIntent(t *testing.T) { eng := newFakeEngine() eng.vms["sbx-lagged-1"] = "/vsock/lagged" @@ -590,10 +535,6 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { } } -// TestReconcileIgnoresStaleOrphanSnapshot: a leftover wake image without a -// journaled intent must never be mistaken for a hibernate — adopting it -// would silently roll the sandbox back to pre-wake state. The claim drops, -// the orphan sweeps. func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -619,8 +560,6 @@ func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { }) } -// TestReconcileDropsIntentWithoutImage: an intent whose wake image is -// verified missing means the hibernate never completed; the claim drops. func TestReconcileDropsIntentWithoutImage(t *testing.T) { eng := newFakeEngine() eng.vms["sbx-noimg-1"] = "/vsock/noimg" diff --git a/sandboxd/pool/idle_test.go b/sandboxd/pool/idle_test.go index 56c68edf..e12537e1 100644 --- a/sandboxd/pool/idle_test.go +++ b/sandboxd/pool/idle_test.go @@ -24,7 +24,6 @@ func TestIdleOnceHibernatesPastThreshold(t *testing.T) { m.idleOnce(t.Context()) waitFor(t, func() bool { return hibernated(m) == 1 }) - // Idempotent on an already-hibernated claim. m.idleOnce(t.Context()) waitFor(t, func() bool { return !m.idleSweep.Load() }) if hibernated(m) != 1 { @@ -37,7 +36,7 @@ func TestIdleOncePolicyScope(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) - m.idleDefault = time.Second // node default must NOT reach pooled keys + m.idleDefault = time.Second m.idleEnabled = true sb := mustClaim(t, m, testKey) @@ -48,7 +47,6 @@ func TestIdleOncePolicyScope(t *testing.T) { t.Fatal("pooled key without the policy was idle-hibernated by the node default") } - // An unpooled key (template claim shape) takes the node default. unpooled := types.PoolKey{Template: "tpl:v1", Net: types.NetNone, Size: types.SizeSmall} sb2, err := m.ClaimProvision(t.Context(), unpooled, time.Hour, "", "", nil) if err != nil { @@ -67,7 +65,6 @@ func TestActivityStampsBlockIdleSweep(t *testing.T) { sb := mustClaim(t, m, testKey) backdate(m, sb, 2*time.Second) - // A data-plane connection refreshes the stamp; the sweep must spare it. if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("WakeAgentSocket: %v", err) } diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index d210e1f2..4a7655e7 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -122,8 +122,7 @@ func TestInterceptPoolAllowsPromote(t *testing.T) { m.mu.Lock() m.claimed[sb.ID] = sb m.mu.Unlock() - // The cluster root is shared, so an interception sandbox's disk carries no - // node-private material: promote/checkpoint are unrestricted. + if _, _, err := m.Promote(t.Context(), sb.ID, Cred{Token: "tok"}, "tpl:x", ""); err != nil { t.Errorf("Promote of an interception-pool sandbox: %v, want success", err) } diff --git a/sandboxd/pool/journal.go b/sandboxd/pool/journal.go index 96ff9fd0..9f32d159 100644 --- a/sandboxd/pool/journal.go +++ b/sandboxd/pool/journal.go @@ -8,14 +8,10 @@ import ( "time" ) -// journalMaxBytes rotates a journal once it crosses this size; one .1 backup -// is kept so a slow collector never loses a window silently. +// journalMaxBytes rotates a journal once it crosses this size; one .1 backup is kept. const journalMaxBytes = 64 * 1024 * 1024 -// usageEvent is one line of usage.jsonl: the product-level billing stream, -// folded by the platform collector (compute time = claim→release minus -// hibernate→wake; hibernated storage = hibernate→wake). VM names ride along -// so cocoon's machine-level metering ledger can audit the fold. +// usageEvent is one line of usage.jsonl, the billing stream the platform collector folds. type usageEvent struct { Time time.Time `json:"t"` Event string `json:"ev"` // claim|hibernate|wake|fork|promote|checkpoint|release|reap @@ -29,8 +25,7 @@ type usageEvent struct { Reference string `json:"ref,omitempty"` // promote: template; checkpoint: ckpt id } -// journal is an append-only JSONL writer with size rotation. Writes happen -// outside the pool mutex; the journal's own lock only orders appends. +// journal is an append-only JSONL writer with size rotation; its lock only orders appends. type journal struct { mu sync.Mutex path string @@ -74,23 +69,22 @@ func (j *journal) close() error { return j.f.Close() } -// rotate moves the live file to .1 (replacing any prior backup) and reopens. -// A failed swap degrades to an oversized journal: reopen, append, retry next time. +// rotate moves the live file to .1 and reopens; the old descriptor closes last. func (j *journal) rotate() error { - if err := j.f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - return err - } renameErr := os.Rename(j.path, j.path+".1") f, err := os.OpenFile(j.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) //nolint:gosec // under data_dir if err != nil { return errors.Join(renameErr, err) } + old := j.f j.f, j.size = f, 0 if renameErr != nil { if st, statErr := f.Stat(); statErr == nil { j.size = st.Size() } - return renameErr } - return nil + if closeErr := old.Close(); closeErr != nil && !errors.Is(closeErr, os.ErrClosed) { + renameErr = errors.Join(renameErr, closeErr) + } + return renameErr } diff --git a/sandboxd/pool/journal_test.go b/sandboxd/pool/journal_test.go index 433f668b..686b7a40 100644 --- a/sandboxd/pool/journal_test.go +++ b/sandboxd/pool/journal_test.go @@ -22,7 +22,7 @@ func TestJournalRotateFailureDegradesAndRecovers(t *testing.T) { t.Fatalf("new journal: %v", err) } defer func() { _ = j.close() }() - // A directory at the backup path makes the rotation rename fail. + if err = os.Mkdir(path+".1", 0o750); err != nil { t.Fatalf("setup: %v", err) } @@ -53,6 +53,32 @@ func TestJournalRotateFailureDegradesAndRecovers(t *testing.T) { } } +func TestJournalRotateReopenFailureKeepsWriting(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "usage.jsonl") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.Truncate(path, journalMaxBytes); err != nil { + t.Fatalf("setup: %v", err) + } + j, err := newJournal(path) + if err != nil { + t.Fatalf("new journal: %v", err) + } + defer func() { _ = j.close() }() + j.path = filepath.Join(dir, "gone", "usage.jsonl") + if err = j.append(map[string]string{"ev": "a"}); err == nil { + t.Fatal("append across a failed reopen must surface the error") + } + if err = j.append(map[string]string{"ev": "b"}); err == nil { + t.Fatal("append while the reopen stays blocked must keep surfacing the error") + } + if got := tail(t, path); !strings.Contains(got, `"ev":"a"`) || !strings.Contains(got, `"ev":"b"`) { + t.Errorf("events lost to a closed descriptor: %q", got) + } +} + func tail(t *testing.T, path string) string { t.Helper() f, err := os.Open(path) diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index fbc31ae7..4c3c2654 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -6,8 +6,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Cred is a caller's resolved authority over one sandbox: the per-sandbox -// token, or Operator for the root api_token the server already verified. +// Cred is a caller's resolved authority over one sandbox: its token, or Operator for root. type Cred struct { Token string Operator bool @@ -21,13 +20,11 @@ func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { if sb == nil { return SandboxSummary{}, false } - // summarize reads fields hibernate and archive mutate under m.mu, so it - // runs here, not after byID has released the lock. + // summarize reads fields hibernate and archive mutate under m.mu return summarize(sb), true } -// Wake restores a hibernated sandbox and leaves it running: waking is -// otherwise only a side effect of opening an agent connection. Idempotent. +// Wake restores a hibernated sandbox and leaves it running. Idempotent. func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { sb, ok := m.resolve(id, cred) if !ok { @@ -36,8 +33,7 @@ func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { return m.wake(ctx, sb) } -// resolve authorizes id under cred: Operator by id alone, otherwise the token -// must match — an unclaimed slot must never match an empty token. +// resolve authorizes id under cred; an unclaimed slot must never match an empty token. func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { if cred.Operator { return m.byID(id) @@ -48,8 +44,7 @@ func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { return m.claim(id, cred.Token) } -// byID resolves a live claim by id alone; callers must already have authorized -// the Operator credential. +// byID resolves a live claim by id alone; callers must have authorized the Operator credential. func (m *Manager) byID(id string) (*types.Sandbox, bool) { m.mu.Lock() defer m.mu.Unlock() @@ -57,8 +52,7 @@ func (m *Manager) byID(id string) (*types.Sandbox, bool) { return sb, sb != nil } -// wake reuses the relay's resolve path, then discards the socket: the caller -// wants the VM running, not a connection to it. +// wake reuses the relay's resolve path, then discards the socket. func (m *Manager) wake(ctx context.Context, sb *types.Sandbox) error { sb.Touch() _, err := m.wakeResolved(ctx, sb) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 7e9ec929..3a6c5136 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -41,9 +41,7 @@ const ( buildRetryDelay = 30 * time.Second claimProbeTimeout = 15 * time.Second coldProbeTimeout = 90 * time.Second - // A refill has no caller waiting and holds a target-accounting slot for - // its whole probe; a clone answers in ~1s even saturated, so a silent one - // is replaced, not waited on. + // a clone answers in ~1s even saturated, so a silent one is replaced, not waited on warmProbeTimeout = 5 * time.Second // One list; a wrong answer only costs an extra sweep. removeVerifyTimeout = 15 * time.Second @@ -54,15 +52,12 @@ const ( maxTTL = 24 * time.Hour recommitBackoff = 20 * time.Millisecond recommitMaxBackoff = 5 * time.Second - // One failed boot is ordinary; only an unbroken run of failures says the - // next attempt will fail too. + // one failed boot is ordinary; only an unbroken run predicts the next failure refillFailStreak = 8 // Doubles per failure past the streak. refillBackoffBase = 250 * time.Millisecond refillBackoffMax = buildRetryDelay - // defaultMaxFork/defaultRefill are the fallbacks when a Manager is built - // from a Config that skipped config.Load's defaulting (direct construction - // in tests). + // fallbacks when a Manager is built from a Config that skipped config.Load's defaulting defaultMaxFork = 16 defaultRefill = 4 @@ -134,8 +129,7 @@ type SandboxSummary struct { Archived bool `json:"archived,omitempty"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` - // ClaimRef echoes the caller reference recorded at claim time; empty for - // fork and checkpoint-branch claims. + // ClaimRef echoes the caller reference; empty for fork and checkpoint-branch claims. ClaimRef string `json:"claim_ref,omitempty"` } @@ -154,8 +148,7 @@ type Gauges struct { Hibernated int Archived int Draining bool - // AtCapacity marks refill parked because the node refused another VM, so a - // placer reads a short warm count as "full", not "still filling". + // AtCapacity marks refill parked because the node refused another VM. AtCapacity bool AtCapacityReason string } @@ -168,11 +161,10 @@ type pendingRemoval struct { } type pool struct { - key types.PoolKey + key types.PoolKey + hash string - // floor and warmMax bound the demand-adaptive target (watermark.go); - // rate/lead/lastArrival are its EWMA inputs, guarded by the manager - // mutex like everything else here. + // floor and warmMax bound the demand-adaptive target computed in watermark.go floor int warmMax int idle time.Duration @@ -189,20 +181,21 @@ type pool struct { warm []*types.Sandbox refilling int - // An unbroken failure streak and the retry gate it earns; without them a - // node-wide dead cause is retried every tick at full concurrency. + // without a streak gate a node-wide dead cause is retried every tick at full concurrency refillFails int nextRefill time.Time } -// refillGated reports whether a refill may not spawn: the backoff wait, or — -// expired — the single probe already in flight (a success resets the streak). +func newPool(key types.PoolKey) *pool { + return &pool{key: key, hash: key.Hash()} +} + +// refillGated reports whether a refill may not spawn: backoff wait, or one probe in flight. func (p *pool) refillGated(now time.Time) bool { return now.Before(p.nextRefill) || (p.refillFails >= refillFailStreak && p.refilling > 0) } -// noteRefillResult records one refill outcome and reports whether this call -// started a backoff, so the caller logs the transition, not every attempt. +// noteRefillResult records one refill outcome and reports whether it started a backoff. func (p *pool) noteRefillResult(now time.Time, failed bool) bool { if !failed { p.refillFails, p.nextRefill = 0, time.Time{} @@ -226,8 +219,7 @@ func (p *pool) applySpec(spec config.PoolSpec) { p.archiveDelete = time.Duration(spec.ArchiveDeleteAfterSeconds) * time.Second } -// trimWarm shrinks p.warm to at most target, returning the trimmed VM names; -// callers hold m.mu and destroy the returned VMs outside it. +// trimWarm shrinks p.warm to at most target and returns the trimmed VM names; callers hold m.mu. func (p *pool) trimWarm(target int) []string { var trim []string for len(p.warm) > target { @@ -249,46 +241,32 @@ type Manager struct { maxFork int store *claimStore volumes map[string]catalogVolume - // volumeAdmission counts the live holders of each volume name, mirroring - // the hypervisor's own per-image lock so a conflicting claim is refused - // before it pays any attach cost; guarded by m.mu. + // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders poolStore *poolStore configSeedHash string // config pools' hash, to warn when a file edit is overridden - // idleDefault is the idle-hibernate threshold for unpooled keys; pooled - // keys carry theirs on the pool struct. Zero means disabled. + // idleDefault is the idle-hibernate threshold for unpooled keys; zero disables. idleDefault time.Duration idleEnabled bool idleSweep atomic.Bool - // archive*Default are the archive thresholds for unpooled keys; pooled keys - // carry theirs on the pool struct. archiveEnabled is set when any pool or - // the node default has archiving on; archiveSweep guards the sweep loop. + // archive*Default are the archive thresholds for unpooled keys. archiveAfterDefault time.Duration archiveDeleteDefault time.Duration archiveEnabled bool archiveSweep atomic.Bool archiveDeleteSweep atomic.Bool - // archiving holds ids with an archive() export in flight, so the reap tick - // and archive sweep don't both re-export the same sandbox; pendingCks pins - // checkpoint ids during archive publish and removal commits. Both guarded - // by m.mu. + // archiving holds ids with an export in flight; pendingCks pins ids mid-commit; both under m.mu. archiving map[string]struct{} pendingCks map[string]struct{} - // egressListeners holds the per-sandbox egress proxy accept point, keyed by - // sandbox id; guarded by m.mu. + // egressListeners holds the per-sandbox egress proxy accept point; guarded by m.mu. egressListeners map[string]*egressListener - // egressTaps holds the nft-locked egress-lane tap per sandbox id, tracked - // apart from the proxy so a locked-but-policyless NIC still unlocks on - // release; guarded by m.mu. + // egressTaps holds the nft-locked egress-lane tap per sandbox id; guarded by m.mu. egressTaps map[string]string - // maxClaims caps live claims node-wide (0 = unlimited); tenantMax holds - // every configured tenant's cap (0 = unlimited) and doubles as the set of - // known tenants; tenantLive counts live claims per tenant. usage is the - // always-on billing event stream, audit the config-gated request tap. + // tenantMax doubles as the set of known tenants; a 0 cap means unlimited. maxClaims int draining bool // guarded by m.mu; deliberately not persisted tenantMax map[string]int @@ -299,14 +277,10 @@ type Manager struct { audit *journal counters counters ckpts store.Store - // A cluster-wide backend resolves every record from every node, so heal - // and the delete broadcast are both no-ops against it. + // a cluster-wide backend makes heal and the delete broadcast no-ops ckptsShared bool healer *peer.Healer - // healSem bounds concurrent transfers node-wide; healFlights dedups - // same-id heals onto one transfer, which owns its own staging dir. - // healPending/healAbort (guarded by recLocksMu) let a delete veto a heal - // still staging the same id (see vetoIfHealPending). + // healPending/healAbort, guarded by recLocksMu, let a delete veto a heal still staging. healSem chan struct{} healFlights singleflight.Group healPending map[string]struct{} @@ -316,41 +290,26 @@ type Manager struct { ckptTTL time.Duration ckptSweeping atomic.Bool - // tplSet caches each template id against its owning tenant ("" = operator), - // so the gossip tick and the claim path never pay a store read. + // tplSet caches each template id against its owning tenant ("" = operator). tplMu sync.Mutex tplSet map[string]string - // recLocks serializes same-id store record mutations and holds off a - // re-publish swap while a clone reads the old generation (per id, RW). - // recLocksMu guards recRefs, the live-holder count that gates eviction: a - // checkpoint id can become live again after a delete (a peer's heal can - // republish one), so an entry is only safe to evict once nothing still - // holds or awaits it — never on a bare "record deleted" signal. - recLocks sync.Map + // recLocks serializes same-id record mutations; an entry evicts only when no holder remains, never on a bare delete, since a peer heal can republish the id. + recLocks map[string]*sync.RWMutex recLocksMu sync.Mutex recRefs map[string]int - // recEvict remembers ids a delete asked to evict while a reference still - // held the lock, so the eviction happens when the last holder leaves rather - // than being lost — otherwise the recLocks entry leaks for that id. + // recEvict defers a delete's eviction to the last holder, else the recLocks entry leaks. recEvict map[string]struct{} - // notifyTemplates, when set (before serving starts), fires after a - // promote or template delete so the mesh republishes immediately - // instead of waiting out a gossip tick. + // notifyTemplates is set before serving starts and fires after a promote or template delete. notifyTemplates func() - // egressSecrets resolves a rule's secret name to the injected header, read-only - // after startup. guardedEgress arms the proxy (some policy exists); lockEgress - // nft-locks every egress-lane NIC default-deny (a bridge lane exists). egressCA - // signs HTTPS-interception leaves and is baked into interception pools' goldens; - // nil unless a pool rule sets intercept. + // egressSecrets is read-only after startup; egressCA is nil unless a pool rule intercepts. egressSecrets *egress.SecretStore egressCA *egress.CA guardedEgress bool lockEgress bool - // dial and sweep are fields as test seams: the SSRF guard blocks loopback - // test origins, and the nft sweep is netlink-only. + // dial and sweep are test seams: the SSRF guard blocks loopback, the nft sweep is netlink-only. dial egress.DialFunc sweep func(map[string]bool) error @@ -359,8 +318,7 @@ type Manager struct { claimed map[string]*types.Sandbox pendingRemovals map[string]pendingRemoval - // Node-wide, unlike the per-pool streak backoff: the resource that ran - // out — bridge ports, disk — is shared by every pool. + // node-wide, unlike the per-pool streak backoff: the exhausted resource is shared atCapacityUntil time.Time atCapacityReason string @@ -369,9 +327,7 @@ type Manager struct { refillKick chan struct{} } -// NewManager builds a manager from the node config; ctx bounds backend -// construction (the s3 store resolves its credential chain). secrets resolves -// egress rule references and is shared by every per-sandbox proxy. +// NewManager builds a manager from the node config; ctx bounds backend construction. func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *egress.SecretStore) (*Manager, error) { maxFork := cfg.MaxForkCount if maxFork < 1 { @@ -399,6 +355,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg pendingCks: map[string]struct{}{}, egressListeners: map[string]*egressListener{}, egressTaps: map[string]string{}, + recLocks: map[string]*sync.RWMutex{}, recRefs: map[string]int{}, recEvict: map[string]struct{}{}, healPending: map[string]struct{}{}, @@ -421,9 +378,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) } - // Checkpoints and templates are two id-namespaced views over ONE store - // root (ck_* vs tp_*): a shared mount or bucket carries both, and each - // instance's listing filters to its own records. + // checkpoints and templates are two id-namespaced views (ck_*, tp_*) over one store root var err error if m.ckpts, err = newStoreView(ctx, cfg, "checkpoint-staging", store.CheckpointIDRe); err != nil { return nil, err @@ -471,7 +426,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.archiveDeleteDefault = time.Duration(cfg.ArchiveDeleteAfterSeconds) * time.Second m.archiveEnabled = m.archiveAfterDefault > 0 for _, spec := range cfg.Pools { - p := &pool{key: spec.PoolKey} + p := newPool(spec.PoolKey) p.applySpec(spec) m.pools[spec.PoolKey] = p if spec.IdleHibernateSeconds > 0 { @@ -499,8 +454,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg return m, nil } -// EgressCAFingerprint is the egress cluster root's fingerprint, or "" when the -// node does no interception. +// EgressCAFingerprint is the egress root's fingerprint, or "" when the node intercepts nothing. func (m *Manager) EgressCAFingerprint() string { if m.egressCA == nil { return "" @@ -514,8 +468,7 @@ func (m *Manager) Run(ctx context.Context) { defer refill.Stop() reap := time.NewTicker(reapInterval) defer reap.Stop() - // Store retention is hourly, not per reap tick: shared roots and buckets - // make each sweep cluster-visible I/O. + // store retention is hourly: each sweep is cluster-visible I/O on a shared root storeSweep := time.NewTicker(time.Hour) defer storeSweep.Stop() if m.ckptTTL > 0 { @@ -545,14 +498,12 @@ func (m *Manager) Run(ctx context.Context) { } } -// FlushClaims synchronously persists the current claim set — the shutdown -// hook that closes the window where a detached recommit has not converged yet. +// FlushClaims synchronously persists the current claim set at shutdown. func (m *Manager) FlushClaims() error { return m.store.commit(m.claimsSnapshot()) } -// SetTemplateNotifier wires the immediate-republish hook; call it before -// the server starts serving. +// SetTemplateNotifier wires the immediate-republish hook; call it before serving starts. func (m *Manager) SetTemplateNotifier(fn func()) { m.notifyTemplates = fn } @@ -562,11 +513,15 @@ func (m *Manager) Info() ([]PoolInfo, Gauges) { m.mu.Lock() defer m.mu.Unlock() now := time.Now() - pools := make([]PoolInfo, 0, len(m.pools)) + live := make([]*pool, 0, len(m.pools)) for _, p := range m.pools { - if p.removed { - continue + if !p.removed { + live = append(live, p) } + } + slices.SortFunc(live, func(a, b *pool) int { return strings.Compare(a.hash, b.hash) }) + pools := make([]PoolInfo, 0, len(live)) + for _, p := range live { pools = append(pools, PoolInfo{ Key: p.key, Warm: len(p.warm), @@ -575,7 +530,6 @@ func (m *Manager) Info() ([]PoolInfo, Gauges) { Golden: p.goldenDir != "", }) } - slices.SortFunc(pools, func(a, b PoolInfo) int { return strings.Compare(a.Key.Hash(), b.Key.Hash()) }) g := Gauges{Claimed: len(m.claimed), Draining: m.draining} if now.Before(m.atCapacityUntil) { g.AtCapacity, g.AtCapacityReason = true, m.atCapacityReason @@ -611,16 +565,15 @@ func (m *Manager) WarmCounts() map[string]int { m.mu.Lock() defer m.mu.Unlock() counts := make(map[string]int, len(m.pools)) - for key, p := range m.pools { + for _, p := range m.pools { if !p.removed { - counts[key.Hash()] = len(p.warm) + counts[p.hash] = len(p.warm) } } return counts } -// WithPeerHeal installs the healer ClaimCheckpointHeal pulls through, so a -// peer transfer is paid only once nothing cheaper resolves the claim. +// WithPeerHeal installs the healer ClaimCheckpointHeal pulls through. func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { if !enabled || owners == nil || m.ckptsShared { return @@ -628,8 +581,7 @@ func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { m.healer = peer.NewHealer(owners, &peer.HTTPPuller{Token: token}) } -// WithPeerDelete wires the broadcast DeleteCheckpoint makes after a local -// delete, so a healed replica does not outlive the source record. +// WithPeerDelete wires the broadcast DeleteCheckpoint makes after a local delete. func (m *Manager) WithPeerDelete(fn PeerDeleteFunc) { m.peerDelete = fn } @@ -645,9 +597,7 @@ func (m *Manager) sweepStoreGenerations(ctx context.Context) { } func (m *Manager) claimsSnapshot() claimSnapshot { - m.mu.Lock() - defer m.mu.Unlock() - return m.store.snapshot(m.claimed) + return m.store.mark() } func (m *Manager) untrack(set map[string]struct{}, key string) { @@ -714,9 +664,7 @@ func dirExists(path string) bool { return err == nil && fi.IsDir() } -// tenantOwns is THE tenancy predicate — every read/delete/overwrite scope -// check goes through it: root (empty tenant) owns everything, a tenant only -// records stamped with its own name. +// tenantOwns is the tenancy predicate: root (empty tenant) owns everything, a tenant its own. func tenantOwns(tenant, owner string) bool { return tenant == "" || tenant == owner } @@ -735,8 +683,7 @@ func logSweepResult(ctx context.Context, logger *log.Fields, err error, okMsg, f } } -// benignSweepErr reports whether err is the expected outcome of a housekeeping -// sweep (victim released, woke mid-sweep, or a lane that never hibernates). +// benignSweepErr reports whether err is the expected outcome of a housekeeping sweep. func benignSweepErr(err error) bool { return errors.Is(err, ErrUnknownSandbox) || errors.Is(err, errWokeMeanwhile) || errors.Is(err, ErrNoEgressHibernate) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index c6a9e766..3a8780bb 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -188,10 +188,6 @@ func TestReleaseValidatesToken(t *testing.T) { } } -// TestReleaseAfterResolveTearsDownOnce: resolve authorizes off the manager -// mutex, so two callers can both hold the same resolved claim before either -// drops it — the state a live double-release reaches. Exactly one may tear the -// VM down; the loser must read as unknown rather than remove it twice. func TestReleaseAfterResolveTearsDownOnce(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -223,11 +219,10 @@ func TestReleaseByOperatorCred(t *testing.T) { m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - // Unknown id is 404-equivalent, exactly like a token release. if err := m.Release(t.Context(), "sb_nope", Cred{Operator: true}); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("got %v, want ErrUnknownSandbox", err) } - // The operator releases by id alone — no per-sandbox token needed. + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { t.Fatalf("Release operator: %v", err) } @@ -239,8 +234,6 @@ func TestReleaseByOperatorCred(t *testing.T) { } } -// TestResolveRejectsEmptyToken documents resolve's explicit guard: an -// unclaimed slot must never match an empty token. func TestResolveRejectsEmptyToken(t *testing.T) { m := newTestManager(t, newFakeEngine()) sb := mustClaim(t, m, testKey) @@ -279,7 +272,7 @@ func TestReapDestroysExpiredClaims(t *testing.T) { m.mu.Unlock() m.reapOnce(t.Context()) - // The destroy batch is fanned off the ticker loop; poll for it. + waitFor(t, func() bool { return eng.removed(sb.VMName) }) if _, g := m.Info(); g.Claimed != 0 { t.Errorf("claimed=%d, want 0", g.Claimed) @@ -427,7 +420,7 @@ func TestRefillServicesEveryPoolUnderTightBudget(t *testing.T) { func TestSetPoolsRemovalShedsArchivePolicy(t *testing.T) { m := newTestManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 0, ArchiveAfterSeconds: 60, ArchiveDeleteAfterSeconds: 120}) m.mu.Lock() - m.pools[testKey].refilling = 1 // keep the removed pool lingering + m.pools[testKey].refilling = 1 m.mu.Unlock() if err := m.SetPools(t.Context(), nil); err != nil { @@ -467,8 +460,6 @@ func TestSetPoolsRejectsInvalidSpec(t *testing.T) { } } -// SetPools neither validates nor keeps egress policy: accepting it would -// report success and silently drop it. func TestSetPoolsRejectsEgress(t *testing.T) { m := newTestManager(t, newFakeEngine()) err := m.SetPools(t.Context(), []config.PoolSpec{{ @@ -603,8 +594,6 @@ func TestRefillBudgetFollowsConfig(t *testing.T) { } } -// TestRefillReactivelyFillsToTarget: one trigger fills the whole target (not -// just the concurrency budget), since a finished refill chains the next. func TestRefillReactivelyFillsToTarget(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -677,9 +666,6 @@ func TestReapBatchDoesNotStallTheLoop(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - // More expired claims than the concurrency budget (defaultRefill here) so - // the batch would block the submit loop if it acquired the semaphore there - // instead of per-goroutine. const n = defaultRefill + 3 var sbs []*types.Sandbox for range n { @@ -699,7 +685,7 @@ func TestReapBatchDoesNotStallTheLoop(t *testing.T) { done := make(chan struct{}) go func() { m.reapOnce(t.Context()); close(done) }() select { - case <-done: // returned while every destroy is parked: the loop never blocked + case <-done: case <-time.After(2 * time.Second): t.Fatal("reapOnce blocked on a batch larger than the budget") } @@ -710,7 +696,7 @@ func TestReapBatchDoesNotStallTheLoop(t *testing.T) { func TestProbeReadyWaitsForVsock(t *testing.T) { eng := newFakeEngine() - eng.vsockLateN = 2 // cocoon reports the socket only once the VMM runs + eng.vsockLateN = 2 m := newTestManager(t, eng) sb, err := claimAny(t.Context(), m, testKey, 0) @@ -727,8 +713,6 @@ func newTestManager(t *testing.T, eng *fakeEngine, pools ...config.PoolSpec) *Ma return newTestManagerAt(t, eng, t.TempDir(), pools...) } -// claimAny composes warm-then-provision the way the server does around the -// redirect decision; production has no single-call form. func claimAny(ctx context.Context, m *Manager, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) { sb, err := m.ClaimWarm(ctx, key, ttl, "", "", nil) if errors.Is(err, ErrNoWarm) { @@ -764,6 +748,19 @@ func testSecrets(t *testing.T, specs ...egress.SecretSpec) *egress.SecretStore { return s } +func lockCount(m *Manager) int { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + return len(m.recLocks) +} + +func hasRecLock(m *Manager, id string) bool { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + _, ok := m.recLocks[id] + return ok +} + func waitFor(t *testing.T, cond func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) @@ -776,65 +773,56 @@ func waitFor(t *testing.T, cond func() bool) { t.Fatal("condition not met within 3s") } -// fakeEngine implements Engine in memory: created VMs land in vms (backing -// List/vsockOf), destroy removes them, and error fields fail one verb at a -// time. type fakeEngine struct { - mu sync.Mutex - vms map[string]string // VM name → vsock socket path - clones []string - cloneFroms []string - colds []string - removes []string - probeTimeouts []time.Duration - volumeSpecs []engine.VolumeSpec - volumeMounts []types.Volume - volumeOps []string - // attachDirty records whether an image's dirty marker was already down when - // it was attached; removeSeenOps and removeSeenDirty snapshot the volume-op - // log and the markers still on disk at each VM removal, so teardown ordering - // is checkable on both sides of the removal. + mu sync.Mutex + vms map[string]string + clones []string + cloneFroms []string + colds []string + removes []string + probeTimeouts []time.Duration + volumeSpecs []engine.VolumeSpec + volumeMounts []types.Volume + volumeOps []string attachDirty map[string]bool removeSeenOps map[string][]string removeSeenDirty map[string][]string - // dirtyPaths are the images removeSeenDirty samples; tests set it to the - // catalog paths the claim under test uses. - dirtyPaths []string - syncs []string // vsock sockets SyncGuest was called on + dirtyPaths []string + syncs []string hibernates, restores, snapRemoves []string snapSaves, exports, snapshots []string exportContent []byte - caInstalls []string // vsock sockets InstallCACert was called on - staleReconciles []string // VM names ReconcileStaleCreate was called on + caInstalls []string + staleReconciles []string installCAErr error diskAttachErr error - diskAttachErrFor string // volume name whose DiskAttach fails; "" = never + diskAttachErrFor string diskAttachCancel context.CancelFunc mountVolumeErr error unmountVolumeErr error stopped map[string]bool - creating map[string]bool // VMs List reports in the creating state + creating map[string]bool staleOutcome engine.StaleCreateOutcome staleErr error - pids map[string]int // VM name → PID, for Stats' resident-set lookup - vsockLateN int // List calls that report no socket yet + pids map[string]int + vsockLateN int cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapSaveErr, snapListErr error - removeErrFor string // VM name whose Remove fails; "" = never - cloneFailNth int // 1-based Clone call to fail; 0 = never - hibernateErrCompletes bool // hibernateErr fires after the snapshot lands (CLI timeout) - tap string // non-empty: lifecycle records carry this NIC tap + removeErrFor string + cloneFailNth int + hibernateErrCompletes bool + tap string listCount int - attachRendezvous *sync.WaitGroup // non-nil: DiskAttach waits there until every attach has arrived + attachRendezvous *sync.WaitGroup - cloneStall chan struct{} // non-nil: Clone blocks until closed - probeStall chan struct{} // non-nil: Probe blocks until closed - hibernateStall chan struct{} // non-nil: Hibernate blocks until closed - removeStall chan struct{} // non-nil: Remove blocks until closed - snapRemoveStall chan struct{} // non-nil: SnapshotRemove blocks until closed - snapListStall chan struct{} // non-nil: SnapshotList blocks until closed + cloneStall chan struct{} + probeStall chan struct{} + hibernateStall chan struct{} + removeStall chan struct{} + snapRemoveStall chan struct{} + snapListStall chan struct{} snapListCount int } @@ -866,7 +854,6 @@ func (f *fakeEngine) RunCold(_ context.Context, name string, _ types.PoolKey) (t } func (f *fakeEngine) Remove(ctx context.Context, name string) error { - // Models exec.CommandContext: a canceled ctx never runs cocoon at all. if err := ctx.Err(); err != nil { return err } @@ -893,7 +880,6 @@ func (f *fakeEngine) Remove(ctx context.Context, name string) error { } func (f *fakeEngine) ReconcileStaleCreate(ctx context.Context, name string) (engine.StaleCreateOutcome, error) { - // Models exec.CommandContext: a canceled ctx never runs cocoon at all. if err := ctx.Err(); err != nil { return "", err } @@ -989,7 +975,7 @@ func (f *fakeEngine) Restore(_ context.Context, name, _ string) (string, error) } f.stopped[name] = false if f.vms[name] == "" { - f.vms[name] = "/vsock/" + name // Restore rebuilds the VM from its snapshot + f.vms[name] = "/vsock/" + name } return f.lateVsock(f.vms[name]), nil } @@ -1114,7 +1100,6 @@ func (f *fakeEngine) clone(from, name string) (types.VMRecord, error) { return f.record(name), nil } -// record builds a lifecycle result for name; callers hold f.mu. func (f *fakeEngine) record(name string) types.VMRecord { rec := types.VMRecord{VsockSocket: f.lateVsock(f.vms[name]), Config: types.VMConfig{Name: name}} if f.tap != "" { @@ -1123,8 +1108,6 @@ func (f *fakeEngine) record(name string) types.VMRecord { return rec } -// lateVsock models cocoon's report-once-running lag: while vsockLateN ticks -// remain it reports no socket (consuming one), forcing the caller's poll. func (f *fakeEngine) lateVsock(sock string) string { if f.vsockLateN > 0 { f.vsockLateN-- @@ -1133,8 +1116,6 @@ func (f *fakeEngine) lateVsock(sock string) string { return sock } -// removed reports whether name was destroyed, under the fake's own lock — -// bounded batches append concurrently with test polls. func (f *fakeEngine) removed(name string) bool { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index 4521dd45..a9db122a 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -21,9 +21,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// poolsFile records the last API-applied pool set so a restart rebuilds from -// operator intent, not the config.json seed. ConfigSeed is the config pools' -// hash at write time; a boot finding it changed warns the edit is overridden. +// poolsFile records the last API-applied pool set so a restart rebuilds from operator intent. type poolsFile struct { ConfigSeed string `json:"config_seed"` Pools []config.PoolSpec `json:"pools"` @@ -41,8 +39,7 @@ func newPoolStore(dataDir string) *poolStore { return &poolStore{path: filepath.Join(dataDir, "pools.json")} } -// load returns the persisted pool file, or nil when the node has never taken a -// PUT /v1/pools (config.json seeds the pools, unchanged behavior). +// load returns the persisted pool file, or nil when the node has never taken a PUT /v1/pools. func (s *poolStore) load() (*poolsFile, error) { raw, err := os.ReadFile(s.path) if errors.Is(err, fs.ErrNotExist) { @@ -58,8 +55,7 @@ func (s *poolStore) load() (*poolsFile, error) { return &pf, nil } -// commit durably writes the applied set, serialized by s.mu; a seq no newer -// than the last written is a no-op, so a superseded concurrent write can't win. +// commit durably writes the applied set; a seq no newer than the last written is a no-op. func (s *poolStore) commit(seq uint64, pf poolsFile) error { s.mu.Lock() defer s.mu.Unlock() @@ -77,9 +73,7 @@ func (s *poolStore) commit(seq uint64, pf poolsFile) error { return nil } -// adoptPersistedPools rebuilds the pools from pools.json (the last API-applied -// set) over the config seed when present. Egress stays config-owned; a restored -// pool that no longer validates fails the boot, matching config-load discipline. +// adoptPersistedPools rebuilds the pools from pools.json over the config seed when present. func (m *Manager) adoptPersistedPools(ctx context.Context) error { pf, err := m.poolStore.load() if err != nil { @@ -103,7 +97,7 @@ func (m *Manager) adoptPersistedPools(ctx context.Context) error { if err := spec.ValidateLimits(); err != nil { return fmt.Errorf("restore pool %q: %w", spec.Template, err) } - p := &pool{key: spec.PoolKey} + p := newPool(spec.PoolKey) p.applySpec(spec) m.adoptGolden(p) m.pools[spec.PoolKey] = p @@ -118,8 +112,7 @@ func (m *Manager) adoptPersistedPools(ctx context.Context) error { return nil } -// poolSeedHash digests a pool set's warm-target shape, order-independent and -// egress-excluded, so it tracks exactly the targets pools.json would override. +// poolSeedHash digests a pool set's warm-target shape, order-independent and egress-excluded. func poolSeedHash(specs []config.PoolSpec) string { shaped := slices.Clone(specs) for i := range shaped { diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index 7a99d8d5..e6fe4b79 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -22,7 +22,7 @@ func TestPersistedPoolsSurviveRestart(t *testing.T) { if err := m.SetPools(t.Context(), []config.PoolSpec{{PoolKey: apiKey, Warm: 3}}); err != nil { t.Fatalf("SetPools: %v", err) } - // Restart on the same data dir, still config-seeded: must rebuild from pools.json. + m2 := newTestManagerAt(t, newFakeEngine(), dir, config.PoolSpec{PoolKey: seedKey, Warm: 1}) m2.mu.Lock() _, hasSeed := m2.pools[seedKey] @@ -67,7 +67,7 @@ func TestConcurrentSetPoolsPersistLatest(t *testing.T) { }) } wg.Wait() - // The last-applied set owns memory; the persisted file must match it. + m.mu.Lock() want := m.pools[apiKey].floor m.mu.Unlock() @@ -92,7 +92,7 @@ func TestRestoredEgressPoolNeedsAttachment(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "pools.json"), raw, 0o600); err != nil { t.Fatalf("write pools.json: %v", err) } - // A node with no bridge/network must refuse to start a restored egress pool. + if _, err := NewManager(t.Context(), &config.Config{DataDir: dir}, newFakeEngine(), testSecrets(t)); err == nil { t.Error("NewManager accepted a restored egress-lane pool without an attachment") } diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index c54f5711..46d66d5a 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -129,8 +129,7 @@ func TestPromoteValidations(t *testing.T) { if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: "wrong"}, "tpl:x", ""); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("bad token: %v, want ErrUnknownSandbox", err) } - // Same template/net/size as the configured pool: the golden path would - // collide with the pool's own. + if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, testKey.Template, ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("pooled key: %v, want ErrPooledTemplate", err) } @@ -160,7 +159,7 @@ func TestDeleteTemplate(t *testing.T) { if m.HasGolden(t.Context(), key, "") { t.Error("template still resolvable after delete") } - // The next claim for the deleted template cold-boots instead of cloning. + before := len(eng.colds) if _, err := claimAny(t.Context(), m, key, 0); err != nil { t.Fatalf("Claim after delete: %v", err) @@ -186,8 +185,6 @@ func TestReconcileSweepsGoldenTmpDirs(t *testing.T) { } func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { - // A template promoted before the egress guard existed must never be resumed: - // resolveGolden returns "" for the egress lane so the claim cold-boots. m := newTestManager(t, newFakeEngine()) id := store.TemplateID(egKey.Hash()) staging, err := m.tpls.Stage(id) @@ -236,9 +233,6 @@ func TestTemplateTenantScopedDelete(t *testing.T) { } } -// TestCommitTemplateRechecksOwnerUnderLock drives the publish-side check -// directly: two racers that both passed Promote's early check must still -// serialize on ownership at publish. func TestCommitTemplateRechecksOwnerUnderLock(t *testing.T) { m := newTestManager(t, newFakeEngine()) id := store.TemplateID(testKey.Hash()) @@ -261,8 +255,6 @@ func TestCommitTemplateRechecksOwnerUnderLock(t *testing.T) { } } -// TestPromoteFailsClosedOnMetaError: an unreadable template meta must refuse -// the promote, never skip the ownership check. func TestPromoteFailsClosedOnMetaError(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root ignores file permissions") @@ -306,7 +298,7 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { if _, _, err := m.Promote(t.Context(), b.ID, Cred{Token: b.Token}, "shared:v1", "beta"); !errors.Is(err, ErrTemplateOwned) { t.Errorf("beta overwrite: %v, want ErrTemplateOwned", err) } - r := claim("") // root may replace anything + r := claim("") if _, _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "shared:v1", ""); err != nil { t.Errorf("root replace: %v, want ok", err) } @@ -370,8 +362,6 @@ func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { t.Fatalf("promote: %v", err) } - // Routing must answer what a claim would: promising beta a golden here - // makes redirectClaim skip the peer hop and cold-boot the name as an image. if m.HasPromotedTemplate(t.Context(), key, "beta") { t.Error("beta sees acme's template as a local golden") } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 1ac31e3a..d67155b0 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -17,11 +17,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Reconcile aligns state after a daemon restart: re-adopt persisted claims -// whose VMs are still running (or hibernated), drop the rest, and remove any -// sbx-prefixed VM nobody owns (stale pool VMs and golden builders from a -// previous life). It must run once at startup, before the server: it swaps -// in fresh records, which would bypass in-flight Transition locks. +// Reconcile aligns claims and VMs after a restart; it runs before the server because it swaps in fresh records that would bypass in-flight Transition locks. func (m *Manager) Reconcile(ctx context.Context) error { claims, err := m.store.load() if err != nil { @@ -45,26 +41,19 @@ func (m *Manager) Reconcile(ctx context.Context) error { rec, ok := live[sb.VMName] switch { case sb.ArchiveCk != "": - // Archived: no local VM by design; the store ck is the durable - // state. Adopt the stub; the first exec wakes it (wakeArchived). + // archived: no local VM by design, so adopt the stub and let the first exec wake it m.claimed[id] = sb m.tenantDelta(sb.Tenant, 1) continue case ok && rec.State == vmStateRunning: sb.VsockSocket = rec.VsockSocket - // Running with either flag set = a wake crashed between restore - // and commit, or a hibernate intent never reached the engine; - // clearing them un-bricks the claim and unreferences the - // snapshot for the sweep below. + // running with either flag set means a crashed wake or an intent the engine never saw sb.HibernateSnap, sb.PendingSnap = "", "" case ok && sb.HibernateSnap != "": // Hibernated: the VM is stopped by design and wakes on demand. sb.PendingSnap = "" case ok && sb.PendingSnap != "" && (snapsErr != nil || slices.Contains(snaps, sb.PendingSnap)): - // Stopped under a journaled hibernate intent whose commit never - // landed: the intent names the wake image, adopt it. A verified- - // missing image means the hibernate never completed — fall - // through and drop; an unverifiable list adopts. + // the journaled intent names the wake image, so adopt it when the snapshot is there sb.HibernateSnap, sb.PendingSnap = sb.PendingSnap, "" default: continue @@ -80,8 +69,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.adoptGolden(p) } m.mu.Unlock() - // A crash mid-export leaves a *.tmp staging dir no build or promote of - // this life will reuse. + // a crash mid-export leaves a *.tmp staging dir nothing in this life reuses tmps, _ := filepath.Glob(filepath.Join(m.goldensDir(), "*.tmp")) if err := m.ckpts.SweepStaging(); err != nil { log.WithFunc("pool.Reconcile").Error(ctx, err, "sweep checkpoint staging") @@ -98,9 +86,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.retryArchiveDeletes(ctx) removed := m.sweepStaleVMs(ctx, live, owned) - // A hibernate snapshot no adopted claim references is an orphan; - // fork/golden-build snapshots never span a restart. A list failure only - // skips the sweep — GC must not brick startup. + // an unreferenced hibernate snapshot is an orphan; fork and golden snapshots never span a restart if snapsErr != nil { logger.Warnf(ctx, "snapshot sweep skipped: %v", snapsErr) } else { @@ -121,8 +107,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { return saveErr } -// sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones -// confirmed gone; a failed remove is omitted so the egress sweep keeps its lock. +// sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRecord, owned map[string]bool) map[string]bool { logger := log.WithFunc("pool.sweepStaleVMs") var stale []string @@ -147,15 +132,11 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec return removed } -// removeStaleVM reclaims one unowned VM, reporting whether it is gone; a -// creating-state record goes through reconcile-stale-create first, since -// rm --force would queue on the ops lock and then delete the VM an -// in-flight clone just produced. +// removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") if rec.State == vmStateCreating { - // Cancellation-immune like removeVM: a canceled ctx must not skip - // the busy check and fall through to the forced remove it guards. + // a canceled ctx must not skip the busy check guarding the forced remove switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); { case err != nil: // Verb missing (cocoon < v0.5.8) or failed: keep the old sweep. @@ -172,11 +153,15 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe return m.removeOrRetry(ctx, name, "", rec.TapDevice(), volumeTeardown{}) } -// resyncEgress re-locks adopted egress claims after a restart, quarantines any -// it cannot lock, and sweeps tables orphaned by VMs confirmed gone (in removed). +// resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) { logger := log.WithFunc("pool.resyncEgress") now := time.Now() + var locked map[string]bool + var lockedErr error + if m.lockEgress { + locked, lockedErr = netfilter.LockedTaps() + } var quarantine []*types.Sandbox for _, sb := range m.claimed { sb.TouchAt(now) @@ -187,7 +172,11 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco quarantine = append(quarantine, sb) continue } - if err := netfilter.EnsureLock(tap); err != nil { + err := lockedErr + if err == nil && !locked[tap] { + err = netfilter.Lock(tap) + } + if err != nil { logger.Errorf(ctx, err, "ensure egress lock %s; quarantining", sb.ID) quarantine = append(quarantine, sb) continue @@ -224,7 +213,7 @@ func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { m.mu.Lock() delete(m.claimed, sb.ID) m.tenantDelta(sb.Tenant, -1) - js := m.store.snapshot(m.claimed) + js := m.store.del(sb.ID) m.mu.Unlock() if err := m.store.commit(js); err != nil { m.recommit(ctx, js) @@ -232,8 +221,7 @@ func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { return gone } -// readoptEgressTap records a live egress-lane claim's tap for lifecycle after a -// restart and returns it; "" for the none lane or a VM whose tap is not listed. +// readoptEgressTap records and returns a live egress claim's tap, "" when there is none. func (m *Manager) readoptEgressTap(sb *types.Sandbox, live map[string]types.VMRecord) string { if sb.Key.Net != types.NetEgress { return "" @@ -246,14 +234,12 @@ func (m *Manager) readoptEgressTap(sb *types.Sandbox, live map[string]types.VMRe m.mu.Lock() m.egressTaps[sb.ID] = tap sb.TAP = tap + m.store.set(sb) m.mu.Unlock() return tap } -// reclaimOrphanArchiveCks reaps archives that crashed between their -// checkpoint publish and the journal commit: the flag hides them from -// listings and deletes, so only the journal identifies ours — the sandbox is -// in it under a different (or no) ArchiveCk. +// reclaimOrphanArchiveCks reaps archives that crashed between publish and journal commit. func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string]*types.Sandbox) { logger := log.WithFunc("pool.reclaimOrphanArchiveCks") metas, err := m.ckpts.Metas(ctx) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 96ce2b1e..d5b23e94 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -152,8 +152,6 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { } } -// staleCreateCase drives both the startup sweep and the reap-tick retry -// through the same outcome/end-state row shape. type staleCreateCase struct { name string outcome engine.StaleCreateOutcome diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index bc1099ea..82ea628e 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -19,8 +19,7 @@ import ( type vmProvisioner func(name string) (types.VMRecord, error) -// kickRefill nudges Run past its refill ticker after a warm claim; the -// 1-buffered channel coalesces bursts and never blocks the claim path. +// kickRefill nudges Run past its refill ticker without blocking the claim path. func (m *Manager) kickRefill() { select { case m.refillKick <- struct{}{}: @@ -35,15 +34,12 @@ func (m *Manager) refillOnce(ctx context.Context) { return } now := time.Now() - // Spawning VMs into a full node still takes the global rtnl lock even when - // doomed to fail. The park gates spawns only — sweeping removed pools is - // pure bookkeeping. + // spawning VMs into a full node still takes the global rtnl lock even when doomed to fail parked := now.Before(m.atCapacityUntil) inFlight := 0 for key, p := range m.pools { inFlight += p.refilling - // SetPools leaves a removed pool in place while a build/refill is in - // flight; sweep it once quiescent. + // SetPools leaves a removed pool in place while a build or refill is in flight if p.removed { if !p.building && p.refilling == 0 { delete(m.pools, key) @@ -106,8 +102,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string) { p.noteLead(time.Since(start)) keep = true } - // A canceled context is the daemon going down, and a capacity failure is - // the node's fault — neither says anything about this pool's health. + // a canceled context or a capacity failure says nothing about this pool's health backedOff := false if ctx.Err() == nil && capReason == "" { backedOff = p.noteRefillResult(now, err != nil) @@ -122,12 +117,11 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string) { switch { case ctx.Err() != nil: case entered: - // One line per episode; the storm this replaces wrote 20k an hour. log.WithFunc("pool.refillOne").Errorf(ctx, err, "node is at capacity; parking refill for %s", capacityBackoff) case parked: default: - hash := p.key.Hash() + hash := p.hash log.WithFunc("pool.refillOne").Errorf(ctx, err, "refill %s", hash) if backedOff { log.WithFunc("pool.refillOne").Warnf(ctx, @@ -145,8 +139,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string) { } } -// noteCapacityLocked parks refill node-wide for a capacity failure, reporting -// whether this call parked it. m.mu held. +// noteCapacityLocked parks refill node-wide, reporting whether this call parked it. func (m *Manager) noteCapacityLocked(now time.Time, reason string) bool { first := !now.Before(m.atCapacityUntil) m.atCapacityUntil = now.Add(capacityBackoff) @@ -156,7 +149,7 @@ func (m *Manager) noteCapacityLocked(now time.Time, reason string) bool { func (m *Manager) buildGolden(ctx context.Context, p *pool) { logger := log.WithFunc("pool.buildGolden") - hash := p.key.Hash() + hash := p.hash name := vmPrefix + "gb-" + hash snap := goldenPrefix + hash final := filepath.Join(m.goldensDir(), hash) @@ -173,8 +166,7 @@ func (m *Manager) buildGolden(ctx context.Context, p *pool) { p.goldenDir = final } else { p.nextBuild = time.Now().Add(buildRetryDelay) - // A cold boot hits the same bridge and disk as a refill; feed the park - // so a node failing only its builds still publishes at-capacity. + // a cold boot hits the same bridge and disk as a refill, so it feeds the park too if reason := engine.CapacitySignature(err); reason != "" { m.noteCapacityLocked(time.Now(), reason) } @@ -211,8 +203,7 @@ func (m *Manager) buildGoldenSteps(ctx context.Context, key types.PoolKey, name, return m.writeGoldenCASidecar(final, caBaked) } -// writeGoldenCASidecar records (or clears) the baked-CA fingerprint, so a -// rotated CA or flipped intercept flag forces a rebuild on restart. +// writeGoldenCASidecar records or clears the baked-CA fingerprint; a rotated CA forces a rebuild. func (m *Manager) writeGoldenCASidecar(final string, caBaked bool) error { path := final + caSidecarSuffix if !caBaked { @@ -227,17 +218,14 @@ func (m *Manager) writeGoldenCASidecar(final string, caBaked bool) error { return nil } -// adoptGolden points p at a golden already on disk from the pool's earlier -// life, when its baked-CA state still fits; buildGolden covers the rest. +// adoptGolden points p at an on-disk golden whose baked-CA state still fits. func (m *Manager) adoptGolden(p *pool) { - if g := filepath.Join(m.goldensDir(), p.key.Hash()); dirExists(g) && m.goldenCAMatches(g, m.poolEgress[p.key].Intercepts()) { + if g := filepath.Join(m.goldensDir(), p.hash); dirExists(g) && m.goldenCAMatches(g, m.poolEgress[p.key].Intercepts()) { p.goldenDir = g } } -// goldenCAMatches reports whether a golden's baked-CA state fits the pool: a -// CA-baked one must carry the current fingerprint, a plain one none. The plain -// case only stats (no read) — it runs under m.mu on the live SetPools path. +// goldenCAMatches reports whether a golden's baked-CA state fits the pool. func (m *Manager) goldenCAMatches(final string, caNeeded bool) bool { if !caNeeded { _, err := os.Stat(final + caSidecarSuffix) @@ -247,10 +235,7 @@ func (m *Manager) goldenCAMatches(final string, caNeeded bool) bool { return err == nil && m.egressCA != nil && string(fp) == m.egressCA.Fingerprint() } -// exportGolden exports snap into final through a unique sibling *.tmp dir: -// a crash mid-export never leaves a half-written dir that would pass for a -// golden, and concurrent promotes to one name cannot clobber each other's -// staging (last rename wins whole). +// exportGolden exports snap into final through a unique sibling *.tmp dir. func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { staging, err := os.MkdirTemp(filepath.Dir(final), filepath.Base(final)+"-*.tmp") if err != nil { @@ -267,9 +252,7 @@ func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { return os.Rename(tmp, final) } -// sourceSnap picks the snapshot to export a claimed sandbox from: the wake -// image of a hibernated one (kept — the wake still needs it), or a transient -// capture of a running one, dropped by the returned cleanup. +// sourceSnap picks the snapshot to export a claimed sandbox from, with its cleanup. func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, func(), error) { if sb.HibernateSnap != "" { return sb.HibernateSnap, func() {}, nil @@ -281,11 +264,7 @@ func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, fu return snap, func() { m.dropSnap(ctx, snap) }, nil } -// exportSource captures a claimed sandbox's state into exportDir, returning -// the snapshot name it exported. Only this window holds the transition lock — -// it pins the source snapshot against a concurrent wake consuming it; the -// minutes-long clone fan-out after it must not block the source's own -// wake/hibernate traffic. +// exportSource captures a claimed sandbox's state into exportDir, returning the snapshot name. func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir string) (string, error) { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -297,8 +276,7 @@ func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir return snap, m.eng.SnapshotExport(ctx, snap, exportDir) } -// provision creates one claim-ready VM: clone from a golden when available, -// cold-boot the template otherwise. +// provision creates one claim-ready VM, cloning from a golden when available. func (m *Manager) provision(ctx context.Context, key types.PoolKey, golden string) (*types.Sandbox, error) { if golden == "" { sb, err := m.provisionVM(ctx, key, coldProbeTimeout, func(name string) (types.VMRecord, error) { @@ -307,8 +285,7 @@ func (m *Manager) provision(ctx context.Context, key types.PoolKey, golden strin if err != nil { return nil, err } - // A pre-golden cold claim must trust the root too, or its intercepted - // hosts fail TLS for the sandbox's whole life. + // a pre-golden cold claim must trust the root, or intercepted hosts fail TLS for its life if m.poolIntercepts(key) { if err := m.eng.InstallCACert(ctx, sb.VsockSocket, m.egressCA.CertPEM()); err != nil { m.destroy(ctx, sb.VMName) @@ -390,9 +367,7 @@ func (m *Manager) cloneBatch(ctx context.Context, count int, key types.PoolKey, return children, nil } -// probeReady waits until a VM's silkd answers, returning its vsock socket. sock -// is what the lifecycle command reported; when empty (a heavy image can boot -// past cocoon's post-start inspect) it polls `vm list` until the socket appears. +// probeReady waits until a VM's silkd answers, polling `vm list` when sock is empty. func (m *Manager) probeReady(ctx context.Context, name, sock string, timeout time.Duration) (string, error) { deadline := time.Now().Add(timeout) if sock == "" { @@ -438,15 +413,11 @@ func (m *Manager) findVM(ctx context.Context, name string) (types.VMRecord, bool return vms[i], true, nil } -// runBounded fans f over n items on the refill semaphore, so engine -// batches (reap destroys, idle hibernates, reconcile sweeps) share the -// same node-wide concurrency budget as refills without blocking the -// caller. Callers that need completion Wait; fire-and-forget drops it. +// runBounded fans f over n items on the refill semaphore, sharing the node-wide budget. func (m *Manager) runBounded(ctx context.Context, n int, f func(context.Context, int)) *sync.WaitGroup { var wg sync.WaitGroup for i := range n { - // Acquire inside the goroutine: a batch larger than the budget - // must not block the caller (Run's select loop). + // acquire inside the goroutine so a batch larger than the budget cannot block the caller wg.Go(func() { select { case m.refillSem <- struct{}{}: diff --git a/sandboxd/pool/refillbackoff_test.go b/sandboxd/pool/refillbackoff_test.go index 2fba3ebb..2b325907 100644 --- a/sandboxd/pool/refillbackoff_test.go +++ b/sandboxd/pool/refillbackoff_test.go @@ -61,18 +61,14 @@ func TestBackoffGrowsAndIsCapped(t *testing.T) { } } -// TestRefillOnceHonorsTheBackoff pins the property that saves the node: -// while a pool is backed off, the ticker spawns nothing. func TestRefillOnceHonorsTheBackoff(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() - // Deliberately not a capacitySignature: that parks the node after one - // failure; the streak backoff is the fallback for unclassifiable errors. + eng.cloneErr = errors.New("clone: guest kernel panicked before vsock came up") m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 50}) m.pools[testKey].goldenDir = "/goldens/x" - // Concurrency is bounded, so the streak accrues over several ticks. backedOff := false for range 50 { m.refillOnce(t.Context()) @@ -92,8 +88,6 @@ func TestRefillOnceHonorsTheBackoff(t *testing.T) { t.Fatal("a pool whose every refill failed is not backed off") } - // Pin the expiry far out so the no-spawn assertions cannot race a short - // first backoff on a loaded machine. m.mu.Lock() m.pools[testKey].nextRefill = time.Now().Add(time.Hour) m.mu.Unlock() @@ -104,7 +98,6 @@ func TestRefillOnceHonorsTheBackoff(t *testing.T) { t.Errorf("clones=%d after backing off, want %d: a backed-off pool must not attempt again", n, attempts) } - // A pause, not a latch: once the wait expires a working engine fills. m.mu.Lock() m.pools[testKey].nextRefill = time.Now().Add(-time.Second) m.mu.Unlock() diff --git a/sandboxd/pool/refillkick_test.go b/sandboxd/pool/refillkick_test.go index 913fcdd1..061d3a21 100644 --- a/sandboxd/pool/refillkick_test.go +++ b/sandboxd/pool/refillkick_test.go @@ -24,8 +24,6 @@ func TestKickRefillCoalescesAndNeverBlocks(t *testing.T) { } } -// TestNodeAtCapacityParksRefillInsteadOfRetrying pins the difference between -// "this VM failed to start" and "this node cannot hold another VM". func TestNodeAtCapacityParksRefillInsteadOfRetrying(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -49,7 +47,6 @@ func TestNodeAtCapacityParksRefillInsteadOfRetrying(t *testing.T) { t.Fatalf("at-capacity not published: %+v", g) } - // Every later tick is a no-op until the backoff expires. for range 5 { m.refillOnce(t.Context()) } @@ -57,7 +54,6 @@ func TestNodeAtCapacityParksRefillInsteadOfRetrying(t *testing.T) { t.Errorf("clones=%d after parking, want %d: a parked node must not attempt again", n, attempts) } - // A park, not a latch: once the backoff expires refill resumes. m.mu.Lock() m.atCapacityUntil = time.Now().Add(-time.Second) m.mu.Unlock() diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 0092d32d..117d3402 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -42,10 +42,7 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { return true } -// removeOrRetry reports whether the VM is confirmed gone; a survivor is -// queued for the reap tick to retry, carrying what its cleanup needs (sandbox -// ID / tap / volume teardown). A restart loses a queued teardown: its markers -// stay until an rw claim clears them, and its holds die with the process. +// removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID, tap string, td volumeTeardown) bool { if m.removeVM(ctx, name) { m.finishVolumeTeardown(ctx, td) @@ -67,9 +64,7 @@ func (m *Manager) queueStaleCreate(name, tap string) { m.mu.Unlock() } -// retryRemovals dispatches by emptying the queue, so absence also means "in -// flight" and the next tick cannot double-dispatch; a failed retry re-queues -// itself on completion. Callers that need completion Wait. +// retryRemovals empties the queue to dispatch, so the next tick cannot double-dispatch. func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { m.mu.Lock() if len(m.pendingRemovals) == 0 { diff --git a/sandboxd/pool/remove_test.go b/sandboxd/pool/remove_test.go index 67f7e3ab..0ae1b39c 100644 --- a/sandboxd/pool/remove_test.go +++ b/sandboxd/pool/remove_test.go @@ -48,7 +48,6 @@ func TestRemovalRetryConvergesConfirmedSurvivor(t *testing.T) { t.Fatal("surviving VM was not retained for retry") } - // A failed retry re-queues itself, so the next tick tries again. m.retryRemovals(t.Context()).Wait() m.mu.Lock() _, pending = m.pendingRemovals["survivor"] diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index 9cd25b21..1a2fa9f9 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -11,8 +11,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// SetPools replaces the node's desired warm targets. Existing claims are not -// affected; only unclaimed warm VMs are trimmed or refilled. +// SetPools replaces the node's desired warm targets; existing claims are unaffected. func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { desired := make(map[types.PoolKey]config.PoolSpec, len(specs)) hashes := make(map[string]types.PoolKey, len(specs)) @@ -24,8 +23,7 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { if err := spec.ValidateLimits(); err != nil { return fmt.Errorf("%w: %v", ErrBadCount, err) } - // Egress is config-owned (validated at load); accepting it here would - // silently drop it. + // egress is config-owned; accepting it here would silently drop it if spec.Egress != nil { return fmt.Errorf("%w: pool %q: egress is set in the config file, not via the API", ErrBadKey, spec.Template) } @@ -46,8 +44,7 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { now := time.Now() for key, p := range m.pools { spec, ok := desired[key] - // A removed key gets the zero spec: a lingering pool sheds its whole - // policy — a stale archiveAfter would keep archiving its claims. + // a removed key gets the zero spec, so a lingering pool sheds a stale archiveAfter p.applySpec(spec) p.removed = !ok trim = append(trim, p.trimWarm(p.effectiveTarget(now))...) @@ -59,13 +56,12 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { if p := m.pools[key]; p != nil { continue } - p := &pool{key: key} + p := newPool(key) p.applySpec(spec) m.adoptGolden(p) m.pools[key] = p } - // Recompute rather than latch: removing every idle pool turns the - // sweep off again. + // recompute rather than latch: removing every idle pool turns the sweep off again m.idleEnabled = m.idleDefault > 0 m.archiveEnabled = m.archiveAfterDefault > 0 for _, p := range m.pools { @@ -88,8 +84,7 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { return m.poolStore.commit(seq, poolsFile{ConfigSeed: m.configSeedHash, Pools: persisted}) } -// normalizePoolSpec fills the wire defaults: API requests default net/size; -// config files stay explicit (Validate rejects empty there). +// normalizePoolSpec fills the wire defaults; config files stay explicit. func normalizePoolSpec(spec config.PoolSpec) config.PoolSpec { spec.PoolKey = spec.Defaulted() return spec diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index e5c67ffd..b258f6e1 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -9,11 +9,7 @@ import ( "time" ) -// SandboxStats is one sandbox's resource usage. CPUCount and MemTotalBytes come -// from the size tier the VM was booted with. MemUsedBytes is the host VMM -// process's resident set, the only usage signal available without a guest agent; -// MemUsedMeasured is false when no VMM process was found (a hibernated sandbox -// has none), so a zero is never read as idle. +// SandboxStats is one sandbox's resource usage; MemUsedMeasured false means the RSS was unreadable. type SandboxStats struct { ID string `json:"id"` CPUCount int `json:"cpu_count"` @@ -26,9 +22,7 @@ type SandboxStats struct { // Stats reports one live claim's resource usage. func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { - // Snapshot the mutable fields under m.mu — hibernate and archive rewrite - // HibernateSnap and VMName there — then do the /proc read, which touches no - // sandbox state, outside the lock. + // hibernate and archive rewrite HibernateSnap and VMName under m.mu; the /proc read needs none m.mu.Lock() sb := m.claimed[id] if sb == nil { @@ -53,9 +47,7 @@ func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { return st, true } -// vmResidentBytes reports the resident set of the hypervisor process serving -// vmName, read straight from engine.List's VMRecord.PID rather than scanning -// /proc for a matching argv. +// vmResidentBytes reports the resident set of the hypervisor process serving vmName. func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bool) { if vmName == "" { return 0, false diff --git a/sandboxd/pool/stats_test.go b/sandboxd/pool/stats_test.go index 4680dba7..3235a95c 100644 --- a/sandboxd/pool/stats_test.go +++ b/sandboxd/pool/stats_test.go @@ -9,7 +9,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" ) -// TestStatsUnknownSandbox: an unclaimed or bad id must not surface any usage. func TestStatsUnknownSandbox(t *testing.T) { m := newTestManager(t, newFakeEngine()) if _, ok := m.Stats(t.Context(), "sb_missing"); ok { @@ -17,9 +16,6 @@ func TestStatsUnknownSandbox(t *testing.T) { } } -// TestOperatorReadsRaceFreeUnderHibernate: Sandbox and Stats read fields that -// hibernate rewrites under m.mu, so they must snapshot under the same lock. -// Run with -race; without the lock this trips the detector. func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -34,8 +30,7 @@ func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { _ = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) } }) - // Read continuously for the writer's whole run, so the operator reads and - // the lifecycle mutations actually overlap for the detector to see. + wg.Go(func() { for { select { @@ -50,9 +45,6 @@ func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { wg.Wait() } -// TestStatsHibernatedIsUnmeasured: a hibernated claim has no VMM process, so -// MemUsedMeasured must stay false rather than reading a stale or zero RSS — -// and Stats must not even ask the engine, which a hibernated claim can't answer. func TestStatsHibernatedIsUnmeasured(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -77,8 +69,6 @@ func TestStatsHibernatedIsUnmeasured(t *testing.T) { } } -// TestStatsNoPIDIsUnmeasured: engine.List answering with no PID (VM not found, -// or not yet running) must not be read as a zero-byte RSS. func TestStatsNoPIDIsUnmeasured(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -93,10 +83,6 @@ func TestStatsNoPIDIsUnmeasured(t *testing.T) { } } -// TestStatsReadsResidentSetFromPID proves the PID engine.List reports is the -// one actually read: pointing it at this test process's own PID, the -// resident-set read must succeed and land on a real value. /proc is -// Linux-only, so this is skipped elsewhere. func TestStatsReadsResidentSetFromPID(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("/proc/PID/statm is Linux-only") diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index ffddd169..b7e46270 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -11,10 +11,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/egress" ) -// AuditLineCap bounds how much of a relayed request frame the audit tap -// captures and records: enough for the op and its addressing fields, never -// file payloads. The relay's tee and Audit share it, so a captured line is -// always recordable. +// AuditLineCap bounds how much of a relayed request frame the audit tap captures and records. const AuditLineCap = 4096 // Counters is a monotonic snapshot for /metrics; gauges come from Info. @@ -34,8 +31,7 @@ type Counters struct { Unarchives uint64 ArchiveDeletes uint64 - // Nanosecond totals paired with the counters above: avg latency for - // dashboards without histogram machinery. + // nanosecond totals paired with the counters above: avg latency without histograms ClaimNanos uint64 WakeNanos uint64 } @@ -86,8 +82,7 @@ func (m *Manager) Counters() Counters { } } -// TenantClaims counts live claims per configured tenant for /metrics — -// configured tenants only, so the label cardinality stays bounded. +// TenantClaims counts live claims per configured tenant, so label cardinality stays bounded. func (m *Manager) TenantClaims() map[string]int { m.mu.Lock() defer m.mu.Unlock() @@ -98,10 +93,7 @@ func (m *Manager) TenantClaims() map[string]int { return counts } -// Audit records one relayed request frame against a sandbox when the audit -// journal is enabled: the op plus its addressing fields, payloads dropped. -// It is the byte-level adapter over recordAudit for the relay's tee; -// non-relay sources (preview) call recordAudit with the frame directly. +// Audit records one relayed request frame against a sandbox: the op and addressing fields only. func (m *Manager) Audit(ctx context.Context, id string, line []byte) { if m.audit == nil || len(line) > AuditLineCap { return @@ -132,8 +124,7 @@ func (m *Manager) recordAudit(ctx context.Context, id string, frame auditFrame) } } -// recordEgress logs one guarded-egress decision to the audit tap (secret name -// only, never its value) and meters it as an always-on usage event. +// recordEgress logs one guarded-egress decision and meters it as a usage event. func (m *Manager) recordEgress(ctx context.Context, id, tenant string, ev egress.Event) { decision := "deny" if ev.Decision == egress.DecisionAllow { @@ -143,9 +134,7 @@ func (m *Manager) recordEgress(ctx context.Context, id, tenant string, ev egress m.recordUsage(ctx, usageEvent{Event: "egress", ID: id, Tenant: tenant, Reference: ev.Host}) } -// recordUsage appends one billing event; failures are logged, never -// propagated — a claim must not fail because the meter hiccuped (cocoon's -// machine-level ledger remains the audit backstop). +// recordUsage appends one billing event; failures are logged, never propagated. func (m *Manager) recordUsage(ctx context.Context, ev usageEvent) { ev.Time = time.Now() if err := m.usage.append(ev); err != nil { diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index f37bd7fa..1a5a8a06 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -62,7 +62,7 @@ func TestJournalRotationKeepsOneBackup(t *testing.T) { t.Fatalf("newJournal: %v", err) } defer func() { _ = j.close() }() - j.size = journalMaxBytes // force the next append to rotate + j.size = journalMaxBytes if err := j.append(usageEvent{Event: "claim", ID: "sb_1"}); err != nil { t.Fatalf("append: %v", err) @@ -85,7 +85,7 @@ func TestQuotaRefusesClaimsPastCap(t *testing.T) { if _, err := claimAny(t.Context(), m, testKey, time.Hour); !errors.Is(err, ErrQuota) { t.Fatalf("claim past cap: %v, want ErrQuota", err) } - // The refused VM must not leak, and the pool state stays sane. + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { t.Fatalf("Release: %v", err) } @@ -109,8 +109,7 @@ func TestTenantQuotaBindsPerTenant(t *testing.T) { if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil); !errors.Is(err, ErrQuota) { t.Fatalf("acme past its cap: %v, want ErrQuota", err) } - // The node-wide cap (unset here) stays untouched: other tenants and root - // keep claiming while acme is full. + if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "beta", "", nil); err != nil { t.Errorf("beta claim while acme is at cap: %v", err) } diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 90c36512..4fa6a334 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -15,19 +15,14 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// templateRecord is a template's meta.json: the id carries the key hash -// (tp_), which is all resolution needs — claims re-derive the hash -// from the requested key, never the reverse. Tenant attributes the promote -// and scopes deletion; empty means the operator (root). +// templateRecord is a template's meta.json; an empty Tenant means the operator (root). type templateRecord struct { ID string `json:"id"` Tenant string `json:"tenant,omitempty"` CreatedAt time.Time `json:"created_at"` } -// Promote publishes a claimed sandbox as a template under (template, parent -// net, parent size); later claims for that key clone from it. Re-promoting the -// same name replaces it, and the caller owns its lifecycle (DeleteTemplate). +// Promote publishes a claimed sandbox as a template under (template, parent net, parent size). func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, tenant string) (types.PoolKey, string, error) { sb, ok := m.resolve(id, cred) if !ok { @@ -44,17 +39,14 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t } key := types.PoolKey{Template: template, Net: sb.Key.Net, Size: sb.Key.Size} if m.pooledHash(key.Hash()) { - // A configured pool owns this key — promoting over it would - // silently change what refills produce. + // a configured pool owns this key; promoting over it would change what refills produce return types.PoolKey{}, "", ErrPooledTemplate } - // Fast-fail before the export; commitTemplate re-checks under the - // template lock, closing the check-then-publish race. + // commitTemplate re-checks under the template lock; this only fast-fails before the export if err := m.checkTemplateOwner(ctx, store.TemplateID(key.Hash()), tenant); err != nil { return types.PoolKey{}, "", err } - // See Fork: the transition lock pins the source snapshot, and a started - // promote must finish even if the caller hangs up. + // the transition lock pins the source snapshot; a started promote must finish uncanceled sb.Transition.Lock() defer sb.Transition.Unlock() ctx = context.WithoutCancel(ctx) @@ -76,10 +68,7 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t return key, digest, nil } -// DeleteTemplate removes a promoted template. Configured pools are refused: -// their goldens are owned by the node config, not an API caller. A tenant -// may delete only templates it promoted — anything else answers -// ErrUnknownTemplate; root (empty tenant) deletes anything. +// DeleteTemplate removes a promoted template; a tenant may delete only what it promoted. func (m *Manager) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant string) error { if err := m.validate(key); err != nil { return err @@ -116,15 +105,12 @@ func (m *Manager) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant return nil } -// TemplateHashes lists the promoted-template key hashes for the mesh's -// template gossip, from the in-memory set — the 1s gossip tick never -// touches the store backend. Sorted: the mesh's unchanged-guard compares -// order-sensitively, and map iteration order would defeat it every tick. +// TemplateHashes lists the promoted-template key hashes for mesh gossip, sorted for its guard. func (m *Manager) TemplateHashes() []string { m.mu.Lock() pooled := make(map[string]struct{}, len(m.pools)) - for key := range m.pools { - pooled[key.Hash()] = struct{}{} + for _, p := range m.pools { + pooled[p.hash] = struct{}{} } m.mu.Unlock() m.tplMu.Lock() @@ -140,16 +126,12 @@ func (m *Manager) TemplateHashes() []string { return hashes } -// HasGolden reports whether this node can provision the key without a cold -// boot — a configured pool golden or a promoted template in the store. The -// tplSet answers without a store read; only a shared-store template promoted -// elsewhere after startup falls through to the backend. +// HasGolden reports whether this node can provision the key without a cold boot. func (m *Manager) HasGolden(ctx context.Context, key types.PoolKey, tenant string) bool { return m.HasPoolGolden(key) || m.HasPromotedTemplate(ctx, key, tenant) } -// HasPoolGolden reports whether a configured pool can serve key from its own -// golden. That golden is what every claim on this node resolves to. +// HasPoolGolden reports whether a configured pool can serve key from its own golden. func (m *Manager) HasPoolGolden(key types.PoolKey) bool { m.mu.Lock() defer m.mu.Unlock() @@ -157,9 +139,7 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { return p != nil && p.goldenDir != "" } -// HasPromotedTemplate reports whether key resolves to a promoted template this -// tenant may claim — resolveGolden's test exactly, so routing never promises -// a golden the claim would then refuse. +// HasPromotedTemplate is resolveGolden's test exactly, so routing never promises a refused golden. func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooledHash(key.Hash()) { return false @@ -183,38 +163,32 @@ func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, te return owner == "" || tenantOwns(tenant, owner) } -// pooledHash reports whether a configured pool occupies this hash — the -// guard is on the HASH, not the key: goldens are stored by hash, so a -// colliding key would reach a pool's golden dir even though the keys differ. +// pooledHash guards on the hash, not the key: a colliding key would reach a pool's golden dir. func (m *Manager) pooledHash(hash string) bool { m.mu.Lock() defer m.mu.Unlock() - for key := range m.pools { - if key.Hash() == hash { + for _, p := range m.pools { + if p.hash == hash { return true } } return false } -// recLock is the per-record mutation lock: template publish/delete serialize -// on it directly; checkpoints take it only for delete/fetch (their ids are -// fresh and unguessable pre-publish). A clone or wake holds it shared, so a -// delete or re-publish swap never runs under an in-flight read. Every call -// counts as a live reference — pair it with recDone (or recDoneEvict) once -// the returned lock is unlocked, so a concurrent evict can never hand a -// waiter a different mutex for the same id. +// recLock takes the per-record lock and a live reference (pair with recDone); clones and wakes hold it shared so a delete or re-publish never runs under an in-flight read. func (m *Manager) recLock(id string) *sync.RWMutex { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() m.recRefs[id]++ - l, _ := m.recLocks.LoadOrStore(id, &sync.RWMutex{}) - return l.(*sync.RWMutex) + l := m.recLocks[id] + if l == nil { + l = &sync.RWMutex{} + m.recLocks[id] = l + } + return l } -// recDone releases a reference taken by recLock; call after Unlock/RUnlock, -// never before — releasing early is what lets a concurrent recLock observe -// a fresh mutex while this call's lock is still logically held. +// recDone releases a reference taken by recLock; call after Unlock/RUnlock, never before. func (m *Manager) recDone(id string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -225,11 +199,7 @@ func (m *Manager) recDone(id string) { } } -// recDoneEvict is recDone for a caller that just deleted id's record: once -// the reference count drops to zero — no one else holds or awaits this id's -// lock — its recLocks slot is removed so the map can't grow per checkpoint. -// Template ids (tp_, reused on re-promote) never call this; they stay -// cached forever, bounded by the promoted set. +// recDoneEvict is recDone for a just-deleted record: the lock slot goes at zero references. func (m *Manager) recDoneEvict(id string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -237,25 +207,22 @@ func (m *Manager) recDoneEvict(id string) { if m.recRefs[id] <= 0 { delete(m.recRefs, id) delete(m.recEvict, id) - m.recLocks.Delete(id) + delete(m.recLocks, id) return } - // Another holder or waiter is still on this lock; defer the eviction to - // whichever call drops the last reference, so it is not lost here. + // a holder or waiter remains; the call that drops the last reference evicts instead m.recEvict[id] = struct{}{} } -// evictIfPending drops id's lock entry if a delete asked for eviction while a -// reference still held it; callers hold recLocksMu with the count at zero. +// evictIfPending drops id's lock entry if a delete asked for eviction; callers hold recLocksMu. func (m *Manager) evictIfPending(id string) { if _, ok := m.recEvict[id]; ok { delete(m.recEvict, id) - m.recLocks.Delete(id) + delete(m.recLocks, id) } } -// checkTemplateOwner rejects publishing or deleting over another tenant's -// record; a meta failure other than absence refuses rather than fail open. +// checkTemplateOwner rejects publishing or deleting over another tenant's record. func (m *Manager) checkTemplateOwner(ctx context.Context, id, tenant string) error { if tenant == "" { return nil @@ -281,10 +248,7 @@ type goldenResolution struct { release func() } -// resolveGolden resolves a key's clone source: the configured pool's local -// golden (no release), else a promoted template fetched from the store; -// empty dir cold-boots. A cross-tenant or absent record cold-boots; a backend -// failure propagates rather than booting a template name as an image ref. +// resolveGolden resolves a key's clone source: the pool golden, else a promoted template. func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant string) (goldenResolution, error) { m.mu.Lock() var dir string @@ -328,7 +292,6 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant s }, nil } -// publishTemplate exports snap into the store under the key's template id. func (m *Manager) publishTemplate(ctx context.Context, snap string, key types.PoolKey, tenant string) (string, error) { id := store.TemplateID(key.Hash()) staging, err := m.tpls.Stage(id) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 3401326d..f92893b1 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -18,8 +18,7 @@ import ( ) const ( - // Sidecar, not data_dir: the marker travels with the image and survives a - // data_dir wipe. + // sidecar, not data_dir: the marker travels with the image and survives a data_dir wipe volumeDirtySuffix = ".dirty" // Caps the per-mount budget below, so teardown can never hang for long. volumeQuiesceMax = 10 * time.Second @@ -35,8 +34,7 @@ func (v catalogVolume) allowed(tenant string) bool { return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) } -// volumeHolders is one name's live admission state: a writer excludes every -// other claim, readers only exclude a writer. +// volumeHolders is one name's live admission state: a writer excludes every other claim. type volumeHolders struct { writers int readers int @@ -47,17 +45,13 @@ type resolvedVolume struct { applied types.Volume } -// volumeTeardown is what a quiesced claim leaves for after its VM is confirmed -// gone: the admission holds to drop, and the marker paths of the mounts that -// came down cleanly — captured rather than re-resolved later, because a claim -// adopted across a restart can name a catalog entry the new config re-pointed. +// volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { holds []types.Volume clears []string } -// Volumes reports the caller-visible fleet catalog, projected through this -// node's ACL metadata and local path state. Empty tenant means root. +// Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { infos := make([]types.VolumeInfo, 0, len(m.volumes)) for name, volume := range m.volumes { @@ -77,8 +71,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI } infos = append(infos, info) } - // Root can safely see peer-only names. Tenant callers need the local catalog - // metadata so the access list can be applied without gossiping it. + // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { if _, ok := m.volumes[name]; !ok { @@ -106,8 +99,7 @@ func (m *Manager) VolumeNames() []string { return names } -// VolumePlacement checks catalog access and reports whether every named image -// is currently available on this node. Empty tenant means root. +// VolumePlacement checks catalog access and reports whether every named image is local. func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) { if err := m.validate(key); err != nil { return false, err @@ -139,8 +131,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - // An attach-only claim arrives with no mounts to default: re-defaulting - // them here would mount what the caller asked to mount itself. + // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) @@ -155,8 +146,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) } if _, statErr := os.Stat(entry.disk.Path); statErr != nil { - // The operator's path stays server-side, and the refusal stays - // byte-identical to the unknown and forbidden ones. + // the operator's path stays server-side; the refusal matches the unknown one log.WithFunc("pool.resolveVolumes").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return nil, ErrVolumeUnavailable } @@ -168,8 +158,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return resolved, nil } -// confirmVolumesClean refuses a read-only claim of a dirty image: run after -// holds are taken, so the marker is stable and can only mean a crashed writer. +// confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. func (m *Manager) confirmVolumesClean(volumes []resolvedVolume) error { for _, volume := range volumes { if !volume.applied.RW() && volumeDirty(volume.disk.Path) { @@ -179,9 +168,7 @@ func (m *Manager) confirmVolumesClean(volumes []resolvedVolume) error { return nil } -// applyVolumes brings the resolved set up concurrently: cocoon serializes the -// attach per VM, so what overlaps is the CLI spawns, settle waits and guest -// mounts. applied is the request-shaped set, recorded once every mount is up. +// applyVolumes brings the resolved set up concurrently; cocoon serializes the attach per VM. func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes []resolvedVolume, applied []types.Volume) error { switch len(volumes) { case 0: @@ -225,9 +212,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res return nil } -// quiesceVolumes unmounts a claim's writable mounts in reverse order and -// returns the teardown its VM removal must finish; a failed unmount keeps the -// image's marker for a recovering writer, never blocking teardown. +// quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { td := volumeTeardown{holds: sb.Volumes} mounts := writableMounts(sb.Volumes) @@ -260,9 +245,7 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT return td } -// finishVolumeTeardown completes teardown once the VM is confirmed gone: only -// then is the hypervisor's image lock released, so an earlier marker clear or -// hold release would hand the name to a claim the still-live writer blocks. +// finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { if len(td.holds) == 0 { return @@ -276,8 +259,7 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { m.unreserveVolumes(td.holds) } -// reserveVolumes admits one claim's volumes; every name is checked before any -// is taken, so a refusal leaves the registry untouched. Callers hold m.mu. +// reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { for _, volume := range volumes { holders := m.volumeAdmission[volume.Name] @@ -289,8 +271,7 @@ func (m *Manager) reserveVolumes(volumes []types.Volume) error { return nil } -// adoptVolumes counts volumes an adopted claim already holds, with no -// admission check. Callers hold m.mu. +// adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { holders := m.volumeAdmission[volume.Name] @@ -329,8 +310,7 @@ func (m *Manager) unreserveVolumes(volumes []types.Volume) { m.releaseVolumes(volumes) } -// quiesceBudget bounds one quiesce: a slice per writable mount plus one for the -// sync fallback, so a wedged multi-volume guest still gets every umount tried. +// quiesceBudget bounds one quiesce: a slice per writable mount plus one for the sync fallback. func quiesceBudget(mounts int) time.Duration { return min(time.Duration(mounts+1)*engine.VolumeCallTimeout, volumeQuiesceMax) } diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index fbf6af24..acf48feb 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -11,9 +11,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// TestAttachOnlyClaimAttachesWithoutMounting: the whole mount contract belongs -// to the caller, so a writable attach-only claim leaves no marker to clear and -// no mount to quiesce — the device only goes away with the VM. func TestAttachOnlyClaimAttachesWithoutMounting(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") eng := newFakeEngine() @@ -76,7 +73,6 @@ func TestAttachOnlyClaimAttachesEveryVolumeConcurrently(t *testing.T) { {Name: "dataset", AttachOnly: true}, {Name: "cache", Mode: types.VolumeModeRW, AttachOnly: true}, } - // Every attach must be in flight at once: a sequential apply blocks here. var attaches sync.WaitGroup attaches.Add(len(requested)) eng.attachRendezvous = &attaches @@ -215,15 +211,11 @@ func TestClaimWarmAttachFailureKeepsHoldsUntilRemoval(t *testing.T) { } } -// TestFinalizeQuotaFailureKeepsHoldsUntilRemoval: the finalize re-check loses -// the quota race with volumes already mounted, so the loser owes a clean umount. func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch") eng := newFakeEngine() m := newVolumeManager(t, eng, []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) m.maxClaims = 1 - // The volume claim parks in its attach, so the volume-less claim below always - // takes the last slot and the volume claim always loses the finalize re-check. var attaches sync.WaitGroup attaches.Add(2) eng.attachRendezvous = &attaches @@ -247,7 +239,7 @@ func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { t.Fatalf("volume-less claim: %v", err) } eng.mu.Lock() - eng.removeErrFor = vmName // the quota loser's VM survives its removal + eng.removeErrFor = vmName eng.attachRendezvous = nil eng.mu.Unlock() attaches.Done() @@ -294,8 +286,6 @@ func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { } } -// TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant: the tenant limb of -// the same re-check; its loser must leave the tenant with nothing counted. func TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch") eng := newFakeEngine() @@ -350,8 +340,6 @@ func TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant(t *testing.T) { } } -// TestAttachOnlyClaimKeepsAdmissionExclusion: what protects other claims is -// unchanged — only this claim's own mount contract moved to the caller. func TestAttachOnlyClaimKeepsAdmissionExclusion(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") eng := newFakeEngine() diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 2e20d820..3ea6e5ee 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -145,8 +145,6 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), []config.VolumeSpec{{Name: "scratch", Path: path, Writable: true}}) readOnly := []types.Volume{{Name: "scratch"}} - // The reader resolves a clean image; a writable claim then fails between - // that resolve and admission, leaving its marker but no hold behind. resolved, err := m.resolveVolumes(t.Context(), "", readOnly) if err != nil { t.Fatalf("resolveVolumes: %v", err) @@ -182,8 +180,7 @@ func TestVolumeAdmissionMatrix(t *testing.T) { wantErr error }{ {"writer excludes writer", types.VolumeModeRW, types.VolumeModeRW, ErrVolumeBusy}, - // Admission answers first: a live writer is a busy conflict, never the - // recovery verdict its own write-ahead marker would otherwise suggest. + {"writer excludes reader", types.VolumeModeRW, "", ErrVolumeBusy}, {"reader excludes writer", "", types.VolumeModeRW, ErrVolumeBusy}, {"readers share", "", "", nil}, @@ -392,7 +389,7 @@ func TestPendingRemovalHoldsVolumesUntilDrained(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - eng.removeErrFor = sb.VMName // survives removal: the VM still holds the image + eng.removeErrFor = sb.VMName if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Release reported success for a VM that survived removal") @@ -448,8 +445,7 @@ func TestVolumeDirtyFailsClosed(t *testing.T) { if !volumeDirty(image) { t.Error("marked image reads clean") } - // The marker of an image behind a non-directory parent cannot be read at - // all; an unreadable marker must never open the image to readers. + if !volumeDirty(filepath.Join(image, "nested.img")) { t.Error("unreadable marker reads clean") } diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 617cc1f0..9a9e7740 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -97,7 +97,6 @@ func TestClaimProvisionBringsMixedVolumesUpConcurrently(t *testing.T) { {Name: "dataset", Mount: "/volumes/dataset"}, {Name: "cache", Mount: "/cache", Mode: types.VolumeModeRW}, } - // Every attach must be in flight at once: a sequential apply blocks here. var attaches sync.WaitGroup attaches.Add(len(requested)) eng.attachRendezvous = &attaches @@ -324,8 +323,6 @@ func TestClaimProvisionPromotedAppliesVolumesFromTemplate(t *testing.T) { } } -// TestPooledKeyOutranksPromotedTemplate: promote, pool, then restart must not -// arm RequirePromoted against every volume claim for the key. func TestPooledKeyOutranksPromotedTemplate(t *testing.T) { path := writeVolumeImage(t, "data.img", "data") catalog := []config.VolumeSpec{{Name: "data", Path: path}} @@ -520,8 +517,6 @@ func TestVolumeClaimUsageAndScopedSummaries(t *testing.T) { t.Fatal("acme claim usage event not found") } -// TestClaimProvisionRefusesUnavailableVolumesIndistinguishably: unknown, -// forbidden, and vanished refuse alike; any difference enumerates the catalog/ACLs. func TestClaimProvisionRefusesUnavailableVolumesIndistinguishably(t *testing.T) { present := writeVolumeImage(t, "present.img", "present") vanished := filepath.Join(t.TempDir(), "vanished.img") @@ -535,7 +530,7 @@ func TestClaimProvisionRefusesUnavailableVolumesIndistinguishably(t *testing.T) for _, principal := range []string{"", "beta"} { for _, name := range []string{"vanished", "unknown", "private"} { if principal == "" && name == "private" { - continue // root is on every access list, so it has no forbidden case + continue } _, err := m.ClaimProvision(t.Context(), testKey, 0, principal, "", []types.Volume{{Name: name}}) if !errors.Is(err, ErrVolumeUnavailable) { @@ -557,8 +552,6 @@ func TestClaimProvisionRefusesUnavailableVolumesIndistinguishably(t *testing.T) } } -// assertVolumeBringUp pins each volume's own marker→attach→mount order and the -// completeness of the set; cross-volume order is free. func assertVolumeBringUp(t *testing.T, eng *fakeEngine, applied []types.Volume) { t.Helper() ops := eng.volumeOpsLog() diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index ab5cb22f..c953d028 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -6,21 +6,15 @@ import ( ) const ( - // ewmaAlpha weights the newest observation in the arrival-rate and - // provision-lead averages; 0.3 tracks bursts within a few samples - // without whiplashing on a single outlier. + // ewmaAlpha weights the newest observation in the arrival-rate and provision-lead averages ewmaAlpha = 0.3 - // rateDecayTau halves the observed arrival rate roughly every 40s of - // silence, so a quiet pool's target drifts back to its floor instead - // of pinning at the last burst's level. + // rateDecayTau halves the observed arrival rate roughly every 40s of silence rateDecayTau = 60 * time.Second - // leadSafety over-provisions against the measured provision lead: - // arrivals are bursty, not uniform, and a warm miss costs a clone. + // leadSafety over-provisions against the measured lead: arrivals are bursty, not uniform leadSafety = 2.0 ) -// noteArrival folds one claim for this pool into the arrival-rate EWMA. -// Caller holds the manager mutex. +// noteArrival folds one claim into the arrival-rate EWMA. Caller holds the manager mutex. func (p *pool) noteArrival(now time.Time) { if !p.lastArrival.IsZero() { if dt := now.Sub(p.lastArrival).Seconds(); dt > 0 { @@ -30,8 +24,7 @@ func (p *pool) noteArrival(now time.Time) { p.lastArrival = now } -// noteLead folds one measured provision duration (clone → claim-ready) into -// the lead EWMA. Caller holds the manager mutex. +// noteLead folds one provision duration into the lead EWMA. Caller holds the manager mutex. func (p *pool) noteLead(d time.Duration) { if p.lead == 0 { p.lead = d @@ -40,10 +33,7 @@ func (p *pool) noteLead(d time.Duration) { p.lead = time.Duration(ewmaAlpha*float64(d) + (1-ewmaAlpha)*float64(p.lead)) } -// effectiveTarget is the demand-adaptive warm watermark: the configured -// floor, raised toward warmMax when the decayed arrival rate times the -// measured provision lead says the floor would miss. warmMax == 0 means the -// feature is off and the floor is the target. Caller holds the manager mutex. +// effectiveTarget raises the floor toward warmMax by rate × lead. Caller holds the manager mutex. func (p *pool) effectiveTarget(now time.Time) int { if p.warmMax <= 0 { return p.floor diff --git a/sandboxd/pool/watermark_test.go b/sandboxd/pool/watermark_test.go index 6d64c016..2ba4201e 100644 --- a/sandboxd/pool/watermark_test.go +++ b/sandboxd/pool/watermark_test.go @@ -15,8 +15,6 @@ func TestEffectiveTargetTracksDemand(t *testing.T) { t.Fatalf("quiet pool target %d, want the floor", got) } - // A burst of one claim every 100ms → ~10/s; ×0.5s lead ×2 safety = 10, - // capped at warmMax. for i := range 20 { p.noteArrival(now.Add(time.Duration(i) * 100 * time.Millisecond)) } @@ -25,7 +23,6 @@ func TestEffectiveTargetTracksDemand(t *testing.T) { t.Errorf("burst target %d, want warmMax 8", got) } - // Silence decays the rate; minutes later the target is the floor again. if got := p.effectiveTarget(burstEnd.Add(5 * time.Minute)); got != 2 { t.Errorf("post-silence target %d, want the floor", got) } diff --git a/sandboxd/server/metrics.go b/sandboxd/server/metrics.go index 84597d92..22a20535 100644 --- a/sandboxd/server/metrics.go +++ b/sandboxd/server/metrics.go @@ -14,9 +14,7 @@ type SandboxListResponse struct { Sandboxes []pool.SandboxSummary `json:"sandboxes"` } -// handleMetrics renders Prometheus text format by hand — counters and -// gauges only, no client library. Latency rides as *_seconds_total next to -// its count. +// handleMetrics renders Prometheus text format by hand, counters and gauges only. func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { pools, g := s.mgr.Info() c := s.mgr.Counters() diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 637d2330..40776397 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -48,6 +48,7 @@ func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewSe } p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} p.transport = &http.Transport{ + // a pooled guest conn skips PreviewDial, and with it wakeResolved and the Transition lock DisableKeepAlives: true, DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { id, portStr, err := net.SplitHostPort(addr) @@ -102,10 +103,10 @@ func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claim rp := &httputil.ReverseProxy{ Rewrite: func(pr *httputil.ProxyRequest) { pr.Out.URL.Scheme = "http" - // The synthetic host carries PreviewDial's target. + // the synthetic host carries PreviewDial's target pr.Out.URL.Host = fmt.Sprintf("%s:%d", claims.ID, claims.Port) pr.Out.URL.Path = "/" + strings.TrimPrefix(pr.In.URL.Path, "/p/"+r.PathValue("token")+"/") - // Browser credentials for the preview domain must not reach guest code. + // browser credentials for the preview domain must not reach guest code pr.Out.Header.Del("Cookie") pr.Out.Header.Del("Authorization") }, diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 1502f15d..1b96a7fb 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -16,16 +16,12 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// drainGrace bounds how long a finished relay waits for the client to -// consume the tail and close; past it the client conn is force-closed. +// drainGrace bounds how long a finished relay waits for the client to close. const drainGrace = 30 * time.Second var switchingProtocols = []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: silkd\r\nConnection: Upgrade\r\n\r\n") -// CloseRelays force-closes every in-flight relay, refuses new ones, and -// waits for the drain; http.Server.Shutdown does not track hijacked -// connections, and a relay hijacked inside the shutdown window would -// otherwise register after this snapshot and hang the drain forever. +// CloseRelays force-closes in-flight relays; http.Server.Shutdown does not track hijacked conns. func (s *Server) CloseRelays() { s.relayMu.Lock() s.relayClosed = true @@ -42,15 +38,13 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { if !ok { return } - // The cheap header check runs first: waking is expensive and consumes - // the hibernate snapshot, so a non-upgrade GET must not trigger it. + // waking consumes the hibernate snapshot, so a non-upgrade GET must not trigger it if !strings.EqualFold(r.Header.Get("Upgrade"), upgradeProto) { w.Header().Set("Upgrade", upgradeProto) writeErr(w, http.StatusUpgradeRequired, "upgrade to "+upgradeProto+" required") return } - // WakeAgentSocket restores a hibernated VM first, so the relay is the - // transparent wake path: the client's next call just works. + // WakeAgentSocket restores a hibernated VM first, making the relay the transparent wake path sock, err := s.mgr.WakeAgentSocket(r.Context(), r.PathValue("id"), token) switch { case writePoolErr(w, err): @@ -76,8 +70,7 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { s.relay(r.Context(), r.PathValue("id"), client, bufrw.Reader, guest) } -// relay writes the 101 and splices the two connections until the guest side -// finishes (silkd closes after the terminal frame) or the client vanishes. +// relay writes the 101 and splices the conns until silkd closes or the client vanishes. func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBuf *bufio.Reader, guest net.Conn) { s.relayMu.Lock() if s.relayClosed { @@ -98,22 +91,19 @@ func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBu s.relayWG.Done() }() - // The http server may have armed a header-read deadline on this conn. + // the http server may have armed a header-read deadline on this conn _ = client.SetDeadline(time.Time{}) if _, err := client.Write(switchingProtocols); err != nil { return } - // Bytes the client pipelined behind the request head already sit in the - // server's bufio reader; drain exactly those, never more — reading the - // bufio past Buffered() would race the direct conn reads below. + // reading the bufio past Buffered() would race the direct conn reads below clientR := io.Reader(client) if n := clientBuf.Buffered(); n > 0 { clientR = io.MultiReader(io.LimitReader(clientBuf, int64(n)), client) } if s.mgr.AuditEnabled() { - // One connection is one RPC: the first line client→guest is the - // request frame. The tee records it as it flows, then passes through. + // one connection is one RPC, so the first line client→guest is the request frame clientR = &auditTee{r: clientR, record: func(line []byte) { s.mgr.Audit(ctx, id, line) }} @@ -123,23 +113,18 @@ func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBu go func() { defer close(done) _, _ = io.Copy(guest, clientR) - // Client done or vanished — either way hard-close: silkd treats - // disconnect as abort, and the protocol delimits input with explicit - // terminal frames (stdin_close, data_end), so a TCP half-close - // carries no meaning worth relaying through the vsock muxer. + // silkd delimits input with terminal frames, so a TCP half-close carries no meaning _ = guest.Close() }() _, _ = io.Copy(client, guest) utils.CloseWrite(client) - // Wait for the client to consume the tail and close its side; the - // deadline unblocks the splice goroutine if it never does. + // the read deadline unblocks the splice goroutine if the client never closes _ = client.SetReadDeadline(time.Now().Add(drainGrace)) <-done } -// auditTee captures the first newline-terminated line (capped) flowing -// through it, hands it to record once, then degrades to a pass-through. +// auditTee captures the first line flowing through it, then degrades to a pass-through. type auditTee struct { r io.Reader record func([]byte) @@ -147,9 +132,6 @@ type auditTee struct { done bool } -// WriteTo hands the splice back to io.Copy's fast path once the first line -// is captured: it reads through Read until the record fires, then delegates -// the rest of the stream to a direct copy. func (t *auditTee) WriteTo(w io.Writer) (int64, error) { var total int64 buf := make([]byte, 4096) diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 3f2833c6..08801ef3 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -79,9 +79,6 @@ func TestRelayRoundTrip(t *testing.T) { } func TestRelayDeliversPipelinedBytes(t *testing.T) { - // The client sends the request head and the first frame in one write, - // before reading the 101 — the frame lands in the server's bufio reader - // and must still reach the guest. ts, _ := newRelayServer(t, func(c net.Conn) { defer c.Close() r := bufio.NewReader(c) @@ -89,7 +86,7 @@ func TestRelayDeliversPipelinedBytes(t *testing.T) { if err != nil { return } - _, _ = io.WriteString(c, line) // echo proves receipt + _, _ = io.WriteString(c, line) }) conn, err := net.Dial("tcp", ts.Listener.Addr().String()) @@ -116,7 +113,7 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { guestClosed := make(chan struct{}) ts, _ := newRelayServer(t, func(c net.Conn) { defer close(guestClosed) - _, _ = io.Copy(io.Discard, c) // block until the relay closes our side + _, _ = io.Copy(io.Discard, c) }) conn, _ := upgradeConn(t, ts) @@ -130,14 +127,14 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { func TestCloseRelaysDrains(t *testing.T) { ts, srv := newRelayServer(t, func(c net.Conn) { - _, _ = io.Copy(io.Discard, c) // idle guest: keeps the relay open + _, _ = io.Copy(io.Discard, c) }) _, r := upgradeConn(t, ts) closed := make(chan struct{}) go func() { defer close(closed) - _, _ = io.Copy(io.Discard, r) // returns when the relay is torn down + _, _ = io.Copy(io.Discard, r) }() srv.CloseRelays() select { @@ -147,8 +144,6 @@ func TestCloseRelaysDrains(t *testing.T) { } } -// newRelayServer wires a Server whose dialer hands each relay one end of a -// net.Pipe and runs guest on the other end. func newRelayServer(t *testing.T, guest func(net.Conn)) (*httptest.Server, *Server) { t.Helper() dialer := &fakeDialer{dial: func(context.Context, string) (net.Conn, error) { @@ -165,8 +160,6 @@ func newRelayServer(t *testing.T, guest func(net.Conn)) (*httptest.Server, *Serv return ts, srv } -// upgradeConn performs the raw HTTP upgrade handshake; further reads must go -// through the returned reader — it may have buffered bytes past the 101. func upgradeConn(t *testing.T, ts *httptest.Server) (net.Conn, *bufio.Reader) { t.Helper() conn, err := net.Dial("tcp", ts.Listener.Addr().String()) @@ -183,8 +176,6 @@ func upgradeConn(t *testing.T, ts *httptest.Server) (net.Conn, *bufio.Reader) { return conn, r } -// readStatus101 consumes exactly the 101 response head; a 1xx response has -// no body, so http.ReadResponse stops right where the guest bytes begin. func readStatus101(t *testing.T, r *bufio.Reader) { t.Helper() resp, err := http.ReadResponse(r, nil) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 56f2a323..83f5f42a 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -33,9 +33,7 @@ const ( previewTTL = time.Hour ) -// poolErrHTTP maps pool sentinels to their HTTP replies; an empty msg -// surfaces err.Error() (4xx detail the caller can act on), a fixed msg -// avoids echoing internals on lookups. +// an empty msg surfaces err.Error(); a fixed msg avoids echoing internals var poolErrHTTP = []struct { err error code int @@ -60,9 +58,7 @@ var poolErrHTTP = []struct { {pool.ErrUnknownCheckpoint, http.StatusNotFound, "unknown checkpoint"}, } -// Manager is the slice of the pool manager the server consumes. tenant -// parameters attribute created resources and scope listings/deletes; empty -// means the operator (root) — unquotaed, unfiltered. +// Manager is the pool manager slice the server consumes; an empty tenant means operator (root). type Manager interface { ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) @@ -106,8 +102,7 @@ type Dialer interface { DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) } -// Placer names peers for redirect placement and lists the mesh for Lookup; -// nil on a single-node deployment (no mesh). +// Placer names peers for redirect placement and lists the mesh; nil on a single-node deployment. type Placer interface { Candidates(keyHash string) []string VolumeCandidates(keyHash string, names []string) []string @@ -119,15 +114,13 @@ type Placer interface { ConfigMismatches() int } -// CheckpointProber answers which peers hold a checkpoint, asked live on a -// redirect decision rather than gossiped. +// CheckpointProber answers which peers hold a checkpoint, asked live rather than gossiped. type CheckpointProber interface { Owners(ctx context.Context, id string) []string Forget(id string) } -// InfoResponse is the wire reply of GET /v1/info. Peers lists the other nodes' -// data-plane addresses so a client can scatter a Lookup across the cluster. +// InfoResponse is the wire reply of GET /v1/info. type InfoResponse struct { Pools []pool.PoolInfo `json:"pools"` Claimed int `json:"claimed"` @@ -135,16 +128,12 @@ type InfoResponse struct { Archived int `json:"archived"` Draining bool `json:"draining,omitempty"` Peers []string `json:"peers,omitempty"` - // AtCapacity marks refill parked because the node refused another VM, so a - // placer reads a short warm count as "full", not "still filling". + // AtCapacity marks refill parked because the node refused another VM. AtCapacity bool `json:"at_capacity,omitempty"` AtCapacityReason string `json:"at_capacity_reason,omitempty"` } -// PoolUpdateRequest is the wire body of PUT /v1/pools. It replaces the node's -// desired warm targets with the supplied list; omitted pools are drained. -// Lives here, not types/api.go: it embeds config.PoolSpec and types cannot -// import config. +// PoolUpdateRequest is the wire body of PUT /v1/pools; omitted pools are drained. type PoolUpdateRequest struct { Pools []config.PoolSpec `json:"pools"` } @@ -162,20 +151,12 @@ type Server struct { preview *PreviewServer relayMu sync.Mutex - relays map[net.Conn]net.Conn // client conn → guest conn, for forced shutdown + relays map[net.Conn]net.Conn // client conn → guest conn relayClosed bool relayWG sync.WaitGroup } -// New returns a Server; an empty apiToken with no tenants leaves the -// node-level endpoints open (per-sandbox tokens still guard sandbox-scoped -// calls). tenants adds per-tenant tokens next to the root apiToken. -// advertise is this node's data-plane address, returned as a claim's owner -// address. A nil placer disables mesh redirects and a nil prober disables -// checkpoint-owner probing (both single-node). probeKey (see -// peer.DeriveProbeKey) authenticates the checkpoint HEAD probe; empty -// leaves it unauthenticated (an unencrypted mesh has no shared secret to -// build one from). +// New returns a Server; an empty apiToken with no tenants leaves node-level endpoints open. func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Manager, dialer Dialer, placer Placer, prober CheckpointProber, probeKey []byte, preview *PreviewServer) *Server { return &Server{ mgr: mgr, @@ -191,9 +172,7 @@ func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Man } } -// Handler builds the route table. Resource-creating verbs and tenant-scoped -// listings/deletes take the api token or a tenant token; pools, info, metrics, -// and per-sandbox operator reads stay root-only. +// Handler builds the route table. func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) @@ -203,17 +182,14 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) - // Fork and promote create node resources, so they take the same token - // class as a claim; the source sandbox's token rides in the body as the - // ownership proof. + // the source sandbox token rides in the body as the ownership proof mux.HandleFunc("POST /v1/sandboxes/{id}/fork", s.requireToken(s.handleFork)) mux.HandleFunc("POST /v1/sandboxes/{id}/promote", s.requireToken(s.handlePromote)) mux.HandleFunc("POST /v1/sandboxes/{id}/preview", s.requireToken(s.handlePreview)) mux.HandleFunc("POST /v1/sandboxes/{id}/checkpoint", s.requireToken(s.handleCheckpoint)) mux.HandleFunc("POST /v1/checkpoints/{id}/claim", s.requireToken(s.handleClaimCheckpoint)) mux.HandleFunc("GET /v1/checkpoints", s.requireToken(s.handleListCheckpoints)) - // GET streams a whole checkpoint with no tenant scoping, so it is - // operator-only; HEAD is the ownership probe and stays unauthenticated. + // GET streams a whole checkpoint with no tenant scoping; HEAD is the unauthenticated probe mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireRoot(s.handleCheckpointBlob)) mux.HandleFunc("HEAD /v1/checkpoints/{id}/blob", s.handleCheckpointProbe) mux.HandleFunc("DELETE /v1/checkpoints/{id}", s.requireToken(s.handleDeleteCheckpoint)) @@ -247,10 +223,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { return } - // Warm hit here is ownership transfer only. On a warm miss with a mesh, a - // peer that reports a warm sandbox gets the claim via redirect (data plane - // must be direct, so redirect beats proxy); only if no peer has one does - // this node provision (golden clone or cold boot). + // the data plane must be direct, so a warm peer gets the claim by redirect, not proxy sb, err := s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) if errors.Is(err, pool.ErrNoWarm) { if s.redirectClaim(r.Context(), w, req, key, hash, tenant) { @@ -258,8 +231,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { } sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) } - // A full node bounces the claim to a warm peer before answering 429 — - // quota is per node, and a peer with capacity is a better answer. + // quota is per node, so a full node bounces the claim to a peer before answering 429 if errors.Is(err, pool.ErrQuota) && s.placer != nil && !req.NoRedirect && writeRedirect(w, s.placer.Candidates(hash)) { return @@ -319,9 +291,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, } localTemplate := s.mgr.HasPromotedTemplate(ctx, key, tenant) - // Per-node content consistency: a peer's promoted template must not escalate - // volume claims off the pool golden every other claim here resolves to. An - // explicit RequirePromoted still passes through, for the manager to refuse. + // a peer's promoted template must not escalate volume claims off the pool golden pooled := s.mgr.HasPoolGolden(key) var templateOwners []string if s.placer != nil && !pooled { @@ -338,9 +308,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, var owners []string if promoted { - // A shared template store lets a volume holder resolve the template - // even before that node has advertised the newly published hash. The - // no_redirect target re-checks both resources before provisioning. + // a shared template store lets a volume holder resolve a hash it has not advertised yet owners = s.templateOwners(func(probe string) []string { return s.placer.TemplateVolumeOwners(probe, names) }, hash, tenant) @@ -377,10 +345,7 @@ func (s *Server) templateOwners(query func(string) []string, hash, tenant string return owners } -// redirectClaim redirects a warm-miss to a better peer — a warm holder, or the -// template owner when we lack a golden (so we don't cold-boot a nonexistent -// image ref). A no_redirect request must resolve locally, never bounce again, -// to avoid a two-node ping-pong. +// redirectClaim bounces a warm-miss to a warm holder, or the template owner when we lack a golden. func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req types.ClaimRequest, key types.PoolKey, hash, tenant string) bool { if s.placer == nil || req.NoRedirect { return false @@ -393,8 +358,6 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t return len(owners) > 0 && !s.mgr.HasGolden(ctx, key, tenant) && writeRedirect(w, owners) } -// handleSandbox reports one live claim, so a reconcile need not scan the -// whole-node listing. func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") sb, ok := s.mgr.Sandbox(id) @@ -405,8 +368,6 @@ func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, sb) } -// handleSandboxStats reports one sandbox's resource usage. It is the only -// per-sandbox usage surface: /metrics is node- and pool-scoped by design. func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") st, ok := s.mgr.Stats(r.Context(), id) @@ -417,7 +378,6 @@ func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, st) } -// handleSandboxVerb shares credential resolution and error mapping for release, hibernate, and wake. func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id string, cred pool.Cred) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) @@ -432,8 +392,6 @@ func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id } } -// handleFork clones a claimed sandbox into fresh child claims, one -// ClaimResponse per child; this node owns them all. func (s *Server) handleFork(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.ForkRequest](w, r) if !ok { @@ -463,7 +421,6 @@ func (s *Server) handlePromote(w http.ResponseWriter, r *http.Request) { }) } -// handleCheckpoint captures a claimed sandbox's state as a new checkpoint. func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointRequest](w, r) if !ok { @@ -476,11 +433,7 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { }) } -// handleClaimCheckpoint claims a fresh sandbox branched from a checkpoint. -// Tier order: the local claim, then a redirect to a probed owner (zero bytes -// moved, but not authoritative — a peer that missed a delete broadcast still -// answers it holds the record until its own TTL sweep runs), then one peer -// transfer if neither answered. +// handleClaimCheckpoint claims a sandbox from a checkpoint; a probed owner is not authoritative. func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointClaimRequest](w, r) if !ok { @@ -506,9 +459,7 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { writeResult(w, r, "fetch checkpoint", ckptID, "fetch checkpoint failed", err, func() { defer release() w.Header().Set("Content-Type", "application/x-tar") - // Status is committed before the walk, so a mid-stream failure cannot change - // it; the tar's completion marker is what tells the reader the record - // arrived whole, and a short transfer is rejected for lacking it. + // the tar completion marker, not the status, tells the reader the record arrived whole w.WriteHeader(http.StatusOK) if err := peer.TarRecord(dir, meta, w); err != nil { log.WithFunc("server.handleCheckpointBlob").Error(r.Context(), err, "stream checkpoint") @@ -516,10 +467,7 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { }) } -// handleCheckpointProbe answers a peer's HEAD probe. With a probeKey -// configured (an encrypted mesh), the caller must present a fresh MAC over -// the id (peer.ProbeHeader) or the probe is rejected before the metadata -// read; without one, the id remains the only capability. +// handleCheckpointProbe answers a HEAD probe; without a probeKey the id is the only capability. func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if len(s.probeKey) > 0 && !peer.VerifyProbeMAC(s.probeKey, id, r.Header.Get(peer.ProbeHeader)) { @@ -533,8 +481,7 @@ func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) } -// handleListCheckpoints lists this node's checkpoints, newest first — a -// tenant caller sees only its own records. +// handleListCheckpoints lists this node's checkpoints, newest first. func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { ckpts, err := s.mgr.Checkpoints(r.Context(), tenantFrom(r.Context())) if err != nil { @@ -545,10 +492,7 @@ func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, types.CheckpointListResponse{Checkpoints: ckpts}) } -// handleDeleteCheckpoint removes a checkpoint; a tenant caller may delete -// only its own records (anything else is 404). no_forward marks a delete -// arriving from another node's own broadcast, so this one does not -// re-broadcast and loop the fleet forever. +// handleDeleteCheckpoint removes a checkpoint; no_forward stops a broadcast delete from looping. func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) { scope := pool.DeleteFleet if r.URL.Query().Get("no_forward") != "" { @@ -556,10 +500,7 @@ func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) } id := r.PathValue("id") err := s.mgr.DeleteCheckpoint(r.Context(), id, tenantFrom(r.Context()), scope) - // Evict after the record is gone, not before: a probe racing the delete - // would otherwise observe the still-present record and cache it just as the - // pre-delete eviction cleared the entry. Unconditional — a forwarded delete - // that finds nothing here still learned the id's ownership changed. + // evict after the delete: a racing probe would otherwise re-cache the still-present record if s.prober != nil { s.prober.Forget(id) } @@ -568,8 +509,7 @@ func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) }) } -// handleDeleteTemplate removes a promoted template; the key axes ride as -// query parameters, defaulted like a claim's. +// handleDeleteTemplate removes a promoted template; the key axes ride as query parameters. func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() req := types.ClaimRequest{ @@ -579,9 +519,7 @@ func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { } key := req.Key() err := s.mgr.DeleteTemplate(r.Context(), key, tenantFrom(r.Context())) - // Unknown here but owned by a peer per gossip: redirect the SDK to the - // owner. no_redirect mirrors the claim protocol — a redirected retry - // carries it, so the owner answers for itself and never bounces again. + // a redirected retry carries no_redirect, so the owner answers for itself and never bounces if errors.Is(err, pool.ErrUnknownTemplate) && s.placer != nil && q.Get("no_redirect") == "" && writeRedirect(w, s.templateOwners(s.placer.TemplateOwners, key.Hash(), tenantFrom(r.Context()))) { return @@ -591,9 +529,7 @@ func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { }) } -// handleOwner answers whether this node owns the sandbox (used by the SDK's -// Lookup scatter to relocate a handle whose owner address was lost). The -// per-sandbox token both authorizes the query and confirms ownership. +// handleOwner answers whether this node owns the sandbox; the token both authorizes and proves it. func (s *Server) handleOwner(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { @@ -632,9 +568,7 @@ func (s *Server) handleUncordon(w http.ResponseWriter, r *http.Request) { s.handleInfo(w, r) } -// handlePeers lists the cluster's node addresses for the SDK's redirect -// follow and Lookup scatter — cluster topology, not operator state, so any -// valid token (root or tenant) may read it. +// handlePeers lists the cluster's node addresses; topology is readable by any valid token. func (s *Server) handlePeers(w http.ResponseWriter, _ *http.Request) { var peers []string if s.placer != nil { @@ -659,9 +593,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") } -// requireToken guards node-level endpoints, resolving the caller's scope — -// root (the api token) or a tenant name (its configured token) — onto the -// request context; sandbox-scoped endpoints use per-sandbox tokens instead. +// requireToken resolves the caller's scope — root or a tenant name — onto the request context. func (s *Server) requireToken(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { tenant, ok := s.resolveScope(r) @@ -673,8 +605,7 @@ func (s *Server) requireToken(next http.HandlerFunc) http.HandlerFunc { } } -// requireRoot guards operator-only surfaces: a tenant token authenticates -// but is not authorized, so it gets 403, never 401. +// requireRoot guards operator-only surfaces: an authenticated tenant token gets 403, not 401. func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { return s.requireToken(func(w http.ResponseWriter, r *http.Request) { if tenantFrom(r.Context()) != "" { @@ -685,22 +616,17 @@ func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { }) } -// rootRequest reports whether this request presented the root api_token. func (s *Server) rootRequest(r *http.Request) bool { token, ok := bearerToken(r) return ok && s.isRootToken(token) } -// isRootToken reports whether token is the configured root api_token (the -// operator credential). An unset api token matches nothing, so an open node -// grants no operator elevation; tenant tokens never match — they live in -// s.tenants. +// isRootToken reports whether token is the root api_token; an unset api token matches nothing. func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } -// sandboxCred resolves a header bearer token to a Cred. The root api_token -// elevates to Operator: the manager must never see it as a sandbox token. +// sandboxCred resolves a header bearer token to a Cred; the root api_token elevates to Operator. func (s *Server) sandboxCred(token string) pool.Cred { if s.isRootToken(token) { return pool.Cred{Operator: true} @@ -708,14 +634,12 @@ func (s *Server) sandboxCred(token string) pool.Cred { return pool.Cred{Token: token} } -// bodyCred resolves a body-carried sandbox token: absent, on a -// root-authenticated request, is Operator; a tenant must still prove ownership. +// bodyCred resolves a body-carried sandbox token; absent on a root request means Operator. func (s *Server) bodyCred(r *http.Request, bodyToken string) pool.Cred { return pool.Cred{Token: bodyToken, Operator: bodyToken == "" && s.rootRequest(r)} } -// resolveScope matches the bearer token to root ("") or a tenant name. With -// no api token and no tenants the node-level endpoints stay open. +// resolveScope matches the bearer token to root ("") or a tenant name. func (s *Server) resolveScope(r *http.Request) (string, bool) { if s.apiToken == "" && len(s.tenants) == 0 { return "", true diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 7c7f5818..3ace7f56 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -15,13 +15,10 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// retryAfterSeconds hints when to retry a 503. A heal slot frees when a -// transfer finishes, which can be seconds or minutes, so this bounds a waiting -// client to a few probes a minute rather than inviting a 1-second hammer. +// retryAfterSeconds bounds a waiting client to a few probes a minute. const retryAfterSeconds = "10" -// tenantKey carries the resolved tenant scope ("" = root) on the request -// context, from requireToken to the handlers. +// tenantKey carries the resolved tenant scope ("" = root) on the request context. type tenantKey struct{} func withTenant(ctx context.Context, tenant string) context.Context { @@ -57,9 +54,7 @@ func decodeBody[T any](w http.ResponseWriter, r *http.Request) (T, bool) { return v, true } -// decodeBodyStrict is decodeBody for operator bodies, where a mistyped or -// repeated key must fail instead of silently no-opping; client-facing bodies -// stay lenient (newer SDK fields must not break older nodes). +// decodeBodyStrict is decodeBody for operator bodies, where a mistyped key must fail. func decodeBodyStrict[T any](w http.ResponseWriter, r *http.Request) (T, bool) { var v T raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBodyBytes)) @@ -74,14 +69,11 @@ func decodeBodyStrict[T any](w http.ResponseWriter, r *http.Request) (T, bool) { return v, true } -// writePoolErr answers a pool-sentinel error per poolErrHTTP, reporting -// whether it handled err; nil and non-sentinel errors stay the caller's. +// writePoolErr answers a pool-sentinel error, reporting whether it handled err. func writePoolErr(w http.ResponseWriter, err error) bool { for _, m := range poolErrHTTP { if errors.Is(err, m.err) { - // 503 here means the node's own resources are momentarily exhausted - // (e.g. the concurrent-heal budget), not a caller error — Retry-After - // tells the client this is worth retrying, not failing the request. + // a 503 is node exhaustion, not a caller error, so the client should retry if m.code == http.StatusServiceUnavailable { w.Header().Set("Retry-After", retryAfterSeconds) } @@ -103,8 +95,7 @@ func writeResult(w http.ResponseWriter, r *http.Request, op, id, failMsg string, } } -// writeRedirect answers with the redirect shape of the claim protocol when -// addrs is non-empty, reporting whether it did. +// writeRedirect answers with the claim protocol's redirect shape, reporting whether it did. func writeRedirect(w http.ResponseWriter, addrs []string) bool { if len(addrs) == 0 { return false diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 878117f9..a3a1738b 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -304,8 +304,6 @@ func TestTenantAuthMatrix(t *testing.T) { } } -// TestMetricsTenantGauge checks the per-tenant live-claim gauge renders with -// the configured tenants only. func TestMetricsTenantGauge(t *testing.T) { mgr := &fakeManager{tenantClaims: map[string]int{"acme": 2, "beta": 0}} ts := newTestServer(t, "", mgr, nil) @@ -323,8 +321,6 @@ func TestMetricsTenantGauge(t *testing.T) { } } -// TestPutPoolsRejectsUnknownFields: the operator body decodes strictly — a -// mistyped key (e.g. "egres") must 400, not succeed with the policy dropped. func TestPutPoolsRejectsUnknownFields(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/pools", @@ -415,7 +411,6 @@ func TestPutPoolsUpdatesTargets(t *testing.T) { } } -// TestSandboxVerbFlows pins shared auth, error mapping, and id/token plumbing for release and hibernate. func TestSandboxVerbFlows(t *testing.T) { verbs := []struct { name string @@ -469,21 +464,16 @@ func TestSandboxVerbFlows(t *testing.T) { } } -// TestReleaseOperatorToken proves the release route's authorization split: the -// node root api_token releases any sandbox by id via an Operator credential (no -// per-sandbox token), while a per-sandbox token still takes the token-checked -// path and a tenant/wrong token gets no operator elevation (it resolves as a -// non-matching sandbox token and 404s). Exactly one hook runs per request. func TestReleaseOperatorToken(t *testing.T) { const rootTok, sbTok = "sekret", "sb-secret" tenants := []config.TenantSpec{{Name: "acme", Token: "acme-tok"}} tests := []struct { name string auth string - wantOp bool // operator (Cred.Operator) release expected - wantRelease bool // per-sandbox release expected - wantToken string // token the per-sandbox release should receive - releaseErr error // error the per-sandbox release returns + wantOp bool + wantRelease bool + wantToken string + releaseErr error want int }{ {"root token releases by id", "Bearer " + rootTok, true, false, "", nil, http.StatusNoContent}, @@ -603,8 +593,7 @@ func TestForkFlow(t *testing.T) { } return children, nil }} - // Fork creates node resources: the header carries the API token like a - // claim, and the sandbox's own token rides in the body. + ts := newTestServer(t, "sekret", mgr, nil) post := func(auth, body string) *http.Response { @@ -741,7 +730,7 @@ func TestPromoteAndDeleteTemplateFlow(t *testing.T) { if got := del("Bearer sekret", "template=nope"); got != http.StatusNotFound { t.Errorf("unknown delete status %d, want 404", got) } - // Node-level endpoint: the API token guards it. + if got := del("", "template=tpl:x"); got != http.StatusUnauthorized { t.Errorf("unauthenticated delete status %d, want 401", got) } @@ -821,10 +810,6 @@ func TestCheckpointFlow(t *testing.T) { } } -// TestDeleteCheckpointNoForwardQueryParam proves the no_forward query param -// reaches the manager, mirroring how handleDeleteTemplate already threads -// no_redirect: a delete arriving from another node's broadcast must not -// re-broadcast, or the fleet loops the delete forever. func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { mgr := &fakeManager{deleteCheckpoint: func(string) error { return nil }} ts := newTestServer(t, "api", mgr, &fakeDialer{}) @@ -857,10 +842,6 @@ func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { } } -// TestDeleteCheckpointForgetsProbeCache proves a delete evicts any cached -// probe answer for the id, whether the delete is the original (fleet scope) -// or a forwarded broadcast (no_forward) that finds nothing to delete locally -// — both cases mean this node just learned id's ownership changed. func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { prober := &fakeProber{} @@ -904,8 +885,6 @@ func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { } func TestClaimRedirectsOnWarmMiss(t *testing.T) { - // Warm miss (fakeManager.ClaimWarm always misses) + a placer with - // candidates → a redirect response, and the local manager never provisions. provisioned := false mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { provisioned = true @@ -936,9 +915,6 @@ func TestClaimRedirectsOnWarmMiss(t *testing.T) { } func TestClaimRedirectsToTemplateOwner(t *testing.T) { - // Warm miss, no warm candidates, no local golden, but gossip names a - // template owner → redirect there instead of cold-booting a nonexistent - // image ref. A local golden suppresses the redirect: provision here. for _, tt := range []struct { name string hasGolden bool @@ -981,9 +957,7 @@ func TestClaimRedirectsToTemplateOwner(t *testing.T) { } func TestDeleteTemplateRedirectsToOwner(t *testing.T) { - // Unknown locally + gossip names an owner → the claim redirect shape; - // unknown everywhere stays 404. - mgr := &fakeManager{} // DeleteTemplate → ErrUnknownTemplate + mgr := &fakeManager{} srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -1008,7 +982,6 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { t.Errorf("redirect %v, want [node-b:7777]", cr.Redirect) } - // no_redirect answers for this node alone, even with owners in gossip. req2, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+"/v1/templates?template=tpl&no_redirect=1", nil) if err != nil { t.Fatalf("request: %v", err) @@ -1040,7 +1013,6 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { } func TestClaimProvisionsWhenNoCandidate(t *testing.T) { - // Warm miss + a placer with no candidates → local provision. mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }} @@ -1058,7 +1030,7 @@ func TestClaimProvisionsWhenNoCandidate(t *testing.T) { if cr.ID != "sb_local" || len(cr.Redirect) != 0 { t.Errorf("got %+v, want local sandbox", cr) } - // The claim_ref on the wire must thread through the handler to the claim. + if mgr.gotClaimRef != "ns1/w1" { t.Errorf("claim_ref not threaded: got %q, want %q", mgr.gotClaimRef, "ns1/w1") } @@ -1253,9 +1225,6 @@ func TestVolumeClaimUsesVolumeAndTemplateIntersections(t *testing.T) { } } -// TestVolumeClaimKeepsPoolContentAgainstPeerTemplates: per-node content -// consistency means a pooled key never asks the mesh what a peer promoted; -// only an explicit RequirePromoted is refused, never served the wrong golden. func TestVolumeClaimKeepsPoolContentAgainstPeerTemplates(t *testing.T) { for _, tt := range []struct { name string @@ -1475,7 +1444,6 @@ func TestVolumeClaimRejectsShapeBeforePlacement(t *testing.T) { } func TestOwnerEndpoint(t *testing.T) { - // AgentSocket succeeds → this node owns it → 200 with owner addr. mgr := &fakeManager{socket: func(id, token string) (string, error) { if token == "good" { return "/v/sock", nil @@ -1516,9 +1484,6 @@ func TestOwnerEndpoint(t *testing.T) { } } -// TestPreviewHandlerZeroDeadlineMintsLiveToken covers a claim with no -// deadline (e.g. archived with ArchiveDeleteAfterSeconds=0): the minted -// preview token must still verify, not be stamped already-expired. func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { mgr := &fakeManager{claimDeadline: func(string, string) (time.Time, error) { return time.Time{}, nil @@ -1551,13 +1516,8 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { } } -// TestCheckpointClaimRedirectsToOwner is L2: checkpoints are node-local, so a -// branch that lands on a node without the record must be pointed at one that -// has it — the clone then runs on the node whose disk already holds the data, -// on its local reflink fast path. Failing here instead would make "snapshot on -// A, branch from B" simply not work. func TestCheckpointClaimRedirectsToOwner(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + mgr := &fakeManager{} prober := &fakeProber{owners: []string{"owner-a:7777"}} ts := newPlacerTestServer(t, "sekret", mgr, prober) @@ -1579,8 +1539,6 @@ func TestCheckpointClaimRedirectsToOwner(t *testing.T) { } } -// TestCheckpointClaimNoRedirectResolvesLocally: the retry at a redirect target -// must resolve there or two nodes would bounce the request between them. func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { prober := &fakeProber{owners: []string{"owner-a:7777"}} ts := newPlacerTestServer(t, "sekret", &fakeManager{}, prober) @@ -1593,11 +1551,8 @@ func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { } } -// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss -// falls through to heal, and a heal miss (item 4) is an honest 404 rather -// than a redirect to nowhere. func TestCheckpointClaimNoOwnersIs404(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpointHeal defaults to ErrUnknownCheckpoint + mgr := &fakeManager{} ts := newPlacerTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) @@ -1611,11 +1566,8 @@ func TestCheckpointClaimNoOwnersIs404(t *testing.T) { } } -// TestCheckpointClaimRedirectBeatsHeal is the tier-order regression test: a -// gossiped owner must redirect (zero bytes moved) and never fall through to -// the heal pull, even though the local ClaimCheckpoint missed. func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + mgr := &fakeManager{} prober := &fakeProber{owners: []string{"owner-a:7777"}} ts := newPlacerTestServer(t, "sekret", mgr, prober) @@ -1637,8 +1589,6 @@ func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { } } -// TestCheckpointClaimFallsBackToHealWhenNoOwner: no gossiped owner to redirect -// to, so the server pulls the record itself via heal. func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { mgr := &fakeManager{ healCheckpoint: func(ckptID string) (*types.Sandbox, error) { @@ -1665,8 +1615,6 @@ func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { } } -// TestCheckpointClaimNoRedirectGoesStraightToHeal: a no_redirect retry must -// not bounce again, but it must still try heal rather than answer 404 outright. func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { mgr := &fakeManager{ healCheckpoint: func(ckptID string) (*types.Sandbox, error) { @@ -1697,9 +1645,6 @@ func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { } } -// TestCheckpointClaimHealBusyIs503WithRetryAfter: a full heal budget is the -// node's own transient exhaustion, not a caller error — the client should -// retry, and Retry-After says so instead of leaving it to guess. func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { mgr := &fakeManager{ healCheckpoint: func(string) (*types.Sandbox, error) { @@ -1719,8 +1664,6 @@ func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { } } -// TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner -// instead of treating a stale gossip entry as a transfer failure. func TestCheckpointBlobUnknownIs404(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) @@ -1741,8 +1684,6 @@ func TestCheckpointBlobUnknownIs404(t *testing.T) { } } -// TestCheckpointBlobStreamsRecord is the serving half of L3: the record leaves -// as a tar the peer can unpack, with its export contents intact. func TestCheckpointBlobStreamsRecord(t *testing.T) { src := t.TempDir() if err := os.WriteFile(filepath.Join(src, "mem"), []byte("guest-pages"), 0o600); err != nil { @@ -1770,9 +1711,7 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { if err := peer.Untar(resp.Body, dst); err != nil { t.Fatalf("Untar the streamed record: %v", err) } - // The blob is a whole record: meta.json at the root plus the export under - // export/. Streaming only the export produced a directory that published - // cleanly and then failed every read as "unknown checkpoint". + if got, err := os.ReadFile(filepath.Join(dst, "export", "mem")); err != nil || string(got) != "guest-pages" { t.Fatalf("streamed export/mem = %q, %v; want the record's bytes", got, err) } @@ -1781,10 +1720,6 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { } } -// TestCheckpointBlobHeadUnauthenticated proves the probe route bypasses -// requireToken while GET still enforces it. With no probe key configured -// (no cluster_key -- an unencrypted mesh has no shared secret to build one -// from), the id remains the only capability, same as before this batch. func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { const ckptID = "ck_00000000000000aa" mgr := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} @@ -1815,10 +1750,6 @@ func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { } } -// TestCheckpointProbeRequiresValidMAC proves that with a probe key -// configured (an encrypted mesh), handleCheckpointProbe rejects a missing or -// wrongly-signed HEAD before it ever reads HasCheckpoint, and accepts one -// signed with the matching key. func TestCheckpointProbeRequiresValidMAC(t *testing.T) { const ckptID = "ck_00000000000000aa" key := peer.DeriveProbeKey([]byte("cluster-secret")) @@ -1848,16 +1779,12 @@ func TestCheckpointProbeRequiresValidMAC(t *testing.T) { if got := head(peer.SignProbe(key, ckptID)); got != http.StatusOK { t.Errorf("valid signature: status = %d, want 200", got) } - // A valid signature for a DIFFERENT checkpoint must not authorize this one. + if got := head(peer.SignProbe(key, "ck_00000000000000ff")); got != http.StatusUnauthorized { t.Errorf("signature for a different id: status = %d, want 401", got) } } -// TestCheckpointClaimProbesRealPeerAndRedirects is the end-to-end probe: a -// live HTTPProber HEADs an actual peer server rather than a fake, proving the -// probe wire protocol (unauthenticated HEAD, addr scheme defaulting) against -// the real handler, not just an interface stub. func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { const ckptID = "ck_00000000000000aa" mgrB := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} @@ -1866,7 +1793,7 @@ func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { t.Cleanup(tsB.Close) prober := &peer.HTTPProber{Peers: func() []string { return []string{tsB.URL} }} - mgrA := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + mgrA := &fakeManager{} srvA := New("sekret", nil, "node-a:7777", mgrA, &fakeDialer{}, nil, prober, nil, nil) tsA := httptest.NewServer(srvA.Handler()) t.Cleanup(tsA.Close) @@ -1925,8 +1852,6 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS return ts } -// newProbeAuthTestServer builds a server with a probe key configured, for -// tests of handleCheckpointProbe's MAC verification specifically. func newProbeAuthTestServer(t *testing.T, mgr Manager, probeKey []byte) *httptest.Server { t.Helper() srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, probeKey, nil) @@ -1938,8 +1863,6 @@ func newProbeAuthTestServer(t *testing.T, mgr Manager, probeKey []byte) *httptes return ts } -// credToken recovers the token a fake hook expects from cred: empty for an -// Operator credential, mirroring the collapsed manager's own translation. func credToken(cred pool.Cred) string { if cred.Operator { return "" @@ -1947,9 +1870,6 @@ func credToken(cred pool.Cred) string { return cred.Token } -// fakeManager implements Manager with overridable behavior. ClaimWarm misses -// unless warmClaim is set; claim stands in for the provision result. -// Tenant-scoped methods record the tenant they were handed in gotTenant. type fakeManager struct { ckptDir string hasCheckpoint map[string]bool @@ -2016,8 +1936,6 @@ func (f *fakeManager) ClaimProvisionPromoted(ctx context.Context, key types.Pool return fakeClaimProvision(f, ctx, key, ttl, tenant, claimRef, volumes, true) } -// Release dispatches to the operator or per-sandbox hook, mirroring the -// collapsed manager's own Cred split. func (f *fakeManager) Release(_ context.Context, id string, cred pool.Cred) error { if cred.Operator { if f.releaseOp == nil { @@ -2207,8 +2125,6 @@ func (f *fakeManager) Drain(context.Context) { f.draining = true } func (f *fakeManager) Uncordon(context.Context) { f.draining = false } -// FetchCheckpoint serves the peer-transfer read; the fake reports every record -// missing unless a test supplies a directory. func (f *fakeManager) FetchCheckpoint(_ context.Context, _ string) (string, []byte, func(), error) { if f.ckptDir == "" { return "", nil, nil, pool.ErrUnknownCheckpoint @@ -2216,8 +2132,6 @@ func (f *fakeManager) FetchCheckpoint(_ context.Context, _ string) (string, []by return f.ckptDir, []byte(`{"id":"ck_00000000000000aa"}`), func() {}, nil } -// HasCheckpoint answers the probe endpoint straight from the fake's map; -// unset ids report false. func (f *fakeManager) HasCheckpoint(_ context.Context, ckptID string) bool { return f.hasCheckpoint[ckptID] } @@ -2285,9 +2199,6 @@ func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } func (f *fakePlacer) PeerAddrs() []string { return f.addrs } func (f *fakePlacer) ConfigMismatches() int { return 0 } -// fakeProber stubs CheckpointProber with a fixed answer, standing in for a -// live HEAD fan-out. forgotten records every id Forget was called with, so -// tests can assert a delete evicted the right cache entry. type fakeProber struct { owners []string forgotten []string @@ -2296,10 +2207,6 @@ type fakeProber struct { func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } -// newPlacerTestServer builds a server with a checkpoint prober, which the -// shared helper deliberately leaves nil (most tests are single-node); the -// mesh placer stays nil throughout — these tests exercise checkpoint claims -// only. func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, prober CheckpointProber) *httptest.Server { t.Helper() srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, nil, prober, nil, nil) diff --git a/sandboxd/store/dir/dir.go b/sandboxd/store/dir/dir.go index 4bbb3a51..395fa521 100644 --- a/sandboxd/store/dir/dir.go +++ b/sandboxd/store/dir/dir.go @@ -31,8 +31,7 @@ const ( digestWorkerLimit = 8 digestReadBufferSize = 128 << 10 - // generationGrace bounds the shared-mount race where another node is - // still cloning a generation it resolved just before a re-publish. + // generationGrace bounds the race where another node still clones a generation being re-published. generationGrace = time.Hour ) @@ -50,9 +49,7 @@ type chunkHasher func(source *digestSource, chunk int, buffer []byte) error var _ store.Store = (*Store)(nil) -// Store keeps records as //{meta.json,export-/...}: the -// generation dir is immutable and the meta rename is the atomic commit -// pointer, so readers and writers need no cross-process locks. +// Store keeps records at /; the meta.json rename is the atomic commit pointer. type Store struct { root string idRe *regexp.Regexp @@ -69,8 +66,6 @@ func (d *Store) Stage(id string) (string, error) { return os.MkdirTemp(d.root, id+"-*.tmp") } -// Publish makes the generation fresh before its install, refreshes the -// outgoing generation at supersession, and commits by renaming meta.json. func (d *Store) Publish(ctx context.Context, staging, id string) error { return d.publish(ctx, staging, id, "") } @@ -86,9 +81,6 @@ func (d *Store) PublishDigested(ctx context.Context, staging, id string) (string return digest, nil } -// Fetch resolves the committed meta and returns its immutable generation -// directory; release is a no-op. Records published before per-generation -// dirs fall back to the flat export layout. func (d *Store) Fetch(ctx context.Context, id string) (string, []byte, string, func(), error) { meta, err := d.ReadMeta(ctx, id) if err != nil { @@ -128,8 +120,6 @@ func (d *Store) Metas(ctx context.Context) ([][]byte, error) { } var metas [][]byte for _, e := range entries { - // Only well-formed record dirs in this instance's namespace — a - // planted foo/meta.json is not a record. if !e.IsDir() || !d.idRe.MatchString(e.Name()) { continue } @@ -145,8 +135,6 @@ func (d *Store) Metas(ctx context.Context) ([][]byte, error) { return metas, nil } -// Delete removes export generations first and the meta last, so a partially -// failed delete stays discoverable and a retry converges. func (d *Store) Delete(_ context.Context, id string) error { final := filepath.Join(d.root, id) entries, err := os.ReadDir(final) @@ -167,7 +155,6 @@ func (d *Store) Delete(_ context.Context, id string) error { return errors.Join(os.RemoveAll(final), os.RemoveAll(final+oldSuffix)) } -// SweepStaging clears crashed staging and delegates generation retention. func (d *Store) SweepStaging() error { // ReadDir + suffix, not Glob: the root path may hold glob metacharacters. if err := utils.RemoveDirEntries(d.root, func(name string) bool { return strings.HasSuffix(name, ".tmp") }); err != nil { @@ -176,7 +163,6 @@ func (d *Store) SweepStaging() error { return d.SweepGenerations() } -// SweepGenerations reclaims expired record generations without touching staging. func (d *Store) SweepGenerations() error { entries, err := os.ReadDir(d.root) if err != nil { @@ -254,9 +240,7 @@ func (d *Store) publish(_ context.Context, staging, id, digest string) error { return os.RemoveAll(staging) } -// sweepGenerations reclaims only entries whose own install or supersession -// time is outside the grace. Without committed meta it never removes the -// record directory, so a peer publishing into the same path remains safe. +// sweepGenerations reclaims only entries whose install or supersession time is outside the grace. func (d *Store) sweepGenerations(id string) (err error) { final := filepath.Join(d.root, id) metaFile, err := os.Open(filepath.Join(final, store.MetaFile)) //nolint:gosec // id pinned by the instance idRe diff --git a/sandboxd/store/dir/dir_test.go b/sandboxd/store/dir/dir_test.go index 0ecbf2f0..69621ee9 100644 --- a/sandboxd/store/dir/dir_test.go +++ b/sandboxd/store/dir/dir_test.go @@ -24,9 +24,6 @@ func TestDirBackendContract(t *testing.T) { storetest.RunContract(t, st) } -// TestFetchPinsGenerationAcrossStoreInstances: a fetched generation must -// survive a concurrent re-publish from another store instance (another -// process on a shared mount) — retention replaces the old cross-process lock. func TestFetchPinsGenerationAcrossStoreInstances(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -300,8 +297,6 @@ func TestSweepGenerationsPairsCurrentAndSupersededDigests(t *testing.T) { } } -// TestPublishRetriesAfterExpiredInstall: an expired uncommitted generation is -// never revived; GC then retry converges, and the committed retry is idempotent. func TestPublishRetriesAfterExpiredInstall(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -346,9 +341,6 @@ func TestPublishRetriesAfterExpiredInstall(t *testing.T) { } } -// TestFetchLegacyFlatLayout: records published before per-generation dirs -// keep a flat /export; Fetch falls back to it, and a re-publish moves the -// record onto the generation layout. func TestFetchLegacyFlatLayout(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -378,8 +370,6 @@ func TestFetchLegacyFlatLayout(t *testing.T) { } } -// TestPublishSweepsGenerationsBySupersessionAge: rolling re-promotes reclaim -// generations outside their own grace without disturbing newer generations. func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -418,8 +408,6 @@ func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { } } -// TestSweepSparesPublishingGeneration: a peer sweep cannot delete a fresh -// generation between its install and meta commit. func TestSweepSparesPublishingGeneration(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -465,8 +453,6 @@ func TestSweepSparesPublishingGeneration(t *testing.T) { } } -// TestSweepSparesLegacyFallback: the flat export dir of a never-re-published -// legacy record is live data; once a re-publish supersedes it, grace applies. func TestSweepSparesLegacyFallback(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -500,8 +486,6 @@ func TestSweepSparesLegacyFallback(t *testing.T) { } } -// TestSweepReclaimsOnlyAgedUncommittedGenerations: an abandoned generation -// is reclaimed across ids while a peer's uncommitted publish remains intact. func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { const ( orphanID = "ck_00000000000000aa" @@ -574,8 +558,6 @@ func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { } } -// TestSweepGenerationsToleratesConcurrentDelete: a record deleted mid-sweep -// must not fail the pass — the first error aborts every remaining record. func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { const id = "ck_00000000000000aa" st, err := New(t.TempDir(), store.CheckpointIDRe) @@ -624,8 +606,6 @@ func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { } } -// TestDeleteRemovesLegacyResidue: Delete clears a pre-generation .old -// crash artifact along with the record, and is idempotent on a missing id. func TestDeleteRemovesLegacyResidue(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -650,8 +630,6 @@ func TestDeleteRemovesLegacyResidue(t *testing.T) { } } -// TestMetasSurfacesReadError: a corrupt/unreadable meta must fail the listing, -// not vanish silently — mirroring the s3 backend. func TestMetasSurfacesReadError(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -659,7 +637,7 @@ func TestMetasSurfacesReadError(t *testing.T) { if err != nil { t.Fatalf("New: %v", err) } - // A directory where meta.json is expected makes ReadFile fail with EISDIR. + if err := os.MkdirAll(filepath.Join(root, id, store.MetaFile), 0o750); err != nil { t.Fatalf("setup: %v", err) } @@ -672,8 +650,6 @@ func metaJSON(metaID string) string { return `{"id":"` + metaID + `"}` } -// installUncommitted installs metaID's generation without committing -// meta.json — the state a crash between install and commit leaves. func installUncommitted(t *testing.T, st *Store, root, id, metaID string) (meta []byte, gen string) { t.Helper() staging, err := st.Stage(id) @@ -699,8 +675,6 @@ func installUncommitted(t *testing.T, st *Store, root, id, metaID string) (meta return meta, gen } -// mustPublish stages and publishes a one-file export whose meta is -// metaJSON(metaID). func mustPublish(t *testing.T, st *Store, id, metaID string) { t.Helper() staging, err := st.Stage(id) diff --git a/sandboxd/store/peer/broadcast.go b/sandboxd/store/peer/broadcast.go index f7126bea..38d1ddc0 100644 --- a/sandboxd/store/peer/broadcast.go +++ b/sandboxd/store/peer/broadcast.go @@ -16,19 +16,15 @@ const ( defaultDeleteClientTimeout = 10 * time.Second ) -// Broadcaster fans a checkpoint delete out to every peer after a local -// delete, so a record healed onto another node does not outlive the source. +// Broadcaster fans a checkpoint delete out to every peer so a healed copy does not outlive it. type Broadcaster struct { - // A nil Client uses a default with a short timeout: one wedged peer must - // not hold up the fan-out. + // A nil Client uses a default with a short timeout: one wedged peer must not hold up the fan-out. Client *http.Client Peers func() []string Token string } -// Delete sends DELETE ?no_forward=1 to every peer in parallel. Failures are -// logged and never returned: a broadcast is best-effort and must never fail -// the local delete it follows. +// Delete sends DELETE ?no_forward=1 to every peer in parallel; failures are logged, not returned. func (b *Broadcaster) Delete(ctx context.Context, id string) { addrs := dedupAddrs(b.Peers()) if len(addrs) == 0 { diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go index d1aa8ed3..8f5d0fa3 100644 --- a/sandboxd/store/peer/broadcast_test.go +++ b/sandboxd/store/peer/broadcast_test.go @@ -7,9 +7,6 @@ import ( "testing" ) -// TestBroadcastDeleteSendsNoForward: a broadcast delete must carry -// no_forward=1 so the receiving peer's own delete does not re-broadcast, -// which would loop forever across the fleet. func TestBroadcastDeleteSendsNoForward(t *testing.T) { var gotNoForward, gotAuth string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -30,8 +27,6 @@ func TestBroadcastDeleteSendsNoForward(t *testing.T) { } } -// TestBroadcastDeleteFansOutToEveryPeer: every distinct peer must be -// contacted, not just the first. func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { var hits atomic.Int32 newSrv := func() *httptest.Server { @@ -52,8 +47,6 @@ func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { } } -// TestBroadcastDeleteDedupsRepeatedAddrs: a duplicated peer list must not -// double-broadcast to the same address. func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { var hits atomic.Int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -70,9 +63,6 @@ func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { } } -// TestBroadcastDeleteSwallowsFailures: a broadcast never fails the local -// delete it follows, so an unreachable or erroring peer must not panic or -// block the caller past the fan-out. func TestBroadcastDeleteSwallowsFailures(t *testing.T) { fail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) @@ -86,14 +76,12 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { defer ok.Close() b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} - b.Delete(t.Context(), testID) // must return without panicking regardless of peer outcomes + b.Delete(t.Context(), testID) if got := reached.Load(); got != 1 { t.Errorf("healthy peer hit %d times, want 1 (fan-out continues past a failure)", got) } } -// TestBroadcastDeleteNoPeersNoOp: a single-node deployment must not dial -// anything. func TestBroadcastDeleteNoPeersNoOp(t *testing.T) { b := &Broadcaster{Peers: func() []string { return nil }} b.Delete(t.Context(), testID) diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index 2db1a134..f049d79b 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -17,15 +17,10 @@ const healBudget = 30 * time.Minute // Owners resolves a record id to the peer addresses that hold it. type Owners func(id string) []string -// Validate checks a staged pull before Pull trusts the owner that sent it; an -// error tries the next owner. +// Validate checks a staged pull before Pull trusts it; an error tries the next owner. type Validate func(staging string) error -// Healer pulls a record this node does not hold into a caller-provided -// staging directory; the caller validates and publishes it. A nil owners or -// puller makes every Pull a miss (no mesh, no heal). Pull never dedups -// concurrent calls: each caller owns its destination dir, so dedup belongs to -// whoever owns the staging — pool.Manager's per-id heal flight. +// Healer pulls a record this node does not hold into a caller-provided staging directory. type Healer struct { owners Owners puller Puller @@ -34,13 +29,11 @@ type Healer struct { budget time.Duration } -// NewHealer builds a Healer wired to owners and puller. func NewHealer(owners Owners, puller Puller) *Healer { return &Healer{owners: owners, puller: puller} } -// Pull fetches id into staging from the first owner whose transfer validates, -// bounding every owner tried to one budget. +// Pull fetches id into staging from the first owner whose transfer validates. func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate) error { if h.owners == nil || h.puller == nil { return store.ErrNotFound diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go index db03dfa8..5761e4e8 100644 --- a/sandboxd/store/peer/peer_test.go +++ b/sandboxd/store/peer/peer_test.go @@ -15,8 +15,6 @@ import ( const testID = "ck_00000000000000aa" -// TestInertWithoutOwnersOrPuller: a node with no mesh must degrade to unable -// to heal, not fail oddly. Healing is an addition, never a dependency. func TestInertWithoutOwnersOrPuller(t *testing.T) { h := NewHealer(nil, nil) if err := h.Pull(t.Context(), testID, t.TempDir(), nil); !errors.Is(err, store.ErrNotFound) { @@ -24,8 +22,6 @@ func TestInertWithoutOwnersOrPuller(t *testing.T) { } } -// TestPullWritesStaging is Pull's whole point: the record lands in the -// caller-provided staging directory for the caller to validate and publish. func TestPullWritesStaging(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -39,9 +35,6 @@ func TestPullWritesStaging(t *testing.T) { } } -// TestHealTriesNextOwner: a gossiped view can be stale or a peer can be -// broken. One bad owner must not fail the pull while another can still -// serve it. func TestHealTriesNextOwner(t *testing.T) { puller := &fakePuller{ records: map[string]map[string]string{"peer-b:7777": record()}, @@ -57,9 +50,6 @@ func TestHealTriesNextOwner(t *testing.T) { } } -// TestHealAllOwnersMissStaysNotFound: when every owner answers "not found" -// the gossiped view was simply stale. That must surface as store.ErrNotFound -// so the caller's existing not-found handling is unchanged. func TestHealAllOwnersMissStaysNotFound(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -69,7 +59,6 @@ func TestHealAllOwnersMissStaysNotFound(t *testing.T) { } } -// TestHealNoOwners: nothing gossiped the record, so there is nobody to ask. func TestHealNoOwners(t *testing.T) { puller := &fakePuller{} h := NewHealer(func(string) []string { return nil }, puller) @@ -82,9 +71,6 @@ func TestHealNoOwners(t *testing.T) { } } -// TestHealErrorIsReported: a peer that fails for a real reason (not a miss) -// must not be flattened into "not found" — that would hide a broken transfer -// behind a 404 and send the operator looking for a deleted checkpoint. func TestHealErrorIsReported(t *testing.T) { puller := &fakePuller{failAddr: "peer-a:7777"} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -98,9 +84,6 @@ func TestHealErrorIsReported(t *testing.T) { } } -// TestPullDoesNotDedupConcurrentCalls: concurrent Pulls for one id but -// different staging dirs must each run their own transfer — sharing a result -// would leave one caller's dir empty but published as real. func TestPullDoesNotDedupConcurrentCalls(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -128,9 +111,6 @@ func TestPullDoesNotDedupConcurrentCalls(t *testing.T) { } } -// TestPullValidateRejectionTriesNextOwner: an owner whose transferred content -// fails validation is treated like a failed transfer — the next owner gets a -// chance instead of the whole heal aborting on one bad copy. func TestPullValidateRejectionTriesNextOwner(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{ "peer-a:7777": record(), @@ -154,8 +134,6 @@ func TestPullValidateRejectionTriesNextOwner(t *testing.T) { } } -// TestPullValidateRejectsEveryOwner: when every owner's copy fails -// validation, Pull must report the failures, not silently succeed. func TestPullValidateRejectsEveryOwner(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -166,13 +144,10 @@ func TestPullValidateRejectsEveryOwner(t *testing.T) { } } -// TestPullBudgetBoundsSlowOwners: two owners that each sleep past their -// per-owner slice must not blow past the overall budget — 30 minutes PER -// owner (the old design) would let N slow owners run N*30 minutes. func TestPullBudgetBoundsSlowOwners(t *testing.T) { puller := &sleepyPuller{sleep: 300 * time.Millisecond} h := NewHealer(func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) - h.budget = 200 * time.Millisecond // each owner's slice (100ms) is shorter than the sleep + h.budget = 200 * time.Millisecond start := time.Now() err := h.Pull(t.Context(), testID, t.TempDir(), nil) @@ -184,10 +159,9 @@ func TestPullBudgetBoundsSlowOwners(t *testing.T) { } } -// fakePuller serves records from an in-memory table, recording who was asked. type fakePuller struct { mu sync.Mutex - records map[string]map[string]string // addr → file → content + records map[string]map[string]string asked []string failAddr string } @@ -215,8 +189,6 @@ func (p *fakePuller) Pull(_ context.Context, addr, _, dst string) error { return nil } -// sleepyPuller blocks until ctx expires or sleep elapses, whichever is -// first — for proving a per-owner slice actually cuts a slow owner off. type sleepyPuller struct { sleep time.Duration } diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index ba0554a0..8fcbdc2e 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -22,17 +22,13 @@ const ( maxHealOwners = 8 // a heal wants the widest source list a few full/corrupt owners can't hide behind redirectCacheTTL = 5 * time.Second - // probeGrace bounds how long a fan-out waits for more owners once it has - // its first: a checkpoint usually lives on one node, and without this the - // collector would wait out every missing peer's timeout to return it. + // probeGrace bounds how long a fan-out waits for more owners once it has its first. probeGrace = 150 * time.Millisecond // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. ProbeHeader = "X-Cocoon-Probe" - // probeBucketSeconds is the coarse time step the probe MAC is bound to; - // VerifyProbeMAC accepts the current bucket plus one on either side, so - // clock skew up to a bucket is tolerated without an unbounded replay window. + // probeBucketSeconds is the coarse time step the probe MAC is bound to. probeBucketSeconds = 30 ) @@ -42,18 +38,13 @@ type redirectCacheEntry struct { expires time.Time } -// HTTPProber finds which peers hold a checkpoint by probing them directly: -// ownership is stable but unbounded, so it is queried on the rare cross-node -// miss instead of gossiped every second by every node. +// HTTPProber finds which peers hold a checkpoint by probing them on a cross-node miss. type HTTPProber struct { - // A nil Client uses a default with a 2-second timeout: a probe must never - // hang on a wedged peer. + // A nil Client uses a default 2-second timeout: a probe must never hang on a wedged peer. Client *http.Client Peers func() []string - // ProbeKey signs and verifies HEAD probes (see DeriveProbeKey); empty - // means an unencrypted mesh, which keeps the older capability-only - // posture rather than losing the probe outright. + // ProbeKey signs and verifies HEAD probes; empty means an unencrypted mesh. ProbeKey []byte redirectFlight singleflight.Group @@ -61,19 +52,13 @@ type HTTPProber struct { cacheMu sync.Mutex cache map[string]redirectCacheEntry - // epoch is bumped by every Forget under cacheMu: a fan-out that started - // before a delete must not write its now-suspect result back into the - // cache, so cachePut only commits when the epoch it read still holds. + // epoch is bumped by every Forget, so a fan-out started before a delete cannot cache its result. epoch uint64 // cacheTTL overrides redirectCacheTTL; a test seam, 5 seconds is slow to wait out. cacheTTL time.Duration } -// Owners fans a HEAD out to every peer and returns up to maxRedirectOwners -// addresses that answered 200. Cache staleness fails safely: a redirect to a -// peer that just deleted the record 404s, and the client's no_redirect -// fallback heals from the origin. func (p *HTTPProber) Owners(ctx context.Context, id string) []string { owners, start, hit := p.cacheLookup(id) if hit { @@ -87,21 +72,15 @@ func (p *HTTPProber) Owners(ctx context.Context, id string) []string { return v.([]string) } -// HealOwners is Owners' wide twin: a heal wants the largest source list, so -// a few full or corrupt owners cannot hide a usable one. Uncached — a heal's -// source list should reflect the fleet now, not a redirect-tuned snapshot. +// HealOwners is Owners' wide twin: a heal wants the largest source list, uncached. func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { v, _, _ := p.healFlight.Do(id, func() (any, error) { - // No grace: a heal wants every owner it can find, so it must not stop - // early on a fast one and hide a slower valid source behind it. + // no grace: a heal must not stop early on a fast owner and hide a slower valid source. return p.fanOut(ctx, id, maxHealOwners, 0), nil }) return v.([]string) } -// Forget evicts any cached Owners result for id -- called after any delete -// (original or forwarded broadcast) touches id here, since a stale positive -// would redirect claims to a node that no longer holds the record. func (p *HTTPProber) Forget(id string) { p.cacheMu.Lock() defer p.cacheMu.Unlock() @@ -109,10 +88,7 @@ func (p *HTTPProber) Forget(id string) { delete(p.cache, id) } -// fanOut probes every peer concurrently, canceling stragglers once maxOwners -// have answered 200. The context is detached and rebounded internally: -// singleflight shares one fan-out across callers whose own contexts cancel -// independently — one client disconnecting must not abort another's probe. +// fanOut probes every peer concurrently, canceling stragglers once maxOwners have answered 200. func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace time.Duration) []string { addrs := dedupAddrs(p.Peers()) if len(addrs) == 0 { @@ -136,10 +112,7 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace } go func() { wg.Wait(); close(wins) }() - // grace opens only after the first win: a genuine miss must hear from - // every peer before concluding nobody holds the record. graceCh stays nil - // when grace is 0 (a heal wants the exhaustive list), so the select then - // runs to the cap or until every peer has answered. + // grace opens only after the first win: a genuine miss must hear from every peer. var owners []string var graceCh <-chan time.Time for { @@ -163,9 +136,7 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace } } -// cacheLookup returns id's owners if a still-live positive entry exists, and -// always the current epoch: the caller carries it into the fan-out so cachePut -// can tell whether a Forget landed while the probe was in flight. +// cacheLookup returns id's live owners and the current epoch the caller carries into the fan-out. func (p *HTTPProber) cacheLookup(id string) (owners []string, epoch uint64, hit bool) { p.cacheMu.Lock() defer p.cacheMu.Unlock() @@ -176,10 +147,7 @@ func (p *HTTPProber) cacheLookup(id string) (owners []string, epoch uint64, hit return entry.owners, p.epoch, true } -// cachePut records a positive result, unless a Forget bumped the epoch since -// the fan-out began — that delete may have removed the very record this -// result names. A miss is never cached: a checkpoint appearing moments later -// must not hide behind a stale negative for the TTL. +// cachePut records a positive result unless a Forget bumped the epoch since the fan-out began. func (p *HTTPProber) cachePut(id string, owners []string, start uint64) { if len(owners) == 0 { return @@ -197,24 +165,19 @@ func (p *HTTPProber) cachePut(id string, owners []string, start uint64) { p.cache[id] = redirectCacheEntry{owners: owners, expires: now.Add(cmp.Or(p.cacheTTL, redirectCacheTTL))} } -// DeriveProbeKey derives the probe-MAC key from the mesh's cluster_key via -// one HMAC step — domain separation from the gossip encryption. Call only -// with a non-empty clusterKey. +// DeriveProbeKey derives the probe-MAC key from the mesh cluster_key, separated from gossip. func DeriveProbeKey(clusterKey []byte) []byte { mac := hmac.New(sha256.New, clusterKey) mac.Write([]byte("cocoon-checkpoint-probe-v1")) return mac.Sum(nil) } -// SignProbe returns id's current probe MAC keyed by key — the ProbeHeader -// value; exported for callers and tests that build a valid probe request. +// SignProbe returns id's current probe MAC keyed by key — the ProbeHeader value. func SignProbe(key []byte, id string) string { return probeMAC(key, id, currentBucket()) } -// VerifyProbeMAC checks sig against id's current time bucket and the one on -// either side, tolerating clock skew up to one bucket without an unbounded -// replay window. +// VerifyProbeMAC checks sig against id's current time bucket and the one on either side. func VerifyProbeMAC(key []byte, id, sig string) bool { if sig == "" { return false @@ -242,8 +205,7 @@ func currentBucket() int64 { return time.Now().Unix() / probeBucketSeconds } -// dedupAddrs drops repeated addresses so a stale or duplicated peer list -// never probes the same node twice. +// dedupAddrs drops repeated addresses so a stale peer list never probes the same node twice. func dedupAddrs(addrs []string) []string { seen := make(map[string]struct{}, len(addrs)) out := make([]string, 0, len(addrs)) @@ -257,9 +219,7 @@ func dedupAddrs(addrs []string) []string { return out } -// probeOwner sends a probe MAC when probeKey is set (see DeriveProbeKey); -// an unencrypted mesh (empty probeKey) falls back to the older -// capability-only posture -- the id is unguessable, but nothing else guards it. +// probeOwner sends a probe MAC when probeKey is set; an empty key falls back to capability-only. func probeOwner(ctx context.Context, client *http.Client, probeKey []byte, addr, id string) bool { ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 5d5d93b8..f76fe3ac 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -9,8 +9,6 @@ import ( "time" ) -// TestOwnersReturnsOnly200: a mixed 200/404 answer set redirects only to the -// peer that actually holds the record. func TestOwnersReturnsOnly200(t *testing.T) { ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) @@ -28,7 +26,6 @@ func TestOwnersReturnsOnly200(t *testing.T) { } } -// TestOwnersAllMissIsEmpty: nobody answering 200 means nobody to redirect to. func TestOwnersAllMissIsEmpty(t *testing.T) { miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) @@ -41,9 +38,6 @@ func TestOwnersAllMissIsEmpty(t *testing.T) { } } -// TestOwnersHungPeerExcludedWithoutBlockingOthers: a wedged peer must not -// stall the whole probe past its own timeout, nor keep a healthy peer's -// answer from coming back. func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { block := make(chan struct{}) hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { @@ -67,10 +61,6 @@ func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { } } -// TestOwnersReturnsPromptlyWithOneOwner: the common case is a single owner, so -// the fan-out must return shortly after that owner answers on the grace window, -// not block on a slow peer's full timeout the way it would waiting for a cap it -// will never reach. func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { block := make(chan struct{}) hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { @@ -94,10 +84,6 @@ func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { } } -// TestForgetDuringFlightPreventsStaleCache: a delete that lands while a probe -// is in flight must keep that probe's result out of the cache — the record it -// names may be the one just deleted, and a cached positive would redirect -// claims to a node that no longer holds it. func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { release := make(chan struct{}) inFlight := make(chan struct{}) @@ -106,7 +92,7 @@ func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) probed() - <-release // hold the probe open so Forget can land mid-flight + <-release w.WriteHeader(http.StatusOK) })) defer owner.Close() @@ -117,21 +103,20 @@ func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { go func() { owners = p.Owners(t.Context(), testID); close(done) }() <-inFlight - p.Forget(testID) // the delete lands while the flight is open + p.Forget(testID) close(release) <-done if len(owners) != 1 { t.Fatalf("owners = %v, want the one that answered", owners) } - // The result must not have been cached: a second call re-probes. + p.Owners(t.Context(), testID) if got := hits.Load(); got != 2 { t.Errorf("owner probed %d times, want 2: a Forget mid-flight must not leave a stale positive cached", got) } } -// TestOwnersDedupsAddrs: a duplicated peer list must probe each address once. func TestOwnersDedupsAddrs(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -150,8 +135,6 @@ func TestOwnersDedupsAddrs(t *testing.T) { } } -// TestOwnersCapsAtThree: a redirect answer must stay small even with a large -// fleet behind the record. func TestOwnersCapsAtThree(t *testing.T) { var addrs []string for range 4 { @@ -168,8 +151,6 @@ func TestOwnersCapsAtThree(t *testing.T) { } } -// TestOwnersCancelsStragglersOnceCapReached: once the redirect cap is met by -// fast peers, a straggler beyond it must not add its own timeout to the result. func TestOwnersCancelsStragglersOnceCapReached(t *testing.T) { var addrs []string for range maxRedirectOwners { @@ -198,8 +179,6 @@ func TestOwnersCancelsStragglersOnceCapReached(t *testing.T) { } } -// TestHealOwnersReturnsMoreThanRedirectCap: a heal wants the widest source -// list, not the redirect's couple of candidates. func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { const want = maxRedirectOwners + 2 var addrs []string @@ -217,16 +196,13 @@ func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { } } -// TestHealOwnersIsExhaustiveDespiteAFastOwner: a heal must collect every owner -// even when one answers fast, so it does not stop on the grace window a redirect -// uses and hide slower valid sources behind the quick one. func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { fast := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer fast.Close() slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - time.Sleep(2 * probeGrace) // answers well after the redirect grace would have closed + time.Sleep(2 * probeGrace) w.WriteHeader(http.StatusOK) })) defer slow.Close() @@ -237,13 +213,11 @@ func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { } } -// TestOwnersCoalescesConcurrentProbes: concurrent redirect probes for one id -// must fan out to each peer once, not once per caller. func TestOwnersCoalescesConcurrentProbes(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) - time.Sleep(50 * time.Millisecond) // widen the window for concurrent callers to overlap + time.Sleep(50 * time.Millisecond) w.WriteHeader(http.StatusOK) })) defer ok.Close() @@ -259,10 +233,6 @@ func TestOwnersCoalescesConcurrentProbes(t *testing.T) { } } -// TestOwnersCacheCollapsesRepeatedRedirects is the hot-checkpoint-behind-a- -// load-balancer case: a fleet of peers, one of them the owner. Ten -// sequential redirect probes for the same id, well within the TTL, must -// fan out to the fleet once total, not once per call. func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { const peers = 5 var hits atomic.Int32 @@ -290,8 +260,6 @@ func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { } } -// TestOwnersCacheExpires: past the TTL, a redirect must re-probe rather than -// serve a result that could be stale forever. func TestOwnersCacheExpires(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -309,8 +277,6 @@ func TestOwnersCacheExpires(t *testing.T) { } } -// TestForgetEvictsCachedEntry: after Forget, the next Owners call must -// re-probe rather than serve the (possibly now-stale) cached answer. func TestForgetEvictsCachedEntry(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -328,8 +294,6 @@ func TestForgetEvictsCachedEntry(t *testing.T) { } } -// TestForgetOtherIDLeavesCacheAlone: Forget must evict only the named id, -// not the whole cache. func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -347,8 +311,6 @@ func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { } } -// TestVerifyProbeMACRejectsWrongKey: a MAC signed with a different key must -// not verify. func TestVerifyProbeMACRejectsWrongKey(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) other := DeriveProbeKey([]byte("different-secret")) @@ -358,8 +320,6 @@ func TestVerifyProbeMACRejectsWrongKey(t *testing.T) { } } -// TestVerifyProbeMACRejectsWrongID: a MAC signed for a different id must not -// verify -- the id itself must be bound into the signature. func TestVerifyProbeMACRejectsWrongID(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) sig := probeMAC(key, testID, currentBucket()) @@ -368,7 +328,6 @@ func TestVerifyProbeMACRejectsWrongID(t *testing.T) { } } -// TestVerifyProbeMACRejectsEmptySignature: a missing header must not verify. func TestVerifyProbeMACRejectsEmptySignature(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) if VerifyProbeMAC(key, testID, "") { @@ -376,8 +335,6 @@ func TestVerifyProbeMACRejectsEmptySignature(t *testing.T) { } } -// TestVerifyProbeMACToleratesAdjacentBucket: a signature from the bucket -// just before or after now must still verify, tolerating clock skew. func TestVerifyProbeMACToleratesAdjacentBucket(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) for _, delta := range []int64{-1, 1} { @@ -388,9 +345,6 @@ func TestVerifyProbeMACToleratesAdjacentBucket(t *testing.T) { } } -// TestVerifyProbeMACRejectsOutsideWindow: a signature two buckets away is -// outside the tolerated skew and must be rejected -- the window is bounded, -// not an unlimited replay allowance. func TestVerifyProbeMACRejectsOutsideWindow(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) sig := probeMAC(key, testID, currentBucket()-2) diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index 8a78d9aa..9ec9c035 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -19,8 +19,7 @@ import ( ) const ( - // A checkpoint carries a guest memory image, so this is generous — but a - // wedged peer must fail the pull so the next owner is tried, not hang it. + // Generous for a guest memory image, but a wedged peer must fail so the next owner is tried. pullTimeout = 30 * time.Minute maxRecordBytes = 1 << 40 // 1 TiB @@ -28,10 +27,7 @@ const ( metaFile = "meta.json" exportPrefix = "export/" - // recordTrailer is written last, only after the whole export streamed, so a - // receiver that reaches EOF without it knows the transfer stopped early — a - // source-side walk or read error leaves a valid but short tar (Close still - // writes the footer), and that must not be published as a complete record. + // recordTrailer is written last, so EOF without it means the transfer stopped early. recordTrailer = ".record-complete" ) @@ -45,8 +41,7 @@ type Puller interface { // HTTPPuller pulls records over sandboxd's control-plane HTTP port. type HTTPPuller struct { - // A nil Client uses a default with no overall timeout: the per-pull - // deadline comes from the context, so a large record is not cut off. + // A nil Client uses a default with no overall timeout; the deadline comes from the context. Client *http.Client Token string } @@ -86,9 +81,7 @@ func (p *HTTPPuller) Pull(ctx context.Context, addr, id, dst string) error { return Untar(resp.Body, dst) } -// TarRecord streams a whole store record: export/ plus meta.json. Both are -// required — the claim path reads the meta's pool key and lineage before it -// ever touches the export, so an export-only copy fails every read. +// TarRecord streams a whole store record: export/ plus meta.json, both required by the claim path. func TarRecord(exportDir string, meta []byte, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() @@ -117,9 +110,7 @@ func TarRecord(exportDir string, meta []byte, w io.Writer) error { return tw.Close() } -// Untar writes a tar stream into dst. An authenticated peer is still not -// allowed to write outside the destination this node chose, so every entry -// name is validated against traversal. +// Untar writes a tar stream into dst, validating every entry name against traversal. func Untar(r io.Reader, dst string) error { tr := tar.NewReader(r) var written int64 @@ -178,8 +169,7 @@ func writeFile(target string, r io.Reader, mode os.FileMode) error { return f.Close() } -// checkpointURL builds the control-plane URL for id's record on addr — the -// one home for the scheme default a mesh member view omits. +// checkpointURL builds the control-plane URL for id's record on addr, defaulting the scheme. func checkpointURL(addr, id, suffix string) string { if !strings.Contains(addr, "://") { addr = "http://" + addr @@ -187,14 +177,12 @@ func checkpointURL(addr, id, suffix string) string { return addr + "/v1/checkpoints/" + url.PathEscape(id) + suffix } -// safeJoin resolves name under root, rejecting absolute paths and any name -// that would escape root. +// safeJoin resolves name under root, rejecting absolute paths and names that escape root. func safeJoin(root, name string) (string, error) { if name == "" || filepath.IsAbs(name) || strings.Contains(name, `\`) { return "", fmt.Errorf("untar: refusing entry %q", name) } - // Clean as-is, never as "/"+name: a leading slash absorbs a leading "..", - // rewriting ../escape to /escape instead of rejecting it. + // clean as-is, never "/"+name: a leading slash absorbs a leading ".." instead of rejecting it. clean := filepath.Clean(name) if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { return "", fmt.Errorf("untar: entry %q escapes destination", name) @@ -206,9 +194,7 @@ func safeJoin(root, name string) (string, error) { return target, nil } -// tarInto walks src into tw under prefix, emitting only regular files and -// directories: a symlink or device node would let the reader be steered outside -// its destination. +// tarInto emits only regular files and directories: a symlink would steer the reader outside dst. func tarInto(src, prefix string, tw *tar.Writer) error { err := filepath.Walk(src, func(path string, fi os.FileInfo, err error) error { if err != nil { diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index 93561919..15638394 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -13,10 +13,6 @@ import ( "testing" ) -// TestTarUntarRoundTrip is the transport's core promise: a record that goes out -// as a tar comes back byte-identical on the other node, directory structure -// included. A checkpoint is a guest memory image plus its meta — a transport -// that silently drops a file would produce a clone of the wrong state. func TestTarUntarRoundTrip(t *testing.T) { src := t.TempDir() files := map[string]string{ @@ -56,10 +52,6 @@ func TestTarUntarRoundTrip(t *testing.T) { } } -// TestUntarRejectsPathTraversal is the security invariant. A peer authenticates -// with the fleet token, but an authenticated peer is still not allowed to write -// outside the destination this node chose — a compromised or buggy one must not -// be able to overwrite the host's files. func TestUntarRejectsPathTraversal(t *testing.T) { for _, name := range []string{ "../escape", @@ -96,9 +88,6 @@ func TestUntarRejectsPathTraversal(t *testing.T) { } } -// TestUntarSkipsNonDataEntries keeps symlinks and device nodes out of a record. -// A record is data; honoring a symlink entry would be an indirect way to make a -// later write land outside the destination. func TestUntarSkipsNonDataEntries(t *testing.T) { var buf bytes.Buffer tw := tar.NewWriter(&buf) @@ -129,14 +118,12 @@ func TestUntarSkipsNonDataEntries(t *testing.T) { if _, err := os.Lstat(filepath.Join(dst, "evil-link")); err == nil { t.Error("symlink entry was materialized; non-data entries must be skipped") } - // The legitimate entry alongside it still lands. + if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { t.Errorf("meta.json missing after skipping the symlink: %v", err) } } -// TestTarSkipsSymlinksAtSource is the same rule on the sending side: a record -// that somehow contains a symlink must not put one on the wire. func TestTarSkipsSymlinksAtSource(t *testing.T) { src := t.TempDir() if err := os.WriteFile(filepath.Join(src, "real"), []byte("x"), 0o600); err != nil { @@ -162,9 +149,6 @@ func TestTarSkipsSymlinksAtSource(t *testing.T) { } } -// TestHTTPPullerNotFound maps a peer's 404 to ErrNotFound so the caller keeps -// trying the next owner instead of failing the whole heal. A gossiped view can -// be stale — one owner not having the record is normal, not an error. func TestHTTPPullerNotFound(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) @@ -178,8 +162,6 @@ func TestHTTPPullerNotFound(t *testing.T) { } } -// TestHTTPPullerPresentsToken proves the fleet token rides on the pull: the -// serving peer authorizes this exactly like any other control-plane read. func TestHTTPPullerPresentsToken(t *testing.T) { var gotAuth, gotPath string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -210,11 +192,6 @@ func TestHTTPPullerPresentsToken(t *testing.T) { } } -// TestTarRecordRoundTripsAWholeRecord reproduces a bug found in the cluster: -// the blob endpoint tarred only the export directory Fetch() returns, so a -// pulled record published cleanly and then failed every read as "unknown -// checkpoint" — the meta carrying the pool key and lineage was never shipped. -// A record on the wire must be exactly what the local store lays out. func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { exportDir := t.TempDir() for name, content := range map[string]string{ @@ -237,7 +214,6 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { t.Fatalf("Untar: %v", err) } - // meta.json at the record root — without it the record is unreadable. got, err := os.ReadFile(filepath.Join(dst, "meta.json")) if err != nil { t.Fatalf("meta.json missing from the shipped record: %v", err) @@ -245,7 +221,7 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { if string(got) != string(meta) { t.Errorf("meta.json = %q, want %q", got, meta) } - // export/ contents under the export prefix, not at the root. + for name, want := range map[string]string{ "memory-ranges": "guest-pages", "cow.raw": "disk", @@ -264,28 +240,21 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { } } -// TestUntarRejectsRecordWithoutCompletionMarker: a transfer cut short by a -// source-side walk/read error still yields a valid short tar (Close writes the -// footer regardless), so the receiver must reject a record that arrives without -// the trailer rather than publish an incomplete one. func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { var buf bytes.Buffer tw := tar.NewWriter(&buf) - // meta plus one complete export file, then the walk "fails" — no trailer. + _ = tw.WriteHeader(&tar.Header{Name: "meta.json", Mode: 0o600, Size: 4, Typeflag: tar.TypeReg}) _, _ = tw.Write([]byte(`{"a"`)) _ = tw.WriteHeader(&tar.Header{Name: "export/mem", Mode: 0o600, Size: 5, Typeflag: tar.TypeReg}) _, _ = tw.Write([]byte("bytes")) - _ = tw.Close() // valid footer, no trailer entry + _ = tw.Close() if err := Untar(&buf, t.TempDir()); err == nil { t.Error("Untar accepted a record with no completion marker (an incomplete transfer)") } } -// TestTarRecordUntarRoundTripComplete: the real send/receive pair carries the -// trailer, so a whole record is accepted and the marker itself is not written -// to disk (validateHealedCheckpoint would reject an unexpected entry). func TestTarRecordUntarRoundTripComplete(t *testing.T) { exportDir := t.TempDir() if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("state"), 0o600); err != nil { @@ -307,8 +276,6 @@ func TestTarRecordUntarRoundTripComplete(t *testing.T) { } } -// tarDir streams src's contents with no record layout, exercising tarInto and -// Untar on their own. func tarDir(src string, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 6985886e..3df01115 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -34,9 +34,7 @@ const ( uploadPartSizeBytes = 16 << 20 ) -// Config selects the bucket and, for MinIO/R2-style endpoints, the -// addressing mode. Credentials come from the standard AWS chain (env, -// IAM role, web identity) — never from sandboxd's config file. +// Config selects the bucket; credentials come from the standard AWS chain, never this config. type Config struct { Bucket string `json:"bucket"` Prefix string `json:"prefix,omitempty"` @@ -58,8 +56,7 @@ type transferManager interface { var _ store.Store = (*Store)(nil) -// Store stages locally under stagingRoot and publishes to the bucket; -// idRe names the instance's id namespace within the shared prefix. +// Store stages locally and publishes to the bucket; idRe names the instance's id namespace. type Store struct { client *awss3.Client tm transferManager @@ -75,8 +72,7 @@ func New(ctx context.Context, cfg Config, stagingRoot string, idRe *regexp.Regex if cfg.Bucket == "" { return nil, fmt.Errorf("s3 checkpoint store needs a bucket") } - // A prefix without its trailing slash would glue onto the id and make - // every record invisible to the delimiter listing. + // a prefix without its trailing slash hides every record from the delimiter listing. if cfg.Prefix != "" && !strings.HasSuffix(cfg.Prefix, "/") { cfg.Prefix += "/" } @@ -93,8 +89,7 @@ func New(ctx context.Context, cfg Config, stagingRoot string, idRe *regexp.Regex } o.UsePathStyle = cfg.ForcePathStyle }) - // Snapshot exports are hundreds of MB: multipart + concurrency keep a - // publish/fetch bandwidth-bound instead of latency-bound. + // snapshot exports are hundreds of MB: multipart concurrency keeps transfers bandwidth-bound. tm := transfermanager.New(client, func(o *transfermanager.Options) { o.PartSizeBytes = uploadPartSizeBytes o.Concurrency = 8 @@ -106,11 +101,6 @@ func (s *Store) Stage(id string) (string, error) { return os.MkdirTemp(s.staging, id+"-*.tmp") } -// Publish uploads every staged file, meta.json last: a lister only sees -// the record once its commit marker exists. Export objects live under a -// per-generation prefix derived from the meta bytes, so a re-publish never -// overwrites keys a concurrent Fetch of the old generation is reading — -// mixed-generation downloads become impossible, not just unlikely. func (s *Store) Publish(ctx context.Context, staging, id string) error { _, err := s.publish(ctx, staging, id, false) return err @@ -120,11 +110,6 @@ func (s *Store) PublishDigested(ctx context.Context, staging, id string) (string return s.publish(ctx, staging, id, true) } -// Fetch materializes the export into a local cache generation keyed by the -// record's meta hash: an unchanged record's repeat fetch is one meta GET, and -// a new generation never disturbs a directory an in-flight clone is reading -// (old generations are reaped at Delete and startup). Concurrent misses share -// one download; release is a no-op; a missing id is ErrNotFound. func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, string, func(), error) { meta, digest, err := s.readMeta(ctx, id) if err != nil { @@ -150,8 +135,7 @@ func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { } func (s *Store) Metas(ctx context.Context) ([][]byte, error) { - // Delimiter listing yields one CommonPrefix per record instead of - // walking every export object of both namespaces under the prefix. + // delimiter listing yields one CommonPrefix per record instead of every export object. var ids []string p := awss3.NewListObjectsV2Paginator(s.client, &awss3.ListObjectsV2Input{ Bucket: &s.bucket, Prefix: &s.prefix, Delimiter: aws.String("/"), @@ -203,14 +187,9 @@ func (s *Store) Delete(ctx context.Context, id string) error { return s.deleteKeys(ctx, []string{metaKey}) } -// SweepStaging clears local staging residue AND stale cache generations — -// it runs at startup, when no clone can be mid-flight. A crash between -// upload and meta.json leaves orphan objects invisible to Metas; an S3 -// lifecycle rule on the bucket reclaims those (documented in deploy). +// SweepStaging clears local staging residue at startup; objects orphaned by a crash before meta.json are reclaimed by the bucket's S3 lifecycle rule (see deploy). func (s *Store) SweepStaging() error { return utils.RemoveDirEntries(s.staging, nil) } -// SweepGenerations is a no-op: Delete reclaims committed S3 generations, and -// bucket lifecycle policy handles invisible upload orphans. func (s *Store) SweepGenerations() error { return nil } func (s *Store) readMeta(ctx context.Context, id string) ([]byte, string, error) { @@ -271,7 +250,7 @@ func (s *Store) publish(ctx context.Context, staging, id string, digested bool) if err = s.uploadReader(ctx, s.key(id, store.MetaFile), bytes.NewReader(metaRaw), int64(len(metaRaw)), metadata); err != nil { return "", err } - // Keep old generations because another node may have selected the previous meta; Delete reclaims them. + // keep old generations: another node may have selected the previous meta; Delete reclaims them. if err := os.RemoveAll(staging); err != nil { return "", err } @@ -427,8 +406,7 @@ func (s *Store) download(ctx context.Context, key, path string) error { return err } defer func() { _ = f.Close() }() - // DownloadObject, not GetObject: the WriterAt form downloads parts in - // parallel; GetObject's io.Reader is a single sequential stream. + // DownloadObject, not GetObject: the WriterAt form downloads parts in parallel. if _, err := s.tm.DownloadObject(ctx, &transfermanager.DownloadObjectInput{Bucket: &s.bucket, Key: &key, WriterAt: f}); err != nil { return fmt.Errorf("download %s: %w", key, err) } @@ -450,9 +428,7 @@ func (s *Store) deleteKeys(ctx context.Context, keys []string) error { return fmt.Errorf("delete %d objects: %w", len(chunk), err) } for _, e := range out.Errors { - // Strict backends report deleting an absent key per-entry where - // AWS succeeds silently; tolerating it keeps Delete retries - // convergent, matching single-object DeleteObject semantics. + // strict backends error on deleting an absent key where AWS succeeds silently. if code := aws.ToString(e.Code); code == "NoSuchKey" || code == "NoSuchVersion" { continue } diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index 9c70fd74..f80f3fc4 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -31,9 +31,6 @@ func TestS3BackendContract(t *testing.T) { storetest.RunContract(t, st) } -// TestDeleteRetryConverges: a Delete retried after a partial failure re-deletes -// keys already gone; strict backends answer NoSuchKey per entry and the retry -// must still converge instead of failing forever. func TestDeleteRetryConverges(t *testing.T) { const id = "ck_00000000000000bb" fake := &fakeS3{objects: map[string][]byte{ @@ -123,8 +120,6 @@ func TestDeleteRetainsMetaUntilExportsAreGone(t *testing.T) { } } -// TestFetchLegacyExportLayout: records published before per-generation -// export prefixes keep flat export/ keys; Fetch must fall back to them. func TestFetchLegacyExportLayout(t *testing.T) { const id = "ck_00000000000000aa" fake := &fakeS3{objects: map[string][]byte{ @@ -358,10 +353,6 @@ func TestRepublishRetainsGenerationSelectedByAnotherStore(t *testing.T) { } } -// TestS3BackendContractRealEndpoint runs the same contract against a real -// S3 implementation (MinIO on a testbed) when SANDBOX_S3_E2E names its -// endpoint — real list pagination, checksums, and path-style behavior the -// in-process fake cannot vouch for. func TestS3BackendContractRealEndpoint(t *testing.T) { endpoint := os.Getenv("SANDBOX_S3_E2E") if endpoint == "" { @@ -413,7 +404,7 @@ func newTestStore(t *testing.T, fake *fakeS3) *Store { t.Cleanup(ts.Close) t.Setenv("AWS_ACCESS_KEY_ID", "test") t.Setenv("AWS_SECRET_ACCESS_KEY", "test") - // Keep PutObject bodies plain instead of adding aws-chunked trailers the fake would need to decode. + t.Setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_required") t.Setenv("AWS_RESPONSE_CHECKSUM_VALIDATION", "when_required") st, err := New(t.Context(), Config{ diff --git a/sandboxd/store/store.go b/sandboxd/store/store.go index ebbaba7f..76dc203d 100644 --- a/sandboxd/store/store.go +++ b/sandboxd/store/store.go @@ -17,22 +17,15 @@ import ( const ( // ExportDir is the snapshot export under a record: /export. ExportDir = "export" - // MetaFile is the record's metadata: /meta.json. It is - // written/uploaded last, so a lister never sees a half-published - // record. + // MetaFile is the record's metadata, written last so a lister never sees a half-published record. MetaFile = "meta.json" ) var ( - // ErrNotFound is the one absence signal: backends normalize their native - // missing-record errors to it, so call sites can tell "gone" from a - // backend failure (which must never silently degrade to a cold boot). + // ErrNotFound is the absence signal every backend normalizes its missing-record error to. ErrNotFound = errors.New("record not found") - // The two id namespaces sharing one store root: backends filter listings - // by their instance's regexp, so checkpoints and templates coexist in the - // same directory/bucket without seeing each other. The pins also keep a - // crafted id from escaping the store's namespace. + // CheckpointIDRe and TemplateIDRe let both id namespaces share one store root. CheckpointIDRe = regexp.MustCompile(`^ck_[0-9a-f]{16}$`) TemplateIDRe = regexp.MustCompile(`^tp_[0-9a-f]{32}$`) ) @@ -41,47 +34,32 @@ var ( type Store interface { // Stage returns a writable staging directory whose Publish is atomic. Stage(id string) (string, error) - // Publish turns a staged directory into the record, replacing any - // previous generation atomically for listers and never disturbing one a - // concurrent Fetch pinned (generations are immutable and retained until - // Delete or a graced sweep). The caller serializes same-id operations - // in-process; request-path callers pass an uncancelable ctx so a started - // publish finishes. + // Publish atomically replaces the record's previous generation with the staged directory. Publish(ctx context.Context, staging, id string) error // PublishDigested applies Publish semantics and returns the export digest. PublishDigested(ctx context.Context, staging, id string) (string, error) - // Fetch materializes a record's snapshot export as a local directory - // cocoon can clone from, plus the meta and digest resolved on the way, - // and a release to hold until clone ends (the generation outlives it). - // The digest is empty for records written by Publish. + // Fetch materializes a record's export locally and returns a release to hold until the clone ends. Fetch(ctx context.Context, id string) (dir string, meta []byte, digest string, release func(), err error) - // ReadMeta returns a record's metadata, or an error when the record - // does not exist. + // ReadMeta returns a record's metadata, or an error when the record does not exist. ReadMeta(ctx context.Context, id string) ([]byte, error) - // Metas lists the metadata of every record in this instance's id - // namespace. + // Metas lists the metadata of every record in this instance's id namespace. Metas(ctx context.Context) ([][]byte, error) // Delete removes a record. Delete(ctx context.Context, id string) error // SweepStaging removes abandoned staging left by a crash mid-publish. SweepStaging() error - // SweepGenerations runs backend-specific generation retention without - // disturbing staging or active fetch caches. + // SweepGenerations runs generation retention without disturbing staging or fetch caches. SweepGenerations() error } -// CheckpointID, TemplateID, and TemplateHash are the one home for the -// ck_/tp_ id scheme the regexps above pin. +// CheckpointID, TemplateID, and TemplateHash are the one home for the ck_/tp_ id scheme. func CheckpointID(suffix string) string { return "ck_" + suffix } func TemplateID(hash string) string { return "tp_" + hash } func TemplateHash(id string) string { return strings.TrimPrefix(id, "tp_") } -// ExportGen names a record generation's export dir/prefix from its meta -// bytes — meta is unique per publish (checkpoint ids are fresh, template -// records carry created_at), so a re-publish never collides with a -// generation a concurrent Fetch is reading. +// ExportGen names a record generation's export dir from its meta bytes, unique per publish. func ExportGen(meta []byte) string { return ExportDir + "-" + ExportGenHash(meta) } func ExportGenHash(meta []byte) string { diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index b565381a..50125215 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -71,8 +71,7 @@ func RunContract(t *testing.T, st store.Store) { t.Fatalf("SweepStaging: %v", err) } - // Re-publish replaces: the second generation's export fully supersedes - // the first (a leftover first-generation file would corrupt a clone). + // re-publish replaces: a leftover first-generation file would corrupt a clone. second, err := st.Stage(id) if err != nil { t.Fatalf("Stage second: %v", err) diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 880ee138..3fbe93e9 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -4,8 +4,7 @@ import ( "time" ) -// TTLField is the requested-lease field shared by the claiming request -// bodies; zero means the server default. +// TTLField is the shared requested-lease field; zero means the server default. type TTLField struct { TTLSeconds int `json:"ttl_seconds,omitempty"` } @@ -15,25 +14,19 @@ func (f TTLField) TTL() time.Duration { return time.Duration(f.TTLSeconds) * time.Second } -// ClaimRequest is the wire body of POST /v1/claim. NoRedirect is set by the -// SDK on a claim it is retrying at a redirect target, so that node warm-or- -// provisions locally instead of bouncing the claim back on a stale view. +// ClaimRequest is the wire body of POST /v1/claim. type ClaimRequest struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` Size Size `json:"size,omitempty"` Volumes []Volume `json:"volumes,omitempty"` - // VolumesAttachOnly attaches every requested volume without mounting it: - // the workload finds the device by its serial and owns the mount contract. + // VolumesAttachOnly attaches every requested volume without mounting it. VolumesAttachOnly bool `json:"volumes_attach_only,omitempty"` TTLField NoRedirect bool `json:"no_redirect,omitempty"` - // RequirePromoted is carried from a promoted-volume redirect to make the - // target refuse a cold-image fallback when template gossip is stale. + // RequirePromoted makes the target refuse a cold-image fallback. RequirePromoted bool `json:"require_promoted,omitempty"` - // ClaimRef is an opaque caller reference (the aggregated apiserver passes - // the k8s "/") recorded on the claim so the read path can - // map a listed sandbox back to the name it was claimed under. + // ClaimRef is an opaque caller reference recorded on the claim. ClaimRef string `json:"claim_ref,omitempty"` } @@ -42,26 +35,21 @@ func (r ClaimRequest) Key() PoolKey { return PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } -// ClaimResponse is the wire reply of POST /v1/claim. A successful claim -// carries ID/Token/Deadline/OwnerAddr; a mesh miss carries Redirect (peer -// addresses to retry, MOVED-style), and the two are mutually exclusive. +// ClaimResponse is the wire reply of POST /v1/claim; success and Redirect are exclusive. type ClaimResponse struct { ID string `json:"id,omitempty"` Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` OwnerAddr string `json:"owner_addr,omitempty"` - // TemplateDigest is the content identity of the promoted-template export - // this claim was cloned from; empty for any other source. + // TemplateDigest is the content identity of the export this claim was cloned from. TemplateDigest string `json:"template_digest,omitempty"` - // FromCheckpoint names the checkpoint a branched claim was born from, - // so clients can reconstruct the checkpoint tree. + // FromCheckpoint names the checkpoint a branched claim was born from. FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []Volume `json:"volumes,omitempty"` Redirect []string `json:"redirect,omitempty"` - // RequirePromoted tells a redirecting client to preserve that requirement - // on its no_redirect retry. It is omitted from successful claims. + // RequirePromoted tells a redirecting client to preserve that requirement on retry. RequirePromoted bool `json:"require_promoted,omitempty"` } @@ -72,8 +60,7 @@ type VolumeInfo struct { SizeBytes int64 `json:"size_bytes"` Available bool `json:"available"` Nodes int `json:"nodes"` - // Writable reports whether the operator allows rw claims of this name; - // unset for a name known only from a peer's advertisement. + // Writable reports whether the operator allows rw claims of this name. Writable bool `json:"writable,omitempty"` } @@ -82,11 +69,7 @@ type VolumeListResponse struct { Volumes []VolumeInfo `json:"volumes"` } -// ForkRequest is the wire body of POST /v1/sandboxes/{id}/fork. The -// Authorization header carries an api or tenant token (forking creates node -// resources, like a claim); Token proves ownership of the source sandbox. -// TTLSeconds applies to every child; zero means the server default, and -// children never inherit the parent's remainder. +// ForkRequest is the wire body of POST /v1/sandboxes/{id}/fork. type ForkRequest struct { Token string `json:"token"` Count int `json:"count"` @@ -99,7 +82,6 @@ type ForkResponse struct { } // CheckpointRequest is the wire body of POST /v1/sandboxes/{id}/checkpoint. -// Auth mirrors fork: api/tenant token in Authorization, sandbox token here. type CheckpointRequest struct { Token string `json:"token"` Name string `json:"name,omitempty"` @@ -111,11 +93,9 @@ type CheckpointResponse struct { } // CheckpointClaimRequest is the wire body of POST /v1/checkpoints/{id}/claim. -// TTLSeconds semantics match a claim's; zero means the server default. type CheckpointClaimRequest struct { TTLField - // NoRedirect is set by a client retrying at a redirect target, so the retry - // resolves locally instead of bouncing between two nodes. + // NoRedirect makes the retry resolve locally instead of bouncing between two nodes. NoRedirect bool `json:"no_redirect,omitempty"` } @@ -124,23 +104,19 @@ type CheckpointListResponse struct { Checkpoints []Checkpoint `json:"checkpoints"` } -// PromoteRequest is the wire body of POST /v1/sandboxes/{id}/promote; auth -// mirrors ForkRequest (control token in the header, ownership in Token). +// PromoteRequest is the wire body of POST /v1/sandboxes/{id}/promote. type PromoteRequest struct { Token string `json:"token"` Template string `json:"template"` } -// PromoteResponse returns the template's full key and stable identity: -// templates are node-local, so a cluster client needs the exact key (and this -// node's address) to claim from or delete the template later. +// PromoteResponse returns the template's full key and stable identity. type PromoteResponse struct { Key PoolKey `json:"key"` ContentDigest string `json:"content_digest"` } -// PreviewRequest is the wire body of POST /v1/sandboxes/{id}/preview; auth -// mirrors ForkRequest (control token in the header, ownership in Token). +// PreviewRequest is the wire body of POST /v1/sandboxes/{id}/preview. type PreviewRequest struct { Token string `json:"token"` Port uint16 `json:"port"` diff --git a/sandboxd/types/bench_test.go b/sandboxd/types/bench_test.go index c29fc7a0..279e571e 100644 --- a/sandboxd/types/bench_test.go +++ b/sandboxd/types/bench_test.go @@ -6,8 +6,6 @@ import ( "time" ) -// BenchmarkSandboxTouch measures the relay hot-path last-activity stamp; the -// lock-free atomic store must stay flat under -cpu parallelism (no contention). func BenchmarkSandboxTouch(b *testing.B) { sb := &Sandbox{} b.RunParallel(func(pb *testing.PB) { @@ -17,8 +15,6 @@ func BenchmarkSandboxTouch(b *testing.B) { }) } -// BenchmarkSandboxTouchMutex is the pre-E1a baseline (mutex-guarded time.Time) -// for contrast: it degrades with parallelism where the atomic stays flat. func BenchmarkSandboxTouchMutex(b *testing.B) { var ( mu sync.Mutex diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 55fccda5..5d3fb180 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -31,8 +31,7 @@ const ( MaxClaimVolumes = 8 - // Also the guest `mount -o` option literals (engine.MountVolume): renaming - // the API vocabulary would silently change the mount flags. + // Also the guest `mount -o` option literals: renaming them changes the mount flags. VolumeModeRO = "ro" VolumeModeRW = "rw" @@ -42,9 +41,7 @@ const ( ) var ( - // NameRe pins caller-chosen names (templates, checkpoints, tenants), - // which ride in journal fields and metric labels: one conservative - // charset, also accepted by cocoon's snapshot naming. + // NameRe pins caller-chosen names to one conservative charset cocoon also accepts. NameRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) // VolumeNameRe is the virtio disk serial grammar shared with cocoon. VolumeNameRe = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,19}$`) @@ -65,8 +62,7 @@ var ( // NetShape selects whether the Cloud Hypervisor guest has a NIC. type NetShape string -// RestoreMode selects cocoon's clone memory-restore strategy; empty leaves -// cocoon's eager-copy default. +// RestoreMode selects cocoon's clone memory-restore strategy; empty is cocoon's default. type RestoreMode string // Validate accepts the empty default plus cocoon's known restore modes. @@ -82,8 +78,7 @@ func (m RestoreMode) Validate() error { // Size is a T-shirt resource tier. type Size string -// SizeSpec is the concrete allocation behind a tier. Memory is in cocoon flag -// units; MemoryBytes is the same figure for callers that report bytes. +// SizeSpec is the concrete allocation behind a tier; Memory is in cocoon flag units. type SizeSpec struct { CPU int Memory string @@ -96,33 +91,26 @@ func (s Size) Spec() (SizeSpec, bool) { return spec, ok } -// PoolKey identifies one warm pool. Every New() parameter that changes VM -// construction is an axis here. +// PoolKey identifies one warm pool. type PoolKey struct { Template string `json:"template"` Net NetShape `json:"net"` Size Size `json:"size"` } -// Capturable reports whether state capture (fork, checkpoint, promote) is -// allowed: the egress lane refuses it — a resumed capture opens an -// unlocked-NIC window before the tap lock can reapply. +// Capturable is false on the egress lane: a resumed capture opens an unlocked-NIC window. func (k PoolKey) Capturable() bool { return k.Net != NetEgress } -// Defaulted fills the wire defaults: the hardened no-NIC lane and the smallest -// tier — the one home for both the claim and pool-spec paths. +// Defaulted fills the wire defaults: the no-NIC lane and the smallest tier. func (k PoolKey) Defaulted() PoolKey { k.Net = cmp.Or(k.Net, NetNone) k.Size = cmp.Or(k.Size, SizeSmall) return k } -// Hash is a stable digest used in VM, snapshot, and golden-dir naming. -// 128 bits, not a short tag: Promote keys goldens by caller-chosen template -// names, so a targeted collision with a configured pool's hash must stay a -// second-preimage problem, never a brute-forceable one. +// Hash is a stable 128-bit digest for naming; a shorter tag would be brute-forceable. func (k PoolKey) Hash() string { sum := sha256.Sum256([]byte(k.Template + "|" + string(k.Net) + "|" + string(k.Size))) return hex.EncodeToString(sum[:16]) @@ -144,13 +132,6 @@ func (k PoolKey) Validate() error { return nil } -// TemplateGossipHash scopes a key hash to its owner for the gossip wire, so -// foreign templates never match an owner query; the tenant itself never travels. -func TemplateGossipHash(keyHash, tenant string) string { - sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) - return hex.EncodeToString(sum[:16]) -} - // Sandbox is the node-local record of one pooled or claimed VM. type Sandbox struct { ID string `json:"id"` @@ -160,52 +141,36 @@ type Sandbox struct { Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` - // Tenant names the owning tenant; empty means the operator (root) - // claimed it. Fork children inherit it from the parent. + // Tenant names the owning tenant; empty means the operator claimed it. Tenant string `json:"tenant,omitempty"` - // ClaimRef is the opaque caller reference recorded at claim time (the - // aggregated apiserver passes the k8s "/"), carried into - // the operator index so a listed sandbox maps back to its claim name. - // Empty for warm-pool, fork, and checkpoint-branch claims. + // ClaimRef is the opaque caller reference recorded at claim time; empty for pool claims. ClaimRef string `json:"claim_ref,omitempty"` // Volumes records the volumes successfully applied to this claim. Volumes []Volume `json:"volumes,omitempty"` VsockSocket string `json:"vsock_socket,omitempty"` - // TAP is the egress-lane NIC's host tap, captured at provision; empty on - // the none lane and on claims adopted from pre-tap journals. + // TAP is the egress-lane NIC's host tap; empty on the none lane. TAP string `json:"tap,omitempty"` - // HibernateSnap names the memory snapshot while the VM is hibernated; - // empty means running. + // HibernateSnap names the memory snapshot while the VM is hibernated; empty means running. HibernateSnap string `json:"hibernate_snap,omitempty"` - // PendingSnap is the journaled intent of a hibernate in flight: written - // before the engine stops the VM, cleared by the commit. Reconcile - // trusts it to adopt a hibernate whose commit never landed. + // PendingSnap is the journaled intent of a hibernate in flight, cleared by its commit. PendingSnap string `json:"pending_snap,omitempty"` - // ArchiveCk names the store checkpoint holding this sandbox's state while - // archived; empty means live or hibernated. While set, VMName/VsockSocket/ - // HibernateSnap are empty (no local VM) and Deadline is the retention deadline. + // ArchiveCk names the store checkpoint holding an archived sandbox; empty means local. ArchiveCk string `json:"archive_ck,omitempty"` - // FromCheckpoint names the checkpoint this sandbox branched from, for - // lineage; empty for pool and template claims. + // FromCheckpoint names the checkpoint this sandbox branched from; empty otherwise. FromCheckpoint string `json:"from_checkpoint,omitempty"` - // TemplateDigest identifies the promoted-template export this sandbox was - // initially cloned from; empty for every other provisioning source. + // TemplateDigest identifies the promoted-template export this sandbox was cloned from. TemplateDigest string `json:"template_digest,omitempty"` - // StaleSnap names a consumed wake snapshot a lagging journal still - // references; dropped once a later write lands. Guarded by Transition. + // StaleSnap names a consumed wake snapshot a lagging journal references; guarded by Transition. StaleSnap string `json:"-"` - // lastActivity is unix-nanos of the last data-plane connection, for the - // idle policy; lock-free, stamped on the relay hot path. Runtime-only, a - // restart resets it to adoption time. + // lastActivity is unix-nanos of the last data-plane connection, stamped lock-free. lastActivity atomic.Int64 - // Transition serializes hibernate/wake so concurrent wakes collapse onto - // one restore. Lock it before (never under) the manager mutex. + // Transition serializes hibernate/wake; lock it before (never under) the manager mutex. Transition sync.Mutex `json:"-"` } @@ -218,8 +183,7 @@ func (s *Sandbox) TouchAt(t time.Time) { s.lastActivity.Store(t.UnixNano()) } // LastSeen returns the last data-plane activity time. func (s *Sandbox) LastSeen() time.Time { return time.Unix(0, s.lastActivity.Load()) } -// Checkpoint is the record of a captured sandbox state: claims cloned from -// it branch off the exact captured moment. Node-local, like a template. +// Checkpoint is the record of a captured sandbox state; node-local, like a template. type Checkpoint struct { ID string `json:"id"` Name string `json:"name,omitempty"` @@ -228,14 +192,11 @@ type Checkpoint struct { Tenant string `json:"tenant,omitempty"` CreatedAt time.Time `json:"created_at"` - // Archive marks a lifecycle-internal wake image: hidden from listings - // and undeletable on every node sharing the store, not just the owner. + // Archive marks a lifecycle-internal wake image: hidden from listings and undeletable. Archive bool `json:"archive,omitempty"` } -// VMRecord is the subset of cocoon's VM records the control plane reads — -// the same shape backs `vm list --format json` rows and the lifecycle -// commands' `--output json` result. +// VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { State string `json:"state"` PID int `json:"pid"` @@ -262,21 +223,24 @@ type VMConfig struct { Name string `json:"name"` } -// Volume is one requested or applied dataset mount. Mount is empty before -// request validation and on an attach-only entry, whose device the caller -// mounts itself. Mode is normalized to "" (read-only) or VolumeModeRW. +// Volume is one dataset mount; Mode is normalized to "" (read-only) or VolumeModeRW. type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` Mode string `json:"mode,omitempty"` - // AttachOnly carries the claim's flag into the pool's own validation pass; - // past it the empty Mount is the signal, so it is never stored or sent. + // AttachOnly carries the claim's flag into validation; past it the empty Mount is the signal. AttachOnly bool `json:"-"` } // RW reports whether the entry asks for write access. func (v Volume) RW() bool { return v.Mode == VolumeModeRW } +// TemplateGossipHash scopes a key hash to its owner so the tenant never travels the wire. +func TemplateGossipHash(keyHash, tenant string) string { + sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) + return hex.EncodeToString(sum[:16]) +} + // ValidVolumeName reports whether name is a legal cocoon data-disk serial. func ValidVolumeName(name string) bool { return VolumeNameRe.MatchString(name) && !strings.HasPrefix(name, "cocoon-") @@ -315,16 +279,12 @@ func VolumeRWNames(volumes []Volume) []string { return names } -// VolumesAttachOnly reports whether request validation marked the set as the -// caller's to mount. +// VolumesAttachOnly reports whether the caller mounts the devices itself. func VolumesAttachOnly(volumes []Volume) bool { return slices.ContainsFunc(volumes, func(v Volume) bool { return v.AttachOnly }) } -// ValidateVolumes validates a request and returns detached entries with every -// default mount filled and every mode normalized. attachOnly instead leaves -// every mount empty for the caller to mount itself, and rejects an entry that -// carries one. The input is not modified. +// ValidateVolumes returns detached entries with defaults filled and modes normalized. func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { if len(volumes) > MaxClaimVolumes { return nil, fmt.Errorf("volumes must contain at most %d entries, got %d", MaxClaimVolumes, len(volumes)) diff --git a/sandboxd/utils/utils.go b/sandboxd/utils/utils.go index a6ab900b..733a6ea1 100644 --- a/sandboxd/utils/utils.go +++ b/sandboxd/utils/utils.go @@ -20,9 +20,7 @@ func CloseWrite(conn net.Conn) { } } -// WriteFileSync durably replaces path with data (temp + fsync + rename + dir -// fsync) so the rename survives a crash. The temp name is per-call unique, so -// concurrent writers to the same path can't race a shared temp into a lost rename. +// WriteFileSync durably replaces path with data (temp + fsync + rename + dir fsync). func WriteFileSync(path string, data []byte, perm os.FileMode) error { dir := filepath.Dir(path) f, err := os.CreateTemp(dir, filepath.Base(path)+".*.tmp") @@ -83,9 +81,7 @@ func RemoveDirEntries(dir string, match func(name string) bool) error { return nil } -// DecodeStrictJSON decodes one JSON value into v, refusing unknown fields, -// duplicated keys, and trailing data — for hand-edited operator input, where -// a typo must fail instead of silently changing what was configured. +// DecodeStrictJSON decodes one JSON value into v, refusing unknown fields and duplicates. func DecodeStrictJSON(raw []byte, v any) error { if err := rejectDuplicateKeys(json.NewDecoder(bytes.NewReader(raw))); err != nil { return err @@ -101,8 +97,7 @@ func DecodeStrictJSON(raw []byte, v any) error { return nil } -// rejectDuplicateKeys walks one JSON value and refuses objects that repeat a -// key at the same level. +// rejectDuplicateKeys refuses objects that repeat a key at the same level. func rejectDuplicateKeys(dec *json.Decoder) error { tok, err := dec.Token() if err != nil { @@ -120,10 +115,7 @@ func rejectDuplicateKeys(dec *json.Decoder) error { if keyErr != nil { return keyErr } - // Fold case: encoding/json matches struct fields case-insensitively, - // so "Methods" and "methods" target the same field and the second - // silently wins — treat them as one key. (The callers decode into - // structs, never into map[string]T where case is significant.) + // encoding/json matches struct fields case-insensitively, so fold case here. keyStr, _ := keyTok.(string) key := strings.ToLower(keyStr) if _, dup := seen[key]; dup { diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go index 4f3f0d2e..705ef5cf 100644 --- a/sdk/go/checkpoint_test.go +++ b/sdk/go/checkpoint_test.go @@ -9,8 +9,6 @@ import ( ) func TestCheckpointNewFollowsRedirect(t *testing.T) { - // The checkpoint's node no longer holds it and redirects; the retry - // must land on node B, bound there, exactly like Client.New's redirect. var nodeACalls int nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req checkpointClaimRequest @@ -48,8 +46,6 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { } func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { - // The probed owner transiently fails (500); once exhausted, New falls - // back to the origin, which this time fails definitively too. broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) @@ -87,8 +83,6 @@ func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { } func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { - // The only probed owner is mid-heal (503); once exhausted, New falls - // back to the origin, which heals (pulls the checkpoint) locally. busy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) @@ -124,8 +118,6 @@ func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { } } -// TestCheckpointNewRedirectNeverYieldsEmptyID: New must fail loudly, not -// return a Sandbox with an empty ID, when no redirect candidate answers. func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) @@ -144,8 +136,6 @@ func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { } func TestCheckpointNewSecondLevelRedirectFails(t *testing.T) { - // A compliant server never redirects a no_redirect retry; if one does - // anyway, it must be treated as a failed candidate rather than followed. nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) })) diff --git a/sdk/go/claimopts_test.go b/sdk/go/claimopts_test.go index c8570732..e224004a 100644 --- a/sdk/go/claimopts_test.go +++ b/sdk/go/claimopts_test.go @@ -5,8 +5,6 @@ import ( "testing" ) -// Checkpoint.New and Template.New pin the network lane and size to the -// snapshot, so WithNetwork/WithSize must fail loud rather than silently no-op. func TestSnapshotClaimRejectsPinnedAxes(t *testing.T) { ck := &Checkpoint{ID: "ck_1", c: &Client{}, addr: "node:1"} tpl := &Template{Name: "rt", c: &Client{}, addr: "node:1", net: "none", size: "small"} diff --git a/sdk/go/client.go b/sdk/go/client.go index 8d76f760..aef31e58 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -426,6 +426,7 @@ func (e *APIError) Error() string { func apiError(verb string, resp *http.Response) error { var er errorResponse _ = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&er) + _, _ = io.Copy(io.Discard, resp.Body) return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 9ad60447..e2232b58 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -2,6 +2,8 @@ package sandbox import ( "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" "reflect" @@ -120,8 +122,6 @@ func TestCloseReleases(t *testing.T) { } func TestNewFollowsRedirect(t *testing.T) { - // The first node redirects; the second answers with the claim. The SDK - // must follow transparently and end up owning the sandbox at node B. var nodeB *httptest.Server nodeB = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{ @@ -150,8 +150,6 @@ func TestNewFollowsRedirect(t *testing.T) { } func TestDeleteTemplateFollowsRedirect(t *testing.T) { - // The entry node answers with the owner's address; the SDK retries the - // delete there, once. var deletedAtB, gotNoRedirect bool nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodDelete { @@ -182,9 +180,6 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { } } -// TestDeleteTemplateRetryPolicy pins the walk semantics shared with the -// Python SDK: a 404 moves to the next owner, a served error surfaces at -// once (walking on would let a later 404 mask it). func TestDeleteTemplateRetryPolicy(t *testing.T) { status := func(code int, hit *bool) *httptest.Server { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -232,8 +227,6 @@ func TestDeleteTemplateRetryPolicy(t *testing.T) { } func TestRedirectSetsNoRedirect(t *testing.T) { - // The retry at a redirect target must carry no_redirect so the target - // warm-or-provisions instead of bouncing the claim back. var gotNoRedirect bool target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req claimRequest @@ -263,7 +256,6 @@ func TestRedirectSetsNoRedirect(t *testing.T) { } func TestRedirectTriesAllCandidates(t *testing.T) { - // First candidate is dead; the SDK must fall through to the second. good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_4", Token: "t"}) })) @@ -286,9 +278,6 @@ func TestRedirectTriesAllCandidates(t *testing.T) { } func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { - // Candidate one is wrong for this claim (403, definitive) but candidate - // two would still succeed — the per-candidate walk retries broadly, so - // one ill-suited candidate must not cost a candidate that would answer. forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) })) @@ -316,8 +305,6 @@ func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { } func TestRedirectFallbackHeals(t *testing.T) { - // The only candidate holds the record but is full (429); once it's - // exhausted, New must fall back to the origin itself, which heals. var fullCalls int full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fullCalls++ @@ -360,8 +347,6 @@ func TestRedirectFallbackHeals(t *testing.T) { } func TestRedirectFallbackAlsoFails(t *testing.T) { - // Every candidate is full (429); the origin fallback answers too, but - // with a definitive error, which must surface as the final error. var fullCalls int full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fullCalls++ @@ -389,8 +374,7 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { if !strings.Contains(err.Error(), "origin also unavailable") { t.Errorf("err %v, want the origin's failure surfaced", err) } - // The candidate's failure must survive into the message too: it is what - // says why the claim left the origin in the first place. + if !strings.Contains(err.Error(), "candidate full") { t.Errorf("err %v, want the candidate's failure surfaced too", err) } @@ -416,9 +400,6 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // A definitive 4xx is not worth trying another candidate, and - // not worth falling back to the origin either — the origin - // would fail the exact same way. bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(tt.code) _ = json.NewEncoder(w).Encode(errorResponse{Error: tt.want}) @@ -481,8 +462,6 @@ func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { } func TestLookupScatter(t *testing.T) { - // The owner is node B; the entry node A doesn't own the sandbox but lists - // B as a peer. Lookup must scatter to B and bind the handle there. var owner *httptest.Server owner = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/sandboxes/sb_1/owner" && r.Header.Get("Authorization") == "Bearer tok" { @@ -496,7 +475,7 @@ func TestLookupScatter(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/sandboxes/sb_1/owner": - w.WriteHeader(http.StatusNotFound) // not owned here + w.WriteHeader(http.StatusNotFound) case "/v1/peers": _ = json.NewEncoder(w).Encode(map[string]any{"peers": []string{strings.TrimPrefix(owner.URL, "http://")}}) } @@ -512,6 +491,19 @@ func TestLookupScatter(t *testing.T) { } } +func TestAPIErrorDrainsOversizedBody(t *testing.T) { + body := strings.NewReader(`{"error":"` + strings.Repeat("x", 3*4096) + `"}`) + err := apiError("claim", &http.Response{StatusCode: http.StatusTooManyRequests, Body: io.NopCloser(body)}) + + var he *APIError + if !errors.As(err, &he) || he.Status != http.StatusTooManyRequests { + t.Fatalf("apiError = %v, want a 429 APIError", err) + } + if body.Len() != 0 { + t.Errorf("apiError left %d bytes unread; the transport cannot reuse the connection", body.Len()) + } +} + func testClient(t *testing.T, ts *httptest.Server, opts ...ClientOption) *Client { t.Helper() c, err := Connect(strings.TrimPrefix(ts.URL, "http://"), opts...) diff --git a/sdk/go/files_test.go b/sdk/go/files_test.go index df56365a..e319c8ef 100644 --- a/sdk/go/files_test.go +++ b/sdk/go/files_test.go @@ -77,7 +77,7 @@ func TestSessionLifecycle(t *testing.T) { if len(ids) != 1 || ids[0] != sess.ID { t.Errorf("sessions %v, want [%s]", ids, sess.ID) } - // exec-in-session runs argv (the fake's echo handler). + out, err := sess.Exec(ctx, "echo", "hi") if err != nil { t.Fatalf("session exec: %v", err) @@ -109,8 +109,6 @@ func TestReadFileMultiChunk(t *testing.T) { } } -// fakeSandbox wires a Sandbox whose data plane is served by a temp-dir-backed -// silkd Fake behind a hijacking agent endpoint. func fakeSandbox(t *testing.T) *Sandbox { t.Helper() fake := silkdtest.NewFake(t.TempDir()) diff --git a/sdk/go/port_test.go b/sdk/go/port_test.go index 2adb57ba..cfd943a7 100644 --- a/sdk/go/port_test.go +++ b/sdk/go/port_test.go @@ -22,7 +22,7 @@ func TestDialPortEchoAndHalfClose(t *testing.T) { if _, err := io.ReadFull(pc, buf); err != nil || string(buf) != "ping" { t.Fatalf("read %q, %v — want the echo", buf, err) } - // Half-close: the fake's server closes, surfacing as EOF. + if err := pc.CloseWrite(); err != nil { t.Fatalf("close write: %v", err) } diff --git a/sdk/go/proc_test.go b/sdk/go/proc_test.go index ee6280e0..1a984a86 100644 --- a/sdk/go/proc_test.go +++ b/sdk/go/proc_test.go @@ -15,12 +15,12 @@ func TestProcVerbs(t *testing.T) { "ps": {`{"type":"procs","procs":[{"pid":41,"argv":["sleep","30"],"detached":true,"state":"running","started_at_epoch_secs":1}]}`}, "kill": {`{"type":"done"}`}, "logs": { - `{"type":"stdout","data":"aGVsbG8="}`, // "hello" - `{"type":"stderr","data":"b29wcw=="}`, // "oops" - `{"type":"done"}`, // still running: no exit + `{"type":"stdout","data":"aGVsbG8="}`, + `{"type":"stderr","data":"b29wcw=="}`, + `{"type":"done"}`, }, "attach": { - `{"type":"stdout","data":"bGF0ZQ=="}`, // "late" + `{"type":"stdout","data":"bGF0ZQ=="}`, `{"type":"exit","code":7}`, }, } @@ -75,8 +75,6 @@ func TestProcVerbsUnknownPid(t *testing.T) { } } -// procServe answers each RPC with a canned frame script keyed by op — the -// proc verbs are pure frame plumbing, so a scripted fake covers them. func procServe(script map[string][]string) func(net.Conn) { return func(conn net.Conn) { defer func() { _ = conn.Close() }() diff --git a/sdk/go/pty.go b/sdk/go/pty.go index a3a31624..8ebc7253 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -38,12 +38,18 @@ func (p *Pty) Read(b []byte) (int, error) { return p.out.Read(b) } -// Write feeds input to the terminal. +// Write feeds input to the terminal, chunked to stay under the frame cap. func (p *Pty) Write(b []byte) (int, error) { - if err := p.conn.Send(&wire.Stdin{Data: b}); err != nil { - return 0, err + sent := 0 + for len(b) > 0 { + chunk := b[:min(len(b), stdinChunk)] + if err := p.conn.Send(&wire.Stdin{Data: chunk}); err != nil { + return sent, err + } + sent += len(chunk) + b = b[len(chunk):] } - return len(b), nil + return sent, nil } // Resize adjusts the terminal window; it is a separate RPC keyed by pid. diff --git a/sdk/go/pty_test.go b/sdk/go/pty_test.go index b65e19b2..f7eb5547 100644 --- a/sdk/go/pty_test.go +++ b/sdk/go/pty_test.go @@ -2,10 +2,16 @@ package sandbox import ( "bufio" + "bytes" "context" "errors" + "fmt" "io" + "net" "testing" + + "github.com/cocoonstack/sandbox/protocol/wire" + "github.com/cocoonstack/sandbox/sdk/go/silkd" ) func TestPtyEchoAndExit(t *testing.T) { @@ -88,3 +94,48 @@ func TestPtyCloseUnblocksUnreadOutput(t *testing.T) { t.Errorf("stop called %d times, want 1", stops) } } + +func TestPtyWriteChunksOversizedInput(t *testing.T) { + client, server := net.Pipe() + defer func() { _ = client.Close() }() + p := &Pty{conn: silkd.NewConn(client)} + payload := bytes.Repeat([]byte("x"), stdinChunk*2+1) + + errc := make(chan error, 1) + go func() { + n, err := p.Write(payload) + if err == nil && n != len(payload) { + err = fmt.Errorf("Write returned %d, want %d", n, len(payload)) + } + errc <- err + }() + + sc := wire.NewFrameScanner(server) + var got []byte + for len(got) < len(payload) { + if !sc.Scan() { + t.Fatalf("scan stdin frame: %v", sc.Err()) + } + if len(sc.Bytes()) > wire.MaxFrame { + t.Fatalf("frame of %d bytes exceeds the %d cap", len(sc.Bytes()), wire.MaxFrame) + } + req, err := wire.DecodeRequest(sc.Bytes()) + if err != nil { + t.Fatalf("decode stdin frame: %v", err) + } + in, ok := req.(*wire.Stdin) + if !ok { + t.Fatalf("frame %q, want stdin", req.Op()) + } + if len(in.Data) > stdinChunk { + t.Fatalf("stdin frame carries %d bytes, want at most %d", len(in.Data), stdinChunk) + } + got = append(got, in.Data...) + } + if err := <-errc; err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Errorf("reassembled %d bytes, want the %d written", len(got), len(payload)) + } +} diff --git a/sdk/go/sandbox_test.go b/sdk/go/sandbox_test.go index 594b24a0..d879c9af 100644 --- a/sdk/go/sandbox_test.go +++ b/sdk/go/sandbox_test.go @@ -78,9 +78,6 @@ func TestRunContextCancel(t *testing.T) { } func TestUpgradeKeepsCoalescedBytes(t *testing.T) { - // The server flushes the 101 and every response frame in a single write - // before reading the request; the frames ride into the handshake reader - // and must survive. mux := http.NewServeMux() mux.HandleFunc("GET /v1/sandboxes/{id}/agent", func(w http.ResponseWriter, r *http.Request) { conn, _, err := http.NewResponseController(w).Hijack() @@ -139,9 +136,6 @@ func TestDialAgentRejectsControlChars(t *testing.T) { } } -// newAgentServer serves the agent endpoint by hijacking and handing the conn -// to serve (a fake silkd), mirroring sandboxd's relay from the SDK's -// perspective. func newAgentServer(t *testing.T, serve func(net.Conn)) *httptest.Server { t.Helper() mux := http.NewServeMux() diff --git a/sdk/go/silkd/conn_test.go b/sdk/go/silkd/conn_test.go index 808a5cd0..d934c814 100644 --- a/sdk/go/silkd/conn_test.go +++ b/sdk/go/silkd/conn_test.go @@ -13,9 +13,6 @@ import ( "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" ) -// TestSendBulkMatchesJSONEncoding pins the hand-built data/stdin envelope to -// byte-for-byte identity with the json.Marshal path — the fast path must not -// drift from the wire contract silkd parses. func TestSendBulkMatchesJSONEncoding(t *testing.T) { for _, req := range []wire.Request{ &wire.Data{Data: []byte("hello\x00\xff binary")}, diff --git a/sdk/go/silkd/silkdtest/fake.go b/sdk/go/silkd/silkdtest/fake.go index 4ff21542..eb424aa8 100644 --- a/sdk/go/silkd/silkdtest/fake.go +++ b/sdk/go/silkd/silkdtest/fake.go @@ -242,7 +242,12 @@ func (f *Fake) fsPull(conn net.Conn, path string) { return } _ = tw.Close() - send(conn, &wire.DataResp{Data: buf.Bytes()}) + data := buf.Bytes() + for len(data) > 0 { + n := min(readChunk, len(data)) + send(conn, &wire.DataResp{Data: data[:n]}) + data = data[n:] + } send(conn, wire.Done{}) } diff --git a/sdk/go/utils.go b/sdk/go/utils.go index 1fbd32d8..c334e78d 100644 --- a/sdk/go/utils.go +++ b/sdk/go/utils.go @@ -12,6 +12,12 @@ import ( // fsChunk matches silkd's BULK_CHUNK. const fsChunk = 256 * 1024 +// respPtr is a frame type's pointer form, so a non-Response T fails to compile. +type respPtr[T any] interface { + *T + wire.Response +} + // doneRPC sends a request that answers with Done or an error frame. func (s *Sandbox) doneRPC(ctx context.Context, req wire.Request) error { conn, done, err := s.call(ctx, req) @@ -46,17 +52,17 @@ func (s *Sandbox) downloadRPC(ctx context.Context, req wire.Request, sink func([ } // oneShotRPC sends req and returns its single typed reply frame. -func oneShotRPC[T any](ctx context.Context, s *Sandbox, req wire.Request) (*T, error) { +func oneShotRPC[T any, PT respPtr[T]](ctx context.Context, s *Sandbox, req wire.Request) (*T, error) { conn, done, err := s.call(ctx, req) if err != nil { return nil, err } defer done() - return expect[T](ctx, conn) + return expect[T, PT](ctx, conn) } // collectRPC sends req and gathers every streamed frame of type T until Done. -func collectRPC[T any](ctx context.Context, s *Sandbox, req wire.Request) ([]T, error) { +func collectRPC[T any, PT respPtr[T]](ctx context.Context, s *Sandbox, req wire.Request) ([]T, error) { conn, done, err := s.call(ctx, req) if err != nil { return nil, err @@ -68,9 +74,7 @@ func collectRPC[T any](ctx context.Context, s *Sandbox, req wire.Request) ([]T, if err != nil { return nil, err } - // Via any: a direct resp.(*T) assertion is rejected at compile time - // (*T is not known to implement Response). - if v, ok := any(resp).(*T); ok { + if v, ok := resp.(PT); ok { out = append(out, *v) continue } @@ -136,12 +140,12 @@ func terminalErr(ctx context.Context, conn *silkd.Conn) error { // expect reads one frame and requires it to be a *T, mapping an error frame // to a Go error. -func expect[T any](ctx context.Context, conn *silkd.Conn) (*T, error) { +func expect[T any, PT respPtr[T]](ctx context.Context, conn *silkd.Conn) (*T, error) { resp, err := recv(ctx, conn) if err != nil { return nil, err } - if v, ok := any(resp).(*T); ok { + if v, ok := resp.(PT); ok { return v, nil } if e, ok := resp.(*wire.ErrorResp); ok { diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index 64ad608e..483b8f1d 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -108,8 +108,7 @@ func TestTemplateNewVolumeClaimFollowsRedirect(t *testing.T) { if !got.NoRedirect || !got.RequirePromoted || !slices.Equal(got.Volumes, want) { t.Errorf("redirected claim = %+v, want promoted no_redirect with %v", got, want) } - // The redirect body carries no principal, so the bearer is what makes the - // peer resolve the same tenant — a dropped one would claim as the operator. + if originAuth != "Bearer tenant-token" || targetAuth != originAuth { t.Errorf("bearer at origin=%q at target=%q, want the caller's token on both hops", originAuth, targetAuth) } diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index 565cda92..9d533b66 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -13,6 +13,9 @@ from langchain_core.tools import StructuredTool from pydantic import BaseModel, Field +# Mirrors the MCP exec contract the tool description states. +CALL_TIMEOUT = 300.0 + class ExecInput(BaseModel): command: str = Field(description="shell command, run via sh -c inside the sandbox") @@ -38,7 +41,7 @@ class CocoonToolkit: def __init__(self, addr: str, api_token: str = "", template: str = "rt:24.04", net: str = "", ttl_seconds: int = 0, from_checkpoint: str = ""): - self._client = Client(addr, api_token=api_token) + self._client = Client(addr, api_token=api_token, timeout=CALL_TIMEOUT) self._template = template self._net = net self._ttl = ttl_seconds @@ -61,6 +64,7 @@ def get_tools(self) -> list[StructuredTool]: "Returns stdout; a non-empty stderr is appended as a 'stderr:' " "line and a non-zero status as an 'exit code: N' line; a " "command that prints nothing and exits 0 returns '(no output)'. " + "The call is cut off after 5 minutes. " "Files and installed packages persist across calls; environment " "variables and the working directory do not.", ExecInput, self._exec), diff --git a/sdk/python/cocoonsandbox/__init__.py b/sdk/python/cocoonsandbox/__init__.py index 46513e08..af31cf0a 100644 --- a/sdk/python/cocoonsandbox/__init__.py +++ b/sdk/python/cocoonsandbox/__init__.py @@ -11,7 +11,7 @@ from .checkpoint import Checkpoint from .client import Client -from .errors import APIError, ExitError, ProtocolError, SandboxError, SilkdError +from .errors import APIError, ExitError, ProtocolError, SandboxError, SilkdError, StreamTimeout from .sandbox import Lsp, PortConn, Pty, Sandbox, Session, Watcher from .template import Template @@ -28,6 +28,7 @@ "SandboxError", "Session", "SilkdError", + "StreamTimeout", "Template", "Watcher", ] diff --git a/sdk/python/cocoonsandbox/checkpoint.py b/sdk/python/cocoonsandbox/checkpoint.py index fb7bf569..0a52b866 100644 --- a/sdk/python/cocoonsandbox/checkpoint.py +++ b/sdk/python/cocoonsandbox/checkpoint.py @@ -46,11 +46,7 @@ def post(peer): return self._client._handle_from(addr, reply) def delete(self) -> None: - """Removes the checkpoint from its node and asks every peer that node - currently sees to drop any replica a heal pulled — best-effort eventual - cleanup, not a fleet-wide revocation. A peer that misses that broadcast - (offline, partitioned, or joined later) keeps serving branches from its - replica until the node's checkpoint_ttl_hours ages it out; with that - TTL at its default of 0 (keep forever), an unreachable peer's replica - has no cleanup bound at all.""" + """Removes the checkpoint and broadcasts the drop; cleanup is + best-effort eventual, bounded by checkpoint_ttl_hours (see + docs/sdk-python.md).""" self._client._request(self._addr, "DELETE", f"/v1/checkpoints/{self.id}", None, "delete checkpoint") diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 4a187622..4f3d2b65 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -60,7 +60,10 @@ def lookup(self, id: str, token: str) -> Sandbox: mesh peer concurrently, binding to whichever confirms ownership first — one dead peer must not cost its full timeout.""" def probe(addr: str) -> Sandbox: - reply = self._request(addr, "GET", f"/v1/sandboxes/{id}/owner", None, "owner", bearer=token) + # Bounded like _peers: a scatter loser must not hold a socket for + # the full client timeout after the winner has answered. + reply = self._request(addr, "GET", f"/v1/sandboxes/{id}/owner", None, "owner", + bearer=token, timeout=min(_PEERS_TIMEOUT, self.timeout)) return Sandbox(client=self, id=id, token=token, owner=reply.get("owner_addr") or addr) @@ -296,9 +299,8 @@ def attempt(addr): except APIError as origin_exc: # Both halves matter to whoever reads this: the peers' failure says # why the claim left the origin, the origin's why returning did not help. - origin_exc.message = f"{origin_exc.message} (after redirect targets failed: {exc.message})" - origin_exc.args = (f"{verb}: {origin_exc.message} (HTTP {origin_exc.status})",) - raise + combined = f"{origin_exc.message} (after redirect targets failed: {exc.message})" + raise APIError(verb, origin_exc.status, combined) from origin_exc def _scatter(addrs, probe): diff --git a/sdk/python/cocoonsandbox/conn.py b/sdk/python/cocoonsandbox/conn.py index e1db2dec..7daa9baf 100644 --- a/sdk/python/cocoonsandbox/conn.py +++ b/sdk/python/cocoonsandbox/conn.py @@ -6,9 +6,9 @@ import contextlib import socket from collections.abc import Iterator -from typing import TypeVar +from typing import BinaryIO, TypeVar -from .errors import APIError, ProtocolError, SilkdError +from .errors import APIError, ProtocolError, SilkdError, StreamTimeout from .frames import MAX_FRAME, decode_response, encode_request _CloseableT = TypeVar("_CloseableT", bound="_Closeable") @@ -27,7 +27,7 @@ def __exit__(self, *exc) -> None: class Conn(_Closeable): """A live frame stream to one sandbox's silkd, via the owner node.""" - def __init__(self, sock: socket.socket, reader): + def __init__(self, sock: socket.socket, reader: BinaryIO): self._sock = sock self._reader = reader @@ -35,14 +35,23 @@ def send(self, op: str, **fields) -> None: self._sock.sendall(encode_request(op, **fields)) def recv(self) -> dict: - """Returns the next frame; raises SilkdError on an error frame and - ProtocolError on EOF or an oversized line.""" - line = self._reader.readline(MAX_FRAME + 1) + """Returns the next frame; raises SilkdError on an error frame, + StreamTimeout when the socket times out, and ProtocolError on EOF, an + oversized line, or an undecodable one — every failure is typed.""" + try: + line = self._reader.readline(MAX_FRAME + 1) + except socket.timeout as exc: + raise StreamTimeout(f"read timed out: {exc}") from exc + except OSError as exc: + raise ProtocolError(f"read failed: {exc}") from exc if not line: raise ProtocolError("connection closed mid-stream") if len(line) > MAX_FRAME: raise ProtocolError("frame exceeds the 8MiB cap") - frame = decode_response(line) + try: + frame = decode_response(line) + except ValueError as exc: + raise ProtocolError(f"malformed frame: {exc}") from exc if frame["type"] == "error": raise SilkdError(frame.get("kind", "internal"), frame.get("message", "")) return frame diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 4b8f16eb..73da6dde 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -41,3 +41,8 @@ def __init__(self, code: int, stderr: str, stdout: str = ""): class ProtocolError(SandboxError): """The peer broke the frame protocol (unexpected frame, oversized line).""" + + +class StreamTimeout(SandboxError): + """A socket operation exceeded the client timeout; the connection is + desynced, so retry the call, not the frame.""" diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index a5baf14e..d536dde4 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -13,7 +13,7 @@ from .checkpoint import Checkpoint from .conn import Conn, _Closeable, dial_agent -from .errors import APIError, ExitError, ProtocolError +from .errors import APIError, ExitError, ProtocolError, StreamTimeout from .frames import BULK_CHUNK, FS_CHUNK from .template import Template @@ -341,14 +341,18 @@ def pump_out(): with contextlib.suppress(OSError): local.close() - threading.Thread(target=pump_out, daemon=True).start() + pump = threading.Thread(target=pump_out, daemon=True) + pump.start() try: while True: chunk = local.recv(FS_CHUNK) if not chunk: break guest.send(chunk) + # Half-close, then let the guest finish answering: closing here + # would cut the reply the local client is still waiting for. guest.close_write() + pump.join() finally: guest.close() @@ -366,7 +370,7 @@ def _drain_proc(self, op: str, pid: int, on_stdout, on_stderr) -> int | None: return _pump_stdio(conn, on_stdout, on_stderr) -class Session: +class Session(_Closeable): """A persistent shell inside the sandbox, addressed by id.""" def __init__(self, sandbox: Sandbox, id: str): @@ -395,7 +399,7 @@ def __iter__(self) -> Iterator[dict]: while True: try: frame = self._conn.recv() - except (ProtocolError, OSError, ValueError) as e: + except (ProtocolError, StreamTimeout) as e: self.error = e return if frame["type"] == "event": @@ -433,7 +437,7 @@ def close(self) -> None: self._conn.close() -class Lsp: +class Lsp(_Closeable): """A language server in the sandbox, spoken to over the relay. silkd is a broker: it pipes JSON-RPC bytes; the caller frames and correlates.""" @@ -452,6 +456,8 @@ def stop(self) -> None: """Kills the language server.""" self._sandbox._done_rpc("lsp_stop", server_id=self.server_id) + close = stop + class PortConn(_Closeable): """A byte stream to a guest port, relayed over the silkd connection.""" diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py new file mode 100644 index 00000000..8155508a --- /dev/null +++ b/sdk/python/tests/conftest.py @@ -0,0 +1,34 @@ +"""Fixtures shared by the cluster-behavior suites: in-process fake nodes and +addresses that refuse a connection.""" + +import socket +import threading +from http.server import HTTPServer + +import pytest +from test_client import FakeNode + + +@pytest.fixture +def spawn_node(): + servers = [] + + def spawn(routes): + handler = type("Node", (FakeNode,), {"routes": routes}) + server = HTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + servers.append(server) + return f"127.0.0.1:{server.server_port}" + + yield spawn + for server in servers: + server.shutdown() + + +@pytest.fixture +def dead_addr(): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return f"127.0.0.1:{port}" diff --git a/sdk/python/tests/test_checkpoint.py b/sdk/python/tests/test_checkpoint.py index 66a9d4a3..0a0bb7b8 100644 --- a/sdk/python/tests/test_checkpoint.py +++ b/sdk/python/tests/test_checkpoint.py @@ -1,41 +1,11 @@ """Checkpoint claim redirect: mirrors the cluster redirect follow tested for Client.new in test_fault_matrix.py, applied to Checkpoint.new.""" -import socket -import threading -from http.server import HTTPServer - import pytest -from test_client import FakeNode from cocoonsandbox import APIError, Client -@pytest.fixture -def spawn_node(): - servers = [] - - def spawn(routes): - handler = type("Node", (FakeNode,), {"routes": routes}) - server = HTTPServer(("127.0.0.1", 0), handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - servers.append(server) - return f"127.0.0.1:{server.server_port}" - - yield spawn - for server in servers: - server.shutdown() - - -@pytest.fixture -def dead_addr(): - sock = socket.socket() - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - sock.close() - return f"127.0.0.1:{port}" - - def test_checkpoint_new_follows_redirect(spawn_node): seen = [] @@ -61,8 +31,6 @@ def redirect(body, path): def test_checkpoint_new_all_candidates_fail(spawn_node): - # The probed owner transiently fails (500); once exhausted, new() falls - # back to the origin, which this time fails definitively too. path = "/v1/checkpoints/ck_2/claim" broken = spawn_node({("POST", path): lambda body, p: (500, {"error": "boom"})}) @@ -82,8 +50,6 @@ def entry_claim(body, p): def test_checkpoint_new_redirect_fallback_heals(spawn_node): - # The only probed owner is mid-heal (503); once exhausted, new() falls - # back to the origin, which heals (pulls the checkpoint) locally. path = "/v1/checkpoints/ck_5/claim" busy = spawn_node({("POST", path): lambda body, p: (503, {"error": "healing"})}) @@ -102,9 +68,6 @@ def entry_claim(body, p): def test_checkpoint_new_redirect_never_yields_empty_id(spawn_node, dead_addr): - # Regression: new() used to hand a bare redirect reply straight to - # _handle_from, producing a Sandbox with no id/token. It must now follow - # the redirect and raise when no candidate answers. entry = spawn_node({("POST", "/v1/checkpoints/ck_3/claim"): lambda body, path: (200, {"redirect": [dead_addr]})}) with pytest.raises(APIError): @@ -112,8 +75,6 @@ def test_checkpoint_new_redirect_never_yields_empty_id(spawn_node, dead_addr): def test_checkpoint_new_second_level_redirect_fails(spawn_node): - # A compliant server never redirects a no_redirect retry; if one does - # anyway, it must be treated as a failed candidate rather than followed. path = "/v1/checkpoints/ck_4/claim" node_b = spawn_node({("POST", path): lambda body, p: (200, {"redirect": ["127.0.0.1:1"]})}) entry = spawn_node({("POST", path): lambda body, p: (200, {"redirect": [node_b]})}) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 0da1356d..4d7b1806 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -121,8 +121,7 @@ def claim(body, path): FakeNode.routes[("POST", "/v1/claim")] = claim Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mode": "ro"}]) Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mode": ""}]) - # "ro" and "" both normalize to an omitted key, byte-identical to a v1 request. - assert seen[0]["volumes"] == seen[1]["volumes"] == [{"name": "imagenet"}] + assert seen[0]["volumes"] == seen[1]["volumes"] == [{"name": "imagenet"}], seen def test_claim_attaches_volumes_without_mounting(node): @@ -232,8 +231,6 @@ def test_volume_catalog(node): def test_volume_catalog_surfaces_writable(node): - # The server omits "writable" for read-only entries (json omitempty); the - # client passes both shapes through as-is, key present or absent. want = [ { "name": "scratch", diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index d6e5b22f..3bcaa512 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -5,41 +5,13 @@ error) — and lookup must not pay a hung or dead peer's full timeout.""" import socket -import threading import time -from http.server import HTTPServer import pytest -from test_client import FakeNode from cocoonsandbox import APIError, Client -@pytest.fixture -def spawn_node(): - servers = [] - - def spawn(routes): - handler = type("Node", (FakeNode,), {"routes": routes}) - server = HTTPServer(("127.0.0.1", 0), handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - servers.append(server) - return f"127.0.0.1:{server.server_port}" - - yield spawn - for server in servers: - server.shutdown() - - -@pytest.fixture -def dead_addr(): - sock = socket.socket() - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - sock.close() - return f"127.0.0.1:{port}" - - @pytest.fixture def hung_addr(): sock = socket.socket() @@ -64,8 +36,6 @@ def test_claim_redirect_skips_404_candidate(spawn_node): def test_claim_redirect_all_candidates_fail_then_origin_fallback_fails(spawn_node): - # Every candidate is a stale miss (404); once exhausted, new() falls - # back to the origin, which this time fails definitively too. a = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone a"})}) b = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone b"})}) @@ -81,15 +51,11 @@ def entry_claim(body, path): with pytest.raises(APIError) as exc: Client(entry).new("rt:24.04") assert exc.value.status == 409 and "origin also failed" in exc.value.message - # The last candidate's failure must survive into the message: it is what - # says why the claim left the origin in the first place. assert "gone b" in exc.value.message assert len(calls) == 2 and calls[1]["no_redirect"] is True def test_claim_redirect_all_candidates_fail_then_origin_heals(spawn_node): - # The only candidate holds the record but is full (429); once exhausted, - # new() falls back to the origin, which heals (provisions locally). full_calls = [] def full_claim(body, path): @@ -115,8 +81,6 @@ def entry_claim(body, path): @pytest.mark.parametrize(("status", "message"), [(409, "no egress"), (403, "tenant not allowed")]) def test_claim_redirect_definitive_error_skips_origin_fallback(spawn_node, status, message): - # A definitive 4xx is not worth trying another candidate, and not worth - # falling back to the origin either -- the origin would fail the same way. bad = spawn_node({("POST", "/v1/claim"): lambda body, path: (status, {"error": message})}) calls = [] @@ -157,9 +121,6 @@ def entry_claim(body, path): def test_claim_redirect_candidate_definitive_error_still_tries_next_candidate(spawn_node): - # Candidate one is wrong for this claim (403, definitive) but candidate - # two would still succeed -- the per-candidate walk retries broadly, so - # one ill-suited candidate must not cost a candidate that would answer. forbidden = spawn_node({("POST", "/v1/claim"): lambda body, path: (403, {"error": "tenant not allowed"})}) good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_3", "token": "t"})}) entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [forbidden, good]})}) diff --git a/sdk/python/tests/test_fixtures.py b/sdk/python/tests/test_fixtures.py index 4e9fd9b7..2f9b4f2a 100644 --- a/sdk/python/tests/test_fixtures.py +++ b/sdk/python/tests/test_fixtures.py @@ -10,7 +10,7 @@ from cocoonsandbox import frames -FIXTURES = pathlib.Path(__file__).parent / ".." / ".." / ".." / "protocol" / "wire" / "fixtures" / "v1" +FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" def fixture_files(prefix): diff --git a/sdk/python/tests/test_hardening.py b/sdk/python/tests/test_hardening.py index 985cd048..78311274 100644 --- a/sdk/python/tests/test_hardening.py +++ b/sdk/python/tests/test_hardening.py @@ -11,7 +11,6 @@ def test_dial_agent_rejects_control_chars_in_identity(): - # A CR/LF in id or token would split the hand-built HTTP request line. with pytest.raises(APIError): dial_agent("127.0.0.1:1", "sb\r\nInjected: 1", "tok", 0.5) with pytest.raises(APIError): @@ -23,7 +22,6 @@ def test_watcher_propagates_silkd_error(): guest_sock.sendall(json.dumps({"type": "error", "kind": "not_found", "message": "gone"}).encode() + b"\n") guest_sock.close() watcher = Watcher(Conn(client_sock, client_sock.makefile("rb"))) - # A real server error frame must surface, not read as a clean stream end. with pytest.raises(SilkdError): for _ in watcher: pass @@ -34,8 +32,6 @@ def test_watcher_ends_cleanly_on_garbage_frame(): guest_sock.sendall(b"not json at all\n") guest_sock.close() watcher = Watcher(Conn(client_sock, client_sock.makefile("rb"))) - # An undecodable frame ends iteration; it must not escape as a raw - # ValueError/JSONDecodeError from the public iterator. assert list(watcher) == [] @@ -46,8 +42,6 @@ def test_request_wraps_truncated_body_as_api_error(): def serve(): conn, _ = server.accept() conn.recv(4096) - # Declare more body than is sent: read() raises IncompleteRead, - # which is an HTTPException, not an OSError. conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort") conn.close() diff --git a/sdk/python/tests/test_wire_binding.py b/sdk/python/tests/test_wire_binding.py index a2edccba..08d74016 100644 --- a/sdk/python/tests/test_wire_binding.py +++ b/sdk/python/tests/test_wire_binding.py @@ -3,7 +3,11 @@ Conn over a socketpair, then asserts the emitted frame carries exactly the fixture's fields with the same values — both directions, so a misspelled field name AND a silently dropped field fail here; the generic encode -round-trip alone cannot see either.""" +round-trip alone cannot see either. + +Each CASES entry is (fixture stem, scripted guest replies, invoke(sb, fixture)); +UNSENT lists the fields no SDK call site emits by design, which silkd defaults +and the Go SDK omits too.""" import contextlib import json @@ -17,11 +21,8 @@ from cocoonsandbox.conn import Conn from cocoonsandbox.frames import PROTO_VERSION -FIXTURES = pathlib.Path(__file__).parent / ".." / ".." / ".." / "protocol" / "wire" / "fixtures" / "v1" +FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" -# op → (fixture stem, replies the fake guest scripts, invoke(sb, fixture)). -# Invocations pass the fixture's own values so the subset assertion binds -# every emitted field name and value to the corpus. CASES = [ ("req_exec", [{"type": "started", "pid": 7}, {"type": "exit", "code": 0}], lambda sb, f: sb.run(f["argv"], cwd=f["cwd"], env=f["env"], user=f["user"], session=f["session"])), @@ -94,7 +95,6 @@ ] -# Fields no SDK call site emits by design (silkd defaults them; Go omits too). UNSENT = {"req_session_create": {"id"}} @@ -124,7 +124,6 @@ def guest(): sent.append(json.loads(line)) for reply in replies: guest_sock.sendall(json.dumps(reply).encode() + b"\n") - # Drain follow-up frames (data/stdin streams) until EOF. while True: line = reader.readline() if not line: diff --git a/silkd/src/exec.rs b/silkd/src/exec.rs index ab16953e..d764249b 100644 --- a/silkd/src/exec.rs +++ b/silkd/src/exec.rs @@ -20,6 +20,8 @@ use crate::sysutil; /// open in a surviving grandchild; a foreground client stalled past this /// window can also lose the not-yet-drained pipe tail (bounded, rare). const POST_EXIT_DRAIN: Duration = Duration::from_secs(2); +/// How long an exited detached process stays in the table, so a late +/// `logs`/`attach` still finds its ring instead of a bare not_found. const REAP_DELAY: Duration = Duration::from_secs(300); /// Foreground output buffer before the child is backpressured. Bounds how far /// ahead of a slow client the child may run, not a loss threshold. diff --git a/silkd/src/find.rs b/silkd/src/find.rs index b069ae55..8ed37ef8 100644 --- a/silkd/src/find.rs +++ b/silkd/src/find.rs @@ -2,11 +2,13 @@ //! rewrites matches in named files. Regex handling lives here so agents pass a //! pattern as data rather than shell-quoting it through exec. +use std::io::Read; use std::path::{Path, PathBuf}; use regex::Regex; use tokio::fs; use tokio::io::AsyncWrite; +use tokio::sync::mpsc; use crate::proto::{self, ErrorKind, Response, err_frame}; @@ -14,6 +16,9 @@ use crate::proto::{self, ErrorKind, Response, err_frame}; /// grep-scale line scanning is for source trees, not blobs. const FIND_MAX_FILE: u64 = 8 * 1024 * 1024; +/// Match frames in flight between the walking thread and the writer. +const MATCH_QUEUE: usize = 256; + /// Streams `match` frames for every line under `path` matching `pattern`, /// terminated by `done`. `glob` narrows the walk to file names matching it /// (`*` and `?` wildcards); an invalid pattern is a bad-request error. @@ -32,42 +37,39 @@ pub async fn find( Some(Ok(re)) => Some(re), Some(Err(e)) => return proto::error_frame(w, ErrorKind::BadRequest, e.to_string()).await, }; - let mut stack = vec![PathBuf::from(path)]; - let mut root = true; - while let Some(dir) = stack.pop() { - let mut rd = match fs::read_dir(&dir).await { - Ok(rd) => rd, - Err(e) if root => return err_frame(w, &e, "read_dir").await, - Err(_) => continue, - }; - loop { - let ent = match rd.next_entry().await { - Ok(Some(ent)) => ent, - Ok(None) => break, - Err(e) if root => return err_frame(w, &e, "read_dir").await, - Err(_) => break, - }; - let Ok(ft) = ent.file_type().await else { - continue; - }; - let p = ent.path(); - if ft.is_dir() { - stack.push(p); - } else if ft.is_file() && name_matches(&p, name_re.as_ref()) { - scan_file(w, &re, &p).await?; - } + // One blocking-pool dispatch for the whole tree, not three per file. + let (tx, mut rx) = mpsc::channel::(MATCH_QUEUE); + let walk = + tokio::task::spawn_blocking(move || walk(PathBuf::from(path), &re, name_re.as_ref(), &tx)); + let mut batch = Vec::new(); + let mut buf = Vec::new(); + let mut failed = None; + while rx.recv_many(&mut batch, MATCH_QUEUE).await > 0 { + let sent = proto::write_frames(w, &mut buf, &batch).await; + batch.clear(); + if let Err(e) = sent { + failed = Some(e); + break; } - root = false; } - proto::write_frame(w, &Response::Done).await + drop(rx); + let walked = walk.await.map_err(std::io::Error::other)?; + match failed { + Some(e) => Err(e), + None => match walked { + Ok(()) => proto::write_frame(w, &Response::Done).await, + Err(e) => err_frame(w, &e, "read_dir").await, + }, + } } /// Rewrites every `pattern` match to `replacement` in each of `files`, /// streaming one `replaced` frame per file (with its match count) and a -/// terminal `done`. A read/write failure on one file ends the stream with an -/// error — files whose `replaced` frame already went out are committed (each -/// file is atomic; the list is not). An invalid pattern is rejected before any -/// file is touched. +/// terminal `done`. A file over the find size bound is skipped with a zero +/// count rather than read into memory. A read/write failure on one file ends +/// the stream with an error — files whose `replaced` frame already went out +/// are committed (each file is atomic; the list is not). An invalid pattern is +/// rejected before any file is touched. pub async fn replace( w: &mut W, files: Vec, @@ -79,23 +81,24 @@ pub async fn replace( Err(e) => return proto::error_frame(w, ErrorKind::BadRequest, e.to_string()).await, }; for file in files { - let body = match fs::read_to_string(&file).await { - Ok(body) => body, - Err(e) => return err_frame(w, &e, "read").await, - }; - // Closure replacer: counts while replacing in one pass; expand keeps - // the &str replacer's $n capture-group semantics. let mut count: u64 = 0; - let new = re.replace_all(&body, |caps: ®ex::Captures| { - count += 1; - let mut expanded = String::new(); - caps.expand(&replacement, &mut expanded); - expanded - }); - if count > 0 - && let Err(e) = crate::fs::write_atomic(Path::new(&file), new.as_bytes()).await - { - return err_frame(w, &e, "write").await; + if !oversized(&file).await { + let body = match fs::read_to_string(&file).await { + Ok(body) => body, + Err(e) => return err_frame(w, &e, "read").await, + }; + // expand keeps the &str replacer's $n capture-group semantics. + let new = re.replace_all(&body, |caps: ®ex::Captures| { + count += 1; + let mut expanded = String::new(); + caps.expand(&replacement, &mut expanded); + expanded + }); + if count > 0 + && let Err(e) = crate::fs::write_atomic(Path::new(&file), new.as_bytes()).await + { + return err_frame(w, &e, "write").await; + } } proto::write_frame( w, @@ -109,35 +112,76 @@ pub async fn replace( proto::write_frame(w, &Response::Done).await } -async fn scan_file( - w: &mut W, - re: &Regex, - path: &Path, -) -> std::io::Result<()> { - if fs::metadata(path) +async fn oversized(file: &str) -> bool { + fs::metadata(file) .await .is_ok_and(|m| m.len() > FIND_MAX_FILE) - { - return Ok(()); +} + +/// Walks `root` depth-first, sending a `match` frame per matching line. A +/// failure on the root propagates; deeper ones skip only that directory. +fn walk( + root: PathBuf, + re: &Regex, + name_re: Option<&Regex>, + tx: &mpsc::Sender, +) -> std::io::Result<()> { + let mut stack = vec![root]; + let mut root = true; + while let Some(dir) = stack.pop() { + let rd = match std::fs::read_dir(&dir) { + Ok(rd) => rd, + Err(e) if root => return Err(e), + Err(_) => continue, + }; + for ent in rd { + let ent = match ent { + Ok(ent) => ent, + Err(e) if root => return Err(e), + Err(_) => break, + }; + let Ok(ft) = ent.file_type() else { + continue; + }; + let p = ent.path(); + if ft.is_dir() { + stack.push(p); + } else if ft.is_file() && name_matches(&p, name_re) && !scan_file(&p, re, tx) { + return Ok(()); + } + } + root = false; } - let Ok(body) = fs::read_to_string(path).await else { - return Ok(()); // non-UTF8 / unreadable: skip, not an error + Ok(()) +} + +/// Scans one file, reporting whether the receiver is still listening. The size +/// bound comes off the open handle, so the check and the read see one file. +fn scan_file(path: &Path, re: &Regex, tx: &mpsc::Sender) -> bool { + let Ok(mut file) = std::fs::File::open(path) else { + return true; }; - let file = path.to_string_lossy().into_owned(); + if file.metadata().is_ok_and(|m| m.len() > FIND_MAX_FILE) { + return true; + } + let mut body = String::new(); + if file.read_to_string(&mut body).is_err() { + return true; + } + let name = path.to_string_lossy(); for (i, line) in body.lines().enumerate() { if re.is_match(line) { - proto::write_frame( - w, - &Response::Match { - file: file.clone(), - line: i as u64 + 1, - content: line.to_string(), - }, - ) - .await?; + let frame = Response::Match { + file: name.to_string(), + line: i as u64 + 1, + content: line.to_string(), + }; + if tx.blocking_send(frame).is_err() { + return false; + } } } - Ok(()) + true } fn name_matches(path: &Path, name_re: Option<&Regex>) -> bool { diff --git a/silkd/src/fs.rs b/silkd/src/fs.rs index 4705bc89..a804cb46 100644 --- a/silkd/src/fs.rs +++ b/silkd/src/fs.rs @@ -9,6 +9,7 @@ use std::time::UNIX_EPOCH; use tokio::fs; use tokio::io::{AsyncBufRead, AsyncWrite, AsyncWriteExt}; +use tokio::sync::mpsc; use crate::proto::{self, DirEntry, FileInfo, FileKind, Response, err_frame}; @@ -68,36 +69,25 @@ pub async fn read(w: &mut W, path: String) -> io::Result< /// Lists a directory as a stream of `entries` frames terminated by `done`, /// batched so an arbitrarily large directory can never exceed the frame cap. pub async fn list(w: &mut W, path: String) -> io::Result<()> { - let mut rd = match fs::read_dir(&path).await { - Ok(r) => r, - Err(e) => return err_frame(w, &e, "read_dir").await, - }; - let mut entries = Vec::new(); - loop { - match rd.next_entry().await { - Ok(None) => break, - Ok(Some(ent)) => { - let (kind, size) = match ent.metadata().await { - Ok(m) => (file_kind(&m), m.len()), - Err(_) => (FileKind::Other, 0), - }; - entries.push(DirEntry { - name: ent.file_name().to_string_lossy().into_owned(), - kind, - size, - }); - if entries.len() == LIST_BATCH { - let batch = std::mem::take(&mut entries); - proto::write_frame(w, &Response::Entries { entries: batch }).await?; - } - } - Err(e) => return err_frame(w, &e, "read_dir").await, + // One blocking-pool dispatch for the whole directory, not one per entry. + let (tx, mut rx) = mpsc::channel::>(2); + let scan = tokio::task::spawn_blocking(move || scan_dir(&path, &tx)); + let mut failed = None; + while let Some(entries) = rx.recv().await { + if let Err(e) = proto::write_frame(w, &Response::Entries { entries }).await { + failed = Some(e); + break; } } - if !entries.is_empty() { - proto::write_frame(w, &Response::Entries { entries }).await?; + drop(rx); + let scanned = scan.await.map_err(io::Error::other)?; + match failed { + Some(e) => Err(e), + None => match scanned { + Ok(()) => proto::write_frame(w, &Response::Done).await, + Err(e) => err_frame(w, &e, "read_dir").await, + }, } - proto::write_frame(w, &Response::Done).await } /// Writes `bytes` to `path` via a sibling temp file committed into place, so @@ -177,6 +167,30 @@ pub async fn rename(w: &mut W, from: String, to: String) } } +/// Reads `path` into `LIST_BATCH`-sized batches, stopping when the writer is gone. +fn scan_dir(path: &str, tx: &mpsc::Sender>) -> io::Result<()> { + let mut entries = Vec::new(); + for ent in std::fs::read_dir(path)? { + let ent = ent?; + let (kind, size) = match ent.metadata() { + Ok(m) => (file_kind(&m), m.len()), + Err(_) => (FileKind::Other, 0), + }; + entries.push(DirEntry { + name: ent.file_name().to_string_lossy().into_owned(), + kind, + size, + }); + if entries.len() == LIST_BATCH && tx.blocking_send(std::mem::take(&mut entries)).is_err() { + return Ok(()); + } + } + if !entries.is_empty() { + let _ = tx.blocking_send(entries); + } + Ok(()) +} + fn tmp_name(path: &str) -> String { format!("{path}.silkd-{}.tmp", crate::sysutil::tmp_suffix()) } diff --git a/silkd/src/git.rs b/silkd/src/git.rs index d52985bb..effd09a8 100644 --- a/silkd/src/git.rs +++ b/silkd/src/git.rs @@ -115,38 +115,36 @@ pub async fn branch( op: GitBranchOp, name: Option, ) -> std::io::Result<()> { - match op { - GitBranchOp::List => { - let out = git(&path, None, &["branch", "--format=%(refname:short)"]).await?; - if !out.status.success() { - return terminal(w, "branch", &out).await; - } - let branches: Vec = String::from_utf8_lossy(&out.stdout) - .lines() - .map(str::to_string) - .collect(); - let cur = git(&path, None, &["rev-parse", "--abbrev-ref", "HEAD"]).await?; - let current = String::from_utf8_lossy(&cur.stdout).trim().to_string(); - proto::write_frame(w, &Response::GitBranches { current, branches }).await - } - GitBranchOp::Create | GitBranchOp::Delete | GitBranchOp::Checkout => { - let Some(name) = name else { - return proto::write_frame( - w, - &Response::error(ErrorKind::BadRequest, "branch op needs a name"), - ) - .await; - }; - let args: Vec<&str> = match op { - GitBranchOp::Create => vec!["branch", &name], - GitBranchOp::Delete => vec!["branch", "-D", &name], - GitBranchOp::Checkout => vec!["checkout", &name], - GitBranchOp::List => unreachable!(), - }; - let out = git(&path, None, &args).await?; - terminal(w, "branch", &out).await + let args: Vec<&str> = match (op, &name) { + (GitBranchOp::List, _) => return list_branches(w, &path).await, + (_, None) => { + return proto::write_frame( + w, + &Response::error(ErrorKind::BadRequest, "branch op needs a name"), + ) + .await; } + (GitBranchOp::Create, Some(name)) => vec!["branch", name], + (GitBranchOp::Delete, Some(name)) => vec!["branch", "-D", name], + (GitBranchOp::Checkout, Some(name)) => vec!["checkout", name], + }; + let out = git(&path, None, &args).await?; + terminal(w, "branch", &out).await +} + +/// Reports every local branch and the checked-out one. +async fn list_branches(w: &mut W, path: &str) -> std::io::Result<()> { + let out = git(path, None, &["branch", "--format=%(refname:short)"]).await?; + if !out.status.success() { + return terminal(w, "branch", &out).await; } + let branches: Vec = String::from_utf8_lossy(&out.stdout) + .lines() + .map(str::to_string) + .collect(); + let cur = git(path, None, &["rev-parse", "--abbrev-ref", "HEAD"]).await?; + let current = String::from_utf8_lossy(&cur.stdout).trim().to_string(); + proto::write_frame(w, &Response::GitBranches { current, branches }).await } /// push/pull shared body: the egress guard, the bare verb, a terminal frame. @@ -327,7 +325,6 @@ mod tests { let untracked = parse_file_line("? build/out.o").unwrap(); assert_eq!(untracked.path, "build/out.o"); - // Unmerged entries (kind "u") must surface as conflicted, not vanish. let conflict = parse_file_line("u UU N... 100644 100644 100644 100644 h1 h2 h3 conflict.rs").unwrap(); assert_eq!(conflict.path, "conflict.rs"); diff --git a/silkd/src/lsp.rs b/silkd/src/lsp.rs index a7738c9d..3b94b1f5 100644 --- a/silkd/src/lsp.rs +++ b/silkd/src/lsp.rs @@ -9,19 +9,20 @@ //! server's stdin, as in port_forward); `lsp_stop` kills it. v1 is //! single-shot per server: the stream ending — clean or dropped — reaps the //! server, since an LSP stream loses frame sync on any mid-request cut and a -//! resynced reattach is not worth the failure surface. The id and `lsp_stop` +//! resynced reattach is not worth the failure surface. A server no +//! `lsp_request` ever attaches is reaped on a TTL. The id and `lsp_stop` //! still earn their place: `lsp_start` for several languages yields several //! ids attached concurrently, and `lsp_stop` tears one down early. The base //! image ships no manifests, so `lsp_start` for any language there answers a //! typed `not_found` naming the flavor that provides one. use std::collections::HashMap; -use std::os::unix::fs::OpenOptionsExt; use std::process::Stdio; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; -use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::process::{Child, ChildStdin, ChildStdout, Command}; use tokio::sync::mpsc; @@ -30,6 +31,10 @@ use crate::sysutil; const MANIFEST_DIR: &str = "/etc/silkd/lsp.d"; +/// A started server nothing attaches is reaped after this: a client that dies +/// between the calls must not pin a 1-2 GB language server for the VM's life. +const ATTACH_TTL: Duration = Duration::from_secs(300); + static NEXT_ID: AtomicU64 = AtomicU64::new(1); /// Table of running language servers, addressed by server id. @@ -51,7 +56,7 @@ impl Broker { language: &str, root: Option<&str>, ) -> std::io::Result<()> { - let argv = match read_manifest(language) { + let argv = match read_manifest(language).await { Some(argv) if !argv.is_empty() => argv, _ => { return proto::error_frame( @@ -82,13 +87,13 @@ impl Broker { .await; }; let server = Server { - child: Arc::new(tokio::sync::Mutex::new(child)), - stdin: Arc::new(tokio::sync::Mutex::new(Some(stdin))), - stdout: Arc::new(tokio::sync::Mutex::new(Some(stdout))), + child, + stdin: Some(stdin), + stdout: Some(stdout), }; let id = format!("lsp-{}", NEXT_ID.fetch_add(1, Ordering::Relaxed)); - crate::sysutil::lock(&self.inner).insert(id.clone(), server); + sysutil::lock(&self.inner).insert(id.clone(), server); // If the client is already gone, don't leak the server we just spawned. if let Err(e) = proto::write_frame( @@ -102,6 +107,11 @@ impl Broker { self.reap(&id).await; return Err(e); } + let broker = self.clone(); + tokio::spawn(async move { + tokio::time::sleep(ATTACH_TTL).await; + broker.reap_unattached(&id).await; + }); Ok(()) } @@ -115,21 +125,25 @@ impl Broker { w: &mut W, server_id: &str, ) -> std::io::Result<()> { - let found = crate::sysutil::lock(&self.inner).get(server_id).cloned(); - let server = match found { - Some(s) => s, + let taken = sysutil::lock(&self.inner) + .get_mut(server_id) + .map(|s| s.stdout.take().zip(s.stdin.take())); + let (stdout, stdin) = match taken { None => return proto::error_frame(w, ErrorKind::NotFound, "no such lsp server").await, - }; - let stdout = match server.stdout.lock().await.take() { - Some(out) => out, - None => { + Some(None) => { return proto::error_frame(w, ErrorKind::BadRequest, "lsp server already attached") .await; } + Some(Some(stdio)) => stdio, }; - proto::write_frame(w, &Response::Ready).await?; + // The stdio halves are out of the table, so the attach TTL no longer + // covers this server: a client already gone must reap it here. + if let Err(e) = proto::write_frame(w, &Response::Ready).await { + self.reap(server_id).await; + return Err(e); + } - let feed = tokio::spawn(feed_stdin(client, server.stdin.clone())); + let feed = tokio::spawn(feed_stdin(client, stdin)); let res = pump_stdout(stdout, w).await; // The session ends with this connection: stop feeding and reap the // server. Aborting is safe here — we are killing the child, so a @@ -155,50 +169,54 @@ impl Broker { /// reap removes a server from the table and kills + waits its child (no /// zombie survives), reporting whether it was there. async fn reap(&self, server_id: &str) -> bool { - let removed = crate::sysutil::lock(&self.inner).remove(server_id); + let removed = sysutil::lock(&self.inner).remove(server_id); + match removed { + Some(server) => { + kill(server).await; + true + } + None => false, + } + } + + /// reap for a server no `lsp_request` ever attached; one that is attached + /// (its stdout taken) belongs to that connection and is left alone. + async fn reap_unattached(&self, server_id: &str) { + let removed = { + let mut table = sysutil::lock(&self.inner); + match table.get(server_id) { + Some(server) if server.stdout.is_some() => table.remove(server_id), + _ => None, + } + }; if let Some(server) = removed { - let mut child = server.child.lock().await; - let _ = child.start_kill(); - let _ = child.wait().await; - true - } else { - false + kill(server).await; } } } -/// One running language server: child + stdio, shared so `lsp_request` reads -/// stdout, `feed_stdin` writes stdin, and `reap` kills the child. stdin is -/// Option so `feed_stdin` can drop it to half-close, like stdout for attach. -#[derive(Clone)] +/// One running language server. `lsp_request` takes both stdio halves, so a +/// present stdout is also the flag that no connection has attached yet. struct Server { - child: Arc>, - stdin: Arc>>, - stdout: Arc>>, + child: Child, + stdin: Option, + stdout: Option, +} + +async fn kill(mut server: Server) { + let _ = server.child.start_kill(); + let _ = server.child.wait().await; } -async fn feed_stdin( - mut client: mpsc::Receiver, - stdin: Arc>>, -) { +async fn feed_stdin(mut client: mpsc::Receiver, mut stdin: ChildStdin) { while let Some(req) = client.recv().await { - match req { - Request::Data { data } => { - let mut guard = stdin.lock().await; - let Some(w) = guard.as_mut() else { return }; - if w.write_all(&data).await.is_err() || w.flush().await.is_err() { - return; - } - } - // DataEnd half-closes, aligned with port_forward: dropping stdin - // delivers EOF so the server can flush remaining output and exit - // (stdout then EOFs -> Done -> reap). A stray non-data frame gets - // the same close — the writer belongs to the stdout pump, so - // there is no channel to answer an error frame on. - _ => { - stdin.lock().await.take(); - return; - } + // Anything but data half-closes by dropping stdin, aligned with + // port_forward: the server sees EOF, flushes, and exits (stdout then + // EOFs -> Done -> reap). A stray frame gets the same close, since the + // writer belongs to the stdout pump. + let Request::Data { data } = req else { return }; + if stdin.write_all(&data).await.is_err() || stdin.flush().await.is_err() { + return; } } } @@ -219,19 +237,20 @@ fn manifest_dir() -> String { std::env::var("SILKD_LSP_DIR").unwrap_or_else(|_| MANIFEST_DIR.to_string()) } -fn read_manifest(language: &str) -> Option> { +async fn read_manifest(language: &str) -> Option> { if language.is_empty() || language.contains(['/', '\\', '\0']) || language.contains("..") { return None; // never let a language name escape the manifest dir } // O_NOFOLLOW: a symlink planted inside lsp.d must not read a target // outside it. - let mut f = std::fs::OpenOptions::new() + let mut f = tokio::fs::OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW) .open(format!("{}/{language}", manifest_dir())) + .await .ok()?; let mut raw = String::new(); - std::io::Read::read_to_string(&mut f, &mut raw).ok()?; + f.read_to_string(&mut raw).await.ok()?; let argv: Vec = raw.split_whitespace().map(str::to_string).collect(); (!argv.is_empty()).then_some(argv) } diff --git a/silkd/src/net.rs b/silkd/src/net.rs index 0058de5e..b035fac3 100644 --- a/silkd/src/net.rs +++ b/silkd/src/net.rs @@ -3,7 +3,7 @@ //! all-builtin kernel auto-creates). Network verbs (git clone/push/pull) and //! proxy-env forwarding consult this. -use std::sync::OnceLock; +use std::sync::LazyLock; use std::sync::atomic::{AtomicI8, Ordering}; static LANE_OVERRIDE: AtomicI8 = AtomicI8::new(-1); @@ -28,13 +28,13 @@ pub fn has_egress() -> bool { _ => {} } // Probed once: the NIC set is fixed for the guest's life. - static DEVICE_BACKED: OnceLock = OnceLock::new(); - *DEVICE_BACKED.get_or_init(|| { + static DEVICE_BACKED: LazyLock = LazyLock::new(|| { let Ok(entries) = std::fs::read_dir("/sys/class/net") else { return true; }; entries.flatten().any(|e| e.path().join("device").exists()) - }) + }); + *DEVICE_BACKED } /// Lane override for tests: set_var would race every concurrent getenv. @@ -44,10 +44,11 @@ pub fn override_egress_for_tests(lane: Option) { /// SILKD_NET decoded once: operator config, fixed at service start. fn silkd_net() -> i8 { - static SILKD_NET: OnceLock = OnceLock::new(); - *SILKD_NET.get_or_init(|| match std::env::var("SILKD_NET").as_deref() { - Ok("none") => 0, - Ok("egress") => 1, - _ => -1, - }) + static SILKD_NET: LazyLock = + LazyLock::new(|| match std::env::var("SILKD_NET").as_deref() { + Ok("none") => 0, + Ok("egress") => 1, + _ => -1, + }); + *SILKD_NET } diff --git a/silkd/src/proc.rs b/silkd/src/proc.rs index 39021fb0..b6ca1388 100644 --- a/silkd/src/proc.rs +++ b/silkd/src/proc.rs @@ -249,17 +249,23 @@ impl Ring { } } + /// Retained output, coalescing adjacent same-stream segments: reads leave + /// tens of thousands of tiny ones and the client concatenates them anyway. fn drain_view(&mut self) -> Vec { let bytes = self.buf.make_contiguous(); - let mut out = Vec::with_capacity(self.tags.len()); + let mut out: Vec = Vec::new(); let mut off = 0; for &(stderr, len) in &self.tags { - let slice = bytes[off..off + len].to_vec(); - out.push(if stderr { - Chunk::Stderr(slice) - } else { - Chunk::Stdout(slice) - }); + let seg = &bytes[off..off + len]; + match out.last_mut() { + Some(Chunk::Stdout(acc)) if !stderr => acc.extend_from_slice(seg), + Some(Chunk::Stderr(acc)) if stderr => acc.extend_from_slice(seg), + _ => out.push(if stderr { + Chunk::Stderr(seg.to_vec()) + } else { + Chunk::Stdout(seg.to_vec()) + }), + } off += len; } out diff --git a/silkd/src/proto.rs b/silkd/src/proto.rs index 01615484..c3051513 100644 --- a/silkd/src/proto.rs +++ b/silkd/src/proto.rs @@ -413,16 +413,41 @@ fn cap_check(len: usize) -> io::Result<()> { Ok(()) } -pub async fn write_frame(w: &mut W, resp: &Response) -> io::Result<()> { - let mut buf = serde_json::to_vec(resp).map_err(io::Error::other)?; +/// Renders one newline-terminated frame onto `buf`, holding the write side to +/// the same cap the read side enforces — every client rejects a larger frame. +fn render_frame(buf: &mut Vec, resp: &Response) -> io::Result<()> { + let start = buf.len(); + serde_json::to_writer(&mut *buf, resp).map_err(io::Error::other)?; + cap_check(buf.len() - start)?; buf.push(b'\n'); + Ok(()) +} + +pub async fn write_frame(w: &mut W, resp: &Response) -> io::Result<()> { + let mut buf = Vec::new(); + render_frame(&mut buf, resp)?; w.write_all(&buf).await?; w.flush().await } +/// Writes `frames` as one buffered batch, so a burst costs one write syscall +/// instead of one per frame. `buf` is reused across calls. +pub async fn write_frames( + w: &mut W, + buf: &mut Vec, + frames: &[Response], +) -> io::Result<()> { + buf.clear(); + for frame in frames { + render_frame(buf, frame)?; + } + w.write_all(buf).await?; + w.flush().await +} + /// Renders `{"type":KIND,"data":""}` into `buf` (reused across calls) /// and writes it — the bulk path skips serde's owned Vec + String per chunk. -/// base64's alphabet needs no JSON escaping. KIND is data|stdout|stderr. +/// base64's alphabet needs no JSON escaping. pub async fn write_chunk_frame( w: &mut W, buf: &mut Vec, @@ -629,8 +654,6 @@ mod tests { #[tokio::test] async fn chunk_frame_parses_back_to_serde_response() { - // Payload lengths 0..=3 cover every base64 padding shape; the full - // byte range covers the whole alphabet. let payloads: Vec> = vec![ Vec::new(), b"a".to_vec(), @@ -675,9 +698,7 @@ mod tests { assert_eq!(got, want); } - /// Perf evidence, not a correctness gate. The serde side pays the - /// per-chunk `to_vec` the old producers paid. Run with: - /// cargo test --release -- --ignored --nocapture bench_chunk_render + /// Run with: cargo test --release -- --ignored --nocapture bench_chunk_render #[tokio::test] #[ignore] async fn bench_chunk_render_old_vs_new() { @@ -712,10 +733,8 @@ mod tests { assert_eq!(err.kind(), io::ErrorKind::InvalidData); } - /// Twin of the Go TestEnumValueSetsMatchFixture: pins each wire enum's - /// full value list (order-sensitive) to enums.json — frame fixtures carry - /// one representative value per enum, so a rename or an added variant on - /// only one side would otherwise drift silently. + /// Twin of the Go TestEnumValueSetsMatchFixture: pins every wire enum's + /// ordered value list to enums.json, which frame fixtures cannot. #[test] fn enum_value_sets_match_fixture() { fn rendered(variants: &[T]) -> Vec { @@ -781,8 +800,6 @@ mod tests { for entry in std::fs::read_dir(dir).expect("fixtures dir") { let path = entry.expect("entry").path(); let name = path.file_name().unwrap().to_string_lossy().into_owned(); - // Only the corpus itself; skip editor/OS cruft (.DS_Store, - // AppleDouble ._ sidecars) that can land in the dir. let Some(stem) = name.strip_suffix(".json") else { continue; }; @@ -799,7 +816,6 @@ mod tests { } seen += 1; } - // Exact, mirroring the Go guard: a new verb lands with its fixture. assert_eq!( seen, 60, "fixture corpus: adding a verb means adding its fixture" diff --git a/silkd/src/pty.rs b/silkd/src/pty.rs index 72f66321..37d0f10d 100644 --- a/silkd/src/pty.rs +++ b/silkd/src/pty.rs @@ -24,9 +24,9 @@ const POST_EXIT_DRAIN: Duration = Duration::from_secs(1); type Master = Arc>; -/// Opens a pty running the guest login shell, streams its output as `stdout` -/// frames and the exit as `exit`, and consumes client `stdin` frames as pty -/// input until the shell exits or the client disconnects. +/// Opens a pty running the guest shell ($SHELL, else bash), streams its output +/// as `stdout` frames and the exit as `exit`, and consumes client `stdin` +/// frames as pty input until the shell exits or the client disconnects. pub async fn open( table: &Table, now: u64, @@ -55,13 +55,15 @@ pub async fn open( { return crate::proto::error_frame(out, ErrorKind::BadRequest, e).await; } - match slave.try_clone().and_then(|i| Ok((i, slave.try_clone()?))) { - Ok((in_fd, out_fd)) => { + match (slave.try_clone(), slave.try_clone()) { + (Ok(in_fd), Ok(out_fd)) => { cmd.stdin(Stdio::from(in_fd)) .stdout(Stdio::from(out_fd)) .stderr(Stdio::from(slave)); } - Err(e) => return crate::proto::err_frame(out, &e, "dup pty slave").await, + (Err(e), _) | (_, Err(e)) => { + return crate::proto::err_frame(out, &e, "dup pty slave").await; + } } // SAFETY: make_controlling_tty runs only async-signal-safe syscalls, which // is the contract for a post-fork pre_exec hook. @@ -86,7 +88,7 @@ pub async fn open( let master: Master = match AsyncFd::new(master) { Ok(m) => Arc::new(m), Err(e) => { - child.start_kill().ok(); + let _ = child.start_kill(); table.remove_if(pid, &proc); return crate::proto::err_frame(out, &e, "watch pty master").await; } @@ -96,7 +98,7 @@ pub async fn open( .is_err() { // Client vanished before we streamed anything — tear down cleanly. - child.start_kill().ok(); + let _ = child.start_kill(); finish(&proc, -1); table.remove_if(pid, &proc); return Ok(()); @@ -163,7 +165,8 @@ async fn pump( Err(_) => { eof = true; continue } }; match guard.try_io(|fd| sysutil::read_fd(fd.get_ref().as_raw_fd(), &mut buf)) { - // 0 (BSD) and EIO (Linux) both mean the slave is fully closed. + // A read of 0 (BSD) or any error (EIO on Linux) means the + // slave is fully closed. Ok(Ok(0)) | Ok(Err(_)) => eof = true, Ok(Ok(n)) => { proc.emit_bytes(false, &buf[..n]); diff --git a/silkd/src/server.rs b/silkd/src/server.rs index eeb1ad22..b37236f5 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -1,7 +1,6 @@ //! Connection handling: read the leading request frame, dispatch to a //! handler, and for exec forward subsequent client frames over a channel. -use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufRead, AsyncWrite}; @@ -29,7 +28,7 @@ impl State { } } - pub async fn serve(self: &Arc, mut reader: R, mut writer: W) -> std::io::Result<()> + pub async fn serve(&self, mut reader: R, mut writer: W) -> std::io::Result<()> where R: AsyncBufRead + Unpin + Send + 'static, W: AsyncWrite + Unpin, diff --git a/silkd/src/session.rs b/silkd/src/session.rs index 0c7c792a..6466eced 100644 --- a/silkd/src/session.rs +++ b/silkd/src/session.rs @@ -74,6 +74,9 @@ impl Table { stdin, stdout, _child: child, + buf: vec![0u8; READ_CHUNK], + acc: Vec::new(), + frame: Vec::new(), }), last_active: Mutex::new(Instant::now()), }); @@ -202,10 +205,15 @@ impl Session { } } +/// The shell's pipes plus converse's scratch buffers, which are per-session +/// and reused across commands rather than reallocated per call. struct Io { stdin: ChildStdin, stdout: ChildStdout, _child: Child, + buf: Vec, + acc: Vec, + frame: Vec, } impl Io { @@ -219,13 +227,10 @@ impl Io { cmd: &str, mut out: Option<&mut W>, ) -> io::Result { - // Unpredictable marker: the shell runs untrusted code that must not be - // able to forge the sentinel and desync the stream. Redirect the - // command's stdin from /dev/null so a command that reads stdin (a REPL, - // `read`) can't swallow the sentinel printf and wedge the read. The - // `{ …; }` group runs in the current shell, so cd/export still persist. - // Trim so a trailing newline doesn't put `;` on its own line inside the - // group; empty becomes a no-op so `{ }` stays valid. + // The marker is unforgeable: the shell runs untrusted code that must + // not be able to fake the sentinel and desync the stream. `{ …; } + // ":", c => c, @@ -240,38 +245,40 @@ impl Io { let mb = marker.as_bytes(); let keep = mb.len().saturating_sub(1); - let mut acc: Vec = Vec::new(); - let mut buf = vec![0u8; READ_CHUNK]; - let mut frame = Vec::new(); + self.acc.clear(); loop { - let n = self.stdout.read(&mut buf).await?; + let n = self.stdout.read(&mut self.buf).await?; if n == 0 { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "shell exited")); } - acc.extend_from_slice(&buf[..n]); - if let Some(pos) = memchr::memmem::find(&acc, mb) { - emit(&mut out, &mut frame, &acc[..pos]).await; - acc.drain(..pos + mb.len()); - while !acc.contains(&b'\n') && acc.len() < EXIT_TAIL_MAX { - let m = self.stdout.read(&mut buf).await?; + self.acc.extend_from_slice(&self.buf[..n]); + if let Some(pos) = memchr::memmem::find(&self.acc, mb) { + emit(&mut out, &mut self.frame, &self.acc[..pos]).await; + self.acc.drain(..pos + mb.len()); + while !self.acc.contains(&b'\n') && self.acc.len() < EXIT_TAIL_MAX { + let m = self.stdout.read(&mut self.buf).await?; if m == 0 { break; } - acc.extend_from_slice(&buf[..m]); + self.acc.extend_from_slice(&self.buf[..m]); } // Parse only the exit-code line; a command that left a // background writer can land bytes after the newline, which // must not corrupt the code. - let end = acc.iter().position(|&b| b == b'\n').unwrap_or(acc.len()); - return Ok(String::from_utf8_lossy(&acc[..end]) + let end = self + .acc + .iter() + .position(|&b| b == b'\n') + .unwrap_or(self.acc.len()); + return Ok(String::from_utf8_lossy(&self.acc[..end]) .trim() .parse() .unwrap_or(-1)); } - if acc.len() > keep { - let upto = acc.len() - keep; - emit(&mut out, &mut frame, &acc[..upto]).await; - acc.drain(..upto); + if self.acc.len() > keep { + let upto = self.acc.len() - keep; + emit(&mut out, &mut self.frame, &self.acc[..upto]).await; + self.acc.drain(..upto); } } } diff --git a/silkd/src/sysutil.rs b/silkd/src/sysutil.rs index 234dddf5..4e5e8c35 100644 --- a/silkd/src/sysutil.rs +++ b/silkd/src/sysutil.rs @@ -3,12 +3,11 @@ //! lives here (pty.rs holds the one other unsafe block, its pre_exec //! registration). -use std::fmt::Write as _; use std::io::Read; use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; use std::process::ExitStatus; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Mutex, OnceLock}; +use std::sync::{LazyLock, Mutex}; use tokio::process::Command; @@ -29,6 +28,8 @@ const FORWARDED: [&str; 6] = [ "NO_PROXY", ]; +const HEX: [u8; 16] = *b"0123456789abcdef"; + /// A suffix unique within this process (pid + monotonic counter), enough to /// name a temp file a concurrent write in the same directory won't collide /// with. Not cryptographic — just unique. @@ -43,19 +44,33 @@ pub fn tmp_suffix() -> String { /// A 128-bit unpredictable hex token from the OS CSPRNG. Used where an /// in-sandbox command must not be able to guess or forge the value (the /// session-command sentinel). Falls back to the predictable tmp_suffix only if -/// /dev/urandom is unreadable, which never happens on a normal guest. +/// the CSPRNG is unreadable, which never happens on a normal guest. pub fn rand_token() -> String { let mut b = [0u8; 16]; - match std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut b)) { - Ok(()) => { - let mut s = String::with_capacity(32); - for x in b { - let _ = write!(s, "{x:02x}"); - } - s + if !fill_random(&mut b) { + return tmp_suffix(); + } + b.iter() + .flat_map(|byte| [HEX[(byte >> 4) as usize], HEX[(byte & 0xf) as usize]]) + .map(char::from) + .collect() +} + +/// Fills `b` from the OS CSPRNG, sparing an async caller the open/read/close a +/// tokio worker would block on. GRND_NONBLOCK never waits on an early-boot +/// pool; /dev/urandom is the same source and covers the fallback. +fn fill_random(b: &mut [u8]) -> bool { + #[cfg(target_os = "linux")] + { + // SAFETY: getrandom writes at most b.len() bytes into b's live buffer. + let n = unsafe { libc::getrandom(b.as_mut_ptr().cast(), b.len(), libc::GRND_NONBLOCK) }; + if n == b.len() as isize { + return true; } - Err(_) => tmp_suffix(), } + std::fs::File::open("/dev/urandom") + .and_then(|mut f| f.read_exact(b)) + .is_ok() } /// SIGKILLs the process group led by `pgid` — a session's shell plus its @@ -133,8 +148,9 @@ fn compose_env<'a>(nic: bool, proxy: &'a [(&'static str, String)]) -> Vec<(&'sta /// The FORWARDED values present in silkd's own environment, read once: the /// unit environment is fixed at service start and silkd never mutates it. fn proxy_vars() -> &'static [(&'static str, String)] { - static PROXY_VARS: OnceLock> = OnceLock::new(); - PROXY_VARS.get_or_init(|| snapshot_proxy(|key| std::env::var(key).ok())) + static PROXY_VARS: LazyLock> = + LazyLock::new(|| snapshot_proxy(|key| std::env::var(key).ok())); + &PROXY_VARS } fn snapshot_proxy(get: impl Fn(&str) -> Option) -> Vec<(&'static str, String)> { @@ -202,7 +218,7 @@ pub fn openpty(cols: u16, rows: u16) -> std::io::Result<(OwnedFd, OwnedFd)> { &mut slave, std::ptr::null_mut(), std::ptr::null_mut(), - std::ptr::addr_of_mut!(ws), + &raw mut ws, ) }; if rc != 0 { @@ -320,8 +336,6 @@ mod tests { assert!(env.iter().any(|(k, _)| *k == "TERM")); } - /// The no-NIC lane only reaches out if the proxy variables arrive without - /// anyone asking; nothing else of silkd's environment is the guest's. #[test] fn base_env_forwards_only_the_proxy_variables() { let proxy = snapshot_proxy(|key| match key { @@ -346,8 +360,6 @@ mod tests { ); } - /// A guest with a working NIC must not be steered into the loopback relay, - /// which is closed unless the host armed a proxy for this sandbox. #[test] fn base_env_forwards_nothing_when_a_nic_is_present() { let proxy = snapshot_proxy(|_| Some("http://127.0.0.1:3128".to_string())); @@ -358,8 +370,6 @@ mod tests { ); } - /// Env-inheriting children (git, language servers) follow the same lane - /// rule as cleared ones: scrubbed with a NIC, inherited without. #[test] fn align_proxy_env_scrubs_only_on_a_nic_lane() { let removed = |cmd: &Command| cmd.as_std().get_envs().filter(|(_, v)| v.is_none()).count(); diff --git a/silkd/src/tree.rs b/silkd/src/tree.rs index 76c9b384..5a5c5e33 100644 --- a/silkd/src/tree.rs +++ b/silkd/src/tree.rs @@ -31,7 +31,7 @@ where if let Err(e) = tokio::fs::create_dir_all(&dest).await { return err_frame(w, &e, "mkdir dest").await; } - let staging = match stage_dir(&dest) { + let staging = match stage_dir(&dest).await { Ok(s) => s, Err(e) => return err_frame(w, &e, "stage push").await, }; @@ -159,9 +159,9 @@ fn spawn_tar( /// Creates a unique staging dir inside `dest` — the same filesystem, so the /// post-extract merge is pure renames. -fn stage_dir(dest: &str) -> io::Result { +async fn stage_dir(dest: &str) -> io::Result { let staging = format!("{dest}/.silkd-push-{}", sysutil::tmp_suffix()); - std::fs::create_dir(&staging)?; + tokio::fs::create_dir(&staging).await?; Ok(staging) } @@ -189,8 +189,8 @@ fn merge_tree(src: &Path, dst: &Path) -> io::Result<()> { Ok(()) } -/// Reads a child's stderr to a string, capped so a pathological tar can't -/// balloon memory; the tail is what a human error message needs. +/// Reads a child's stderr to a string, keeping the first CAP bytes so a +/// pathological tar can't balloon memory — tar's first error is the useful one. async fn drain(mut stderr: tokio::process::ChildStderr) -> String { const CAP: usize = 16 * 1024; let mut out = Vec::new(); diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index e5b120f2..c618325d 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -10,6 +10,10 @@ use crate::proto::{self, ErrorKind, EventKind, Response}; const OVERFLOW_MESSAGE: &str = "watch event queue overflow"; +/// Events written per syscall. A build tool emits them in tight bursts, so +/// draining the queue in batches also keeps it far from its 256-slot cap. +const EVENT_BATCH: usize = 64; + /// Watches `path`, writing `ready`, ordered events, or a terminal error until disconnect. pub async fn watch( reader: &mut R, @@ -23,8 +27,10 @@ where { let (tx, mut rx) = mpsc::channel::(256); let mut tx = Some(tx); + let mut frames = Vec::new(); let mut watcher = match notify::recommended_watcher(move |res| { - forward_frames(&mut tx, to_frames(res)); + to_frames(res, &mut frames); + forward_frames(&mut tx, &mut frames); }) { Ok(watcher) => watcher, Err(e) => return proto::error_frame(w, ErrorKind::Internal, e.to_string()).await, @@ -38,18 +44,23 @@ where return proto::error_frame(w, map_kind(&e), e.to_string()).await; } proto::write_frame(w, &Response::Ready).await?; + let mut batch = Vec::new(); + let mut buf = Vec::new(); loop { tokio::select! { - frame = rx.recv() => match frame { - Some(frame) => { - let terminal = matches!(frame, Response::Error { .. }); - // A failed write is the disconnect the EOF arm can lose the select to. - if proto::write_frame(w, &frame).await.is_err() || terminal { - return Ok(()); - } - } + n = rx.recv_many(&mut batch, EVENT_BATCH) => { // Only overflow drops the sender mid-watch; the buffered prefix is already out. - None => return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await, + if n == 0 { + return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await; + } + let terminal = batch.iter().position(|f| matches!(f, Response::Error { .. })); + let upto = terminal.map_or(batch.len(), |i| i + 1); + // A failed write is the disconnect the EOF arm can lose the select to. + let sent = proto::write_frames(w, &mut buf, &batch[..upto]).await; + batch.clear(); + if sent.is_err() || terminal.is_some() { + return Ok(()); + } }, // The client sends nothing during a watch, so any readable state — // EOF (disconnect), a stray frame, or an error — ends the watch. @@ -58,9 +69,12 @@ where } } -fn forward_frames(tx: &mut Option>, frames: Vec) { - let Some(sender) = tx.as_ref() else { return }; - for frame in frames { +fn forward_frames(tx: &mut Option>, frames: &mut Vec) { + let Some(sender) = tx.as_ref() else { + frames.clear(); + return; + }; + for frame in frames.drain(..) { if sender.try_send(frame).is_err() { *tx = None; return; @@ -68,27 +82,28 @@ fn forward_frames(tx: &mut Option>, frames: Vec } } -fn to_frames(res: notify::Result) -> Vec { +/// Renders one notify event onto `out`, which the caller reuses — nearly every +/// event carries a single path. +fn to_frames(res: notify::Result, out: &mut Vec) { use notify::EventKind as N; let event = match res { Ok(event) => event, - Err(e) => return vec![Response::error(ErrorKind::Internal, e.to_string())], + Err(e) => { + out.push(Response::error(ErrorKind::Internal, e.to_string())); + return; + } }; let kind = match event.kind { N::Create(_) => EventKind::Created, N::Modify(notify::event::ModifyKind::Name(_)) => EventKind::Renamed, N::Modify(_) => EventKind::Modified, N::Remove(_) => EventKind::Deleted, - _ => return Vec::new(), + _ => return, }; - event - .paths - .into_iter() - .map(|p| Response::Event { - kind, - path: p.to_string_lossy().into_owned(), - }) - .collect() + out.extend(event.paths.into_iter().map(|p| Response::Event { + kind, + path: p.to_string_lossy().into_owned(), + })); } fn map_kind(e: ¬ify::Error) -> ErrorKind { @@ -104,7 +119,8 @@ mod tests { #[test] fn watcher_error_becomes_terminal_error_frame() { - let frames = to_frames(Err(notify::Error::generic("inotify overflow"))); + let mut frames = Vec::new(); + to_frames(Err(notify::Error::generic("inotify overflow")), &mut frames); assert_eq!(frames.len(), 1); match &frames[0] { Response::Error { message, .. } => assert!(message.contains("inotify overflow")), @@ -116,7 +132,8 @@ mod tests { fn create_event_becomes_event_frames() { let event = notify::Event::new(notify::EventKind::Create(notify::event::CreateKind::File)) .add_path("/x/y.txt".into()); - let frames = to_frames(Ok(event)); + let mut frames = Vec::new(); + to_frames(Ok(event), &mut frames); assert_eq!(frames.len(), 1); match &frames[0] { Response::Event { kind, path } => { @@ -132,9 +149,11 @@ mod tests { let (tx, mut rx) = mpsc::channel(1); let mut tx = Some(tx); - forward_frames(&mut tx, vec![Response::Ready, Response::Ready]); + let mut frames = vec![Response::Ready, Response::Ready]; + forward_frames(&mut tx, &mut frames); assert!(tx.is_none()); + assert!(frames.is_empty()); assert!(matches!(rx.try_recv(), Ok(Response::Ready))); assert!(matches!( rx.try_recv(), diff --git a/silkd/tests/common/mod.rs b/silkd/tests/common/mod.rs index 4536f449..c4eb9884 100644 --- a/silkd/tests/common/mod.rs +++ b/silkd/tests/common/mod.rs @@ -3,7 +3,7 @@ //! //! Compiled into each test binary separately, so not every helper is used by //! every binary — allow the resulting dead_code rather than fragment this. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] #![allow(dead_code)] use std::sync::Arc; @@ -19,9 +19,8 @@ use tokio::task::JoinHandle; pub type FrameWriter = WriteHalf; pub type FrameLines = Lines>>; -/// Opens a framed connection to a task serving `state`: the write half, a -/// line reader over the response frames, and the serve task handle. The split -/// halves let streaming verbs (pty, watch) interleave writes with reads. +/// Opens a framed connection to a task serving `state`: write half, response +/// lines, and the serve task handle, split so streaming verbs can interleave. pub fn connect(state: &Arc) -> (FrameWriter, FrameLines, JoinHandle>) { let (client, server) = tokio::io::duplex(1 << 20); let state = Arc::clone(state); @@ -36,9 +35,7 @@ pub async fn roundtrip(request_line: &str) -> Vec { request_on(&Arc::new(State::new()), &[request_line.to_string()]).await } -/// Runs one or more request lines on a single connection against `state` (so a -/// write/push can be followed by its data frames, and calls can be chained on -/// a shared state), returning the response frames. +/// Runs request lines on one connection against `state`, returning the frames. pub async fn request_on(state: &Arc, lines: &[String]) -> Vec { let (mut cw, mut out, handle) = connect(state); for line in lines { @@ -93,8 +90,7 @@ pub fn stdout_body(frames: &[Value]) -> String { String::from_utf8_lossy(&payload(frames, "stdout")).into_owned() } -/// A `data`-frame stream (16K chunks) terminated by `data_end`, for feeding -/// fs.write / fs.push. +/// A `data`-frame stream (16K chunks) terminated by `data_end`. pub fn data_frames(bytes: &[u8]) -> Vec { let mut lines: Vec = bytes .chunks(16 * 1024) diff --git a/silkd/tests/exec_e2e.rs b/silkd/tests/exec_e2e.rs index 6bfcc026..7606fb9d 100644 --- a/silkd/tests/exec_e2e.rs +++ b/silkd/tests/exec_e2e.rs @@ -2,7 +2,7 @@ //! (echo, sh), so it runs on any Unix — keep it that way so exec-path bugs //! surface on the dev host, not only on Linux CI. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -23,8 +23,6 @@ async fn exec_streams_stdout_then_exit() { #[tokio::test] async fn large_output_is_delivered_without_loss() { - // Exercises the backpressured foreground mpsc with far more chunks than - // its buffer; every line must arrive in order, none dropped. let frames = roundtrip(r#"{"op":"exec","argv":["/bin/sh","-c","seq 1 20000"]}"#).await; let body = stdout_body(&frames); let lines: Vec<&str> = body.lines().collect(); @@ -71,8 +69,6 @@ async fn kill_unknown_pid_is_not_found() { #[tokio::test] async fn daemonizing_child_does_not_wedge_the_exec() { - // sh forks a process that inherits stdout and outlives it. The exec must - // still return an exit frame promptly instead of blocking on pipe EOF. let frames = tokio::time::timeout( Duration::from_secs(8), roundtrip(r#"{"op":"exec","argv":["/bin/sh","-c","sleep 30 & exit 0"]}"#), @@ -86,9 +82,6 @@ async fn daemonizing_child_does_not_wedge_the_exec() { #[tokio::test] async fn daemonizer_exit_is_the_last_frame() { - // A child that leaves a grandchild holding stdout must still end its frame - // stream with exit and nothing after it — the post-exit drain aborts the - // pump, so no stray stdout can arrive once Exit is sent. let frames = tokio::time::timeout( Duration::from_secs(8), roundtrip(r#"{"op":"exec","argv":["/bin/sh","-c","(sleep 5; echo late) & exit 0"]}"#), @@ -124,8 +117,6 @@ async fn detached_pid(state: &Arc, script: &str) -> u64 { async fn attach_streams_live_output_then_exit() { let state = Arc::new(State::new()); let pid = detached_pid(&state, "sleep 0.3; echo live-chunk").await; - // Attach before the echo fires: the chunk must arrive live and the stream - // must end with exit. let frames = tokio::time::timeout( Duration::from_secs(8), one(&state, &format!(r#"{{"op":"attach","pid":{pid}}}"#)), @@ -171,8 +162,6 @@ async fn attach_unknown_pid_is_not_found() { async fn kill_actually_terminates_a_live_process() { let state = Arc::new(State::new()); let pid = detached_pid(&state, "sleep 30").await; - // Wait for it to register, then kill and confirm it reaches exited well - // before the 30s sleep would end. for _ in 0..50 { let ps = one(&state, r#"{"op":"ps"}"#).await; if ps[0]["procs"] @@ -217,7 +206,6 @@ async fn exec_missing_binary_is_an_internal_error() { #[tokio::test] async fn exec_stdin_is_piped_to_the_child() { - // cat echoes its stdin; feed a stdin frame, close it, expect the bytes back. let frames = exchange(&[ r#"{"op":"exec","argv":["/bin/cat"]}"#.to_string(), serde_json::json!({"op":"stdin","data":common::b64(b"piped-in\n")}).to_string(), @@ -292,9 +280,6 @@ async fn detached_exec_is_listed_then_logs_replay_output_and_exit() { #[tokio::test] async fn disconnect_during_drain_publishes_real_exit_code() { - // A grandchild holds stdout so the supervisor sits in the post-exit - // drain; the foreground client then vanishes mid-drain. The disconnect - // fallback must publish the child's real code to attachers, not -1. use tokio::io::AsyncWriteExt; let state = Arc::new(State::new()); @@ -312,9 +297,6 @@ async fn disconnect_during_drain_publishes_real_exit_code() { assert_eq!(type_of(&started), "started"); let pid = started["pid"].as_u64().unwrap(); - // Attach while the proc is live, then drop the exec client: writing the - // grandchild's "late" chunk fails and takes the disconnect path while - // the supervisor is still mid-drain. let st = Arc::clone(&state); let attach = tokio::spawn(async move { one( diff --git a/silkd/tests/find_e2e.rs b/silkd/tests/find_e2e.rs index a9b82b34..57f581ee 100644 --- a/silkd/tests/find_e2e.rs +++ b/silkd/tests/find_e2e.rs @@ -1,6 +1,6 @@ //! Search verb E2E: fs.find streams matches, fs.replace rewrites files. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use common::{exchange, type_of}; @@ -34,8 +34,8 @@ async fn find_streams_line_matches() { .iter() .all(|m| m["content"].as_str().unwrap().contains("TODO")) ); - assert!(matches.iter().any(|m| m["line"] == 2)); // a.rs - assert!(matches.iter().any(|m| m["line"] == 1)); // sub/c.rs + assert!(matches.iter().any(|m| m["line"] == 2)); + assert!(matches.iter().any(|m| m["line"] == 1)); assert_eq!(type_of(frames.last().unwrap()), "done"); } @@ -45,7 +45,6 @@ async fn find_glob_is_a_real_glob_not_a_substring() { write_file(dir.path(), "a.rs", "TODO\n").await; write_file(dir.path(), "a.rs.bak", "TODO\n").await; - // Anchored: "*.rs" must not match the .bak; "?" is a single character. let frames = exchange(&[json!({ "op": "fs_find", "path": dir.path().to_str().unwrap(), @@ -123,6 +122,55 @@ async fn replace_rewrites_and_counts() { ); } +#[tokio::test] +async fn replace_skips_a_file_over_the_size_bound() { + let dir = tempfile::tempdir().unwrap(); + let big = dir.path().join("big.log"); + let body = format!("foo\n{}", "x".repeat(8 * 1024 * 1024)); + tokio::fs::write(&big, &body).await.unwrap(); + + let frames = exchange(&[json!({ + "op": "fs_replace", + "files": [big.to_str().unwrap()], + "pattern": "foo", + "replacement": "bar" + }) + .to_string()]) + .await; + + let replaced: Vec<_> = frames.iter().filter(|f| type_of(f) == "replaced").collect(); + assert_eq!(replaced.len(), 1, "{frames:?}"); + assert_eq!(replaced[0]["replacements"], 0); + assert_eq!(type_of(frames.last().unwrap()), "done"); + assert_eq!( + tokio::fs::read_to_string(&big).await.unwrap().len(), + body.len(), + "an oversized file must not be rewritten" + ); +} + +#[tokio::test] +async fn find_skips_a_file_over_the_size_bound() { + let dir = tempfile::tempdir().unwrap(); + tokio::fs::write( + dir.path().join("big.log"), + format!("foo\n{}", "x".repeat(8 * 1024 * 1024)), + ) + .await + .unwrap(); + + let frames = exchange(&[json!({ + "op": "fs_find", + "path": dir.path().to_str().unwrap(), + "pattern": "foo" + }) + .to_string()]) + .await; + + assert_eq!(frames.iter().filter(|f| type_of(f) == "match").count(), 0); + assert_eq!(type_of(frames.last().unwrap()), "done"); +} + #[tokio::test] async fn replace_preserves_exec_bit() { use std::os::unix::fs::PermissionsExt; diff --git a/silkd/tests/forward_e2e.rs b/silkd/tests/forward_e2e.rs index 50e9ba48..cdc4979e 100644 --- a/silkd/tests/forward_e2e.rs +++ b/silkd/tests/forward_e2e.rs @@ -2,7 +2,9 @@ //! test runs under a deadline — the regressions this suite exists to catch //! (relay deadlocks) would otherwise hang CI instead of failing it. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] +mod common; + use std::sync::Arc; use std::time::Duration; @@ -14,8 +16,6 @@ use tokio::time::timeout; use silkd::server::State; -mod common; - const DEADLINE: Duration = Duration::from_secs(30); async fn echo_listener() -> u16 { @@ -54,7 +54,6 @@ async fn forward_round_trips_and_done_on_server_close() { let ready = lines.next_line().await.expect("read").expect("ready"); assert!(ready.contains("\"ready\""), "got {ready}"); - // "hi" → base64 aGk= ; the echo server must send it straight back. cw.write_all(b"{\"v\":1,\"op\":\"data\",\"data\":\"aGk=\"}\n") .await .expect("send data"); @@ -64,7 +63,6 @@ async fn forward_round_trips_and_done_on_server_close() { "got {echoed}" ); - // Half-close: the echo server sees EOF, closes, and the verb ends with done. cw.write_all(b"{\"v\":1,\"op\":\"data_end\"}\n") .await .expect("send data_end"); @@ -75,15 +73,10 @@ async fn forward_round_trips_and_done_on_server_close() { .expect("test deadline"); } -// Bidirectional bulk transfer must not head-of-line deadlock: the client -// streams > socket-buffer worth of input into an echo server that is -// simultaneously streaming it all back, so both directions have to make -// progress concurrently. A single-select relay (write_all stalling the read -// arm) hangs this test. #[tokio::test] async fn forward_bidirectional_bulk_no_deadlock() { - const CHUNK: usize = 3072; // multiple of 3: base64 with no padding - const FRAMES: usize = 400; // ~1.2 MiB, well past socket + duplex buffers + const CHUNK: usize = 3072; + const FRAMES: usize = 400; timeout(DEADLINE, async { let port = echo_listener().await; @@ -138,7 +131,6 @@ async fn forward_refused_port_is_not_found() { let state = Arc::new(State::new()); let (mut cw, mut lines, _) = common::connect(&state); - // Port 1 on loopback: nothing listens there in the test environment. cw.write_all(b"{\"v\":1,\"op\":\"port_forward\",\"port\":1}\n") .await .expect("send request"); @@ -165,7 +157,6 @@ async fn forward_stray_frame_is_bad_request() { let ready = lines.next_line().await.expect("read").expect("ready"); assert!(ready.contains("\"ready\""), "got {ready}"); - // A non-data frame mid-relay is a protocol violation, not a clean end. cw.write_all(b"{\"v\":1,\"op\":\"ps\"}\n") .await .expect("send stray"); diff --git a/silkd/tests/fs_e2e.rs b/silkd/tests/fs_e2e.rs index 7b12e2c1..ff660959 100644 --- a/silkd/tests/fs_e2e.rs +++ b/silkd/tests/fs_e2e.rs @@ -1,6 +1,6 @@ //! Filesystem verb E2E over the in-memory duplex, against real temp paths. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use common::{b64, exchange, type_of}; @@ -174,7 +174,6 @@ async fn overwrite_preserves_destination_mode() { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("script.sh"); - // Original file is executable (0755); overwrite without specifying a mode. std::fs::write(&path, b"old").unwrap(); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); let f = exchange(&[ diff --git a/silkd/tests/git_e2e.rs b/silkd/tests/git_e2e.rs index c4847afe..6c758db6 100644 --- a/silkd/tests/git_e2e.rs +++ b/silkd/tests/git_e2e.rs @@ -1,7 +1,7 @@ //! git verb E2E against a real git binary in temp repos. Local verbs run on //! any host; the none-lane guard is exercised via the test lane override. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use common::{exchange, type_of}; @@ -39,7 +39,6 @@ async fn add_commit_status_reports_structured_state() { let p = repo.path().to_str().unwrap(); std::fs::write(repo.path().join("a.txt"), "hello\n").unwrap(); - // Untracked file shows in status. let st = exchange(&[json!({"op":"git_status","path":p}).to_string()]).await; assert_eq!(type_of(&st[0]), "git_status_result"); assert_eq!(st[0]["branch"], "main"); @@ -50,7 +49,6 @@ async fn add_commit_status_reports_structured_state() { .any(|f| f["path"] == "a.txt" && f["staged"] == "?") ); - // Stage and commit; a hash comes back. let add = exchange(&[json!({"op":"git_add","path":p,"files":["a.txt"]}).to_string()]).await; assert_eq!(type_of(last(&add)), "done"); let commit = exchange(&[json!({ @@ -61,7 +59,6 @@ async fn add_commit_status_reports_structured_state() { assert_eq!(type_of(&commit[0]), "git_commit_result"); assert_eq!(commit[0]["hash"].as_str().unwrap().len(), 40); - // Clean tree now. let st2 = exchange(&[json!({"op":"git_status","path":p}).to_string()]).await; assert!( st2[0]["files"].as_array().unwrap().is_empty(), @@ -103,7 +100,7 @@ async fn branch_create_list_checkout() { #[tokio::test] async fn commit_failure_surfaces_git_error() { - let repo = init_repo(); // nothing staged + let repo = init_repo(); let p = repo.path().to_str().unwrap(); let frames = exchange(&[json!({ "op":"git_commit","path":p,"message":"empty","author":"Dev " @@ -113,10 +110,8 @@ async fn commit_failure_surfaces_git_error() { assert_eq!(type_of(&frames[0]), "error"); } -// One test owns the lane override so the two cases cannot fight over it. #[tokio::test] async fn clone_lane_behavior() { - // none lane refuses with a typed error pointing at fs.push. silkd::net::override_egress_for_tests(Some(false)); let none = exchange(&[json!({ "op":"git_clone","url":"https://example.com/x.git","path":"/tmp/silkd-x" @@ -127,7 +122,6 @@ async fn clone_lane_behavior() { assert_eq!(none[0]["kind"], "unimplemented"); assert!(none[0]["message"].as_str().unwrap().contains("fs.push")); - // egress lane clones (exercised against a local source repo). let src = init_repo(); std::fs::write(src.path().join("r.txt"), "body\n").unwrap(); git(src.path(), &["add", "r.txt"]); diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index 9a4fb90f..e45c91d6 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -4,7 +4,7 @@ //! real language server. Every test runs under a deadline so a relay deadlock //! fails CI instead of hanging it. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] use std::io::Write; use std::sync::Arc; use std::time::Duration; @@ -21,16 +21,14 @@ use common::{b64, connect, decode, one, type_of}; const DEADLINE: Duration = Duration::from_secs(30); -// SILKD_LSP_DIR is process-global; serialize the tests that set it. -static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); +static ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); -// A fake server that echoes each line it reads, prefixed — enough to prove -// the round trip without LSP semantics. const FAKE_SERVER: &str = r#"#!/bin/sh IFS= read -r line printf 'reply:%s\n' "$line" "#; +/// Creates a fake language server and points SILKD_LSP_DIR at its dir. fn manifest_env(server_body: &str) -> TempDir { let dir = tempfile::tempdir().unwrap(); let bin = dir.path().join("fake-lsp"); @@ -41,11 +39,15 @@ fn manifest_env(server_body: &str) -> TempDir { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); } + // SAFETY: ENV_LOCK, held by the caller for its whole test, serializes every + // writer and reader of SILKD_LSP_DIR in this binary. + unsafe { std::env::set_var("SILKD_LSP_DIR", dir.path()) }; dir } #[tokio::test] async fn lsp_start_missing_manifest_is_not_found() { + let _env_lock = ENV_LOCK.lock().await; let frames = timeout( DEADLINE, one( @@ -77,12 +79,8 @@ async fn lsp_start_language_name_cannot_escape() { } #[tokio::test] -#[allow(clippy::await_holding_lock)] // ENV_LOCK serializes SILKD_LSP_DIR across the whole test async fn lsp_broker_relays_to_the_server() { - // Point the broker at a manifest dir under our control: SILKD_LSP_DIR - // overrides /etc/silkd/lsp.d, and the planted manifest names the fake - // server as the language server for "faketest". - let _env_lock = ENV_LOCK.lock().unwrap(); + let _env_lock = ENV_LOCK.lock().await; let env = manifest_env(FAKE_SERVER); let bin = env.path().join("fake-lsp"); std::fs::write( @@ -90,9 +88,6 @@ async fn lsp_broker_relays_to_the_server() { bin.to_string_lossy().as_bytes(), ) .unwrap(); - // SAFETY: ENV_LOCK (held for this test's whole body) serializes every - // writer/reader of SILKD_LSP_DIR in this binary. - unsafe { std::env::set_var("SILKD_LSP_DIR", env.path()) }; let state = Arc::new(State::new()); let start = one( @@ -105,7 +100,6 @@ async fn lsp_broker_relays_to_the_server() { .unwrap() .to_string(); - // Attach and send one JSON-RPC-ish line; expect the echoed reply back. let (mut cw, mut out, handle) = connect(&state); cw.write_all( json!({"op":"lsp_request","server_id":server_id}) @@ -136,8 +130,6 @@ async fn lsp_broker_relays_to_the_server() { assert_eq!(type_of(&frame), "data"); assert_eq!(decode(&frame), b"reply:ping\n"); - // The fake server exits after one reply: its stdout EOFs, so the request - // ends with Done and the broker reaps the server. let done = timeout(DEADLINE, out.next_line()) .await .expect("deadline") @@ -150,7 +142,6 @@ async fn lsp_broker_relays_to_the_server() { cw.shutdown().await.unwrap(); let _ = handle.await; - // Reaped: a later lsp_stop is not_found. let gone = one( &state, &json!({"op":"lsp_stop","server_id":server_id}).to_string(), @@ -160,18 +151,14 @@ async fn lsp_broker_relays_to_the_server() { } #[tokio::test] -#[allow(clippy::await_holding_lock)] // ENV_LOCK serializes SILKD_LSP_DIR across the whole test async fn lsp_stop_kills_an_idle_server() { - let _env_lock = ENV_LOCK.lock().unwrap(); + let _env_lock = ENV_LOCK.lock().await; let env = manifest_env("#!/bin/sh\nsleep 60\n"); std::fs::write( env.path().join("idletest"), env.path().join("fake-lsp").to_string_lossy().as_bytes(), ) .unwrap(); - // SAFETY: ENV_LOCK (held for this test's whole body) serializes every - // writer/reader of SILKD_LSP_DIR in this binary. - unsafe { std::env::set_var("SILKD_LSP_DIR", env.path()) }; let state = Arc::new(State::new()); let start = one( &state, @@ -197,21 +184,57 @@ async fn lsp_stop_kills_an_idle_server() { } #[tokio::test] -#[allow(clippy::await_holding_lock)] // ENV_LOCK serializes SILKD_LSP_DIR across the whole test +async fn lsp_request_reaps_when_the_client_vanishes_before_ready() { + let _env_lock = ENV_LOCK.lock().await; + let env = manifest_env("#!/bin/sh\nsleep 60\n"); + std::fs::write( + env.path().join("gonetest"), + env.path().join("fake-lsp").to_string_lossy().as_bytes(), + ) + .unwrap(); + let state = Arc::new(State::new()); + let start = one( + &state, + &json!({"op":"lsp_start","language":"gonetest"}).to_string(), + ) + .await; + let server_id = start.last().unwrap()["server_id"] + .as_str() + .unwrap() + .to_string(); + + let (mut client, server) = tokio::io::duplex(1 << 20); + let request = json!({"op":"lsp_request","server_id":server_id}).to_string(); + client.write_all(request.as_bytes()).await.unwrap(); + client.write_all(b"\n").await.unwrap(); + drop(client); + let (sr, sw) = tokio::io::split(server); + let served = timeout(DEADLINE, state.serve(tokio::io::BufReader::new(sr), sw)) + .await + .expect("deadline"); + assert!(served.is_err(), "the ready write must fail: {served:?}"); + + let gone = one( + &state, + &json!({"op":"lsp_stop","server_id":server_id}).to_string(), + ) + .await; + assert_eq!( + gone.last().unwrap()["kind"], + "not_found", + "a server whose client vanished before ready must be reaped: {gone:?}" + ); +} + +#[tokio::test] async fn lsp_data_end_half_closes_stdin() { - // The fake server drains stdin to EOF before replying — it can only - // answer if data_end actually closes its stdin. - let _env_lock = ENV_LOCK.lock().unwrap(); - // unquoted $(...) word-splits away wc's platform-dependent padding + let _env_lock = ENV_LOCK.lock().await; let env = manifest_env("#!/bin/sh\nprintf 'ate:%s\\n' $(cat | wc -c)\n"); std::fs::write( env.path().join("eoftest"), env.path().join("fake-lsp").to_string_lossy().as_bytes(), ) .unwrap(); - // SAFETY: ENV_LOCK (held for this test's whole body) serializes every - // writer/reader of SILKD_LSP_DIR in this binary. - unsafe { std::env::set_var("SILKD_LSP_DIR", env.path()) }; let state = Arc::new(State::new()); let start = one( &state, diff --git a/silkd/tests/pty_e2e.rs b/silkd/tests/pty_e2e.rs index c8052503..571ca87d 100644 --- a/silkd/tests/pty_e2e.rs +++ b/silkd/tests/pty_e2e.rs @@ -1,7 +1,7 @@ //! pty.open E2E: a shell under a pseudo-terminal, driven over a duplex. //! openpty exists on macOS and Linux, so this runs on the dev host. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -42,7 +42,6 @@ async fn pty_runs_a_shell_and_echoes() { let pid = started["pid"].as_u64().unwrap(); assert!(pid > 0); - // A pty echoes input; run a command and look for its output. send( &mut cw, json!({"op":"stdin","data":common::b64(b"echo silk-pty-marker\n")}), @@ -72,7 +71,6 @@ async fn pty_appears_in_ps_and_resizes() { .as_u64() .unwrap(); - // ps (shared table) sees the pty as a process; resize succeeds on it. let ps = common::one(&state, r#"{"op":"ps"}"#).await; assert!( ps[0]["procs"] @@ -105,9 +103,6 @@ async fn resize_unknown_pid_is_not_found() { #[tokio::test] async fn pty_disconnect_tears_down_without_spin() { - // Client disconnects while the shell is idle. The pty must tear down (kill - // the shell, drop the table entry, end the serve task) instead of - // busy-spinning on the closed client channel. let state = Arc::new(State::new()); let (mut cw, mut lines, handle) = common::connect(&state); send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; @@ -115,13 +110,10 @@ async fn pty_disconnect_tears_down_without_spin() { .as_u64() .unwrap(); - // Let the shell settle, then disconnect. tokio::time::sleep(Duration::from_millis(200)).await; drop(lines); drop(cw); - // The serve task returns promptly (no spin); the final Exit-write to the - // gone client may error, which is fine — the point is it returns. let _ = tokio::time::timeout(Duration::from_secs(5), handle) .await .expect("pty did not tear down on disconnect"); diff --git a/silkd/tests/session_e2e.rs b/silkd/tests/session_e2e.rs index b16b4018..677ec2a7 100644 --- a/silkd/tests/session_e2e.rs +++ b/silkd/tests/session_e2e.rs @@ -2,7 +2,7 @@ //! exec calls. Spawns real bash, so Unix-only in practice; kept ungated so //! the persistence contract is checked on the dev host too. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -59,7 +59,6 @@ async fn session_applies_cwd_and_env_at_create() { let g = sh(&state, &id, &["sh", "-c", "echo $GREETING"]).await; assert_eq!(stdout_body(&g).trim(), "hello-silk"); - // cwd applied: a relative touch lands in the session's directory. let t = sh(&state, &id, &["touch", "marker"]).await; assert_eq!(type_of(t.last().unwrap()), "exit"); assert!(dir.path().join("marker").exists(), "cwd was not applied"); @@ -100,16 +99,12 @@ async fn session_list_and_rm() { #[tokio::test] async fn stdin_reading_command_does_not_wedge_the_session() { - // A command that reads stdin (here `cat`, which would block forever on the - // control pipe) must not hang the session — its stdin is isolated. Bound - // with a timeout: a regression would hang instead of returning. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let r = tokio::time::timeout(std::time::Duration::from_secs(8), sh(&state, &id, &["cat"])) .await .expect("session wedged on a stdin-reading command"); assert_eq!(type_of(r.last().unwrap()), "exit"); - // The session is still usable afterward. let after = sh(&state, &id, &["echo", "alive"]).await; assert_eq!(stdout_body(&after).trim(), "alive"); } @@ -118,7 +113,6 @@ async fn stdin_reading_command_does_not_wedge_the_session() { async fn shell_killing_command_removes_the_session() { let state = Arc::new(State::new()); let id = create(&state, json!({})).await; - // `exit` kills the shell; the session must then stop resolving. let _ = sh(&state, &id, &["exit"]).await; let again = sh(&state, &id, &["echo", "hi"]).await; assert_eq!(type_of(&again[0]), "error"); @@ -127,9 +121,6 @@ async fn shell_killing_command_removes_the_session() { #[tokio::test] async fn forged_sentinel_in_output_does_not_desync() { - // A command printing a plausible sentinel must not be mistaken for the - // real one (random token) — the real command output is preserved and the - // next command still frames correctly. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let r = sh( @@ -155,7 +146,6 @@ async fn reap_idle_removes_idle_sessions() { use std::time::Duration; let state = Arc::new(State::new()); let id = create(&state, json!({})).await; - // Everything is "idle" against a zero TTL. let reaped = state.sessions.reap_idle(Duration::ZERO); assert_eq!(reaped, 1); let gone = sh(&state, &id, &["echo", "hi"]).await; @@ -171,8 +161,6 @@ async fn reap_skips_a_session_running_a_command() { use std::time::Duration; let state = Arc::new(State::new()); let id = create(&state, json!({})).await; - // Hold the session busy with a slow command while a zero-TTL reap runs; - // an actively-running session must not be reaped. let busy = { let s = Arc::clone(&state); let sid = id.clone(); @@ -183,14 +171,11 @@ async fn reap_skips_a_session_running_a_command() { assert_eq!(reaped, 0, "a running session must not be reaped"); let done = busy.await.unwrap(); assert_eq!(type_of(done.last().unwrap()), "exit"); - // After it finishes it becomes reapable again. assert_eq!(state.sessions.reap_idle(Duration::ZERO), 1); } #[tokio::test] async fn session_argv_with_metacharacters_is_one_literal_token() { - // A single argv element containing shell metacharacters must be passed as - // one literal argument, not interpreted — shell_quote must neutralize it. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let evil = "a b; echo INJECTED $(id) `whoami` 'q'"; @@ -208,10 +193,6 @@ async fn session_argv_with_metacharacters_is_one_literal_token() { #[tokio::test] async fn session_rm_unwedges_a_running_external_command() { - // A session running a long external command blocks its run() until the - // command exits. session_rm must group-kill the shell AND the command so - // the blocked run() unwinds — killing only bash would leave the sleep - // holding the stdout pipe open. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let busy = { @@ -222,7 +203,6 @@ async fn session_rm_unwedges_a_running_external_command() { tokio::time::sleep(std::time::Duration::from_millis(200)).await; let rm = one(&state, &json!({"op":"session_rm","id":id}).to_string()).await; assert_eq!(type_of(&rm[0]), "done"); - // The blocked command must now return (shell group killed) within a bound. let done = tokio::time::timeout(std::time::Duration::from_secs(5), busy) .await .expect("session_rm did not unwedge the external command"); @@ -232,8 +212,6 @@ async fn session_rm_unwedges_a_running_external_command() { #[tokio::test] async fn empty_argv_in_session_is_a_bad_request() { - // The dispatcher validates argv ahead of the session/process split: an - // empty argv must not reach the shell as a successful no-op. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let f = one( @@ -247,8 +225,6 @@ async fn empty_argv_in_session_is_a_bad_request() { #[tokio::test] async fn detach_with_session_is_a_bad_request() { - // silkd's session path has no Started frame to answer a detached exec, so - // the combination must be rejected up front rather than silently run. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let f = one( @@ -270,8 +246,6 @@ async fn exec_in_unknown_session_is_not_found() { #[tokio::test] async fn session_merges_stderr_into_stdout() { - // The shell's first init line is `exec 2>&1`: command stderr must arrive - // on the session's single output stream. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; diff --git a/silkd/tests/tree_e2e.rs b/silkd/tests/tree_e2e.rs index aa27a6dc..6dd4ddc7 100644 --- a/silkd/tests/tree_e2e.rs +++ b/silkd/tests/tree_e2e.rs @@ -1,7 +1,7 @@ //! Whole-tree transfer E2E: push a tar stream in, pull a tar stream out, //! using the system tar to build/extract the reference archives. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::path::Path; @@ -76,7 +76,6 @@ async fn pull_streams_a_tar_of_the_path() { let out = tempfile::tempdir().unwrap(); sys_tar_extract(&archive, out.path()); - // the archive carries the entry under its own name (proj/…) assert_eq!( std::fs::read(out.path().join("proj/x.txt")).unwrap(), b"hello-pull" @@ -108,8 +107,6 @@ async fn push_then_pull_roundtrips_a_tree() { #[tokio::test] async fn push_truncated_stream_errors() { - // Send fs_push + a data frame but no data_end: tar gets a partial archive - // and the handler must report an error, not done. let dest = tempfile::tempdir().unwrap(); let frames = exchange(&[ json!({"op":"fs_push","dest":dest.path().join("p").to_str().unwrap()}).to_string(), @@ -125,7 +122,6 @@ async fn push_truncated_stream_errors() { #[tokio::test] async fn push_bad_archive_reports_tar_failure() { - // A complete stream that isn't a valid tar → tar exits nonzero → Internal. let dest = tempfile::tempdir().unwrap(); let frames = exchange(&[ json!({"op":"fs_push","dest":dest.path().join("p").to_str().unwrap()}).to_string(), @@ -139,7 +135,6 @@ async fn push_bad_archive_reports_tar_failure() { #[tokio::test] async fn pull_missing_path_errors() { - // Parent exists; the leaf itself is missing. let dir = tempfile::tempdir().unwrap(); let missing = dir.path().join("xyz"); let frames = @@ -195,8 +190,6 @@ async fn pull_dangling_symlink_archives_the_link() { #[tokio::test] async fn push_failure_leaves_dest_untouched() { - // Atomicity: a truncated stream must not mutate a populated dest at all — - // no partial files, no staging leftovers. let dest = tempfile::tempdir().unwrap(); std::fs::write(dest.path().join("keep.txt"), b"KEEP").unwrap(); std::fs::create_dir(dest.path().join("sub")).unwrap(); @@ -228,8 +221,6 @@ async fn push_failure_leaves_dest_untouched() { #[tokio::test] async fn push_overlays_existing_tree_and_cleans_staging() { - // tar semantics survive the staged merge: dirs merge recursively, an - // existing file is replaced, unrelated files survive, staging vanishes. let src = tempfile::tempdir().unwrap(); std::fs::create_dir(src.path().join("sub")).unwrap(); std::fs::write(src.path().join("sub/new.txt"), b"NEW").unwrap(); diff --git a/silkd/tests/watch_e2e.rs b/silkd/tests/watch_e2e.rs index 3bd5fb9b..a7ddf41c 100644 --- a/silkd/tests/watch_e2e.rs +++ b/silkd/tests/watch_e2e.rs @@ -1,6 +1,6 @@ //! fs.watch E2E: a connection-bound event stream ended by client disconnect. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -24,7 +24,6 @@ async fn watch_streams_events_until_disconnect() { cw.write_all(req.as_bytes()).await.unwrap(); cw.write_all(b"\n").await.unwrap(); - // The ready frame guarantees the watch is armed — no arming sleep needed. let ready = tokio::time::timeout(Duration::from_secs(5), lines.next_line()) .await .expect("no ready within 5s") @@ -36,16 +35,23 @@ async fn watch_streams_events_until_disconnect() { .await .unwrap(); - let event = tokio::time::timeout(Duration::from_secs(5), lines.next_line()) - .await - .expect("no event within 5s") - .unwrap() - .expect("stream closed before an event"); - let frame: serde_json::Value = serde_json::from_str(&event).unwrap(); - assert_eq!(frame["type"], "event", "got {frame:?}"); - assert!(frame["path"].as_str().unwrap().ends_with("new.txt")); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let line = lines + .next_line() + .await + .unwrap() + .expect("stream closed before an event"); + let frame: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(frame["type"], "event", "got {frame:?}"); + if frame["path"].as_str().unwrap().ends_with("new.txt") { + return; + } + } + }) + .await + .expect("no new.txt event within 5s"); - // Disconnect ends the watch: the server task returns promptly. drop(lines); drop(cw); tokio::time::timeout(Duration::from_secs(5), handle)