review: whole-repo round on main 21dc4e7

.github/workflows/python.yml

@@ -7,6 +7,7 @@ on:
       - "sdk/openai/**"
       - "sdk/langchain/**"
       - "mcp/**/*.py"
+      - "e2e/**/*.py"
       - "ruff.toml"
       - "protocol/**"
       - ".github/workflows/python.yml"
@@ -16,6 +17,7 @@ on:
       - "sdk/openai/**"
       - "sdk/langchain/**"
       - "mcp/**/*.py"
+      - "e2e/**/*.py"
       - "ruff.toml"
       - "protocol/**"
 

README.md

@@ -4,7 +4,7 @@ MicroVM sandboxes for AI agents, built on
 [cocoon](https://github.com/cocoonstack/cocoon): a fast-boot guest stack, an
 in-guest product daemon, a per-node control plane with warm pools, and SDKs
 for Go and Python. Warm claims are sub-millisecond; a pool miss clones from a
-golden snapshot in tens of milliseconds; cold boot is ~200ms on bare metal.
+golden snapshot in tens of milliseconds; cold boot is ~0.2–0.4 s on bare metal.
 
 ```
 SDK (Go/Python)         sandboxd (per node)              guest microVM
@@ -74,16 +74,18 @@ performance) — source in
 - `scripts/` — `boot-bench.sh` (boot phase timing), `bench.sh` (the published
   benchmark procedure), `sandboxd-e2e.sh` (bare-metal e2e, below), plus the
   `archive`/`egress`/`intercept`/`socks` e2e drivers
+- `packaging/` — the systemd unit deploy installs
 
 ## Build & test
 
 ```bash
-make help          # this list
+make help          # every target; the ones below are the daily set
 make lint test     # Rust: boot/init + silkd (fmt --check, clippy -D warnings, tests)
 make go-lint       # Go: protocol/wire + sandboxd + sdk/go + e2e + mcp, GOOS linux AND darwin
 make go-test       # Go: go test -race across the Go modules
 make sh-lint       # shellcheck every tracked shell script
 make sandboxd      # build dist/sandboxd
+make bench         # claim-tier + data-plane benchmarks on this node
 make boot          # kernel + initramfs artifact image (docker)
                    #   KERNEL_MIRROR=… if kernel.org tarball paths 404 locally
 make silkd-image   # silkd release binary in a scratch carrier image
@@ -148,9 +150,9 @@ Boot contract (cmdline keys consumed by sandbox-init):
 
 | cmdline key | meaning |
 |---|---|
-| `cocoon.layers=a,b,…` | EROFS layer disks resolved from virtio-blk serials, lowerdir order |
+| `cocoon.layers=a,b,…` | EROFS layer disks resolved from virtio-blk serials (or `/dev/vdX` paths, as the Firecracker lane passes them), lowerdir order |
 | `cocoon.cow=x` | writable ext4 COW disk (same resolution rules) |
-| `cocoon.timeout=10` | per-disk wait budget, seconds |
+| `cocoon.timeout=10` | wait budget for the whole disk set, seconds (NIC MACs get a fixed 200 ms) |
 | `cocoon.hostname=h` | set via `sethostname(2)` before handoff |
 | `ip=addr::gw:mask:host:ethN:off[:dns0[:dns1]]` | cocoon CNI static config: persisted as a MAC-matched networkd unit in the new root (not applied in the initramfs); absent → the image's DHCP fallback covers the NIC |
 | `sandbox.init=/path` | handoff target, default `/sbin/init` |

boot/init/src/boot.rs

@@ -202,9 +202,11 @@ fn mkdir_all(path: &str) -> Result<(), String> {
     fs::create_dir_all(path).map_err(|err| format!("mkdir {path}: {err}"))
 }
 
-/// Resolves every virtio-blk serial in one sysfs sweep per poll iteration.
 fn resolve_disks(ids: &[&str], timeout: Duration) -> Result<Vec<String>, String> {
-    let found = poll_slots(ids.len(), timeout, |found| scan_serials(ids, found));
+    let found = poll_slots(ids.len(), timeout, |found| {
+        record_devices(ids, found);
+        scan_serials(ids, found);
+    });
     if found.iter().all(Option::is_some) {
         return Ok(found.into_iter().flatten().collect());
     }
@@ -246,6 +248,15 @@ fn scan_serials(ids: &[&str], found: &mut [Option<String>]) {
     }
 }
 
+// cocoon's Firecracker lane names the device node itself; there is no virtio serial to match
+fn record_devices(ids: &[&str], found: &mut [Option<String>]) {
+    for (slot, id) in found.iter_mut().zip(ids) {
+        if slot.is_none() && id.starts_with("/dev/") && Path::new(id).exists() {
+            *slot = Some((*id).into());
+        }
+    }
+}
+
 fn record_serial(ids: &[&str], found: &mut [Option<String>], serial: &str, device: &str) {
     for (i, id) in ids.iter().enumerate() {
         if found[i].is_none() && *id == serial && Path::new(device).exists() {
@@ -281,4 +292,15 @@ mod tests {
         record_serial(&ids, &mut found, "layer", "/dev/null");
         assert_eq!(found[0].as_deref(), Some("/dev/null"));
     }
+
+    #[test]
+    fn device_path_ids_resolve_once_the_node_exists() {
+        let ids = ["/dev/null", "/dev/sandbox-init-missing-device", "layer"];
+        let mut found = [None, None, None];
+
+        record_devices(&ids, &mut found);
+        assert_eq!(found[0].as_deref(), Some("/dev/null"));
+        assert!(found[1].is_none());
+        assert!(found[2].is_none());
+    }
 }

boot/init/src/cfg.rs

@@ -14,7 +14,7 @@ pub struct BootCfg {
     pub layers: Vec<String>,
     /// Writable ext4 COW disk ID (same resolution rules as layers).
     pub cow: String,
-    /// Per-device wait budget.
+    /// Wait budget for the whole disk set; NICs get a fixed 200 ms.
     pub timeout: Duration,
     pub hostname: Option<String>,
     /// Static per-NIC config, persisted as networkd units and never applied in the initramfs.
@@ -27,7 +27,7 @@ pub struct BootCfg {
     pub trace: bool,
 }
 
-/// One `ip=<addr>::<gw>:<mask>:<host>:<dev>:off[:dns0[:dns1]]` param, as cocoon emits it per NIC.
+/// One kernel `ip=` param, as cocoon emits it per NIC.
 #[derive(Debug, PartialEq, Eq)]
 pub struct IpParam {
     pub addr: String,
@@ -123,7 +123,6 @@ fn params(cmdline: &str) -> impl DoubleEndedIterator<Item = (&str, &str)> {
         .map(|tok| tok.split_once('=').unwrap_or((tok, "")))
 }
 
-/// sandbox.debug value semantics, shared by parse() and debug_requested().
 fn debug_token(val: &str) -> bool {
     val.is_empty() || val == "1"
 }
@@ -136,9 +135,7 @@ fn parse_ip_param(val: &str) -> Option<IpParam> {
     }
     let prefix = mask_to_prefix(f[3])?;
     let gateway = (!f[2].is_empty() && f[2] != "0.0.0.0").then(|| f[2].to_string());
-    let dns = f
-        .get(7..)
-        .unwrap_or(&[])
+    let dns = f[7..]
         .iter()
         .filter(|d| !d.is_empty() && **d != "0.0.0.0")
         .map(|d| d.to_string())

boot/init/src/sys.rs

@@ -5,7 +5,6 @@ use std::io;
 use std::ptr;
 use std::time::Duration;
 
-/// Mount flags for the kernel API filesystems.
 pub const MNT_SECURE: libc::c_ulong = libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC;
 
 pub fn mount(
@@ -83,7 +82,7 @@ pub fn exec_init(path: &str) -> String {
     io::Error::last_os_error().to_string()
 }
 
-/// Routes PID 1 stdio to /dev/console, covering a cpio built without the console node.
+/// Routes PID 1 stdio to /dev/console once devtmpfs is mounted.
 pub fn claim_console() {
     // SAFETY: the literal is a live 'static CStr; the fd is validated (>=0)
     // before any dup2/close and only closed when it is not already a std stream.

docs/benchmarks.md

@@ -265,7 +265,7 @@ powersave, whose C-state/freq-ramp latency lands in the tail. The guest is
 up 0.26s (its own /proc/uptime) when the first cold exec returns — silkd
 answers long before a console login prompt would appear.
 
-### 2026-07-10 — bare metal
+### 2026-07-10 — bare metal (commit not recorded)
 
 | environment | |
 |---|---|
@@ -285,7 +285,7 @@ answers long before a console login prompt would appear.
 | fs_stat RTT (dial per RPC) | n=200 p50=0.22ms p90=1.71ms p99=3.40ms |
 | fs_pull throughput (128 MiB) | 595.9 MiB/s best of 3 |
 
-### 2026-07-08 — bare metal
+### 2026-07-08 — bare metal (commit not recorded)
 
 | environment | |
 |---|---|
@@ -305,7 +305,7 @@ answers long before a console login prompt would appear.
 | fs_stat RTT (dial per RPC) | n=200 p50=0.17ms p90=0.22ms p99=0.26ms |
 | fs_pull throughput (128 MiB) | 232.6 MiB/s best of 3 |
 
-### 2026-07-07 — nested (google)
+### 2026-07-07 — nested (google, commit not recorded)
 
 | environment | |
 |---|---|

docs/deploy.md

@@ -107,13 +107,13 @@ sandboxd reads one JSON file (`-config`, default
 | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant). Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) |
 | `egress_ca` | unset | [HTTPS-interception](egress.md#https-interception) PKI: `root_cert` (the cluster root baked into intercepted guests; may bundle old+new roots during rotation) plus this node's `intermediate_cert`/`intermediate_key` from `sandboxd ca issue-intermediate`. Required when any pool rule sets `intercept` |
 | `api_token` | unset | the operator (root) credential: when set, guards the node-level endpoints (Bearer) with full access, including release-by-id cleanup. Per-sandbox tokens guard ordinary sandbox-scoped calls |
-| `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50}]`. A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview), catalog discovery, and its own sandbox/checkpoint listings; everything it creates is stamped with the tenant name. Root-only surfaces (per-id sandbox reads, `GET /v1/checkpoints/{id}/blob`, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set. Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays whichever token authorized a redirect), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior |
+| `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50, "egress": {…}}]` (the optional `egress` policy is intersected with the pool's, see [egress](egress.md)). A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview), catalog discovery, and its own sandbox/checkpoint listings; everything it creates is stamped with the tenant name. Root-only surfaces (per-id sandbox reads and stats, `GET /v1/checkpoints/{id}/blob`, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set. Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays whichever token authorized a redirect), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior |
 | `max_fork_count` | 16 | children a single `fork` may create; each is a full-RAM VM, so this bounds one request's memory blast radius to the node's capacity |
 | `refill_concurrency` | 0 (auto) | concurrent VM provisioning budget, shared by warm-pool refills, fork clones, and the reap/hibernate/reconcile engine batches. 0 sizes it from the node: `NumCPU*2/3` clamped to [4, 256] — a 384-core node gets 256; small nodes keep a floor of 4 |
 | `release_delay_seconds` | 0 | seconds a released VM waits in the removal queue before `cocoon vm rm`. The claim is dropped and journaled and its egress listener closed at release; the VM removal, the volume hold release and the egress tap unlock run on the first reap tick (5 s) after the delay, on the `refill_concurrency` budget like every other batch teardown, so a burst of releases does not compete with the claims still running. Until then the VM holds its memory and its volume reservations without counting as a claim. 0 removes inline |
 | `preview_listen` | (off) | address for a preview HTTP server that serves guest ports under signed URLs; needs `preview_secret` |
 | `preview_secret` | — | cluster-shared HMAC secret signing preview tokens (all nodes share one) |
-| `preview_advertise` | = `preview_listen` | the browser-facing preview base URL; nodes behind one TLS proxy may share it, while signed tokens route internally through each owner's `advertise_addr` |
+| `preview_advertise` | = `preview_listen` | the browser-facing preview base URL, minted into every preview URL, so it must name a routable host (a wildcard `preview_listen` needs it set explicitly); nodes behind one TLS proxy may share it, while signed tokens route internally through each owner's `advertise_addr` |
 | `checkpoint_dir` | `<data_dir>/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — records are generation-addressed with `meta.json` as the atomic commit pointer, so no cross-node locking is required of the filesystem. A re-publish retains its superseded export generation for at least ~1h and until a following hourly sweep so an in-flight clone that resolved the old metadata can finish; budget the current generation plus every generation retained across that grace-and-sweep window. An explicit delete can make a concurrent clone fail visibly. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide) |
 | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. Re-publish retains prior export generations until Delete so an in-flight fetch that selected old metadata can finish; budget storage for those generations. An explicit S3 Delete can still make a concurrent fetch that has not finished materializing fail visibly. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` |
 | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the expiry eligibility point for a healed replica a delete broadcast missed, after which its next successful hourly sweep removes it; persistent sweep failure extends retention until one succeeds, so it is not a hard ceiling |

docs/desktop.md

@@ -31,7 +31,7 @@ before it joins the warm pool, so the snapshot already carries a listening
 server:
 
 ```json
-"warmup": ["bash", "-lc", "for i in $(seq 1 1100); do curl -sf -m 2 -o /dev/null http://127.0.0.1:5000/screenshot && exit 0; sleep 0.1; done; exit 1"]
+"warmup": ["bash", "-lc", "end=$((SECONDS+100)); until curl -sf -m 2 -o /dev/null http://127.0.0.1:5000/screenshot; do [ $SECONDS -ge $end ] && exit 1; sleep 0.1; done"]
 ```
 
 Measured on a 192-core node running the OSWorld task set at 96 concurrent

docs/egress.md

@@ -3,8 +3,9 @@
 A sandbox gets the **effect** of a credential, never the credential. Outbound
 access is an allow-listed, audited resource enforced on the host: the secret
 lives host-side and enters no guest memory, so prompt injection can exfiltrate
-at most the proxy's answers, and every credentialed call lands in the audit and
-usage journals keyed by sandbox and tenant. Default-deny.
+at most the proxy's answers, and every credentialed call lands in the usage
+journal keyed by sandbox and tenant (and in the audit journal when `audit_log`
+is on). Default-deny.
 
 The same host proxy also serves the **none lane** (a NIC-less Cloud Hypervisor
 guest) over vsock, so a network-less sandbox can call approved APIs with
@@ -61,7 +62,7 @@ opts in with rules that all carry a nonempty `methods` list without
 `CONNECT`, or all `intercept`, is rejected at load. A pool whose policy does
 not opt in pays
 nothing for the door on the claim path, whatever its tenants' policies say.
-Audit lines carry `"method":"SOCKS5"`.
+Audit lines (with `audit_log` on) carry `"method":"SOCKS5"`.
 
 ```sh
 curl --socks5-hostname 127.0.0.1:1080 imaps://imap.example.com/   # allowed: {"socks5": true, "allow": [{"host": "imap.example.com"}]}
@@ -203,9 +204,10 @@ woken sandbox binds at arm time.
   method and the secret injected (see below). Only a pool rule may set it.
 - No policy on a claim ⇒ no egress at all (the proxy is not started).
 
-Each decision is written to `audit.jsonl` (`op:"egress"`, `dest`, `port`,
-`decision`, and the secret **name** in `secret`) and metered as an `egress`
-usage event.
+Each decision is metered as an `egress` usage event and, when `audit_log` is
+on, written to `audit.jsonl` (`op:"egress"`, `dest`, `port`, `method`,
+`decision`, and the secret **name** in `secret`); an intercepted `CONNECT` is
+recorded as a decision of its own before its inner requests.
 
 ## HTTPS interception
 

docs/index.md

@@ -3,7 +3,7 @@
 MicroVM sandboxes for AI agents, built on
 [cocoon](https://github.com/cocoonstack/cocoon). Warm claims are
 sub-millisecond; a pool miss clones from a golden snapshot in tens of
-milliseconds; cold boot is ~200ms on bare metal.
+milliseconds; cold boot is ~0.2–0.4 s on bare metal.
 
 ```
 SDK (Go/Python)         sandboxd (per node)              guest microVM

docs/openai-adapter.md

@@ -33,7 +33,7 @@ pair over the sync [Python SDK](sdk-python.md), bridged with
 |---|---|
 | `create` | `Client.new` — one claimed sandbox per session |
 | session `exec` | `Sandbox.run` (stdout/stderr/exit); a per-call `timeout` is `run`'s wall clock and surfaces as `TimeoutError` |
-| `read` / `write` | `Sandbox.read_file` / `write_file`; a missing path on `read` → `FileNotFoundError` |
+| `read` / `write` | `Sandbox.read_file` / `write_file`; a missing path on `read` → `FileNotFoundError`; a per-call `user` raises `NotImplementedError` (fs verbs carry no user on the wire) |
 | `persist_workspace` / `hydrate_workspace` | `Sandbox.pull` / `push` (tar) |
 | exposed port | `Sandbox.proxy_port` |
 | `delete` | `Sandbox.close` (release) |

docs/performance.md

@@ -24,8 +24,8 @@ bare metal, `small` tier:
 | cold boot (no golden yet) | **~215–400 ms** | full boot from the template image to silkd answering |
 
 A guarded-egress claim binds its proxy doors at refill rather than at claim
-(#177): on bare metal that took the warm claim with the HTTP door from 307 to
-263 µs p50, and with both doors from 351 to 274 µs.
+(#177, cocoon-test2 bare metal, 2026-09-14): that took the warm claim with the
+HTTP door from 307 to 263 µs p50, and with both doors from 351 to 274 µs.
 
 Cloud Hypervisor lifecycle latency (bare metal, vsock agent-ready):
 
@@ -72,8 +72,9 @@ TLS edge, a handshake. From proto 2 silkd serves RPCs back to back, and both
 SDKs keep one relay connection per handle with a 30s idle window.
 
 `e2e/cmd/rpcbench`, warm `rt:24.04` sandbox, `net=none`, n=200 `fs_stat`
-RPCs per arm, bare metal, client on the node. The two arms interleave sample
-by sample and swap which one leads, so drift cannot land on one of them.
+RPCs per arm, cocoon-test1 bare metal, client on the node, 2026-09-16 at
+5158a14. The two arms interleave sample by sample and swap which one leads,
+so drift cannot land on one of them.
 
 | per-RPC wall | plain p50 | plain p90 | TLS edge p50 | TLS edge p90 |
 | --- | --- | --- | --- | --- |
@@ -175,7 +176,8 @@ pre-dialer declined — behind a TLS edge it tracks the dial arm, because it
 hides a handshake only while RPCs arrive slower than a dial completes.
 
 **Template pre-check meta GET**: a single `ReadMeta` against MinIO
-measures 4.76ms cross-host (sub-ms node-local, 20-50ms on WAN S3). It
+measured 4.76ms cross-host (sub-ms node-local, 20-50ms on WAN S3; a
+2026-07 spot check, host and commit not recorded). It
 would fire once per warm-miss claim of a promoted template whose key
 gossip advertises, against a provision that already costs a >=48ms clone
 plus the export fetch. Declined until a measurement shows it matters.