From 51a7236feb115cf66bc655f10fa436ad1dae4129 Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 3 Sep 2026 03:39:36 +0800 Subject: [PATCH 1/6] =?UTF-8?q?review:=20sandboxd=20=E2=80=94=20claims=20p?= =?UTF-8?q?rojection=20off=20the=20claim=20path,=20netfilter=20single=20li?= =?UTF-8?q?sting,=20leased=20mesh=20epoch,=20comment=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim store keeps its persisted projection incrementally (set/del/mark under the store mutex, the file write under a second one), so a claim no longer copies every live claim under the manager mutex (5000 claims: 580 µs and 5018 allocs → 30 ns, 0 allocs). Startup egress resync lists locked taps once instead of one netlink socket per claim; the mesh epoch is persisted as a leased ceiling so gossip churn stops fsyncing twice a second; the guest port handshake reads through the existing buffered reader; docs/sandboxd-api.md states that reconcile records nothing for dropped claims. Test files carry no comments; the comment budget is applied across the daemon (−1683 lines). --- docs/sandboxd-api.md | 5 +- sandboxd/ca.go | 6 +- sandboxd/config/config.go | 138 ++++++---------------- sandboxd/config/config_test.go | 5 +- sandboxd/egress/bench_test.go | 3 - sandboxd/egress/ca.go | 18 +-- sandboxd/egress/ca_issue.go | 6 +- sandboxd/egress/intercept.go | 23 ++-- sandboxd/egress/intercept_test.go | 13 -- sandboxd/egress/policy.go | 35 ++---- sandboxd/egress/policy_test.go | 7 +- sandboxd/egress/proxy.go | 51 +++----- sandboxd/egress/proxy_test.go | 5 - sandboxd/egress/secrets.go | 16 +-- sandboxd/engine/capacity_test.go | 2 - sandboxd/engine/cloneargs_test.go | 8 -- sandboxd/engine/engine.go | 92 ++++++--------- sandboxd/engine/engine_test.go | 11 +- sandboxd/engine/installca.go | 8 +- sandboxd/engine/installca_test.go | 8 +- sandboxd/engine/portconn.go | 37 +++--- sandboxd/engine/portconn_test.go | 18 +-- sandboxd/engine/silkd.go | 9 +- sandboxd/engine/volume.go | 9 +- sandboxd/main.go | 6 +- sandboxd/mesh/epoch.go | 4 +- sandboxd/mesh/mesh.go | 87 ++++++-------- sandboxd/mesh/mesh_test.go | 16 ++- sandboxd/mesh/state_test.go | 35 +++++- sandboxd/netfilter/nft_linux.go | 32 ++--- sandboxd/netfilter/nft_other.go | 5 +- sandboxd/pool/archive.go | 40 ++----- sandboxd/pool/archive_test.go | 110 ++--------------- sandboxd/pool/checkpoint.go | 109 +++++------------ sandboxd/pool/checkpoint_heal_test.go | 132 +++------------------ sandboxd/pool/checkpoint_test.go | 36 ++---- sandboxd/pool/claim.go | 93 ++++++--------- sandboxd/pool/claims.go | 110 +++++++++++------ sandboxd/pool/claims_bench_test.go | 37 +++--- sandboxd/pool/claims_test.go | 69 ++++++++--- sandboxd/pool/drain.go | 4 +- sandboxd/pool/egress.go | 44 ++----- sandboxd/pool/egress_test.go | 93 +++++---------- sandboxd/pool/fork.go | 15 +-- sandboxd/pool/fork_test.go | 28 ++--- sandboxd/pool/hibernate.go | 87 ++++---------- sandboxd/pool/hibernate_test.go | 77 ++---------- sandboxd/pool/idle_test.go | 5 +- sandboxd/pool/intercept_test.go | 3 +- sandboxd/pool/journal.go | 24 ++-- sandboxd/pool/journal_test.go | 28 ++++- sandboxd/pool/operator.go | 18 +-- sandboxd/pool/pool.go | 163 +++++++++----------------- sandboxd/pool/pool_test.go | 115 ++++++++---------- sandboxd/pool/poolstore.go | 19 +-- sandboxd/pool/poolstore_test.go | 6 +- sandboxd/pool/promote_test.go | 16 +-- sandboxd/pool/reconcile.go | 62 ++++------ sandboxd/pool/reconcile_test.go | 2 - sandboxd/pool/refill.go | 69 ++++------- sandboxd/pool/refillbackoff_test.go | 9 +- sandboxd/pool/refillkick_test.go | 4 - sandboxd/pool/remove.go | 9 +- sandboxd/pool/remove_test.go | 1 - sandboxd/pool/setpools.go | 17 +-- sandboxd/pool/stats.go | 14 +-- sandboxd/pool/stats_test.go | 16 +-- sandboxd/pool/telemetry.go | 23 +--- sandboxd/pool/telemetry_test.go | 7 +- sandboxd/pool/template.go | 97 +++++---------- sandboxd/pool/volume.go | 50 +++----- sandboxd/pool/volume_attach_test.go | 14 +-- sandboxd/pool/volume_rw_test.go | 10 +- sandboxd/pool/volume_test.go | 9 +- sandboxd/pool/watermark.go | 22 +--- sandboxd/pool/watermark_test.go | 3 - sandboxd/server/metrics.go | 4 +- sandboxd/server/preview.go | 5 +- sandboxd/server/relay.go | 40 ++----- sandboxd/server/relay_test.go | 17 +-- sandboxd/server/server.go | 142 ++++++---------------- sandboxd/server/server_http.go | 21 +--- sandboxd/server/server_test.go | 121 +++---------------- sandboxd/store/dir/dir.go | 22 +--- sandboxd/store/dir/dir_test.go | 28 +---- sandboxd/store/peer/broadcast.go | 10 +- sandboxd/store/peer/broadcast_test.go | 14 +-- sandboxd/store/peer/peer.go | 13 +- sandboxd/store/peer/peer_test.go | 32 +---- sandboxd/store/peer/probe.go | 74 +++--------- sandboxd/store/peer/probe_test.go | 56 +-------- sandboxd/store/peer/transport.go | 32 ++--- sandboxd/store/peer/transport_test.go | 41 +------ sandboxd/store/s3/s3.go | 41 ++----- sandboxd/store/s3/s3_test.go | 11 +- sandboxd/store/store.go | 42 ++----- sandboxd/store/storetest/storetest.go | 3 +- sandboxd/types/api.go | 54 +++------ sandboxd/types/bench_test.go | 4 - sandboxd/types/types.go | 104 +++++----------- sandboxd/utils/utils.go | 16 +-- 101 files changed, 1083 insertions(+), 2501 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index dbe2ade2..542f2aa6 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -500,8 +500,9 @@ silently. Folding rules: billable compute seconds per sandbox = Σ(hibernate→wake) minus the archived span; archived storage seconds = Σ(archive→unarchive/archive_delete), a cheaper store tier than a hibernated VM's RAM. An interval left open by a crash clamps to the claim's -deadline, and the next reconcile emits `reap` for claims it drops. The `vm` -name joins cocoon's machine-level metering ledger for audit cross-checks. +deadline; reconcile records nothing for the claims it drops, so a restart +leaves those intervals without a terminal event. The `vm` name joins +cocoon's machine-level metering ledger for audit cross-checks. ## GET /v1/sandboxes/{id}/agent diff --git a/sandboxd/ca.go b/sandboxd/ca.go index b2c806e2..840ee6cf 100644 --- a/sandboxd/ca.go +++ b/sandboxd/ca.go @@ -10,8 +10,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/egress" ) -// runCA is the operator PKI tool: mint the cluster root once, then issue a -// per-node intermediate from it. The root key stays with the operator. +// runCA is the operator PKI tool: mint the cluster root, then per-node intermediates. func runCA(args []string) error { if len(args) == 0 { return fmt.Errorf("usage: sandboxd ca {init|issue-intermediate}") @@ -103,8 +102,7 @@ func writeCAFiles(dir, name string, certPEM, keyPEM []byte, force bool) error { return nil } -// writeKeyMaterial writes path with O_EXCL unless force; the Stat precheck -// above is a courtesy. +// writeKeyMaterial writes path with O_EXCL unless force. func writeKeyMaterial(path string, data []byte, perm os.FileMode, force bool) error { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC if !force { diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index ec4973c7..2d4cd98d 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -39,33 +39,23 @@ type PoolSpec struct { types.PoolKey Warm int `json:"warm"` - // WarmMax, when >0, turns on the demand-adaptive watermark: the warm - // target may rise from Warm (the floor) toward WarmMax while claims - // arrive faster than the pool provisions, and decays back in silence. + // WarmMax, when >0, lets the warm target rise from Warm toward it under demand. WarmMax int `json:"warm_max,omitempty"` - // Egress is this pool's allow-list, intersected with the tenant's for the - // effective policy (a request must pass both); when both matched rules - // name a secret, the pool's is injected. Nil denies all egress. + // Egress is this pool's allow-list, intersected with the tenant's; nil denies all egress. Egress *egress.Policy `json:"egress,omitempty"` - // IdleHibernateSeconds, when >0, hibernates this none-lane pool's idle claims - // after that many seconds without a data-plane connection; the next - // call wakes them transparently. Zero disables. + // IdleHibernateSeconds, when >0, hibernates idle claims after that many seconds. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` - // ArchiveAfterSeconds, when >0, checkpoints a hibernated claim to the - // store after that many idle seconds and drops its local VM; a later call - // restores it. Requires IdleHibernateSeconds>0 and must exceed it. + // ArchiveAfterSeconds, when >0, archives a hibernated claim; must exceed IdleHibernateSeconds. ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` - // ArchiveDeleteAfterSeconds, when >0, purges an archived claim's store - // checkpoint that many seconds after archiving. Zero keeps it forever. + // ArchiveDeleteAfterSeconds, when >0, purges the checkpoint that long after archiving. ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` } -// ValidateLimits checks the warm/watermark/idle bounds — shared by the -// config file path and the PUT /v1/pools path so the two cannot drift. +// ValidateLimits checks the warm/watermark/idle bounds shared by config and PUT /v1/pools. func (s PoolSpec) ValidateLimits() error { if s.Warm < 0 { return fmt.Errorf("warm must not be negative") @@ -88,9 +78,7 @@ type StoreConfig struct { S3 *s3.Config `json:"s3,omitempty"` } -// EgressCAConfig provisions HTTPS interception: RootCert is the cluster root -// baked into guests; IntermediateCert/Key are this node's signing CA. The root -// private key never appears here. +// EgressCAConfig provisions HTTPS interception; the root private key never appears here. type EgressCAConfig struct { RootCert string `json:"root_cert"` IntermediateCert string `json:"intermediate_cert"` @@ -103,9 +91,6 @@ func (e *EgressCAConfig) Set() bool { } // TenantSpec declares one tenant: its bearer token and its live-claim quota. -// APIToken stays the operator (root) credential with full access; tenant -// tokens reach the resource-creating verbs only, and everything a tenant -// creates is stamped with its name. type TenantSpec struct { Name string `json:"name"` Token string `json:"token"` //nolint:gosec // config field, not a hardcoded credential @@ -115,9 +100,7 @@ type TenantSpec struct { Egress *egress.Policy `json:"egress,omitempty"` } -// VolumeSpec declares one operator-managed dataset disk. The entry — name, -// access list, mode — is declared fleet-wide; Writable admits rw claims, and -// the image at Path is held by exactly one node. +// VolumeSpec declares one operator-managed dataset disk, held by exactly one node. type VolumeSpec struct { Name string `json:"name"` Path string `json:"path"` @@ -126,10 +109,7 @@ type VolumeSpec struct { Tenants []string `json:"tenants,omitempty"` } -// MeshConfig configures cluster membership. Two v1 constraints: every node -// shares one APIToken and Tenants set (a redirect replays the claim's token, -// so a peer missing that tenant answers 401), and only a node with its own -// egress attachment can take a redirected egress claim. +// MeshConfig configures cluster membership; every node shares one APIToken and tenant set. type MeshConfig struct { NodeID string `json:"node_id"` // unique name; defaults to Bind Bind string `json:"bind"` // memberlist host:port @@ -137,8 +117,7 @@ type MeshConfig struct { ClusterKey string `json:"cluster_key,omitempty"` // base64 gossip-encryption key } -// ParsedBind splits Bind into host and port, rejecting a wildcard host — -// memberlist would advertise an unroutable address. +// ParsedBind splits Bind into host and port, rejecting a wildcard host. func (mc *MeshConfig) ParsedBind() (string, int, error) { host, portStr, err := net.SplitHostPort(mc.Bind) if err != nil { @@ -177,50 +156,34 @@ type Config struct { DataDir string `json:"data_dir"` CocoonBin string `json:"cocoon_bin"` - // AdvertiseAddr is the host:port the data plane reaches this node at; it - // is returned as a claim's owner address (and, at M2c, gossiped). Defaults - // to Listen, which is correct when Listen is a routable host:port. + // AdvertiseAddr is the host:port the data plane reaches this node at; defaults to Listen. AdvertiseAddr string `json:"advertise_addr,omitempty"` - // Bridges shards egress-lane VMs over several host bridge devices with - // the raw TAP-on-bridge attachment: taps stay in the root netns, so - // guarded egress can lock them. Mutually exclusive with Networks; with - // neither set the node serves only the no-network lane. + // Bridges shards egress-lane VMs over host bridges; their taps stay lockable in the root netns. Bridges []string `json:"bridges,omitempty"` - // Networks shards egress-lane VMs over several CNI conflists, one Linux - // bridge each: a bridge holds at most 1024 ports (BR_MAX_PORTS, no - // sysctl), so N conflists raise the node's egress ceiling to N×1024. + // Networks shards egress-lane VMs over CNI conflists; a bridge holds at most 1024 ports. Networks []string `json:"networks,omitempty"` - // RestoreMode rides clones and wake restores as cocoon's --restore-mode. Opt-in: mmap - // needs every node's cloud-hypervisor to carry CoW restore — an older CH - // silently eager-copies while reporting success. + // RestoreMode is cocoon's --restore-mode; an older CH silently eager-copies an mmap restore. RestoreMode types.RestoreMode `json:"restore_mode,omitempty"` // NoDirectIO enables buffered writable disks for cold boots and clones. NoDirectIO bool `json:"no_direct_io,omitempty"` - // APIToken, when set, guards claim and info; per-sandbox tokens guard - // sandbox-scoped calls regardless. + // APIToken, when set, guards claim and info. APIToken string `json:"api_token,omitempty"` //nolint:gosec // config field, not a hardcoded credential - // Tenants adds per-tenant bearer tokens next to APIToken; empty keeps - // the single-token behavior. + // Tenants adds per-tenant bearer tokens next to APIToken. Tenants []TenantSpec `json:"tenants,omitempty"` - // Secrets registers node-side credentials the egress proxy injects by - // name; values come from the environment (value_env), never this file. + // Secrets registers node-side credentials; values come from value_env, never this file. Secrets []egress.SecretSpec `json:"secrets,omitempty"` - // IdleHibernateSeconds is the idle policy for unpooled none-lane claims - // (template and checkpoint claims); per-pool settings override it for - // pooled keys. Zero disables. + // IdleHibernateSeconds is the idle policy for unpooled claims; per-pool settings override. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` - // ArchiveAfterSeconds / ArchiveDeleteAfterSeconds are the archive policy - // for unpooled keys; per-pool settings override them. Same rules as - // PoolSpec (archive_after must exceed idle_hibernate). + // ArchiveAfterSeconds and ArchiveDeleteAfterSeconds are the archive policy for unpooled keys. ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` @@ -229,58 +192,40 @@ type Config struct { // PreviewAdvertise is the browser-facing base, shareable fleet-wide behind one proxy. PreviewAdvertise string `json:"preview_advertise,omitempty"` - // CheckpointDir is where checkpoints live; defaults to - // /checkpoints. Point it at a shared FUSE mount (JuiceFS over - // object storage, NFS) and every node sharing the mount resolves every - // checkpoint — the path's filesystem is the operator's choice. + // CheckpointDir is where checkpoints live; defaults to /checkpoints. CheckpointDir string `json:"checkpoint_dir,omitempty"` - // CheckpointStore selects the checkpoint backend; absent means the - // dir backend at CheckpointDir. Kind "s3" stores checkpoints in object - // storage (credentials from the AWS chain, never this file). + // CheckpointStore selects the checkpoint backend; absent means the dir backend. CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` - // CheckpointPeerHeal lets a node pull a checkpoint it lacks from a peer - // rather than failing the branch. Requires cluster_key (the pull presents - // the fleet token to a probed address) and a nonzero checkpoint TTL (the - // revocation bound for a replica a delete broadcast missed). Off by default. + // CheckpointPeerHeal lets a node pull a checkpoint it lacks from a peer; off by default. CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` - // EgressInternalAllow re-admits CIDRs through the proxy's SSRF guard, - // node-wide (every pool and tenant). Prefixes, not a permit-private - // switch: the guest bridges are themselves ULA/RFC1918. + // EgressInternalAllow re-admits CIDRs through the proxy's SSRF guard, node-wide. EgressInternalAllow []string `json:"egress_internal_allow,omitempty"` - // CheckpointTTLHours ages out checkpoints (0 = keep forever); the - // sweep runs hourly and on startup. + // CheckpointTTLHours ages out checkpoints; 0 keeps them forever. CheckpointTTLHours int `json:"checkpoint_ttl_hours,omitempty"` - // MaxClaims caps live claims node-wide; 0 means unlimited. Claim, - // fork, and checkpoint-branch requests beyond it answer 429. + // MaxClaims caps live claims node-wide; 0 means unlimited. MaxClaims int `json:"max_claims,omitempty"` - // AuditLog, when true, appends every relayed request frame's op and - // addressing fields (never payloads) to /audit.jsonl. + // AuditLog, when true, appends relayed request ops, never payloads, to audit.jsonl. AuditLog bool `json:"audit_log,omitempty"` - // MaxForkCount caps children per fork call — each child is a full-RAM VM. - // Defaults to 16. + // MaxForkCount caps children per fork call; each child is a full-RAM VM. MaxForkCount int `json:"max_fork_count,omitempty"` // Volumes is the node-local catalog of operator-managed dataset images. Volumes []VolumeSpec `json:"volumes,omitempty"` - // RefillConcurrency caps concurrent VM provisioning node-wide — warm-pool - // refills, fork clones, and reap/hibernate/reconcile batches share one - // budget; 0 auto-scales with the node's CPU count. + // RefillConcurrency caps concurrent VM provisioning node-wide; 0 auto-scales with CPUs. RefillConcurrency int `json:"refill_concurrency,omitempty"` - // Mesh, when set, joins this node to a memberlist cluster for redirect - // placement; nil is a single node (mesh of one, no gossip). + // Mesh, when set, joins this node to a memberlist cluster; nil is a mesh of one. Mesh *MeshConfig `json:"mesh,omitempty"` - // EgressCA provisions HTTPS interception; required when any pool has an - // intercept rule. + // EgressCA provisions HTTPS interception; required for any intercept rule. EgressCA *EgressCAConfig `json:"egress_ca,omitempty"` Pools []PoolSpec `json:"pools"` @@ -291,10 +236,7 @@ func (c *Config) HasEgress() bool { return len(c.Bridges) > 0 || len(c.Networks) > 0 } -// ClusterDigest fingerprints the must-match config so a divergent node is -// caught early, not at a later 401 or an outlived replica. With cluster_key it -// HMACs token material too; without one only tenant names, the CA root and the -// checkpoint TTL ride the (maybe cleartext) wire. +// ClusterDigest fingerprints the must-match config; without cluster_key it omits tokens. func (c *Config) ClusterDigest(caFingerprint string) string { names := make([]string, len(c.Tenants)) for i, t := range c.Tenants { @@ -320,8 +262,7 @@ func (c *Config) ClusterDigest(caFingerprint string) string { return hex.EncodeToString(sum[:]) } -// guardsEgressLane: any tenant policy counts (claims mint egress keys without a -// pool), a none-lane pool policy does not (it locks no tap). +// guardsEgressLane counts any tenant policy, but only an egress-lane pool policy. func (c *Config) guardsEgressLane() bool { return slices.ContainsFunc(c.Tenants, func(t TenantSpec) bool { return t.Egress != nil }) || slices.ContainsFunc(c.Pools, func(p PoolSpec) bool { return p.Net == types.NetEgress && p.Egress != nil }) @@ -350,8 +291,7 @@ func (c *Config) validate() error { if err := c.validateAttachment(); err != nil { return err } - // A CNI network's tap lives in the VM netns, unreachable from the root-netns - // nft lock; guarded egress needs a bridge. + // a CNI network's tap lives in the VM netns, unreachable from the root-netns nft lock. if len(c.Networks) > 0 && c.guardsEgressLane() { return fmt.Errorf("guarded egress needs a bridge lane, not a CNI network: the tap lives in the VM netns and cannot be locked") } @@ -502,8 +442,7 @@ func (c *Config) validateSecrets() (map[string]struct{}, error) { return names, nil } -// validateAttachment checks the egress-lane attachment; a repeated shard -// would report N shards while filling one bridge. +// validateAttachment checks the egress-lane attachment; a repeated shard fills one bridge. func (c *Config) validateAttachment() error { if len(c.Bridges) > 0 && len(c.Networks) > 0 { return fmt.Errorf("bridges and networks are mutually exclusive") @@ -514,8 +453,7 @@ func (c *Config) validateAttachment() error { return validateShards(c.Networks, "networks", "conflist") } -// validateEgressAllow rejects a malformed prefix at load: a typo would -// otherwise surface as a refused dial long after startup. +// validateEgressAllow rejects a malformed prefix at load, not at a later refused dial. func (c *Config) validateEgressAllow() error { for _, cidr := range c.EgressInternalAllow { if _, err := netip.ParsePrefix(cidr); err != nil { @@ -574,8 +512,7 @@ func Load(path string) (*Config, error) { return cfg, nil } -// validatePolicy checks the rules and that each secret ref is registered; a -// nil policy is valid (deny-all). +// validatePolicy checks the rules and secret refs; a nil policy is deny-all. func validatePolicy(p *egress.Policy, secrets map[string]struct{}) error { if p == nil { return nil @@ -594,8 +531,7 @@ func validatePolicy(p *egress.Policy, secrets map[string]struct{}) error { return nil } -// validateArchiveWindow checks the archive thresholds shared by PoolSpec and -// the node Config: non-negative, and archive_after must sit past idle_hibernate. +// validateArchiveWindow requires archive_after past idle_hibernate, both non-negative. func validateArchiveWindow(idle, after, del int) error { if after < 0 || del < 0 { return fmt.Errorf("archive seconds must not be negative") diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index ac37e344..aebb4345 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -23,7 +23,7 @@ func TestClusterDigest(t *testing.T) { if base.ClusterDigest("other-fp") == d { t.Error("an egress CA root change is not reflected") } - // Keyless: token material must stay off the (maybe cleartext) wire. + if (&Config{APIToken: "other", PreviewSecret: "ps", Tenants: base.Tenants}).ClusterDigest("ca-fp") != d { t.Error("api_token leaked into the keyless digest") } @@ -207,7 +207,6 @@ func TestLoadAcceptsEgressPolicy(t *testing.T) { } func TestLoadAcceptsUnguardedCNINetwork(t *testing.T) { - // Only guarded egress needs a bridge; an unguarded CNI network lane is fine. path := writeConfig(t, `{"networks":["cni"],"pools":[{"template":"rt:24.04","net":"egress","size":"small"}]}`) if _, err := Load(path); err != nil { t.Fatalf("Load: %v", err) @@ -215,8 +214,6 @@ func TestLoadAcceptsUnguardedCNINetwork(t *testing.T) { } func TestLoadAcceptsNoneLanePolicyOnCNI(t *testing.T) { - // A none-lane policy rides the vsock proxy and locks no tap, so it is valid on - // a CNI network — the bridge requirement is only for a guarded egress lane. for _, body := range []string{ `{"networks":["cni"],"pools":[{"template":"rt:24.04","net":"none","size":"small","egress":{"allow":[{"host":"x"}]}}]}`, `{"networks":["cni"],"pools":[{"template":"rt:24.04","net":"egress","size":"small"},{"template":"rt:24.04","net":"none","size":"medium","egress":{"allow":[{"host":"x"}]}}]}`, diff --git a/sandboxd/egress/bench_test.go b/sandboxd/egress/bench_test.go index 6ac3a146..54f3bf9a 100644 --- a/sandboxd/egress/bench_test.go +++ b/sandboxd/egress/bench_test.go @@ -73,9 +73,6 @@ func benchCA(b *testing.B) *CA { return ca } -// benchFront serves an HTTPS echo behind the proxy; intercept toggles the rule, -// and guest roots match: the cluster root when intercepted, the origin's own -// cert when spliced. func benchFront(b *testing.B, intercept bool) (proxyAddr string, roots *x509.CertPool) { b.Helper() upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/sandboxd/egress/ca.go b/sandboxd/egress/ca.go index 8fe93fe1..0faa2259 100644 --- a/sandboxd/egress/ca.go +++ b/sandboxd/egress/ca.go @@ -24,8 +24,7 @@ const ( pemTypeCertificate = "CERTIFICATE" ) -// CA signs per-host interception leaves with the node's intermediate. The -// shared cluster root is baked into guests; the root key never reaches a node. +// CA signs per-host interception leaves with the node's intermediate. type CA struct { rootPEM []byte rootFP string @@ -35,8 +34,7 @@ type CA struct { leafKey *ecdsa.PrivateKey } -// LoadCA builds the node CA; the intermediate must be signed by the bundle's -// first root cert (a bundle carries old+new roots during a rotation). +// LoadCA builds the node CA; the intermediate must be signed by the bundle's first cert. func LoadCA(rootCertPEM, interCertPEM, interKeyPEM []byte) (*CA, error) { root, err := parseRootBundle(rootCertPEM) if err != nil { @@ -59,8 +57,7 @@ func LoadCA(rootCertPEM, interCertPEM, interKeyPEM []byte) (*CA, error) { if err = inter.CheckSignatureFrom(root); err != nil { return nil, fmt.Errorf("intermediate not signed by root: %w", err) } - // Reject a self-signed intermediate: the root must not double as one, which - // would put the root key on the node. + // reject a self-signed intermediate: it would put the root key on the node. if inter.CheckSignatureFrom(inter) == nil { return nil, fmt.Errorf("intermediate must not be a self-signed root") } @@ -86,12 +83,10 @@ func LoadCA(rootCertPEM, interCertPEM, interKeyPEM []byte) (*CA, error) { // CertPEM is the cluster root to bake into an intercepted guest's trust store. func (c *CA) CertPEM() []byte { return c.rootPEM } -// Fingerprint is the SHA-256 of the baked root PEM; changed bytes rebuild the -// golden, a rotated intermediate does not. +// Fingerprint is the SHA-256 of the baked root PEM, so a rotated intermediate keeps it. func (c *CA) Fingerprint() string { return c.rootFP } -// SignLeaf issues a fresh interception leaf for host, chained [leaf, -// intermediate] for a guest that trusts the root. +// SignLeaf issues a fresh interception leaf for host, chained [leaf, intermediate]. func (c *CA) SignLeaf(host string) (*tls.Certificate, error) { serial, err := serialNumber() if err != nil { @@ -146,8 +141,7 @@ func parseCert(pemBytes []byte, what string) (*x509.Certificate, error) { return cert, nil } -// parseRootBundle returns the first cert (the anchor) and rejects any non-CA -// block: the whole file lands in guest trust stores. +// parseRootBundle returns the anchor and rejects a non-CA block: guests trust the whole file. func parseRootBundle(pemBytes []byte) (*x509.Certificate, error) { var anchor *x509.Certificate for rest := pemBytes; ; { diff --git a/sandboxd/egress/ca_issue.go b/sandboxd/egress/ca_issue.go index 6d2677c7..696aee02 100644 --- a/sandboxd/egress/ca_issue.go +++ b/sandboxd/egress/ca_issue.go @@ -16,8 +16,7 @@ const ( interValidity = 5 * 365 * 24 * time.Hour ) -// GenerateRoot mints a cluster root CA (cert + key PEM). Run once per cluster; -// the key stays with the operator and never reaches a node. +// GenerateRoot mints a cluster root CA; the key stays with the operator, never a node. func GenerateRoot(commonName string) (certPEM, keyPEM []byte, err error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { @@ -40,8 +39,7 @@ func GenerateRoot(commonName string) (certPEM, keyPEM []byte, err error) { return marshalCA(tmpl, tmpl, &key.PublicKey, key, key) } -// IssueIntermediate signs a per-node intermediate CA (leaves only, path length -// zero) from the cluster root, to provision to nodeID. +// IssueIntermediate signs a per-node intermediate CA from the cluster root. func IssueIntermediate(rootCertPEM, rootKeyPEM []byte, nodeID string) (certPEM, keyPEM []byte, err error) { root, err := parseCert(rootCertPEM, "root") if err != nil { diff --git a/sandboxd/egress/intercept.go b/sandboxd/egress/intercept.go index 93f68184..21b223c8 100644 --- a/sandboxd/egress/intercept.go +++ b/sandboxd/egress/intercept.go @@ -15,14 +15,11 @@ const ( interceptTimeout = 30 * time.Second leafRenewBefore = 24 * time.Hour - // maxLeaves bounds one sandbox's leaf cache against a wildcard rule minting - // unbounded per-host leaves. + // maxLeaves bounds one sandbox's leaf cache against a wildcard rule. maxLeaves = 1024 ) -// serveIntercept terminates a matched CONNECT's TLS with a node-signed leaf the -// guest trusts, then serves the decrypted HTTP/1.1 through interceptHandler. -// Upstream is re-originated with real-root verification (mitmTr), never trusted. +// serveIntercept terminates a matched CONNECT's TLS with a node-signed leaf. func (p *Proxy) serveIntercept(w http.ResponseWriter, r *http.Request, host string) { leaf, err := p.leafFor(host) if err != nil { @@ -47,8 +44,7 @@ func (p *Proxy) serveIntercept(w http.ResponseWriter, r *http.Request, host stri if _, err := io.WriteString(client, "HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { return } - // A guest that never sends a ClientHello would pin this goroutine until the - // sandbox dies; cleared once the handshake lands. + // a guest that never sends a ClientHello would pin this goroutine until the sandbox dies. _ = client.SetDeadline(time.Now().Add(interceptTimeout)) tlsConn := tls.Server(client, cfg) if err := tlsConn.HandshakeContext(r.Context()); err != nil { @@ -63,8 +59,7 @@ func (p *Proxy) serveIntercept(w http.ResponseWriter, r *http.Request, host stri _ = srv.Serve(ln) } -// leafFor returns this sandbox's cached leaf for host, signing a fresh one on -// miss or near-expiry; the cache and its flush are per-sandbox. +// leafFor returns the cached leaf for host, signing a fresh one on miss or near-expiry. func (p *Proxy) leafFor(host string) (*tls.Certificate, error) { p.leafMu.Lock() defer p.leafMu.Unlock() @@ -82,8 +77,7 @@ func (p *Proxy) leafFor(host string) (*tls.Certificate, error) { return crt, nil } -// interceptHandler forwards one decrypted request per call: host is the CONNECT -// authority's hostname (policy + leaf key), authority its host:port (upstream). +// interceptHandler forwards one decrypted request per call; host is bare, authority host:port. type interceptHandler struct { proxy *Proxy host string @@ -96,17 +90,14 @@ func (h *interceptHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.relay(w, r, h.host, rule, decision, p.mitmTr, func(out *http.Request) { out.URL.Scheme = "https" out.URL.Host = h.authority - // Keep the guest's Host when it names the CONNECT host (SigV4-style - // signing breaks on the ":443" rewrite); a foreign Host snaps to the - // authority. + // keep the guest's Host when it names the CONNECT host: SigV4 signing breaks on a rewrite. if !strings.EqualFold(hostOnly(r.Host), h.host) { out.Host = h.authority } }) } -// singleConnListener feeds one already-accepted conn to an http.Server and then -// blocks until that conn closes, so Serve exits once the tunnel ends. +// singleConnListener feeds one accepted conn to an http.Server, then blocks until it closes. type singleConnListener struct { conn net.Conn handed atomic.Bool diff --git a/sandboxd/egress/intercept_test.go b/sandboxd/egress/intercept_test.go index 79058635..109b2f10 100644 --- a/sandboxd/egress/intercept_test.go +++ b/sandboxd/egress/intercept_test.go @@ -70,9 +70,6 @@ func TestInterceptInjectsSecretIntoHTTPS(t *testing.T) { } func TestInterceptBindsInnerRequestToInterceptRule(t *testing.T) { - // A broad plain rule precedes the specific intercept rule for the same - // host: the inner request must still bind to the intercept rule (inject its - // secret), not first-match the shadowing plain rule. policy := Policy{Allow: []Rule{ {Host: "*.example.com"}, {Host: "api.example.com", Secret: "gh", Intercept: true}, @@ -93,9 +90,6 @@ func TestInterceptBindsInnerRequestToInterceptRule(t *testing.T) { } func TestInterceptCaptureEnforcesRuleMethod(t *testing.T) { - // The intercept rule is GET-only; a broad plain rule would allow POST. Once - // a host is captured for interception, the intercept rule governs its - // decrypted traffic — POST must be denied, not routed via the plain rule. policy := Policy{Allow: []Rule{ {Host: "*.example.com"}, {Host: "api.example.com", Methods: []string{"GET"}, Intercept: true}, @@ -116,9 +110,6 @@ func TestInterceptCaptureEnforcesRuleMethod(t *testing.T) { } func TestInterceptReachesLaterInterceptRuleByMethod(t *testing.T) { - // A wildcard intercept rule (GET) precedes a specific intercept rule (POST) - // for the same host. The inner POST must reach the second rule by host+method - // — binding the whole tunnel to the first intercept rule makes it unreachable. policy := Policy{Allow: []Rule{ {Host: "*.example.com", Methods: []string{"GET"}, Intercept: true}, {Host: "api.example.com", Methods: []string{"POST"}, Intercept: true}, @@ -257,8 +248,6 @@ func trustUpstream(upstream *httptest.Server) *x509.CertPool { return pool } -// connectTLS opens a CONNECT tunnel through the proxy and completes the guest -// TLS handshake against the leaf the proxy presents. func connectTLS(t *testing.T, proxyAddr, target, serverName string, roots *x509.CertPool) *tls.Conn { t.Helper() conn, err := net.Dial("tcp", proxyAddr) @@ -318,8 +307,6 @@ func roundTripTLSHost(t *testing.T, tc *tls.Conn, method, host string) *http.Res return resp } -// readPreamble reads the CONNECT reply byte-wise up to the blank line, so no -// TLS bytes are consumed; the proxy sends nothing more until the ClientHello. func readPreamble(t *testing.T, conn net.Conn) string { t.Helper() var sb strings.Builder diff --git a/sandboxd/egress/policy.go b/sandboxd/egress/policy.go index da3b914e..49f175f7 100644 --- a/sandboxd/egress/policy.go +++ b/sandboxd/egress/policy.go @@ -18,13 +18,7 @@ const ( // Decision is the policy verdict for one request. type Decision int -// Rule allows requests whose host matches Host — an exact name, a -// "*."-prefixed suffix wildcard ("*.example.com" matches sub.example.com but -// not the apex), or "*" for any host — and whose method is in Methods (empty -// means any). Secret names a node-side credential the proxy injects into a -// matched request; empty injects nothing. Intercept terminates a matched -// HTTPS CONNECT so the request is filtered by method and the secret injected; -// only a pool rule may set it. Host matching is case-insensitive. +// Rule allows requests matching Host (exact, "*." suffix, or "*") and Methods; empty means any. type Rule struct { Host string `json:"host"` Methods []string `json:"methods,omitempty"` @@ -53,20 +47,17 @@ func (r Rule) matchMethod(method string) bool { slices.ContainsFunc(r.Methods, func(m string) bool { return strings.EqualFold(m, method) }) } -// Policy is one sandbox's egress allow-list. Default-deny is implicit: a -// request matching no Allow rule is denied. +// Policy is one sandbox's egress allow-list; a request matching no rule is denied. type Policy struct { Allow []Rule `json:"allow"` } -// Intercepts reports whether any rule terminates HTTPS; nil is false. It gates -// baking the cluster root cert into a pool's golden. +// Intercepts reports whether any rule terminates HTTPS; nil is false. func (p *Policy) Intercepts() bool { return p != nil && slices.ContainsFunc(p.Allow, func(r Rule) bool { return r.Intercept }) } -// Validate rejects a rule with an empty or bare-wildcard host; an empty -// allow-list is valid (it denies everything). +// Validate rejects an empty or bare-wildcard host; an empty allow-list denies everything. func (p Policy) Validate() error { for i, r := range p.Allow { switch r.Host { @@ -79,10 +70,7 @@ func (p Policy) Validate() error { return nil } -// Eval returns the first matching non-intercept allow rule with DecisionAllow, -// or a zero rule with DecisionDeny. host is a bare hostname (no port). Intercept -// rules answer only the CONNECT paths (EvalHost/EvalInner) — matching one here -// would inject its secret over cleartext. +// Eval returns the first matching non-intercept rule; matching one here would leak its secret. func (p Policy) Eval(host, method string) (Rule, Decision) { host = strings.ToLower(host) for _, r := range p.Allow { @@ -93,8 +81,7 @@ func (p Policy) Eval(host, method string) (Rule, Decision) { return Rule{}, DecisionDeny } -// EvalHost matches by host only (a CONNECT tunnel carries no method), preferring -// an intercept rule over an earlier plain match: interception is a host property. +// EvalHost matches by host only, preferring an intercept rule over an earlier plain match. func (p Policy) EvalHost(host string) (Rule, Decision) { host = strings.ToLower(host) first := -1 @@ -113,9 +100,7 @@ func (p Policy) EvalHost(host string) (Rule, Decision) { return Rule{}, DecisionDeny } -// EvalInner decides a decrypted request over an intercepted tunnel: it matches -// only intercept rules, by host+method, so a plain rule can neither shadow the -// injection nor rescue a method the intercept rules reject. +// EvalInner matches only intercept rules, so a plain rule cannot shadow or rescue one. func (p Policy) EvalInner(host, method string) (Rule, Decision) { host = strings.ToLower(host) for _, r := range p.Allow { @@ -126,16 +111,14 @@ func (p Policy) EvalInner(host, method string) (Rule, Decision) { return Rule{}, DecisionDeny } -// Evaluator is what the proxy consults per request; Policy is one, Compose -// intersects two. +// Evaluator is what the proxy consults per request. type Evaluator interface { Eval(host, method string) (Rule, Decision) EvalHost(host string) (Rule, Decision) EvalInner(host, method string) (Rule, Decision) } -// Compose intersects a pool and a tenant policy: a request must pass both, and -// the pool rule (with its secret) wins on a double allow. +// Compose intersects a pool and a tenant policy; the pool rule wins on a double allow. func Compose(pool, tenant Policy) Evaluator { return composite{pool: pool, tenant: tenant} } diff --git a/sandboxd/egress/policy_test.go b/sandboxd/egress/policy_test.go index d2f1ae6f..6f9dd9ff 100644 --- a/sandboxd/egress/policy_test.go +++ b/sandboxd/egress/policy_test.go @@ -19,7 +19,7 @@ func TestPolicyEval(t *testing.T) { {"exact host case-insensitive", "API.GitHub.com", "POST", DecisionAllow, "gh"}, {"exact host wrong method falls through to catch-all deny", "api.github.com", "DELETE", DecisionDeny, ""}, {"wildcard subdomain", "storage.googleapis.com", "PUT", DecisionAllow, ""}, - {"wildcard does not match apex", "googleapis.com", "GET", DecisionAllow, ""}, // caught by "*"/GET + {"wildcard does not match apex", "googleapis.com", "GET", DecisionAllow, ""}, {"catch-all get", "example.com", "GET", DecisionAllow, ""}, {"catch-all denies non-get", "example.com", "POST", DecisionDeny, ""}, } @@ -41,8 +41,7 @@ func TestEvalSkipsInterceptRules(t *testing.T) { {Host: "api.github.com", Secret: "gh", Intercept: true}, {Host: "plain.github.com"}, }} - // An intercept-only host must deny the plaintext forward path outright — - // matching would inject the rule's secret over cleartext. + if rule, d := p.Eval("api.github.com", "GET"); d != DecisionDeny || rule.Secret != "" { t.Errorf("Eval on an intercept-only host = %+v/%v, want deny", rule, d) } @@ -70,7 +69,7 @@ func TestEvalHostPrefersInterceptRule(t *testing.T) { func TestEvalInnerMatchesOnlyInterceptRulesByMethod(t *testing.T) { p := Policy{Allow: []Rule{ - {Host: "*.example.com"}, // plain: must not shadow or rescue + {Host: "*.example.com"}, {Host: "api.example.com", Methods: []string{"GET"}, Secret: "gh", Intercept: true}, {Host: "api.example.com", Methods: []string{"POST"}, Intercept: true}, }} diff --git a/sandboxd/egress/proxy.go b/sandboxd/egress/proxy.go index 65b0a2e6..1bdda05f 100644 --- a/sandboxd/egress/proxy.go +++ b/sandboxd/egress/proxy.go @@ -19,27 +19,21 @@ import ( const idleConnTimeout = 90 * time.Second -// hopHeaders are hop-by-hop and proxy-scoped headers this hop owns: stripped -// from forwarded requests and from relayed responses rather than passed on. +// hopHeaders are hop-by-hop and proxy-scoped headers this hop owns, never passed on. var hopHeaders = []string{ "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Proxy-Connection", "TE", "Trailer", "Transfer-Encoding", "Upgrade", } -// DialFunc opens the upstream connection for a permitted request. The proxy -// stays transport-agnostic: the node wires the real egress path (a bridge -// dial, or a checked resolver) behind it. +// DialFunc opens the upstream connection for a permitted request. type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) -// Secrets resolves a rule's Secret name to the header it injects. The value -// lives node-side and the proxy is its only holder, so a matched request -// carries the credential's effect without the guest ever seeing it. +// Secrets resolves a rule's Secret name to the header it injects. type Secrets interface { Header(name string) (header, value string, ok bool) } -// Event is one audited egress attempt, keyed by sandbox and tenant. Injected -// names the credential added to a permitted request, never its value. +// Event is one audited egress attempt; Injected names the credential, never its value. type Event struct { Sandbox string Tenant string @@ -49,10 +43,7 @@ type Event struct { Injected string } -// Proxy is one sandbox's forward proxy: an http.Handler serving CONNECT -// tunnels and absolute-form HTTP, gated by Policy and audited per request. -// One Proxy carries one sandbox's identity, so the connection a listener -// hands it needs no in-band token. +// Proxy is one sandbox's forward proxy, gated by Policy and audited per request. type Proxy struct { sandbox string tenant string @@ -64,8 +55,7 @@ type Proxy struct { tr *http.Transport mitmTr *http.Transport - // leaves caches this sandbox's interception leaves; per-proxy, so one - // sandbox's host churn never evicts or blocks another's. + // leaves caches this sandbox's interception leaves; per-proxy, never shared. leafMu sync.Mutex leaves map[string]*tls.Certificate @@ -75,8 +65,7 @@ type Proxy struct { closed bool } -// New builds a Proxy for one sandbox. secrets, ca, and audit may be nil (no -// injection, no HTTPS interception, no audit). dial must be set. +// New builds a Proxy for one sandbox; secrets, ca, and audit may be nil, dial must not. func New(sandbox, tenant string, policy Evaluator, secrets Secrets, ca *CA, dial DialFunc, audit func(Event)) *Proxy { p := &Proxy{ sandbox: sandbox, @@ -86,8 +75,7 @@ func New(sandbox, tenant string, policy Evaluator, secrets Secrets, ca *CA, dial ca: ca, audit: audit, dial: dial, - // The stdlib default of 2 idle conns per host re-dials bursty - // same-host plaintext traffic. + // the stdlib default of 2 idle conns per host re-dials bursty same-host traffic. tr: &http.Transport{DialContext: dial, MaxIdleConnsPerHost: 8, IdleConnTimeout: idleConnTimeout}, conns: map[net.Conn]struct{}{}, } @@ -107,8 +95,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.serveForward(w, r) } -// Close ends every hijacked tunnel, which outlives http.Server.Close, so a -// released claim's guest cannot keep an authorized session. Idempotent. +// Close ends every hijacked tunnel, which outlives http.Server.Close; idempotent. func (p *Proxy) Close() { p.connMu.Lock() conns := slices.Collect(maps.Keys(p.conns)) @@ -145,9 +132,7 @@ func (p *Proxy) untrack(conn net.Conn) { // serveConnect gates an HTTPS/opaque tunnel by host. func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request) { host := hostOnly(r.Host) - // Host-gate the interception decision: the tunnel's CONNECT verb is not the - // request method, so a matched intercept rule enforces its methods on the - // decrypted inner request. The plain splice keeps the host+method gate. + // host-gate interception: the tunnel's CONNECT verb is not the request method. if rule, d := p.policy.EvalHost(host); d == DecisionAllow && rule.Intercept && p.ca != nil { p.serveIntercept(w, r, host) return @@ -190,8 +175,7 @@ func (p *Proxy) serveForward(w http.ResponseWriter, r *http.Request) { p.relay(w, r, host, rule, decision, p.tr, nil) } -// relay forwards one policy-evaluated request upstream and mirrors the -// response; prepare, when set, rewrites the clone before it leaves. +// relay forwards one policy-evaluated request upstream and mirrors the response. func (p *Proxy) relay(w http.ResponseWriter, r *http.Request, host string, rule Rule, decision Decision, tr *http.Transport, prepare func(*http.Request)) { if decision == DecisionDeny { p.record(Event{Method: r.Method, Host: host, Decision: decision}) @@ -220,9 +204,7 @@ func (p *Proxy) relay(w http.ResponseWriter, r *http.Request, host string, rule _, _ = io.Copy(w, resp.Body) } -// inject sets the header the rule's Secret resolves to, overwriting any -// guest-supplied value, and returns the secret name it applied (empty when -// the rule names none or the store cannot resolve it). +// inject sets the rule's secret header, overwriting any guest-supplied value. func (p *Proxy) inject(rule Rule, h http.Header) string { if rule.Secret == "" || p.secrets == nil { return "" @@ -257,8 +239,7 @@ func stripHop(h http.Header) { } } -// splice copies bytes both ways until either side ends, then returns; each -// direction's end half-closes the peer so its EOF propagates. +// splice copies both ways until either side ends, half-closing the peer so EOF propagates. func splice(a, b net.Conn) { done := make(chan struct{}) go func() { @@ -271,14 +252,12 @@ func splice(a, b net.Conn) { <-done } -// denied answers a policy rejection with a typed 403 the guest's client -// surfaces as a distinct failure rather than a hang. +// denied answers a policy rejection with a 403 rather than a hang. func denied(w http.ResponseWriter, host string) { http.Error(w, fmt.Sprintf("egress denied: %s", host), http.StatusForbidden) } -// hostOnly strips a port from an authority, tolerating a bare host and IPv6 -// literals. +// hostOnly strips a port from an authority, tolerating a bare host and IPv6 literals. func hostOnly(authority string) string { if host, _, err := net.SplitHostPort(authority); err == nil { return host diff --git a/sandboxd/egress/proxy_test.go b/sandboxd/egress/proxy_test.go index 2e9c3a8a..d568cc53 100644 --- a/sandboxd/egress/proxy_test.go +++ b/sandboxd/egress/proxy_test.go @@ -115,8 +115,6 @@ func TestForwardNeverInjectsInterceptSecret(t *testing.T) { } } -// TestForwardStripsHopHeaders: fixed and Connection-named hop headers die at -// this hop, in both directions. func TestForwardStripsHopHeaders(t *testing.T) { var gotHop, gotConn string upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -250,8 +248,6 @@ func (f fakeSecrets) Header(name string) (string, string, bool) { return hv[0], hv[1], ok } -// fixedDial ignores the requested address and connects to target, so a test -// policy can name any host while the bytes land on a local backend. func fixedDial(target string) DialFunc { return func(ctx context.Context, network, _ string) (net.Conn, error) { var d net.Dialer @@ -298,7 +294,6 @@ func readStatus(t *testing.T, br *bufio.Reader) string { return line } -// echoServer starts a one-shot TCP echo backend and returns its address. func echoServer(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/sandboxd/egress/secrets.go b/sandboxd/egress/secrets.go index deb43157..a26486ae 100644 --- a/sandboxd/egress/secrets.go +++ b/sandboxd/egress/secrets.go @@ -8,9 +8,7 @@ import ( "golang.org/x/net/http/httpguts" ) -// nonInjectable rejects secret headers the transport owns or strips — -// injecting one would audit as done while the request carries something -// else. Derived from the proxy's hop set so the two lists cannot drift. +// nonInjectable rejects secret headers the transport owns or strips. var nonInjectable = func() map[string]struct{} { m := map[string]struct{}{"host": {}, "content-length": {}} for _, h := range hopHeaders { @@ -19,9 +17,7 @@ var nonInjectable = func() map[string]struct{} { return m }() -// SecretSpec declares a node-side credential the proxy injects: Header is the -// request header it sets, valued from the ValueEnv env var — the literal -// never sits in the config file. +// SecretSpec declares a node-side credential the proxy injects, valued from ValueEnv. type SecretSpec struct { Name string `json:"name"` Header string `json:"header"` @@ -51,16 +47,12 @@ type resolvedSecret struct { value string } -// SecretStore is the resolved node-side credential registry, implementing the -// Proxy's Secrets interface. Values live only here — never in a policy or a -// gossiped struct. +// SecretStore is the resolved node-side credential registry; values live only here. type SecretStore struct { byName map[string]resolvedSecret } -// NewSecretStore resolves each spec's value; an unset or empty ValueEnv is an -// error, never a silently-empty credential, and a value a header cannot carry -// (CTLs would split the injected request) is rejected before it can be sent. +// NewSecretStore resolves each spec's value; an unset or invalid one is an error. func NewSecretStore(specs []SecretSpec) (*SecretStore, error) { byName := make(map[string]resolvedSecret, len(specs)) for _, s := range specs { diff --git a/sandboxd/engine/capacity_test.go b/sandboxd/engine/capacity_test.go index 5c1be760..5595fabe 100644 --- a/sandboxd/engine/capacity_test.go +++ b/sandboxd/engine/capacity_test.go @@ -5,8 +5,6 @@ import ( "testing" ) -// TestOnlyNodeWideFailuresCarryACapacitySignature guards the classifier. -// Parking on an ordinary clone failure would stall a healthy node over one bad VM. func TestOnlyNodeWideFailuresCarryACapacitySignature(t *testing.T) { for _, tc := range []struct { name string diff --git a/sandboxd/engine/cloneargs_test.go b/sandboxd/engine/cloneargs_test.go index ffa2540e..fe333429 100644 --- a/sandboxd/engine/cloneargs_test.go +++ b/sandboxd/engine/cloneargs_test.go @@ -64,8 +64,6 @@ func TestLifecycleArgsApplyDirectIOPolicy(t *testing.T) { } } -// TestEgressVMsSpreadOverEveryConfiguredShard pins the point of both shard -// lists: a shard the hash never picks is bridge capacity that does not exist. func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { shards := []string{"sbx0", "sbx1", "sbx2", "sbx3"} key := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} @@ -87,7 +85,6 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { counts[args[j+1]]++ } for _, s := range shards { - // The bound catches a starved shard, not non-uniformity. if counts[s] < 4000/len(shards)/2 { t.Errorf("shard %s got %d of 4000, want a fair share: %v", s, counts[s], counts) } @@ -96,8 +93,6 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { } } -// TestNetworkChoiceIsStableForAName guards restore and teardown: the record -// persists the network a VM was built on, so the choice must not move. func TestNetworkChoiceIsStableForAName(t *testing.T) { shards := []string{"a", "b", "c"} first := shardOf(shards, "sbx-pool1-42") @@ -108,9 +103,6 @@ func TestNetworkChoiceIsStableForAName(t *testing.T) { } } -// TestNetArgsHonorsTheLaneAndTheAttachment keeps the three attachment shapes -// distinct: no NIC on the none lane, a device on the bridge lane, and a -// single-entry list byte-for-byte the old scalar. func TestNetArgsHonorsTheLaneAndTheAttachment(t *testing.T) { none := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeMedium} egress := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 06c3e9b8..330264d2 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -28,7 +28,6 @@ const ( // RequiredCocoon carries the snapshot/store performance baseline. RequiredCocoon = "v0.5.2" - // Stale-create verdicts, mirroring cocoon's reconcile-stale-create verb. StaleCreateCollected StaleCreateOutcome = "collected" StaleCreateBusy StaleCreateOutcome = "busy" StaleCreateNotCreating StaleCreateOutcome = "not-creating" @@ -47,15 +46,11 @@ const ( infoMax = 4096 // info response frame cap outputTail = 400 - // portForwardMax caps the port_forward handshake reply line; the byte - // stream follows on the same conn. + // portForwardMax caps the port_forward handshake reply line. portForwardMax = 4096 ) -// capacitySignatures mean "this node cannot attach another VM", not "this VM -// failed"; matched as text because cocoon is a subprocess and the distinction -// survives only in its stderr. "exchange full" is EXFULL from a bridge at -// BR_MAX_PORTS — permanent until VMs are removed. +// capacitySignatures mean the node cannot attach another VM, not that this VM failed. var capacitySignatures = []string{ "exchange full", "no space left on device", @@ -73,15 +68,12 @@ type Engine struct { restoreMode types.RestoreMode } -// New returns a cocoon engine with node-wide network and disk policy. bridges -// and networks are the egress-lane shard lists (host bridge devices vs CNI -// conflists) egress VMs spread over; see shardOf. +// New returns a cocoon engine with node-wide network and disk policy. func New(bin string, bridges, networks []string, noDirectIO bool, restoreMode types.RestoreMode) *Engine { return &Engine{bin: bin, bridges: bridges, networks: networks, noDirectIO: noDirectIO, restoreMode: restoreMode} } -// Version reports cocoon's version string — a "vX.Y.Z" release or a -// "master-" dev build. +// Version reports cocoon's version string: a "vX.Y.Z" release or a "master-" dev build. func (e *Engine) Version(ctx context.Context) (string, error) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -97,9 +89,7 @@ func (e *Engine) Version(ctx context.Context) (string, error) { return "", fmt.Errorf("cocoon version: no Version line in output") } -// VersionWarning probes cocoon and returns a non-empty warning when its -// version is below RequiredCocoon (or unreadable); version is set when known. -// A dev build (no vX.Y.Z) is assumed current. +// VersionWarning returns a non-empty warning when cocoon's version is below RequiredCocoon. func (e *Engine) VersionWarning(ctx context.Context) (version, warning string) { v, err := e.Version(ctx) if err != nil { @@ -120,8 +110,7 @@ func (e *Engine) Clone(ctx context.Context, fromDir, name string, key types.Pool return parseRecord(ctx, out), nil } -// CloneSnap clones from a local-store snapshot by name — fork's fast path, -// which skips the export-to-dir copy that `Clone --from-dir` requires. +// CloneSnap clones from a local-store snapshot by name — fork's fast path. func (e *Engine) CloneSnap(ctx context.Context, snap, name string, key types.PoolKey) (types.VMRecord, error) { out, err := e.run(ctx, e.cloneSnapArgs(snap, name, key)...) if err != nil { @@ -130,8 +119,7 @@ func (e *Engine) CloneSnap(ctx context.Context, snap, name string, key types.Poo return parseRecord(ctx, out), nil } -// RunCold boots a VM from the template image (golden builds and cache-miss -// claims), returning its lifecycle record. +// RunCold boots a VM from the template image, returning its lifecycle record. func (e *Engine) RunCold(ctx context.Context, name string, key types.PoolKey) (types.VMRecord, error) { out, err := e.run(ctx, e.runColdArgs(name, key)...) if err != nil { @@ -146,8 +134,7 @@ func (e *Engine) Remove(ctx context.Context, name string) error { return err } -// ReconcileStaleCreate reclaims a creating-state record via cocoon's -// free-ops-lock predicate — safe against an in-flight clone where rm --force is not. +// ReconcileStaleCreate reclaims a creating-state record via cocoon's free-ops-lock predicate. func (e *Engine) ReconcileStaleCreate(ctx context.Context, name string) (StaleCreateOutcome, error) { out, err := e.run(ctx, "vm", "reconcile-stale-create", name, argOutput, formatJSON) if err != nil { @@ -168,15 +155,13 @@ func (e *Engine) SnapshotSave(ctx context.Context, vmName, snapName string) erro return err } -// Hibernate atomically snapshots a running VM under snapName and stops it, -// freeing its memory; Restore with the snapshot resumes it. +// Hibernate atomically snapshots a running VM under snapName and stops it. func (e *Engine) Hibernate(ctx context.Context, vmName, snapName string) error { _, err := e.run(ctx, "vm", "hibernate", argName, snapName, vmName) return err } -// Restore resumes a VM from a snapshot with its memory state and identity -// intact (cocoon reseeds entropy only on restore), returning its vsock UDS. +// Restore resumes a VM from a snapshot, returning its vsock UDS. func (e *Engine) Restore(ctx context.Context, vmName, snapRef string) (string, error) { out, err := e.run(ctx, e.restoreCmdArgs(vmName, snapRef)...) if err != nil { @@ -185,8 +170,7 @@ func (e *Engine) Restore(ctx context.Context, vmName, snapRef string) (string, e return parseRecord(ctx, out).VsockSocket, nil } -// SnapshotExport exports a snapshot into toDir (cocoon requires it absent or -// empty); the result pairs with `vm clone --from-dir`. +// SnapshotExport exports a snapshot into toDir, which cocoon requires absent or empty. func (e *Engine) SnapshotExport(ctx context.Context, snapName, toDir string) error { _, err := e.run(ctx, "snapshot", "export", snapName, "--to-dir", toDir) return err @@ -238,8 +222,7 @@ func (e *Engine) List(ctx context.Context, filters ...string) ([]types.VMRecord, return vms, nil } -// DialSilkd connects to a VM's silkd through the hybrid-vsock UDS: -// dial, send "CONNECT ", expect an "OK" reply. +// DialSilkd connects to a VM's silkd through the hybrid-vsock UDS. func (e *Engine) DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) { var d net.Dialer conn, err := d.DialContext(ctx, "unix", vsockSocket) @@ -267,8 +250,7 @@ func (e *Engine) DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, e return conn, nil } -// Probe polls until silkd completes an info round-trip — the claim-ready -// signal (probe what the product uses, not cocoon-agent). +// Probe polls until silkd completes an info round-trip — the claim-ready signal. func (e *Engine) Probe(ctx context.Context, vsockSocket string, timeout time.Duration) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -285,9 +267,7 @@ func (e *Engine) Probe(ctx context.Context, vsockSocket string, timeout time.Dur } } -// DialGuestPort opens a raw byte stream to 127.0.0.1:port inside the guest -// by driving silkd's port_forward handshake, then returns the connection -// carrying the relayed TCP bytes — the preview server proxies HTTP over it. +// DialGuestPort opens a raw byte stream to 127.0.0.1:port inside the guest. func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uint16) (net.Conn, error) { conn, err := e.DialSilkd(ctx, vsockSocket) if err != nil { @@ -304,7 +284,9 @@ func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uin _ = conn.Close() return nil, fmt.Errorf("write port_forward: %w", err) } - line, readErr := readLine(conn, portForwardMax) + // the forwarded stream's own reader takes the handshake reply, so an over-read stays buffered. + r := bufio.NewReaderSize(conn, portReadBuf) + line, readErr := readBufLine(r, portForwardMax) if readErr != nil { _ = conn.Close() if ctxErr := ctx.Err(); ctxErr != nil { @@ -321,13 +303,11 @@ func (e *Engine) DialGuestPort(ctx context.Context, vsockSocket string, port uin _ = conn.Close() return nil, fmt.Errorf("port_forward %d: %s", port, respFail(resp)) } - return newGuestPortConn(conn), nil + return newGuestPortConn(conn, r), nil } func (e *Engine) cloneArgs(fromDir, name string, key types.PoolKey) []string { - // --pull: a checkpoint/template export carries only COW + memory; on a - // cross-node claim the base image blobs resolve locally or are pulled - // by digest. + // --pull: a checkpoint/template export carries only COW + memory, not the base image. args := []string{"vm", "clone", "--from-dir", fromDir, argName, name, "--pull", argOutput, formatJSON, e.directIOArg()} args = append(args, e.restoreArgs()...) return append(args, e.netArgs(name, key, false)...) @@ -403,8 +383,7 @@ func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string) error { if _, err = conn.Write(append(probe, '\n')); err != nil { return fmt.Errorf("write info: %w", err) } - // Unlike the handshake reader, buffered over-read is safe here: the conn - // is discarded after this one reply. + // buffered over-read is safe here: the conn is discarded after this one reply. reply, err := bufio.NewReader(io.LimitReader(conn, infoMax)).ReadBytes('\n') if err != nil { return fmt.Errorf("read info reply: %w", err) @@ -419,8 +398,7 @@ func (e *Engine) infoRoundTrip(ctx context.Context, vsockSocket string) error { return nil } -// EgressSocketPath is the host UDS the VMM connects when the guest dials -// CID2:egressPort — sandboxd listens here to serve the egress proxy. +// EgressSocketPath is the host UDS the VMM connects when the guest dials CID2:egressPort. func EgressSocketPath(vsockSocket string) string { return fmt.Sprintf("%s_%d", vsockSocket, egressPort) } @@ -439,9 +417,7 @@ func CapacitySignature(err error) string { return "" } -// shardOf picks a shard by hashing the VM name, not by counter: the record -// persists the attachment a VM was built on, so the choice must be -// reproducible without process state. +// shardOf picks a shard by hashing the VM name, so the choice needs no process state. func shardOf(shards []string, name string) string { if len(shards) == 0 { return "" @@ -452,8 +428,7 @@ func shardOf(shards []string, name string) string { return shards[int(h.Sum32()>>1)%len(shards)] } -// respFail renders a non-success reply: the error frame's own text, or the -// unexpected frame's type. +// respFail renders a non-success reply: the error text, or the unexpected frame's type. func respFail(resp wire.Response) string { if errResp, ok := resp.(*wire.ErrorResp); ok { return errResp.Error() @@ -461,8 +436,7 @@ func respFail(resp wire.Response) string { return "unexpected frame " + resp.RespType() } -// parseRecord reads a lifecycle command's --output json VM record; best-effort, -// so an unparseable record yields the zero value (callers fall back to vm list). +// parseRecord is best-effort: an unparseable record yields the zero value. func parseRecord(ctx context.Context, out []byte) types.VMRecord { var rec types.VMRecord if err := json.Unmarshal(out, &rec); err != nil { @@ -472,8 +446,7 @@ func parseRecord(ctx context.Context, out []byte) types.VMRecord { return rec } -// belowFloor reports whether cocoon version v is below RequiredCocoon; -// comparable is false for a non-release build (no vX.Y.Z), assumed current. +// belowFloor reports whether v is below RequiredCocoon; comparable is false for a dev build. func belowFloor(v string) (below, comparable bool) { cur, ok := parseSemver(v) if !ok { @@ -503,8 +476,19 @@ func parseSemver(s string) ([3]int, bool) { return out, true } -// readLine reads byte-wise so nothing past the newline is consumed — -// the same conn carries the silkd protocol right after the handshake. +// readBufLine takes one line from r, capped at max bytes. +func readBufLine(r *bufio.Reader, max int) (string, error) { + line, err := r.ReadSlice('\n') + if err != nil { + return "", err + } + if len(line) > max { + return "", fmt.Errorf("reply exceeds %d bytes", max) + } + return string(line[:len(line)-1]), nil +} + +// readLine reads byte-wise so nothing past the newline is consumed. func readLine(conn net.Conn, max int) (string, error) { var sb strings.Builder var b [1]byte diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index d16967a4..790a9bfb 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -20,8 +20,7 @@ const ( func TestDialSilkdConsumesOnlyHandshake(t *testing.T) { path := sockPath(t) - // Sentinel rides in the same write as the handshake: if DialSilkd - // over-reads past the newline, the first Read below loses it. + listenMuxer(t, path, "OK 2048\nX") conn, err := New("cocoon", nil, nil, false, "").DialSilkd(t.Context(), path) @@ -112,7 +111,7 @@ func TestDialGuestPortCtxCancel(t *testing.T) { if _, err := io.WriteString(conn, "OK 2048\n"); err != nil { return } - _, _ = r.ReadString('\n') // consume port_forward, never answer + _, _ = r.ReadString('\n') <-hold }() @@ -132,10 +131,6 @@ func TestProbeTimeout(t *testing.T) { } } -// listenMuxer fakes the hybrid-vsock muxer on a UDS: per connection it -// consumes the CONNECT line, replies with handshake, then answers the first -// frame line of connection i with frames[i] (last frame repeats; no frames -// means handshake only). func listenMuxer(t *testing.T, path, handshake string, frames ...string) { t.Helper() ln, err := net.Listen("unix", path) @@ -173,8 +168,6 @@ func serveConn(conn net.Conn, handshake, frame string) { _, _ = io.WriteString(conn, frame) } -// sockPath returns a socket path in a fresh short-prefix temp dir — -// t.TempDir() paths can exceed darwin's 104-byte sun_path limit. func sockPath(t *testing.T) string { t.Helper() dir, err := os.MkdirTemp("", "sbx") diff --git a/sandboxd/engine/installca.go b/sandboxd/engine/installca.go index 9269b8dc..439cebc1 100644 --- a/sandboxd/engine/installca.go +++ b/sandboxd/engine/installca.go @@ -13,15 +13,11 @@ import ( const ( caCertGuestPath = "/usr/local/share/ca-certificates/sandbox-egress.crt" caBundlePath = "/etc/ssl/certs/ca-certificates.crt" - // guestExecPATH is set on the guest command because silkd starts it with an - // empty environment. + // guestExecPATH is set because silkd starts the guest command with an empty environment. guestExecPATH = "/usr/sbin:/usr/bin:/sbin:/bin" ) -// InstallCACert makes the guest trust the cluster root: drop the cert in the -// source dir and append it to the active bundle. It avoids update-ca-certificates -// on purpose — silkd runs before systemd-tmpfiles clears /tmp, where that script -// stages temp files. Off the claim path (golden build / cold provision). +// InstallCACert makes the guest trust the cluster root without update-ca-certificates. func (e *Engine) InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error { ctx, cancel := context.WithTimeout(ctx, cmdTimeout) defer cancel() diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 8fa3cb0c..8c502c43 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -2,10 +2,12 @@ package engine import ( "bufio" + "bytes" "encoding/json" "fmt" "io" "net" + "slices" "strings" "sync" "testing" @@ -117,7 +119,7 @@ func serveFakeSilkd(t *testing.T, path string) *fakeSilkd { func (f *fakeSilkd) serve(conn net.Conn) { defer func() { _ = conn.Close() }() r := bufio.NewReader(conn) - if _, err := r.ReadString('\n'); err != nil { // CONNECT line + if _, err := r.ReadString('\n'); err != nil { return } if _, err := io.WriteString(conn, "OK 2048\n"); err != nil { @@ -202,7 +204,7 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri func (f *fakeSilkd) handleList(conn net.Conn, path string) { f.mu.Lock() f.listCalls++ - entries, reply := append([]wire.DirEntry(nil), f.block...), f.listErr + entries, reply := slices.Clone(f.block), f.listErr f.mu.Unlock() if reply != "" { writeFakeSilkdResponse(conn, &wire.ErrorResp{Kind: wire.KindInternal, Message: reply}) @@ -225,7 +227,7 @@ func (f *fakeSilkd) handleRead(conn net.Conn, path string) { f.readMisses[path]-- ok = false } - data = append([]byte(nil), data...) + data = bytes.Clone(data) f.mu.Unlock() if reply != "" { writeFakeSilkdResponse(conn, &wire.ErrorResp{Kind: wire.KindInternal, Message: reply}) diff --git a/sandboxd/engine/portconn.go b/sandboxd/engine/portconn.go index c0b34b0f..16176a95 100644 --- a/sandboxd/engine/portconn.go +++ b/sandboxd/engine/portconn.go @@ -13,30 +13,26 @@ import ( ) const ( - // portWriteChunk keeps each data frame (base64 ×4/3 + envelope) well - // under silkd's 8MiB frame cap, so a large HTTP body is split across - // frames. + // portWriteChunk keeps each data frame well under silkd's 8MiB frame cap. portWriteChunk = 1 << 20 - // portReadBuf fits silkd's data frames (32KiB chunks, base64-expanded) - // in one buffered read. + // portReadBuf fits silkd's data frames in one buffered read. portReadBuf = 64 << 10 ) var portDataHead = []byte(`{"type":"data","data":"`) -// guestPortConn adapts silkd's newline-JSON data-frame port_forward channel to -// a plain net.Conn (base64 payloads), the server-side twin of the SDK's -// PortConn, so a reverse proxy can splice HTTP straight through it. +// guestPortConn adapts silkd's port_forward data frames to a plain net.Conn. type guestPortConn struct { net.Conn r *bufio.Reader wbuf []byte + rbuf []byte pending []byte } -func newGuestPortConn(conn net.Conn) *guestPortConn { - return &guestPortConn{Conn: conn, r: bufio.NewReaderSize(conn, portReadBuf)} +func newGuestPortConn(conn net.Conn, r *bufio.Reader) *guestPortConn { + return &guestPortConn{Conn: conn, r: r} } func (g *guestPortConn) Read(p []byte) (int, error) { @@ -45,7 +41,7 @@ func (g *guestPortConn) Read(p []byte) (int, error) { if err != nil { return 0, err } - if data, ok := fastPortData(line); ok { + if data, ok := g.fastPortData(line); ok { g.pending = data continue } @@ -74,8 +70,7 @@ func (g *guestPortConn) Write(p []byte) (int, error) { written := 0 for len(p) > 0 { n := min(len(p), portWriteChunk) - // The hot relay path renders into one reused buffer instead of - // allocating two frame-sized slices per chunk. + // the hot relay path reuses one buffer instead of allocating per chunk. g.wbuf = wire.AppendBulkRequest(g.wbuf, "data", p[:n]) if _, err := g.Conn.Write(g.wbuf); err != nil { return written, err @@ -86,10 +81,8 @@ func (g *guestPortConn) Write(p []byte) (int, error) { return written, nil } -// fastPortData slices the canonical data frame's base64 out without a JSON -// parse; the SDK's fastBulk sets the contract: only the exact canonical -// shape takes the slice, anything else falls back to the full parse. -func fastPortData(line []byte) ([]byte, bool) { +// fastPortData slices the canonical data frame's base64 out without a JSON parse. +func (g *guestPortConn) fastPortData(line []byte) ([]byte, bool) { after, ok := bytes.CutPrefix(line, portDataHead) if !ok { return nil, false @@ -98,10 +91,14 @@ func fastPortData(line []byte) ([]byte, bool) { if !ok || !bytes.Equal(rest, []byte("}\n")) { return nil, false } - out := make([]byte, base64.StdEncoding.DecodedLen(len(b64))) - n, err := base64.StdEncoding.Decode(out, b64) + if size := base64.StdEncoding.DecodedLen(len(b64)); cap(g.rbuf) < size { + g.rbuf = make([]byte, size) + } else { + g.rbuf = g.rbuf[:size] + } + n, err := base64.StdEncoding.Decode(g.rbuf, b64) if err != nil || base64.StdEncoding.EncodedLen(n) != len(b64) { return nil, false } - return out[:n], true + return g.rbuf[:n], true } diff --git a/sandboxd/engine/portconn_test.go b/sandboxd/engine/portconn_test.go index 0311e9b8..759488de 100644 --- a/sandboxd/engine/portconn_test.go +++ b/sandboxd/engine/portconn_test.go @@ -1,6 +1,7 @@ package engine import ( + "bufio" "bytes" "encoding/base64" "encoding/json" @@ -14,9 +15,8 @@ import ( func TestGuestPortConnFramesBothWays(t *testing.T) { client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - gc := newGuestPortConn(client) + gc := newGuestPortConn(client, bufio.NewReaderSize(client, portReadBuf)) - // Read: a silkd data frame decodes to raw bytes; done maps to EOF. go func() { frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("HTTP/1.1 200 OK"))}) _, _ = silk.Write(append(frame, '\n')) @@ -35,7 +35,7 @@ func TestGuestPortConnFramesBothWays(t *testing.T) { func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - gc := newGuestPortConn(client) + gc := newGuestPortConn(client, bufio.NewReaderSize(client, portReadBuf)) go func() { _, _ = gc.Write([]byte("GET / HTTP/1.1\r\n")) }() @@ -54,9 +54,6 @@ func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { } } -// TestGuestPortConnRoundTripsFixtures pins the engine's hand-rolled framing -// to the shared protocol corpus: portconn is not built on sdk/go/silkd, so -// without this test it could drift from the frames both real peers agree on. func TestGuestPortConnRoundTripsFixtures(t *testing.T) { const fixtureDir = "../../protocol/wire/fixtures/v1" fixture := func(name string) []byte { @@ -70,9 +67,8 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - gc := newGuestPortConn(client) + gc := newGuestPortConn(client, bufio.NewReaderSize(client, portReadBuf)) - // Read path: the corpus data frame decodes to its payload, done is EOF. go func() { _, _ = silk.Write(fixture("resp_data.json")) _, _ = silk.Write(fixture("resp_done.json")) @@ -86,19 +82,17 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { t.Errorf("fixture done frame: %v, want EOF", err) } - // Error frames surface their type, not a silent close. client2, silk2 := net.Pipe() t.Cleanup(func() { _ = client2.Close(); _ = silk2.Close() }) - gc2 := newGuestPortConn(client2) + gc2 := newGuestPortConn(client2, bufio.NewReaderSize(client2, portReadBuf)) go func() { _, _ = silk2.Write(fixture("resp_error.json")) }() if _, err := gc2.Read(buf); err == nil || err == io.EOF { t.Errorf("fixture error frame: %v, want a typed error", err) } - // Write path: the emitted frame is byte-identical to the corpus request. client3, silk3 := net.Pipe() t.Cleanup(func() { _ = client3.Close(); _ = silk3.Close() }) - gc3 := newGuestPortConn(client3) + gc3 := newGuestPortConn(client3, bufio.NewReaderSize(client3, portReadBuf)) go func() { _, _ = gc3.Write([]byte("hello")) }() line := make([]byte, 256) n, _ = silk3.Read(line) diff --git a/sandboxd/engine/silkd.go b/sandboxd/engine/silkd.go index c0c521f7..c9afd760 100644 --- a/sandboxd/engine/silkd.go +++ b/sandboxd/engine/silkd.go @@ -12,8 +12,7 @@ import ( const silkdChunk = 256 * 1024 -// silkdSession is a dialed silkd conn bound to a ctx, with wire-typed -// request/reply helpers. The hot port-forward relay (portconn.go) does not use it. +// silkdSession is a dialed silkd conn bound to a ctx, with wire-typed request/reply helpers. type silkdSession struct { conn net.Conn sc *bufio.Scanner @@ -58,8 +57,7 @@ func (e *Engine) dialSilkdSession(ctx context.Context, vsockSocket string) (*sil }, nil } -// silkdStream serves one request per dial: silkd handles a single request -// per connection. +// silkdStream serves one request per dial: silkd handles one request per connection. func (e *Engine) silkdStream(ctx context.Context, vsockSocket string, req wire.Request, onFrame func(wire.Response) error) error { s, err := e.dialSilkdSession(ctx, vsockSocket) if err != nil { @@ -119,8 +117,7 @@ func (e *Engine) silkdReadFile(ctx context.Context, vsockSocket, path string) ([ return data, nil } -// silkdStat answers with a single Stat frame and no terminal Done, so it -// cannot ride silkdStream. +// silkdStat answers with a single Stat frame and no Done, so it cannot ride silkdStream. func (e *Engine) silkdStat(ctx context.Context, vsockSocket, path string) error { s, err := e.dialSilkdSession(ctx, vsockSocket) if err != nil { diff --git a/sandboxd/engine/volume.go b/sandboxd/engine/volume.go index 1255bf04..16fa51d5 100644 --- a/sandboxd/engine/volume.go +++ b/sandboxd/engine/volume.go @@ -13,8 +13,7 @@ import ( ) const ( - // VolumeCallTimeout bounds one guest teardown exec; the pool scales its - // whole quiesce budget from it. + // VolumeCallTimeout bounds one guest teardown exec. VolumeCallTimeout = 2 * time.Second volumePollInterval = 10 * time.Millisecond @@ -69,8 +68,7 @@ func (e *Engine) MountVolume(ctx context.Context, vsockSocket, name, mount strin return nil } -// UnmountVolume flushes and detaches a guest mount, so a writable image's -// dirty state reaches the backing file before the VM is removed. +// UnmountVolume flushes and detaches a guest mount before the VM is removed. func (e *Engine) UnmountVolume(ctx context.Context, vsockSocket, mount string) error { ctx, cancel := context.WithTimeout(ctx, VolumeCallTimeout) defer cancel() @@ -80,8 +78,7 @@ func (e *Engine) UnmountVolume(ctx context.Context, vsockSocket, mount string) e return nil } -// SyncGuest flushes the guest page cache — the only flush left for a mount that -// refused to unmount, since a lazy unmount would keep writing behind us. +// SyncGuest flushes the guest page cache, the only flush left for a failed unmount. func (e *Engine) SyncGuest(ctx context.Context, vsockSocket string) error { ctx, cancel := context.WithTimeout(ctx, VolumeCallTimeout) defer cancel() diff --git a/sandboxd/main.go b/sandboxd/main.go index 14a58c92..a866bc9b 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -34,8 +34,7 @@ import ( const ( shutdownGrace = 5 * time.Second gossipInterval = time.Second - // Slowloris protection; ReadTimeout/WriteTimeout must stay zero — cold - // claims block up to the cold probe timeout and relays stream forever. + // Slowloris protection; ReadTimeout/WriteTimeout must stay zero for streaming relays. readHeaderTimeout = 5 * time.Second ) @@ -208,8 +207,7 @@ func startMesh(ctx context.Context, cfg *config.Config, mgr *pool.Manager) (*mes return msh, nil } -// gossipNodeState republishes this node's warm-pool counts, templates, and -// locally available volumes every tick so placement tracks filesystem changes. +// gossipNodeState republishes this node's counts, templates, and volumes every tick. func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { t := time.NewTicker(gossipInterval) defer t.Stop() diff --git a/sandboxd/mesh/epoch.go b/sandboxd/mesh/epoch.go index f9439505..c0ec53cc 100644 --- a/sandboxd/mesh/epoch.go +++ b/sandboxd/mesh/epoch.go @@ -9,9 +9,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// loadEpoch reads the persisted gossip epoch, or 0 when the file is absent, -// unreadable, or above MaxInt64 — a UnixNano-derived counter cannot legitimately -// exceed it, so a corrupt value falls back to the wall-clock seed. +// loadEpoch reads the persisted gossip epoch, or 0 when it is absent, unreadable, or corrupt. func loadEpoch(path string) uint64 { raw, err := os.ReadFile(path) //nolint:gosec // node-local data-dir path if err != nil { diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 83df2e27..d915d02c 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -20,12 +20,15 @@ import ( "github.com/projecteru2/core/log" ) -// leaveTimeout bounds the graceful-leave broadcast on shutdown so a wedged -// network can't hang the exit path. -const leaveTimeout = time.Second +const ( + // leaveTimeout bounds the graceful-leave broadcast so a wedged network cannot hang the exit path. + leaveTimeout = time.Second -// NodeState is one node's gossiped placement view. Epoch resolves merges: the -// higher epoch for a given node wins. + // epochLease keeps the durable floor ahead of the counter, off the gossip tick. + epochLease = 1024 +) + +// NodeState is one node's gossiped placement view; the higher Epoch wins a merge. type NodeState struct { NodeID string `json:"node_id"` Addr string `json:"addr"` // data-plane advertise address @@ -40,13 +43,12 @@ type NodeState struct { type Mesh struct { ml *memberlist.Memberlist epochPath string - // ctx is the daemon's, for logging inside memberlist callbacks — the - // delegate interfaces carry no context of their own. + // ctx is the daemon's, for logging inside memberlist callbacks. ctx context.Context - // updateMu serializes UpdateSelf end-to-end: two updates reading one epoch - // would both bump to E+1 and the loser's payload would be dropped. + // updateMu serializes UpdateSelf and guards leased; one shared epoch read drops a payload. updateMu sync.Mutex + leased uint64 mu sync.Mutex self NodeState @@ -54,18 +56,15 @@ type Mesh struct { live map[string]struct{} // members SWIM reports; gossip about anyone else is ignored } -// New starts a mesh member listening per cfg. selfAddr is the data-plane -// address peers should dial for a redirect; secretKey (16/24/32 bytes) enables -// gossip encryption when non-empty; dataDir holds the persisted epoch. +// New starts a mesh member listening per cfg. func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, secretKey []byte, dataDir string) (*Mesh, error) { epochPath := filepath.Join(dataDir, "mesh-epoch") - // Seed strictly above the persisted floor (the last epoch peers saw): - // seeding at it ties their stale copy and merge's `>` rejects the restart's - // fresh state. loadEpoch caps the floor so the +1 cannot wrap. + // seed strictly above the persisted floor: merge's `>` rejects a tie with a stale copy. epoch := max(uint64(time.Now().UnixNano()), loadEpoch(epochPath)+1) //nolint:gosec // UnixNano is positive for current times m := &Mesh{ ctx: ctx, epochPath: epochPath, + leased: epoch + epochLease, self: NodeState{ NodeID: nodeID, Addr: selfAddr, @@ -75,7 +74,7 @@ func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, s view: map[string]NodeState{}, live: map[string]struct{}{}, } - if err := m.persistEpoch(epoch); err != nil { + if err := m.persistEpoch(m.leased); err != nil { return nil, fmt.Errorf("persist mesh epoch: %w", err) } m.view[nodeID] = m.self @@ -105,9 +104,7 @@ func (m *Mesh) Join(seeds []string) error { return nil } -// UpdateSelf republishes this node's warm-pool counts, promoted-template set, -// and locally available volumes. An unchanged view does not bump the epoch. -// templates and volumes must arrive sorted: the compare is order-sensitive. +// UpdateSelf republishes this node's counts, templates, and volumes; both slices must be sorted. func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, volumes []string) { m.updateMu.Lock() defer m.updateMu.Unlock() @@ -118,12 +115,14 @@ func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, } epoch := m.self.Epoch + 1 m.mu.Unlock() - // Persist the candidate before publishing it: memberlist gossips self the - // instant it enters the view, so a crash before the write would strand peers - // on an epoch a backwards-clock restart can't beat. - if err := m.persistEpoch(epoch); err != nil { - log.WithFunc("mesh.UpdateSelf").Warnf(ctx, "persist epoch: %v", err) - return + // the durable floor must stay above anything gossiped: memberlist publishes self at once. + if epoch > m.leased { + leased := epoch + epochLease + if err := m.persistEpoch(leased); err != nil { + log.WithFunc("mesh.UpdateSelf").Warnf(ctx, "persist epoch: %v", err) + return + } + m.leased = leased } m.mu.Lock() m.self.Epoch = epoch @@ -134,8 +133,7 @@ func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, m.mu.Unlock() } -// SetSelfDigest records this node's cluster-invariant config digest so peers can -// detect divergence; call it once before Join, before any gossip ships. +// SetSelfDigest records this node's config digest; call it once before Join. func (m *Mesh) SetSelfDigest(digest string) { m.mu.Lock() defer m.mu.Unlock() @@ -143,8 +141,7 @@ func (m *Mesh) SetSelfDigest(digest string) { m.view[m.self.NodeID] = m.self } -// ConfigMismatches counts peers whose config digest differs from this node's — -// the gauge for alerting on a divergent cluster, recomputed per read. +// ConfigMismatches counts peers whose config digest differs from this node's. func (m *Mesh) ConfigMismatches() int { m.mu.Lock() defer m.mu.Unlock() @@ -160,9 +157,7 @@ func (m *Mesh) ConfigMismatches() int { return n } -// Candidates returns up to two peer addresses that report warm(keyHash) > 0, -// chosen power-of-two-choices to avoid herding every waiter onto one node. -// Self is never a candidate — the caller has already missed locally. +// Candidates returns up to two peers reporting warm(keyHash) > 0, self excluded. func (m *Mesh) Candidates(keyHash string) []string { return m.warmCandidates(keyHash, func(NodeState) bool { return true }) } @@ -172,22 +167,17 @@ func (m *Mesh) VolumeCandidates(keyHash string, names []string) []string { return m.warmCandidates(keyHash, func(st NodeState) bool { return containsAll(st.Volumes, names) }) } -// TemplateOwners returns up to two peer addresses whose gossiped template -// set contains keyHash — the redirect targets for a name-based claim or -// delete of a template this node does not hold. Self is excluded: the caller -// has already checked its own disk. +// TemplateOwners returns up to two peers whose gossiped template set contains keyHash. func (m *Mesh) TemplateOwners(keyHash string) []string { return m.owners(func(st NodeState) bool { return slices.Contains(st.Templates, keyHash) }) } -// VolumeOwners returns peers that currently advertise every requested volume. -// Self is excluded because the caller checks local availability first. +// VolumeOwners returns peers that advertise every requested volume; self is excluded. func (m *Mesh) VolumeOwners(names []string) []string { return m.owners(func(st NodeState) bool { return containsAll(st.Volumes, names) }) } -// TemplateVolumeOwners returns peers that hold both the promoted template and -// every requested volume, avoiding an incorrect intersection after truncation. +// TemplateVolumeOwners returns peers holding both the template and every requested volume. func (m *Mesh) TemplateVolumeOwners(keyHash string, names []string) []string { return m.owners(func(st NodeState) bool { return slices.Contains(st.Templates, keyHash) && containsAll(st.Volumes, names) @@ -214,8 +204,7 @@ func (m *Mesh) Members() []NodeState { return slices.Collect(maps.Values(m.view)) } -// PeerAddrs returns the data-plane addresses of the other nodes, for a -// client-side Lookup scatter. +// PeerAddrs returns the data-plane addresses of the other nodes. func (m *Mesh) PeerAddrs() []string { m.mu.Lock() defer m.mu.Unlock() @@ -291,8 +280,7 @@ func (m *Mesh) admit(nodeID string) { m.live[nodeID] = struct{}{} } -// forget drops a departed node from the placement view so redirects stop -// targeting a dead peer; SWIM detected the death, the view must follow. +// forget drops a departed node from the placement view so redirects stop targeting it. func (m *Mesh) forget(nodeID string) { m.mu.Lock() defer m.mu.Unlock() @@ -302,8 +290,7 @@ func (m *Mesh) forget(nodeID string) { } } -// merge absorbs a peer's view, keeping the higher epoch per node and never -// letting a peer overwrite this node's own authoritative self entry. +// merge absorbs a peer's view, keeping the higher epoch per node; self is never overwritten. func (m *Mesh) merge(states []NodeState) { m.mu.Lock() defer m.mu.Unlock() @@ -318,10 +305,7 @@ func (m *Mesh) merge(states []NodeState) { if ok && st.Epoch <= cur.Epoch { continue } - // Warn on each distinct divergent digest (not once per lifetime): a - // mismatched api_token/tenants/preview_secret/CA root 401s cross-node - // redirects and fails interception. Warn-only — refusing would partition - // a rolling credential rotation. + // warn-only: refusing a divergent digest would partition a rolling credential rotation. if m.self.Digest != "" && st.Digest != "" && st.Digest != m.self.Digest && (!ok || cur.Digest != st.Digest) { log.WithFunc("mesh.merge").Warnf(m.ctx, "peer %s config digest %s differs from this node's %s: cluster-invariant config diverges (redirects may 401, interception may fail)", @@ -343,8 +327,7 @@ func containsAll(have, need []string) bool { var _ memberlist.Delegate = (*delegate)(nil) -// delegate carries this node's full view on each memberlist push/pull sync, so -// state propagates transitively across the cluster. +// delegate carries this node's full view on each memberlist push/pull sync. type delegate Mesh func (d *delegate) NodeMeta(int) []byte { return nil } diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index b00aae50..9901d426 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -17,7 +17,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { m := newTestMesh(t, "a") mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}}) mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 3, Pools: map[string]int{"k": 5}}}) - mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) // stale + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) got := 0 for _, st := range m.Members() { @@ -33,7 +33,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil, nil) - // A peer claiming to be "a" must not clobber our authoritative self entry. + mergeStates(t, m, []NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) for _, st := range m.Members() { @@ -45,10 +45,10 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate + m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm + {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"other": 4}}, }) @@ -63,7 +63,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate + m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, @@ -87,7 +87,7 @@ func TestForgetPrunesDeadNode(t *testing.T) { if got := m.Candidates("k"); got != nil { t.Errorf("candidates after forget %v, want nil (b pruned)", got) } - // forgetting self is a no-op. + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) m.forget("a") if len(m.Members()) != 1 { @@ -126,7 +126,7 @@ func TestCandidatesPowerOfTwo(t *testing.T) { {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, }) - // With ≥2 warm peers, exactly two distinct candidates come back. + for range 20 { cands := m.Candidates("k") if len(cands) != 2 || cands[0] == cands[1] { @@ -148,8 +148,6 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { } a.mesh.UpdateSelf(t.Context(), map[string]int{"kk": 4}, []string{"tpl-hash"}, []string{"dataset"}) - // Push/pull sync propagates a's warm counts and template set to b within - // a few intervals. deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { cands := b.mesh.Candidates("kk") diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index 43c2b281..17a9dbd4 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -24,7 +24,7 @@ func TestEpochRoundTrip(t *testing.T) { func TestLoadEpochRejectsImplausibleValue(t *testing.T) { p := filepath.Join(t.TempDir(), "e") - // A saturated value must not seed a counter that +1 can't climb past. + if err := os.WriteFile(p, []byte("18446744073709551615"), 0o600); err != nil { t.Fatalf("write: %v", err) } @@ -35,12 +35,12 @@ func TestLoadEpochRejectsImplausibleValue(t *testing.T) { func TestEpochSeededAbovePersistedFloor(t *testing.T) { dir := t.TempDir() - huge := uint64(1) << 62 // above any plausible wall-clock nanos + huge := uint64(1) << 62 if err := storeEpoch(filepath.Join(dir, "mesh-epoch"), huge); err != nil { t.Fatalf("store: %v", err) } m := newBoundMesh(t, dir) - // Strictly above: seeding at the persisted value would tie peers' stale copy. + if m.self.Epoch != huge+1 { t.Errorf("epoch seeded %d, want persisted floor + 1 = %d (a backwards clock must land above peers' last-seen epoch)", m.self.Epoch, huge+1) } @@ -50,8 +50,9 @@ func TestUpdateSelfPersistFailKeepsOldState(t *testing.T) { dir := t.TempDir() m := newBoundMesh(t, dir) before := m.self.Epoch - // A non-existent dir makes the durable write fail: the epoch must not publish. + m.epochPath = filepath.Join(dir, "gone", "mesh-epoch") + m.leased = before m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) if m.self.Epoch != before { t.Errorf("self epoch advanced to %d on a failed persist, want %d held", m.self.Epoch, before) @@ -71,6 +72,29 @@ func TestUpdateSelfPersistsEpoch(t *testing.T) { } } +func TestUpdateSelfLeasesEpochFloor(t *testing.T) { + dir := t.TempDir() + m := newBoundMesh(t, dir) + path := filepath.Join(dir, "mesh-epoch") + seeded := loadEpoch(path) + if seeded <= m.self.Epoch { + t.Fatalf("seeded floor %d, want above the live epoch %d", seeded, m.self.Epoch) + } + for i := range 8 { + m.UpdateSelf(t.Context(), map[string]int{"k": i + 1}, nil, nil) + } + if got := loadEpoch(path); got != seeded { + t.Errorf("floor rewritten to %d during %d leased ticks, want %d held", got, 8, seeded) + } + m.mu.Lock() + m.self.Epoch = seeded + m.mu.Unlock() + m.UpdateSelf(t.Context(), map[string]int{"k": 99}, nil, nil) + if got := loadEpoch(path); got <= m.self.Epoch { + t.Errorf("floor %d not above the published epoch %d after crossing the lease", got, m.self.Epoch) + } +} + func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { m := newBoundMesh(t, t.TempDir()) base := m.self.Epoch @@ -81,8 +105,7 @@ func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { wg.Go(func() { m.UpdateSelf(t.Context(), a, nil, nil) }) wg.Go(func() { m.UpdateSelf(t.Context(), b, nil, nil) }) wg.Wait() - // Serialized updates with distinct payloads must both land: the loser - // of the old TOCTOU race silently dropped its payload and one bump. + want := base + uint64(2*(i+1)) //nolint:gosec // loop index is small and positive if m.self.Epoch != want { t.Fatalf("round %d: epoch %d, want %d (an update was dropped)", i, m.self.Epoch, want) diff --git a/sandboxd/netfilter/nft_linux.go b/sandboxd/netfilter/nft_linux.go index c85a5d40..3c493919 100644 --- a/sandboxd/netfilter/nft_linux.go +++ b/sandboxd/netfilter/nft_linux.go @@ -4,7 +4,6 @@ package netfilter import ( "fmt" - "slices" "strings" "github.com/google/nftables" @@ -14,9 +13,7 @@ import ( const tablePrefix = "sandbox_egress_" -// Lock drops a tap's guest-initiated egress (DHCP renewal excepted) at its -// ingress hook, forcing all egress through the vsock proxy. One netdev table -// per tap in the root netns; idempotent. +// Lock drops a tap's guest egress (DHCP excepted) at its ingress hook; idempotent. func Lock(tap string) error { _ = Unlock(tap) // a prior table for this tap would block a clean re-lock c, err := nftables.New() @@ -34,10 +31,7 @@ func Lock(tap string) error { Policy: &policy, Device: tap, }) - // Accept only IPv4 broadcast DHCP (sport 68 → dport 67, dst 255.255.255.255); - // the nfproto guard blocks an IPv6 datagram carrying 0xffffffff at the - // IPv4-daddr offset. The unchecked payload can only reach the local L2 - // segment, so egress VMs must not share a broadcast domain. + // accept only IPv4 broadcast DHCP; egress VMs must not share a broadcast domain. c.AddRule(&nftables.Rule{Table: t, Chain: ch, Exprs: []expr.Any{ &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.NFPROTO_IPV4}}, @@ -60,21 +54,23 @@ func Lock(tap string) error { return nil } -// EnsureLock applies Lock only if the tap has no table yet, so a restart can -// re-adopt a live VM's surviving lock without a re-lock window. -func EnsureLock(tap string) error { +// LockedTaps reports the taps that already carry a table, so a restart re-adopts them. +func LockedTaps() (map[string]bool, error) { c, err := nftables.New() if err != nil { - return fmt.Errorf("nft conn: %w", err) + return nil, fmt.Errorf("nft conn: %w", err) } tables, err := c.ListTablesOfFamily(nftables.TableFamilyNetdev) if err != nil { - return fmt.Errorf("list netdev tables: %w", err) + return nil, fmt.Errorf("list netdev tables: %w", err) } - if slices.ContainsFunc(tables, func(t *nftables.Table) bool { return t.Name == tablePrefix+tap }) { - return nil + locked := make(map[string]bool, len(tables)) + for _, t := range tables { + if tap, ok := strings.CutPrefix(t.Name, tablePrefix); ok { + locked[tap] = true + } } - return Lock(tap) + return locked, nil } // Unlock removes the tap's table; a no-op when it is already gone. @@ -88,9 +84,7 @@ func Unlock(tap string) error { return nil } -// SweepExcept drops every per-tap egress table whose tap is not in keep, clearing -// locks orphaned by VMs that went away while the daemon was down. It owns the -// whole sandbox_egress_* namespace in the root netns — one sandboxd per host. +// SweepExcept drops every per-tap egress table whose tap is not in keep; one sandboxd per host. func SweepExcept(keep map[string]bool) error { c, err := nftables.New() if err != nil { diff --git a/sandboxd/netfilter/nft_other.go b/sandboxd/netfilter/nft_other.go index e448ad79..92c241e5 100644 --- a/sandboxd/netfilter/nft_other.go +++ b/sandboxd/netfilter/nft_other.go @@ -4,12 +4,11 @@ package netfilter import "errors" -// Lock and Unlock fail off Linux (nftables is Linux-only), so a caller cannot -// mistake a missing lock for an applied one. +// Lock fails off Linux, so a caller cannot mistake a missing lock for an applied one. func Lock(string) error { return errors.ErrUnsupported } func Unlock(string) error { return errors.ErrUnsupported } -func EnsureLock(string) error { return errors.ErrUnsupported } +func LockedTaps() (map[string]bool, error) { return nil, errors.ErrUnsupported } func SweepExcept(keep map[string]bool) error { return nil } diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 168f7953..39a24bc5 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -37,8 +37,7 @@ func (m *Manager) archiveEnabledFor(key types.PoolKey) bool { return m.archiveAfterFor(key) > 0 } -// archiveOnce checkpoints hibernated claims idle past their archive threshold -// to the store and drops their local VM; the next access restores them. +// archiveOnce checkpoints hibernated claims idle past their archive threshold and drops their VM. func (m *Manager) archiveOnce(ctx context.Context) { if !m.archiveEnabled { return @@ -78,9 +77,7 @@ func (m *Manager) archiveOnce(ctx context.Context) { }() } -// archive publishes a hibernated claim's wake image to the store, then swaps -// its compute backing from local VM to store checkpoint (ArchiveCk), preserving -// id/token/tenant so a later access restores it (wakeArchived). +// archive swaps a hibernated claim's backing from local VM to a store checkpoint. func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // Must finish even if the sweep ctx is canceled, or the record diverges from the store. ctx = context.WithoutCancel(ctx) @@ -91,8 +88,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { if !ok { return errWokeMeanwhile } - // Pin the wake image before it exists: from the moment Publish makes it - // listable, a tenant/root delete would strand the claim it will back. + // pin the ck before Publish makes it listable, or a delete would strand the claim ckID := store.CheckpointID(randHex(8)) m.mu.Lock() m.pendingCks[ckID] = struct{}{} @@ -103,9 +99,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { if err != nil { return err } - // Re-validate under Transition+m.mu: the claim must still be hibernated on - // the exact snapshot the export captured — a wake, or a wake+re-hibernate - // landing inside the publish window, means the checkpoint holds stale state. + // the claim must still hold the exact snapshot the export captured, or the ck is stale sb.Transition.Lock() m.mu.Lock() if m.claimed[sb.ID] != sb || sb.HibernateSnap != srcSnap || sb.ArchiveCk != "" { @@ -122,7 +116,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { } else { sb.Deadline = time.Time{} // keep forever: no retention (reap skips a zero deadline) } - js := m.store.snapshot(m.claimed) + js := m.store.set(sb) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { // Roll back so memory matches the still-durable hibernated record; drop the orphan ck. @@ -130,15 +124,14 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { sb.ArchiveCk = "" sb.HibernateSnap, sb.VsockSocket, sb.VMName = snap, sock, vmName sb.Deadline = prevDeadline - rb := m.store.snapshot(m.claimed) + rb := m.store.set(sb) m.mu.Unlock() sb.Transition.Unlock() m.recommit(ctx, rb) m.deleteOrphanArchiveCk(ctx, ck.ID) return fmt.Errorf("archive %s: persist claims: %w", sb.ID, saveErr) } - // Disarm under Transition so a wake that runs the instant we release it - // re-arms cleanly instead of being clobbered by a late disarm. + // disarm under Transition so a wake right after the release is not clobbered by a late disarm m.disarmEgress(sb.ID, true) sb.Transition.Unlock() // Committed: the store ck is authoritative now; reclaim the local footprint. @@ -149,8 +142,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { return nil } -// wakeArchived restores an archived claim from its store checkpoint into a -// fresh local VM, keeping id/token/tenant; the caller holds the Transition lock. +// wakeArchived restores an archived claim into a fresh local VM; caller holds Transition. func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, error) { // Egress never archives; a corrupt archived egress claim must fail closed. if sb.Key.Net == types.NetEgress { @@ -158,10 +150,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, } ctx = context.WithoutCancel(ctx) ck := sb.ArchiveCk - // Exclusive, not shared: this path deletes the consumed ck below. The - // lock releases before eviction is considered, below — evicting while - // still held would let a concurrent recLock for ck split onto a - // different mutex. + // evicting while the lock is held would split a concurrent recLock onto a different mutex l := m.recLock(ck) l.Lock() consumed := false @@ -206,8 +195,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, log.WithFunc("pool.wakeArchived").Warnf(ctx, "clear archive ck %s: %v", ck, clearErr) } } - // Only the none lane reaches here (the egress guard above fails closed), so - // this rebinds the none-lane proxy; there is no NIC to re-lock. + // only the none lane reaches here, so there is no NIC to re-lock if proxyErr := m.armEgressProxy(ctx, sb); proxyErr != nil { log.WithFunc("pool.wakeArchived").Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) } @@ -219,9 +207,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, return built.VsockSocket, nil } -// commitWake swaps an archived record back to a live VM if still claimed -// (Release/reap skip the Transition lock). A failed persist rolls the record -// back to archived so memory matches disk and the ck stays pinned. +// commitWake swaps an archived record back to a live VM if the claim is still live. func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, sock string) (live, saved bool) { m.mu.Lock() live = m.claimed[sb.ID] == sb @@ -232,12 +218,12 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc ck, deadline := sb.ArchiveCk, sb.Deadline sb.VMName, sb.VsockSocket, sb.ArchiveCk = vmName, sock, "" sb.Deadline = time.Now().Add(clampTTL(0)) // a woken sandbox is a fresh lease - js := m.store.snapshot(m.claimed) + js := m.store.set(sb) m.mu.Unlock() if err := m.store.commit(js); err != nil { m.mu.Lock() sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.Deadline = "", "", ck, deadline - rb := m.store.snapshot(m.claimed) + rb := m.store.set(sb) m.mu.Unlock() m.recommit(ctx, rb) log.WithFunc("pool.commitWake").Warnf(ctx, "persist claims: %v", err) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 7d0fdc1d..220a7f05 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -20,10 +20,6 @@ const ( testArchiveReap = "reap" ) -// TestArchivePublishWindowPinsCheckpoint guards the pre-pin: between an -// archive's checkpoint publish and the ArchiveCk commit, the checkpoint must -// be invisible to listings and refuse deletion — a delete landing in that -// window stranded the claim. func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -53,11 +49,6 @@ func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { } } -// TestArchiveWakeRehibernateWindowAborts guards the commit's identity check: a -// wake + re-hibernate landing inside the publish window leaves HibernateSnap -// non-empty but pointing at NEWER state than the exported checkpoint — -// committing would archive stale data and destroy the only copy of the new -// snapshot. func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -105,10 +96,6 @@ func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { } } -// TestArchiveCkHiddenAcrossNodes: on a shared store, another node has no -// in-memory pin for this node's archive wake images — the meta Archive flag -// must hide them from its listings, refuse its deletes, and spare its TTL -// sweep, or a routine cleanup elsewhere strands the claim. func TestArchiveCkHiddenAcrossNodes(t *testing.T) { shared := t.TempDir() mkMgr := func() *Manager { @@ -142,11 +129,6 @@ func TestArchiveCkHiddenAcrossNodes(t *testing.T) { } } -// TestReconcileReclaimsOrphanArchiveCk: a crash between an archive's publish -// and its journal commit leaves a flagged record listings hide and deletes -// refuse; reconcile identifies it by the journal (the sandbox is ours, under -// a different ArchiveCk) and reclaims it. A flagged record of an unknown -// sandbox is left alone. func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { eng := newFakeEngine() dataDir := t.TempDir() @@ -188,8 +170,6 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { } } -// TestClaimCheckpointRefusesArchive: wake images are lifecycle-internal — -// branching from one would race its consumption by the owner's wake. func TestClaimCheckpointRefusesArchive(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -204,9 +184,6 @@ func TestClaimCheckpointRefusesArchive(t *testing.T) { } } -// TestArchiveLifecycleRoundTrip exercises the real sweep wiring: -// claim→idle-hibernate→archive→wake, and asserts the id/token survive the -// store round-trip while the local VM and its wake image are reclaimed. func TestArchiveLifecycleRoundTrip(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -257,12 +234,10 @@ func TestArchiveLifecycleRoundTrip(t *testing.T) { }) } -// TestArchiveTTLSweepSparesLiveCheckpoint guards the fix where the checkpoint -// TTL sweep deleted the store image backing a live archived claim, bricking it. func TestArchiveTTLSweepSparesLiveCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) - m.ckptTTL = time.Nanosecond // every checkpoint is instantly age-expired + m.ckptTTL = time.Nanosecond sb := mustClaim(t, m, testKey) id, token := sb.ID, sb.Token mustArchive(t, m, sb) @@ -277,8 +252,6 @@ func TestArchiveTTLSweepSparesLiveCheckpoint(t *testing.T) { } } -// TestArchiveKeepsForeverWhenDeleteZero guards the fix where a zero -// archive-delete window purged archives immediately instead of keeping them. func TestArchiveKeepsForeverWhenDeleteZero(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(0)) @@ -303,9 +276,6 @@ func TestArchiveKeepsForeverWhenDeleteZero(t *testing.T) { } } -// TestArchiveRetentionPurge covers the reapPurge path: an archive past its -// retention deadline drops from claims, deletes its checkpoint, and releases -// its tenant slot. func TestArchiveRetentionPurge(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -318,14 +288,12 @@ func TestArchiveRetentionPurge(t *testing.T) { ck := sb.ArchiveCk m.mu.Lock() - sb.Deadline = time.Now().Add(-time.Second) // retention window elapsed + sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() m.reapOnce(t.Context()) waitFor(t, func() bool { return archivedCount(m) == 0 }) - // archiveDeletes is the purge goroutine's last side effect, sequenced after - // the checkpoint delete; waiting on it makes both observable (a bare read - // races the async purge, flaking under -race). + waitFor(t, func() bool { return m.Counters().ArchiveDeletes > 0 }) if ckExists(t, m, ck) { t.Error("purge did not delete the archived checkpoint") @@ -340,9 +308,6 @@ func TestArchiveRetentionPurge(t *testing.T) { }) } -// TestReleaseArchivedDeletesCheckpoint guards the fix where releasing an -// archived claim orphaned its store checkpoint and called Remove on an empty -// VM name. func TestReleaseArchivedDeletesCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -525,9 +490,6 @@ func TestArchiveWakeRequiresDeleteMarker(t *testing.T) { } } -// Not a synctest candidate: each subtest holds m.store.mu itself to force the -// release/reap goroutine to block acquiring it — same plain-sync.Mutex shape -// as TestArchiveDeleteRetryRechecksWakeRollback, already verified to deadlock. func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { for _, action := range []string{testArchiveRelease, testArchiveReap} { t.Run(action, func(t *testing.T) { @@ -542,11 +504,11 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { m.mu.Unlock() } - m.store.mu.Lock() + m.store.writeMu.Lock() locked := true t.Cleanup(func() { if locked { - m.store.mu.Unlock() + m.store.writeMu.Unlock() } }) done := make(chan error, 1) @@ -570,7 +532,7 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { t.Error("retry deleted the checkpoint before the claims commit") } m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") - m.store.mu.Unlock() + m.store.writeMu.Unlock() locked = false if err := <-done; action == testArchiveRelease && err == nil { t.Fatal("release succeeded despite a persist failure") @@ -639,10 +601,6 @@ func TestArchiveRemovalRequiresDeleteMarker(t *testing.T) { } } -// Not a synctest candidate: this test holds m.store.mu itself to force the -// wake goroutine to block acquiring it — a plain sync.Mutex, never "durably -// blocked" by synctest's rules, so the waitFor below can't fast-forward past -// it. Verified as a real deadlock (times out), not a flake. func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -650,11 +608,11 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { mustArchive(t, m, sb) id, token, ck := sb.ID, sb.Token, sb.ArchiveCk - m.store.mu.Lock() + m.store.writeMu.Lock() locked := true t.Cleanup(func() { if locked { - m.store.mu.Unlock() + m.store.writeMu.Unlock() } }) woke := make(chan error, 1) @@ -681,7 +639,7 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { return m.recRefs[ck] >= 2 }) m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") - m.store.mu.Unlock() + m.store.writeMu.Unlock() locked = false if err := <-woke; err == nil { t.Fatal("wake succeeded despite a persist failure") @@ -702,8 +660,6 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { } } -// TestIdleOnceSkipsArchived guards the fix where the idle sweep tried to -// hibernate an archived (VM-less) claim, corrupting its record. func TestIdleOnceSkipsArchived(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -724,9 +680,6 @@ func TestIdleOnceSkipsArchived(t *testing.T) { }) } -// TestDeleteCheckpointCannotBrickArchive guards the fix where the archive -// image was listable and deletable through the ordinary checkpoint API, -// letting a tenant permanently strand its own archived sandbox. func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -735,8 +688,6 @@ func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { mustArchive(t, m, sb) ck := sb.ArchiveCk - // The archive image is lifecycle-internal — it must not surface as a user - // checkpoint, or be deletable as one. if ckpts, err := m.Checkpoints(t.Context(), ""); err != nil || len(ckpts) != 0 { t.Fatalf("Checkpoints listed %d records, want the archive image hidden (%v)", len(ckpts), err) } @@ -751,9 +702,6 @@ func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { } } -// TestArchivePersistFailureRollsBack guards the durability fix where archive -// destroyed the local VM+snapshot before its transition reached disk: a failed -// persist must roll the record back to hibernated and keep the local backing. func TestArchivePersistFailureRollsBack(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -784,9 +732,6 @@ func TestArchivePersistFailureRollsBack(t *testing.T) { }) } -// TestReleaseArchivedRollsBackOnPersistFailure guards the durability fix where -// a release whose removal did not persist must roll back — the claim survives -// and its ck stays pinned, so a restart still wakes it. func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -810,9 +755,6 @@ func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { }) } -// TestWakeArchivedRollsBackOnPersistFailure guards the durability fix where a -// wake whose cleared-ArchiveCk record did not persist must roll back to -// archived, keeping the ck pinned. func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -839,8 +781,6 @@ func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { }) } -// TestReapPurgeRollsBackOnPersistFailure guards the same rollback on the reaper: -// a retention purge that does not persist keeps the claim and its pinned ck. func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -850,7 +790,7 @@ func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { mustArchive(t, m, sb) ck := sb.ArchiveCk m.mu.Lock() - sb.Deadline = time.Now().Add(-time.Second) // retention elapsed + sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() breakStore(t, m) @@ -868,8 +808,6 @@ func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { }) } -// TestArchiveOnceSkipsInFlight guards the in-flight dedup: a sandbox already -// being archived (marked by a racing reap) must not be exported twice. func TestArchiveOnceSkipsInFlight(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -881,7 +819,7 @@ func TestArchiveOnceSkipsInFlight(t *testing.T) { backdate(m, sb, time.Hour) m.mu.Lock() - m.archiving[sb.ID] = struct{}{} // a reap-triggered archive is already exporting it + m.archiving[sb.ID] = struct{}{} m.mu.Unlock() exportsBefore := eng.exportCount() @@ -896,10 +834,6 @@ func TestArchiveOnceSkipsInFlight(t *testing.T) { }) } -// TestArchiveWakeEvictsRecLock proves the archive->wake cycle leaves no -// recLocks residue: wakeArchived recLocks the consumed ck, so it must evict it -// after the delete or a repeatedly idle->archive->wake workload leaks a lock -// per cycle. func TestArchiveWakeEvictsRecLock(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -907,25 +841,18 @@ func TestArchiveWakeEvictsRecLock(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } - countLocks := func() int { - n := 0 - m.recLocks.Range(func(_, _ any) bool { n++; return true }) - return n - } - base := countLocks() + base := lockCount(m) if err := m.archive(t.Context(), sb); err != nil { t.Fatalf("archive: %v", err) } if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake archived: %v", err) } - if got := countLocks(); got != base { + if got := lockCount(m); got != base { t.Errorf("recLocks grew %d->%d over an archive+wake cycle, want no growth", base, got) } } -// stallingStore parks Publish after the record becomes visible, exposing the -// window between an archive's publish and its ArchiveCk commit. type stallingStore struct { store.Store published chan string @@ -954,15 +881,11 @@ func (s *failingDeleteStore) Delete(_ context.Context, id string) error { return errors.New("delete failed") } -// breakStore points the claim store at a path whose parent is missing, so the -// next save fails — the fault used to exercise the persist-failure paths. func breakStore(t *testing.T, m *Manager) { t.Helper() m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") } -// healStore repairs breakStore's path and waits for the detached recommit to -// converge, so its retry loop does not outlive the test. func healStore(t *testing.T, m *Manager) { t.Helper() if err := os.MkdirAll(filepath.Dir(m.store.path), 0o750); err != nil { @@ -976,8 +899,6 @@ func archivedCount(m *Manager) int { return g.Archived } -// archivePool is the pool shape the archive tests share: idle→hibernate at 1s, -// archive at 2s, a long retention window (overridden per test). func archivePool(delete int) config.PoolSpec { return config.PoolSpec{ PoolKey: testKey, @@ -988,9 +909,6 @@ func archivePool(delete int) config.PoolSpec { } } -// mustArchive drives a fresh claim to the archived state deterministically -// (hibernate then archive directly), bypassing the sweep thresholds so a test -// can start from a known archived record. func mustArchive(t *testing.T, m *Manager, sb *types.Sandbox) { t.Helper() if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { @@ -1016,8 +934,6 @@ func archiveCkMarked(m *Manager, ck string) bool { return err == nil } -// pinnedHidden reports whether the store holds exactly one checkpoint and it is -// hidden from listings — i.e. a live claim's ArchiveCk still pins it. func pinnedHidden(t *testing.T, m *Manager, ck string) bool { t.Helper() ckpts, err := m.Checkpoints(t.Context(), "") diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index bc294b61..75e66e27 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -16,8 +16,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// DeleteScope selects how far a checkpoint delete reaches. Fleet-wide is the -// zero value, so the narrower local-only delete must be asked for. +// DeleteScope selects how far a checkpoint delete reaches; fleet-wide is the zero value. type DeleteScope int const ( @@ -31,9 +30,7 @@ var ( ErrHealBusy = errors.New("too many checkpoint heals in flight") ) -// Checkpoint captures a claimed sandbox's state under a fresh id; the source -// keeps running (a hibernated one is captured from its wake image). tenant -// attributes the record; empty means the operator (root). +// Checkpoint captures a claimed sandbox's state under a fresh id; the source keeps running. func (m *Manager) Checkpoint(ctx context.Context, id string, cred Cred, name, tenant string) (types.Checkpoint, error) { sb, ok := m.resolve(id, cred) if !ok { @@ -59,15 +56,9 @@ func (m *Manager) Checkpoint(ctx context.Context, id string, cred Cred, name, te return ckpt, nil } -// ClaimCheckpoint provisions a fresh claim cloned from a checkpoint — a -// branch. The checkpoint's recorded key applies (snapshots pin size and -// lane); the checkpoint itself is read-only and reusable. The claim is -// attributed to tenant, not the checkpoint's recorder — the unguessable id -// is the capability to branch. +// ClaimCheckpoint provisions a fresh claim cloned from a checkpoint, attributed to tenant. func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { - // Reject a bad or unknown id before recLock: a rejected id must not leave - // a lock-map entry (only a delete evicts one). It precedes quota: a full - // node must still answer "not here". + // a rejected id must not leave a lock-map entry, so resolve before recLock and quota ckpt, err := m.loadCheckpoint(ctx, ckptID) if err != nil { return nil, err @@ -78,15 +69,12 @@ func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.D return m.claimLoaded(ctx, ckpt, ttl, tenant) } -// ClaimCheckpointHeal claims a checkpoint this node does not hold locally, -// pulling it from a peer first. The server calls it only after the local -// claim missed and a redirect could not answer. +// ClaimCheckpointHeal claims a checkpoint held only by a peer, pulling it first. func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { if m.healer == nil || !store.CheckpointIDRe.MatchString(ckptID) { return nil, ErrUnknownCheckpoint } - // Resolving here means moving a guest memory image, so quota rejects a - // full node before that cost, not after. + // resolving moves a guest memory image, so quota rejects a full node before that cost if err := m.overQuota(1, tenant); err != nil { return nil, err } @@ -97,11 +85,7 @@ func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl ti return m.claimLoaded(ctx, ckpt, ttl, tenant) } -// Checkpoints lists the store's checkpoints, newest first — on a shared -// backend (a FUSE mount, a bucket), that is the cluster's set, not one -// node's. A non-empty tenant filters to that tenant's records; empty (root) -// lists everything. Checkpoints backing a live archived claim are hidden: -// they are lifecycle-internal wake images, not user checkpoints. +// Checkpoints lists tenant's checkpoints, newest first, hiding archive wake images. func (m *Manager) Checkpoints(ctx context.Context, tenant string) ([]types.Checkpoint, error) { metas, err := m.ckpts.Metas(ctx) if err != nil { @@ -123,13 +107,9 @@ func (m *Manager) Checkpoints(ctx context.Context, tenant string) ([]types.Check return ckpts, nil } -// DeleteCheckpoint removes a checkpoint's snapshot and record, then -// broadcasts to peers when fleet-scoped so a healed copy does not outlive it. -// A tenant may delete only its own records — anything else answers -// ErrUnknownCheckpoint, never a hint the id exists; root deletes anything. +// DeleteCheckpoint removes a checkpoint's snapshot and record, broadcasting when fleet-scoped. func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { - // Reject a bad id before recLock: a rejected id must not leave a - // lock-map entry. + // a rejected id must not leave a lock-map entry if !store.CheckpointIDRe.MatchString(ckptID) { return ErrUnknownCheckpoint } @@ -144,8 +124,7 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s if !tenantOwns(tenant, ckpt.Tenant) { return ErrUnknownCheckpoint } - // ckpt.Archive guards wake images across every node sharing the store; - // the pin set guards this node's not-yet-committed ones. + // ckpt.Archive guards wake images store-wide; the pin set guards uncommitted local ones if _, pinned := m.pinnedArchiveCks()[ckptID]; pinned || ckpt.Archive { return ErrUnknownCheckpoint // backs an archived sandbox, not a deletable checkpoint } @@ -159,15 +138,13 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s return nil } -// HasCheckpoint answers the ownership probe: a branchable local record, never -// a fetch. +// HasCheckpoint answers the ownership probe: a branchable local record, never a fetch. func (m *Manager) HasCheckpoint(ctx context.Context, ckptID string) bool { ckpt, err := m.loadCheckpoint(ctx, ckptID) return err == nil && !ckpt.Archive } -// FetchCheckpoint materializes a checkpoint's export for a peer transfer, -// returning the local directory, its meta, and the release to call when done. +// FetchCheckpoint materializes a checkpoint's export for a peer transfer. func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, []byte, func(), error) { if _, err := m.loadCheckpoint(ctx, ckptID); err != nil { return "", nil, nil, err @@ -183,15 +160,11 @@ func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, [ } return "", nil, nil, fmt.Errorf("fetch checkpoint: %w", err) } - // The read lock spans the transfer: a delete must not pull the export out - // from under a stream already writing it to a peer. + // the read lock spans the transfer so a delete cannot pull the export from under it return dir, meta, func() { release(); l.RUnlock(); m.recDone(ckptID) }, nil } -// publishCheckpoint stages the sandbox's exported state, writes the meta, and -// publishes it to the store, returning the record and the source snapshot the -// export captured. Shared by Checkpoint and archive; a hibernated source -// exports its wake image directly (no VM start — refill.sourceSnap). +// publishCheckpoint stages, writes the meta, and publishes, returning the record and source snap. func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID, name, tenant string, archive bool) (types.Checkpoint, string, error) { ckpt := types.Checkpoint{ ID: ckID, @@ -224,8 +197,7 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID return ckpt, srcSnap, nil } -// claimLoaded is the shared body once ckpt's meta is resolved: it re-fetches -// under the record lock, so a delete racing the pre-check cannot slip through. +// claimLoaded re-fetches under the record lock, so a delete racing the pre-check cannot slip in. func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl time.Duration, tenant string) (*types.Sandbox, error) { l := m.recLock(ckpt.ID) l.RLock() @@ -257,10 +229,7 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti return out, err } -// healCheckpoint dedups concurrent heals of ckptID onto one flight (which -// owns its own staging dir — never a caller's) and lets THIS call abandon -// waiting the moment ctx is done; the flight itself runs detached (runHeal), -// so a client hanging up never abandons a transfer already paid for. +// healCheckpoint dedups concurrent heals onto one detached flight; only the wait honors ctx. func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { return ckpt, nil @@ -279,10 +248,7 @@ func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Chec } } -// runHeal is healCheckpoint's flight body: it stages and pulls WITHOUT -// holding ckptID's record lock — a heal budget runs up to 30 minutes, and -// holding the lock across it would pin every same-id operation behind an -// uncancellable wait. +// runHeal stages and pulls without ckptID's record lock: a pull runs up to 30 minutes. func (m *Manager) runHeal(ckptID string) (types.Checkpoint, error) { select { case m.healSem <- struct{}{}: @@ -325,18 +291,14 @@ func (m *Manager) runHeal(ckptID string) (types.Checkpoint, error) { return m.loadCheckpoint(ctx, ckptID) } -// markHealPending records ckptID as staging or pulling, unlocked, so -// vetoIfHealPending knows a concurrent delete must veto rather than ignore -// it. Bounded by healSem: at most maxConcurrentHeals entries ever exist. +// markHealPending records ckptID as staging or pulling, so a delete can veto it. func (m *Manager) markHealPending(ckptID string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() m.healPending[ckptID] = struct{}{} } -// clearHealPending un-marks ckptID and reports whether vetoIfHealPending -// vetoed it in the meantime; call under ckptID's recLock, so the check and -// the eventual publish decide together. +// clearHealPending un-marks ckptID and reports a veto; call under ckptID's recLock. func (m *Manager) clearHealPending(ckptID string) (aborted bool) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -347,10 +309,7 @@ func (m *Manager) clearHealPending(ckptID string) (aborted bool) { return aborted } -// vetoIfHealPending marks ckptID aborted when a heal is pending, so the -// heal's locked decide phase (clearHealPending) abandons its publish instead -// of resurrecting a checkpoint this delete just answered "not here" for. A -// no-op when no heal is pending, so an unrelated miss leaves no residue. +// vetoIfHealPending aborts a pending heal so its publish cannot resurrect a deleted checkpoint. func (m *Manager) vetoIfHealPending(ckptID string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -359,9 +318,7 @@ func (m *Manager) vetoIfHealPending(ckptID string) { } } -// pinnedArchiveCks is the set of checkpoint ids backing a live archived claim -// or an archive transition in flight (pendingCks): wake images the listing hides -// and delete/TTL must spare (deleting one would strand its sandbox). +// pinnedArchiveCks is the set of wake-image ids the listing hides and delete/TTL must spare. func (m *Manager) pinnedArchiveCks() map[string]struct{} { m.mu.Lock() defer m.mu.Unlock() @@ -377,17 +334,14 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { return pinned } -// sweepExpiredCheckpoints ages out checkpoints older than the configured -// TTL; explicit deletes never wait for it. It runs detached from the Run loop. +// sweepExpiredCheckpoints ages out checkpoints older than the configured TTL. func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { if !m.ckptSweeping.CompareAndSwap(false, true) { return } defer m.ckptSweeping.Store(false) logger := log.WithFunc("pool.sweepExpiredCheckpoints") - // Checkpoints already hides archive images backing a live claim, so the - // TTL never reaches one; their retention is the claim's own Deadline - // (reapPurge). + // Checkpoints hides archive images, so the TTL never reaches one; reapPurge retires them ckpts, err := m.Checkpoints(ctx, "") if err != nil { logger.Error(ctx, err, "list for retention") @@ -404,9 +358,7 @@ func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { } } -// deleteCkLocked removes a checkpoint under its record lock, so the delete -// never runs beneath an in-flight branch clone, heal, or archived wake; the -// lock is released before eviction is considered (see recDoneEvict). +// deleteCkLocked removes a checkpoint under its record lock, releasing it before eviction. func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { l := m.recLock(ckID) l.Lock() @@ -434,8 +386,7 @@ func (m *Manager) loadCheckpoint(ctx context.Context, ckptID string) (types.Chec return parseCheckpoint(raw) } -// validateHealedCheckpoint checks a staged pull's shape before publishing it: -// an unreadable or misattributed record would suppress every later heal. +// validateHealedCheckpoint checks a staged pull's shape: a bad record suppresses later heals. func validateHealedCheckpoint(staging, wantID string) error { raw, err := os.ReadFile(filepath.Join(staging, store.MetaFile)) //nolint:gosec // staging dir is this manager's own if err != nil { @@ -455,17 +406,14 @@ func validateHealedCheckpoint(staging, wantID string) error { return fmt.Errorf("healed record %s has an invalid key: %w", wantID, keyErr) } if !ckpt.Key.Capturable() { - // A well-formed egress key is not branchable: publishing it would - // poison the id and suppress a good owner; reject, try the next owner. + // publishing a non-branchable key would poison the id and suppress a good owner return fmt.Errorf("healed record %s has a non-branchable (egress) key", wantID) } export, err := os.ReadDir(filepath.Join(staging, store.ExportDir)) if err != nil { return fmt.Errorf("healed record %s missing export dir: %w", wantID, err) } - // A present export with no regular file clones to nothing (an empty dir, or - // only empty subdirs); the byte content is cocoon's own format and not - // sandboxd's to validate further. + // an export with no regular file clones to nothing; deeper content is cocoon's own format if !hasRegularFile(export) { return fmt.Errorf("healed record %s has no export content", wantID) } @@ -481,8 +429,7 @@ func validateHealedCheckpoint(staging, wantID string) error { return nil } -// hasRegularFile reports whether entries holds at least one regular file, so an -// export of only empty subdirectories is treated as empty. +// hasRegularFile reports whether entries holds at least one regular file. func hasRegularFile(entries []os.DirEntry) bool { return slices.ContainsFunc(entries, func(e os.DirEntry) bool { return e.Type().IsRegular() }) } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index e9e2d8c1..0c2c94e7 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -16,9 +16,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// TestClaimCheckpointNeverPullsOnMiss: ClaimCheckpoint answers from the local -// store alone — a redirect exists precisely so this path never pays a peer -// transfer; only ClaimCheckpointHeal may. func TestClaimCheckpointNeverPullsOnMiss(t *testing.T) { ckpt := types.Checkpoint{ID: "ck_00000000000000bb", Key: testKey, CreatedAt: time.Now()} m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) @@ -31,8 +28,6 @@ func TestClaimCheckpointNeverPullsOnMiss(t *testing.T) { } } -// TestClaimCheckpointHealPullsOnce: a heal pays the transfer once and leaves -// the record served from the local store from then on. func TestClaimCheckpointHealPullsOnce(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -53,8 +48,6 @@ func TestClaimCheckpointHealPullsOnce(t *testing.T) { } } -// TestClaimCheckpointAfterHealStaysLocal: once healed, the record is served -// from the local store — a later branch must not pull again. func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -71,12 +64,6 @@ func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { } } -// TestClaimCheckpointHealDedupsConcurrentPulls: two branches racing to heal -// the same missing checkpoint must share one transfer, not each pay for it — -// healFlights (not the record lock, which no longer spans the transfer) -// dedups them onto one Manager-owned staging dir, and each branch must -// still get a genuinely valid, non-empty checkpoint out of it (the exact -// case a caller-owned-staging-dir singleflight used to get wrong). func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { synctest.Test(t, func(t *testing.T) { id := "ck_00000000000000bb" @@ -93,7 +80,7 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { }) } waitFor(t, func() bool { return puller.count() >= 1 }) - time.Sleep(50 * time.Millisecond) // let the second goroutine join the same flight + time.Sleep(50 * time.Millisecond) close(puller.release) wg.Wait() @@ -112,14 +99,6 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { }) } -// TestDeleteVetoesInFlightHeal covers the Codex r2 blocker: a heal's transfer -// runs unlocked (holding the lock across a 30-minute budget would pin every -// other operation on the id behind it), so a delete arriving while it stages -// must not block on that transfer — but it must also not let the heal -// resurrect the checkpoint moments after the delete answered "not here" for -// it. The delete returns immediately (nothing local to delete yet) and -// vetoes the pending heal, so the heal's own locked decide phase abandons -// its publish instead of racing the delete to a stale conclusion. func TestDeleteVetoesInFlightHeal(t *testing.T) { synctest.Test(t, func(t *testing.T) { id := "ck_00000000000000bb" @@ -132,7 +111,7 @@ func TestDeleteVetoesInFlightHeal(t *testing.T) { _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") healDone <- err }() - waitFor(t, func() bool { return puller.count() >= 1 }) // heal is staging, unpublished + waitFor(t, func() bool { return puller.count() >= 1 }) if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteLocal); !errors.Is(err, ErrUnknownCheckpoint) { t.Fatalf("delete while heal is staging: %v, want ErrUnknownCheckpoint (nothing local yet, and it must not block)", err) @@ -148,11 +127,6 @@ func TestDeleteVetoesInFlightHeal(t *testing.T) { }) } -// TestClaimCheckpointHealCtxCancelReturnsPromptly: a caller that gives up -// waiting must not be stuck behind the whole heal budget — the flight runs -// detached (context.Background() in runHeal), so this call returns -// ctx.Err() as soon as its own ctx is canceled, while the transfer keeps -// running and still lands. func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { synctest.Test(t, func(t *testing.T) { id := "ck_00000000000000bb" @@ -166,7 +140,7 @@ func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { _, err := m.ClaimCheckpointHeal(ctx, id, time.Hour, "") done <- err }() - waitFor(t, func() bool { return puller.count() >= 1 }) // transfer started, unlocked + waitFor(t, func() bool { return puller.count() >= 1 }) cancel() select { @@ -178,72 +152,58 @@ func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { t.Fatal("ClaimCheckpointHeal did not return promptly after ctx cancellation") } - // Detached from the canceled caller, the transfer must still complete - // and publish. close(puller.release) waitFor(t, func() bool { return m.HasCheckpoint(t.Context(), id) }) }) } -// TestRecLockEvictionWaitsForAllHolders covers the lock-identity-split half -// of the same Codex r2 blocker: evicting an id's entry while another holder -// still references it would let that holder's (or a new caller's) next -// recLock for the same id return a different mutex, splitting mutual -// exclusion between them. Eviction must wait until every holder has -// released, not just the one that happened to delete the record. func TestRecLockEvictionWaitsForAllHolders(t *testing.T) { m := newTestManager(t, newFakeEngine()) const id = "ck_00000000000000aa" l1 := m.recLock(id) l1.Lock() - l2 := m.recLock(id) // a second, concurrent holder registers interest first + l2 := m.recLock(id) if l2 != l1 { t.Fatalf("recLock returned a different mutex while the first holder is still outstanding") } l1.Unlock() - m.recDoneEvict(id) // first holder's release must not evict: l2 is still outstanding - if _, ok := m.recLocks.Load(id); !ok { + m.recDoneEvict(id) + if !hasRecLock(m, id) { t.Fatal("entry evicted while a second holder still references it") } - l3 := m.recLock(id) // a third caller arriving now must still find the SAME mutex + l3 := m.recLock(id) if l3 != l1 { t.Fatal("a concurrent recLock diverged onto a different mutex for the same id — the lock-identity split") } - m.recDone(id) // l2's release - m.recDoneEvict(id) // l3's release, last one out - if _, ok := m.recLocks.Load(id); ok { + m.recDone(id) + m.recDoneEvict(id) + if hasRecLock(m, id) { t.Error("entry not evicted once every holder released") } } -// TestRecLockEvictDeferredToLastHolder is the production sequence the test -// above sidesteps: a delete (recDoneEvict) releases while an ordinary holder -// (recDone) still references the id, so the ordinary holder is the last one -// out. The eviction the delete asked for must not be lost. func TestRecLockEvictDeferredToLastHolder(t *testing.T) { m := newTestManager(t, newFakeEngine()) const id = "ck_00000000000000aa" - l1 := m.recLock(id) // the delete - l2 := m.recLock(id) // a concurrent claim/fetch + l1 := m.recLock(id) + l2 := m.recLock(id) if l2 != l1 { t.Fatal("recLock diverged for the same id") } - m.recDoneEvict(id) // delete releases first, wants eviction, but l2 still holds - if _, ok := m.recLocks.Load(id); !ok { + m.recDoneEvict(id) + if !hasRecLock(m, id) { t.Fatal("entry evicted while an ordinary holder still references it") } - m.recDone(id) // the ordinary holder, last one out - if _, ok := m.recLocks.Load(id); ok { + m.recDone(id) + if hasRecLock(m, id) { t.Error("recLocks entry leaked: a delete's deferred eviction was lost when a plain recDone dropped the last reference") } } -// TestFetchCheckpointNeverPulls: the peer-transfer blob endpoint must never -// trigger a recursive pull, even with a healer installed that could serve it. func TestFetchCheckpointNeverPulls(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -257,8 +217,6 @@ func TestFetchCheckpointNeverPulls(t *testing.T) { } } -// TestHasCheckpoint covers the probe answer across a present, missing, and -// archive record: only a live branchable checkpoint reports true. func TestHasCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) @@ -282,10 +240,6 @@ func TestHasCheckpoint(t *testing.T) { } } -// TestClaimCheckpointQuotaDoesNotMaskMiss: a full node must still answer -// "not here" for a checkpoint it does not hold, or the handler's -// redirect/heal tiers behind ErrUnknownCheckpoint never run; a full node -// that DOES hold the record still answers the quota error. func TestClaimCheckpointQuotaDoesNotMaskMiss(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -304,9 +258,6 @@ func TestClaimCheckpointQuotaDoesNotMaskMiss(t *testing.T) { } } -// TestClaimCheckpointHealQuotaBeforePull: unlike ClaimCheckpoint's cheap -// local read, resolving a heal means a peer transfer — quota must reject a -// full node before that cost, so the puller must never be called. func TestClaimCheckpointHealQuotaBeforePull(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -321,10 +272,6 @@ func TestClaimCheckpointHealQuotaBeforePull(t *testing.T) { } } -// TestClaimCheckpointHealConcurrencyCapRejectsExtra: heals of DIFFERENT ids -// (so the recLock does not serialize them) must still be capped node-wide — -// each pulls a whole guest memory image, so distinct ids are not free just -// because they don't contend on the same lock. func TestClaimCheckpointHealConcurrencyCapRejectsExtra(t *testing.T) { m := newTestManager(t, newFakeEngine()) m.healSem = make(chan struct{}, 1) @@ -347,8 +294,6 @@ func TestClaimCheckpointHealConcurrencyCapRejectsExtra(t *testing.T) { } } -// TestValidateHealedCheckpointAcceptsValid: a well-formed staged record must -// pass every check. func TestValidateHealedCheckpointAcceptsValid(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa"}) @@ -357,9 +302,6 @@ func TestValidateHealedCheckpointAcceptsValid(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsMismatchedID: a peer's meta.json must -// name the id that was actually requested, or a hostile peer could serve a -// crafted record under someone else's id. func TestValidateHealedCheckpointRejectsMismatchedID(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000bb"}) @@ -368,8 +310,6 @@ func TestValidateHealedCheckpointRejectsMismatchedID(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsMissingMeta: no meta.json means nothing -// to trust. func TestValidateHealedCheckpointRejectsMissingMeta(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { @@ -380,12 +320,9 @@ func TestValidateHealedCheckpointRejectsMissingMeta(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsMissingExport: a meta with no export -// dir cannot back a branch. func TestValidateHealedCheckpointRejectsMissingExport(t *testing.T) { dir := t.TempDir() - // A valid key, so the record reaches the export check rather than failing - // earlier on the key. + meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa", Key: testKey}) if err != nil { t.Fatalf("setup: %v", err) @@ -398,8 +335,6 @@ func TestValidateHealedCheckpointRejectsMissingExport(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsArchive: a wake image must never be -// healed in as a branchable checkpoint. func TestValidateHealedCheckpointRejectsArchive(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Archive: true}) @@ -408,10 +343,6 @@ func TestValidateHealedCheckpointRejectsArchive(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsEmptyExport: an export directory that is -// present but empty clones to nothing — publishing it would suppress a good -// owner and fail every later local claim, so it must be rejected and the next -// owner tried. func TestValidateHealedCheckpointRejectsEmptyExport(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { @@ -429,8 +360,6 @@ func TestValidateHealedCheckpointRejectsEmptyExport(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsInvalidKey: a record whose key does not -// name a branchable pool must not be published. func TestValidateHealedCheckpointRejectsInvalidKey(t *testing.T) { dir := t.TempDir() plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Key: types.PoolKey{Net: "bogus"}}) @@ -439,9 +368,6 @@ func TestValidateHealedCheckpointRejectsInvalidKey(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsEgressKey: an egress-lane key is valid but -// not branchable (claimLoaded refuses it), so a healed record carrying one must -// be rejected rather than published to poison the id. func TestValidateHealedCheckpointRejectsEgressKey(t *testing.T) { dir := t.TempDir() egress := testKey @@ -452,8 +378,6 @@ func TestValidateHealedCheckpointRejectsEgressKey(t *testing.T) { } } -// TestValidateHealedCheckpointRejectsContentlessExport: an export directory -// holding only an empty subdirectory clones to nothing and must be rejected. func TestValidateHealedCheckpointRejectsContentlessExport(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, store.ExportDir, "sub"), 0o750); err != nil { @@ -471,8 +395,6 @@ func TestValidateHealedCheckpointRejectsContentlessExport(t *testing.T) { } } -// TestDeleteCheckpointBroadcasts: a local delete calls the peer-delete hook -// once so a copy healed onto another node is removed too. func TestDeleteCheckpointBroadcasts(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -492,9 +414,6 @@ func TestDeleteCheckpointBroadcasts(t *testing.T) { } } -// TestDeleteCheckpointNoForwardSkipsBroadcast: a delete arriving as a -// broadcast (no_forward) must not re-broadcast, or the fleet would loop -// forever. func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -514,9 +433,6 @@ func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { } } -// TestDeleteCheckpointSharedStoreSkipsBroadcast: an s3 (or other shared) -// backend has one copy every node already sees, so broadcasting would be a -// no-op fan-out; DeleteCheckpoint must skip it. func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -537,9 +453,6 @@ func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { } } -// newHealManager builds a Manager with a healer wired to a puller serving -// ckpt at addrs, mirroring WithPeerHeal's construction directly (tests are in -// the same package). func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manager, *healPuller) { t.Helper() m := newTestManager(t, newFakeEngine()) @@ -548,14 +461,11 @@ func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manag return m, puller } -// healPuller fakes peer.Puller: it writes a valid record (meta.json + an -// export/ dir the fakeEngine "clones" from) and can block until -// release, so a test can prove concurrent misses dedup to one pull. type healPuller struct { mu sync.Mutex calls int ckpt types.Checkpoint - release chan struct{} // non-nil: Pull blocks until closed + release chan struct{} } func (p *healPuller) Pull(_ context.Context, _, _, dst string) error { @@ -584,9 +494,6 @@ func (p *healPuller) count() int { return p.calls } -// blockingPuller writes a valid record stamped with the requested id, after -// signaling started and then blocking until release — for proving the -// node-wide heal concurrency cap against distinct, non-contending ids. type blockingPuller struct { started chan struct{} release chan struct{} @@ -609,11 +516,10 @@ func (p *blockingPuller) Pull(_ context.Context, _, id, dst string) error { return os.WriteFile(filepath.Join(dst, store.MetaFile), meta, 0o600) } -// plantHealedRecord writes a valid-shaped staged record for validate tests. func plantHealedRecord(t *testing.T, dir string, ckpt types.Checkpoint) { t.Helper() if ckpt.Key == (types.PoolKey{}) { - ckpt.Key = testKey // a healed record carries a real, branchable key + ckpt.Key = testKey } if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { t.Fatalf("setup: %v", err) diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index a924463a..8ee529aa 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -106,8 +106,8 @@ func TestRetentionSweepsExpiredCheckpoints(t *testing.T) { t.Fatalf("setup: %v", err) } } - plant("ck_000000000000000a", 2*time.Hour) // expired - plant("ck_000000000000000b", time.Minute) // fresh + plant("ck_000000000000000a", 2*time.Hour) + plant("ck_000000000000000b", time.Minute) m.sweepExpiredCheckpoints(t.Context()) @@ -147,7 +147,6 @@ func TestCheckpointTenantIsolation(t *testing.T) { t.Errorf("root listing %+v, %v; want both records", ckpts, err) } - // A tenant cannot delete another's record — and learns nothing beyond 404. if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "beta", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("cross-tenant delete: %v, want ErrUnknownCheckpoint", err) } @@ -159,20 +158,12 @@ func TestCheckpointTenantIsolation(t *testing.T) { } } -// TestCheckpointDeleteEvictsRecLock proves recLocks does not retain a lock per -// checkpoint: ck ids are single-use, so a create+delete cycle must leave the -// map at its prior size, or a long-running node leaks one RWMutex per ck. func TestCheckpointDeleteEvictsRecLock(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) src := mustClaim(t, m, testKey) - countLocks := func() int { - n := 0 - m.recLocks.Range(func(_, _ any) bool { n++; return true }) - return n - } - base := countLocks() // the golden-resolve's template lock: bounded, never evicted + base := lockCount(m) var ids []string for range 5 { @@ -186,26 +177,18 @@ func TestCheckpointDeleteEvictsRecLock(t *testing.T) { if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteFleet); err != nil { t.Fatalf("delete %s: %v", id, err) } - if _, ok := m.recLocks.Load(id); ok { + if hasRecLock(m, id) { t.Errorf("recLocks retained a lock for deleted checkpoint %s", id) } } - if got := countLocks(); got != base { + if got := lockCount(m); got != base { t.Errorf("recLocks grew %d->%d over 5 checkpoint create+delete cycles, want no net growth", base, got) } } -// TestRejectedCheckpointOpsLeakNoRecLock: a rejected claim or delete (bad -// format, unknown, wrong tenant) must not create a lock-map entry — otherwise -// a guess-loop grows recLocks without bound. func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { m := newTestManager(t, newFakeEngine()) - countLocks := func() int { - n := 0 - m.recLocks.Range(func(_, _ any) bool { n++; return true }) - return n - } - base := countLocks() + base := lockCount(m) for _, id := range []string{"../../etc", "ck_zz", "", "ck_00112233445566ff", "ck_deadbeefdeadbeef"} { if _, err := m.ClaimCheckpoint(t.Context(), id, time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { @@ -215,21 +198,20 @@ func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { t.Errorf("delete %q: %v, want ErrUnknownCheckpoint", id, err) } } - if got := countLocks(); got != base { + if got := lockCount(m); got != base { t.Errorf("recLocks grew %d->%d over rejected checkpoint ops, want no growth", base, got) } - // A wrong-tenant delete of a real checkpoint must also leak nothing. src := mustClaim(t, m, testKey) ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "acme") if err != nil { t.Fatalf("checkpoint: %v", err) } - before := countLocks() + before := lockCount(m) if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "other", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("wrong-tenant delete: %v, want ErrUnknownCheckpoint", err) } - if got := countLocks(); got != before { + if got := lockCount(m); got != before { t.Errorf("recLocks grew %d->%d over a wrong-tenant delete, want no growth", before, got) } } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index cd42f0cc..f7d248a6 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -21,10 +21,7 @@ const ( reapPurge // archived past retention: delete the store checkpoint ) -// ClaimWarm transfers ownership of a warm sandbox without provisioning; -// ErrNoWarm means the pool is empty (the caller may redirect or provision). -// tenant attributes the claim; empty means the operator (root). claimRef is an -// opaque caller reference recorded on the claim; empty means none. +// ClaimWarm transfers ownership of a warm sandbox without provisioning; ErrNoWarm means empty. func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { start := time.Now() if err := m.validate(key); err != nil { @@ -71,13 +68,11 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Dur } // ClaimProvision creates a claim-ready sandbox (golden clone or cold boot). -// claimRef is an opaque caller reference recorded on the claim; empty means none. func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { return m.claimProvision(ctx, key, ttl, tenant, claimRef, volumes, false) } -// ClaimProvisionPromoted requires key to resolve from a promoted template and -// never falls through to a cold image boot. +// ClaimProvisionPromoted requires key to resolve from a promoted template, never a cold boot. func (m *Manager) ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { return m.claimProvision(ctx, key, ttl, tenant, claimRef, volumes, true) } @@ -91,8 +86,7 @@ func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { return m.releaseResolved(ctx, id, sb) } -// ClaimDeadline authorizes a sandbox by token and returns its lease -// deadline — the preview mint clamps a URL's life to it. +// ClaimDeadline authorizes a sandbox by token and returns its lease deadline. func (m *Manager) ClaimDeadline(id, token string) (time.Time, error) { m.mu.Lock() defer m.mu.Unlock() @@ -120,8 +114,7 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. return m.eng.DialGuestPort(ctx, sock, port) } -// AgentSocket resolves a claimed sandbox's vsock UDS without waking it (the -// ownership probe must not restore a hibernated VM). +// AgentSocket resolves a claimed sandbox's vsock UDS without waking it. func (m *Manager) AgentSocket(id, token string) (string, error) { m.mu.Lock() defer m.mu.Unlock() @@ -129,15 +122,11 @@ func (m *Manager) AgentSocket(id, token string) (string, error) { if !ok { return "", ErrUnknownSandbox } - // No activity stamp here: owner/Lookup probes use this path, and a - // control-plane poll must not keep an idle sandbox awake. The relay's - // stamp lives in WakeAgentSocket. + // no activity stamp: a control-plane poll must not keep an idle sandbox awake return sb.VsockSocket, nil } -// releaseResolved drops the resolved claim and tears down its VM. resolve -// unlocks before this runs, so it re-checks under m.mu that sb is still the -// live claim: a second release racing in must not tear down twice. +// releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { m.mu.Lock() if m.claimed[id] != sb { @@ -154,14 +143,14 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand } delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) - js := m.store.snapshot(m.claimed) + js := m.store.del(id) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() - m.claimed[id] = sb // roll back so memory matches the still-durable claim; the restored claim re-pins ck + m.claimed[id] = sb // roll back so memory matches the still-durable claim m.tenantDelta(sb.Tenant, 1) delete(m.pendingCks, ck) - rb := m.store.snapshot(m.claimed) + rb := m.store.set(sb) m.mu.Unlock() if ck != "" { if clearErr := m.clearArchiveCk(ck); clearErr != nil { @@ -192,15 +181,14 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand return err } -// overQuota is a cheap advisory precheck; finalizeBatch does the authoritative -// admission check, so this just spares a doomed request the provision cost. +// overQuota is an advisory precheck; finalizeBatch does the authoritative admission check. func (m *Manager) overQuota(extra int, tenant string) error { m.mu.Lock() defer m.mu.Unlock() return m.quotaErr(extra, tenant) } -// admitVolumes takes one claim's holds, then the marker check; a refusal on either leaves nothing held. +// admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) if err := m.admitClaim(tenant, applied); err != nil { @@ -213,9 +201,7 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types return applied, nil } -// admitClaim is the provision path's one admission section: the advisory quota -// precheck plus the authoritative volume reservation, so a busy volume is -// refused before any VM is built. +// admitClaim refuses a busy volume or an over-quota tenant before any VM is built. func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { m.mu.Lock() defer m.mu.Unlock() @@ -225,8 +211,7 @@ func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { return m.reserveVolumes(volumes) } -// quotaErr answers ErrQuota over the node cap, the tenant cap, or a -// draining node; callers hold m.mu. +// quotaErr answers ErrQuota over a cap or a draining node; callers hold m.mu. func (m *Manager) quotaErr(extra int, tenant string) error { if m.draining { return fmt.Errorf("%w: node draining", ErrQuota) @@ -256,8 +241,7 @@ func (m *Manager) tenantDelta(tenant string, delta int) { } } -// finalize stamps identity, persists the claim, and destroys the VM if the -// store write fails so a durable claim always matches a live VM. +// finalize stamps identity and persists the claim; a failed write destroys the VM. func (m *Manager) finalize(ctx context.Context, sb *types.Sandbox, ttl time.Duration) (*types.Sandbox, error) { if err := m.finalizeBatch(ctx, []*types.Sandbox{sb}, ttl); err != nil { return nil, err @@ -265,9 +249,7 @@ func (m *Manager) finalize(ctx context.Context, sb *types.Sandbox, ttl time.Dura return sb, nil } -// finalizeBatch stamps identities and persists the batch as one journal write; -// a failed write destroys every VM in it — a durable claim always matches a -// live VM, all-or-nothing. One tenant per batch (fork children inherit it). +// finalizeBatch persists the batch as one write, all-or-nothing; one tenant per batch. func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl time.Duration) error { now := time.Now() for _, sb := range sbs { @@ -278,8 +260,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t if quotaErr := m.quotaErr(len(sbs), sbs[0].Tenant); quotaErr != nil { m.mu.Unlock() for _, sb := range sbs { - // Never handed out, but the rw mount itself dirtied the journal — the - // clean umount is owed exactly as on the rollback path. + // the rw mount dirtied the journal, so the clean umount is still owed td := m.quiesceVolumes(ctx, sb) m.removeOrRetry(ctx, sb.VMName, "", "", td) } @@ -289,7 +270,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t m.claimed[sb.ID] = sb m.tenantDelta(sb.Tenant, 1) } - js := m.store.snapshot(m.claimed) + js := m.store.set(sbs...) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.rollbackClaim(ctx, sbs) @@ -301,8 +282,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t return fmt.Errorf("arm egress %s: %w", sb.ID, armErr) } } - // Usage lands only after the whole batch armed: a rollback must not leave - // claim events with no terminal release/reap in the billing stream. + // usage lands only after the batch armed, so a rollback leaves no unterminated claim event for _, sb := range sbs { m.recordUsage(ctx, usageEvent{ Event: "claim", //nolint:goconst // event name; other occurrences are test assertions @@ -314,16 +294,16 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t return nil } -// rollbackClaim unwinds a claim batch after a persist or egress-arm failure: -// drop the claims, reconverge the journal, destroy the VMs — a NIC that -// cannot be locked must never be handed out. +// rollbackClaim unwinds a claim batch: a NIC that cannot be locked is never handed out. func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { m.mu.Lock() - for _, sb := range sbs { + ids := make([]string, len(sbs)) + for i, sb := range sbs { delete(m.claimed, sb.ID) m.tenantDelta(sb.Tenant, -1) + ids[i] = sb.ID } - rb := m.store.snapshot(m.claimed) + rb := m.store.del(ids...) m.mu.Unlock() m.recommit(ctx, rb) for _, sb := range sbs { @@ -332,8 +312,7 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { } } -// abortVolumeClaim removes a failed claim's VM, holds riding the teardown; -// clearing reserved stops the deferred unreserve from double-releasing them. +// abortVolumeClaim removes a failed claim's VM and clears reserved against a double release. func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved *[]types.Volume) { m.removeOrRetry(ctx, vmName, "", "", volumeTeardown{holds: *reserved}) *reserved = nil @@ -348,6 +327,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() var expired []victim + var dropped []string for id, sb := range m.claimed { // A zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { @@ -357,8 +337,7 @@ func (m *Manager) reapOnce(ctx context.Context) { case sb.ArchiveCk != "": expired = append(expired, victim{action: reapPurge, id: id, ck: sb.ArchiveCk, tenant: sb.Tenant, sb: sb}) case sb.HibernateSnap != "" && m.archiveEnabledFor(sb.Key): - // End-of-lease hibernated + archive-enabled: archive instead of - // destroy, kept in m.claimed for archive() to re-validate and move. + // archive instead of destroy, kept in m.claimed for archive() to move if _, busy := m.archiving[id]; busy { continue // an archive is already exporting this sandbox } @@ -368,8 +347,12 @@ func (m *Manager) reapOnce(ctx context.Context) { expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb}) delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) + dropped = append(dropped, id) } } + if len(dropped) > 0 { + m.store.del(dropped...) + } m.mu.Unlock() if len(expired) == 0 { return @@ -392,7 +375,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() keep = expired[:0] - var purgeCks []string + var purgeCks, purged []string for _, v := range expired { if v.action == reapPurge { if m.claimed[v.id] != v.sb || v.sb.ArchiveCk != v.ck { @@ -402,6 +385,7 @@ func (m *Manager) reapOnce(ctx context.Context) { delete(m.claimed, v.id) m.tenantDelta(v.tenant, -1) purgeCks = append(purgeCks, v.ck) + purged = append(purged, v.id) } keep = append(keep, v) } @@ -410,10 +394,11 @@ func (m *Manager) reapOnce(ctx context.Context) { m.mu.Unlock() return } - js := m.store.snapshot(m.claimed) + js := m.store.del(purged...) m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { m.mu.Lock() + var restored []*types.Sandbox for _, v := range expired { if v.action == reapArchive { delete(m.archiving, v.id) // never removed from m.claimed @@ -421,9 +406,10 @@ func (m *Manager) reapOnce(ctx context.Context) { m.claimed[v.id] = v.sb // roll back so memory matches the still-durable claim m.tenantDelta(v.tenant, 1) delete(m.pendingCks, v.ck) + restored = append(restored, v.sb) } } - rb := m.store.snapshot(m.claimed) + rb := m.store.set(restored...) m.mu.Unlock() for _, ck := range purgeCks { if clearErr := m.clearArchiveCk(ck); clearErr != nil { @@ -434,8 +420,7 @@ func (m *Manager) reapOnce(ctx context.Context) { logger.Error(ctx, saveErr, "persist reap; rolled back") return } - // Engine subprocesses (worst case minutes on a hung stop): fan out bounded - // so a big batch never stalls the ticker loop. + // engine teardown can take minutes, so fan out bounded and never stall the ticker loop m.runBounded(ctx, len(expired), func(ctx context.Context, i int) { v := expired[i] switch v.action { @@ -468,9 +453,7 @@ func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { m.recordUsage(ctx, usageEvent{Event: "archive_delete", ID: id, Reference: ck, Tenant: tenant}) } -// recommit re-persists a snapshot in the background until its own success or -// any newer durable write ends the loop (commit coalesces by sequence); -// detached, because callers may hold the transition lock. +// recommit re-persists in the background until success or a newer durable write; detached. func (m *Manager) recommit(ctx context.Context, snap claimSnapshot) { go func() { backoff := recommitBackoff diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 5542df8f..9b1c2c7d 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -5,26 +5,22 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "path/filepath" "slices" "sync" - "sync/atomic" "time" "github.com/cocoonstack/sandbox/sandboxd/types" ) -// claimSnapshot is a sequenced, detached copy of the claim set awaiting a -// durable write; commit() marshals it off the manager mutex. +// claimSnapshot sequences one persist request; commit skips it once a newer write has landed. type claimSnapshot struct { - claims map[string]claimDTO - seq uint64 + seq uint64 } -// claimDTO is the persisted projection of a Sandbox (its json fields only), -// copied by value so commit's marshal runs off the manager mutex without -// racing the live record's Transition mutex and lastActivity. +// claimDTO is the persisted projection of a Sandbox, copied so commit marshals off m.mu. type claimDTO struct { ID string `json:"id"` VMName string `json:"vm_name"` @@ -42,19 +38,20 @@ type claimDTO struct { FromCheckpoint string `json:"from_checkpoint,omitempty"` } -// claimStore persists claimed sandboxes across daemon restarts. Warm pool -// VMs are deliberately not persisted: they are cheap to rebuild and unsafe -// to trust after an unsupervised gap. +// claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string - seq atomic.Uint64 // sequence source; bumped in snapshot (mutation order) + writeMu sync.Mutex + mu sync.Mutex - written uint64 // highest sequence on disk; guarded by mu + dtos map[string]claimDTO + seq uint64 + written uint64 } func newClaimStore(dataDir string) *claimStore { - return &claimStore{path: filepath.Join(dataDir, "claims.json")} + return &claimStore{path: filepath.Join(dataDir, "claims.json"), dtos: map[string]claimDTO{}} } func (s *claimStore) load() (map[string]*types.Sandbox, error) { @@ -72,33 +69,59 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { return claims, nil } -// snapshot copies the claim set into a detached, sequenced value; the caller -// holds the manager mutex, so sequences are handed out in mutation order. -func (s *claimStore) snapshot(claims map[string]*types.Sandbox) claimSnapshot { - seq := s.seq.Add(1) - dtos := make(map[string]claimDTO, len(claims)) +// set records each sandbox's projection and sequences the write; callers may hold m.mu. +func (s *claimStore) set(sbs ...*types.Sandbox) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + for _, sb := range sbs { + s.dtos[sb.ID] = dtoOf(sb) + } + s.seq++ + return claimSnapshot{seq: s.seq} +} + +// del drops the projections of ids and sequences the write; callers may hold m.mu. +func (s *claimStore) del(ids ...string) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + for _, id := range ids { + delete(s.dtos, id) + } + s.seq++ + return claimSnapshot{seq: s.seq} +} + +// mark sequences a persist of the projection as it already stands. +func (s *claimStore) mark() claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + return claimSnapshot{seq: s.seq} +} + +// reset rebuilds the whole projection; the startup path, before contention exists. +func (s *claimStore) reset(claims map[string]*types.Sandbox) claimSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + s.dtos = make(map[string]claimDTO, len(claims)) for id, sb := range claims { - dtos[id] = claimDTO{ - ID: sb.ID, VMName: sb.VMName, Key: sb.Key, Token: sb.Token, - Deadline: sb.Deadline, Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, - Volumes: slices.Clone(sb.Volumes), VsockSocket: sb.VsockSocket, - TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, - ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, - } + s.dtos[id] = dtoOf(sb) } - return claimSnapshot{claims: dtos, seq: seq} + s.seq++ + return claimSnapshot{seq: s.seq} } -// commit marshals a snapshot and atomically replaces claims.json, off the -// manager mutex; serialized by s.mu, a stale snapshot is a no-op. Unsynced by -// design (#21): an fsync would sit on the sub-ms warm-claim path. +// commit atomically replaces claims.json; unsynced by design (#21) to keep the claim path fast. func (s *claimStore) commit(snap claimSnapshot) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() s.mu.Lock() - defer s.mu.Unlock() if snap.seq <= s.written { + s.mu.Unlock() return nil } - raw, err := json.Marshal(snap.claims) + dtos, seq := maps.Clone(s.dtos), s.seq + s.mu.Unlock() + raw, err := json.Marshal(dtos) if err != nil { return fmt.Errorf("encode claims: %w", err) } @@ -109,19 +132,30 @@ func (s *claimStore) commit(snap claimSnapshot) error { if err := os.Rename(tmp, s.path); err != nil { return fmt.Errorf("commit claims: %w", err) } - s.written = snap.seq + s.mu.Lock() + s.written = seq + s.mu.Unlock() return nil } -// save is the combined form for the startup Reconcile pass, before contention -// exists; hot-path callers split snapshot()/commit() around the manager mutex. +// save is the combined form for the startup Reconcile pass, before contention exists. func (s *claimStore) save(claims map[string]*types.Sandbox) error { - return s.commit(s.snapshot(claims)) + return s.commit(s.reset(claims)) } -// synced reports whether every handed-out snapshot has reached disk. +// synced reports whether every sequenced change has reached disk. func (s *claimStore) synced() bool { s.mu.Lock() defer s.mu.Unlock() - return s.written == s.seq.Load() + return s.written == s.seq +} + +func dtoOf(sb *types.Sandbox) claimDTO { + return claimDTO{ + ID: sb.ID, VMName: sb.VMName, Key: sb.Key, Token: sb.Token, + Deadline: sb.Deadline, Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, + Volumes: slices.Clone(sb.Volumes), VsockSocket: sb.VsockSocket, + TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, + ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, + } } diff --git a/sandboxd/pool/claims_bench_test.go b/sandboxd/pool/claims_bench_test.go index 03fb1df9..c8e8ec2f 100644 --- a/sandboxd/pool/claims_bench_test.go +++ b/sandboxd/pool/claims_bench_test.go @@ -10,8 +10,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// BenchmarkStoreSaveScaling measures the claims-journal persist cost (marshal -// + write + rename) as the live-claim set grows. func BenchmarkStoreSaveScaling(b *testing.B) { for _, n := range []int{10, 100, 1000} { b.Run(fmt.Sprintf("claims=%d", n), func(b *testing.B) { @@ -26,19 +24,14 @@ func BenchmarkStoreSaveScaling(b *testing.B) { } } -// BenchmarkStorePersistContention reports how long a concurrent manager-mutex -// acquire (a data-plane op) waits while the journal is being persisted, under -// the old "write under the lock" path (save) vs the new "marshal under the lock, -// write off it" split (snapshot+commit). The ns/acquire metric is the win. func BenchmarkStorePersistContention(b *testing.B) { for _, n := range []int{100, 1000} { claims := benchClaims(n) - b.Run(fmt.Sprintf("under-lock/n=%d", n), func(b *testing.B) { - benchPersistContention(b, claims, false) - }) - b.Run(fmt.Sprintf("off-lock/n=%d", n), func(b *testing.B) { - benchPersistContention(b, claims, true) - }) + for _, arm := range []string{"under-lock", "rebuild", "incremental"} { + b.Run(fmt.Sprintf("%s/n=%d", arm, n), func(b *testing.B) { + benchPersistContention(b, claims, arm) + }) + } } } @@ -55,8 +48,10 @@ func benchClaims(n int) map[string]*types.Sandbox { return claims } -func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, offLock bool) { +func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, arm string) { s := newClaimStore(b.TempDir()) + s.reset(claims) + one := claims[fmt.Sprintf("sb_%016x", 0)] var mu sync.Mutex var waitNs, waits atomic.Int64 done := make(chan struct{}) @@ -76,15 +71,21 @@ func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, offL }() for b.Loop() { - if offLock { + switch arm { + case "under-lock": + mu.Lock() + _ = s.save(claims) + mu.Unlock() + case "rebuild": mu.Lock() - snap := s.snapshot(claims) // marshal under the lock + snap := s.reset(claims) mu.Unlock() - _ = s.commit(snap) // write off the lock - } else { + _ = s.commit(snap) + default: mu.Lock() - _ = s.save(claims) // marshal + write both under the lock + snap := s.set(one) mu.Unlock() + _ = s.commit(snap) } } close(done) diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index c13d2418..72f3d744 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -34,7 +34,7 @@ func TestStoreRoundTrip(t *testing.T) { func TestClaimStoreSnapshotDetachesVolumes(t *testing.T) { s := newClaimStore(t.TempDir()) sb := &types.Sandbox{ID: "sb_a", Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}} - snap := s.snapshot(map[string]*types.Sandbox{sb.ID: sb}) + snap := s.set(sb) sb.Volumes[0].Mount = "/mutated" if err := s.commit(snap); err != nil { t.Fatalf("commit: %v", err) @@ -73,20 +73,14 @@ func TestStoreLoadCorruptFile(t *testing.T) { } } -// TestClaimStoreCommitCoalesces asserts an older snapshot never overwrites a -// newer one already on disk — the guard that lets the write leave the manager -// mutex without losing the latest state. func TestClaimStoreCommitCoalesces(t *testing.T) { s := newClaimStore(t.TempDir()) - one := map[string]*types.Sandbox{"sb_a": {ID: "sb_a"}} - two := map[string]*types.Sandbox{"sb_a": {ID: "sb_a"}, "sb_b": {ID: "sb_b"}} - - older := s.snapshot(one) // seq 1 - newer := s.snapshot(two) // seq 2 + older := s.set(&types.Sandbox{ID: "sb_a"}) + newer := s.set(&types.Sandbox{ID: "sb_b"}) if err := s.commit(newer); err != nil { t.Fatalf("commit newer: %v", err) } - if err := s.commit(older); err != nil { // must be a no-op, not an overwrite + if err := s.commit(older); err != nil { t.Fatalf("commit older: %v", err) } if got, err := s.load(); err != nil || len(got) != 2 { @@ -94,14 +88,61 @@ func TestClaimStoreCommitCoalesces(t *testing.T) { } } -// TestClaimStoreConcurrentCommits exercises the writer off the manager mutex: -// many snapshot+commit pairs race, and the file must stay parseable (-race). +func TestClaimStoreIncrementalProjection(t *testing.T) { + s := newClaimStore(t.TempDir()) + a := &types.Sandbox{ID: "sb_a", VMName: "sbx-1", Key: testKey} + b := &types.Sandbox{ID: "sb_b", VMName: "sbx-2", Key: testKey} + if err := s.commit(s.set(a, b)); err != nil { + t.Fatalf("commit set: %v", err) + } + a.HibernateSnap = "snap-a" + if err := s.commit(s.set(a)); err != nil { + t.Fatalf("commit update: %v", err) + } + got, err := s.load() + if err != nil { + t.Fatalf("load: %v", err) + } + if len(got) != 2 || got["sb_a"].HibernateSnap != "snap-a" || got["sb_b"].VMName != "sbx-2" { + t.Fatalf("after incremental update: %+v", got) + } + if delErr := s.commit(s.del("sb_a")); delErr != nil { + t.Fatalf("commit del: %v", delErr) + } + if got, err = s.load(); err != nil || len(got) != 1 || got["sb_b"] == nil { + t.Fatalf("after del: %+v, %v", got, err) + } + if resetErr := s.commit(s.reset(map[string]*types.Sandbox{"sb_c": {ID: "sb_c", Key: testKey}})); resetErr != nil { + t.Fatalf("commit reset: %v", resetErr) + } + if got, err = s.load(); err != nil || len(got) != 1 || got["sb_c"] == nil { + t.Fatalf("after reset: %+v, %v", got, err) + } +} + +func TestClaimStoreMarkRepersists(t *testing.T) { + s := newClaimStore(t.TempDir()) + s.set(&types.Sandbox{ID: "sb_a", Key: testKey}) + if s.synced() { + t.Fatal("synced with an unwritten change") + } + if err := s.commit(s.mark()); err != nil { + t.Fatalf("commit mark: %v", err) + } + if !s.synced() { + t.Error("not synced after flushing the mark") + } + if got, err := s.load(); err != nil || len(got) != 1 { + t.Errorf("after mark flush: %+v, %v", got, err) + } +} + func TestClaimStoreConcurrentCommits(t *testing.T) { s := newClaimStore(t.TempDir()) - claims := map[string]*types.Sandbox{"sb_a": {ID: "sb_a"}} + sb := &types.Sandbox{ID: "sb_a"} var wg sync.WaitGroup for range 50 { - wg.Go(func() { _ = s.commit(s.snapshot(claims)) }) + wg.Go(func() { _ = s.commit(s.set(sb)) }) } wg.Wait() if got, err := s.load(); err != nil || len(got) != 1 { diff --git a/sandboxd/pool/drain.go b/sandboxd/pool/drain.go index 5580df10..6b55fce6 100644 --- a/sandboxd/pool/drain.go +++ b/sandboxd/pool/drain.go @@ -2,9 +2,7 @@ package pool import "context" -// Drain cordons the node: claim/fork/branch answer 429, warm VMs are -// destroyed, live claims run to their leases. Not persisted — a restarted -// node serves again. +// Drain cordons the node: claims answer 429 and warm VMs are destroyed. Not persisted. func (m *Manager) Drain(ctx context.Context) { var trim []string m.mu.Lock() diff --git a/sandboxd/pool/egress.go b/sandboxd/pool/egress.go index 706ee4bc..8941d074 100644 --- a/sandboxd/pool/egress.go +++ b/sandboxd/pool/egress.go @@ -20,11 +20,7 @@ import ( var ( nat64Range = netip.MustParsePrefix("64:ff9b::/96") // RFC 6052 NAT64; the embedded v4 is checked instead - // internalRanges is every IANA special-purpose prefix with Globally Reachable = - // false that IsGlobalUnicast/IsPrivate do not already exclude, plus the - // deprecated IPv4-compatible embedding, mirrored from the IPv4/IPv6 - // Special-Purpose registries (snapshot 2026-07-11). 2001::/23 is blocked whole - // (its handful of globally-reachable anycast carve-outs are not egress targets). + // internalRanges lists the IANA special-purpose prefixes IsGlobalUnicast/IsPrivate leave in. internalRanges = []netip.Prefix{ netip.MustParsePrefix("0.0.0.0/8"), // "this network" netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT; some clouds host metadata here @@ -48,9 +44,7 @@ var ( } ) -// egressListener is one sandbox's egress accept point: an http.Server serving -// egress.Proxy over the per-sandbox UDS the VMM connects when the guest dials -// CID2:egressPort. +// egressListener is one sandbox's egress accept point over the per-sandbox UDS. type egressListener struct { srv *http.Server proxy *egress.Proxy @@ -65,9 +59,7 @@ func (e *egressListener) close() { _ = os.Remove(e.path) } -// armEgress locks the egress-lane NIC then binds the proxy: fail-closed (a -// lock error aborts the claim), and no policy still yields a locked NIC with -// no proxy — default-deny, never a free NIC. +// armEgress locks the egress-lane NIC then binds the proxy: fail-closed, never a free NIC. func (m *Manager) armEgress(ctx context.Context, sb *types.Sandbox) error { if err := m.lockEgressNIC(ctx, sb); err != nil { return err @@ -75,8 +67,7 @@ func (m *Manager) armEgress(ctx context.Context, sb *types.Sandbox) error { return m.armEgressProxy(ctx, sb) } -// lockEgressNIC nft-locks the egress-lane NIC and records the tap for unlock; -// the engine lookup is the fallback for claims from pre-tap journals. +// lockEgressNIC nft-locks the egress-lane NIC and records the tap for unlock. func (m *Manager) lockEgressNIC(ctx context.Context, sb *types.Sandbox) error { if !m.lockEgress || sb.Key.Net != types.NetEgress { return nil @@ -111,9 +102,7 @@ func (m *Manager) tapOf(ctx context.Context, vmName string) (string, error) { return "", fmt.Errorf("no tap for %s", vmName) } -// armEgressProxy binds the vsock egress proxy for a claim whose effective policy -// permits something; a no-op otherwise, so no listener means default-deny (the -// egress-lane NIC stays nft-locked, the none lane's proxy dial is refused). +// armEgressProxy binds the egress proxy only when the effective policy permits something. func (m *Manager) armEgressProxy(ctx context.Context, sb *types.Sandbox) error { if !m.guardedEgress || sb.VsockSocket == "" { return nil @@ -140,8 +129,7 @@ func (m *Manager) armEgressProxy(ctx context.Context, sb *types.Sandbox) error { return nil } -// disarmIfReleased tears down a proxy armed on a wake path if Release dropped -// the claim in the arm window (Release skips the Transition lock); reports gone. +// disarmIfReleased tears down a wake-path proxy if Release dropped the claim in the arm window. func (m *Manager) disarmIfReleased(sb *types.Sandbox) bool { m.mu.Lock() live := m.claimed[sb.ID] == sb @@ -152,8 +140,7 @@ func (m *Manager) disarmIfReleased(sb *types.Sandbox) bool { return !live } -// disarmEgress tears down a sandbox's egress proxy listener, and its NIC lock -// only when removed (a failed removal keeps a running guest locked). Idempotent. +// disarmEgress tears down a sandbox's egress listener, and its NIC lock only when removed. func (m *Manager) disarmEgress(id string, removed bool) { if !m.guardedEgress && !m.lockEgress { return @@ -175,17 +162,14 @@ func (m *Manager) disarmEgress(id string, removed bool) { } } -// poolIntercepts reports whether the configured pool for key HTTPS-intercepts, -// so its golden bakes the node CA. +// poolIntercepts reports whether the configured pool for key HTTPS-intercepts. func (m *Manager) poolIntercepts(key types.PoolKey) bool { m.mu.Lock() defer m.mu.Unlock() return m.poolEgress[key].Intercepts() } -// effectivePolicy resolves pool ∩ tenant; root has no tenant layer, and a key -// no pool is configured for (a promoted template, a cold-booted image) has no -// pool layer. +// effectivePolicy resolves pool ∩ tenant; root has no tenant layer, an unpooled key no pool one. func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { m.mu.Lock() defer m.mu.Unlock() @@ -206,9 +190,7 @@ func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { return egress.Compose(*poolPol, *tenantPol), true } -// newEgressDialer builds the proxy's upstream dialer: internal targets are -// blocked (the proxy must not be an SSRF), then exactly the node-named -// prefixes re-admitted. +// newEgressDialer blocks internal targets, then re-admits exactly the node-named prefixes. func newEgressDialer(allow []netip.Prefix) *net.Dialer { return &net.Dialer{Control: func(_, address string, _ syscall.RawConn) error { host, _, err := net.SplitHostPort(address) @@ -224,8 +206,7 @@ func newEgressDialer(allow []netip.Prefix) *net.Dialer { b := ip.As16() ip = netip.AddrFrom4([4]byte{b[12], b[13], b[14], b[15]}) } - // After the NAT64 unwrap (a v4-in-v6 target matches as the v4 it is), - // before the block, so a named prefix wins. + // after the NAT64 unwrap and before the block, so a named prefix wins if slices.ContainsFunc(allow, func(p netip.Prefix) bool { return p.Contains(ip) }) { return nil } @@ -237,8 +218,7 @@ func newEgressDialer(allow []netip.Prefix) *net.Dialer { }} } -// parsePrefixes turns the allow-list into prefixes; config validation already -// rejected malformed entries, so a leftover is dropped rather than failing dials. +// parsePrefixes turns the allow-list into prefixes; config validation already rejected bad ones. func parsePrefixes(cidrs []string) []netip.Prefix { out := make([]netip.Prefix, 0, len(cidrs)) for _, c := range cidrs { diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index dba61fff..b9d6245d 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -35,9 +35,8 @@ func TestEgressProxyInjectsAndGates(t *testing.T) { pol := &egress.Policy{Allow: []egress.Rule{{Host: host, Secret: "gh"}}} m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: pol}) - m.dial = (&net.Dialer{}).DialContext // the test origin is on loopback, which the SSRF guard blocks + m.dial = (&net.Dialer{}).DialContext - // A short socket dir: the UDS path + "_2049" must fit the OS sun_path cap. sockDir, err := os.MkdirTemp("/tmp", "eg") if err != nil { t.Fatalf("sockdir: %v", err) @@ -78,8 +77,7 @@ func TestEgressProxyInjectsAndGates(t *testing.T) { func TestArmEgressFailsClosedWhenNICUnlockable(t *testing.T) { m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - // The engine reports no NIC tap, so the nft lock cannot apply; arming must - // fail rather than hand out an egress-lane NIC that bypasses the proxy. + sb := &types.Sandbox{ID: "sb_eg_no_tap", Key: egKey, VMName: "sbx-no-tap-1"} if armErr := m.armEgress(t.Context(), sb); armErr == nil { t.Fatal("armEgress must fail closed when the egress-lane NIC cannot be locked") @@ -87,12 +85,9 @@ func TestArmEgressFailsClosedWhenNICUnlockable(t *testing.T) { } func TestArmEgressLocksEgressLaneWithoutPolicy(t *testing.T) { - // guardedEgress on (one pool carries a policy) plus a policyless egress-lane - // pool: that lane must still lock the NIC (default-deny), so arming a claim - // whose NIC cannot be locked fails rather than handing out a free NIC. m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}}}}, - config.PoolSpec{PoolKey: egKey}, // egress lane, no egress policy + config.PoolSpec{PoolKey: egKey}, ) sb := &types.Sandbox{ID: "sb_eg_np", Key: egKey, VMName: "sbx-np-1"} if armErr := m.armEgress(t.Context(), sb); armErr == nil { @@ -101,8 +96,6 @@ func TestArmEgressLocksEgressLaneWithoutPolicy(t *testing.T) { } func TestEgressLaneDoesNotHibernate(t *testing.T) { - // cocoon resumes a guest before its fresh tap can be re-locked, so an - // egress-lane sandbox must refuse to hibernate rather than wake unlocked. m := newTestManager(t, newFakeEngine()) sb := &types.Sandbox{ID: "sb_eg_h", Key: types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall}} sb.Transition.Lock() @@ -114,7 +107,6 @@ func TestEgressLaneDoesNotHibernate(t *testing.T) { } func TestEgressLaneWakeFailsClosed(t *testing.T) { - // A hibernated/archived egress claim (corrupt state) must fail closed on wake. m := newTestManager(t, newFakeEngine()) cases := []struct { name string @@ -147,7 +139,7 @@ func TestEffectivePolicyComposition(t *testing.T) { }{ {"root takes the pool policy whole", "", true, both, nil, "a.test", "z.test", true}, {"root without a pool policy", "", true, nil, nil, "", "", false}, - {"tenant intersects", "acme", true, both, tenantOnly, "b.test", "a.test", true}, // a.test allowed by pool, denied by tenant + {"tenant intersects", "acme", true, both, tenantOnly, "b.test", "a.test", true}, {"tenant declaring no policy", "acme", true, both, nil, "", "", false}, {"tenant on a policyless pool", "acme", true, nil, tenantOnly, "", "", false}, {"tenant on a promoted template", "acme", false, nil, tenantOnly, "b.test", "a.test", true}, @@ -158,7 +150,7 @@ func TestEffectivePolicyComposition(t *testing.T) { t.Run(tc.name, func(t *testing.T) { m.pools = map[types.PoolKey]*pool{} if tc.pooled { - m.pools[testKey] = &pool{key: testKey} + m.pools[testKey] = newPool(testKey) } m.poolEgress = map[types.PoolKey]*egress.Policy{} if tc.pool != nil { @@ -204,7 +196,7 @@ func TestLockUsesProvisionedTapWithoutList(t *testing.T) { } switch { case lockErr != nil && !strings.Contains(lockErr.Error(), "tap-fake0"): - t.Errorf("lock failed outside nft: %v", lockErr) // nft itself may fail off-linux/unprivileged + t.Errorf("lock failed outside nft: %v", lockErr) case lockErr == nil: m.mu.Lock() got := m.egressTaps[sb.ID] @@ -222,7 +214,7 @@ func TestLockFallsBackToListForPreTapClaims(t *testing.T) { if _, err := eng.RunCold(t.Context(), "sbx-old", egKey); err != nil { t.Fatalf("seed vm: %v", err) } - // A pre-tap-journal claim has no TAP; the lock resolves it through the engine. + sb := &types.Sandbox{ID: "sb_old", Key: egKey, VMName: "sbx-old"} lockErr := m.lockEgressNIC(t.Context(), sb) if calls := eng.listCalls(); calls != 1 { @@ -236,8 +228,7 @@ func TestLockFallsBackToListForPreTapClaims(t *testing.T) { func TestBatchArmFailureRecordsNoUsage(t *testing.T) { eng := newFakeEngine() m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - // Second member (egress, no VM) fails to arm; the rolled-back batch must - // leave no claim event, else it stays open forever for the collector. + sbs := []*types.Sandbox{ {VMName: "sbx-ok", Key: testKey}, {VMName: "sbx-eg", Key: egKey}, @@ -264,11 +255,11 @@ func TestBatchArmFailureRecordsNoUsage(t *testing.T) { func TestRollbackKeepsLockWhenRemoveFails(t *testing.T) { eng := newFakeEngine() eng.removeErrFor = "sbx-r1" - // Both VMs are running; sbx-r1's remove fails and the engine keeps listing it. + eng.vms["sbx-r1"] = "/run/sbx-r1.sock" eng.vms["sbx-r2"] = "/run/sbx-r2.sock" m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) - // Both members armed before a later batch member failed; sbx-r1's remove fails. + sbs := []*types.Sandbox{ {ID: "sb_r1", VMName: "sbx-r1", Key: egKey}, {ID: "sb_r2", VMName: "sbx-r2", Key: egKey}, @@ -300,7 +291,7 @@ func TestRollbackKeepsLockWhenRemoveFails(t *testing.T) { func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { eng := newFakeEngine() eng.removeErrFor = "sbx-q1" - // Running: removeVM reports "still here" only while the engine lists it. + eng.vms["sbx-q1"] = "/run/sbx-q1.sock" m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) sb := &types.Sandbox{ID: "sb_q1", VMName: "sbx-q1", Key: egKey, TAP: "tap-q1"} @@ -328,11 +319,9 @@ func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { } func TestEgressLaneLocksWithNoPolicyAnywhere(t *testing.T) { - // A bridge egress lane with zero policy on the node: guardedEgress is false, - // but the NIC must still be nft-locked (default-deny), never handed out free. eng := newFakeEngine() eng.tap = "tap-nopol" - m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey}) // egress lane, no policy + m := egressManager(t, eng, config.PoolSpec{PoolKey: egKey}) if m.guardedEgress { t.Fatal("no policy configured; guardedEgress should be false") } @@ -342,8 +331,6 @@ func TestEgressLaneLocksWithNoPolicyAnywhere(t *testing.T) { attempted := m.egressTaps[sb.ID] == "tap-nopol" m.mu.Unlock() if lockErr != nil { - // nft is linux-only; off Linux the lock fails but must name the tap it - // tried, proving lockEgressNIC ran rather than short-circuiting as before. attempted = strings.Contains(lockErr.Error(), "tap-nopol") } if !attempted { @@ -352,8 +339,6 @@ func TestEgressLaneLocksWithNoPolicyAnywhere(t *testing.T) { } func TestDisarmKeepsLockWhenRemoveFailed(t *testing.T) { - // A failed VM removal must tear down the proxy listener (stop credential - // injection for the dropped claim) but keep the NIC lock (the guest runs on). m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Egress: egPolicy}) m.dial = (&net.Dialer{}).DialContext sockDir, err := os.MkdirTemp("/tmp", "eg") @@ -366,7 +351,7 @@ func TestDisarmKeepsLockWhenRemoveFailed(t *testing.T) { t.Fatalf("arm proxy: %v", armErr) } m.mu.Lock() - m.egressTaps[sb.ID] = "tap-dz" // as lockEgressNIC would record at claim + m.egressTaps[sb.ID] = "tap-dz" m.mu.Unlock() path := engine.EgressSocketPath(sb.VsockSocket) @@ -383,11 +368,9 @@ func TestDisarmKeepsLockWhenRemoveFailed(t *testing.T) { } func TestSetPoolsPreservesEgressPolicy(t *testing.T) { - // The pool API cannot carry egress (config-only), so no SetPools call — warm - // change or drain-then-re-add — may drop a pool's policy. m := egressManager(t, newFakeEngine(), config.PoolSpec{PoolKey: egKey, Egress: egPolicy}) gd := filepath.Join(m.goldensDir(), egKey.Hash()) - if err := os.MkdirAll(gd, 0o750); err != nil { // on disk so re-add adopts it, sparing an async build + if err := os.MkdirAll(gd, 0o750); err != nil { t.Fatalf("golden dir: %v", err) } m.mu.Lock() @@ -403,7 +386,7 @@ func TestSetPoolsPreservesEgressPolicy(t *testing.T) { if !policyLive() { t.Fatal("a warm change wiped the pool egress policy") } - if err := m.SetPools(t.Context(), nil); err != nil { // drain: the pool object is deleted + if err := m.SetPools(t.Context(), nil); err != nil { t.Fatalf("SetPools drain: %v", err) } if err := m.SetPools(t.Context(), []config.PoolSpec{{PoolKey: egKey}}); err != nil { @@ -416,31 +399,30 @@ func TestSetPoolsPreservesEgressPolicy(t *testing.T) { func TestEgressDialerBlocksInternal(t *testing.T) { blocked := []string{ - // Caught by IsGlobalUnicast/IsPrivate/IsLinkLocal. "127.0.0.1:80", "169.254.169.254:80", "10.0.0.5:443", "192.168.1.1:22", "172.16.0.1:80", "255.255.255.255:80", "[::1]:80", "[::ffff:127.0.0.1]:80", "[fe80::1]:80", "[fc00::1]:80", "[ff02::1]:80", - // IANA IPv4 special-purpose, Globally Reachable = false (one per table prefix). + "0.6.6.6:80", "100.100.100.200:80", "192.0.0.1:80", "192.0.2.1:80", "192.88.99.2:80", "198.18.0.1:80", "198.19.255.255:80", "198.51.100.1:80", "203.0.113.1:80", "240.1.2.3:80", - // IANA IPv6 special-purpose + deprecated v4-in-v6 embeddings. + "[100::1]:80", "[100:0:0:1::1]:80", "[2001:db8::1]:80", "[3fff::1]:80", "[5f00::1]:80", "[::127.0.0.1]:80", - "[2001::a9fe:a9fe]:80", // Teredo, within 2001::/23 - "[2001:2::1]:80", // benchmarking, within 2001::/23 - "[2001:10::1]:80", // ORCHID, within 2001::/23 - "[64:ff9b:1::8.8.8.8]:80", // RFC 8215 local-use NAT64 - "[2002:a9fe:a9fe::]:80", // 6to4-embedded 169.254.169.254 - "[64:ff9b::a9fe:a9fe]:80", // NAT64-embedded 169.254.169.254 - "[64:ff9b::a00:1]:80", // NAT64-embedded 10.0.0.1 + "[2001::a9fe:a9fe]:80", + "[2001:2::1]:80", + "[2001:10::1]:80", + "[64:ff9b:1::8.8.8.8]:80", + "[2002:a9fe:a9fe::]:80", + "[64:ff9b::a9fe:a9fe]:80", + "[64:ff9b::a00:1]:80", } for _, addr := range blocked { if err := newEgressDialer(nil).Control("tcp", addr, nil); err == nil { t.Errorf("dial to internal %s allowed; SSRF not blocked", addr) } } - // public v4 direct, public v6, and a public v4 via the NAT64 well-known prefix + for _, addr := range []string{"93.184.216.34:443", "[2606:4700:4700::1111]:443", "[64:ff9b::5db8:d822]:443"} { if err := newEgressDialer(nil).Control("tcp", addr, nil); err != nil { t.Errorf("dial to public %s blocked: %v", addr, err) @@ -465,9 +447,6 @@ func TestEgressLaneCannotForkOrCheckpoint(t *testing.T) { } } -// TestEgressDialerAdmitsOnlyNamedInternalPrefixes pins the SSRF boundary: -// named prefixes re-admit corporate services without re-admitting the guest -// bridges, which are themselves ULA/RFC1918. func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { d := newEgressDialer(parsePrefixes([]string{"fdc8::/16", "10.8.0.0/16"})) check := func(addr string) error { @@ -481,7 +460,7 @@ func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { "public v6": {"[2606:2800:220:1:248:1893:25c8:1946]:443", true}, "corporate v6": {"[fdc8:17:9:200f::1]:443", true}, "corporate v4": {"10.8.7.149:443", true}, - // Guests and the host gateway share fc00::/7 with the corporate range. + "another sandbox on the bridge": {"[fd00:c0c0:38::5]:443", false}, "the host's bridge gateway": {"[fd00:c0c0:38::1]:443", false}, "other private v4": {"192.168.1.1:443", false}, @@ -501,18 +480,15 @@ func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { } } -// TestEgressDialerWildcardAllowsEverything pins the fully-open form as a -// deliberate choice — loopback and cloud metadata included — for fleets with -// a policy-enforcing forward proxy in front. func TestEgressDialerWildcardAllowsEverything(t *testing.T) { d := newEgressDialer(parsePrefixes([]string{"0.0.0.0/0", "::/0"})) for _, addr := range []string{ - "93.184.216.34:443", // public - "[fdc8:17:9:200f::1]:443", // corporate v6 - "10.8.7.149:443", // corporate v4 - "[fd00:c0c0:38::5]:443", // another sandbox - "127.0.0.1:7777", // the node itself - "169.254.169.254:80", // cloud metadata + "93.184.216.34:443", + "[fdc8:17:9:200f::1]:443", + "10.8.7.149:443", + "[fd00:c0c0:38::5]:443", + "127.0.0.1:7777", + "169.254.169.254:80", } { if err := d.Control("tcp", addr, nil); err != nil { t.Errorf("%s blocked under a wildcard allow-list: %v", addr, err) @@ -520,9 +496,6 @@ func TestEgressDialerWildcardAllowsEverything(t *testing.T) { } } -// egressClient builds an HTTP client that reaches the origin through the -// per-sandbox egress proxy UDS, exactly as the guest's proxy client would over -// vsock. Playing the VMM's role lets the host-side path run without a real VM. func egressClient(path string) *http.Client { return &http.Client{ Timeout: 5 * time.Second, @@ -547,8 +520,6 @@ func egressManager(t *testing.T, eng *fakeEngine, pools ...config.PoolSpec) *Man return m } -// writeTestEgressCA provisions a cluster root + this node's intermediate to temp -// files, as `sandboxd ca` would, and returns the config pointing at them. func writeTestEgressCA(t *testing.T) *config.EgressCAConfig { t.Helper() rootCert, rootKey, err := egress.GenerateRoot("test cluster ca") diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index d7613620..06462f42 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -10,12 +10,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Fork clones a claimed sandbox into count children, each a fresh claim with -// its own lease: memory, disk, and guest state (sessions, processes, tmpfs) -// duplicate at the snapshot point, and cocoon's clone reseed gives every -// child a distinct machine identity. Children inherit the parent's tenant -// and count against its quota. All-or-nothing: any child failing destroys -// the ones already built, so an error means no child survived. +// Fork clones a claimed sandbox into count children, each a fresh claim; all-or-nothing. func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl time.Duration) ([]*types.Sandbox, error) { sb, ok := m.resolve(id, cred) if !ok { @@ -56,9 +51,7 @@ func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl return children, nil } -// forkClones builds count children from a claimed source: a running source is -// cloned straight from a fresh store snapshot (no export copy); a hibernated -// source stays on the export path (its shared wake image needs a private copy). +// forkClones clones a running source from a fresh snapshot, a hibernated one from an export. func (m *Manager) forkClones(ctx context.Context, sb *types.Sandbox, count int) ([]*types.Sandbox, error) { create, cleanup, err := m.forkSource(ctx, sb) if err != nil { @@ -68,9 +61,7 @@ func (m *Manager) forkClones(ctx context.Context, sb *types.Sandbox, count int) return m.cloneBatch(ctx, count, sb.Key, create) } -// forkSource decides and captures the fork source under the transition lock -// (a racing hibernate cannot swap the snapshot out from under the fan-out), -// returning the per-child provisioner and its cleanup. +// forkSource captures the fork source under the transition lock, against a racing hibernate. func (m *Manager) forkSource(ctx context.Context, sb *types.Sandbox) (vmProvisioner, func(), error) { sb.Transition.Lock() defer sb.Transition.Unlock() diff --git a/sandboxd/pool/fork_test.go b/sandboxd/pool/fork_test.go index 0b09e194..ef97e87e 100644 --- a/sandboxd/pool/fork_test.go +++ b/sandboxd/pool/fork_test.go @@ -10,14 +10,6 @@ import ( "time" ) -// TestForkRacingHibernateUsesPrivateSource pins the transition-locked source -// decision: a hibernate landing between Fork's entry and its capture must -// never let the fan-out clone the shared wake snapshot. -// -// Not a synctest candidate: Fork's block on sb.Transition (a plain -// sync.Mutex) is never "durably blocked" by synctest's rules, so the bubble -// can't fast-forward past the 20ms sleep that lets Fork reach it — verified -// as a real deadlock (times out at 120s), not a flake. func TestForkRacingHibernateUsesPrivateSource(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -46,7 +38,7 @@ func TestForkRacingHibernateUsesPrivateSource(t *testing.T) { children = append(children, sb.ID) } }) - time.Sleep(20 * time.Millisecond) // let Fork reach the transition lock + time.Sleep(20 * time.Millisecond) close(eng.hibernateStall) wg.Wait() @@ -78,7 +70,7 @@ func TestForkFromRunning(t *testing.T) { if len(eng.snapSaves) != 1 || !strings.HasPrefix(eng.snapSaves[0], forkPrefix) { t.Errorf("snapSaves %v, want one %s* snapshot", eng.snapSaves, forkPrefix) } - // Fast path: clone the children straight from the store snapshot, no export. + if len(eng.exports) != 0 { t.Errorf("exported %v, want none (fork clones from the snapshot directly)", eng.exports) } @@ -135,7 +127,7 @@ func TestForkFromHibernatedUsesWakeImage(t *testing.T) { if slices.Contains(eng.snapRemoves, hibSnap) { t.Error("hibernate snapshot dropped by fork — the parent could never wake") } - // The parent stays hibernated and must still wake. + if _, err := m.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after fork: %v", err) } @@ -167,17 +159,16 @@ func TestForkUnknownSandbox(t *testing.T) { func TestForkAllOrNothing(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - parent, err := claimAny(t.Context(), m, testKey, 0) // cold boot, no Clone calls yet + parent, err := claimAny(t.Context(), m, testKey, 0) if err != nil { t.Fatalf("Claim: %v", err) } - eng.cloneFailNth = 2 // fail one of the three fork clones + eng.cloneFailNth = 2 if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 3, 0); err == nil { t.Fatal("Fork succeeded despite a failed clone") } - // Every successfully built child is destroyed; only the parent's claim - // survives in memory and on disk. + if _, g := m.Info(); g.Claimed != 1 { t.Errorf("claimed=%d, want only the parent", g.Claimed) } @@ -206,11 +197,10 @@ func TestReconcileSweepsOrphanSnapshots(t *testing.T) { hibernatePrefix + "orphan-1", forkPrefix + "stale-2", goldenPrefix + "stale-3", - "user-snapshot", // not ours: never touched + "user-snapshot", } eng.mu.Unlock() - // A fresh manager over the same journal models the post-crash restart. m2 := newTestManagerAt(t, eng, dataDir) if err := m2.Reconcile(t.Context()); err != nil { t.Fatalf("Reconcile: %v", err) @@ -225,7 +215,7 @@ func TestReconcileSweepsOrphanSnapshots(t *testing.T) { t.Errorf("snapshot %q swept but must be kept", keep) } } - // The adopted hibernated claim still wakes after the restart. + if _, err := m2.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after reconcile: %v", err) } @@ -249,7 +239,7 @@ func TestForkChildrenInheritTenantAndQuota(t *testing.T) { t.Errorf("child %d tenant %q, want acme", i, c.Tenant) } } - // Parent plus two children fill acme's cap of 3; one more child is over. + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 1, 0); !errors.Is(err, ErrQuota) { t.Errorf("fork past the tenant cap: %v, want ErrQuota", err) } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 02c470ee..62d7fa80 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -13,9 +13,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Hibernate atomically snapshots a claimed sandbox and stops its VM, freeing -// memory; the next agent access wakes it. Idempotent on an already-hibernated -// sandbox. func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { sb, ok := m.resolve(id, cred) if !ok { @@ -26,9 +23,6 @@ func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { return m.hibernateLocked(ctx, sb) } -// WakeAgentSocket resolves the sandbox's vsock UDS for the relay, first -// restoring the VM if it is hibernated; concurrent wakes queue on the -// transition lock and find the fast path. func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string, error) { sb, ok := m.claim(id, token) if !ok { @@ -44,12 +38,10 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return ErrVolumeCapture } if sb.Key.Net == types.NetEgress { - // cocoon resumes the guest before its fresh tap can be re-locked, so a - // woken egress guest would egress unlocked; keep the lane live instead. + // cocoon resumes the guest before its fresh tap can be re-locked return ErrNoEgressHibernate } - // An adopted hibernate was never billed (its confirming list had failed), - // so record it even if this resolve's own persist is still converging. + // an adopted hibernate was never billed, so record it here adopted, resolveErr := m.resolvePendingSnap(ctx, sb) if adopted { m.recordHibernate(ctx, sb) @@ -64,24 +56,20 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error } return nil } - // A started transition must finish even if the caller hangs up (the - // engine bounds every step), or the record would disagree with the VM. + // a started transition must finish, or the record disagrees with the VM ctx = context.WithoutCancel(ctx) - // VMName-based (cocoon rejects sb.ID's "sb_" underscore) plus a random - // suffix so the wake's async snapshot drop can't collide with a re-hibernate. + // cocoon rejects sb.ID's underscore, so the snapshot name is VMName-based snap := hibernatePrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-" + randHex(3) - // Journal the intent before the engine stops anything: an intent that - // cannot be written aborts with the VM untouched, and a committed intent - // lets Reconcile adopt a hibernate whose final commit never landed. + // journal the intent first so Reconcile can adopt a hibernate whose commit never landed if err := m.store.commit(m.setPendingSnap(sb, snap)); err != nil { m.mu.Lock() sb.PendingSnap = "" + m.store.set(sb) m.mu.Unlock() return fmt.Errorf("hibernate %s: persist intent: %w", sb.ID, err) } if hibErr := m.eng.Hibernate(ctx, sb.VMName, snap); hibErr != nil { - // The engine can report failure after the snapshot landed (a CLI - // timeout); resolve the intent against the real snapshot. + // the engine can report failure after the snapshot landed adopted, resolveErr := m.resolvePendingSnap(ctx, sb) if adopted { m.recordHibernate(ctx, sb) @@ -119,8 +107,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, if sb.ArchiveCk != "" { return m.wakeArchived(ctx, sb) } - // Settle a dangling intent before choosing running vs hibernated; bill an - // adopted hibernate before the wake closes its window. + // settle a dangling intent before choosing running vs hibernated adopted, err := m.resolvePendingSnap(ctx, sb) if adopted { m.recordHibernate(ctx, sb) @@ -129,15 +116,13 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, return "", fmt.Errorf("wake %s: %w", sb.ID, err) } if sb.HibernateSnap == "" { - // No journal sync here — data-plane fast path; a lagging journal only - // says hibernated while the VM runs, which restart Reconcile heals. + // no journal sync on this data-plane fast path; restart Reconcile heals the lag if sb.StaleSnap != "" && m.store.synced() { m.dropStale(ctx, sb) } return sb.VsockSocket, nil } - // The egress lane never hibernates (its fresh tap can't be locked before the - // guest resumes); a hibernated one is corrupt state — fail closed. + // the egress lane never hibernates, so a hibernated one is corrupt state if sb.Key.Net == types.NetEgress { return "", fmt.Errorf("wake %s: egress lane cannot resume from hibernation", sb.ID) } @@ -147,9 +132,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, snap := sb.HibernateSnap restoredSock, err := m.eng.Restore(ctx, sb.VMName, snap) if err != nil { - // Restore may have booted the VM before the engine errored (a CLI - // timeout); tear it down, keeping the snapshot, so the next wake - // restores cleanly instead of re-restoring an already-running VM. + // restore may have booted the VM before the engine errored m.destroy(ctx, sb.VMName) return "", fmt.Errorf("wake %s: %w", sb.ID, err) } @@ -174,21 +157,17 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, return "", ErrUnknownSandbox } if err != nil { - // The lagging journal still references the snapshot; park it for - // reclaim after a later write lands, not just the restart sweep. + // the lagging journal still references the snapshot, so park it for later reclaim sb.StaleSnap = snap return "", fmt.Errorf("wake %s: persist claims: %w", sb.ID, err) } m.dropStale(ctx, sb) - // The resume consumed the memory image; reclaim its disk off the wake- - // return path (name reuse guarded by randHex, failed drops by the sweep). + // the resume consumed the memory image; reclaim its disk off the wake-return path go m.dropSnap(ctx, snap) return sock, nil } -// idleOnce hibernates claims idle past their pool's (or the node's) -// threshold. Best-effort: a connection racing the sweep may see its sandbox -// hibernate right after — the next call wakes it transparently. +// idleOnce hibernates claims idle past their pool's (or the node's) threshold. func (m *Manager) idleOnce(ctx context.Context) { if !m.idleEnabled { return @@ -215,8 +194,7 @@ func (m *Manager) idleOnce(ctx context.Context) { m.idleSweep.Store(false) return } - // Hibernates are seconds-long engine snapshots: fan out off the - // housekeeping loop on the refill budget. + // hibernates are seconds-long engine snapshots, so fan out off the housekeeping loop go func() { defer m.idleSweep.Store(false) logger := log.WithFunc("pool.idleOnce") @@ -227,9 +205,7 @@ func (m *Manager) idleOnce(ctx context.Context) { }() } -// idleHibernate re-validates a sweep victim under the Transition lock: a -// data-plane connection that arrived after the sweep's snapshot refreshes -// LastActivity, and hibernating underneath it would cut a live call. +// idleHibernate re-validates a sweep victim under the Transition lock. func (m *Manager) idleHibernate(ctx context.Context, id, token string, sweepStart time.Time) error { sb, ok := m.claim(id, token) if !ok { @@ -246,10 +222,7 @@ func (m *Manager) idleHibernate(ctx context.Context, id, token string, sweepStar return m.hibernateLocked(ctx, sb) } -// commitTransition publishes a hibernate/wake result and persists the journal -// only if the claim is still live — Release/reap skip the transition lock, and -// publishing after them resurrects state nobody owns. Returns a failed write -// (caller must not report success); recommit converges disk in the background. +// commitTransition publishes a hibernate/wake result only if the claim is still live. func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, sock string) (live bool, err error) { m.mu.Lock() live = m.claimed[sb.ID] == sb @@ -258,7 +231,7 @@ func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, sb.HibernateSnap = snap sb.VsockSocket = sock sb.PendingSnap = "" - js = m.store.snapshot(m.claimed) + js = m.store.set(sb) } m.mu.Unlock() if !live { @@ -271,9 +244,7 @@ func (m *Manager) commitTransition(ctx context.Context, sb *types.Sandbox, snap, return true, nil } -// syncClaims backs the idempotent hibernate fast path: a retry after a -// failed persist must not answer success while the journal still lags the -// transition it reports. The caller holds sb.Transition. +// syncClaims flushes a lagging journal so a hibernate retry cannot report a false success. func (m *Manager) syncClaims(ctx context.Context, sb *types.Sandbox) error { if !m.store.synced() { if err := m.store.commit(m.claimsSnapshot()); err != nil { @@ -295,13 +266,10 @@ func (m *Manager) setPendingSnap(sb *types.Sandbox, snap string) claimSnapshot { m.mu.Lock() defer m.mu.Unlock() sb.PendingSnap = snap - return m.store.snapshot(m.claimed) + return m.store.set(sb) } -// resolvePendingSnap settles a hibernate intent whose engine result was never -// confirmed: the snapshot's presence decides whether to adopt it as -// HibernateSnap or clear the intent. An unusable snapshot list keeps the -// intent and errors. The caller holds sb.Transition. +// resolvePendingSnap settles an unconfirmed hibernate intent against the engine's snapshot list. func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (adopted bool, err error) { if sb.PendingSnap == "" { return false, nil @@ -314,17 +282,17 @@ func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (ad live := m.claimed[sb.ID] == sb pending := sb.PendingSnap exists := slices.Contains(snaps, pending) + var js claimSnapshot if live { if exists { sb.HibernateSnap = pending } sb.PendingSnap = "" + js = m.store.set(sb) } - js := m.store.snapshot(m.claimed) m.mu.Unlock() if !live { - // Released mid-resolve: Release captured an empty HibernateSnap, so a - // completed hibernate's snapshot is an orphan it could not drop. + // Release captured an empty HibernateSnap, so a completed hibernate's snapshot is an orphan if exists { m.dropSnap(ctx, pending) } @@ -332,8 +300,7 @@ func (m *Manager) resolvePendingSnap(ctx context.Context, sb *types.Sandbox) (ad } if err := m.store.commit(js); err != nil { m.recommit(ctx, js) - // The transition is decided in memory: report the adoption so the - // caller bills it, surfacing the persist error — commitTransition contract. + // the transition is decided in memory, so report the adoption for the caller to bill return exists, fmt.Errorf("resolve hibernate intent %s: persist: %w", sb.ID, err) } return exists, nil @@ -344,9 +311,7 @@ func (m *Manager) recordHibernate(ctx context.Context, sb *types.Sandbox) { m.recordUsage(ctx, usageEvent{Event: "hibernate", ID: sb.ID, VMName: sb.VMName}) } -// skipIdle reports the claims an idle sweep must leave alone: the egress lane -// cannot resume, a mounted volume cannot be captured, and an already -// hibernated or archived claim has nothing left to do. +// skipIdle reports the claims an idle sweep must leave alone. func skipIdle(sb *types.Sandbox, idle time.Duration, now time.Time) bool { return idle <= 0 || sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index bfa27520..ff92f83e 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -13,11 +13,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// TestHibernatePersistFailureSurfacesAndConverges guards the durability fix -// where a hibernate whose journal write failed still reported success — a -// restart then dropped the claim and destroyed the VM and snapshot. With the -// intent write first, an unwritable journal aborts before the engine stops -// anything; a retry after the store heals runs the full transition. func TestHibernatePersistFailureSurfacesAndConverges(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -46,10 +41,6 @@ func TestHibernatePersistFailureSurfacesAndConverges(t *testing.T) { } } -// TestWakePersistFailureSurfaces is the wake side of the same fix: retries -// must surface it, the snapshot the lagging journal references must survive -// (a fast-path retry keeps serving the awake VM — a restart adopts running -// VMs, so the lag is harmless there), and it is reclaimed once a write lands. func TestWakePersistFailureSurfaces(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -77,8 +68,7 @@ func TestWakePersistFailureSurfaces(t *testing.T) { if err := os.MkdirAll(filepath.Dir(broken), 0o750); err != nil { t.Fatalf("heal store dir: %v", err) } - // The background recommit converges the journal; the next fast-path wake - // then reclaims the parked snapshot. + waitFor(t, func() bool { if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("fast-path wake: %v", err) @@ -92,16 +82,12 @@ func TestWakePersistFailureSurfaces(t *testing.T) { }) } -// TestHibernateAmbiguousErrorTreatedAsDone: the engine can report failure -// after the snapshot landed (a CLI timeout). A snapshot the list confirms -// present means the hibernate happened — the call must not clear the intent -// or fail, or a restart would destroy a genuinely hibernated claim. func TestHibernateAmbiguousErrorTreatedAsDone(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) eng.hibernateErr = errors.New("cli timeout") - eng.hibernateErrCompletes = true // the snapshot landed before the timeout + eng.hibernateErrCompletes = true if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v, want nil (the snapshot landed despite the error)", err) @@ -115,14 +101,11 @@ func TestHibernateAmbiguousErrorTreatedAsDone(t *testing.T) { } } -// TestHibernateVerifiedFailureClearsIntent: only a snapshot the list confirms -// absent proves the hibernate did not happen — then the intent clears, the -// claim stays running, and a retry hibernates cleanly. func TestHibernateVerifiedFailureClearsIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - eng.hibernateErr = errors.New("cli failed") // completes=false: no snapshot + eng.hibernateErr = errors.New("cli failed") if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success despite a verified failure") @@ -140,10 +123,6 @@ func TestHibernateVerifiedFailureClearsIntent(t *testing.T) { } } -// TestHibernateUnverifiableErrorKeepsIntent: when the engine errors and the -// snapshot list is also unusable (the same cocoon fault), the outcome is -// unknown — the call must surface the error and keep the intent for Reconcile, -// never report success on an unconfirmed transition. func TestHibernateUnverifiableErrorKeepsIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -163,10 +142,6 @@ func TestHibernateUnverifiableErrorKeepsIntent(t *testing.T) { } } -// TestWakeRestoreErrorDestroysAndRetries: Restore can boot the VM before the -// engine errors (a CLI timeout); the failed wake must tear it down (keeping -// the snapshot) so the next wake restores cleanly instead of re-restoring an -// already-running VM. func TestWakeRestoreErrorDestroysAndRetries(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -196,15 +171,11 @@ func TestWakeRestoreErrorDestroysAndRetries(t *testing.T) { } } -// TestRetryResolvesDanglingIntent: a hibernate that actually completed but -// left a dangling intent (CLI and list both timed out) must be resolved by a -// later retry against the real snapshot — adopted, not overwritten with a -// fresh name that would strand the real snapshot and drop the claim on restart. func TestRetryResolvesDanglingIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - // First hibernate completes, but the CLI and the list both time out. + eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") @@ -216,7 +187,6 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { t.Fatalf("intent %q, want the real snapshot %q kept", got[sb.ID].PendingSnap, realSnap) } - // cocoon recovers; the retry must adopt the real snapshot, not re-hibernate. eng.hibernateErr = nil eng.snapListErr = nil if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { @@ -240,9 +210,6 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { } } -// TestWakeResolvesDanglingIntent: a Wake facing a dangling intent whose -// snapshot exists must resolve it and restore, not return the stopped VM's -// stale socket as if it were live. func TestWakeResolvesDanglingIntent(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -271,16 +238,12 @@ func TestWakeResolvesDanglingIntent(t *testing.T) { } } -// TestResolvePendingSnapReleaseRaceDropsOrphan: a Release landing while a wake -// is mid-resolve (Release captured an empty HibernateSnap, so it drops -// nothing) must not strand the intent's real snapshot — resolvePendingSnap -// sees the claim gone and drops the orphan itself. func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - // Drive to a dangling intent whose snapshot really exists. + eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") @@ -289,7 +252,6 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { } realSnap := eng.hibernates[0] - // A wake resolves, but stalls inside SnapshotList; Release lands first. eng.snapListErr = nil eng.snapListStall = make(chan struct{}) callsBefore := eng.snapListCalls() @@ -311,9 +273,6 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { }) } -// TestResolveAdoptBillsWhenPersistFails: a real transition must be billed even -// when the adopting resolve's own journal write fails — memory is authoritative -// and recommit converges disk, so it must not silently escape metering. func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -325,7 +284,7 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } - // The list recovers, but the claims journal is now unwritable. + eng.snapListErr = nil broken := filepath.Join(t.TempDir(), "gone", "claims.json") m.store.path = broken @@ -335,7 +294,7 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { if n := m.Counters().Hibernates; n != 1 { t.Errorf("hibernates billed=%d, want 1 (billed even while the persist lags)", n) } - // Heal so the background recommit converges (and its goroutine exits). + if err := os.MkdirAll(filepath.Dir(broken), 0o750); err != nil { t.Fatalf("heal store dir: %v", err) } @@ -346,9 +305,6 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { }) } -// TestWakeProbeFailureDestroysAndRetries: a readiness-probe timeout after a -// successful Restore must tear the booted VM down (keeping the snapshot) so -// the next wake restores cleanly, instead of re-restoring a running VM. func TestWakeProbeFailureDestroysAndRetries(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -378,8 +334,6 @@ func TestWakeProbeFailureDestroysAndRetries(t *testing.T) { } } -// TestFlushClaimsClosesShutdownWindow: the shutdown hook persists what the -// background recommit has not converged yet (here, a wake whose commit failed). func TestFlushClaimsClosesShutdownWindow(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -441,12 +395,11 @@ func TestHibernateWakeCycle(t *testing.T) { if len(eng.restores) != 1 { t.Errorf("engine restored %d times, want 1", len(eng.restores)) } - waitFor(t, func() bool { return eng.snapRemoved(hibSnap) }) // wake drops it off the return path + waitFor(t, func() bool { return eng.snapRemoved(hibSnap) }) if _, g := m.Info(); g.Hibernated != 0 { t.Errorf("hibernated count %d after wake, want 0", g.Hibernated) } - // Awake sandbox: the fast path must not touch the engine again. if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("fast-path WakeAgentSocket: %v", err) } @@ -456,8 +409,6 @@ func TestHibernateWakeCycle(t *testing.T) { }) } -// TestWakeDropsSnapshotOffReturnPath: wake returns the socket without waiting -// for the consumed snapshot's (subprocess) removal. func TestWakeDropsSnapshotOffReturnPath(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -537,8 +488,6 @@ func TestReleaseMidHibernateDropsOrphanSnapshot(t *testing.T) { return len(eng.hibernates) == 1 }) - // Release lands while the engine transition is in flight: the claim is - // gone before Hibernate can commit, so its snapshot must not leak. if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Release: %v", err) } @@ -555,10 +504,6 @@ func TestReleaseMidHibernateDropsOrphanSnapshot(t *testing.T) { }) } -// TestReconcileAdoptsJournaledIntent: a hibernate that stopped the VM but -// whose final commit never landed leaves a journaled intent (PendingSnap) -// over a stopped VM; reconcile adopts it as hibernated — no heuristics, the -// intent names the wake image. func TestReconcileAdoptsJournaledIntent(t *testing.T) { eng := newFakeEngine() eng.vms["sbx-lagged-1"] = "/vsock/lagged" @@ -590,10 +535,6 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { } } -// TestReconcileIgnoresStaleOrphanSnapshot: a leftover wake image without a -// journaled intent must never be mistaken for a hibernate — adopting it -// would silently roll the sandbox back to pre-wake state. The claim drops, -// the orphan sweeps. func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -619,8 +560,6 @@ func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { }) } -// TestReconcileDropsIntentWithoutImage: an intent whose wake image is -// verified missing means the hibernate never completed; the claim drops. func TestReconcileDropsIntentWithoutImage(t *testing.T) { eng := newFakeEngine() eng.vms["sbx-noimg-1"] = "/vsock/noimg" diff --git a/sandboxd/pool/idle_test.go b/sandboxd/pool/idle_test.go index 56c68edf..e12537e1 100644 --- a/sandboxd/pool/idle_test.go +++ b/sandboxd/pool/idle_test.go @@ -24,7 +24,6 @@ func TestIdleOnceHibernatesPastThreshold(t *testing.T) { m.idleOnce(t.Context()) waitFor(t, func() bool { return hibernated(m) == 1 }) - // Idempotent on an already-hibernated claim. m.idleOnce(t.Context()) waitFor(t, func() bool { return !m.idleSweep.Load() }) if hibernated(m) != 1 { @@ -37,7 +36,7 @@ func TestIdleOncePolicyScope(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) - m.idleDefault = time.Second // node default must NOT reach pooled keys + m.idleDefault = time.Second m.idleEnabled = true sb := mustClaim(t, m, testKey) @@ -48,7 +47,6 @@ func TestIdleOncePolicyScope(t *testing.T) { t.Fatal("pooled key without the policy was idle-hibernated by the node default") } - // An unpooled key (template claim shape) takes the node default. unpooled := types.PoolKey{Template: "tpl:v1", Net: types.NetNone, Size: types.SizeSmall} sb2, err := m.ClaimProvision(t.Context(), unpooled, time.Hour, "", "", nil) if err != nil { @@ -67,7 +65,6 @@ func TestActivityStampsBlockIdleSweep(t *testing.T) { sb := mustClaim(t, m, testKey) backdate(m, sb, 2*time.Second) - // A data-plane connection refreshes the stamp; the sweep must spare it. if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("WakeAgentSocket: %v", err) } diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index d210e1f2..4a7655e7 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -122,8 +122,7 @@ func TestInterceptPoolAllowsPromote(t *testing.T) { m.mu.Lock() m.claimed[sb.ID] = sb m.mu.Unlock() - // The cluster root is shared, so an interception sandbox's disk carries no - // node-private material: promote/checkpoint are unrestricted. + if _, _, err := m.Promote(t.Context(), sb.ID, Cred{Token: "tok"}, "tpl:x", ""); err != nil { t.Errorf("Promote of an interception-pool sandbox: %v, want success", err) } diff --git a/sandboxd/pool/journal.go b/sandboxd/pool/journal.go index 96ff9fd0..9f32d159 100644 --- a/sandboxd/pool/journal.go +++ b/sandboxd/pool/journal.go @@ -8,14 +8,10 @@ import ( "time" ) -// journalMaxBytes rotates a journal once it crosses this size; one .1 backup -// is kept so a slow collector never loses a window silently. +// journalMaxBytes rotates a journal once it crosses this size; one .1 backup is kept. const journalMaxBytes = 64 * 1024 * 1024 -// usageEvent is one line of usage.jsonl: the product-level billing stream, -// folded by the platform collector (compute time = claim→release minus -// hibernate→wake; hibernated storage = hibernate→wake). VM names ride along -// so cocoon's machine-level metering ledger can audit the fold. +// usageEvent is one line of usage.jsonl, the billing stream the platform collector folds. type usageEvent struct { Time time.Time `json:"t"` Event string `json:"ev"` // claim|hibernate|wake|fork|promote|checkpoint|release|reap @@ -29,8 +25,7 @@ type usageEvent struct { Reference string `json:"ref,omitempty"` // promote: template; checkpoint: ckpt id } -// journal is an append-only JSONL writer with size rotation. Writes happen -// outside the pool mutex; the journal's own lock only orders appends. +// journal is an append-only JSONL writer with size rotation; its lock only orders appends. type journal struct { mu sync.Mutex path string @@ -74,23 +69,22 @@ func (j *journal) close() error { return j.f.Close() } -// rotate moves the live file to .1 (replacing any prior backup) and reopens. -// A failed swap degrades to an oversized journal: reopen, append, retry next time. +// rotate moves the live file to .1 and reopens; the old descriptor closes last. func (j *journal) rotate() error { - if err := j.f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - return err - } renameErr := os.Rename(j.path, j.path+".1") f, err := os.OpenFile(j.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) //nolint:gosec // under data_dir if err != nil { return errors.Join(renameErr, err) } + old := j.f j.f, j.size = f, 0 if renameErr != nil { if st, statErr := f.Stat(); statErr == nil { j.size = st.Size() } - return renameErr } - return nil + if closeErr := old.Close(); closeErr != nil && !errors.Is(closeErr, os.ErrClosed) { + renameErr = errors.Join(renameErr, closeErr) + } + return renameErr } diff --git a/sandboxd/pool/journal_test.go b/sandboxd/pool/journal_test.go index 433f668b..686b7a40 100644 --- a/sandboxd/pool/journal_test.go +++ b/sandboxd/pool/journal_test.go @@ -22,7 +22,7 @@ func TestJournalRotateFailureDegradesAndRecovers(t *testing.T) { t.Fatalf("new journal: %v", err) } defer func() { _ = j.close() }() - // A directory at the backup path makes the rotation rename fail. + if err = os.Mkdir(path+".1", 0o750); err != nil { t.Fatalf("setup: %v", err) } @@ -53,6 +53,32 @@ func TestJournalRotateFailureDegradesAndRecovers(t *testing.T) { } } +func TestJournalRotateReopenFailureKeepsWriting(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "usage.jsonl") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.Truncate(path, journalMaxBytes); err != nil { + t.Fatalf("setup: %v", err) + } + j, err := newJournal(path) + if err != nil { + t.Fatalf("new journal: %v", err) + } + defer func() { _ = j.close() }() + j.path = filepath.Join(dir, "gone", "usage.jsonl") + if err = j.append(map[string]string{"ev": "a"}); err == nil { + t.Fatal("append across a failed reopen must surface the error") + } + if err = j.append(map[string]string{"ev": "b"}); err == nil { + t.Fatal("append while the reopen stays blocked must keep surfacing the error") + } + if got := tail(t, path); !strings.Contains(got, `"ev":"a"`) || !strings.Contains(got, `"ev":"b"`) { + t.Errorf("events lost to a closed descriptor: %q", got) + } +} + func tail(t *testing.T, path string) string { t.Helper() f, err := os.Open(path) diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index fbc31ae7..4c3c2654 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -6,8 +6,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Cred is a caller's resolved authority over one sandbox: the per-sandbox -// token, or Operator for the root api_token the server already verified. +// Cred is a caller's resolved authority over one sandbox: its token, or Operator for root. type Cred struct { Token string Operator bool @@ -21,13 +20,11 @@ func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { if sb == nil { return SandboxSummary{}, false } - // summarize reads fields hibernate and archive mutate under m.mu, so it - // runs here, not after byID has released the lock. + // summarize reads fields hibernate and archive mutate under m.mu return summarize(sb), true } -// Wake restores a hibernated sandbox and leaves it running: waking is -// otherwise only a side effect of opening an agent connection. Idempotent. +// Wake restores a hibernated sandbox and leaves it running. Idempotent. func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { sb, ok := m.resolve(id, cred) if !ok { @@ -36,8 +33,7 @@ func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { return m.wake(ctx, sb) } -// resolve authorizes id under cred: Operator by id alone, otherwise the token -// must match — an unclaimed slot must never match an empty token. +// resolve authorizes id under cred; an unclaimed slot must never match an empty token. func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { if cred.Operator { return m.byID(id) @@ -48,8 +44,7 @@ func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { return m.claim(id, cred.Token) } -// byID resolves a live claim by id alone; callers must already have authorized -// the Operator credential. +// byID resolves a live claim by id alone; callers must have authorized the Operator credential. func (m *Manager) byID(id string) (*types.Sandbox, bool) { m.mu.Lock() defer m.mu.Unlock() @@ -57,8 +52,7 @@ func (m *Manager) byID(id string) (*types.Sandbox, bool) { return sb, sb != nil } -// wake reuses the relay's resolve path, then discards the socket: the caller -// wants the VM running, not a connection to it. +// wake reuses the relay's resolve path, then discards the socket. func (m *Manager) wake(ctx context.Context, sb *types.Sandbox) error { sb.Touch() _, err := m.wakeResolved(ctx, sb) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 7e9ec929..53985349 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -41,9 +41,7 @@ const ( buildRetryDelay = 30 * time.Second claimProbeTimeout = 15 * time.Second coldProbeTimeout = 90 * time.Second - // A refill has no caller waiting and holds a target-accounting slot for - // its whole probe; a clone answers in ~1s even saturated, so a silent one - // is replaced, not waited on. + // a clone answers in ~1s even saturated, so a silent one is replaced, not waited on warmProbeTimeout = 5 * time.Second // One list; a wrong answer only costs an extra sweep. removeVerifyTimeout = 15 * time.Second @@ -54,15 +52,12 @@ const ( maxTTL = 24 * time.Hour recommitBackoff = 20 * time.Millisecond recommitMaxBackoff = 5 * time.Second - // One failed boot is ordinary; only an unbroken run of failures says the - // next attempt will fail too. + // one failed boot is ordinary; only an unbroken run predicts the next failure refillFailStreak = 8 // Doubles per failure past the streak. refillBackoffBase = 250 * time.Millisecond refillBackoffMax = buildRetryDelay - // defaultMaxFork/defaultRefill are the fallbacks when a Manager is built - // from a Config that skipped config.Load's defaulting (direct construction - // in tests). + // fallbacks when a Manager is built from a Config that skipped config.Load's defaulting defaultMaxFork = 16 defaultRefill = 4 @@ -134,8 +129,7 @@ type SandboxSummary struct { Archived bool `json:"archived,omitempty"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` - // ClaimRef echoes the caller reference recorded at claim time; empty for - // fork and checkpoint-branch claims. + // ClaimRef echoes the caller reference; empty for fork and checkpoint-branch claims. ClaimRef string `json:"claim_ref,omitempty"` } @@ -154,8 +148,7 @@ type Gauges struct { Hibernated int Archived int Draining bool - // AtCapacity marks refill parked because the node refused another VM, so a - // placer reads a short warm count as "full", not "still filling". + // AtCapacity marks refill parked because the node refused another VM. AtCapacity bool AtCapacityReason string } @@ -168,11 +161,10 @@ type pendingRemoval struct { } type pool struct { - key types.PoolKey + key types.PoolKey + hash string - // floor and warmMax bound the demand-adaptive target (watermark.go); - // rate/lead/lastArrival are its EWMA inputs, guarded by the manager - // mutex like everything else here. + // floor and warmMax bound the demand-adaptive target computed in watermark.go floor int warmMax int idle time.Duration @@ -189,20 +181,21 @@ type pool struct { warm []*types.Sandbox refilling int - // An unbroken failure streak and the retry gate it earns; without them a - // node-wide dead cause is retried every tick at full concurrency. + // without a streak gate a node-wide dead cause is retried every tick at full concurrency refillFails int nextRefill time.Time } -// refillGated reports whether a refill may not spawn: the backoff wait, or — -// expired — the single probe already in flight (a success resets the streak). +func newPool(key types.PoolKey) *pool { + return &pool{key: key, hash: key.Hash()} +} + +// refillGated reports whether a refill may not spawn: backoff wait, or one probe in flight. func (p *pool) refillGated(now time.Time) bool { return now.Before(p.nextRefill) || (p.refillFails >= refillFailStreak && p.refilling > 0) } -// noteRefillResult records one refill outcome and reports whether this call -// started a backoff, so the caller logs the transition, not every attempt. +// noteRefillResult records one refill outcome and reports whether it started a backoff. func (p *pool) noteRefillResult(now time.Time, failed bool) bool { if !failed { p.refillFails, p.nextRefill = 0, time.Time{} @@ -226,8 +219,7 @@ func (p *pool) applySpec(spec config.PoolSpec) { p.archiveDelete = time.Duration(spec.ArchiveDeleteAfterSeconds) * time.Second } -// trimWarm shrinks p.warm to at most target, returning the trimmed VM names; -// callers hold m.mu and destroy the returned VMs outside it. +// trimWarm shrinks p.warm to at most target and returns the trimmed VM names; callers hold m.mu. func (p *pool) trimWarm(target int) []string { var trim []string for len(p.warm) > target { @@ -249,46 +241,32 @@ type Manager struct { maxFork int store *claimStore volumes map[string]catalogVolume - // volumeAdmission counts the live holders of each volume name, mirroring - // the hypervisor's own per-image lock so a conflicting claim is refused - // before it pays any attach cost; guarded by m.mu. + // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders poolStore *poolStore configSeedHash string // config pools' hash, to warn when a file edit is overridden - // idleDefault is the idle-hibernate threshold for unpooled keys; pooled - // keys carry theirs on the pool struct. Zero means disabled. + // idleDefault is the idle-hibernate threshold for unpooled keys; zero disables. idleDefault time.Duration idleEnabled bool idleSweep atomic.Bool - // archive*Default are the archive thresholds for unpooled keys; pooled keys - // carry theirs on the pool struct. archiveEnabled is set when any pool or - // the node default has archiving on; archiveSweep guards the sweep loop. + // archive*Default are the archive thresholds for unpooled keys. archiveAfterDefault time.Duration archiveDeleteDefault time.Duration archiveEnabled bool archiveSweep atomic.Bool archiveDeleteSweep atomic.Bool - // archiving holds ids with an archive() export in flight, so the reap tick - // and archive sweep don't both re-export the same sandbox; pendingCks pins - // checkpoint ids during archive publish and removal commits. Both guarded - // by m.mu. + // archiving holds ids with an export in flight; pendingCks pins ids mid-commit; both under m.mu. archiving map[string]struct{} pendingCks map[string]struct{} - // egressListeners holds the per-sandbox egress proxy accept point, keyed by - // sandbox id; guarded by m.mu. + // egressListeners holds the per-sandbox egress proxy accept point; guarded by m.mu. egressListeners map[string]*egressListener - // egressTaps holds the nft-locked egress-lane tap per sandbox id, tracked - // apart from the proxy so a locked-but-policyless NIC still unlocks on - // release; guarded by m.mu. + // egressTaps holds the nft-locked egress-lane tap per sandbox id; guarded by m.mu. egressTaps map[string]string - // maxClaims caps live claims node-wide (0 = unlimited); tenantMax holds - // every configured tenant's cap (0 = unlimited) and doubles as the set of - // known tenants; tenantLive counts live claims per tenant. usage is the - // always-on billing event stream, audit the config-gated request tap. + // tenantMax doubles as the set of known tenants; a 0 cap means unlimited. maxClaims int draining bool // guarded by m.mu; deliberately not persisted tenantMax map[string]int @@ -299,14 +277,10 @@ type Manager struct { audit *journal counters counters ckpts store.Store - // A cluster-wide backend resolves every record from every node, so heal - // and the delete broadcast are both no-ops against it. + // a cluster-wide backend makes heal and the delete broadcast no-ops ckptsShared bool healer *peer.Healer - // healSem bounds concurrent transfers node-wide; healFlights dedups - // same-id heals onto one transfer, which owns its own staging dir. - // healPending/healAbort (guarded by recLocksMu) let a delete veto a heal - // still staging the same id (see vetoIfHealPending). + // healPending/healAbort, guarded by recLocksMu, let a delete veto a heal still staging. healSem chan struct{} healFlights singleflight.Group healPending map[string]struct{} @@ -316,41 +290,26 @@ type Manager struct { ckptTTL time.Duration ckptSweeping atomic.Bool - // tplSet caches each template id against its owning tenant ("" = operator), - // so the gossip tick and the claim path never pay a store read. + // tplSet caches each template id against its owning tenant ("" = operator). tplMu sync.Mutex tplSet map[string]string - // recLocks serializes same-id store record mutations and holds off a - // re-publish swap while a clone reads the old generation (per id, RW). - // recLocksMu guards recRefs, the live-holder count that gates eviction: a - // checkpoint id can become live again after a delete (a peer's heal can - // republish one), so an entry is only safe to evict once nothing still - // holds or awaits it — never on a bare "record deleted" signal. - recLocks sync.Map + // recLocks serializes same-id record mutations; an entry evicts only when no holder remains. + recLocks map[string]*sync.RWMutex recLocksMu sync.Mutex recRefs map[string]int - // recEvict remembers ids a delete asked to evict while a reference still - // held the lock, so the eviction happens when the last holder leaves rather - // than being lost — otherwise the recLocks entry leaks for that id. + // recEvict defers a delete's eviction to the last holder, else the recLocks entry leaks. recEvict map[string]struct{} - // notifyTemplates, when set (before serving starts), fires after a - // promote or template delete so the mesh republishes immediately - // instead of waiting out a gossip tick. + // notifyTemplates is set before serving starts and fires after a promote or template delete. notifyTemplates func() - // egressSecrets resolves a rule's secret name to the injected header, read-only - // after startup. guardedEgress arms the proxy (some policy exists); lockEgress - // nft-locks every egress-lane NIC default-deny (a bridge lane exists). egressCA - // signs HTTPS-interception leaves and is baked into interception pools' goldens; - // nil unless a pool rule sets intercept. + // egressSecrets is read-only after startup; egressCA is nil unless a pool rule intercepts. egressSecrets *egress.SecretStore egressCA *egress.CA guardedEgress bool lockEgress bool - // dial and sweep are fields as test seams: the SSRF guard blocks loopback - // test origins, and the nft sweep is netlink-only. + // dial and sweep are test seams: the SSRF guard blocks loopback, the nft sweep is netlink-only. dial egress.DialFunc sweep func(map[string]bool) error @@ -359,8 +318,7 @@ type Manager struct { claimed map[string]*types.Sandbox pendingRemovals map[string]pendingRemoval - // Node-wide, unlike the per-pool streak backoff: the resource that ran - // out — bridge ports, disk — is shared by every pool. + // node-wide, unlike the per-pool streak backoff: the exhausted resource is shared atCapacityUntil time.Time atCapacityReason string @@ -369,9 +327,7 @@ type Manager struct { refillKick chan struct{} } -// NewManager builds a manager from the node config; ctx bounds backend -// construction (the s3 store resolves its credential chain). secrets resolves -// egress rule references and is shared by every per-sandbox proxy. +// NewManager builds a manager from the node config; ctx bounds backend construction. func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *egress.SecretStore) (*Manager, error) { maxFork := cfg.MaxForkCount if maxFork < 1 { @@ -399,6 +355,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg pendingCks: map[string]struct{}{}, egressListeners: map[string]*egressListener{}, egressTaps: map[string]string{}, + recLocks: map[string]*sync.RWMutex{}, recRefs: map[string]int{}, recEvict: map[string]struct{}{}, healPending: map[string]struct{}{}, @@ -421,9 +378,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) } - // Checkpoints and templates are two id-namespaced views over ONE store - // root (ck_* vs tp_*): a shared mount or bucket carries both, and each - // instance's listing filters to its own records. + // checkpoints and templates are two id-namespaced views (ck_*, tp_*) over one store root var err error if m.ckpts, err = newStoreView(ctx, cfg, "checkpoint-staging", store.CheckpointIDRe); err != nil { return nil, err @@ -471,7 +426,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.archiveDeleteDefault = time.Duration(cfg.ArchiveDeleteAfterSeconds) * time.Second m.archiveEnabled = m.archiveAfterDefault > 0 for _, spec := range cfg.Pools { - p := &pool{key: spec.PoolKey} + p := newPool(spec.PoolKey) p.applySpec(spec) m.pools[spec.PoolKey] = p if spec.IdleHibernateSeconds > 0 { @@ -499,8 +454,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg return m, nil } -// EgressCAFingerprint is the egress cluster root's fingerprint, or "" when the -// node does no interception. +// EgressCAFingerprint is the egress root's fingerprint, or "" when the node intercepts nothing. func (m *Manager) EgressCAFingerprint() string { if m.egressCA == nil { return "" @@ -514,8 +468,7 @@ func (m *Manager) Run(ctx context.Context) { defer refill.Stop() reap := time.NewTicker(reapInterval) defer reap.Stop() - // Store retention is hourly, not per reap tick: shared roots and buckets - // make each sweep cluster-visible I/O. + // store retention is hourly: each sweep is cluster-visible I/O on a shared root storeSweep := time.NewTicker(time.Hour) defer storeSweep.Stop() if m.ckptTTL > 0 { @@ -545,14 +498,12 @@ func (m *Manager) Run(ctx context.Context) { } } -// FlushClaims synchronously persists the current claim set — the shutdown -// hook that closes the window where a detached recommit has not converged yet. +// FlushClaims synchronously persists the current claim set at shutdown. func (m *Manager) FlushClaims() error { return m.store.commit(m.claimsSnapshot()) } -// SetTemplateNotifier wires the immediate-republish hook; call it before -// the server starts serving. +// SetTemplateNotifier wires the immediate-republish hook; call it before serving starts. func (m *Manager) SetTemplateNotifier(fn func()) { m.notifyTemplates = fn } @@ -562,11 +513,15 @@ func (m *Manager) Info() ([]PoolInfo, Gauges) { m.mu.Lock() defer m.mu.Unlock() now := time.Now() - pools := make([]PoolInfo, 0, len(m.pools)) + live := make([]*pool, 0, len(m.pools)) for _, p := range m.pools { - if p.removed { - continue + if !p.removed { + live = append(live, p) } + } + slices.SortFunc(live, func(a, b *pool) int { return strings.Compare(a.hash, b.hash) }) + pools := make([]PoolInfo, 0, len(live)) + for _, p := range live { pools = append(pools, PoolInfo{ Key: p.key, Warm: len(p.warm), @@ -575,7 +530,6 @@ func (m *Manager) Info() ([]PoolInfo, Gauges) { Golden: p.goldenDir != "", }) } - slices.SortFunc(pools, func(a, b PoolInfo) int { return strings.Compare(a.Key.Hash(), b.Key.Hash()) }) g := Gauges{Claimed: len(m.claimed), Draining: m.draining} if now.Before(m.atCapacityUntil) { g.AtCapacity, g.AtCapacityReason = true, m.atCapacityReason @@ -611,16 +565,15 @@ func (m *Manager) WarmCounts() map[string]int { m.mu.Lock() defer m.mu.Unlock() counts := make(map[string]int, len(m.pools)) - for key, p := range m.pools { + for _, p := range m.pools { if !p.removed { - counts[key.Hash()] = len(p.warm) + counts[p.hash] = len(p.warm) } } return counts } -// WithPeerHeal installs the healer ClaimCheckpointHeal pulls through, so a -// peer transfer is paid only once nothing cheaper resolves the claim. +// WithPeerHeal installs the healer ClaimCheckpointHeal pulls through. func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { if !enabled || owners == nil || m.ckptsShared { return @@ -628,8 +581,7 @@ func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { m.healer = peer.NewHealer(owners, &peer.HTTPPuller{Token: token}) } -// WithPeerDelete wires the broadcast DeleteCheckpoint makes after a local -// delete, so a healed replica does not outlive the source record. +// WithPeerDelete wires the broadcast DeleteCheckpoint makes after a local delete. func (m *Manager) WithPeerDelete(fn PeerDeleteFunc) { m.peerDelete = fn } @@ -645,9 +597,7 @@ func (m *Manager) sweepStoreGenerations(ctx context.Context) { } func (m *Manager) claimsSnapshot() claimSnapshot { - m.mu.Lock() - defer m.mu.Unlock() - return m.store.snapshot(m.claimed) + return m.store.mark() } func (m *Manager) untrack(set map[string]struct{}, key string) { @@ -714,9 +664,7 @@ func dirExists(path string) bool { return err == nil && fi.IsDir() } -// tenantOwns is THE tenancy predicate — every read/delete/overwrite scope -// check goes through it: root (empty tenant) owns everything, a tenant only -// records stamped with its own name. +// tenantOwns is the tenancy predicate: root (empty tenant) owns everything, a tenant its own. func tenantOwns(tenant, owner string) bool { return tenant == "" || tenant == owner } @@ -735,8 +683,7 @@ func logSweepResult(ctx context.Context, logger *log.Fields, err error, okMsg, f } } -// benignSweepErr reports whether err is the expected outcome of a housekeeping -// sweep (victim released, woke mid-sweep, or a lane that never hibernates). +// benignSweepErr reports whether err is the expected outcome of a housekeeping sweep. func benignSweepErr(err error) bool { return errors.Is(err, ErrUnknownSandbox) || errors.Is(err, errWokeMeanwhile) || errors.Is(err, ErrNoEgressHibernate) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index c6a9e766..3a8780bb 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -188,10 +188,6 @@ func TestReleaseValidatesToken(t *testing.T) { } } -// TestReleaseAfterResolveTearsDownOnce: resolve authorizes off the manager -// mutex, so two callers can both hold the same resolved claim before either -// drops it — the state a live double-release reaches. Exactly one may tear the -// VM down; the loser must read as unknown rather than remove it twice. func TestReleaseAfterResolveTearsDownOnce(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -223,11 +219,10 @@ func TestReleaseByOperatorCred(t *testing.T) { m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - // Unknown id is 404-equivalent, exactly like a token release. if err := m.Release(t.Context(), "sb_nope", Cred{Operator: true}); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("got %v, want ErrUnknownSandbox", err) } - // The operator releases by id alone — no per-sandbox token needed. + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { t.Fatalf("Release operator: %v", err) } @@ -239,8 +234,6 @@ func TestReleaseByOperatorCred(t *testing.T) { } } -// TestResolveRejectsEmptyToken documents resolve's explicit guard: an -// unclaimed slot must never match an empty token. func TestResolveRejectsEmptyToken(t *testing.T) { m := newTestManager(t, newFakeEngine()) sb := mustClaim(t, m, testKey) @@ -279,7 +272,7 @@ func TestReapDestroysExpiredClaims(t *testing.T) { m.mu.Unlock() m.reapOnce(t.Context()) - // The destroy batch is fanned off the ticker loop; poll for it. + waitFor(t, func() bool { return eng.removed(sb.VMName) }) if _, g := m.Info(); g.Claimed != 0 { t.Errorf("claimed=%d, want 0", g.Claimed) @@ -427,7 +420,7 @@ func TestRefillServicesEveryPoolUnderTightBudget(t *testing.T) { func TestSetPoolsRemovalShedsArchivePolicy(t *testing.T) { m := newTestManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 0, ArchiveAfterSeconds: 60, ArchiveDeleteAfterSeconds: 120}) m.mu.Lock() - m.pools[testKey].refilling = 1 // keep the removed pool lingering + m.pools[testKey].refilling = 1 m.mu.Unlock() if err := m.SetPools(t.Context(), nil); err != nil { @@ -467,8 +460,6 @@ func TestSetPoolsRejectsInvalidSpec(t *testing.T) { } } -// SetPools neither validates nor keeps egress policy: accepting it would -// report success and silently drop it. func TestSetPoolsRejectsEgress(t *testing.T) { m := newTestManager(t, newFakeEngine()) err := m.SetPools(t.Context(), []config.PoolSpec{{ @@ -603,8 +594,6 @@ func TestRefillBudgetFollowsConfig(t *testing.T) { } } -// TestRefillReactivelyFillsToTarget: one trigger fills the whole target (not -// just the concurrency budget), since a finished refill chains the next. func TestRefillReactivelyFillsToTarget(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -677,9 +666,6 @@ func TestReapBatchDoesNotStallTheLoop(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - // More expired claims than the concurrency budget (defaultRefill here) so - // the batch would block the submit loop if it acquired the semaphore there - // instead of per-goroutine. const n = defaultRefill + 3 var sbs []*types.Sandbox for range n { @@ -699,7 +685,7 @@ func TestReapBatchDoesNotStallTheLoop(t *testing.T) { done := make(chan struct{}) go func() { m.reapOnce(t.Context()); close(done) }() select { - case <-done: // returned while every destroy is parked: the loop never blocked + case <-done: case <-time.After(2 * time.Second): t.Fatal("reapOnce blocked on a batch larger than the budget") } @@ -710,7 +696,7 @@ func TestReapBatchDoesNotStallTheLoop(t *testing.T) { func TestProbeReadyWaitsForVsock(t *testing.T) { eng := newFakeEngine() - eng.vsockLateN = 2 // cocoon reports the socket only once the VMM runs + eng.vsockLateN = 2 m := newTestManager(t, eng) sb, err := claimAny(t.Context(), m, testKey, 0) @@ -727,8 +713,6 @@ func newTestManager(t *testing.T, eng *fakeEngine, pools ...config.PoolSpec) *Ma return newTestManagerAt(t, eng, t.TempDir(), pools...) } -// claimAny composes warm-then-provision the way the server does around the -// redirect decision; production has no single-call form. func claimAny(ctx context.Context, m *Manager, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) { sb, err := m.ClaimWarm(ctx, key, ttl, "", "", nil) if errors.Is(err, ErrNoWarm) { @@ -764,6 +748,19 @@ func testSecrets(t *testing.T, specs ...egress.SecretSpec) *egress.SecretStore { return s } +func lockCount(m *Manager) int { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + return len(m.recLocks) +} + +func hasRecLock(m *Manager, id string) bool { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + _, ok := m.recLocks[id] + return ok +} + func waitFor(t *testing.T, cond func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) @@ -776,65 +773,56 @@ func waitFor(t *testing.T, cond func() bool) { t.Fatal("condition not met within 3s") } -// fakeEngine implements Engine in memory: created VMs land in vms (backing -// List/vsockOf), destroy removes them, and error fields fail one verb at a -// time. type fakeEngine struct { - mu sync.Mutex - vms map[string]string // VM name → vsock socket path - clones []string - cloneFroms []string - colds []string - removes []string - probeTimeouts []time.Duration - volumeSpecs []engine.VolumeSpec - volumeMounts []types.Volume - volumeOps []string - // attachDirty records whether an image's dirty marker was already down when - // it was attached; removeSeenOps and removeSeenDirty snapshot the volume-op - // log and the markers still on disk at each VM removal, so teardown ordering - // is checkable on both sides of the removal. + mu sync.Mutex + vms map[string]string + clones []string + cloneFroms []string + colds []string + removes []string + probeTimeouts []time.Duration + volumeSpecs []engine.VolumeSpec + volumeMounts []types.Volume + volumeOps []string attachDirty map[string]bool removeSeenOps map[string][]string removeSeenDirty map[string][]string - // dirtyPaths are the images removeSeenDirty samples; tests set it to the - // catalog paths the claim under test uses. - dirtyPaths []string - syncs []string // vsock sockets SyncGuest was called on + dirtyPaths []string + syncs []string hibernates, restores, snapRemoves []string snapSaves, exports, snapshots []string exportContent []byte - caInstalls []string // vsock sockets InstallCACert was called on - staleReconciles []string // VM names ReconcileStaleCreate was called on + caInstalls []string + staleReconciles []string installCAErr error diskAttachErr error - diskAttachErrFor string // volume name whose DiskAttach fails; "" = never + diskAttachErrFor string diskAttachCancel context.CancelFunc mountVolumeErr error unmountVolumeErr error stopped map[string]bool - creating map[string]bool // VMs List reports in the creating state + creating map[string]bool staleOutcome engine.StaleCreateOutcome staleErr error - pids map[string]int // VM name → PID, for Stats' resident-set lookup - vsockLateN int // List calls that report no socket yet + pids map[string]int + vsockLateN int cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapSaveErr, snapListErr error - removeErrFor string // VM name whose Remove fails; "" = never - cloneFailNth int // 1-based Clone call to fail; 0 = never - hibernateErrCompletes bool // hibernateErr fires after the snapshot lands (CLI timeout) - tap string // non-empty: lifecycle records carry this NIC tap + removeErrFor string + cloneFailNth int + hibernateErrCompletes bool + tap string listCount int - attachRendezvous *sync.WaitGroup // non-nil: DiskAttach waits there until every attach has arrived + attachRendezvous *sync.WaitGroup - cloneStall chan struct{} // non-nil: Clone blocks until closed - probeStall chan struct{} // non-nil: Probe blocks until closed - hibernateStall chan struct{} // non-nil: Hibernate blocks until closed - removeStall chan struct{} // non-nil: Remove blocks until closed - snapRemoveStall chan struct{} // non-nil: SnapshotRemove blocks until closed - snapListStall chan struct{} // non-nil: SnapshotList blocks until closed + cloneStall chan struct{} + probeStall chan struct{} + hibernateStall chan struct{} + removeStall chan struct{} + snapRemoveStall chan struct{} + snapListStall chan struct{} snapListCount int } @@ -866,7 +854,6 @@ func (f *fakeEngine) RunCold(_ context.Context, name string, _ types.PoolKey) (t } func (f *fakeEngine) Remove(ctx context.Context, name string) error { - // Models exec.CommandContext: a canceled ctx never runs cocoon at all. if err := ctx.Err(); err != nil { return err } @@ -893,7 +880,6 @@ func (f *fakeEngine) Remove(ctx context.Context, name string) error { } func (f *fakeEngine) ReconcileStaleCreate(ctx context.Context, name string) (engine.StaleCreateOutcome, error) { - // Models exec.CommandContext: a canceled ctx never runs cocoon at all. if err := ctx.Err(); err != nil { return "", err } @@ -989,7 +975,7 @@ func (f *fakeEngine) Restore(_ context.Context, name, _ string) (string, error) } f.stopped[name] = false if f.vms[name] == "" { - f.vms[name] = "/vsock/" + name // Restore rebuilds the VM from its snapshot + f.vms[name] = "/vsock/" + name } return f.lateVsock(f.vms[name]), nil } @@ -1114,7 +1100,6 @@ func (f *fakeEngine) clone(from, name string) (types.VMRecord, error) { return f.record(name), nil } -// record builds a lifecycle result for name; callers hold f.mu. func (f *fakeEngine) record(name string) types.VMRecord { rec := types.VMRecord{VsockSocket: f.lateVsock(f.vms[name]), Config: types.VMConfig{Name: name}} if f.tap != "" { @@ -1123,8 +1108,6 @@ func (f *fakeEngine) record(name string) types.VMRecord { return rec } -// lateVsock models cocoon's report-once-running lag: while vsockLateN ticks -// remain it reports no socket (consuming one), forcing the caller's poll. func (f *fakeEngine) lateVsock(sock string) string { if f.vsockLateN > 0 { f.vsockLateN-- @@ -1133,8 +1116,6 @@ func (f *fakeEngine) lateVsock(sock string) string { return sock } -// removed reports whether name was destroyed, under the fake's own lock — -// bounded batches append concurrently with test polls. func (f *fakeEngine) removed(name string) bool { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index 4521dd45..a9db122a 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -21,9 +21,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// poolsFile records the last API-applied pool set so a restart rebuilds from -// operator intent, not the config.json seed. ConfigSeed is the config pools' -// hash at write time; a boot finding it changed warns the edit is overridden. +// poolsFile records the last API-applied pool set so a restart rebuilds from operator intent. type poolsFile struct { ConfigSeed string `json:"config_seed"` Pools []config.PoolSpec `json:"pools"` @@ -41,8 +39,7 @@ func newPoolStore(dataDir string) *poolStore { return &poolStore{path: filepath.Join(dataDir, "pools.json")} } -// load returns the persisted pool file, or nil when the node has never taken a -// PUT /v1/pools (config.json seeds the pools, unchanged behavior). +// load returns the persisted pool file, or nil when the node has never taken a PUT /v1/pools. func (s *poolStore) load() (*poolsFile, error) { raw, err := os.ReadFile(s.path) if errors.Is(err, fs.ErrNotExist) { @@ -58,8 +55,7 @@ func (s *poolStore) load() (*poolsFile, error) { return &pf, nil } -// commit durably writes the applied set, serialized by s.mu; a seq no newer -// than the last written is a no-op, so a superseded concurrent write can't win. +// commit durably writes the applied set; a seq no newer than the last written is a no-op. func (s *poolStore) commit(seq uint64, pf poolsFile) error { s.mu.Lock() defer s.mu.Unlock() @@ -77,9 +73,7 @@ func (s *poolStore) commit(seq uint64, pf poolsFile) error { return nil } -// adoptPersistedPools rebuilds the pools from pools.json (the last API-applied -// set) over the config seed when present. Egress stays config-owned; a restored -// pool that no longer validates fails the boot, matching config-load discipline. +// adoptPersistedPools rebuilds the pools from pools.json over the config seed when present. func (m *Manager) adoptPersistedPools(ctx context.Context) error { pf, err := m.poolStore.load() if err != nil { @@ -103,7 +97,7 @@ func (m *Manager) adoptPersistedPools(ctx context.Context) error { if err := spec.ValidateLimits(); err != nil { return fmt.Errorf("restore pool %q: %w", spec.Template, err) } - p := &pool{key: spec.PoolKey} + p := newPool(spec.PoolKey) p.applySpec(spec) m.adoptGolden(p) m.pools[spec.PoolKey] = p @@ -118,8 +112,7 @@ func (m *Manager) adoptPersistedPools(ctx context.Context) error { return nil } -// poolSeedHash digests a pool set's warm-target shape, order-independent and -// egress-excluded, so it tracks exactly the targets pools.json would override. +// poolSeedHash digests a pool set's warm-target shape, order-independent and egress-excluded. func poolSeedHash(specs []config.PoolSpec) string { shaped := slices.Clone(specs) for i := range shaped { diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index 7a99d8d5..e6fe4b79 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -22,7 +22,7 @@ func TestPersistedPoolsSurviveRestart(t *testing.T) { if err := m.SetPools(t.Context(), []config.PoolSpec{{PoolKey: apiKey, Warm: 3}}); err != nil { t.Fatalf("SetPools: %v", err) } - // Restart on the same data dir, still config-seeded: must rebuild from pools.json. + m2 := newTestManagerAt(t, newFakeEngine(), dir, config.PoolSpec{PoolKey: seedKey, Warm: 1}) m2.mu.Lock() _, hasSeed := m2.pools[seedKey] @@ -67,7 +67,7 @@ func TestConcurrentSetPoolsPersistLatest(t *testing.T) { }) } wg.Wait() - // The last-applied set owns memory; the persisted file must match it. + m.mu.Lock() want := m.pools[apiKey].floor m.mu.Unlock() @@ -92,7 +92,7 @@ func TestRestoredEgressPoolNeedsAttachment(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "pools.json"), raw, 0o600); err != nil { t.Fatalf("write pools.json: %v", err) } - // A node with no bridge/network must refuse to start a restored egress pool. + if _, err := NewManager(t.Context(), &config.Config{DataDir: dir}, newFakeEngine(), testSecrets(t)); err == nil { t.Error("NewManager accepted a restored egress-lane pool without an attachment") } diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index c54f5711..46d66d5a 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -129,8 +129,7 @@ func TestPromoteValidations(t *testing.T) { if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: "wrong"}, "tpl:x", ""); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("bad token: %v, want ErrUnknownSandbox", err) } - // Same template/net/size as the configured pool: the golden path would - // collide with the pool's own. + if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, testKey.Template, ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("pooled key: %v, want ErrPooledTemplate", err) } @@ -160,7 +159,7 @@ func TestDeleteTemplate(t *testing.T) { if m.HasGolden(t.Context(), key, "") { t.Error("template still resolvable after delete") } - // The next claim for the deleted template cold-boots instead of cloning. + before := len(eng.colds) if _, err := claimAny(t.Context(), m, key, 0); err != nil { t.Fatalf("Claim after delete: %v", err) @@ -186,8 +185,6 @@ func TestReconcileSweepsGoldenTmpDirs(t *testing.T) { } func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { - // A template promoted before the egress guard existed must never be resumed: - // resolveGolden returns "" for the egress lane so the claim cold-boots. m := newTestManager(t, newFakeEngine()) id := store.TemplateID(egKey.Hash()) staging, err := m.tpls.Stage(id) @@ -236,9 +233,6 @@ func TestTemplateTenantScopedDelete(t *testing.T) { } } -// TestCommitTemplateRechecksOwnerUnderLock drives the publish-side check -// directly: two racers that both passed Promote's early check must still -// serialize on ownership at publish. func TestCommitTemplateRechecksOwnerUnderLock(t *testing.T) { m := newTestManager(t, newFakeEngine()) id := store.TemplateID(testKey.Hash()) @@ -261,8 +255,6 @@ func TestCommitTemplateRechecksOwnerUnderLock(t *testing.T) { } } -// TestPromoteFailsClosedOnMetaError: an unreadable template meta must refuse -// the promote, never skip the ownership check. func TestPromoteFailsClosedOnMetaError(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root ignores file permissions") @@ -306,7 +298,7 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { if _, _, err := m.Promote(t.Context(), b.ID, Cred{Token: b.Token}, "shared:v1", "beta"); !errors.Is(err, ErrTemplateOwned) { t.Errorf("beta overwrite: %v, want ErrTemplateOwned", err) } - r := claim("") // root may replace anything + r := claim("") if _, _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "shared:v1", ""); err != nil { t.Errorf("root replace: %v, want ok", err) } @@ -370,8 +362,6 @@ func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { t.Fatalf("promote: %v", err) } - // Routing must answer what a claim would: promising beta a golden here - // makes redirectClaim skip the peer hop and cold-boot the name as an image. if m.HasPromotedTemplate(t.Context(), key, "beta") { t.Error("beta sees acme's template as a local golden") } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 1ac31e3a..63199475 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -17,11 +17,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Reconcile aligns state after a daemon restart: re-adopt persisted claims -// whose VMs are still running (or hibernated), drop the rest, and remove any -// sbx-prefixed VM nobody owns (stale pool VMs and golden builders from a -// previous life). It must run once at startup, before the server: it swaps -// in fresh records, which would bypass in-flight Transition locks. +// Reconcile aligns claims and VMs after a restart; it must run before the server starts. func (m *Manager) Reconcile(ctx context.Context) error { claims, err := m.store.load() if err != nil { @@ -45,26 +41,19 @@ func (m *Manager) Reconcile(ctx context.Context) error { rec, ok := live[sb.VMName] switch { case sb.ArchiveCk != "": - // Archived: no local VM by design; the store ck is the durable - // state. Adopt the stub; the first exec wakes it (wakeArchived). + // archived: no local VM by design, so adopt the stub and let the first exec wake it m.claimed[id] = sb m.tenantDelta(sb.Tenant, 1) continue case ok && rec.State == vmStateRunning: sb.VsockSocket = rec.VsockSocket - // Running with either flag set = a wake crashed between restore - // and commit, or a hibernate intent never reached the engine; - // clearing them un-bricks the claim and unreferences the - // snapshot for the sweep below. + // running with either flag set means a crashed wake or an intent the engine never saw sb.HibernateSnap, sb.PendingSnap = "", "" case ok && sb.HibernateSnap != "": // Hibernated: the VM is stopped by design and wakes on demand. sb.PendingSnap = "" case ok && sb.PendingSnap != "" && (snapsErr != nil || slices.Contains(snaps, sb.PendingSnap)): - // Stopped under a journaled hibernate intent whose commit never - // landed: the intent names the wake image, adopt it. A verified- - // missing image means the hibernate never completed — fall - // through and drop; an unverifiable list adopts. + // the journaled intent names the wake image, so adopt it when the snapshot is there sb.HibernateSnap, sb.PendingSnap = sb.PendingSnap, "" default: continue @@ -80,8 +69,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.adoptGolden(p) } m.mu.Unlock() - // A crash mid-export leaves a *.tmp staging dir no build or promote of - // this life will reuse. + // a crash mid-export leaves a *.tmp staging dir nothing in this life reuses tmps, _ := filepath.Glob(filepath.Join(m.goldensDir(), "*.tmp")) if err := m.ckpts.SweepStaging(); err != nil { log.WithFunc("pool.Reconcile").Error(ctx, err, "sweep checkpoint staging") @@ -98,9 +86,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.retryArchiveDeletes(ctx) removed := m.sweepStaleVMs(ctx, live, owned) - // A hibernate snapshot no adopted claim references is an orphan; - // fork/golden-build snapshots never span a restart. A list failure only - // skips the sweep — GC must not brick startup. + // an unreferenced hibernate snapshot is an orphan; fork and golden snapshots never span a restart if snapsErr != nil { logger.Warnf(ctx, "snapshot sweep skipped: %v", snapsErr) } else { @@ -121,8 +107,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { return saveErr } -// sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones -// confirmed gone; a failed remove is omitted so the egress sweep keeps its lock. +// sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRecord, owned map[string]bool) map[string]bool { logger := log.WithFunc("pool.sweepStaleVMs") var stale []string @@ -147,15 +132,11 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec return removed } -// removeStaleVM reclaims one unowned VM, reporting whether it is gone; a -// creating-state record goes through reconcile-stale-create first, since -// rm --force would queue on the ops lock and then delete the VM an -// in-flight clone just produced. +// removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") if rec.State == vmStateCreating { - // Cancellation-immune like removeVM: a canceled ctx must not skip - // the busy check and fall through to the forced remove it guards. + // a canceled ctx must not skip the busy check guarding the forced remove switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); { case err != nil: // Verb missing (cocoon < v0.5.8) or failed: keep the old sweep. @@ -172,11 +153,15 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe return m.removeOrRetry(ctx, name, "", rec.TapDevice(), volumeTeardown{}) } -// resyncEgress re-locks adopted egress claims after a restart, quarantines any -// it cannot lock, and sweeps tables orphaned by VMs confirmed gone (in removed). +// resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) { logger := log.WithFunc("pool.resyncEgress") now := time.Now() + var locked map[string]bool + var lockedErr error + if m.lockEgress { + locked, lockedErr = netfilter.LockedTaps() + } var quarantine []*types.Sandbox for _, sb := range m.claimed { sb.TouchAt(now) @@ -187,7 +172,11 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco quarantine = append(quarantine, sb) continue } - if err := netfilter.EnsureLock(tap); err != nil { + err := lockedErr + if err == nil && !locked[tap] { + err = netfilter.Lock(tap) + } + if err != nil { logger.Errorf(ctx, err, "ensure egress lock %s; quarantining", sb.ID) quarantine = append(quarantine, sb) continue @@ -224,7 +213,7 @@ func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { m.mu.Lock() delete(m.claimed, sb.ID) m.tenantDelta(sb.Tenant, -1) - js := m.store.snapshot(m.claimed) + js := m.store.del(sb.ID) m.mu.Unlock() if err := m.store.commit(js); err != nil { m.recommit(ctx, js) @@ -232,8 +221,7 @@ func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { return gone } -// readoptEgressTap records a live egress-lane claim's tap for lifecycle after a -// restart and returns it; "" for the none lane or a VM whose tap is not listed. +// readoptEgressTap records and returns a live egress claim's tap, "" when there is none. func (m *Manager) readoptEgressTap(sb *types.Sandbox, live map[string]types.VMRecord) string { if sb.Key.Net != types.NetEgress { return "" @@ -246,14 +234,12 @@ func (m *Manager) readoptEgressTap(sb *types.Sandbox, live map[string]types.VMRe m.mu.Lock() m.egressTaps[sb.ID] = tap sb.TAP = tap + m.store.set(sb) m.mu.Unlock() return tap } -// reclaimOrphanArchiveCks reaps archives that crashed between their -// checkpoint publish and the journal commit: the flag hides them from -// listings and deletes, so only the journal identifies ours — the sandbox is -// in it under a different (or no) ArchiveCk. +// reclaimOrphanArchiveCks reaps archives that crashed between publish and journal commit. func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string]*types.Sandbox) { logger := log.WithFunc("pool.reclaimOrphanArchiveCks") metas, err := m.ckpts.Metas(ctx) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 96ce2b1e..d5b23e94 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -152,8 +152,6 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { } } -// staleCreateCase drives both the startup sweep and the reap-tick retry -// through the same outcome/end-state row shape. type staleCreateCase struct { name string outcome engine.StaleCreateOutcome diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index bc1099ea..82ea628e 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -19,8 +19,7 @@ import ( type vmProvisioner func(name string) (types.VMRecord, error) -// kickRefill nudges Run past its refill ticker after a warm claim; the -// 1-buffered channel coalesces bursts and never blocks the claim path. +// kickRefill nudges Run past its refill ticker without blocking the claim path. func (m *Manager) kickRefill() { select { case m.refillKick <- struct{}{}: @@ -35,15 +34,12 @@ func (m *Manager) refillOnce(ctx context.Context) { return } now := time.Now() - // Spawning VMs into a full node still takes the global rtnl lock even when - // doomed to fail. The park gates spawns only — sweeping removed pools is - // pure bookkeeping. + // spawning VMs into a full node still takes the global rtnl lock even when doomed to fail parked := now.Before(m.atCapacityUntil) inFlight := 0 for key, p := range m.pools { inFlight += p.refilling - // SetPools leaves a removed pool in place while a build/refill is in - // flight; sweep it once quiescent. + // SetPools leaves a removed pool in place while a build or refill is in flight if p.removed { if !p.building && p.refilling == 0 { delete(m.pools, key) @@ -106,8 +102,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string) { p.noteLead(time.Since(start)) keep = true } - // A canceled context is the daemon going down, and a capacity failure is - // the node's fault — neither says anything about this pool's health. + // a canceled context or a capacity failure says nothing about this pool's health backedOff := false if ctx.Err() == nil && capReason == "" { backedOff = p.noteRefillResult(now, err != nil) @@ -122,12 +117,11 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string) { switch { case ctx.Err() != nil: case entered: - // One line per episode; the storm this replaces wrote 20k an hour. log.WithFunc("pool.refillOne").Errorf(ctx, err, "node is at capacity; parking refill for %s", capacityBackoff) case parked: default: - hash := p.key.Hash() + hash := p.hash log.WithFunc("pool.refillOne").Errorf(ctx, err, "refill %s", hash) if backedOff { log.WithFunc("pool.refillOne").Warnf(ctx, @@ -145,8 +139,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string) { } } -// noteCapacityLocked parks refill node-wide for a capacity failure, reporting -// whether this call parked it. m.mu held. +// noteCapacityLocked parks refill node-wide, reporting whether this call parked it. func (m *Manager) noteCapacityLocked(now time.Time, reason string) bool { first := !now.Before(m.atCapacityUntil) m.atCapacityUntil = now.Add(capacityBackoff) @@ -156,7 +149,7 @@ func (m *Manager) noteCapacityLocked(now time.Time, reason string) bool { func (m *Manager) buildGolden(ctx context.Context, p *pool) { logger := log.WithFunc("pool.buildGolden") - hash := p.key.Hash() + hash := p.hash name := vmPrefix + "gb-" + hash snap := goldenPrefix + hash final := filepath.Join(m.goldensDir(), hash) @@ -173,8 +166,7 @@ func (m *Manager) buildGolden(ctx context.Context, p *pool) { p.goldenDir = final } else { p.nextBuild = time.Now().Add(buildRetryDelay) - // A cold boot hits the same bridge and disk as a refill; feed the park - // so a node failing only its builds still publishes at-capacity. + // a cold boot hits the same bridge and disk as a refill, so it feeds the park too if reason := engine.CapacitySignature(err); reason != "" { m.noteCapacityLocked(time.Now(), reason) } @@ -211,8 +203,7 @@ func (m *Manager) buildGoldenSteps(ctx context.Context, key types.PoolKey, name, return m.writeGoldenCASidecar(final, caBaked) } -// writeGoldenCASidecar records (or clears) the baked-CA fingerprint, so a -// rotated CA or flipped intercept flag forces a rebuild on restart. +// writeGoldenCASidecar records or clears the baked-CA fingerprint; a rotated CA forces a rebuild. func (m *Manager) writeGoldenCASidecar(final string, caBaked bool) error { path := final + caSidecarSuffix if !caBaked { @@ -227,17 +218,14 @@ func (m *Manager) writeGoldenCASidecar(final string, caBaked bool) error { return nil } -// adoptGolden points p at a golden already on disk from the pool's earlier -// life, when its baked-CA state still fits; buildGolden covers the rest. +// adoptGolden points p at an on-disk golden whose baked-CA state still fits. func (m *Manager) adoptGolden(p *pool) { - if g := filepath.Join(m.goldensDir(), p.key.Hash()); dirExists(g) && m.goldenCAMatches(g, m.poolEgress[p.key].Intercepts()) { + if g := filepath.Join(m.goldensDir(), p.hash); dirExists(g) && m.goldenCAMatches(g, m.poolEgress[p.key].Intercepts()) { p.goldenDir = g } } -// goldenCAMatches reports whether a golden's baked-CA state fits the pool: a -// CA-baked one must carry the current fingerprint, a plain one none. The plain -// case only stats (no read) — it runs under m.mu on the live SetPools path. +// goldenCAMatches reports whether a golden's baked-CA state fits the pool. func (m *Manager) goldenCAMatches(final string, caNeeded bool) bool { if !caNeeded { _, err := os.Stat(final + caSidecarSuffix) @@ -247,10 +235,7 @@ func (m *Manager) goldenCAMatches(final string, caNeeded bool) bool { return err == nil && m.egressCA != nil && string(fp) == m.egressCA.Fingerprint() } -// exportGolden exports snap into final through a unique sibling *.tmp dir: -// a crash mid-export never leaves a half-written dir that would pass for a -// golden, and concurrent promotes to one name cannot clobber each other's -// staging (last rename wins whole). +// exportGolden exports snap into final through a unique sibling *.tmp dir. func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { staging, err := os.MkdirTemp(filepath.Dir(final), filepath.Base(final)+"-*.tmp") if err != nil { @@ -267,9 +252,7 @@ func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { return os.Rename(tmp, final) } -// sourceSnap picks the snapshot to export a claimed sandbox from: the wake -// image of a hibernated one (kept — the wake still needs it), or a transient -// capture of a running one, dropped by the returned cleanup. +// sourceSnap picks the snapshot to export a claimed sandbox from, with its cleanup. func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, func(), error) { if sb.HibernateSnap != "" { return sb.HibernateSnap, func() {}, nil @@ -281,11 +264,7 @@ func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, fu return snap, func() { m.dropSnap(ctx, snap) }, nil } -// exportSource captures a claimed sandbox's state into exportDir, returning -// the snapshot name it exported. Only this window holds the transition lock — -// it pins the source snapshot against a concurrent wake consuming it; the -// minutes-long clone fan-out after it must not block the source's own -// wake/hibernate traffic. +// exportSource captures a claimed sandbox's state into exportDir, returning the snapshot name. func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir string) (string, error) { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -297,8 +276,7 @@ func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir return snap, m.eng.SnapshotExport(ctx, snap, exportDir) } -// provision creates one claim-ready VM: clone from a golden when available, -// cold-boot the template otherwise. +// provision creates one claim-ready VM, cloning from a golden when available. func (m *Manager) provision(ctx context.Context, key types.PoolKey, golden string) (*types.Sandbox, error) { if golden == "" { sb, err := m.provisionVM(ctx, key, coldProbeTimeout, func(name string) (types.VMRecord, error) { @@ -307,8 +285,7 @@ func (m *Manager) provision(ctx context.Context, key types.PoolKey, golden strin if err != nil { return nil, err } - // A pre-golden cold claim must trust the root too, or its intercepted - // hosts fail TLS for the sandbox's whole life. + // a pre-golden cold claim must trust the root, or intercepted hosts fail TLS for its life if m.poolIntercepts(key) { if err := m.eng.InstallCACert(ctx, sb.VsockSocket, m.egressCA.CertPEM()); err != nil { m.destroy(ctx, sb.VMName) @@ -390,9 +367,7 @@ func (m *Manager) cloneBatch(ctx context.Context, count int, key types.PoolKey, return children, nil } -// probeReady waits until a VM's silkd answers, returning its vsock socket. sock -// is what the lifecycle command reported; when empty (a heavy image can boot -// past cocoon's post-start inspect) it polls `vm list` until the socket appears. +// probeReady waits until a VM's silkd answers, polling `vm list` when sock is empty. func (m *Manager) probeReady(ctx context.Context, name, sock string, timeout time.Duration) (string, error) { deadline := time.Now().Add(timeout) if sock == "" { @@ -438,15 +413,11 @@ func (m *Manager) findVM(ctx context.Context, name string) (types.VMRecord, bool return vms[i], true, nil } -// runBounded fans f over n items on the refill semaphore, so engine -// batches (reap destroys, idle hibernates, reconcile sweeps) share the -// same node-wide concurrency budget as refills without blocking the -// caller. Callers that need completion Wait; fire-and-forget drops it. +// runBounded fans f over n items on the refill semaphore, sharing the node-wide budget. func (m *Manager) runBounded(ctx context.Context, n int, f func(context.Context, int)) *sync.WaitGroup { var wg sync.WaitGroup for i := range n { - // Acquire inside the goroutine: a batch larger than the budget - // must not block the caller (Run's select loop). + // acquire inside the goroutine so a batch larger than the budget cannot block the caller wg.Go(func() { select { case m.refillSem <- struct{}{}: diff --git a/sandboxd/pool/refillbackoff_test.go b/sandboxd/pool/refillbackoff_test.go index 2fba3ebb..2b325907 100644 --- a/sandboxd/pool/refillbackoff_test.go +++ b/sandboxd/pool/refillbackoff_test.go @@ -61,18 +61,14 @@ func TestBackoffGrowsAndIsCapped(t *testing.T) { } } -// TestRefillOnceHonorsTheBackoff pins the property that saves the node: -// while a pool is backed off, the ticker spawns nothing. func TestRefillOnceHonorsTheBackoff(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() - // Deliberately not a capacitySignature: that parks the node after one - // failure; the streak backoff is the fallback for unclassifiable errors. + eng.cloneErr = errors.New("clone: guest kernel panicked before vsock came up") m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 50}) m.pools[testKey].goldenDir = "/goldens/x" - // Concurrency is bounded, so the streak accrues over several ticks. backedOff := false for range 50 { m.refillOnce(t.Context()) @@ -92,8 +88,6 @@ func TestRefillOnceHonorsTheBackoff(t *testing.T) { t.Fatal("a pool whose every refill failed is not backed off") } - // Pin the expiry far out so the no-spawn assertions cannot race a short - // first backoff on a loaded machine. m.mu.Lock() m.pools[testKey].nextRefill = time.Now().Add(time.Hour) m.mu.Unlock() @@ -104,7 +98,6 @@ func TestRefillOnceHonorsTheBackoff(t *testing.T) { t.Errorf("clones=%d after backing off, want %d: a backed-off pool must not attempt again", n, attempts) } - // A pause, not a latch: once the wait expires a working engine fills. m.mu.Lock() m.pools[testKey].nextRefill = time.Now().Add(-time.Second) m.mu.Unlock() diff --git a/sandboxd/pool/refillkick_test.go b/sandboxd/pool/refillkick_test.go index 913fcdd1..061d3a21 100644 --- a/sandboxd/pool/refillkick_test.go +++ b/sandboxd/pool/refillkick_test.go @@ -24,8 +24,6 @@ func TestKickRefillCoalescesAndNeverBlocks(t *testing.T) { } } -// TestNodeAtCapacityParksRefillInsteadOfRetrying pins the difference between -// "this VM failed to start" and "this node cannot hold another VM". func TestNodeAtCapacityParksRefillInsteadOfRetrying(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -49,7 +47,6 @@ func TestNodeAtCapacityParksRefillInsteadOfRetrying(t *testing.T) { t.Fatalf("at-capacity not published: %+v", g) } - // Every later tick is a no-op until the backoff expires. for range 5 { m.refillOnce(t.Context()) } @@ -57,7 +54,6 @@ func TestNodeAtCapacityParksRefillInsteadOfRetrying(t *testing.T) { t.Errorf("clones=%d after parking, want %d: a parked node must not attempt again", n, attempts) } - // A park, not a latch: once the backoff expires refill resumes. m.mu.Lock() m.atCapacityUntil = time.Now().Add(-time.Second) m.mu.Unlock() diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 0092d32d..117d3402 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -42,10 +42,7 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { return true } -// removeOrRetry reports whether the VM is confirmed gone; a survivor is -// queued for the reap tick to retry, carrying what its cleanup needs (sandbox -// ID / tap / volume teardown). A restart loses a queued teardown: its markers -// stay until an rw claim clears them, and its holds die with the process. +// removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID, tap string, td volumeTeardown) bool { if m.removeVM(ctx, name) { m.finishVolumeTeardown(ctx, td) @@ -67,9 +64,7 @@ func (m *Manager) queueStaleCreate(name, tap string) { m.mu.Unlock() } -// retryRemovals dispatches by emptying the queue, so absence also means "in -// flight" and the next tick cannot double-dispatch; a failed retry re-queues -// itself on completion. Callers that need completion Wait. +// retryRemovals empties the queue to dispatch, so the next tick cannot double-dispatch. func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { m.mu.Lock() if len(m.pendingRemovals) == 0 { diff --git a/sandboxd/pool/remove_test.go b/sandboxd/pool/remove_test.go index 67f7e3ab..0ae1b39c 100644 --- a/sandboxd/pool/remove_test.go +++ b/sandboxd/pool/remove_test.go @@ -48,7 +48,6 @@ func TestRemovalRetryConvergesConfirmedSurvivor(t *testing.T) { t.Fatal("surviving VM was not retained for retry") } - // A failed retry re-queues itself, so the next tick tries again. m.retryRemovals(t.Context()).Wait() m.mu.Lock() _, pending = m.pendingRemovals["survivor"] diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index 9cd25b21..1a2fa9f9 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -11,8 +11,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// SetPools replaces the node's desired warm targets. Existing claims are not -// affected; only unclaimed warm VMs are trimmed or refilled. +// SetPools replaces the node's desired warm targets; existing claims are unaffected. func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { desired := make(map[types.PoolKey]config.PoolSpec, len(specs)) hashes := make(map[string]types.PoolKey, len(specs)) @@ -24,8 +23,7 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { if err := spec.ValidateLimits(); err != nil { return fmt.Errorf("%w: %v", ErrBadCount, err) } - // Egress is config-owned (validated at load); accepting it here would - // silently drop it. + // egress is config-owned; accepting it here would silently drop it if spec.Egress != nil { return fmt.Errorf("%w: pool %q: egress is set in the config file, not via the API", ErrBadKey, spec.Template) } @@ -46,8 +44,7 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { now := time.Now() for key, p := range m.pools { spec, ok := desired[key] - // A removed key gets the zero spec: a lingering pool sheds its whole - // policy — a stale archiveAfter would keep archiving its claims. + // a removed key gets the zero spec, so a lingering pool sheds a stale archiveAfter p.applySpec(spec) p.removed = !ok trim = append(trim, p.trimWarm(p.effectiveTarget(now))...) @@ -59,13 +56,12 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { if p := m.pools[key]; p != nil { continue } - p := &pool{key: key} + p := newPool(key) p.applySpec(spec) m.adoptGolden(p) m.pools[key] = p } - // Recompute rather than latch: removing every idle pool turns the - // sweep off again. + // recompute rather than latch: removing every idle pool turns the sweep off again m.idleEnabled = m.idleDefault > 0 m.archiveEnabled = m.archiveAfterDefault > 0 for _, p := range m.pools { @@ -88,8 +84,7 @@ func (m *Manager) SetPools(ctx context.Context, specs []config.PoolSpec) error { return m.poolStore.commit(seq, poolsFile{ConfigSeed: m.configSeedHash, Pools: persisted}) } -// normalizePoolSpec fills the wire defaults: API requests default net/size; -// config files stay explicit (Validate rejects empty there). +// normalizePoolSpec fills the wire defaults; config files stay explicit. func normalizePoolSpec(spec config.PoolSpec) config.PoolSpec { spec.PoolKey = spec.Defaulted() return spec diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index e5c67ffd..b258f6e1 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -9,11 +9,7 @@ import ( "time" ) -// SandboxStats is one sandbox's resource usage. CPUCount and MemTotalBytes come -// from the size tier the VM was booted with. MemUsedBytes is the host VMM -// process's resident set, the only usage signal available without a guest agent; -// MemUsedMeasured is false when no VMM process was found (a hibernated sandbox -// has none), so a zero is never read as idle. +// SandboxStats is one sandbox's resource usage; MemUsedMeasured false means the RSS was unreadable. type SandboxStats struct { ID string `json:"id"` CPUCount int `json:"cpu_count"` @@ -26,9 +22,7 @@ type SandboxStats struct { // Stats reports one live claim's resource usage. func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { - // Snapshot the mutable fields under m.mu — hibernate and archive rewrite - // HibernateSnap and VMName there — then do the /proc read, which touches no - // sandbox state, outside the lock. + // hibernate and archive rewrite HibernateSnap and VMName under m.mu; the /proc read needs none m.mu.Lock() sb := m.claimed[id] if sb == nil { @@ -53,9 +47,7 @@ func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { return st, true } -// vmResidentBytes reports the resident set of the hypervisor process serving -// vmName, read straight from engine.List's VMRecord.PID rather than scanning -// /proc for a matching argv. +// vmResidentBytes reports the resident set of the hypervisor process serving vmName. func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bool) { if vmName == "" { return 0, false diff --git a/sandboxd/pool/stats_test.go b/sandboxd/pool/stats_test.go index 4680dba7..3235a95c 100644 --- a/sandboxd/pool/stats_test.go +++ b/sandboxd/pool/stats_test.go @@ -9,7 +9,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" ) -// TestStatsUnknownSandbox: an unclaimed or bad id must not surface any usage. func TestStatsUnknownSandbox(t *testing.T) { m := newTestManager(t, newFakeEngine()) if _, ok := m.Stats(t.Context(), "sb_missing"); ok { @@ -17,9 +16,6 @@ func TestStatsUnknownSandbox(t *testing.T) { } } -// TestOperatorReadsRaceFreeUnderHibernate: Sandbox and Stats read fields that -// hibernate rewrites under m.mu, so they must snapshot under the same lock. -// Run with -race; without the lock this trips the detector. func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -34,8 +30,7 @@ func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { _ = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) } }) - // Read continuously for the writer's whole run, so the operator reads and - // the lifecycle mutations actually overlap for the detector to see. + wg.Go(func() { for { select { @@ -50,9 +45,6 @@ func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { wg.Wait() } -// TestStatsHibernatedIsUnmeasured: a hibernated claim has no VMM process, so -// MemUsedMeasured must stay false rather than reading a stale or zero RSS — -// and Stats must not even ask the engine, which a hibernated claim can't answer. func TestStatsHibernatedIsUnmeasured(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -77,8 +69,6 @@ func TestStatsHibernatedIsUnmeasured(t *testing.T) { } } -// TestStatsNoPIDIsUnmeasured: engine.List answering with no PID (VM not found, -// or not yet running) must not be read as a zero-byte RSS. func TestStatsNoPIDIsUnmeasured(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -93,10 +83,6 @@ func TestStatsNoPIDIsUnmeasured(t *testing.T) { } } -// TestStatsReadsResidentSetFromPID proves the PID engine.List reports is the -// one actually read: pointing it at this test process's own PID, the -// resident-set read must succeed and land on a real value. /proc is -// Linux-only, so this is skipped elsewhere. func TestStatsReadsResidentSetFromPID(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("/proc/PID/statm is Linux-only") diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index ffddd169..b7e46270 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -11,10 +11,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/egress" ) -// AuditLineCap bounds how much of a relayed request frame the audit tap -// captures and records: enough for the op and its addressing fields, never -// file payloads. The relay's tee and Audit share it, so a captured line is -// always recordable. +// AuditLineCap bounds how much of a relayed request frame the audit tap captures and records. const AuditLineCap = 4096 // Counters is a monotonic snapshot for /metrics; gauges come from Info. @@ -34,8 +31,7 @@ type Counters struct { Unarchives uint64 ArchiveDeletes uint64 - // Nanosecond totals paired with the counters above: avg latency for - // dashboards without histogram machinery. + // nanosecond totals paired with the counters above: avg latency without histograms ClaimNanos uint64 WakeNanos uint64 } @@ -86,8 +82,7 @@ func (m *Manager) Counters() Counters { } } -// TenantClaims counts live claims per configured tenant for /metrics — -// configured tenants only, so the label cardinality stays bounded. +// TenantClaims counts live claims per configured tenant, so label cardinality stays bounded. func (m *Manager) TenantClaims() map[string]int { m.mu.Lock() defer m.mu.Unlock() @@ -98,10 +93,7 @@ func (m *Manager) TenantClaims() map[string]int { return counts } -// Audit records one relayed request frame against a sandbox when the audit -// journal is enabled: the op plus its addressing fields, payloads dropped. -// It is the byte-level adapter over recordAudit for the relay's tee; -// non-relay sources (preview) call recordAudit with the frame directly. +// Audit records one relayed request frame against a sandbox: the op and addressing fields only. func (m *Manager) Audit(ctx context.Context, id string, line []byte) { if m.audit == nil || len(line) > AuditLineCap { return @@ -132,8 +124,7 @@ func (m *Manager) recordAudit(ctx context.Context, id string, frame auditFrame) } } -// recordEgress logs one guarded-egress decision to the audit tap (secret name -// only, never its value) and meters it as an always-on usage event. +// recordEgress logs one guarded-egress decision and meters it as a usage event. func (m *Manager) recordEgress(ctx context.Context, id, tenant string, ev egress.Event) { decision := "deny" if ev.Decision == egress.DecisionAllow { @@ -143,9 +134,7 @@ func (m *Manager) recordEgress(ctx context.Context, id, tenant string, ev egress m.recordUsage(ctx, usageEvent{Event: "egress", ID: id, Tenant: tenant, Reference: ev.Host}) } -// recordUsage appends one billing event; failures are logged, never -// propagated — a claim must not fail because the meter hiccuped (cocoon's -// machine-level ledger remains the audit backstop). +// recordUsage appends one billing event; failures are logged, never propagated. func (m *Manager) recordUsage(ctx context.Context, ev usageEvent) { ev.Time = time.Now() if err := m.usage.append(ev); err != nil { diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index f37bd7fa..1a5a8a06 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -62,7 +62,7 @@ func TestJournalRotationKeepsOneBackup(t *testing.T) { t.Fatalf("newJournal: %v", err) } defer func() { _ = j.close() }() - j.size = journalMaxBytes // force the next append to rotate + j.size = journalMaxBytes if err := j.append(usageEvent{Event: "claim", ID: "sb_1"}); err != nil { t.Fatalf("append: %v", err) @@ -85,7 +85,7 @@ func TestQuotaRefusesClaimsPastCap(t *testing.T) { if _, err := claimAny(t.Context(), m, testKey, time.Hour); !errors.Is(err, ErrQuota) { t.Fatalf("claim past cap: %v, want ErrQuota", err) } - // The refused VM must not leak, and the pool state stays sane. + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { t.Fatalf("Release: %v", err) } @@ -109,8 +109,7 @@ func TestTenantQuotaBindsPerTenant(t *testing.T) { if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil); !errors.Is(err, ErrQuota) { t.Fatalf("acme past its cap: %v, want ErrQuota", err) } - // The node-wide cap (unset here) stays untouched: other tenants and root - // keep claiming while acme is full. + if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "beta", "", nil); err != nil { t.Errorf("beta claim while acme is at cap: %v", err) } diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 90c36512..b05fed94 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -15,19 +15,14 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// templateRecord is a template's meta.json: the id carries the key hash -// (tp_), which is all resolution needs — claims re-derive the hash -// from the requested key, never the reverse. Tenant attributes the promote -// and scopes deletion; empty means the operator (root). +// templateRecord is a template's meta.json; an empty Tenant means the operator (root). type templateRecord struct { ID string `json:"id"` Tenant string `json:"tenant,omitempty"` CreatedAt time.Time `json:"created_at"` } -// Promote publishes a claimed sandbox as a template under (template, parent -// net, parent size); later claims for that key clone from it. Re-promoting the -// same name replaces it, and the caller owns its lifecycle (DeleteTemplate). +// Promote publishes a claimed sandbox as a template under (template, parent net, parent size). func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, tenant string) (types.PoolKey, string, error) { sb, ok := m.resolve(id, cred) if !ok { @@ -44,17 +39,14 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t } key := types.PoolKey{Template: template, Net: sb.Key.Net, Size: sb.Key.Size} if m.pooledHash(key.Hash()) { - // A configured pool owns this key — promoting over it would - // silently change what refills produce. + // a configured pool owns this key; promoting over it would change what refills produce return types.PoolKey{}, "", ErrPooledTemplate } - // Fast-fail before the export; commitTemplate re-checks under the - // template lock, closing the check-then-publish race. + // commitTemplate re-checks under the template lock; this only fast-fails before the export if err := m.checkTemplateOwner(ctx, store.TemplateID(key.Hash()), tenant); err != nil { return types.PoolKey{}, "", err } - // See Fork: the transition lock pins the source snapshot, and a started - // promote must finish even if the caller hangs up. + // the transition lock pins the source snapshot; a started promote must finish uncanceled sb.Transition.Lock() defer sb.Transition.Unlock() ctx = context.WithoutCancel(ctx) @@ -76,10 +68,7 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t return key, digest, nil } -// DeleteTemplate removes a promoted template. Configured pools are refused: -// their goldens are owned by the node config, not an API caller. A tenant -// may delete only templates it promoted — anything else answers -// ErrUnknownTemplate; root (empty tenant) deletes anything. +// DeleteTemplate removes a promoted template; a tenant may delete only what it promoted. func (m *Manager) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant string) error { if err := m.validate(key); err != nil { return err @@ -116,15 +105,12 @@ func (m *Manager) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant return nil } -// TemplateHashes lists the promoted-template key hashes for the mesh's -// template gossip, from the in-memory set — the 1s gossip tick never -// touches the store backend. Sorted: the mesh's unchanged-guard compares -// order-sensitively, and map iteration order would defeat it every tick. +// TemplateHashes lists the promoted-template key hashes for mesh gossip, sorted for its guard. func (m *Manager) TemplateHashes() []string { m.mu.Lock() pooled := make(map[string]struct{}, len(m.pools)) - for key := range m.pools { - pooled[key.Hash()] = struct{}{} + for _, p := range m.pools { + pooled[p.hash] = struct{}{} } m.mu.Unlock() m.tplMu.Lock() @@ -140,16 +126,12 @@ func (m *Manager) TemplateHashes() []string { return hashes } -// HasGolden reports whether this node can provision the key without a cold -// boot — a configured pool golden or a promoted template in the store. The -// tplSet answers without a store read; only a shared-store template promoted -// elsewhere after startup falls through to the backend. +// HasGolden reports whether this node can provision the key without a cold boot. func (m *Manager) HasGolden(ctx context.Context, key types.PoolKey, tenant string) bool { return m.HasPoolGolden(key) || m.HasPromotedTemplate(ctx, key, tenant) } -// HasPoolGolden reports whether a configured pool can serve key from its own -// golden. That golden is what every claim on this node resolves to. +// HasPoolGolden reports whether a configured pool can serve key from its own golden. func (m *Manager) HasPoolGolden(key types.PoolKey) bool { m.mu.Lock() defer m.mu.Unlock() @@ -157,9 +139,7 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { return p != nil && p.goldenDir != "" } -// HasPromotedTemplate reports whether key resolves to a promoted template this -// tenant may claim — resolveGolden's test exactly, so routing never promises -// a golden the claim would then refuse. +// HasPromotedTemplate is resolveGolden's test exactly, so routing never promises a refused golden. func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooledHash(key.Hash()) { return false @@ -183,38 +163,32 @@ func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, te return owner == "" || tenantOwns(tenant, owner) } -// pooledHash reports whether a configured pool occupies this hash — the -// guard is on the HASH, not the key: goldens are stored by hash, so a -// colliding key would reach a pool's golden dir even though the keys differ. +// pooledHash guards on the hash, not the key: a colliding key would reach a pool's golden dir. func (m *Manager) pooledHash(hash string) bool { m.mu.Lock() defer m.mu.Unlock() - for key := range m.pools { - if key.Hash() == hash { + for _, p := range m.pools { + if p.hash == hash { return true } } return false } -// recLock is the per-record mutation lock: template publish/delete serialize -// on it directly; checkpoints take it only for delete/fetch (their ids are -// fresh and unguessable pre-publish). A clone or wake holds it shared, so a -// delete or re-publish swap never runs under an in-flight read. Every call -// counts as a live reference — pair it with recDone (or recDoneEvict) once -// the returned lock is unlocked, so a concurrent evict can never hand a -// waiter a different mutex for the same id. +// recLock takes the per-record lock and a live reference; pair every call with recDone. func (m *Manager) recLock(id string) *sync.RWMutex { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() m.recRefs[id]++ - l, _ := m.recLocks.LoadOrStore(id, &sync.RWMutex{}) - return l.(*sync.RWMutex) + l := m.recLocks[id] + if l == nil { + l = &sync.RWMutex{} + m.recLocks[id] = l + } + return l } -// recDone releases a reference taken by recLock; call after Unlock/RUnlock, -// never before — releasing early is what lets a concurrent recLock observe -// a fresh mutex while this call's lock is still logically held. +// recDone releases a reference taken by recLock; call after Unlock/RUnlock, never before. func (m *Manager) recDone(id string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -225,11 +199,7 @@ func (m *Manager) recDone(id string) { } } -// recDoneEvict is recDone for a caller that just deleted id's record: once -// the reference count drops to zero — no one else holds or awaits this id's -// lock — its recLocks slot is removed so the map can't grow per checkpoint. -// Template ids (tp_, reused on re-promote) never call this; they stay -// cached forever, bounded by the promoted set. +// recDoneEvict is recDone for a just-deleted record: the lock slot goes at zero references. func (m *Manager) recDoneEvict(id string) { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() @@ -237,25 +207,22 @@ func (m *Manager) recDoneEvict(id string) { if m.recRefs[id] <= 0 { delete(m.recRefs, id) delete(m.recEvict, id) - m.recLocks.Delete(id) + delete(m.recLocks, id) return } - // Another holder or waiter is still on this lock; defer the eviction to - // whichever call drops the last reference, so it is not lost here. + // a holder or waiter remains; the call that drops the last reference evicts instead m.recEvict[id] = struct{}{} } -// evictIfPending drops id's lock entry if a delete asked for eviction while a -// reference still held it; callers hold recLocksMu with the count at zero. +// evictIfPending drops id's lock entry if a delete asked for eviction; callers hold recLocksMu. func (m *Manager) evictIfPending(id string) { if _, ok := m.recEvict[id]; ok { delete(m.recEvict, id) - m.recLocks.Delete(id) + delete(m.recLocks, id) } } -// checkTemplateOwner rejects publishing or deleting over another tenant's -// record; a meta failure other than absence refuses rather than fail open. +// checkTemplateOwner rejects publishing or deleting over another tenant's record. func (m *Manager) checkTemplateOwner(ctx context.Context, id, tenant string) error { if tenant == "" { return nil @@ -281,10 +248,7 @@ type goldenResolution struct { release func() } -// resolveGolden resolves a key's clone source: the configured pool's local -// golden (no release), else a promoted template fetched from the store; -// empty dir cold-boots. A cross-tenant or absent record cold-boots; a backend -// failure propagates rather than booting a template name as an image ref. +// resolveGolden resolves a key's clone source: the pool golden, else a promoted template. func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant string) (goldenResolution, error) { m.mu.Lock() var dir string @@ -328,7 +292,6 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant s }, nil } -// publishTemplate exports snap into the store under the key's template id. func (m *Manager) publishTemplate(ctx context.Context, snap string, key types.PoolKey, tenant string) (string, error) { id := store.TemplateID(key.Hash()) staging, err := m.tpls.Stage(id) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 3401326d..f92893b1 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -18,8 +18,7 @@ import ( ) const ( - // Sidecar, not data_dir: the marker travels with the image and survives a - // data_dir wipe. + // sidecar, not data_dir: the marker travels with the image and survives a data_dir wipe volumeDirtySuffix = ".dirty" // Caps the per-mount budget below, so teardown can never hang for long. volumeQuiesceMax = 10 * time.Second @@ -35,8 +34,7 @@ func (v catalogVolume) allowed(tenant string) bool { return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) } -// volumeHolders is one name's live admission state: a writer excludes every -// other claim, readers only exclude a writer. +// volumeHolders is one name's live admission state: a writer excludes every other claim. type volumeHolders struct { writers int readers int @@ -47,17 +45,13 @@ type resolvedVolume struct { applied types.Volume } -// volumeTeardown is what a quiesced claim leaves for after its VM is confirmed -// gone: the admission holds to drop, and the marker paths of the mounts that -// came down cleanly — captured rather than re-resolved later, because a claim -// adopted across a restart can name a catalog entry the new config re-pointed. +// volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { holds []types.Volume clears []string } -// Volumes reports the caller-visible fleet catalog, projected through this -// node's ACL metadata and local path state. Empty tenant means root. +// Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { infos := make([]types.VolumeInfo, 0, len(m.volumes)) for name, volume := range m.volumes { @@ -77,8 +71,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI } infos = append(infos, info) } - // Root can safely see peer-only names. Tenant callers need the local catalog - // metadata so the access list can be applied without gossiping it. + // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { if _, ok := m.volumes[name]; !ok { @@ -106,8 +99,7 @@ func (m *Manager) VolumeNames() []string { return names } -// VolumePlacement checks catalog access and reports whether every named image -// is currently available on this node. Empty tenant means root. +// VolumePlacement checks catalog access and reports whether every named image is local. func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) { if err := m.validate(key); err != nil { return false, err @@ -139,8 +131,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - // An attach-only claim arrives with no mounts to default: re-defaulting - // them here would mount what the caller asked to mount itself. + // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) @@ -155,8 +146,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) } if _, statErr := os.Stat(entry.disk.Path); statErr != nil { - // The operator's path stays server-side, and the refusal stays - // byte-identical to the unknown and forbidden ones. + // the operator's path stays server-side; the refusal matches the unknown one log.WithFunc("pool.resolveVolumes").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return nil, ErrVolumeUnavailable } @@ -168,8 +158,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return resolved, nil } -// confirmVolumesClean refuses a read-only claim of a dirty image: run after -// holds are taken, so the marker is stable and can only mean a crashed writer. +// confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. func (m *Manager) confirmVolumesClean(volumes []resolvedVolume) error { for _, volume := range volumes { if !volume.applied.RW() && volumeDirty(volume.disk.Path) { @@ -179,9 +168,7 @@ func (m *Manager) confirmVolumesClean(volumes []resolvedVolume) error { return nil } -// applyVolumes brings the resolved set up concurrently: cocoon serializes the -// attach per VM, so what overlaps is the CLI spawns, settle waits and guest -// mounts. applied is the request-shaped set, recorded once every mount is up. +// applyVolumes brings the resolved set up concurrently; cocoon serializes the attach per VM. func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes []resolvedVolume, applied []types.Volume) error { switch len(volumes) { case 0: @@ -225,9 +212,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res return nil } -// quiesceVolumes unmounts a claim's writable mounts in reverse order and -// returns the teardown its VM removal must finish; a failed unmount keeps the -// image's marker for a recovering writer, never blocking teardown. +// quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { td := volumeTeardown{holds: sb.Volumes} mounts := writableMounts(sb.Volumes) @@ -260,9 +245,7 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT return td } -// finishVolumeTeardown completes teardown once the VM is confirmed gone: only -// then is the hypervisor's image lock released, so an earlier marker clear or -// hold release would hand the name to a claim the still-live writer blocks. +// finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { if len(td.holds) == 0 { return @@ -276,8 +259,7 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { m.unreserveVolumes(td.holds) } -// reserveVolumes admits one claim's volumes; every name is checked before any -// is taken, so a refusal leaves the registry untouched. Callers hold m.mu. +// reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { for _, volume := range volumes { holders := m.volumeAdmission[volume.Name] @@ -289,8 +271,7 @@ func (m *Manager) reserveVolumes(volumes []types.Volume) error { return nil } -// adoptVolumes counts volumes an adopted claim already holds, with no -// admission check. Callers hold m.mu. +// adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { holders := m.volumeAdmission[volume.Name] @@ -329,8 +310,7 @@ func (m *Manager) unreserveVolumes(volumes []types.Volume) { m.releaseVolumes(volumes) } -// quiesceBudget bounds one quiesce: a slice per writable mount plus one for the -// sync fallback, so a wedged multi-volume guest still gets every umount tried. +// quiesceBudget bounds one quiesce: a slice per writable mount plus one for the sync fallback. func quiesceBudget(mounts int) time.Duration { return min(time.Duration(mounts+1)*engine.VolumeCallTimeout, volumeQuiesceMax) } diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index fbf6af24..acf48feb 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -11,9 +11,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// TestAttachOnlyClaimAttachesWithoutMounting: the whole mount contract belongs -// to the caller, so a writable attach-only claim leaves no marker to clear and -// no mount to quiesce — the device only goes away with the VM. func TestAttachOnlyClaimAttachesWithoutMounting(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") eng := newFakeEngine() @@ -76,7 +73,6 @@ func TestAttachOnlyClaimAttachesEveryVolumeConcurrently(t *testing.T) { {Name: "dataset", AttachOnly: true}, {Name: "cache", Mode: types.VolumeModeRW, AttachOnly: true}, } - // Every attach must be in flight at once: a sequential apply blocks here. var attaches sync.WaitGroup attaches.Add(len(requested)) eng.attachRendezvous = &attaches @@ -215,15 +211,11 @@ func TestClaimWarmAttachFailureKeepsHoldsUntilRemoval(t *testing.T) { } } -// TestFinalizeQuotaFailureKeepsHoldsUntilRemoval: the finalize re-check loses -// the quota race with volumes already mounted, so the loser owes a clean umount. func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch") eng := newFakeEngine() m := newVolumeManager(t, eng, []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) m.maxClaims = 1 - // The volume claim parks in its attach, so the volume-less claim below always - // takes the last slot and the volume claim always loses the finalize re-check. var attaches sync.WaitGroup attaches.Add(2) eng.attachRendezvous = &attaches @@ -247,7 +239,7 @@ func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { t.Fatalf("volume-less claim: %v", err) } eng.mu.Lock() - eng.removeErrFor = vmName // the quota loser's VM survives its removal + eng.removeErrFor = vmName eng.attachRendezvous = nil eng.mu.Unlock() attaches.Done() @@ -294,8 +286,6 @@ func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { } } -// TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant: the tenant limb of -// the same re-check; its loser must leave the tenant with nothing counted. func TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch") eng := newFakeEngine() @@ -350,8 +340,6 @@ func TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant(t *testing.T) { } } -// TestAttachOnlyClaimKeepsAdmissionExclusion: what protects other claims is -// unchanged — only this claim's own mount contract moved to the caller. func TestAttachOnlyClaimKeepsAdmissionExclusion(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") eng := newFakeEngine() diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 2e20d820..3ea6e5ee 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -145,8 +145,6 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), []config.VolumeSpec{{Name: "scratch", Path: path, Writable: true}}) readOnly := []types.Volume{{Name: "scratch"}} - // The reader resolves a clean image; a writable claim then fails between - // that resolve and admission, leaving its marker but no hold behind. resolved, err := m.resolveVolumes(t.Context(), "", readOnly) if err != nil { t.Fatalf("resolveVolumes: %v", err) @@ -182,8 +180,7 @@ func TestVolumeAdmissionMatrix(t *testing.T) { wantErr error }{ {"writer excludes writer", types.VolumeModeRW, types.VolumeModeRW, ErrVolumeBusy}, - // Admission answers first: a live writer is a busy conflict, never the - // recovery verdict its own write-ahead marker would otherwise suggest. + {"writer excludes reader", types.VolumeModeRW, "", ErrVolumeBusy}, {"reader excludes writer", "", types.VolumeModeRW, ErrVolumeBusy}, {"readers share", "", "", nil}, @@ -392,7 +389,7 @@ func TestPendingRemovalHoldsVolumesUntilDrained(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - eng.removeErrFor = sb.VMName // survives removal: the VM still holds the image + eng.removeErrFor = sb.VMName if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Release reported success for a VM that survived removal") @@ -448,8 +445,7 @@ func TestVolumeDirtyFailsClosed(t *testing.T) { if !volumeDirty(image) { t.Error("marked image reads clean") } - // The marker of an image behind a non-directory parent cannot be read at - // all; an unreadable marker must never open the image to readers. + if !volumeDirty(filepath.Join(image, "nested.img")) { t.Error("unreadable marker reads clean") } diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 617cc1f0..9a9e7740 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -97,7 +97,6 @@ func TestClaimProvisionBringsMixedVolumesUpConcurrently(t *testing.T) { {Name: "dataset", Mount: "/volumes/dataset"}, {Name: "cache", Mount: "/cache", Mode: types.VolumeModeRW}, } - // Every attach must be in flight at once: a sequential apply blocks here. var attaches sync.WaitGroup attaches.Add(len(requested)) eng.attachRendezvous = &attaches @@ -324,8 +323,6 @@ func TestClaimProvisionPromotedAppliesVolumesFromTemplate(t *testing.T) { } } -// TestPooledKeyOutranksPromotedTemplate: promote, pool, then restart must not -// arm RequirePromoted against every volume claim for the key. func TestPooledKeyOutranksPromotedTemplate(t *testing.T) { path := writeVolumeImage(t, "data.img", "data") catalog := []config.VolumeSpec{{Name: "data", Path: path}} @@ -520,8 +517,6 @@ func TestVolumeClaimUsageAndScopedSummaries(t *testing.T) { t.Fatal("acme claim usage event not found") } -// TestClaimProvisionRefusesUnavailableVolumesIndistinguishably: unknown, -// forbidden, and vanished refuse alike; any difference enumerates the catalog/ACLs. func TestClaimProvisionRefusesUnavailableVolumesIndistinguishably(t *testing.T) { present := writeVolumeImage(t, "present.img", "present") vanished := filepath.Join(t.TempDir(), "vanished.img") @@ -535,7 +530,7 @@ func TestClaimProvisionRefusesUnavailableVolumesIndistinguishably(t *testing.T) for _, principal := range []string{"", "beta"} { for _, name := range []string{"vanished", "unknown", "private"} { if principal == "" && name == "private" { - continue // root is on every access list, so it has no forbidden case + continue } _, err := m.ClaimProvision(t.Context(), testKey, 0, principal, "", []types.Volume{{Name: name}}) if !errors.Is(err, ErrVolumeUnavailable) { @@ -557,8 +552,6 @@ func TestClaimProvisionRefusesUnavailableVolumesIndistinguishably(t *testing.T) } } -// assertVolumeBringUp pins each volume's own marker→attach→mount order and the -// completeness of the set; cross-volume order is free. func assertVolumeBringUp(t *testing.T, eng *fakeEngine, applied []types.Volume) { t.Helper() ops := eng.volumeOpsLog() diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index ab5cb22f..c953d028 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -6,21 +6,15 @@ import ( ) const ( - // ewmaAlpha weights the newest observation in the arrival-rate and - // provision-lead averages; 0.3 tracks bursts within a few samples - // without whiplashing on a single outlier. + // ewmaAlpha weights the newest observation in the arrival-rate and provision-lead averages ewmaAlpha = 0.3 - // rateDecayTau halves the observed arrival rate roughly every 40s of - // silence, so a quiet pool's target drifts back to its floor instead - // of pinning at the last burst's level. + // rateDecayTau halves the observed arrival rate roughly every 40s of silence rateDecayTau = 60 * time.Second - // leadSafety over-provisions against the measured provision lead: - // arrivals are bursty, not uniform, and a warm miss costs a clone. + // leadSafety over-provisions against the measured lead: arrivals are bursty, not uniform leadSafety = 2.0 ) -// noteArrival folds one claim for this pool into the arrival-rate EWMA. -// Caller holds the manager mutex. +// noteArrival folds one claim into the arrival-rate EWMA. Caller holds the manager mutex. func (p *pool) noteArrival(now time.Time) { if !p.lastArrival.IsZero() { if dt := now.Sub(p.lastArrival).Seconds(); dt > 0 { @@ -30,8 +24,7 @@ func (p *pool) noteArrival(now time.Time) { p.lastArrival = now } -// noteLead folds one measured provision duration (clone → claim-ready) into -// the lead EWMA. Caller holds the manager mutex. +// noteLead folds one provision duration into the lead EWMA. Caller holds the manager mutex. func (p *pool) noteLead(d time.Duration) { if p.lead == 0 { p.lead = d @@ -40,10 +33,7 @@ func (p *pool) noteLead(d time.Duration) { p.lead = time.Duration(ewmaAlpha*float64(d) + (1-ewmaAlpha)*float64(p.lead)) } -// effectiveTarget is the demand-adaptive warm watermark: the configured -// floor, raised toward warmMax when the decayed arrival rate times the -// measured provision lead says the floor would miss. warmMax == 0 means the -// feature is off and the floor is the target. Caller holds the manager mutex. +// effectiveTarget raises the floor toward warmMax by rate × lead. Caller holds the manager mutex. func (p *pool) effectiveTarget(now time.Time) int { if p.warmMax <= 0 { return p.floor diff --git a/sandboxd/pool/watermark_test.go b/sandboxd/pool/watermark_test.go index 6d64c016..2ba4201e 100644 --- a/sandboxd/pool/watermark_test.go +++ b/sandboxd/pool/watermark_test.go @@ -15,8 +15,6 @@ func TestEffectiveTargetTracksDemand(t *testing.T) { t.Fatalf("quiet pool target %d, want the floor", got) } - // A burst of one claim every 100ms → ~10/s; ×0.5s lead ×2 safety = 10, - // capped at warmMax. for i := range 20 { p.noteArrival(now.Add(time.Duration(i) * 100 * time.Millisecond)) } @@ -25,7 +23,6 @@ func TestEffectiveTargetTracksDemand(t *testing.T) { t.Errorf("burst target %d, want warmMax 8", got) } - // Silence decays the rate; minutes later the target is the floor again. if got := p.effectiveTarget(burstEnd.Add(5 * time.Minute)); got != 2 { t.Errorf("post-silence target %d, want the floor", got) } diff --git a/sandboxd/server/metrics.go b/sandboxd/server/metrics.go index 84597d92..22a20535 100644 --- a/sandboxd/server/metrics.go +++ b/sandboxd/server/metrics.go @@ -14,9 +14,7 @@ type SandboxListResponse struct { Sandboxes []pool.SandboxSummary `json:"sandboxes"` } -// handleMetrics renders Prometheus text format by hand — counters and -// gauges only, no client library. Latency rides as *_seconds_total next to -// its count. +// handleMetrics renders Prometheus text format by hand, counters and gauges only. func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { pools, g := s.mgr.Info() c := s.mgr.Counters() diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 637d2330..40776397 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -48,6 +48,7 @@ func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewSe } p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} p.transport = &http.Transport{ + // a pooled guest conn skips PreviewDial, and with it wakeResolved and the Transition lock DisableKeepAlives: true, DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { id, portStr, err := net.SplitHostPort(addr) @@ -102,10 +103,10 @@ func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claim rp := &httputil.ReverseProxy{ Rewrite: func(pr *httputil.ProxyRequest) { pr.Out.URL.Scheme = "http" - // The synthetic host carries PreviewDial's target. + // the synthetic host carries PreviewDial's target pr.Out.URL.Host = fmt.Sprintf("%s:%d", claims.ID, claims.Port) pr.Out.URL.Path = "/" + strings.TrimPrefix(pr.In.URL.Path, "/p/"+r.PathValue("token")+"/") - // Browser credentials for the preview domain must not reach guest code. + // browser credentials for the preview domain must not reach guest code pr.Out.Header.Del("Cookie") pr.Out.Header.Del("Authorization") }, diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 1502f15d..1b96a7fb 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -16,16 +16,12 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// drainGrace bounds how long a finished relay waits for the client to -// consume the tail and close; past it the client conn is force-closed. +// drainGrace bounds how long a finished relay waits for the client to close. const drainGrace = 30 * time.Second var switchingProtocols = []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: silkd\r\nConnection: Upgrade\r\n\r\n") -// CloseRelays force-closes every in-flight relay, refuses new ones, and -// waits for the drain; http.Server.Shutdown does not track hijacked -// connections, and a relay hijacked inside the shutdown window would -// otherwise register after this snapshot and hang the drain forever. +// CloseRelays force-closes in-flight relays; http.Server.Shutdown does not track hijacked conns. func (s *Server) CloseRelays() { s.relayMu.Lock() s.relayClosed = true @@ -42,15 +38,13 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { if !ok { return } - // The cheap header check runs first: waking is expensive and consumes - // the hibernate snapshot, so a non-upgrade GET must not trigger it. + // waking consumes the hibernate snapshot, so a non-upgrade GET must not trigger it if !strings.EqualFold(r.Header.Get("Upgrade"), upgradeProto) { w.Header().Set("Upgrade", upgradeProto) writeErr(w, http.StatusUpgradeRequired, "upgrade to "+upgradeProto+" required") return } - // WakeAgentSocket restores a hibernated VM first, so the relay is the - // transparent wake path: the client's next call just works. + // WakeAgentSocket restores a hibernated VM first, making the relay the transparent wake path sock, err := s.mgr.WakeAgentSocket(r.Context(), r.PathValue("id"), token) switch { case writePoolErr(w, err): @@ -76,8 +70,7 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { s.relay(r.Context(), r.PathValue("id"), client, bufrw.Reader, guest) } -// relay writes the 101 and splices the two connections until the guest side -// finishes (silkd closes after the terminal frame) or the client vanishes. +// relay writes the 101 and splices the conns until silkd closes or the client vanishes. func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBuf *bufio.Reader, guest net.Conn) { s.relayMu.Lock() if s.relayClosed { @@ -98,22 +91,19 @@ func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBu s.relayWG.Done() }() - // The http server may have armed a header-read deadline on this conn. + // the http server may have armed a header-read deadline on this conn _ = client.SetDeadline(time.Time{}) if _, err := client.Write(switchingProtocols); err != nil { return } - // Bytes the client pipelined behind the request head already sit in the - // server's bufio reader; drain exactly those, never more — reading the - // bufio past Buffered() would race the direct conn reads below. + // reading the bufio past Buffered() would race the direct conn reads below clientR := io.Reader(client) if n := clientBuf.Buffered(); n > 0 { clientR = io.MultiReader(io.LimitReader(clientBuf, int64(n)), client) } if s.mgr.AuditEnabled() { - // One connection is one RPC: the first line client→guest is the - // request frame. The tee records it as it flows, then passes through. + // one connection is one RPC, so the first line client→guest is the request frame clientR = &auditTee{r: clientR, record: func(line []byte) { s.mgr.Audit(ctx, id, line) }} @@ -123,23 +113,18 @@ func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBu go func() { defer close(done) _, _ = io.Copy(guest, clientR) - // Client done or vanished — either way hard-close: silkd treats - // disconnect as abort, and the protocol delimits input with explicit - // terminal frames (stdin_close, data_end), so a TCP half-close - // carries no meaning worth relaying through the vsock muxer. + // silkd delimits input with terminal frames, so a TCP half-close carries no meaning _ = guest.Close() }() _, _ = io.Copy(client, guest) utils.CloseWrite(client) - // Wait for the client to consume the tail and close its side; the - // deadline unblocks the splice goroutine if it never does. + // the read deadline unblocks the splice goroutine if the client never closes _ = client.SetReadDeadline(time.Now().Add(drainGrace)) <-done } -// auditTee captures the first newline-terminated line (capped) flowing -// through it, hands it to record once, then degrades to a pass-through. +// auditTee captures the first line flowing through it, then degrades to a pass-through. type auditTee struct { r io.Reader record func([]byte) @@ -147,9 +132,6 @@ type auditTee struct { done bool } -// WriteTo hands the splice back to io.Copy's fast path once the first line -// is captured: it reads through Read until the record fires, then delegates -// the rest of the stream to a direct copy. func (t *auditTee) WriteTo(w io.Writer) (int64, error) { var total int64 buf := make([]byte, 4096) diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 3f2833c6..08801ef3 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -79,9 +79,6 @@ func TestRelayRoundTrip(t *testing.T) { } func TestRelayDeliversPipelinedBytes(t *testing.T) { - // The client sends the request head and the first frame in one write, - // before reading the 101 — the frame lands in the server's bufio reader - // and must still reach the guest. ts, _ := newRelayServer(t, func(c net.Conn) { defer c.Close() r := bufio.NewReader(c) @@ -89,7 +86,7 @@ func TestRelayDeliversPipelinedBytes(t *testing.T) { if err != nil { return } - _, _ = io.WriteString(c, line) // echo proves receipt + _, _ = io.WriteString(c, line) }) conn, err := net.Dial("tcp", ts.Listener.Addr().String()) @@ -116,7 +113,7 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { guestClosed := make(chan struct{}) ts, _ := newRelayServer(t, func(c net.Conn) { defer close(guestClosed) - _, _ = io.Copy(io.Discard, c) // block until the relay closes our side + _, _ = io.Copy(io.Discard, c) }) conn, _ := upgradeConn(t, ts) @@ -130,14 +127,14 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { func TestCloseRelaysDrains(t *testing.T) { ts, srv := newRelayServer(t, func(c net.Conn) { - _, _ = io.Copy(io.Discard, c) // idle guest: keeps the relay open + _, _ = io.Copy(io.Discard, c) }) _, r := upgradeConn(t, ts) closed := make(chan struct{}) go func() { defer close(closed) - _, _ = io.Copy(io.Discard, r) // returns when the relay is torn down + _, _ = io.Copy(io.Discard, r) }() srv.CloseRelays() select { @@ -147,8 +144,6 @@ func TestCloseRelaysDrains(t *testing.T) { } } -// newRelayServer wires a Server whose dialer hands each relay one end of a -// net.Pipe and runs guest on the other end. func newRelayServer(t *testing.T, guest func(net.Conn)) (*httptest.Server, *Server) { t.Helper() dialer := &fakeDialer{dial: func(context.Context, string) (net.Conn, error) { @@ -165,8 +160,6 @@ func newRelayServer(t *testing.T, guest func(net.Conn)) (*httptest.Server, *Serv return ts, srv } -// upgradeConn performs the raw HTTP upgrade handshake; further reads must go -// through the returned reader — it may have buffered bytes past the 101. func upgradeConn(t *testing.T, ts *httptest.Server) (net.Conn, *bufio.Reader) { t.Helper() conn, err := net.Dial("tcp", ts.Listener.Addr().String()) @@ -183,8 +176,6 @@ func upgradeConn(t *testing.T, ts *httptest.Server) (net.Conn, *bufio.Reader) { return conn, r } -// readStatus101 consumes exactly the 101 response head; a 1xx response has -// no body, so http.ReadResponse stops right where the guest bytes begin. func readStatus101(t *testing.T, r *bufio.Reader) { t.Helper() resp, err := http.ReadResponse(r, nil) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 56f2a323..83f5f42a 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -33,9 +33,7 @@ const ( previewTTL = time.Hour ) -// poolErrHTTP maps pool sentinels to their HTTP replies; an empty msg -// surfaces err.Error() (4xx detail the caller can act on), a fixed msg -// avoids echoing internals on lookups. +// an empty msg surfaces err.Error(); a fixed msg avoids echoing internals var poolErrHTTP = []struct { err error code int @@ -60,9 +58,7 @@ var poolErrHTTP = []struct { {pool.ErrUnknownCheckpoint, http.StatusNotFound, "unknown checkpoint"}, } -// Manager is the slice of the pool manager the server consumes. tenant -// parameters attribute created resources and scope listings/deletes; empty -// means the operator (root) — unquotaed, unfiltered. +// Manager is the pool manager slice the server consumes; an empty tenant means operator (root). type Manager interface { ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) @@ -106,8 +102,7 @@ type Dialer interface { DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) } -// Placer names peers for redirect placement and lists the mesh for Lookup; -// nil on a single-node deployment (no mesh). +// Placer names peers for redirect placement and lists the mesh; nil on a single-node deployment. type Placer interface { Candidates(keyHash string) []string VolumeCandidates(keyHash string, names []string) []string @@ -119,15 +114,13 @@ type Placer interface { ConfigMismatches() int } -// CheckpointProber answers which peers hold a checkpoint, asked live on a -// redirect decision rather than gossiped. +// CheckpointProber answers which peers hold a checkpoint, asked live rather than gossiped. type CheckpointProber interface { Owners(ctx context.Context, id string) []string Forget(id string) } -// InfoResponse is the wire reply of GET /v1/info. Peers lists the other nodes' -// data-plane addresses so a client can scatter a Lookup across the cluster. +// InfoResponse is the wire reply of GET /v1/info. type InfoResponse struct { Pools []pool.PoolInfo `json:"pools"` Claimed int `json:"claimed"` @@ -135,16 +128,12 @@ type InfoResponse struct { Archived int `json:"archived"` Draining bool `json:"draining,omitempty"` Peers []string `json:"peers,omitempty"` - // AtCapacity marks refill parked because the node refused another VM, so a - // placer reads a short warm count as "full", not "still filling". + // AtCapacity marks refill parked because the node refused another VM. AtCapacity bool `json:"at_capacity,omitempty"` AtCapacityReason string `json:"at_capacity_reason,omitempty"` } -// PoolUpdateRequest is the wire body of PUT /v1/pools. It replaces the node's -// desired warm targets with the supplied list; omitted pools are drained. -// Lives here, not types/api.go: it embeds config.PoolSpec and types cannot -// import config. +// PoolUpdateRequest is the wire body of PUT /v1/pools; omitted pools are drained. type PoolUpdateRequest struct { Pools []config.PoolSpec `json:"pools"` } @@ -162,20 +151,12 @@ type Server struct { preview *PreviewServer relayMu sync.Mutex - relays map[net.Conn]net.Conn // client conn → guest conn, for forced shutdown + relays map[net.Conn]net.Conn // client conn → guest conn relayClosed bool relayWG sync.WaitGroup } -// New returns a Server; an empty apiToken with no tenants leaves the -// node-level endpoints open (per-sandbox tokens still guard sandbox-scoped -// calls). tenants adds per-tenant tokens next to the root apiToken. -// advertise is this node's data-plane address, returned as a claim's owner -// address. A nil placer disables mesh redirects and a nil prober disables -// checkpoint-owner probing (both single-node). probeKey (see -// peer.DeriveProbeKey) authenticates the checkpoint HEAD probe; empty -// leaves it unauthenticated (an unencrypted mesh has no shared secret to -// build one from). +// New returns a Server; an empty apiToken with no tenants leaves node-level endpoints open. func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Manager, dialer Dialer, placer Placer, prober CheckpointProber, probeKey []byte, preview *PreviewServer) *Server { return &Server{ mgr: mgr, @@ -191,9 +172,7 @@ func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Man } } -// Handler builds the route table. Resource-creating verbs and tenant-scoped -// listings/deletes take the api token or a tenant token; pools, info, metrics, -// and per-sandbox operator reads stay root-only. +// Handler builds the route table. func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) @@ -203,17 +182,14 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) - // Fork and promote create node resources, so they take the same token - // class as a claim; the source sandbox's token rides in the body as the - // ownership proof. + // the source sandbox token rides in the body as the ownership proof mux.HandleFunc("POST /v1/sandboxes/{id}/fork", s.requireToken(s.handleFork)) mux.HandleFunc("POST /v1/sandboxes/{id}/promote", s.requireToken(s.handlePromote)) mux.HandleFunc("POST /v1/sandboxes/{id}/preview", s.requireToken(s.handlePreview)) mux.HandleFunc("POST /v1/sandboxes/{id}/checkpoint", s.requireToken(s.handleCheckpoint)) mux.HandleFunc("POST /v1/checkpoints/{id}/claim", s.requireToken(s.handleClaimCheckpoint)) mux.HandleFunc("GET /v1/checkpoints", s.requireToken(s.handleListCheckpoints)) - // GET streams a whole checkpoint with no tenant scoping, so it is - // operator-only; HEAD is the ownership probe and stays unauthenticated. + // GET streams a whole checkpoint with no tenant scoping; HEAD is the unauthenticated probe mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireRoot(s.handleCheckpointBlob)) mux.HandleFunc("HEAD /v1/checkpoints/{id}/blob", s.handleCheckpointProbe) mux.HandleFunc("DELETE /v1/checkpoints/{id}", s.requireToken(s.handleDeleteCheckpoint)) @@ -247,10 +223,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { return } - // Warm hit here is ownership transfer only. On a warm miss with a mesh, a - // peer that reports a warm sandbox gets the claim via redirect (data plane - // must be direct, so redirect beats proxy); only if no peer has one does - // this node provision (golden clone or cold boot). + // the data plane must be direct, so a warm peer gets the claim by redirect, not proxy sb, err := s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) if errors.Is(err, pool.ErrNoWarm) { if s.redirectClaim(r.Context(), w, req, key, hash, tenant) { @@ -258,8 +231,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { } sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) } - // A full node bounces the claim to a warm peer before answering 429 — - // quota is per node, and a peer with capacity is a better answer. + // quota is per node, so a full node bounces the claim to a peer before answering 429 if errors.Is(err, pool.ErrQuota) && s.placer != nil && !req.NoRedirect && writeRedirect(w, s.placer.Candidates(hash)) { return @@ -319,9 +291,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, } localTemplate := s.mgr.HasPromotedTemplate(ctx, key, tenant) - // Per-node content consistency: a peer's promoted template must not escalate - // volume claims off the pool golden every other claim here resolves to. An - // explicit RequirePromoted still passes through, for the manager to refuse. + // a peer's promoted template must not escalate volume claims off the pool golden pooled := s.mgr.HasPoolGolden(key) var templateOwners []string if s.placer != nil && !pooled { @@ -338,9 +308,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, var owners []string if promoted { - // A shared template store lets a volume holder resolve the template - // even before that node has advertised the newly published hash. The - // no_redirect target re-checks both resources before provisioning. + // a shared template store lets a volume holder resolve a hash it has not advertised yet owners = s.templateOwners(func(probe string) []string { return s.placer.TemplateVolumeOwners(probe, names) }, hash, tenant) @@ -377,10 +345,7 @@ func (s *Server) templateOwners(query func(string) []string, hash, tenant string return owners } -// redirectClaim redirects a warm-miss to a better peer — a warm holder, or the -// template owner when we lack a golden (so we don't cold-boot a nonexistent -// image ref). A no_redirect request must resolve locally, never bounce again, -// to avoid a two-node ping-pong. +// redirectClaim bounces a warm-miss to a warm holder, or the template owner when we lack a golden. func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req types.ClaimRequest, key types.PoolKey, hash, tenant string) bool { if s.placer == nil || req.NoRedirect { return false @@ -393,8 +358,6 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t return len(owners) > 0 && !s.mgr.HasGolden(ctx, key, tenant) && writeRedirect(w, owners) } -// handleSandbox reports one live claim, so a reconcile need not scan the -// whole-node listing. func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") sb, ok := s.mgr.Sandbox(id) @@ -405,8 +368,6 @@ func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, sb) } -// handleSandboxStats reports one sandbox's resource usage. It is the only -// per-sandbox usage surface: /metrics is node- and pool-scoped by design. func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") st, ok := s.mgr.Stats(r.Context(), id) @@ -417,7 +378,6 @@ func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, st) } -// handleSandboxVerb shares credential resolution and error mapping for release, hibernate, and wake. func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id string, cred pool.Cred) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) @@ -432,8 +392,6 @@ func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id } } -// handleFork clones a claimed sandbox into fresh child claims, one -// ClaimResponse per child; this node owns them all. func (s *Server) handleFork(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.ForkRequest](w, r) if !ok { @@ -463,7 +421,6 @@ func (s *Server) handlePromote(w http.ResponseWriter, r *http.Request) { }) } -// handleCheckpoint captures a claimed sandbox's state as a new checkpoint. func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointRequest](w, r) if !ok { @@ -476,11 +433,7 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { }) } -// handleClaimCheckpoint claims a fresh sandbox branched from a checkpoint. -// Tier order: the local claim, then a redirect to a probed owner (zero bytes -// moved, but not authoritative — a peer that missed a delete broadcast still -// answers it holds the record until its own TTL sweep runs), then one peer -// transfer if neither answered. +// handleClaimCheckpoint claims a sandbox from a checkpoint; a probed owner is not authoritative. func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointClaimRequest](w, r) if !ok { @@ -506,9 +459,7 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { writeResult(w, r, "fetch checkpoint", ckptID, "fetch checkpoint failed", err, func() { defer release() w.Header().Set("Content-Type", "application/x-tar") - // Status is committed before the walk, so a mid-stream failure cannot change - // it; the tar's completion marker is what tells the reader the record - // arrived whole, and a short transfer is rejected for lacking it. + // the tar completion marker, not the status, tells the reader the record arrived whole w.WriteHeader(http.StatusOK) if err := peer.TarRecord(dir, meta, w); err != nil { log.WithFunc("server.handleCheckpointBlob").Error(r.Context(), err, "stream checkpoint") @@ -516,10 +467,7 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { }) } -// handleCheckpointProbe answers a peer's HEAD probe. With a probeKey -// configured (an encrypted mesh), the caller must present a fresh MAC over -// the id (peer.ProbeHeader) or the probe is rejected before the metadata -// read; without one, the id remains the only capability. +// handleCheckpointProbe answers a HEAD probe; without a probeKey the id is the only capability. func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if len(s.probeKey) > 0 && !peer.VerifyProbeMAC(s.probeKey, id, r.Header.Get(peer.ProbeHeader)) { @@ -533,8 +481,7 @@ func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) } -// handleListCheckpoints lists this node's checkpoints, newest first — a -// tenant caller sees only its own records. +// handleListCheckpoints lists this node's checkpoints, newest first. func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { ckpts, err := s.mgr.Checkpoints(r.Context(), tenantFrom(r.Context())) if err != nil { @@ -545,10 +492,7 @@ func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, types.CheckpointListResponse{Checkpoints: ckpts}) } -// handleDeleteCheckpoint removes a checkpoint; a tenant caller may delete -// only its own records (anything else is 404). no_forward marks a delete -// arriving from another node's own broadcast, so this one does not -// re-broadcast and loop the fleet forever. +// handleDeleteCheckpoint removes a checkpoint; no_forward stops a broadcast delete from looping. func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) { scope := pool.DeleteFleet if r.URL.Query().Get("no_forward") != "" { @@ -556,10 +500,7 @@ func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) } id := r.PathValue("id") err := s.mgr.DeleteCheckpoint(r.Context(), id, tenantFrom(r.Context()), scope) - // Evict after the record is gone, not before: a probe racing the delete - // would otherwise observe the still-present record and cache it just as the - // pre-delete eviction cleared the entry. Unconditional — a forwarded delete - // that finds nothing here still learned the id's ownership changed. + // evict after the delete: a racing probe would otherwise re-cache the still-present record if s.prober != nil { s.prober.Forget(id) } @@ -568,8 +509,7 @@ func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) }) } -// handleDeleteTemplate removes a promoted template; the key axes ride as -// query parameters, defaulted like a claim's. +// handleDeleteTemplate removes a promoted template; the key axes ride as query parameters. func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() req := types.ClaimRequest{ @@ -579,9 +519,7 @@ func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { } key := req.Key() err := s.mgr.DeleteTemplate(r.Context(), key, tenantFrom(r.Context())) - // Unknown here but owned by a peer per gossip: redirect the SDK to the - // owner. no_redirect mirrors the claim protocol — a redirected retry - // carries it, so the owner answers for itself and never bounces again. + // a redirected retry carries no_redirect, so the owner answers for itself and never bounces if errors.Is(err, pool.ErrUnknownTemplate) && s.placer != nil && q.Get("no_redirect") == "" && writeRedirect(w, s.templateOwners(s.placer.TemplateOwners, key.Hash(), tenantFrom(r.Context()))) { return @@ -591,9 +529,7 @@ func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { }) } -// handleOwner answers whether this node owns the sandbox (used by the SDK's -// Lookup scatter to relocate a handle whose owner address was lost). The -// per-sandbox token both authorizes the query and confirms ownership. +// handleOwner answers whether this node owns the sandbox; the token both authorizes and proves it. func (s *Server) handleOwner(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { @@ -632,9 +568,7 @@ func (s *Server) handleUncordon(w http.ResponseWriter, r *http.Request) { s.handleInfo(w, r) } -// handlePeers lists the cluster's node addresses for the SDK's redirect -// follow and Lookup scatter — cluster topology, not operator state, so any -// valid token (root or tenant) may read it. +// handlePeers lists the cluster's node addresses; topology is readable by any valid token. func (s *Server) handlePeers(w http.ResponseWriter, _ *http.Request) { var peers []string if s.placer != nil { @@ -659,9 +593,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") } -// requireToken guards node-level endpoints, resolving the caller's scope — -// root (the api token) or a tenant name (its configured token) — onto the -// request context; sandbox-scoped endpoints use per-sandbox tokens instead. +// requireToken resolves the caller's scope — root or a tenant name — onto the request context. func (s *Server) requireToken(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { tenant, ok := s.resolveScope(r) @@ -673,8 +605,7 @@ func (s *Server) requireToken(next http.HandlerFunc) http.HandlerFunc { } } -// requireRoot guards operator-only surfaces: a tenant token authenticates -// but is not authorized, so it gets 403, never 401. +// requireRoot guards operator-only surfaces: an authenticated tenant token gets 403, not 401. func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { return s.requireToken(func(w http.ResponseWriter, r *http.Request) { if tenantFrom(r.Context()) != "" { @@ -685,22 +616,17 @@ func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { }) } -// rootRequest reports whether this request presented the root api_token. func (s *Server) rootRequest(r *http.Request) bool { token, ok := bearerToken(r) return ok && s.isRootToken(token) } -// isRootToken reports whether token is the configured root api_token (the -// operator credential). An unset api token matches nothing, so an open node -// grants no operator elevation; tenant tokens never match — they live in -// s.tenants. +// isRootToken reports whether token is the root api_token; an unset api token matches nothing. func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } -// sandboxCred resolves a header bearer token to a Cred. The root api_token -// elevates to Operator: the manager must never see it as a sandbox token. +// sandboxCred resolves a header bearer token to a Cred; the root api_token elevates to Operator. func (s *Server) sandboxCred(token string) pool.Cred { if s.isRootToken(token) { return pool.Cred{Operator: true} @@ -708,14 +634,12 @@ func (s *Server) sandboxCred(token string) pool.Cred { return pool.Cred{Token: token} } -// bodyCred resolves a body-carried sandbox token: absent, on a -// root-authenticated request, is Operator; a tenant must still prove ownership. +// bodyCred resolves a body-carried sandbox token; absent on a root request means Operator. func (s *Server) bodyCred(r *http.Request, bodyToken string) pool.Cred { return pool.Cred{Token: bodyToken, Operator: bodyToken == "" && s.rootRequest(r)} } -// resolveScope matches the bearer token to root ("") or a tenant name. With -// no api token and no tenants the node-level endpoints stay open. +// resolveScope matches the bearer token to root ("") or a tenant name. func (s *Server) resolveScope(r *http.Request) (string, bool) { if s.apiToken == "" && len(s.tenants) == 0 { return "", true diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 7c7f5818..3ace7f56 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -15,13 +15,10 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -// retryAfterSeconds hints when to retry a 503. A heal slot frees when a -// transfer finishes, which can be seconds or minutes, so this bounds a waiting -// client to a few probes a minute rather than inviting a 1-second hammer. +// retryAfterSeconds bounds a waiting client to a few probes a minute. const retryAfterSeconds = "10" -// tenantKey carries the resolved tenant scope ("" = root) on the request -// context, from requireToken to the handlers. +// tenantKey carries the resolved tenant scope ("" = root) on the request context. type tenantKey struct{} func withTenant(ctx context.Context, tenant string) context.Context { @@ -57,9 +54,7 @@ func decodeBody[T any](w http.ResponseWriter, r *http.Request) (T, bool) { return v, true } -// decodeBodyStrict is decodeBody for operator bodies, where a mistyped or -// repeated key must fail instead of silently no-opping; client-facing bodies -// stay lenient (newer SDK fields must not break older nodes). +// decodeBodyStrict is decodeBody for operator bodies, where a mistyped key must fail. func decodeBodyStrict[T any](w http.ResponseWriter, r *http.Request) (T, bool) { var v T raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBodyBytes)) @@ -74,14 +69,11 @@ func decodeBodyStrict[T any](w http.ResponseWriter, r *http.Request) (T, bool) { return v, true } -// writePoolErr answers a pool-sentinel error per poolErrHTTP, reporting -// whether it handled err; nil and non-sentinel errors stay the caller's. +// writePoolErr answers a pool-sentinel error, reporting whether it handled err. func writePoolErr(w http.ResponseWriter, err error) bool { for _, m := range poolErrHTTP { if errors.Is(err, m.err) { - // 503 here means the node's own resources are momentarily exhausted - // (e.g. the concurrent-heal budget), not a caller error — Retry-After - // tells the client this is worth retrying, not failing the request. + // a 503 is node exhaustion, not a caller error, so the client should retry if m.code == http.StatusServiceUnavailable { w.Header().Set("Retry-After", retryAfterSeconds) } @@ -103,8 +95,7 @@ func writeResult(w http.ResponseWriter, r *http.Request, op, id, failMsg string, } } -// writeRedirect answers with the redirect shape of the claim protocol when -// addrs is non-empty, reporting whether it did. +// writeRedirect answers with the claim protocol's redirect shape, reporting whether it did. func writeRedirect(w http.ResponseWriter, addrs []string) bool { if len(addrs) == 0 { return false diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 878117f9..a3a1738b 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -304,8 +304,6 @@ func TestTenantAuthMatrix(t *testing.T) { } } -// TestMetricsTenantGauge checks the per-tenant live-claim gauge renders with -// the configured tenants only. func TestMetricsTenantGauge(t *testing.T) { mgr := &fakeManager{tenantClaims: map[string]int{"acme": 2, "beta": 0}} ts := newTestServer(t, "", mgr, nil) @@ -323,8 +321,6 @@ func TestMetricsTenantGauge(t *testing.T) { } } -// TestPutPoolsRejectsUnknownFields: the operator body decodes strictly — a -// mistyped key (e.g. "egres") must 400, not succeed with the policy dropped. func TestPutPoolsRejectsUnknownFields(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, ts.URL+"/v1/pools", @@ -415,7 +411,6 @@ func TestPutPoolsUpdatesTargets(t *testing.T) { } } -// TestSandboxVerbFlows pins shared auth, error mapping, and id/token plumbing for release and hibernate. func TestSandboxVerbFlows(t *testing.T) { verbs := []struct { name string @@ -469,21 +464,16 @@ func TestSandboxVerbFlows(t *testing.T) { } } -// TestReleaseOperatorToken proves the release route's authorization split: the -// node root api_token releases any sandbox by id via an Operator credential (no -// per-sandbox token), while a per-sandbox token still takes the token-checked -// path and a tenant/wrong token gets no operator elevation (it resolves as a -// non-matching sandbox token and 404s). Exactly one hook runs per request. func TestReleaseOperatorToken(t *testing.T) { const rootTok, sbTok = "sekret", "sb-secret" tenants := []config.TenantSpec{{Name: "acme", Token: "acme-tok"}} tests := []struct { name string auth string - wantOp bool // operator (Cred.Operator) release expected - wantRelease bool // per-sandbox release expected - wantToken string // token the per-sandbox release should receive - releaseErr error // error the per-sandbox release returns + wantOp bool + wantRelease bool + wantToken string + releaseErr error want int }{ {"root token releases by id", "Bearer " + rootTok, true, false, "", nil, http.StatusNoContent}, @@ -603,8 +593,7 @@ func TestForkFlow(t *testing.T) { } return children, nil }} - // Fork creates node resources: the header carries the API token like a - // claim, and the sandbox's own token rides in the body. + ts := newTestServer(t, "sekret", mgr, nil) post := func(auth, body string) *http.Response { @@ -741,7 +730,7 @@ func TestPromoteAndDeleteTemplateFlow(t *testing.T) { if got := del("Bearer sekret", "template=nope"); got != http.StatusNotFound { t.Errorf("unknown delete status %d, want 404", got) } - // Node-level endpoint: the API token guards it. + if got := del("", "template=tpl:x"); got != http.StatusUnauthorized { t.Errorf("unauthenticated delete status %d, want 401", got) } @@ -821,10 +810,6 @@ func TestCheckpointFlow(t *testing.T) { } } -// TestDeleteCheckpointNoForwardQueryParam proves the no_forward query param -// reaches the manager, mirroring how handleDeleteTemplate already threads -// no_redirect: a delete arriving from another node's broadcast must not -// re-broadcast, or the fleet loops the delete forever. func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { mgr := &fakeManager{deleteCheckpoint: func(string) error { return nil }} ts := newTestServer(t, "api", mgr, &fakeDialer{}) @@ -857,10 +842,6 @@ func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { } } -// TestDeleteCheckpointForgetsProbeCache proves a delete evicts any cached -// probe answer for the id, whether the delete is the original (fleet scope) -// or a forwarded broadcast (no_forward) that finds nothing to delete locally -// — both cases mean this node just learned id's ownership changed. func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { prober := &fakeProber{} @@ -904,8 +885,6 @@ func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { } func TestClaimRedirectsOnWarmMiss(t *testing.T) { - // Warm miss (fakeManager.ClaimWarm always misses) + a placer with - // candidates → a redirect response, and the local manager never provisions. provisioned := false mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { provisioned = true @@ -936,9 +915,6 @@ func TestClaimRedirectsOnWarmMiss(t *testing.T) { } func TestClaimRedirectsToTemplateOwner(t *testing.T) { - // Warm miss, no warm candidates, no local golden, but gossip names a - // template owner → redirect there instead of cold-booting a nonexistent - // image ref. A local golden suppresses the redirect: provision here. for _, tt := range []struct { name string hasGolden bool @@ -981,9 +957,7 @@ func TestClaimRedirectsToTemplateOwner(t *testing.T) { } func TestDeleteTemplateRedirectsToOwner(t *testing.T) { - // Unknown locally + gossip names an owner → the claim redirect shape; - // unknown everywhere stays 404. - mgr := &fakeManager{} // DeleteTemplate → ErrUnknownTemplate + mgr := &fakeManager{} srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -1008,7 +982,6 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { t.Errorf("redirect %v, want [node-b:7777]", cr.Redirect) } - // no_redirect answers for this node alone, even with owners in gossip. req2, err := http.NewRequestWithContext(t.Context(), http.MethodDelete, ts.URL+"/v1/templates?template=tpl&no_redirect=1", nil) if err != nil { t.Fatalf("request: %v", err) @@ -1040,7 +1013,6 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { } func TestClaimProvisionsWhenNoCandidate(t *testing.T) { - // Warm miss + a placer with no candidates → local provision. mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }} @@ -1058,7 +1030,7 @@ func TestClaimProvisionsWhenNoCandidate(t *testing.T) { if cr.ID != "sb_local" || len(cr.Redirect) != 0 { t.Errorf("got %+v, want local sandbox", cr) } - // The claim_ref on the wire must thread through the handler to the claim. + if mgr.gotClaimRef != "ns1/w1" { t.Errorf("claim_ref not threaded: got %q, want %q", mgr.gotClaimRef, "ns1/w1") } @@ -1253,9 +1225,6 @@ func TestVolumeClaimUsesVolumeAndTemplateIntersections(t *testing.T) { } } -// TestVolumeClaimKeepsPoolContentAgainstPeerTemplates: per-node content -// consistency means a pooled key never asks the mesh what a peer promoted; -// only an explicit RequirePromoted is refused, never served the wrong golden. func TestVolumeClaimKeepsPoolContentAgainstPeerTemplates(t *testing.T) { for _, tt := range []struct { name string @@ -1475,7 +1444,6 @@ func TestVolumeClaimRejectsShapeBeforePlacement(t *testing.T) { } func TestOwnerEndpoint(t *testing.T) { - // AgentSocket succeeds → this node owns it → 200 with owner addr. mgr := &fakeManager{socket: func(id, token string) (string, error) { if token == "good" { return "/v/sock", nil @@ -1516,9 +1484,6 @@ func TestOwnerEndpoint(t *testing.T) { } } -// TestPreviewHandlerZeroDeadlineMintsLiveToken covers a claim with no -// deadline (e.g. archived with ArchiveDeleteAfterSeconds=0): the minted -// preview token must still verify, not be stamped already-expired. func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { mgr := &fakeManager{claimDeadline: func(string, string) (time.Time, error) { return time.Time{}, nil @@ -1551,13 +1516,8 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { } } -// TestCheckpointClaimRedirectsToOwner is L2: checkpoints are node-local, so a -// branch that lands on a node without the record must be pointed at one that -// has it — the clone then runs on the node whose disk already holds the data, -// on its local reflink fast path. Failing here instead would make "snapshot on -// A, branch from B" simply not work. func TestCheckpointClaimRedirectsToOwner(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + mgr := &fakeManager{} prober := &fakeProber{owners: []string{"owner-a:7777"}} ts := newPlacerTestServer(t, "sekret", mgr, prober) @@ -1579,8 +1539,6 @@ func TestCheckpointClaimRedirectsToOwner(t *testing.T) { } } -// TestCheckpointClaimNoRedirectResolvesLocally: the retry at a redirect target -// must resolve there or two nodes would bounce the request between them. func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { prober := &fakeProber{owners: []string{"owner-a:7777"}} ts := newPlacerTestServer(t, "sekret", &fakeManager{}, prober) @@ -1593,11 +1551,8 @@ func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { } } -// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss -// falls through to heal, and a heal miss (item 4) is an honest 404 rather -// than a redirect to nowhere. func TestCheckpointClaimNoOwnersIs404(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpointHeal defaults to ErrUnknownCheckpoint + mgr := &fakeManager{} ts := newPlacerTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) @@ -1611,11 +1566,8 @@ func TestCheckpointClaimNoOwnersIs404(t *testing.T) { } } -// TestCheckpointClaimRedirectBeatsHeal is the tier-order regression test: a -// gossiped owner must redirect (zero bytes moved) and never fall through to -// the heal pull, even though the local ClaimCheckpoint missed. func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + mgr := &fakeManager{} prober := &fakeProber{owners: []string{"owner-a:7777"}} ts := newPlacerTestServer(t, "sekret", mgr, prober) @@ -1637,8 +1589,6 @@ func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { } } -// TestCheckpointClaimFallsBackToHealWhenNoOwner: no gossiped owner to redirect -// to, so the server pulls the record itself via heal. func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { mgr := &fakeManager{ healCheckpoint: func(ckptID string) (*types.Sandbox, error) { @@ -1665,8 +1615,6 @@ func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { } } -// TestCheckpointClaimNoRedirectGoesStraightToHeal: a no_redirect retry must -// not bounce again, but it must still try heal rather than answer 404 outright. func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { mgr := &fakeManager{ healCheckpoint: func(ckptID string) (*types.Sandbox, error) { @@ -1697,9 +1645,6 @@ func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { } } -// TestCheckpointClaimHealBusyIs503WithRetryAfter: a full heal budget is the -// node's own transient exhaustion, not a caller error — the client should -// retry, and Retry-After says so instead of leaving it to guess. func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { mgr := &fakeManager{ healCheckpoint: func(string) (*types.Sandbox, error) { @@ -1719,8 +1664,6 @@ func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { } } -// TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner -// instead of treating a stale gossip entry as a transfer failure. func TestCheckpointBlobUnknownIs404(t *testing.T) { ts := newTestServer(t, "sekret", &fakeManager{}, nil) @@ -1741,8 +1684,6 @@ func TestCheckpointBlobUnknownIs404(t *testing.T) { } } -// TestCheckpointBlobStreamsRecord is the serving half of L3: the record leaves -// as a tar the peer can unpack, with its export contents intact. func TestCheckpointBlobStreamsRecord(t *testing.T) { src := t.TempDir() if err := os.WriteFile(filepath.Join(src, "mem"), []byte("guest-pages"), 0o600); err != nil { @@ -1770,9 +1711,7 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { if err := peer.Untar(resp.Body, dst); err != nil { t.Fatalf("Untar the streamed record: %v", err) } - // The blob is a whole record: meta.json at the root plus the export under - // export/. Streaming only the export produced a directory that published - // cleanly and then failed every read as "unknown checkpoint". + if got, err := os.ReadFile(filepath.Join(dst, "export", "mem")); err != nil || string(got) != "guest-pages" { t.Fatalf("streamed export/mem = %q, %v; want the record's bytes", got, err) } @@ -1781,10 +1720,6 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { } } -// TestCheckpointBlobHeadUnauthenticated proves the probe route bypasses -// requireToken while GET still enforces it. With no probe key configured -// (no cluster_key -- an unencrypted mesh has no shared secret to build one -// from), the id remains the only capability, same as before this batch. func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { const ckptID = "ck_00000000000000aa" mgr := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} @@ -1815,10 +1750,6 @@ func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { } } -// TestCheckpointProbeRequiresValidMAC proves that with a probe key -// configured (an encrypted mesh), handleCheckpointProbe rejects a missing or -// wrongly-signed HEAD before it ever reads HasCheckpoint, and accepts one -// signed with the matching key. func TestCheckpointProbeRequiresValidMAC(t *testing.T) { const ckptID = "ck_00000000000000aa" key := peer.DeriveProbeKey([]byte("cluster-secret")) @@ -1848,16 +1779,12 @@ func TestCheckpointProbeRequiresValidMAC(t *testing.T) { if got := head(peer.SignProbe(key, ckptID)); got != http.StatusOK { t.Errorf("valid signature: status = %d, want 200", got) } - // A valid signature for a DIFFERENT checkpoint must not authorize this one. + if got := head(peer.SignProbe(key, "ck_00000000000000ff")); got != http.StatusUnauthorized { t.Errorf("signature for a different id: status = %d, want 401", got) } } -// TestCheckpointClaimProbesRealPeerAndRedirects is the end-to-end probe: a -// live HTTPProber HEADs an actual peer server rather than a fake, proving the -// probe wire protocol (unauthenticated HEAD, addr scheme defaulting) against -// the real handler, not just an interface stub. func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { const ckptID = "ck_00000000000000aa" mgrB := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} @@ -1866,7 +1793,7 @@ func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { t.Cleanup(tsB.Close) prober := &peer.HTTPProber{Peers: func() []string { return []string{tsB.URL} }} - mgrA := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + mgrA := &fakeManager{} srvA := New("sekret", nil, "node-a:7777", mgrA, &fakeDialer{}, nil, prober, nil, nil) tsA := httptest.NewServer(srvA.Handler()) t.Cleanup(tsA.Close) @@ -1925,8 +1852,6 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS return ts } -// newProbeAuthTestServer builds a server with a probe key configured, for -// tests of handleCheckpointProbe's MAC verification specifically. func newProbeAuthTestServer(t *testing.T, mgr Manager, probeKey []byte) *httptest.Server { t.Helper() srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, probeKey, nil) @@ -1938,8 +1863,6 @@ func newProbeAuthTestServer(t *testing.T, mgr Manager, probeKey []byte) *httptes return ts } -// credToken recovers the token a fake hook expects from cred: empty for an -// Operator credential, mirroring the collapsed manager's own translation. func credToken(cred pool.Cred) string { if cred.Operator { return "" @@ -1947,9 +1870,6 @@ func credToken(cred pool.Cred) string { return cred.Token } -// fakeManager implements Manager with overridable behavior. ClaimWarm misses -// unless warmClaim is set; claim stands in for the provision result. -// Tenant-scoped methods record the tenant they were handed in gotTenant. type fakeManager struct { ckptDir string hasCheckpoint map[string]bool @@ -2016,8 +1936,6 @@ func (f *fakeManager) ClaimProvisionPromoted(ctx context.Context, key types.Pool return fakeClaimProvision(f, ctx, key, ttl, tenant, claimRef, volumes, true) } -// Release dispatches to the operator or per-sandbox hook, mirroring the -// collapsed manager's own Cred split. func (f *fakeManager) Release(_ context.Context, id string, cred pool.Cred) error { if cred.Operator { if f.releaseOp == nil { @@ -2207,8 +2125,6 @@ func (f *fakeManager) Drain(context.Context) { f.draining = true } func (f *fakeManager) Uncordon(context.Context) { f.draining = false } -// FetchCheckpoint serves the peer-transfer read; the fake reports every record -// missing unless a test supplies a directory. func (f *fakeManager) FetchCheckpoint(_ context.Context, _ string) (string, []byte, func(), error) { if f.ckptDir == "" { return "", nil, nil, pool.ErrUnknownCheckpoint @@ -2216,8 +2132,6 @@ func (f *fakeManager) FetchCheckpoint(_ context.Context, _ string) (string, []by return f.ckptDir, []byte(`{"id":"ck_00000000000000aa"}`), func() {}, nil } -// HasCheckpoint answers the probe endpoint straight from the fake's map; -// unset ids report false. func (f *fakeManager) HasCheckpoint(_ context.Context, ckptID string) bool { return f.hasCheckpoint[ckptID] } @@ -2285,9 +2199,6 @@ func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } func (f *fakePlacer) PeerAddrs() []string { return f.addrs } func (f *fakePlacer) ConfigMismatches() int { return 0 } -// fakeProber stubs CheckpointProber with a fixed answer, standing in for a -// live HEAD fan-out. forgotten records every id Forget was called with, so -// tests can assert a delete evicted the right cache entry. type fakeProber struct { owners []string forgotten []string @@ -2296,10 +2207,6 @@ type fakeProber struct { func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } -// newPlacerTestServer builds a server with a checkpoint prober, which the -// shared helper deliberately leaves nil (most tests are single-node); the -// mesh placer stays nil throughout — these tests exercise checkpoint claims -// only. func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, prober CheckpointProber) *httptest.Server { t.Helper() srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, nil, prober, nil, nil) diff --git a/sandboxd/store/dir/dir.go b/sandboxd/store/dir/dir.go index 4bbb3a51..395fa521 100644 --- a/sandboxd/store/dir/dir.go +++ b/sandboxd/store/dir/dir.go @@ -31,8 +31,7 @@ const ( digestWorkerLimit = 8 digestReadBufferSize = 128 << 10 - // generationGrace bounds the shared-mount race where another node is - // still cloning a generation it resolved just before a re-publish. + // generationGrace bounds the race where another node still clones a generation being re-published. generationGrace = time.Hour ) @@ -50,9 +49,7 @@ type chunkHasher func(source *digestSource, chunk int, buffer []byte) error var _ store.Store = (*Store)(nil) -// Store keeps records as //{meta.json,export-/...}: the -// generation dir is immutable and the meta rename is the atomic commit -// pointer, so readers and writers need no cross-process locks. +// Store keeps records at /; the meta.json rename is the atomic commit pointer. type Store struct { root string idRe *regexp.Regexp @@ -69,8 +66,6 @@ func (d *Store) Stage(id string) (string, error) { return os.MkdirTemp(d.root, id+"-*.tmp") } -// Publish makes the generation fresh before its install, refreshes the -// outgoing generation at supersession, and commits by renaming meta.json. func (d *Store) Publish(ctx context.Context, staging, id string) error { return d.publish(ctx, staging, id, "") } @@ -86,9 +81,6 @@ func (d *Store) PublishDigested(ctx context.Context, staging, id string) (string return digest, nil } -// Fetch resolves the committed meta and returns its immutable generation -// directory; release is a no-op. Records published before per-generation -// dirs fall back to the flat export layout. func (d *Store) Fetch(ctx context.Context, id string) (string, []byte, string, func(), error) { meta, err := d.ReadMeta(ctx, id) if err != nil { @@ -128,8 +120,6 @@ func (d *Store) Metas(ctx context.Context) ([][]byte, error) { } var metas [][]byte for _, e := range entries { - // Only well-formed record dirs in this instance's namespace — a - // planted foo/meta.json is not a record. if !e.IsDir() || !d.idRe.MatchString(e.Name()) { continue } @@ -145,8 +135,6 @@ func (d *Store) Metas(ctx context.Context) ([][]byte, error) { return metas, nil } -// Delete removes export generations first and the meta last, so a partially -// failed delete stays discoverable and a retry converges. func (d *Store) Delete(_ context.Context, id string) error { final := filepath.Join(d.root, id) entries, err := os.ReadDir(final) @@ -167,7 +155,6 @@ func (d *Store) Delete(_ context.Context, id string) error { return errors.Join(os.RemoveAll(final), os.RemoveAll(final+oldSuffix)) } -// SweepStaging clears crashed staging and delegates generation retention. func (d *Store) SweepStaging() error { // ReadDir + suffix, not Glob: the root path may hold glob metacharacters. if err := utils.RemoveDirEntries(d.root, func(name string) bool { return strings.HasSuffix(name, ".tmp") }); err != nil { @@ -176,7 +163,6 @@ func (d *Store) SweepStaging() error { return d.SweepGenerations() } -// SweepGenerations reclaims expired record generations without touching staging. func (d *Store) SweepGenerations() error { entries, err := os.ReadDir(d.root) if err != nil { @@ -254,9 +240,7 @@ func (d *Store) publish(_ context.Context, staging, id, digest string) error { return os.RemoveAll(staging) } -// sweepGenerations reclaims only entries whose own install or supersession -// time is outside the grace. Without committed meta it never removes the -// record directory, so a peer publishing into the same path remains safe. +// sweepGenerations reclaims only entries whose install or supersession time is outside the grace. func (d *Store) sweepGenerations(id string) (err error) { final := filepath.Join(d.root, id) metaFile, err := os.Open(filepath.Join(final, store.MetaFile)) //nolint:gosec // id pinned by the instance idRe diff --git a/sandboxd/store/dir/dir_test.go b/sandboxd/store/dir/dir_test.go index 0ecbf2f0..69621ee9 100644 --- a/sandboxd/store/dir/dir_test.go +++ b/sandboxd/store/dir/dir_test.go @@ -24,9 +24,6 @@ func TestDirBackendContract(t *testing.T) { storetest.RunContract(t, st) } -// TestFetchPinsGenerationAcrossStoreInstances: a fetched generation must -// survive a concurrent re-publish from another store instance (another -// process on a shared mount) — retention replaces the old cross-process lock. func TestFetchPinsGenerationAcrossStoreInstances(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -300,8 +297,6 @@ func TestSweepGenerationsPairsCurrentAndSupersededDigests(t *testing.T) { } } -// TestPublishRetriesAfterExpiredInstall: an expired uncommitted generation is -// never revived; GC then retry converges, and the committed retry is idempotent. func TestPublishRetriesAfterExpiredInstall(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -346,9 +341,6 @@ func TestPublishRetriesAfterExpiredInstall(t *testing.T) { } } -// TestFetchLegacyFlatLayout: records published before per-generation dirs -// keep a flat /export; Fetch falls back to it, and a re-publish moves the -// record onto the generation layout. func TestFetchLegacyFlatLayout(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -378,8 +370,6 @@ func TestFetchLegacyFlatLayout(t *testing.T) { } } -// TestPublishSweepsGenerationsBySupersessionAge: rolling re-promotes reclaim -// generations outside their own grace without disturbing newer generations. func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -418,8 +408,6 @@ func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { } } -// TestSweepSparesPublishingGeneration: a peer sweep cannot delete a fresh -// generation between its install and meta commit. func TestSweepSparesPublishingGeneration(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -465,8 +453,6 @@ func TestSweepSparesPublishingGeneration(t *testing.T) { } } -// TestSweepSparesLegacyFallback: the flat export dir of a never-re-published -// legacy record is live data; once a re-publish supersedes it, grace applies. func TestSweepSparesLegacyFallback(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -500,8 +486,6 @@ func TestSweepSparesLegacyFallback(t *testing.T) { } } -// TestSweepReclaimsOnlyAgedUncommittedGenerations: an abandoned generation -// is reclaimed across ids while a peer's uncommitted publish remains intact. func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { const ( orphanID = "ck_00000000000000aa" @@ -574,8 +558,6 @@ func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { } } -// TestSweepGenerationsToleratesConcurrentDelete: a record deleted mid-sweep -// must not fail the pass — the first error aborts every remaining record. func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { const id = "ck_00000000000000aa" st, err := New(t.TempDir(), store.CheckpointIDRe) @@ -624,8 +606,6 @@ func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { } } -// TestDeleteRemovesLegacyResidue: Delete clears a pre-generation .old -// crash artifact along with the record, and is idempotent on a missing id. func TestDeleteRemovesLegacyResidue(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -650,8 +630,6 @@ func TestDeleteRemovesLegacyResidue(t *testing.T) { } } -// TestMetasSurfacesReadError: a corrupt/unreadable meta must fail the listing, -// not vanish silently — mirroring the s3 backend. func TestMetasSurfacesReadError(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -659,7 +637,7 @@ func TestMetasSurfacesReadError(t *testing.T) { if err != nil { t.Fatalf("New: %v", err) } - // A directory where meta.json is expected makes ReadFile fail with EISDIR. + if err := os.MkdirAll(filepath.Join(root, id, store.MetaFile), 0o750); err != nil { t.Fatalf("setup: %v", err) } @@ -672,8 +650,6 @@ func metaJSON(metaID string) string { return `{"id":"` + metaID + `"}` } -// installUncommitted installs metaID's generation without committing -// meta.json — the state a crash between install and commit leaves. func installUncommitted(t *testing.T, st *Store, root, id, metaID string) (meta []byte, gen string) { t.Helper() staging, err := st.Stage(id) @@ -699,8 +675,6 @@ func installUncommitted(t *testing.T, st *Store, root, id, metaID string) (meta return meta, gen } -// mustPublish stages and publishes a one-file export whose meta is -// metaJSON(metaID). func mustPublish(t *testing.T, st *Store, id, metaID string) { t.Helper() staging, err := st.Stage(id) diff --git a/sandboxd/store/peer/broadcast.go b/sandboxd/store/peer/broadcast.go index f7126bea..38d1ddc0 100644 --- a/sandboxd/store/peer/broadcast.go +++ b/sandboxd/store/peer/broadcast.go @@ -16,19 +16,15 @@ const ( defaultDeleteClientTimeout = 10 * time.Second ) -// Broadcaster fans a checkpoint delete out to every peer after a local -// delete, so a record healed onto another node does not outlive the source. +// Broadcaster fans a checkpoint delete out to every peer so a healed copy does not outlive it. type Broadcaster struct { - // A nil Client uses a default with a short timeout: one wedged peer must - // not hold up the fan-out. + // A nil Client uses a default with a short timeout: one wedged peer must not hold up the fan-out. Client *http.Client Peers func() []string Token string } -// Delete sends DELETE ?no_forward=1 to every peer in parallel. Failures are -// logged and never returned: a broadcast is best-effort and must never fail -// the local delete it follows. +// Delete sends DELETE ?no_forward=1 to every peer in parallel; failures are logged, not returned. func (b *Broadcaster) Delete(ctx context.Context, id string) { addrs := dedupAddrs(b.Peers()) if len(addrs) == 0 { diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go index d1aa8ed3..8f5d0fa3 100644 --- a/sandboxd/store/peer/broadcast_test.go +++ b/sandboxd/store/peer/broadcast_test.go @@ -7,9 +7,6 @@ import ( "testing" ) -// TestBroadcastDeleteSendsNoForward: a broadcast delete must carry -// no_forward=1 so the receiving peer's own delete does not re-broadcast, -// which would loop forever across the fleet. func TestBroadcastDeleteSendsNoForward(t *testing.T) { var gotNoForward, gotAuth string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -30,8 +27,6 @@ func TestBroadcastDeleteSendsNoForward(t *testing.T) { } } -// TestBroadcastDeleteFansOutToEveryPeer: every distinct peer must be -// contacted, not just the first. func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { var hits atomic.Int32 newSrv := func() *httptest.Server { @@ -52,8 +47,6 @@ func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { } } -// TestBroadcastDeleteDedupsRepeatedAddrs: a duplicated peer list must not -// double-broadcast to the same address. func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { var hits atomic.Int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -70,9 +63,6 @@ func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { } } -// TestBroadcastDeleteSwallowsFailures: a broadcast never fails the local -// delete it follows, so an unreachable or erroring peer must not panic or -// block the caller past the fan-out. func TestBroadcastDeleteSwallowsFailures(t *testing.T) { fail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) @@ -86,14 +76,12 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { defer ok.Close() b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} - b.Delete(t.Context(), testID) // must return without panicking regardless of peer outcomes + b.Delete(t.Context(), testID) if got := reached.Load(); got != 1 { t.Errorf("healthy peer hit %d times, want 1 (fan-out continues past a failure)", got) } } -// TestBroadcastDeleteNoPeersNoOp: a single-node deployment must not dial -// anything. func TestBroadcastDeleteNoPeersNoOp(t *testing.T) { b := &Broadcaster{Peers: func() []string { return nil }} b.Delete(t.Context(), testID) diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index 2db1a134..f049d79b 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -17,15 +17,10 @@ const healBudget = 30 * time.Minute // Owners resolves a record id to the peer addresses that hold it. type Owners func(id string) []string -// Validate checks a staged pull before Pull trusts the owner that sent it; an -// error tries the next owner. +// Validate checks a staged pull before Pull trusts it; an error tries the next owner. type Validate func(staging string) error -// Healer pulls a record this node does not hold into a caller-provided -// staging directory; the caller validates and publishes it. A nil owners or -// puller makes every Pull a miss (no mesh, no heal). Pull never dedups -// concurrent calls: each caller owns its destination dir, so dedup belongs to -// whoever owns the staging — pool.Manager's per-id heal flight. +// Healer pulls a record this node does not hold into a caller-provided staging directory. type Healer struct { owners Owners puller Puller @@ -34,13 +29,11 @@ type Healer struct { budget time.Duration } -// NewHealer builds a Healer wired to owners and puller. func NewHealer(owners Owners, puller Puller) *Healer { return &Healer{owners: owners, puller: puller} } -// Pull fetches id into staging from the first owner whose transfer validates, -// bounding every owner tried to one budget. +// Pull fetches id into staging from the first owner whose transfer validates. func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate) error { if h.owners == nil || h.puller == nil { return store.ErrNotFound diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go index db03dfa8..5761e4e8 100644 --- a/sandboxd/store/peer/peer_test.go +++ b/sandboxd/store/peer/peer_test.go @@ -15,8 +15,6 @@ import ( const testID = "ck_00000000000000aa" -// TestInertWithoutOwnersOrPuller: a node with no mesh must degrade to unable -// to heal, not fail oddly. Healing is an addition, never a dependency. func TestInertWithoutOwnersOrPuller(t *testing.T) { h := NewHealer(nil, nil) if err := h.Pull(t.Context(), testID, t.TempDir(), nil); !errors.Is(err, store.ErrNotFound) { @@ -24,8 +22,6 @@ func TestInertWithoutOwnersOrPuller(t *testing.T) { } } -// TestPullWritesStaging is Pull's whole point: the record lands in the -// caller-provided staging directory for the caller to validate and publish. func TestPullWritesStaging(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -39,9 +35,6 @@ func TestPullWritesStaging(t *testing.T) { } } -// TestHealTriesNextOwner: a gossiped view can be stale or a peer can be -// broken. One bad owner must not fail the pull while another can still -// serve it. func TestHealTriesNextOwner(t *testing.T) { puller := &fakePuller{ records: map[string]map[string]string{"peer-b:7777": record()}, @@ -57,9 +50,6 @@ func TestHealTriesNextOwner(t *testing.T) { } } -// TestHealAllOwnersMissStaysNotFound: when every owner answers "not found" -// the gossiped view was simply stale. That must surface as store.ErrNotFound -// so the caller's existing not-found handling is unchanged. func TestHealAllOwnersMissStaysNotFound(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -69,7 +59,6 @@ func TestHealAllOwnersMissStaysNotFound(t *testing.T) { } } -// TestHealNoOwners: nothing gossiped the record, so there is nobody to ask. func TestHealNoOwners(t *testing.T) { puller := &fakePuller{} h := NewHealer(func(string) []string { return nil }, puller) @@ -82,9 +71,6 @@ func TestHealNoOwners(t *testing.T) { } } -// TestHealErrorIsReported: a peer that fails for a real reason (not a miss) -// must not be flattened into "not found" — that would hide a broken transfer -// behind a 404 and send the operator looking for a deleted checkpoint. func TestHealErrorIsReported(t *testing.T) { puller := &fakePuller{failAddr: "peer-a:7777"} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -98,9 +84,6 @@ func TestHealErrorIsReported(t *testing.T) { } } -// TestPullDoesNotDedupConcurrentCalls: concurrent Pulls for one id but -// different staging dirs must each run their own transfer — sharing a result -// would leave one caller's dir empty but published as real. func TestPullDoesNotDedupConcurrentCalls(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -128,9 +111,6 @@ func TestPullDoesNotDedupConcurrentCalls(t *testing.T) { } } -// TestPullValidateRejectionTriesNextOwner: an owner whose transferred content -// fails validation is treated like a failed transfer — the next owner gets a -// chance instead of the whole heal aborting on one bad copy. func TestPullValidateRejectionTriesNextOwner(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{ "peer-a:7777": record(), @@ -154,8 +134,6 @@ func TestPullValidateRejectionTriesNextOwner(t *testing.T) { } } -// TestPullValidateRejectsEveryOwner: when every owner's copy fails -// validation, Pull must report the failures, not silently succeed. func TestPullValidateRejectsEveryOwner(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) @@ -166,13 +144,10 @@ func TestPullValidateRejectsEveryOwner(t *testing.T) { } } -// TestPullBudgetBoundsSlowOwners: two owners that each sleep past their -// per-owner slice must not blow past the overall budget — 30 minutes PER -// owner (the old design) would let N slow owners run N*30 minutes. func TestPullBudgetBoundsSlowOwners(t *testing.T) { puller := &sleepyPuller{sleep: 300 * time.Millisecond} h := NewHealer(func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) - h.budget = 200 * time.Millisecond // each owner's slice (100ms) is shorter than the sleep + h.budget = 200 * time.Millisecond start := time.Now() err := h.Pull(t.Context(), testID, t.TempDir(), nil) @@ -184,10 +159,9 @@ func TestPullBudgetBoundsSlowOwners(t *testing.T) { } } -// fakePuller serves records from an in-memory table, recording who was asked. type fakePuller struct { mu sync.Mutex - records map[string]map[string]string // addr → file → content + records map[string]map[string]string asked []string failAddr string } @@ -215,8 +189,6 @@ func (p *fakePuller) Pull(_ context.Context, addr, _, dst string) error { return nil } -// sleepyPuller blocks until ctx expires or sleep elapses, whichever is -// first — for proving a per-owner slice actually cuts a slow owner off. type sleepyPuller struct { sleep time.Duration } diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index ba0554a0..8fcbdc2e 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -22,17 +22,13 @@ const ( maxHealOwners = 8 // a heal wants the widest source list a few full/corrupt owners can't hide behind redirectCacheTTL = 5 * time.Second - // probeGrace bounds how long a fan-out waits for more owners once it has - // its first: a checkpoint usually lives on one node, and without this the - // collector would wait out every missing peer's timeout to return it. + // probeGrace bounds how long a fan-out waits for more owners once it has its first. probeGrace = 150 * time.Millisecond // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. ProbeHeader = "X-Cocoon-Probe" - // probeBucketSeconds is the coarse time step the probe MAC is bound to; - // VerifyProbeMAC accepts the current bucket plus one on either side, so - // clock skew up to a bucket is tolerated without an unbounded replay window. + // probeBucketSeconds is the coarse time step the probe MAC is bound to. probeBucketSeconds = 30 ) @@ -42,18 +38,13 @@ type redirectCacheEntry struct { expires time.Time } -// HTTPProber finds which peers hold a checkpoint by probing them directly: -// ownership is stable but unbounded, so it is queried on the rare cross-node -// miss instead of gossiped every second by every node. +// HTTPProber finds which peers hold a checkpoint by probing them on a cross-node miss. type HTTPProber struct { - // A nil Client uses a default with a 2-second timeout: a probe must never - // hang on a wedged peer. + // A nil Client uses a default 2-second timeout: a probe must never hang on a wedged peer. Client *http.Client Peers func() []string - // ProbeKey signs and verifies HEAD probes (see DeriveProbeKey); empty - // means an unencrypted mesh, which keeps the older capability-only - // posture rather than losing the probe outright. + // ProbeKey signs and verifies HEAD probes; empty means an unencrypted mesh. ProbeKey []byte redirectFlight singleflight.Group @@ -61,19 +52,13 @@ type HTTPProber struct { cacheMu sync.Mutex cache map[string]redirectCacheEntry - // epoch is bumped by every Forget under cacheMu: a fan-out that started - // before a delete must not write its now-suspect result back into the - // cache, so cachePut only commits when the epoch it read still holds. + // epoch is bumped by every Forget, so a fan-out started before a delete cannot cache its result. epoch uint64 // cacheTTL overrides redirectCacheTTL; a test seam, 5 seconds is slow to wait out. cacheTTL time.Duration } -// Owners fans a HEAD out to every peer and returns up to maxRedirectOwners -// addresses that answered 200. Cache staleness fails safely: a redirect to a -// peer that just deleted the record 404s, and the client's no_redirect -// fallback heals from the origin. func (p *HTTPProber) Owners(ctx context.Context, id string) []string { owners, start, hit := p.cacheLookup(id) if hit { @@ -87,21 +72,15 @@ func (p *HTTPProber) Owners(ctx context.Context, id string) []string { return v.([]string) } -// HealOwners is Owners' wide twin: a heal wants the largest source list, so -// a few full or corrupt owners cannot hide a usable one. Uncached — a heal's -// source list should reflect the fleet now, not a redirect-tuned snapshot. +// HealOwners is Owners' wide twin: a heal wants the largest source list, uncached. func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { v, _, _ := p.healFlight.Do(id, func() (any, error) { - // No grace: a heal wants every owner it can find, so it must not stop - // early on a fast one and hide a slower valid source behind it. + // no grace: a heal must not stop early on a fast owner and hide a slower valid source. return p.fanOut(ctx, id, maxHealOwners, 0), nil }) return v.([]string) } -// Forget evicts any cached Owners result for id -- called after any delete -// (original or forwarded broadcast) touches id here, since a stale positive -// would redirect claims to a node that no longer holds the record. func (p *HTTPProber) Forget(id string) { p.cacheMu.Lock() defer p.cacheMu.Unlock() @@ -109,10 +88,7 @@ func (p *HTTPProber) Forget(id string) { delete(p.cache, id) } -// fanOut probes every peer concurrently, canceling stragglers once maxOwners -// have answered 200. The context is detached and rebounded internally: -// singleflight shares one fan-out across callers whose own contexts cancel -// independently — one client disconnecting must not abort another's probe. +// fanOut probes every peer concurrently, canceling stragglers once maxOwners have answered 200. func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace time.Duration) []string { addrs := dedupAddrs(p.Peers()) if len(addrs) == 0 { @@ -136,10 +112,7 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace } go func() { wg.Wait(); close(wins) }() - // grace opens only after the first win: a genuine miss must hear from - // every peer before concluding nobody holds the record. graceCh stays nil - // when grace is 0 (a heal wants the exhaustive list), so the select then - // runs to the cap or until every peer has answered. + // grace opens only after the first win: a genuine miss must hear from every peer. var owners []string var graceCh <-chan time.Time for { @@ -163,9 +136,7 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace } } -// cacheLookup returns id's owners if a still-live positive entry exists, and -// always the current epoch: the caller carries it into the fan-out so cachePut -// can tell whether a Forget landed while the probe was in flight. +// cacheLookup returns id's live owners and the current epoch the caller carries into the fan-out. func (p *HTTPProber) cacheLookup(id string) (owners []string, epoch uint64, hit bool) { p.cacheMu.Lock() defer p.cacheMu.Unlock() @@ -176,10 +147,7 @@ func (p *HTTPProber) cacheLookup(id string) (owners []string, epoch uint64, hit return entry.owners, p.epoch, true } -// cachePut records a positive result, unless a Forget bumped the epoch since -// the fan-out began — that delete may have removed the very record this -// result names. A miss is never cached: a checkpoint appearing moments later -// must not hide behind a stale negative for the TTL. +// cachePut records a positive result unless a Forget bumped the epoch since the fan-out began. func (p *HTTPProber) cachePut(id string, owners []string, start uint64) { if len(owners) == 0 { return @@ -197,24 +165,19 @@ func (p *HTTPProber) cachePut(id string, owners []string, start uint64) { p.cache[id] = redirectCacheEntry{owners: owners, expires: now.Add(cmp.Or(p.cacheTTL, redirectCacheTTL))} } -// DeriveProbeKey derives the probe-MAC key from the mesh's cluster_key via -// one HMAC step — domain separation from the gossip encryption. Call only -// with a non-empty clusterKey. +// DeriveProbeKey derives the probe-MAC key from the mesh cluster_key, separated from gossip. func DeriveProbeKey(clusterKey []byte) []byte { mac := hmac.New(sha256.New, clusterKey) mac.Write([]byte("cocoon-checkpoint-probe-v1")) return mac.Sum(nil) } -// SignProbe returns id's current probe MAC keyed by key — the ProbeHeader -// value; exported for callers and tests that build a valid probe request. +// SignProbe returns id's current probe MAC keyed by key — the ProbeHeader value. func SignProbe(key []byte, id string) string { return probeMAC(key, id, currentBucket()) } -// VerifyProbeMAC checks sig against id's current time bucket and the one on -// either side, tolerating clock skew up to one bucket without an unbounded -// replay window. +// VerifyProbeMAC checks sig against id's current time bucket and the one on either side. func VerifyProbeMAC(key []byte, id, sig string) bool { if sig == "" { return false @@ -242,8 +205,7 @@ func currentBucket() int64 { return time.Now().Unix() / probeBucketSeconds } -// dedupAddrs drops repeated addresses so a stale or duplicated peer list -// never probes the same node twice. +// dedupAddrs drops repeated addresses so a stale peer list never probes the same node twice. func dedupAddrs(addrs []string) []string { seen := make(map[string]struct{}, len(addrs)) out := make([]string, 0, len(addrs)) @@ -257,9 +219,7 @@ func dedupAddrs(addrs []string) []string { return out } -// probeOwner sends a probe MAC when probeKey is set (see DeriveProbeKey); -// an unencrypted mesh (empty probeKey) falls back to the older -// capability-only posture -- the id is unguessable, but nothing else guards it. +// probeOwner sends a probe MAC when probeKey is set; an empty key falls back to capability-only. func probeOwner(ctx context.Context, client *http.Client, probeKey []byte, addr, id string) bool { ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 5d5d93b8..f76fe3ac 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -9,8 +9,6 @@ import ( "time" ) -// TestOwnersReturnsOnly200: a mixed 200/404 answer set redirects only to the -// peer that actually holds the record. func TestOwnersReturnsOnly200(t *testing.T) { ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) @@ -28,7 +26,6 @@ func TestOwnersReturnsOnly200(t *testing.T) { } } -// TestOwnersAllMissIsEmpty: nobody answering 200 means nobody to redirect to. func TestOwnersAllMissIsEmpty(t *testing.T) { miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) @@ -41,9 +38,6 @@ func TestOwnersAllMissIsEmpty(t *testing.T) { } } -// TestOwnersHungPeerExcludedWithoutBlockingOthers: a wedged peer must not -// stall the whole probe past its own timeout, nor keep a healthy peer's -// answer from coming back. func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { block := make(chan struct{}) hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { @@ -67,10 +61,6 @@ func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { } } -// TestOwnersReturnsPromptlyWithOneOwner: the common case is a single owner, so -// the fan-out must return shortly after that owner answers on the grace window, -// not block on a slow peer's full timeout the way it would waiting for a cap it -// will never reach. func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { block := make(chan struct{}) hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { @@ -94,10 +84,6 @@ func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { } } -// TestForgetDuringFlightPreventsStaleCache: a delete that lands while a probe -// is in flight must keep that probe's result out of the cache — the record it -// names may be the one just deleted, and a cached positive would redirect -// claims to a node that no longer holds it. func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { release := make(chan struct{}) inFlight := make(chan struct{}) @@ -106,7 +92,7 @@ func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) probed() - <-release // hold the probe open so Forget can land mid-flight + <-release w.WriteHeader(http.StatusOK) })) defer owner.Close() @@ -117,21 +103,20 @@ func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { go func() { owners = p.Owners(t.Context(), testID); close(done) }() <-inFlight - p.Forget(testID) // the delete lands while the flight is open + p.Forget(testID) close(release) <-done if len(owners) != 1 { t.Fatalf("owners = %v, want the one that answered", owners) } - // The result must not have been cached: a second call re-probes. + p.Owners(t.Context(), testID) if got := hits.Load(); got != 2 { t.Errorf("owner probed %d times, want 2: a Forget mid-flight must not leave a stale positive cached", got) } } -// TestOwnersDedupsAddrs: a duplicated peer list must probe each address once. func TestOwnersDedupsAddrs(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -150,8 +135,6 @@ func TestOwnersDedupsAddrs(t *testing.T) { } } -// TestOwnersCapsAtThree: a redirect answer must stay small even with a large -// fleet behind the record. func TestOwnersCapsAtThree(t *testing.T) { var addrs []string for range 4 { @@ -168,8 +151,6 @@ func TestOwnersCapsAtThree(t *testing.T) { } } -// TestOwnersCancelsStragglersOnceCapReached: once the redirect cap is met by -// fast peers, a straggler beyond it must not add its own timeout to the result. func TestOwnersCancelsStragglersOnceCapReached(t *testing.T) { var addrs []string for range maxRedirectOwners { @@ -198,8 +179,6 @@ func TestOwnersCancelsStragglersOnceCapReached(t *testing.T) { } } -// TestHealOwnersReturnsMoreThanRedirectCap: a heal wants the widest source -// list, not the redirect's couple of candidates. func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { const want = maxRedirectOwners + 2 var addrs []string @@ -217,16 +196,13 @@ func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { } } -// TestHealOwnersIsExhaustiveDespiteAFastOwner: a heal must collect every owner -// even when one answers fast, so it does not stop on the grace window a redirect -// uses and hide slower valid sources behind the quick one. func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { fast := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer fast.Close() slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - time.Sleep(2 * probeGrace) // answers well after the redirect grace would have closed + time.Sleep(2 * probeGrace) w.WriteHeader(http.StatusOK) })) defer slow.Close() @@ -237,13 +213,11 @@ func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { } } -// TestOwnersCoalescesConcurrentProbes: concurrent redirect probes for one id -// must fan out to each peer once, not once per caller. func TestOwnersCoalescesConcurrentProbes(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) - time.Sleep(50 * time.Millisecond) // widen the window for concurrent callers to overlap + time.Sleep(50 * time.Millisecond) w.WriteHeader(http.StatusOK) })) defer ok.Close() @@ -259,10 +233,6 @@ func TestOwnersCoalescesConcurrentProbes(t *testing.T) { } } -// TestOwnersCacheCollapsesRepeatedRedirects is the hot-checkpoint-behind-a- -// load-balancer case: a fleet of peers, one of them the owner. Ten -// sequential redirect probes for the same id, well within the TTL, must -// fan out to the fleet once total, not once per call. func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { const peers = 5 var hits atomic.Int32 @@ -290,8 +260,6 @@ func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { } } -// TestOwnersCacheExpires: past the TTL, a redirect must re-probe rather than -// serve a result that could be stale forever. func TestOwnersCacheExpires(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -309,8 +277,6 @@ func TestOwnersCacheExpires(t *testing.T) { } } -// TestForgetEvictsCachedEntry: after Forget, the next Owners call must -// re-probe rather than serve the (possibly now-stale) cached answer. func TestForgetEvictsCachedEntry(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -328,8 +294,6 @@ func TestForgetEvictsCachedEntry(t *testing.T) { } } -// TestForgetOtherIDLeavesCacheAlone: Forget must evict only the named id, -// not the whole cache. func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { var hits atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -347,8 +311,6 @@ func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { } } -// TestVerifyProbeMACRejectsWrongKey: a MAC signed with a different key must -// not verify. func TestVerifyProbeMACRejectsWrongKey(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) other := DeriveProbeKey([]byte("different-secret")) @@ -358,8 +320,6 @@ func TestVerifyProbeMACRejectsWrongKey(t *testing.T) { } } -// TestVerifyProbeMACRejectsWrongID: a MAC signed for a different id must not -// verify -- the id itself must be bound into the signature. func TestVerifyProbeMACRejectsWrongID(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) sig := probeMAC(key, testID, currentBucket()) @@ -368,7 +328,6 @@ func TestVerifyProbeMACRejectsWrongID(t *testing.T) { } } -// TestVerifyProbeMACRejectsEmptySignature: a missing header must not verify. func TestVerifyProbeMACRejectsEmptySignature(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) if VerifyProbeMAC(key, testID, "") { @@ -376,8 +335,6 @@ func TestVerifyProbeMACRejectsEmptySignature(t *testing.T) { } } -// TestVerifyProbeMACToleratesAdjacentBucket: a signature from the bucket -// just before or after now must still verify, tolerating clock skew. func TestVerifyProbeMACToleratesAdjacentBucket(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) for _, delta := range []int64{-1, 1} { @@ -388,9 +345,6 @@ func TestVerifyProbeMACToleratesAdjacentBucket(t *testing.T) { } } -// TestVerifyProbeMACRejectsOutsideWindow: a signature two buckets away is -// outside the tolerated skew and must be rejected -- the window is bounded, -// not an unlimited replay allowance. func TestVerifyProbeMACRejectsOutsideWindow(t *testing.T) { key := DeriveProbeKey([]byte("cluster-secret")) sig := probeMAC(key, testID, currentBucket()-2) diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index 8a78d9aa..9ec9c035 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -19,8 +19,7 @@ import ( ) const ( - // A checkpoint carries a guest memory image, so this is generous — but a - // wedged peer must fail the pull so the next owner is tried, not hang it. + // Generous for a guest memory image, but a wedged peer must fail so the next owner is tried. pullTimeout = 30 * time.Minute maxRecordBytes = 1 << 40 // 1 TiB @@ -28,10 +27,7 @@ const ( metaFile = "meta.json" exportPrefix = "export/" - // recordTrailer is written last, only after the whole export streamed, so a - // receiver that reaches EOF without it knows the transfer stopped early — a - // source-side walk or read error leaves a valid but short tar (Close still - // writes the footer), and that must not be published as a complete record. + // recordTrailer is written last, so EOF without it means the transfer stopped early. recordTrailer = ".record-complete" ) @@ -45,8 +41,7 @@ type Puller interface { // HTTPPuller pulls records over sandboxd's control-plane HTTP port. type HTTPPuller struct { - // A nil Client uses a default with no overall timeout: the per-pull - // deadline comes from the context, so a large record is not cut off. + // A nil Client uses a default with no overall timeout; the deadline comes from the context. Client *http.Client Token string } @@ -86,9 +81,7 @@ func (p *HTTPPuller) Pull(ctx context.Context, addr, id, dst string) error { return Untar(resp.Body, dst) } -// TarRecord streams a whole store record: export/ plus meta.json. Both are -// required — the claim path reads the meta's pool key and lineage before it -// ever touches the export, so an export-only copy fails every read. +// TarRecord streams a whole store record: export/ plus meta.json, both required by the claim path. func TarRecord(exportDir string, meta []byte, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() @@ -117,9 +110,7 @@ func TarRecord(exportDir string, meta []byte, w io.Writer) error { return tw.Close() } -// Untar writes a tar stream into dst. An authenticated peer is still not -// allowed to write outside the destination this node chose, so every entry -// name is validated against traversal. +// Untar writes a tar stream into dst, validating every entry name against traversal. func Untar(r io.Reader, dst string) error { tr := tar.NewReader(r) var written int64 @@ -178,8 +169,7 @@ func writeFile(target string, r io.Reader, mode os.FileMode) error { return f.Close() } -// checkpointURL builds the control-plane URL for id's record on addr — the -// one home for the scheme default a mesh member view omits. +// checkpointURL builds the control-plane URL for id's record on addr, defaulting the scheme. func checkpointURL(addr, id, suffix string) string { if !strings.Contains(addr, "://") { addr = "http://" + addr @@ -187,14 +177,12 @@ func checkpointURL(addr, id, suffix string) string { return addr + "/v1/checkpoints/" + url.PathEscape(id) + suffix } -// safeJoin resolves name under root, rejecting absolute paths and any name -// that would escape root. +// safeJoin resolves name under root, rejecting absolute paths and names that escape root. func safeJoin(root, name string) (string, error) { if name == "" || filepath.IsAbs(name) || strings.Contains(name, `\`) { return "", fmt.Errorf("untar: refusing entry %q", name) } - // Clean as-is, never as "/"+name: a leading slash absorbs a leading "..", - // rewriting ../escape to /escape instead of rejecting it. + // clean as-is, never "/"+name: a leading slash absorbs a leading ".." instead of rejecting it. clean := filepath.Clean(name) if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { return "", fmt.Errorf("untar: entry %q escapes destination", name) @@ -206,9 +194,7 @@ func safeJoin(root, name string) (string, error) { return target, nil } -// tarInto walks src into tw under prefix, emitting only regular files and -// directories: a symlink or device node would let the reader be steered outside -// its destination. +// tarInto emits only regular files and directories: a symlink would steer the reader outside dst. func tarInto(src, prefix string, tw *tar.Writer) error { err := filepath.Walk(src, func(path string, fi os.FileInfo, err error) error { if err != nil { diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index 93561919..15638394 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -13,10 +13,6 @@ import ( "testing" ) -// TestTarUntarRoundTrip is the transport's core promise: a record that goes out -// as a tar comes back byte-identical on the other node, directory structure -// included. A checkpoint is a guest memory image plus its meta — a transport -// that silently drops a file would produce a clone of the wrong state. func TestTarUntarRoundTrip(t *testing.T) { src := t.TempDir() files := map[string]string{ @@ -56,10 +52,6 @@ func TestTarUntarRoundTrip(t *testing.T) { } } -// TestUntarRejectsPathTraversal is the security invariant. A peer authenticates -// with the fleet token, but an authenticated peer is still not allowed to write -// outside the destination this node chose — a compromised or buggy one must not -// be able to overwrite the host's files. func TestUntarRejectsPathTraversal(t *testing.T) { for _, name := range []string{ "../escape", @@ -96,9 +88,6 @@ func TestUntarRejectsPathTraversal(t *testing.T) { } } -// TestUntarSkipsNonDataEntries keeps symlinks and device nodes out of a record. -// A record is data; honoring a symlink entry would be an indirect way to make a -// later write land outside the destination. func TestUntarSkipsNonDataEntries(t *testing.T) { var buf bytes.Buffer tw := tar.NewWriter(&buf) @@ -129,14 +118,12 @@ func TestUntarSkipsNonDataEntries(t *testing.T) { if _, err := os.Lstat(filepath.Join(dst, "evil-link")); err == nil { t.Error("symlink entry was materialized; non-data entries must be skipped") } - // The legitimate entry alongside it still lands. + if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { t.Errorf("meta.json missing after skipping the symlink: %v", err) } } -// TestTarSkipsSymlinksAtSource is the same rule on the sending side: a record -// that somehow contains a symlink must not put one on the wire. func TestTarSkipsSymlinksAtSource(t *testing.T) { src := t.TempDir() if err := os.WriteFile(filepath.Join(src, "real"), []byte("x"), 0o600); err != nil { @@ -162,9 +149,6 @@ func TestTarSkipsSymlinksAtSource(t *testing.T) { } } -// TestHTTPPullerNotFound maps a peer's 404 to ErrNotFound so the caller keeps -// trying the next owner instead of failing the whole heal. A gossiped view can -// be stale — one owner not having the record is normal, not an error. func TestHTTPPullerNotFound(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) @@ -178,8 +162,6 @@ func TestHTTPPullerNotFound(t *testing.T) { } } -// TestHTTPPullerPresentsToken proves the fleet token rides on the pull: the -// serving peer authorizes this exactly like any other control-plane read. func TestHTTPPullerPresentsToken(t *testing.T) { var gotAuth, gotPath string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -210,11 +192,6 @@ func TestHTTPPullerPresentsToken(t *testing.T) { } } -// TestTarRecordRoundTripsAWholeRecord reproduces a bug found in the cluster: -// the blob endpoint tarred only the export directory Fetch() returns, so a -// pulled record published cleanly and then failed every read as "unknown -// checkpoint" — the meta carrying the pool key and lineage was never shipped. -// A record on the wire must be exactly what the local store lays out. func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { exportDir := t.TempDir() for name, content := range map[string]string{ @@ -237,7 +214,6 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { t.Fatalf("Untar: %v", err) } - // meta.json at the record root — without it the record is unreadable. got, err := os.ReadFile(filepath.Join(dst, "meta.json")) if err != nil { t.Fatalf("meta.json missing from the shipped record: %v", err) @@ -245,7 +221,7 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { if string(got) != string(meta) { t.Errorf("meta.json = %q, want %q", got, meta) } - // export/ contents under the export prefix, not at the root. + for name, want := range map[string]string{ "memory-ranges": "guest-pages", "cow.raw": "disk", @@ -264,28 +240,21 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { } } -// TestUntarRejectsRecordWithoutCompletionMarker: a transfer cut short by a -// source-side walk/read error still yields a valid short tar (Close writes the -// footer regardless), so the receiver must reject a record that arrives without -// the trailer rather than publish an incomplete one. func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { var buf bytes.Buffer tw := tar.NewWriter(&buf) - // meta plus one complete export file, then the walk "fails" — no trailer. + _ = tw.WriteHeader(&tar.Header{Name: "meta.json", Mode: 0o600, Size: 4, Typeflag: tar.TypeReg}) _, _ = tw.Write([]byte(`{"a"`)) _ = tw.WriteHeader(&tar.Header{Name: "export/mem", Mode: 0o600, Size: 5, Typeflag: tar.TypeReg}) _, _ = tw.Write([]byte("bytes")) - _ = tw.Close() // valid footer, no trailer entry + _ = tw.Close() if err := Untar(&buf, t.TempDir()); err == nil { t.Error("Untar accepted a record with no completion marker (an incomplete transfer)") } } -// TestTarRecordUntarRoundTripComplete: the real send/receive pair carries the -// trailer, so a whole record is accepted and the marker itself is not written -// to disk (validateHealedCheckpoint would reject an unexpected entry). func TestTarRecordUntarRoundTripComplete(t *testing.T) { exportDir := t.TempDir() if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("state"), 0o600); err != nil { @@ -307,8 +276,6 @@ func TestTarRecordUntarRoundTripComplete(t *testing.T) { } } -// tarDir streams src's contents with no record layout, exercising tarInto and -// Untar on their own. func tarDir(src string, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 6985886e..ab0a4757 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -34,9 +34,7 @@ const ( uploadPartSizeBytes = 16 << 20 ) -// Config selects the bucket and, for MinIO/R2-style endpoints, the -// addressing mode. Credentials come from the standard AWS chain (env, -// IAM role, web identity) — never from sandboxd's config file. +// Config selects the bucket; credentials come from the standard AWS chain, never this config. type Config struct { Bucket string `json:"bucket"` Prefix string `json:"prefix,omitempty"` @@ -58,8 +56,7 @@ type transferManager interface { var _ store.Store = (*Store)(nil) -// Store stages locally under stagingRoot and publishes to the bucket; -// idRe names the instance's id namespace within the shared prefix. +// Store stages locally and publishes to the bucket; idRe names the instance's id namespace. type Store struct { client *awss3.Client tm transferManager @@ -75,8 +72,7 @@ func New(ctx context.Context, cfg Config, stagingRoot string, idRe *regexp.Regex if cfg.Bucket == "" { return nil, fmt.Errorf("s3 checkpoint store needs a bucket") } - // A prefix without its trailing slash would glue onto the id and make - // every record invisible to the delimiter listing. + // a prefix without its trailing slash hides every record from the delimiter listing. if cfg.Prefix != "" && !strings.HasSuffix(cfg.Prefix, "/") { cfg.Prefix += "/" } @@ -93,8 +89,7 @@ func New(ctx context.Context, cfg Config, stagingRoot string, idRe *regexp.Regex } o.UsePathStyle = cfg.ForcePathStyle }) - // Snapshot exports are hundreds of MB: multipart + concurrency keep a - // publish/fetch bandwidth-bound instead of latency-bound. + // snapshot exports are hundreds of MB: multipart concurrency keeps transfers bandwidth-bound. tm := transfermanager.New(client, func(o *transfermanager.Options) { o.PartSizeBytes = uploadPartSizeBytes o.Concurrency = 8 @@ -106,11 +101,6 @@ func (s *Store) Stage(id string) (string, error) { return os.MkdirTemp(s.staging, id+"-*.tmp") } -// Publish uploads every staged file, meta.json last: a lister only sees -// the record once its commit marker exists. Export objects live under a -// per-generation prefix derived from the meta bytes, so a re-publish never -// overwrites keys a concurrent Fetch of the old generation is reading — -// mixed-generation downloads become impossible, not just unlikely. func (s *Store) Publish(ctx context.Context, staging, id string) error { _, err := s.publish(ctx, staging, id, false) return err @@ -120,11 +110,6 @@ func (s *Store) PublishDigested(ctx context.Context, staging, id string) (string return s.publish(ctx, staging, id, true) } -// Fetch materializes the export into a local cache generation keyed by the -// record's meta hash: an unchanged record's repeat fetch is one meta GET, and -// a new generation never disturbs a directory an in-flight clone is reading -// (old generations are reaped at Delete and startup). Concurrent misses share -// one download; release is a no-op; a missing id is ErrNotFound. func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, string, func(), error) { meta, digest, err := s.readMeta(ctx, id) if err != nil { @@ -150,8 +135,7 @@ func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { } func (s *Store) Metas(ctx context.Context) ([][]byte, error) { - // Delimiter listing yields one CommonPrefix per record instead of - // walking every export object of both namespaces under the prefix. + // delimiter listing yields one CommonPrefix per record instead of every export object. var ids []string p := awss3.NewListObjectsV2Paginator(s.client, &awss3.ListObjectsV2Input{ Bucket: &s.bucket, Prefix: &s.prefix, Delimiter: aws.String("/"), @@ -203,14 +187,8 @@ func (s *Store) Delete(ctx context.Context, id string) error { return s.deleteKeys(ctx, []string{metaKey}) } -// SweepStaging clears local staging residue AND stale cache generations — -// it runs at startup, when no clone can be mid-flight. A crash between -// upload and meta.json leaves orphan objects invisible to Metas; an S3 -// lifecycle rule on the bucket reclaims those (documented in deploy). func (s *Store) SweepStaging() error { return utils.RemoveDirEntries(s.staging, nil) } -// SweepGenerations is a no-op: Delete reclaims committed S3 generations, and -// bucket lifecycle policy handles invisible upload orphans. func (s *Store) SweepGenerations() error { return nil } func (s *Store) readMeta(ctx context.Context, id string) ([]byte, string, error) { @@ -271,7 +249,7 @@ func (s *Store) publish(ctx context.Context, staging, id string, digested bool) if err = s.uploadReader(ctx, s.key(id, store.MetaFile), bytes.NewReader(metaRaw), int64(len(metaRaw)), metadata); err != nil { return "", err } - // Keep old generations because another node may have selected the previous meta; Delete reclaims them. + // keep old generations: another node may have selected the previous meta; Delete reclaims them. if err := os.RemoveAll(staging); err != nil { return "", err } @@ -427,8 +405,7 @@ func (s *Store) download(ctx context.Context, key, path string) error { return err } defer func() { _ = f.Close() }() - // DownloadObject, not GetObject: the WriterAt form downloads parts in - // parallel; GetObject's io.Reader is a single sequential stream. + // DownloadObject, not GetObject: the WriterAt form downloads parts in parallel. if _, err := s.tm.DownloadObject(ctx, &transfermanager.DownloadObjectInput{Bucket: &s.bucket, Key: &key, WriterAt: f}); err != nil { return fmt.Errorf("download %s: %w", key, err) } @@ -450,9 +427,7 @@ func (s *Store) deleteKeys(ctx context.Context, keys []string) error { return fmt.Errorf("delete %d objects: %w", len(chunk), err) } for _, e := range out.Errors { - // Strict backends report deleting an absent key per-entry where - // AWS succeeds silently; tolerating it keeps Delete retries - // convergent, matching single-object DeleteObject semantics. + // strict backends error on deleting an absent key where AWS succeeds silently. if code := aws.ToString(e.Code); code == "NoSuchKey" || code == "NoSuchVersion" { continue } diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index 9c70fd74..f80f3fc4 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -31,9 +31,6 @@ func TestS3BackendContract(t *testing.T) { storetest.RunContract(t, st) } -// TestDeleteRetryConverges: a Delete retried after a partial failure re-deletes -// keys already gone; strict backends answer NoSuchKey per entry and the retry -// must still converge instead of failing forever. func TestDeleteRetryConverges(t *testing.T) { const id = "ck_00000000000000bb" fake := &fakeS3{objects: map[string][]byte{ @@ -123,8 +120,6 @@ func TestDeleteRetainsMetaUntilExportsAreGone(t *testing.T) { } } -// TestFetchLegacyExportLayout: records published before per-generation -// export prefixes keep flat export/ keys; Fetch must fall back to them. func TestFetchLegacyExportLayout(t *testing.T) { const id = "ck_00000000000000aa" fake := &fakeS3{objects: map[string][]byte{ @@ -358,10 +353,6 @@ func TestRepublishRetainsGenerationSelectedByAnotherStore(t *testing.T) { } } -// TestS3BackendContractRealEndpoint runs the same contract against a real -// S3 implementation (MinIO on a testbed) when SANDBOX_S3_E2E names its -// endpoint — real list pagination, checksums, and path-style behavior the -// in-process fake cannot vouch for. func TestS3BackendContractRealEndpoint(t *testing.T) { endpoint := os.Getenv("SANDBOX_S3_E2E") if endpoint == "" { @@ -413,7 +404,7 @@ func newTestStore(t *testing.T, fake *fakeS3) *Store { t.Cleanup(ts.Close) t.Setenv("AWS_ACCESS_KEY_ID", "test") t.Setenv("AWS_SECRET_ACCESS_KEY", "test") - // Keep PutObject bodies plain instead of adding aws-chunked trailers the fake would need to decode. + t.Setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_required") t.Setenv("AWS_RESPONSE_CHECKSUM_VALIDATION", "when_required") st, err := New(t.Context(), Config{ diff --git a/sandboxd/store/store.go b/sandboxd/store/store.go index ebbaba7f..76dc203d 100644 --- a/sandboxd/store/store.go +++ b/sandboxd/store/store.go @@ -17,22 +17,15 @@ import ( const ( // ExportDir is the snapshot export under a record: /export. ExportDir = "export" - // MetaFile is the record's metadata: /meta.json. It is - // written/uploaded last, so a lister never sees a half-published - // record. + // MetaFile is the record's metadata, written last so a lister never sees a half-published record. MetaFile = "meta.json" ) var ( - // ErrNotFound is the one absence signal: backends normalize their native - // missing-record errors to it, so call sites can tell "gone" from a - // backend failure (which must never silently degrade to a cold boot). + // ErrNotFound is the absence signal every backend normalizes its missing-record error to. ErrNotFound = errors.New("record not found") - // The two id namespaces sharing one store root: backends filter listings - // by their instance's regexp, so checkpoints and templates coexist in the - // same directory/bucket without seeing each other. The pins also keep a - // crafted id from escaping the store's namespace. + // CheckpointIDRe and TemplateIDRe let both id namespaces share one store root. CheckpointIDRe = regexp.MustCompile(`^ck_[0-9a-f]{16}$`) TemplateIDRe = regexp.MustCompile(`^tp_[0-9a-f]{32}$`) ) @@ -41,47 +34,32 @@ var ( type Store interface { // Stage returns a writable staging directory whose Publish is atomic. Stage(id string) (string, error) - // Publish turns a staged directory into the record, replacing any - // previous generation atomically for listers and never disturbing one a - // concurrent Fetch pinned (generations are immutable and retained until - // Delete or a graced sweep). The caller serializes same-id operations - // in-process; request-path callers pass an uncancelable ctx so a started - // publish finishes. + // Publish atomically replaces the record's previous generation with the staged directory. Publish(ctx context.Context, staging, id string) error // PublishDigested applies Publish semantics and returns the export digest. PublishDigested(ctx context.Context, staging, id string) (string, error) - // Fetch materializes a record's snapshot export as a local directory - // cocoon can clone from, plus the meta and digest resolved on the way, - // and a release to hold until clone ends (the generation outlives it). - // The digest is empty for records written by Publish. + // Fetch materializes a record's export locally and returns a release to hold until the clone ends. Fetch(ctx context.Context, id string) (dir string, meta []byte, digest string, release func(), err error) - // ReadMeta returns a record's metadata, or an error when the record - // does not exist. + // ReadMeta returns a record's metadata, or an error when the record does not exist. ReadMeta(ctx context.Context, id string) ([]byte, error) - // Metas lists the metadata of every record in this instance's id - // namespace. + // Metas lists the metadata of every record in this instance's id namespace. Metas(ctx context.Context) ([][]byte, error) // Delete removes a record. Delete(ctx context.Context, id string) error // SweepStaging removes abandoned staging left by a crash mid-publish. SweepStaging() error - // SweepGenerations runs backend-specific generation retention without - // disturbing staging or active fetch caches. + // SweepGenerations runs generation retention without disturbing staging or fetch caches. SweepGenerations() error } -// CheckpointID, TemplateID, and TemplateHash are the one home for the -// ck_/tp_ id scheme the regexps above pin. +// CheckpointID, TemplateID, and TemplateHash are the one home for the ck_/tp_ id scheme. func CheckpointID(suffix string) string { return "ck_" + suffix } func TemplateID(hash string) string { return "tp_" + hash } func TemplateHash(id string) string { return strings.TrimPrefix(id, "tp_") } -// ExportGen names a record generation's export dir/prefix from its meta -// bytes — meta is unique per publish (checkpoint ids are fresh, template -// records carry created_at), so a re-publish never collides with a -// generation a concurrent Fetch is reading. +// ExportGen names a record generation's export dir from its meta bytes, unique per publish. func ExportGen(meta []byte) string { return ExportDir + "-" + ExportGenHash(meta) } func ExportGenHash(meta []byte) string { diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index b565381a..50125215 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -71,8 +71,7 @@ func RunContract(t *testing.T, st store.Store) { t.Fatalf("SweepStaging: %v", err) } - // Re-publish replaces: the second generation's export fully supersedes - // the first (a leftover first-generation file would corrupt a clone). + // re-publish replaces: a leftover first-generation file would corrupt a clone. second, err := st.Stage(id) if err != nil { t.Fatalf("Stage second: %v", err) diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 880ee138..3fbe93e9 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -4,8 +4,7 @@ import ( "time" ) -// TTLField is the requested-lease field shared by the claiming request -// bodies; zero means the server default. +// TTLField is the shared requested-lease field; zero means the server default. type TTLField struct { TTLSeconds int `json:"ttl_seconds,omitempty"` } @@ -15,25 +14,19 @@ func (f TTLField) TTL() time.Duration { return time.Duration(f.TTLSeconds) * time.Second } -// ClaimRequest is the wire body of POST /v1/claim. NoRedirect is set by the -// SDK on a claim it is retrying at a redirect target, so that node warm-or- -// provisions locally instead of bouncing the claim back on a stale view. +// ClaimRequest is the wire body of POST /v1/claim. type ClaimRequest struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` Size Size `json:"size,omitempty"` Volumes []Volume `json:"volumes,omitempty"` - // VolumesAttachOnly attaches every requested volume without mounting it: - // the workload finds the device by its serial and owns the mount contract. + // VolumesAttachOnly attaches every requested volume without mounting it. VolumesAttachOnly bool `json:"volumes_attach_only,omitempty"` TTLField NoRedirect bool `json:"no_redirect,omitempty"` - // RequirePromoted is carried from a promoted-volume redirect to make the - // target refuse a cold-image fallback when template gossip is stale. + // RequirePromoted makes the target refuse a cold-image fallback. RequirePromoted bool `json:"require_promoted,omitempty"` - // ClaimRef is an opaque caller reference (the aggregated apiserver passes - // the k8s "/") recorded on the claim so the read path can - // map a listed sandbox back to the name it was claimed under. + // ClaimRef is an opaque caller reference recorded on the claim. ClaimRef string `json:"claim_ref,omitempty"` } @@ -42,26 +35,21 @@ func (r ClaimRequest) Key() PoolKey { return PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } -// ClaimResponse is the wire reply of POST /v1/claim. A successful claim -// carries ID/Token/Deadline/OwnerAddr; a mesh miss carries Redirect (peer -// addresses to retry, MOVED-style), and the two are mutually exclusive. +// ClaimResponse is the wire reply of POST /v1/claim; success and Redirect are exclusive. type ClaimResponse struct { ID string `json:"id,omitempty"` Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` OwnerAddr string `json:"owner_addr,omitempty"` - // TemplateDigest is the content identity of the promoted-template export - // this claim was cloned from; empty for any other source. + // TemplateDigest is the content identity of the export this claim was cloned from. TemplateDigest string `json:"template_digest,omitempty"` - // FromCheckpoint names the checkpoint a branched claim was born from, - // so clients can reconstruct the checkpoint tree. + // FromCheckpoint names the checkpoint a branched claim was born from. FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []Volume `json:"volumes,omitempty"` Redirect []string `json:"redirect,omitempty"` - // RequirePromoted tells a redirecting client to preserve that requirement - // on its no_redirect retry. It is omitted from successful claims. + // RequirePromoted tells a redirecting client to preserve that requirement on retry. RequirePromoted bool `json:"require_promoted,omitempty"` } @@ -72,8 +60,7 @@ type VolumeInfo struct { SizeBytes int64 `json:"size_bytes"` Available bool `json:"available"` Nodes int `json:"nodes"` - // Writable reports whether the operator allows rw claims of this name; - // unset for a name known only from a peer's advertisement. + // Writable reports whether the operator allows rw claims of this name. Writable bool `json:"writable,omitempty"` } @@ -82,11 +69,7 @@ type VolumeListResponse struct { Volumes []VolumeInfo `json:"volumes"` } -// ForkRequest is the wire body of POST /v1/sandboxes/{id}/fork. The -// Authorization header carries an api or tenant token (forking creates node -// resources, like a claim); Token proves ownership of the source sandbox. -// TTLSeconds applies to every child; zero means the server default, and -// children never inherit the parent's remainder. +// ForkRequest is the wire body of POST /v1/sandboxes/{id}/fork. type ForkRequest struct { Token string `json:"token"` Count int `json:"count"` @@ -99,7 +82,6 @@ type ForkResponse struct { } // CheckpointRequest is the wire body of POST /v1/sandboxes/{id}/checkpoint. -// Auth mirrors fork: api/tenant token in Authorization, sandbox token here. type CheckpointRequest struct { Token string `json:"token"` Name string `json:"name,omitempty"` @@ -111,11 +93,9 @@ type CheckpointResponse struct { } // CheckpointClaimRequest is the wire body of POST /v1/checkpoints/{id}/claim. -// TTLSeconds semantics match a claim's; zero means the server default. type CheckpointClaimRequest struct { TTLField - // NoRedirect is set by a client retrying at a redirect target, so the retry - // resolves locally instead of bouncing between two nodes. + // NoRedirect makes the retry resolve locally instead of bouncing between two nodes. NoRedirect bool `json:"no_redirect,omitempty"` } @@ -124,23 +104,19 @@ type CheckpointListResponse struct { Checkpoints []Checkpoint `json:"checkpoints"` } -// PromoteRequest is the wire body of POST /v1/sandboxes/{id}/promote; auth -// mirrors ForkRequest (control token in the header, ownership in Token). +// PromoteRequest is the wire body of POST /v1/sandboxes/{id}/promote. type PromoteRequest struct { Token string `json:"token"` Template string `json:"template"` } -// PromoteResponse returns the template's full key and stable identity: -// templates are node-local, so a cluster client needs the exact key (and this -// node's address) to claim from or delete the template later. +// PromoteResponse returns the template's full key and stable identity. type PromoteResponse struct { Key PoolKey `json:"key"` ContentDigest string `json:"content_digest"` } -// PreviewRequest is the wire body of POST /v1/sandboxes/{id}/preview; auth -// mirrors ForkRequest (control token in the header, ownership in Token). +// PreviewRequest is the wire body of POST /v1/sandboxes/{id}/preview. type PreviewRequest struct { Token string `json:"token"` Port uint16 `json:"port"` diff --git a/sandboxd/types/bench_test.go b/sandboxd/types/bench_test.go index c29fc7a0..279e571e 100644 --- a/sandboxd/types/bench_test.go +++ b/sandboxd/types/bench_test.go @@ -6,8 +6,6 @@ import ( "time" ) -// BenchmarkSandboxTouch measures the relay hot-path last-activity stamp; the -// lock-free atomic store must stay flat under -cpu parallelism (no contention). func BenchmarkSandboxTouch(b *testing.B) { sb := &Sandbox{} b.RunParallel(func(pb *testing.PB) { @@ -17,8 +15,6 @@ func BenchmarkSandboxTouch(b *testing.B) { }) } -// BenchmarkSandboxTouchMutex is the pre-E1a baseline (mutex-guarded time.Time) -// for contrast: it degrades with parallelism where the atomic stays flat. func BenchmarkSandboxTouchMutex(b *testing.B) { var ( mu sync.Mutex diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 55fccda5..5d3fb180 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -31,8 +31,7 @@ const ( MaxClaimVolumes = 8 - // Also the guest `mount -o` option literals (engine.MountVolume): renaming - // the API vocabulary would silently change the mount flags. + // Also the guest `mount -o` option literals: renaming them changes the mount flags. VolumeModeRO = "ro" VolumeModeRW = "rw" @@ -42,9 +41,7 @@ const ( ) var ( - // NameRe pins caller-chosen names (templates, checkpoints, tenants), - // which ride in journal fields and metric labels: one conservative - // charset, also accepted by cocoon's snapshot naming. + // NameRe pins caller-chosen names to one conservative charset cocoon also accepts. NameRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) // VolumeNameRe is the virtio disk serial grammar shared with cocoon. VolumeNameRe = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,19}$`) @@ -65,8 +62,7 @@ var ( // NetShape selects whether the Cloud Hypervisor guest has a NIC. type NetShape string -// RestoreMode selects cocoon's clone memory-restore strategy; empty leaves -// cocoon's eager-copy default. +// RestoreMode selects cocoon's clone memory-restore strategy; empty is cocoon's default. type RestoreMode string // Validate accepts the empty default plus cocoon's known restore modes. @@ -82,8 +78,7 @@ func (m RestoreMode) Validate() error { // Size is a T-shirt resource tier. type Size string -// SizeSpec is the concrete allocation behind a tier. Memory is in cocoon flag -// units; MemoryBytes is the same figure for callers that report bytes. +// SizeSpec is the concrete allocation behind a tier; Memory is in cocoon flag units. type SizeSpec struct { CPU int Memory string @@ -96,33 +91,26 @@ func (s Size) Spec() (SizeSpec, bool) { return spec, ok } -// PoolKey identifies one warm pool. Every New() parameter that changes VM -// construction is an axis here. +// PoolKey identifies one warm pool. type PoolKey struct { Template string `json:"template"` Net NetShape `json:"net"` Size Size `json:"size"` } -// Capturable reports whether state capture (fork, checkpoint, promote) is -// allowed: the egress lane refuses it — a resumed capture opens an -// unlocked-NIC window before the tap lock can reapply. +// Capturable is false on the egress lane: a resumed capture opens an unlocked-NIC window. func (k PoolKey) Capturable() bool { return k.Net != NetEgress } -// Defaulted fills the wire defaults: the hardened no-NIC lane and the smallest -// tier — the one home for both the claim and pool-spec paths. +// Defaulted fills the wire defaults: the no-NIC lane and the smallest tier. func (k PoolKey) Defaulted() PoolKey { k.Net = cmp.Or(k.Net, NetNone) k.Size = cmp.Or(k.Size, SizeSmall) return k } -// Hash is a stable digest used in VM, snapshot, and golden-dir naming. -// 128 bits, not a short tag: Promote keys goldens by caller-chosen template -// names, so a targeted collision with a configured pool's hash must stay a -// second-preimage problem, never a brute-forceable one. +// Hash is a stable 128-bit digest for naming; a shorter tag would be brute-forceable. func (k PoolKey) Hash() string { sum := sha256.Sum256([]byte(k.Template + "|" + string(k.Net) + "|" + string(k.Size))) return hex.EncodeToString(sum[:16]) @@ -144,13 +132,6 @@ func (k PoolKey) Validate() error { return nil } -// TemplateGossipHash scopes a key hash to its owner for the gossip wire, so -// foreign templates never match an owner query; the tenant itself never travels. -func TemplateGossipHash(keyHash, tenant string) string { - sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) - return hex.EncodeToString(sum[:16]) -} - // Sandbox is the node-local record of one pooled or claimed VM. type Sandbox struct { ID string `json:"id"` @@ -160,52 +141,36 @@ type Sandbox struct { Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` - // Tenant names the owning tenant; empty means the operator (root) - // claimed it. Fork children inherit it from the parent. + // Tenant names the owning tenant; empty means the operator claimed it. Tenant string `json:"tenant,omitempty"` - // ClaimRef is the opaque caller reference recorded at claim time (the - // aggregated apiserver passes the k8s "/"), carried into - // the operator index so a listed sandbox maps back to its claim name. - // Empty for warm-pool, fork, and checkpoint-branch claims. + // ClaimRef is the opaque caller reference recorded at claim time; empty for pool claims. ClaimRef string `json:"claim_ref,omitempty"` // Volumes records the volumes successfully applied to this claim. Volumes []Volume `json:"volumes,omitempty"` VsockSocket string `json:"vsock_socket,omitempty"` - // TAP is the egress-lane NIC's host tap, captured at provision; empty on - // the none lane and on claims adopted from pre-tap journals. + // TAP is the egress-lane NIC's host tap; empty on the none lane. TAP string `json:"tap,omitempty"` - // HibernateSnap names the memory snapshot while the VM is hibernated; - // empty means running. + // HibernateSnap names the memory snapshot while the VM is hibernated; empty means running. HibernateSnap string `json:"hibernate_snap,omitempty"` - // PendingSnap is the journaled intent of a hibernate in flight: written - // before the engine stops the VM, cleared by the commit. Reconcile - // trusts it to adopt a hibernate whose commit never landed. + // PendingSnap is the journaled intent of a hibernate in flight, cleared by its commit. PendingSnap string `json:"pending_snap,omitempty"` - // ArchiveCk names the store checkpoint holding this sandbox's state while - // archived; empty means live or hibernated. While set, VMName/VsockSocket/ - // HibernateSnap are empty (no local VM) and Deadline is the retention deadline. + // ArchiveCk names the store checkpoint holding an archived sandbox; empty means local. ArchiveCk string `json:"archive_ck,omitempty"` - // FromCheckpoint names the checkpoint this sandbox branched from, for - // lineage; empty for pool and template claims. + // FromCheckpoint names the checkpoint this sandbox branched from; empty otherwise. FromCheckpoint string `json:"from_checkpoint,omitempty"` - // TemplateDigest identifies the promoted-template export this sandbox was - // initially cloned from; empty for every other provisioning source. + // TemplateDigest identifies the promoted-template export this sandbox was cloned from. TemplateDigest string `json:"template_digest,omitempty"` - // StaleSnap names a consumed wake snapshot a lagging journal still - // references; dropped once a later write lands. Guarded by Transition. + // StaleSnap names a consumed wake snapshot a lagging journal references; guarded by Transition. StaleSnap string `json:"-"` - // lastActivity is unix-nanos of the last data-plane connection, for the - // idle policy; lock-free, stamped on the relay hot path. Runtime-only, a - // restart resets it to adoption time. + // lastActivity is unix-nanos of the last data-plane connection, stamped lock-free. lastActivity atomic.Int64 - // Transition serializes hibernate/wake so concurrent wakes collapse onto - // one restore. Lock it before (never under) the manager mutex. + // Transition serializes hibernate/wake; lock it before (never under) the manager mutex. Transition sync.Mutex `json:"-"` } @@ -218,8 +183,7 @@ func (s *Sandbox) TouchAt(t time.Time) { s.lastActivity.Store(t.UnixNano()) } // LastSeen returns the last data-plane activity time. func (s *Sandbox) LastSeen() time.Time { return time.Unix(0, s.lastActivity.Load()) } -// Checkpoint is the record of a captured sandbox state: claims cloned from -// it branch off the exact captured moment. Node-local, like a template. +// Checkpoint is the record of a captured sandbox state; node-local, like a template. type Checkpoint struct { ID string `json:"id"` Name string `json:"name,omitempty"` @@ -228,14 +192,11 @@ type Checkpoint struct { Tenant string `json:"tenant,omitempty"` CreatedAt time.Time `json:"created_at"` - // Archive marks a lifecycle-internal wake image: hidden from listings - // and undeletable on every node sharing the store, not just the owner. + // Archive marks a lifecycle-internal wake image: hidden from listings and undeletable. Archive bool `json:"archive,omitempty"` } -// VMRecord is the subset of cocoon's VM records the control plane reads — -// the same shape backs `vm list --format json` rows and the lifecycle -// commands' `--output json` result. +// VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { State string `json:"state"` PID int `json:"pid"` @@ -262,21 +223,24 @@ type VMConfig struct { Name string `json:"name"` } -// Volume is one requested or applied dataset mount. Mount is empty before -// request validation and on an attach-only entry, whose device the caller -// mounts itself. Mode is normalized to "" (read-only) or VolumeModeRW. +// Volume is one dataset mount; Mode is normalized to "" (read-only) or VolumeModeRW. type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` Mode string `json:"mode,omitempty"` - // AttachOnly carries the claim's flag into the pool's own validation pass; - // past it the empty Mount is the signal, so it is never stored or sent. + // AttachOnly carries the claim's flag into validation; past it the empty Mount is the signal. AttachOnly bool `json:"-"` } // RW reports whether the entry asks for write access. func (v Volume) RW() bool { return v.Mode == VolumeModeRW } +// TemplateGossipHash scopes a key hash to its owner so the tenant never travels the wire. +func TemplateGossipHash(keyHash, tenant string) string { + sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) + return hex.EncodeToString(sum[:16]) +} + // ValidVolumeName reports whether name is a legal cocoon data-disk serial. func ValidVolumeName(name string) bool { return VolumeNameRe.MatchString(name) && !strings.HasPrefix(name, "cocoon-") @@ -315,16 +279,12 @@ func VolumeRWNames(volumes []Volume) []string { return names } -// VolumesAttachOnly reports whether request validation marked the set as the -// caller's to mount. +// VolumesAttachOnly reports whether the caller mounts the devices itself. func VolumesAttachOnly(volumes []Volume) bool { return slices.ContainsFunc(volumes, func(v Volume) bool { return v.AttachOnly }) } -// ValidateVolumes validates a request and returns detached entries with every -// default mount filled and every mode normalized. attachOnly instead leaves -// every mount empty for the caller to mount itself, and rejects an entry that -// carries one. The input is not modified. +// ValidateVolumes returns detached entries with defaults filled and modes normalized. func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { if len(volumes) > MaxClaimVolumes { return nil, fmt.Errorf("volumes must contain at most %d entries, got %d", MaxClaimVolumes, len(volumes)) diff --git a/sandboxd/utils/utils.go b/sandboxd/utils/utils.go index a6ab900b..733a6ea1 100644 --- a/sandboxd/utils/utils.go +++ b/sandboxd/utils/utils.go @@ -20,9 +20,7 @@ func CloseWrite(conn net.Conn) { } } -// WriteFileSync durably replaces path with data (temp + fsync + rename + dir -// fsync) so the rename survives a crash. The temp name is per-call unique, so -// concurrent writers to the same path can't race a shared temp into a lost rename. +// WriteFileSync durably replaces path with data (temp + fsync + rename + dir fsync). func WriteFileSync(path string, data []byte, perm os.FileMode) error { dir := filepath.Dir(path) f, err := os.CreateTemp(dir, filepath.Base(path)+".*.tmp") @@ -83,9 +81,7 @@ func RemoveDirEntries(dir string, match func(name string) bool) error { return nil } -// DecodeStrictJSON decodes one JSON value into v, refusing unknown fields, -// duplicated keys, and trailing data — for hand-edited operator input, where -// a typo must fail instead of silently changing what was configured. +// DecodeStrictJSON decodes one JSON value into v, refusing unknown fields and duplicates. func DecodeStrictJSON(raw []byte, v any) error { if err := rejectDuplicateKeys(json.NewDecoder(bytes.NewReader(raw))); err != nil { return err @@ -101,8 +97,7 @@ func DecodeStrictJSON(raw []byte, v any) error { return nil } -// rejectDuplicateKeys walks one JSON value and refuses objects that repeat a -// key at the same level. +// rejectDuplicateKeys refuses objects that repeat a key at the same level. func rejectDuplicateKeys(dec *json.Decoder) error { tok, err := dec.Token() if err != nil { @@ -120,10 +115,7 @@ func rejectDuplicateKeys(dec *json.Decoder) error { if keyErr != nil { return keyErr } - // Fold case: encoding/json matches struct fields case-insensitively, - // so "Methods" and "methods" target the same field and the second - // silently wins — treat them as one key. (The callers decode into - // structs, never into map[string]T where case is significant.) + // encoding/json matches struct fields case-insensitively, so fold case here. keyStr, _ := keyTok.(string) key := strings.ToLower(keyStr) if _, dup := seen[key]; dup { From 431d55bd006cda673843624ded80486baffcdd8e Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 3 Sep 2026 03:39:36 +0800 Subject: [PATCH 2/6] =?UTF-8?q?review:=20silkd,=20init,=20Python=20SDK=20?= =?UTF-8?q?=E2=80=94=20single-pass=20find=20and=20list,=20outbound=20frame?= =?UTF-8?q?=20cap,=20LSP=20attach=20TTL,=20typed=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fs.find walks the tree in one spawn_blocking and streams batched matches through a bounded channel (10k files on .79: 350–670 ms → 51–76 ms); fs.list scans a directory in one blocking pass; fs.replace skips files over the find bound; the write side enforces the same 8 MiB frame cap every client enforces; watch drains up to 64 events per write; the LSP broker drops its three Arc and reaps a server nothing attaches within 300 s; session ids come from getrandom; init waits for every NIC against one deadline. The Python SDK wraps socket and decode failures as ProtocolError/StreamTimeout, joins the proxy pump before close, and the langchain toolkit states its 5-minute exec bound. --- boot/init/src/boot.rs | 31 +-- boot/init/src/cfg.rs | 2 - docs/langchain.md | 2 +- docs/sdk-python.md | 13 +- docs/silkd.md | 4 +- .../cocoonsandbox_langchain/toolkit.py | 6 +- sdk/python/cocoonsandbox/__init__.py | 3 +- sdk/python/cocoonsandbox/checkpoint.py | 10 +- sdk/python/cocoonsandbox/client.py | 10 +- sdk/python/cocoonsandbox/conn.py | 23 ++- sdk/python/cocoonsandbox/errors.py | 5 + sdk/python/cocoonsandbox/sandbox.py | 16 +- sdk/python/tests/conftest.py | 34 ++++ sdk/python/tests/test_checkpoint.py | 39 ---- sdk/python/tests/test_client.py | 5 +- sdk/python/tests/test_fault_matrix.py | 39 ---- sdk/python/tests/test_fixtures.py | 2 +- sdk/python/tests/test_hardening.py | 6 - sdk/python/tests/test_wire_binding.py | 13 +- silkd/src/exec.rs | 2 + silkd/src/find.rs | 178 +++++++++++------- silkd/src/fs.rs | 68 ++++--- silkd/src/git.rs | 59 +++--- silkd/src/lsp.rs | 122 ++++++------ silkd/src/net.rs | 21 ++- silkd/src/proc.rs | 20 +- silkd/src/proto.rs | 46 +++-- silkd/src/pty.rs | 21 ++- silkd/src/server.rs | 3 +- silkd/src/session.rs | 55 +++--- silkd/src/sysutil.rs | 50 +++-- silkd/src/tree.rs | 10 +- silkd/src/watch.rs | 73 ++++--- silkd/tests/common/mod.rs | 14 +- silkd/tests/exec_e2e.rs | 20 +- silkd/tests/find_e2e.rs | 56 +++++- silkd/tests/forward_e2e.rs | 19 +- silkd/tests/fs_e2e.rs | 3 +- silkd/tests/git_e2e.rs | 10 +- silkd/tests/lsp_e2e.rs | 26 +-- silkd/tests/pty_e2e.rs | 10 +- silkd/tests/session_e2e.rs | 28 +-- silkd/tests/tree_e2e.rs | 11 +- silkd/tests/watch_e2e.rs | 4 +- 44 files changed, 629 insertions(+), 563 deletions(-) create mode 100644 sdk/python/tests/conftest.py diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index bae1325a..22eb62b0 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -43,6 +43,7 @@ impl Marks { } pub fn run() -> ! { + let mut marks = Marks::new(); // Best-effort: if devtmpfs fails there is no console either; later // failures then power off silently, which is still the right end state. let _ = sys::mount( @@ -65,7 +66,7 @@ pub fn run() -> ! { Ok(cfg) => cfg, Err(err) => sys::fatal(&err, cfg::debug_requested(&cmdline)), }; - let mut marks = Marks::new(); + marks.mark("early"); if let Err(err) = assemble(&cfg, &mut marks) { sys::fatal(&err, cfg.debug); } @@ -158,8 +159,9 @@ fn persist_network(cfg: &BootCfg) { eprintln!("sandbox-init: WARN: mkdir {dir}: {err}"); return; } - for ip in &cfg.ips { - let Some(mac) = wait_nic_mac(&ip.device, NIC_TIMEOUT) else { + let devices: Vec<&str> = cfg.ips.iter().map(|ip| ip.device.as_str()).collect(); + for (ip, mac) in cfg.ips.iter().zip(wait_nic_macs(&devices, NIC_TIMEOUT)) { + let Some(mac) = mac else { eprintln!( "sandbox-init: WARN: NIC {} not found, static config skipped", ip.device @@ -173,23 +175,30 @@ fn persist_network(cfg: &BootCfg) { } } -fn wait_nic_mac(device: &str, timeout: Duration) -> Option { - let path = format!("/sys/class/net/{device}/address"); +/// Resolves every NIC's MAC in one sysfs sweep per poll iteration, against one +/// shared deadline — a missing NIC must cost the timeout once, not once each. +fn wait_nic_macs(devices: &[&str], timeout: Duration) -> Vec> { let deadline = Instant::now() + timeout; + let mut found: Vec> = vec![None; devices.len()]; loop { - if let Ok(mac) = fs::read_to_string(&path) { - let mac = mac.trim_end().to_string(); - if !mac.is_empty() && mac != "00:00:00:00:00:00" { - return Some(mac); + for (slot, device) in found.iter_mut().zip(devices) { + if slot.is_none() { + *slot = read_nic_mac(device); } } - if Instant::now() >= deadline { - return None; + if found.iter().all(Option::is_some) || Instant::now() >= deadline { + return found; } std::thread::sleep(POLL_INTERVAL); } } +fn read_nic_mac(device: &str) -> Option { + let mac = fs::read_to_string(format!("/sys/class/net/{device}/address")).ok()?; + let mac = mac.trim_end(); + (!mac.is_empty() && mac != "00:00:00:00:00:00").then(|| mac.to_string()) +} + fn mkdir_all(path: &str) -> Result<(), String> { fs::create_dir_all(path).map_err(|err| format!("mkdir {path}: {err}")) } diff --git a/boot/init/src/cfg.rs b/boot/init/src/cfg.rs index 91c021a7..aa24d1ef 100644 --- a/boot/init/src/cfg.rs +++ b/boot/init/src/cfg.rs @@ -243,7 +243,6 @@ mod tests { assert!(!debug_requested("a sandbox.debug=0 b")); assert!(!debug_requested("a sandbox.debugger b")); assert!(!debug_requested("")); - // Last occurrence wins, like parse(). assert!(!debug_requested("sandbox.debug=1 x sandbox.debug=0")); assert!(debug_requested("sandbox.debug=0 x sandbox.debug")); } @@ -298,7 +297,6 @@ mod tests { #[test] fn parse_cocoon_ip_param() { - // Exact shape BuildIPParams emits, with both DNS servers. let cfg = parse( "cocoon.layers=l0 cocoon.cow=cow \ ip=172.20.100.5::172.20.100.1:255.255.255.0:vm1:eth0:off:8.8.8.8:8.8.4.4", diff --git a/docs/langchain.md b/docs/langchain.md index 04481b5d..aafdb362 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -19,7 +19,7 @@ schemas, sync-native with `asyncio.to_thread` async bridges): | tool | what it does | |---|---| -| `sandbox_exec` | run a shell command; stdout/stderr/exit code; disk state persists across calls | +| `sandbox_exec` | run a shell command, cut off after 5 minutes; stdout/stderr/exit code; disk state persists across calls | | `sandbox_write_file` | write a text file (atomic on the guest) | | `sandbox_read_file` | read a text file | | `sandbox_list_dir` | list a directory as JSON | diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 8848d0e4..99e3c75e 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -237,6 +237,13 @@ records, while `client.checkpoint(id)` creates an entry-node-bound handle for an already known id without listing. `new()` follows an owner redirect and may heal a missing record locally; `delete()` acts on the handle's bound node. +`delete()` also asks every peer that node currently sees to drop any replica a +heal pulled — best-effort eventual cleanup, not a fleet-wide revocation. A peer +that misses the broadcast (offline, partitioned, or joined later) keeps serving +branches from its replica until the node's `checkpoint_ttl_hours` ages it out; +with that TTL at its default of 0 (keep forever), an unreachable peer's replica +has no cleanup bound at all. + ## Language servers (LSP) ```python @@ -244,6 +251,7 @@ lsp = sb.start_lsp("python", "/work") # flavor image provides the server stream = lsp.request() # JSON-RPC byte stream (frame it yourself) # ... speak Content-Length-framed JSON-RPC over stream.send()/recv() ... stream.close() # ends the session and reaps the server +lsp.close() # same as lsp.stop(); both are context-managers ``` `start_lsp` spawns the language server the flavor image ships for the @@ -319,7 +327,7 @@ A session is a real persistent shell: cwd, env and shell state survive across calls. ```python -sess = sb.session(cwd="/work", env={"PATH": "..."}) +sess = sb.session(cwd="/work", env={"PATH": "..."}) # context-manager sess.exec("export", "MARK=1") # persists sess.exec("sh", "-c", "echo $MARK") # "1\n" sess.close() @@ -435,7 +443,8 @@ zero. - `SilkdError(kind, message)` — typed guest failure; `kind` is `bad_request` / `not_found` / `unimplemented` / `internal` - `ExitError(code, stderr, stdout)` — non-zero exit from `exec` -- `ProtocolError` — broken stream +- `ProtocolError` — broken stream (EOF, oversized or undecodable frame) +- `StreamTimeout` — a socket operation hit the client `timeout`; retry the call ```python try: diff --git a/docs/silkd.md b/docs/silkd.md index befa4be9..4f096a58 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -35,12 +35,12 @@ a frame only one side can parse fails CI. | sessions | `session_create {id?, cwd?, env?}` / `session_list` / `session_rm {id}` | a session is a real persistent bash; `exec` with `session` runs inside it. Idle sessions are reaped after 30 minutes | | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | streaming both directions; write commits atomically via temp+rename and inherits an overwritten file's mode; `fs_list` streams 4096-entry batches | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar | -| search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files | +| search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count | | watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | | pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | a shell under a pseudo-terminal; output as `stdout` frames, input as `stdin` frames, exit terminal. PTYs register in the proc table like any exec | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | structured results (porcelain-v2 status, commit hash, branch list). `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways (`data_end` half-closes the guest socket); the server closing ends the stream with `done`. Works on both lanes — the no-network lane's only way in | -| lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early | +| lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early, and a server nothing attaches within 5 minutes is reaped | | misc | `info` | `{version, proto, uptime_secs, procs, sessions}` — the in-guest readiness probe (sandboxd consumes it; no SDK surface). Distinct from the control plane's `GET /v1/info`, which reports node pools and claims | ## Error kinds diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index 565cda92..9d533b66 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -13,6 +13,9 @@ from langchain_core.tools import StructuredTool from pydantic import BaseModel, Field +# Mirrors the MCP exec contract the tool description states. +CALL_TIMEOUT = 300.0 + class ExecInput(BaseModel): command: str = Field(description="shell command, run via sh -c inside the sandbox") @@ -38,7 +41,7 @@ class CocoonToolkit: def __init__(self, addr: str, api_token: str = "", template: str = "rt:24.04", net: str = "", ttl_seconds: int = 0, from_checkpoint: str = ""): - self._client = Client(addr, api_token=api_token) + self._client = Client(addr, api_token=api_token, timeout=CALL_TIMEOUT) self._template = template self._net = net self._ttl = ttl_seconds @@ -61,6 +64,7 @@ def get_tools(self) -> list[StructuredTool]: "Returns stdout; a non-empty stderr is appended as a 'stderr:' " "line and a non-zero status as an 'exit code: N' line; a " "command that prints nothing and exits 0 returns '(no output)'. " + "The call is cut off after 5 minutes. " "Files and installed packages persist across calls; environment " "variables and the working directory do not.", ExecInput, self._exec), diff --git a/sdk/python/cocoonsandbox/__init__.py b/sdk/python/cocoonsandbox/__init__.py index 46513e08..af31cf0a 100644 --- a/sdk/python/cocoonsandbox/__init__.py +++ b/sdk/python/cocoonsandbox/__init__.py @@ -11,7 +11,7 @@ from .checkpoint import Checkpoint from .client import Client -from .errors import APIError, ExitError, ProtocolError, SandboxError, SilkdError +from .errors import APIError, ExitError, ProtocolError, SandboxError, SilkdError, StreamTimeout from .sandbox import Lsp, PortConn, Pty, Sandbox, Session, Watcher from .template import Template @@ -28,6 +28,7 @@ "SandboxError", "Session", "SilkdError", + "StreamTimeout", "Template", "Watcher", ] diff --git a/sdk/python/cocoonsandbox/checkpoint.py b/sdk/python/cocoonsandbox/checkpoint.py index fb7bf569..0a52b866 100644 --- a/sdk/python/cocoonsandbox/checkpoint.py +++ b/sdk/python/cocoonsandbox/checkpoint.py @@ -46,11 +46,7 @@ def post(peer): return self._client._handle_from(addr, reply) def delete(self) -> None: - """Removes the checkpoint from its node and asks every peer that node - currently sees to drop any replica a heal pulled — best-effort eventual - cleanup, not a fleet-wide revocation. A peer that misses that broadcast - (offline, partitioned, or joined later) keeps serving branches from its - replica until the node's checkpoint_ttl_hours ages it out; with that - TTL at its default of 0 (keep forever), an unreachable peer's replica - has no cleanup bound at all.""" + """Removes the checkpoint and broadcasts the drop; cleanup is + best-effort eventual, bounded by checkpoint_ttl_hours (see + docs/sdk-python.md).""" self._client._request(self._addr, "DELETE", f"/v1/checkpoints/{self.id}", None, "delete checkpoint") diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 4a187622..4f3d2b65 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -60,7 +60,10 @@ def lookup(self, id: str, token: str) -> Sandbox: mesh peer concurrently, binding to whichever confirms ownership first — one dead peer must not cost its full timeout.""" def probe(addr: str) -> Sandbox: - reply = self._request(addr, "GET", f"/v1/sandboxes/{id}/owner", None, "owner", bearer=token) + # Bounded like _peers: a scatter loser must not hold a socket for + # the full client timeout after the winner has answered. + reply = self._request(addr, "GET", f"/v1/sandboxes/{id}/owner", None, "owner", + bearer=token, timeout=min(_PEERS_TIMEOUT, self.timeout)) return Sandbox(client=self, id=id, token=token, owner=reply.get("owner_addr") or addr) @@ -296,9 +299,8 @@ def attempt(addr): except APIError as origin_exc: # Both halves matter to whoever reads this: the peers' failure says # why the claim left the origin, the origin's why returning did not help. - origin_exc.message = f"{origin_exc.message} (after redirect targets failed: {exc.message})" - origin_exc.args = (f"{verb}: {origin_exc.message} (HTTP {origin_exc.status})",) - raise + combined = f"{origin_exc.message} (after redirect targets failed: {exc.message})" + raise APIError(verb, origin_exc.status, combined) from origin_exc def _scatter(addrs, probe): diff --git a/sdk/python/cocoonsandbox/conn.py b/sdk/python/cocoonsandbox/conn.py index e1db2dec..7daa9baf 100644 --- a/sdk/python/cocoonsandbox/conn.py +++ b/sdk/python/cocoonsandbox/conn.py @@ -6,9 +6,9 @@ import contextlib import socket from collections.abc import Iterator -from typing import TypeVar +from typing import BinaryIO, TypeVar -from .errors import APIError, ProtocolError, SilkdError +from .errors import APIError, ProtocolError, SilkdError, StreamTimeout from .frames import MAX_FRAME, decode_response, encode_request _CloseableT = TypeVar("_CloseableT", bound="_Closeable") @@ -27,7 +27,7 @@ def __exit__(self, *exc) -> None: class Conn(_Closeable): """A live frame stream to one sandbox's silkd, via the owner node.""" - def __init__(self, sock: socket.socket, reader): + def __init__(self, sock: socket.socket, reader: BinaryIO): self._sock = sock self._reader = reader @@ -35,14 +35,23 @@ def send(self, op: str, **fields) -> None: self._sock.sendall(encode_request(op, **fields)) def recv(self) -> dict: - """Returns the next frame; raises SilkdError on an error frame and - ProtocolError on EOF or an oversized line.""" - line = self._reader.readline(MAX_FRAME + 1) + """Returns the next frame; raises SilkdError on an error frame, + StreamTimeout when the socket times out, and ProtocolError on EOF, an + oversized line, or an undecodable one — every failure is typed.""" + try: + line = self._reader.readline(MAX_FRAME + 1) + except socket.timeout as exc: + raise StreamTimeout(f"read timed out: {exc}") from exc + except OSError as exc: + raise ProtocolError(f"read failed: {exc}") from exc if not line: raise ProtocolError("connection closed mid-stream") if len(line) > MAX_FRAME: raise ProtocolError("frame exceeds the 8MiB cap") - frame = decode_response(line) + try: + frame = decode_response(line) + except ValueError as exc: + raise ProtocolError(f"malformed frame: {exc}") from exc if frame["type"] == "error": raise SilkdError(frame.get("kind", "internal"), frame.get("message", "")) return frame diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 4b8f16eb..73da6dde 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -41,3 +41,8 @@ def __init__(self, code: int, stderr: str, stdout: str = ""): class ProtocolError(SandboxError): """The peer broke the frame protocol (unexpected frame, oversized line).""" + + +class StreamTimeout(SandboxError): + """A socket operation exceeded the client timeout; the connection is + desynced, so retry the call, not the frame.""" diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index a5baf14e..d536dde4 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -13,7 +13,7 @@ from .checkpoint import Checkpoint from .conn import Conn, _Closeable, dial_agent -from .errors import APIError, ExitError, ProtocolError +from .errors import APIError, ExitError, ProtocolError, StreamTimeout from .frames import BULK_CHUNK, FS_CHUNK from .template import Template @@ -341,14 +341,18 @@ def pump_out(): with contextlib.suppress(OSError): local.close() - threading.Thread(target=pump_out, daemon=True).start() + pump = threading.Thread(target=pump_out, daemon=True) + pump.start() try: while True: chunk = local.recv(FS_CHUNK) if not chunk: break guest.send(chunk) + # Half-close, then let the guest finish answering: closing here + # would cut the reply the local client is still waiting for. guest.close_write() + pump.join() finally: guest.close() @@ -366,7 +370,7 @@ def _drain_proc(self, op: str, pid: int, on_stdout, on_stderr) -> int | None: return _pump_stdio(conn, on_stdout, on_stderr) -class Session: +class Session(_Closeable): """A persistent shell inside the sandbox, addressed by id.""" def __init__(self, sandbox: Sandbox, id: str): @@ -395,7 +399,7 @@ def __iter__(self) -> Iterator[dict]: while True: try: frame = self._conn.recv() - except (ProtocolError, OSError, ValueError) as e: + except (ProtocolError, StreamTimeout) as e: self.error = e return if frame["type"] == "event": @@ -433,7 +437,7 @@ def close(self) -> None: self._conn.close() -class Lsp: +class Lsp(_Closeable): """A language server in the sandbox, spoken to over the relay. silkd is a broker: it pipes JSON-RPC bytes; the caller frames and correlates.""" @@ -452,6 +456,8 @@ def stop(self) -> None: """Kills the language server.""" self._sandbox._done_rpc("lsp_stop", server_id=self.server_id) + close = stop + class PortConn(_Closeable): """A byte stream to a guest port, relayed over the silkd connection.""" diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py new file mode 100644 index 00000000..8155508a --- /dev/null +++ b/sdk/python/tests/conftest.py @@ -0,0 +1,34 @@ +"""Fixtures shared by the cluster-behavior suites: in-process fake nodes and +addresses that refuse a connection.""" + +import socket +import threading +from http.server import HTTPServer + +import pytest +from test_client import FakeNode + + +@pytest.fixture +def spawn_node(): + servers = [] + + def spawn(routes): + handler = type("Node", (FakeNode,), {"routes": routes}) + server = HTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + servers.append(server) + return f"127.0.0.1:{server.server_port}" + + yield spawn + for server in servers: + server.shutdown() + + +@pytest.fixture +def dead_addr(): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return f"127.0.0.1:{port}" diff --git a/sdk/python/tests/test_checkpoint.py b/sdk/python/tests/test_checkpoint.py index 66a9d4a3..0a0bb7b8 100644 --- a/sdk/python/tests/test_checkpoint.py +++ b/sdk/python/tests/test_checkpoint.py @@ -1,41 +1,11 @@ """Checkpoint claim redirect: mirrors the cluster redirect follow tested for Client.new in test_fault_matrix.py, applied to Checkpoint.new.""" -import socket -import threading -from http.server import HTTPServer - import pytest -from test_client import FakeNode from cocoonsandbox import APIError, Client -@pytest.fixture -def spawn_node(): - servers = [] - - def spawn(routes): - handler = type("Node", (FakeNode,), {"routes": routes}) - server = HTTPServer(("127.0.0.1", 0), handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - servers.append(server) - return f"127.0.0.1:{server.server_port}" - - yield spawn - for server in servers: - server.shutdown() - - -@pytest.fixture -def dead_addr(): - sock = socket.socket() - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - sock.close() - return f"127.0.0.1:{port}" - - def test_checkpoint_new_follows_redirect(spawn_node): seen = [] @@ -61,8 +31,6 @@ def redirect(body, path): def test_checkpoint_new_all_candidates_fail(spawn_node): - # The probed owner transiently fails (500); once exhausted, new() falls - # back to the origin, which this time fails definitively too. path = "/v1/checkpoints/ck_2/claim" broken = spawn_node({("POST", path): lambda body, p: (500, {"error": "boom"})}) @@ -82,8 +50,6 @@ def entry_claim(body, p): def test_checkpoint_new_redirect_fallback_heals(spawn_node): - # The only probed owner is mid-heal (503); once exhausted, new() falls - # back to the origin, which heals (pulls the checkpoint) locally. path = "/v1/checkpoints/ck_5/claim" busy = spawn_node({("POST", path): lambda body, p: (503, {"error": "healing"})}) @@ -102,9 +68,6 @@ def entry_claim(body, p): def test_checkpoint_new_redirect_never_yields_empty_id(spawn_node, dead_addr): - # Regression: new() used to hand a bare redirect reply straight to - # _handle_from, producing a Sandbox with no id/token. It must now follow - # the redirect and raise when no candidate answers. entry = spawn_node({("POST", "/v1/checkpoints/ck_3/claim"): lambda body, path: (200, {"redirect": [dead_addr]})}) with pytest.raises(APIError): @@ -112,8 +75,6 @@ def test_checkpoint_new_redirect_never_yields_empty_id(spawn_node, dead_addr): def test_checkpoint_new_second_level_redirect_fails(spawn_node): - # A compliant server never redirects a no_redirect retry; if one does - # anyway, it must be treated as a failed candidate rather than followed. path = "/v1/checkpoints/ck_4/claim" node_b = spawn_node({("POST", path): lambda body, p: (200, {"redirect": ["127.0.0.1:1"]})}) entry = spawn_node({("POST", path): lambda body, p: (200, {"redirect": [node_b]})}) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 0da1356d..4d7b1806 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -121,8 +121,7 @@ def claim(body, path): FakeNode.routes[("POST", "/v1/claim")] = claim Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mode": "ro"}]) Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mode": ""}]) - # "ro" and "" both normalize to an omitted key, byte-identical to a v1 request. - assert seen[0]["volumes"] == seen[1]["volumes"] == [{"name": "imagenet"}] + assert seen[0]["volumes"] == seen[1]["volumes"] == [{"name": "imagenet"}], seen def test_claim_attaches_volumes_without_mounting(node): @@ -232,8 +231,6 @@ def test_volume_catalog(node): def test_volume_catalog_surfaces_writable(node): - # The server omits "writable" for read-only entries (json omitempty); the - # client passes both shapes through as-is, key present or absent. want = [ { "name": "scratch", diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index d6e5b22f..3bcaa512 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -5,41 +5,13 @@ error) — and lookup must not pay a hung or dead peer's full timeout.""" import socket -import threading import time -from http.server import HTTPServer import pytest -from test_client import FakeNode from cocoonsandbox import APIError, Client -@pytest.fixture -def spawn_node(): - servers = [] - - def spawn(routes): - handler = type("Node", (FakeNode,), {"routes": routes}) - server = HTTPServer(("127.0.0.1", 0), handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - servers.append(server) - return f"127.0.0.1:{server.server_port}" - - yield spawn - for server in servers: - server.shutdown() - - -@pytest.fixture -def dead_addr(): - sock = socket.socket() - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - sock.close() - return f"127.0.0.1:{port}" - - @pytest.fixture def hung_addr(): sock = socket.socket() @@ -64,8 +36,6 @@ def test_claim_redirect_skips_404_candidate(spawn_node): def test_claim_redirect_all_candidates_fail_then_origin_fallback_fails(spawn_node): - # Every candidate is a stale miss (404); once exhausted, new() falls - # back to the origin, which this time fails definitively too. a = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone a"})}) b = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone b"})}) @@ -81,15 +51,11 @@ def entry_claim(body, path): with pytest.raises(APIError) as exc: Client(entry).new("rt:24.04") assert exc.value.status == 409 and "origin also failed" in exc.value.message - # The last candidate's failure must survive into the message: it is what - # says why the claim left the origin in the first place. assert "gone b" in exc.value.message assert len(calls) == 2 and calls[1]["no_redirect"] is True def test_claim_redirect_all_candidates_fail_then_origin_heals(spawn_node): - # The only candidate holds the record but is full (429); once exhausted, - # new() falls back to the origin, which heals (provisions locally). full_calls = [] def full_claim(body, path): @@ -115,8 +81,6 @@ def entry_claim(body, path): @pytest.mark.parametrize(("status", "message"), [(409, "no egress"), (403, "tenant not allowed")]) def test_claim_redirect_definitive_error_skips_origin_fallback(spawn_node, status, message): - # A definitive 4xx is not worth trying another candidate, and not worth - # falling back to the origin either -- the origin would fail the same way. bad = spawn_node({("POST", "/v1/claim"): lambda body, path: (status, {"error": message})}) calls = [] @@ -157,9 +121,6 @@ def entry_claim(body, path): def test_claim_redirect_candidate_definitive_error_still_tries_next_candidate(spawn_node): - # Candidate one is wrong for this claim (403, definitive) but candidate - # two would still succeed -- the per-candidate walk retries broadly, so - # one ill-suited candidate must not cost a candidate that would answer. forbidden = spawn_node({("POST", "/v1/claim"): lambda body, path: (403, {"error": "tenant not allowed"})}) good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_3", "token": "t"})}) entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [forbidden, good]})}) diff --git a/sdk/python/tests/test_fixtures.py b/sdk/python/tests/test_fixtures.py index 4e9fd9b7..2f9b4f2a 100644 --- a/sdk/python/tests/test_fixtures.py +++ b/sdk/python/tests/test_fixtures.py @@ -10,7 +10,7 @@ from cocoonsandbox import frames -FIXTURES = pathlib.Path(__file__).parent / ".." / ".." / ".." / "protocol" / "wire" / "fixtures" / "v1" +FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" def fixture_files(prefix): diff --git a/sdk/python/tests/test_hardening.py b/sdk/python/tests/test_hardening.py index 985cd048..78311274 100644 --- a/sdk/python/tests/test_hardening.py +++ b/sdk/python/tests/test_hardening.py @@ -11,7 +11,6 @@ def test_dial_agent_rejects_control_chars_in_identity(): - # A CR/LF in id or token would split the hand-built HTTP request line. with pytest.raises(APIError): dial_agent("127.0.0.1:1", "sb\r\nInjected: 1", "tok", 0.5) with pytest.raises(APIError): @@ -23,7 +22,6 @@ def test_watcher_propagates_silkd_error(): guest_sock.sendall(json.dumps({"type": "error", "kind": "not_found", "message": "gone"}).encode() + b"\n") guest_sock.close() watcher = Watcher(Conn(client_sock, client_sock.makefile("rb"))) - # A real server error frame must surface, not read as a clean stream end. with pytest.raises(SilkdError): for _ in watcher: pass @@ -34,8 +32,6 @@ def test_watcher_ends_cleanly_on_garbage_frame(): guest_sock.sendall(b"not json at all\n") guest_sock.close() watcher = Watcher(Conn(client_sock, client_sock.makefile("rb"))) - # An undecodable frame ends iteration; it must not escape as a raw - # ValueError/JSONDecodeError from the public iterator. assert list(watcher) == [] @@ -46,8 +42,6 @@ def test_request_wraps_truncated_body_as_api_error(): def serve(): conn, _ = server.accept() conn.recv(4096) - # Declare more body than is sent: read() raises IncompleteRead, - # which is an HTTPException, not an OSError. conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort") conn.close() diff --git a/sdk/python/tests/test_wire_binding.py b/sdk/python/tests/test_wire_binding.py index a2edccba..08d74016 100644 --- a/sdk/python/tests/test_wire_binding.py +++ b/sdk/python/tests/test_wire_binding.py @@ -3,7 +3,11 @@ Conn over a socketpair, then asserts the emitted frame carries exactly the fixture's fields with the same values — both directions, so a misspelled field name AND a silently dropped field fail here; the generic encode -round-trip alone cannot see either.""" +round-trip alone cannot see either. + +Each CASES entry is (fixture stem, scripted guest replies, invoke(sb, fixture)); +UNSENT lists the fields no SDK call site emits by design, which silkd defaults +and the Go SDK omits too.""" import contextlib import json @@ -17,11 +21,8 @@ from cocoonsandbox.conn import Conn from cocoonsandbox.frames import PROTO_VERSION -FIXTURES = pathlib.Path(__file__).parent / ".." / ".." / ".." / "protocol" / "wire" / "fixtures" / "v1" +FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" -# op → (fixture stem, replies the fake guest scripts, invoke(sb, fixture)). -# Invocations pass the fixture's own values so the subset assertion binds -# every emitted field name and value to the corpus. CASES = [ ("req_exec", [{"type": "started", "pid": 7}, {"type": "exit", "code": 0}], lambda sb, f: sb.run(f["argv"], cwd=f["cwd"], env=f["env"], user=f["user"], session=f["session"])), @@ -94,7 +95,6 @@ ] -# Fields no SDK call site emits by design (silkd defaults them; Go omits too). UNSENT = {"req_session_create": {"id"}} @@ -124,7 +124,6 @@ def guest(): sent.append(json.loads(line)) for reply in replies: guest_sock.sendall(json.dumps(reply).encode() + b"\n") - # Drain follow-up frames (data/stdin streams) until EOF. while True: line = reader.readline() if not line: diff --git a/silkd/src/exec.rs b/silkd/src/exec.rs index ab16953e..d764249b 100644 --- a/silkd/src/exec.rs +++ b/silkd/src/exec.rs @@ -20,6 +20,8 @@ use crate::sysutil; /// open in a surviving grandchild; a foreground client stalled past this /// window can also lose the not-yet-drained pipe tail (bounded, rare). const POST_EXIT_DRAIN: Duration = Duration::from_secs(2); +/// How long an exited detached process stays in the table, so a late +/// `logs`/`attach` still finds its ring instead of a bare not_found. const REAP_DELAY: Duration = Duration::from_secs(300); /// Foreground output buffer before the child is backpressured. Bounds how far /// ahead of a slow client the child may run, not a loss threshold. diff --git a/silkd/src/find.rs b/silkd/src/find.rs index b069ae55..8ed37ef8 100644 --- a/silkd/src/find.rs +++ b/silkd/src/find.rs @@ -2,11 +2,13 @@ //! rewrites matches in named files. Regex handling lives here so agents pass a //! pattern as data rather than shell-quoting it through exec. +use std::io::Read; use std::path::{Path, PathBuf}; use regex::Regex; use tokio::fs; use tokio::io::AsyncWrite; +use tokio::sync::mpsc; use crate::proto::{self, ErrorKind, Response, err_frame}; @@ -14,6 +16,9 @@ use crate::proto::{self, ErrorKind, Response, err_frame}; /// grep-scale line scanning is for source trees, not blobs. const FIND_MAX_FILE: u64 = 8 * 1024 * 1024; +/// Match frames in flight between the walking thread and the writer. +const MATCH_QUEUE: usize = 256; + /// Streams `match` frames for every line under `path` matching `pattern`, /// terminated by `done`. `glob` narrows the walk to file names matching it /// (`*` and `?` wildcards); an invalid pattern is a bad-request error. @@ -32,42 +37,39 @@ pub async fn find( Some(Ok(re)) => Some(re), Some(Err(e)) => return proto::error_frame(w, ErrorKind::BadRequest, e.to_string()).await, }; - let mut stack = vec![PathBuf::from(path)]; - let mut root = true; - while let Some(dir) = stack.pop() { - let mut rd = match fs::read_dir(&dir).await { - Ok(rd) => rd, - Err(e) if root => return err_frame(w, &e, "read_dir").await, - Err(_) => continue, - }; - loop { - let ent = match rd.next_entry().await { - Ok(Some(ent)) => ent, - Ok(None) => break, - Err(e) if root => return err_frame(w, &e, "read_dir").await, - Err(_) => break, - }; - let Ok(ft) = ent.file_type().await else { - continue; - }; - let p = ent.path(); - if ft.is_dir() { - stack.push(p); - } else if ft.is_file() && name_matches(&p, name_re.as_ref()) { - scan_file(w, &re, &p).await?; - } + // One blocking-pool dispatch for the whole tree, not three per file. + let (tx, mut rx) = mpsc::channel::(MATCH_QUEUE); + let walk = + tokio::task::spawn_blocking(move || walk(PathBuf::from(path), &re, name_re.as_ref(), &tx)); + let mut batch = Vec::new(); + let mut buf = Vec::new(); + let mut failed = None; + while rx.recv_many(&mut batch, MATCH_QUEUE).await > 0 { + let sent = proto::write_frames(w, &mut buf, &batch).await; + batch.clear(); + if let Err(e) = sent { + failed = Some(e); + break; } - root = false; } - proto::write_frame(w, &Response::Done).await + drop(rx); + let walked = walk.await.map_err(std::io::Error::other)?; + match failed { + Some(e) => Err(e), + None => match walked { + Ok(()) => proto::write_frame(w, &Response::Done).await, + Err(e) => err_frame(w, &e, "read_dir").await, + }, + } } /// Rewrites every `pattern` match to `replacement` in each of `files`, /// streaming one `replaced` frame per file (with its match count) and a -/// terminal `done`. A read/write failure on one file ends the stream with an -/// error — files whose `replaced` frame already went out are committed (each -/// file is atomic; the list is not). An invalid pattern is rejected before any -/// file is touched. +/// terminal `done`. A file over the find size bound is skipped with a zero +/// count rather than read into memory. A read/write failure on one file ends +/// the stream with an error — files whose `replaced` frame already went out +/// are committed (each file is atomic; the list is not). An invalid pattern is +/// rejected before any file is touched. pub async fn replace( w: &mut W, files: Vec, @@ -79,23 +81,24 @@ pub async fn replace( Err(e) => return proto::error_frame(w, ErrorKind::BadRequest, e.to_string()).await, }; for file in files { - let body = match fs::read_to_string(&file).await { - Ok(body) => body, - Err(e) => return err_frame(w, &e, "read").await, - }; - // Closure replacer: counts while replacing in one pass; expand keeps - // the &str replacer's $n capture-group semantics. let mut count: u64 = 0; - let new = re.replace_all(&body, |caps: ®ex::Captures| { - count += 1; - let mut expanded = String::new(); - caps.expand(&replacement, &mut expanded); - expanded - }); - if count > 0 - && let Err(e) = crate::fs::write_atomic(Path::new(&file), new.as_bytes()).await - { - return err_frame(w, &e, "write").await; + if !oversized(&file).await { + let body = match fs::read_to_string(&file).await { + Ok(body) => body, + Err(e) => return err_frame(w, &e, "read").await, + }; + // expand keeps the &str replacer's $n capture-group semantics. + let new = re.replace_all(&body, |caps: ®ex::Captures| { + count += 1; + let mut expanded = String::new(); + caps.expand(&replacement, &mut expanded); + expanded + }); + if count > 0 + && let Err(e) = crate::fs::write_atomic(Path::new(&file), new.as_bytes()).await + { + return err_frame(w, &e, "write").await; + } } proto::write_frame( w, @@ -109,35 +112,76 @@ pub async fn replace( proto::write_frame(w, &Response::Done).await } -async fn scan_file( - w: &mut W, - re: &Regex, - path: &Path, -) -> std::io::Result<()> { - if fs::metadata(path) +async fn oversized(file: &str) -> bool { + fs::metadata(file) .await .is_ok_and(|m| m.len() > FIND_MAX_FILE) - { - return Ok(()); +} + +/// Walks `root` depth-first, sending a `match` frame per matching line. A +/// failure on the root propagates; deeper ones skip only that directory. +fn walk( + root: PathBuf, + re: &Regex, + name_re: Option<&Regex>, + tx: &mpsc::Sender, +) -> std::io::Result<()> { + let mut stack = vec![root]; + let mut root = true; + while let Some(dir) = stack.pop() { + let rd = match std::fs::read_dir(&dir) { + Ok(rd) => rd, + Err(e) if root => return Err(e), + Err(_) => continue, + }; + for ent in rd { + let ent = match ent { + Ok(ent) => ent, + Err(e) if root => return Err(e), + Err(_) => break, + }; + let Ok(ft) = ent.file_type() else { + continue; + }; + let p = ent.path(); + if ft.is_dir() { + stack.push(p); + } else if ft.is_file() && name_matches(&p, name_re) && !scan_file(&p, re, tx) { + return Ok(()); + } + } + root = false; } - let Ok(body) = fs::read_to_string(path).await else { - return Ok(()); // non-UTF8 / unreadable: skip, not an error + Ok(()) +} + +/// Scans one file, reporting whether the receiver is still listening. The size +/// bound comes off the open handle, so the check and the read see one file. +fn scan_file(path: &Path, re: &Regex, tx: &mpsc::Sender) -> bool { + let Ok(mut file) = std::fs::File::open(path) else { + return true; }; - let file = path.to_string_lossy().into_owned(); + if file.metadata().is_ok_and(|m| m.len() > FIND_MAX_FILE) { + return true; + } + let mut body = String::new(); + if file.read_to_string(&mut body).is_err() { + return true; + } + let name = path.to_string_lossy(); for (i, line) in body.lines().enumerate() { if re.is_match(line) { - proto::write_frame( - w, - &Response::Match { - file: file.clone(), - line: i as u64 + 1, - content: line.to_string(), - }, - ) - .await?; + let frame = Response::Match { + file: name.to_string(), + line: i as u64 + 1, + content: line.to_string(), + }; + if tx.blocking_send(frame).is_err() { + return false; + } } } - Ok(()) + true } fn name_matches(path: &Path, name_re: Option<&Regex>) -> bool { diff --git a/silkd/src/fs.rs b/silkd/src/fs.rs index 4705bc89..a804cb46 100644 --- a/silkd/src/fs.rs +++ b/silkd/src/fs.rs @@ -9,6 +9,7 @@ use std::time::UNIX_EPOCH; use tokio::fs; use tokio::io::{AsyncBufRead, AsyncWrite, AsyncWriteExt}; +use tokio::sync::mpsc; use crate::proto::{self, DirEntry, FileInfo, FileKind, Response, err_frame}; @@ -68,36 +69,25 @@ pub async fn read(w: &mut W, path: String) -> io::Result< /// Lists a directory as a stream of `entries` frames terminated by `done`, /// batched so an arbitrarily large directory can never exceed the frame cap. pub async fn list(w: &mut W, path: String) -> io::Result<()> { - let mut rd = match fs::read_dir(&path).await { - Ok(r) => r, - Err(e) => return err_frame(w, &e, "read_dir").await, - }; - let mut entries = Vec::new(); - loop { - match rd.next_entry().await { - Ok(None) => break, - Ok(Some(ent)) => { - let (kind, size) = match ent.metadata().await { - Ok(m) => (file_kind(&m), m.len()), - Err(_) => (FileKind::Other, 0), - }; - entries.push(DirEntry { - name: ent.file_name().to_string_lossy().into_owned(), - kind, - size, - }); - if entries.len() == LIST_BATCH { - let batch = std::mem::take(&mut entries); - proto::write_frame(w, &Response::Entries { entries: batch }).await?; - } - } - Err(e) => return err_frame(w, &e, "read_dir").await, + // One blocking-pool dispatch for the whole directory, not one per entry. + let (tx, mut rx) = mpsc::channel::>(2); + let scan = tokio::task::spawn_blocking(move || scan_dir(&path, &tx)); + let mut failed = None; + while let Some(entries) = rx.recv().await { + if let Err(e) = proto::write_frame(w, &Response::Entries { entries }).await { + failed = Some(e); + break; } } - if !entries.is_empty() { - proto::write_frame(w, &Response::Entries { entries }).await?; + drop(rx); + let scanned = scan.await.map_err(io::Error::other)?; + match failed { + Some(e) => Err(e), + None => match scanned { + Ok(()) => proto::write_frame(w, &Response::Done).await, + Err(e) => err_frame(w, &e, "read_dir").await, + }, } - proto::write_frame(w, &Response::Done).await } /// Writes `bytes` to `path` via a sibling temp file committed into place, so @@ -177,6 +167,30 @@ pub async fn rename(w: &mut W, from: String, to: String) } } +/// Reads `path` into `LIST_BATCH`-sized batches, stopping when the writer is gone. +fn scan_dir(path: &str, tx: &mpsc::Sender>) -> io::Result<()> { + let mut entries = Vec::new(); + for ent in std::fs::read_dir(path)? { + let ent = ent?; + let (kind, size) = match ent.metadata() { + Ok(m) => (file_kind(&m), m.len()), + Err(_) => (FileKind::Other, 0), + }; + entries.push(DirEntry { + name: ent.file_name().to_string_lossy().into_owned(), + kind, + size, + }); + if entries.len() == LIST_BATCH && tx.blocking_send(std::mem::take(&mut entries)).is_err() { + return Ok(()); + } + } + if !entries.is_empty() { + let _ = tx.blocking_send(entries); + } + Ok(()) +} + fn tmp_name(path: &str) -> String { format!("{path}.silkd-{}.tmp", crate::sysutil::tmp_suffix()) } diff --git a/silkd/src/git.rs b/silkd/src/git.rs index d52985bb..effd09a8 100644 --- a/silkd/src/git.rs +++ b/silkd/src/git.rs @@ -115,38 +115,36 @@ pub async fn branch( op: GitBranchOp, name: Option, ) -> std::io::Result<()> { - match op { - GitBranchOp::List => { - let out = git(&path, None, &["branch", "--format=%(refname:short)"]).await?; - if !out.status.success() { - return terminal(w, "branch", &out).await; - } - let branches: Vec = String::from_utf8_lossy(&out.stdout) - .lines() - .map(str::to_string) - .collect(); - let cur = git(&path, None, &["rev-parse", "--abbrev-ref", "HEAD"]).await?; - let current = String::from_utf8_lossy(&cur.stdout).trim().to_string(); - proto::write_frame(w, &Response::GitBranches { current, branches }).await - } - GitBranchOp::Create | GitBranchOp::Delete | GitBranchOp::Checkout => { - let Some(name) = name else { - return proto::write_frame( - w, - &Response::error(ErrorKind::BadRequest, "branch op needs a name"), - ) - .await; - }; - let args: Vec<&str> = match op { - GitBranchOp::Create => vec!["branch", &name], - GitBranchOp::Delete => vec!["branch", "-D", &name], - GitBranchOp::Checkout => vec!["checkout", &name], - GitBranchOp::List => unreachable!(), - }; - let out = git(&path, None, &args).await?; - terminal(w, "branch", &out).await + let args: Vec<&str> = match (op, &name) { + (GitBranchOp::List, _) => return list_branches(w, &path).await, + (_, None) => { + return proto::write_frame( + w, + &Response::error(ErrorKind::BadRequest, "branch op needs a name"), + ) + .await; } + (GitBranchOp::Create, Some(name)) => vec!["branch", name], + (GitBranchOp::Delete, Some(name)) => vec!["branch", "-D", name], + (GitBranchOp::Checkout, Some(name)) => vec!["checkout", name], + }; + let out = git(&path, None, &args).await?; + terminal(w, "branch", &out).await +} + +/// Reports every local branch and the checked-out one. +async fn list_branches(w: &mut W, path: &str) -> std::io::Result<()> { + let out = git(path, None, &["branch", "--format=%(refname:short)"]).await?; + if !out.status.success() { + return terminal(w, "branch", &out).await; } + let branches: Vec = String::from_utf8_lossy(&out.stdout) + .lines() + .map(str::to_string) + .collect(); + let cur = git(path, None, &["rev-parse", "--abbrev-ref", "HEAD"]).await?; + let current = String::from_utf8_lossy(&cur.stdout).trim().to_string(); + proto::write_frame(w, &Response::GitBranches { current, branches }).await } /// push/pull shared body: the egress guard, the bare verb, a terminal frame. @@ -327,7 +325,6 @@ mod tests { let untracked = parse_file_line("? build/out.o").unwrap(); assert_eq!(untracked.path, "build/out.o"); - // Unmerged entries (kind "u") must surface as conflicted, not vanish. let conflict = parse_file_line("u UU N... 100644 100644 100644 100644 h1 h2 h3 conflict.rs").unwrap(); assert_eq!(conflict.path, "conflict.rs"); diff --git a/silkd/src/lsp.rs b/silkd/src/lsp.rs index a7738c9d..5080098d 100644 --- a/silkd/src/lsp.rs +++ b/silkd/src/lsp.rs @@ -9,19 +9,20 @@ //! server's stdin, as in port_forward); `lsp_stop` kills it. v1 is //! single-shot per server: the stream ending — clean or dropped — reaps the //! server, since an LSP stream loses frame sync on any mid-request cut and a -//! resynced reattach is not worth the failure surface. The id and `lsp_stop` +//! resynced reattach is not worth the failure surface. A server no +//! `lsp_request` ever attaches is reaped on a TTL. The id and `lsp_stop` //! still earn their place: `lsp_start` for several languages yields several //! ids attached concurrently, and `lsp_stop` tears one down early. The base //! image ships no manifests, so `lsp_start` for any language there answers a //! typed `not_found` naming the flavor that provides one. use std::collections::HashMap; -use std::os::unix::fs::OpenOptionsExt; use std::process::Stdio; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; -use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::process::{Child, ChildStdin, ChildStdout, Command}; use tokio::sync::mpsc; @@ -30,6 +31,10 @@ use crate::sysutil; const MANIFEST_DIR: &str = "/etc/silkd/lsp.d"; +/// A started server nothing attaches is reaped after this: a client that dies +/// between the calls must not pin a 1-2 GB language server for the VM's life. +const ATTACH_TTL: Duration = Duration::from_secs(300); + static NEXT_ID: AtomicU64 = AtomicU64::new(1); /// Table of running language servers, addressed by server id. @@ -51,7 +56,7 @@ impl Broker { language: &str, root: Option<&str>, ) -> std::io::Result<()> { - let argv = match read_manifest(language) { + let argv = match read_manifest(language).await { Some(argv) if !argv.is_empty() => argv, _ => { return proto::error_frame( @@ -82,13 +87,13 @@ impl Broker { .await; }; let server = Server { - child: Arc::new(tokio::sync::Mutex::new(child)), - stdin: Arc::new(tokio::sync::Mutex::new(Some(stdin))), - stdout: Arc::new(tokio::sync::Mutex::new(Some(stdout))), + child, + stdin: Some(stdin), + stdout: Some(stdout), }; let id = format!("lsp-{}", NEXT_ID.fetch_add(1, Ordering::Relaxed)); - crate::sysutil::lock(&self.inner).insert(id.clone(), server); + sysutil::lock(&self.inner).insert(id.clone(), server); // If the client is already gone, don't leak the server we just spawned. if let Err(e) = proto::write_frame( @@ -102,6 +107,11 @@ impl Broker { self.reap(&id).await; return Err(e); } + let broker = self.clone(); + tokio::spawn(async move { + tokio::time::sleep(ATTACH_TTL).await; + broker.reap_unattached(&id).await; + }); Ok(()) } @@ -115,21 +125,20 @@ impl Broker { w: &mut W, server_id: &str, ) -> std::io::Result<()> { - let found = crate::sysutil::lock(&self.inner).get(server_id).cloned(); - let server = match found { - Some(s) => s, + let taken = sysutil::lock(&self.inner) + .get_mut(server_id) + .map(|s| s.stdout.take().zip(s.stdin.take())); + let (stdout, stdin) = match taken { None => return proto::error_frame(w, ErrorKind::NotFound, "no such lsp server").await, - }; - let stdout = match server.stdout.lock().await.take() { - Some(out) => out, - None => { + Some(None) => { return proto::error_frame(w, ErrorKind::BadRequest, "lsp server already attached") .await; } + Some(Some(stdio)) => stdio, }; proto::write_frame(w, &Response::Ready).await?; - let feed = tokio::spawn(feed_stdin(client, server.stdin.clone())); + let feed = tokio::spawn(feed_stdin(client, stdin)); let res = pump_stdout(stdout, w).await; // The session ends with this connection: stop feeding and reap the // server. Aborting is safe here — we are killing the child, so a @@ -155,50 +164,54 @@ impl Broker { /// reap removes a server from the table and kills + waits its child (no /// zombie survives), reporting whether it was there. async fn reap(&self, server_id: &str) -> bool { - let removed = crate::sysutil::lock(&self.inner).remove(server_id); + let removed = sysutil::lock(&self.inner).remove(server_id); + match removed { + Some(server) => { + kill(server).await; + true + } + None => false, + } + } + + /// reap for a server no `lsp_request` ever attached; one that is attached + /// (its stdout taken) belongs to that connection and is left alone. + async fn reap_unattached(&self, server_id: &str) { + let removed = { + let mut table = sysutil::lock(&self.inner); + match table.get(server_id) { + Some(server) if server.stdout.is_some() => table.remove(server_id), + _ => None, + } + }; if let Some(server) = removed { - let mut child = server.child.lock().await; - let _ = child.start_kill(); - let _ = child.wait().await; - true - } else { - false + kill(server).await; } } } -/// One running language server: child + stdio, shared so `lsp_request` reads -/// stdout, `feed_stdin` writes stdin, and `reap` kills the child. stdin is -/// Option so `feed_stdin` can drop it to half-close, like stdout for attach. -#[derive(Clone)] +/// One running language server. `lsp_request` takes both stdio halves, so a +/// present stdout is also the flag that no connection has attached yet. struct Server { - child: Arc>, - stdin: Arc>>, - stdout: Arc>>, + child: Child, + stdin: Option, + stdout: Option, +} + +async fn kill(mut server: Server) { + let _ = server.child.start_kill(); + let _ = server.child.wait().await; } -async fn feed_stdin( - mut client: mpsc::Receiver, - stdin: Arc>>, -) { +async fn feed_stdin(mut client: mpsc::Receiver, mut stdin: ChildStdin) { while let Some(req) = client.recv().await { - match req { - Request::Data { data } => { - let mut guard = stdin.lock().await; - let Some(w) = guard.as_mut() else { return }; - if w.write_all(&data).await.is_err() || w.flush().await.is_err() { - return; - } - } - // DataEnd half-closes, aligned with port_forward: dropping stdin - // delivers EOF so the server can flush remaining output and exit - // (stdout then EOFs -> Done -> reap). A stray non-data frame gets - // the same close — the writer belongs to the stdout pump, so - // there is no channel to answer an error frame on. - _ => { - stdin.lock().await.take(); - return; - } + // Anything but data half-closes by dropping stdin, aligned with + // port_forward: the server sees EOF, flushes, and exits (stdout then + // EOFs -> Done -> reap). A stray frame gets the same close, since the + // writer belongs to the stdout pump. + let Request::Data { data } = req else { return }; + if stdin.write_all(&data).await.is_err() || stdin.flush().await.is_err() { + return; } } } @@ -219,19 +232,20 @@ fn manifest_dir() -> String { std::env::var("SILKD_LSP_DIR").unwrap_or_else(|_| MANIFEST_DIR.to_string()) } -fn read_manifest(language: &str) -> Option> { +async fn read_manifest(language: &str) -> Option> { if language.is_empty() || language.contains(['/', '\\', '\0']) || language.contains("..") { return None; // never let a language name escape the manifest dir } // O_NOFOLLOW: a symlink planted inside lsp.d must not read a target // outside it. - let mut f = std::fs::OpenOptions::new() + let mut f = tokio::fs::OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW) .open(format!("{}/{language}", manifest_dir())) + .await .ok()?; let mut raw = String::new(); - std::io::Read::read_to_string(&mut f, &mut raw).ok()?; + f.read_to_string(&mut raw).await.ok()?; let argv: Vec = raw.split_whitespace().map(str::to_string).collect(); (!argv.is_empty()).then_some(argv) } diff --git a/silkd/src/net.rs b/silkd/src/net.rs index 0058de5e..b035fac3 100644 --- a/silkd/src/net.rs +++ b/silkd/src/net.rs @@ -3,7 +3,7 @@ //! all-builtin kernel auto-creates). Network verbs (git clone/push/pull) and //! proxy-env forwarding consult this. -use std::sync::OnceLock; +use std::sync::LazyLock; use std::sync::atomic::{AtomicI8, Ordering}; static LANE_OVERRIDE: AtomicI8 = AtomicI8::new(-1); @@ -28,13 +28,13 @@ pub fn has_egress() -> bool { _ => {} } // Probed once: the NIC set is fixed for the guest's life. - static DEVICE_BACKED: OnceLock = OnceLock::new(); - *DEVICE_BACKED.get_or_init(|| { + static DEVICE_BACKED: LazyLock = LazyLock::new(|| { let Ok(entries) = std::fs::read_dir("/sys/class/net") else { return true; }; entries.flatten().any(|e| e.path().join("device").exists()) - }) + }); + *DEVICE_BACKED } /// Lane override for tests: set_var would race every concurrent getenv. @@ -44,10 +44,11 @@ pub fn override_egress_for_tests(lane: Option) { /// SILKD_NET decoded once: operator config, fixed at service start. fn silkd_net() -> i8 { - static SILKD_NET: OnceLock = OnceLock::new(); - *SILKD_NET.get_or_init(|| match std::env::var("SILKD_NET").as_deref() { - Ok("none") => 0, - Ok("egress") => 1, - _ => -1, - }) + static SILKD_NET: LazyLock = + LazyLock::new(|| match std::env::var("SILKD_NET").as_deref() { + Ok("none") => 0, + Ok("egress") => 1, + _ => -1, + }); + *SILKD_NET } diff --git a/silkd/src/proc.rs b/silkd/src/proc.rs index 39021fb0..b6ca1388 100644 --- a/silkd/src/proc.rs +++ b/silkd/src/proc.rs @@ -249,17 +249,23 @@ impl Ring { } } + /// Retained output, coalescing adjacent same-stream segments: reads leave + /// tens of thousands of tiny ones and the client concatenates them anyway. fn drain_view(&mut self) -> Vec { let bytes = self.buf.make_contiguous(); - let mut out = Vec::with_capacity(self.tags.len()); + let mut out: Vec = Vec::new(); let mut off = 0; for &(stderr, len) in &self.tags { - let slice = bytes[off..off + len].to_vec(); - out.push(if stderr { - Chunk::Stderr(slice) - } else { - Chunk::Stdout(slice) - }); + let seg = &bytes[off..off + len]; + match out.last_mut() { + Some(Chunk::Stdout(acc)) if !stderr => acc.extend_from_slice(seg), + Some(Chunk::Stderr(acc)) if stderr => acc.extend_from_slice(seg), + _ => out.push(if stderr { + Chunk::Stderr(seg.to_vec()) + } else { + Chunk::Stdout(seg.to_vec()) + }), + } off += len; } out diff --git a/silkd/src/proto.rs b/silkd/src/proto.rs index 01615484..c3051513 100644 --- a/silkd/src/proto.rs +++ b/silkd/src/proto.rs @@ -413,16 +413,41 @@ fn cap_check(len: usize) -> io::Result<()> { Ok(()) } -pub async fn write_frame(w: &mut W, resp: &Response) -> io::Result<()> { - let mut buf = serde_json::to_vec(resp).map_err(io::Error::other)?; +/// Renders one newline-terminated frame onto `buf`, holding the write side to +/// the same cap the read side enforces — every client rejects a larger frame. +fn render_frame(buf: &mut Vec, resp: &Response) -> io::Result<()> { + let start = buf.len(); + serde_json::to_writer(&mut *buf, resp).map_err(io::Error::other)?; + cap_check(buf.len() - start)?; buf.push(b'\n'); + Ok(()) +} + +pub async fn write_frame(w: &mut W, resp: &Response) -> io::Result<()> { + let mut buf = Vec::new(); + render_frame(&mut buf, resp)?; w.write_all(&buf).await?; w.flush().await } +/// Writes `frames` as one buffered batch, so a burst costs one write syscall +/// instead of one per frame. `buf` is reused across calls. +pub async fn write_frames( + w: &mut W, + buf: &mut Vec, + frames: &[Response], +) -> io::Result<()> { + buf.clear(); + for frame in frames { + render_frame(buf, frame)?; + } + w.write_all(buf).await?; + w.flush().await +} + /// Renders `{"type":KIND,"data":""}` into `buf` (reused across calls) /// and writes it — the bulk path skips serde's owned Vec + String per chunk. -/// base64's alphabet needs no JSON escaping. KIND is data|stdout|stderr. +/// base64's alphabet needs no JSON escaping. pub async fn write_chunk_frame( w: &mut W, buf: &mut Vec, @@ -629,8 +654,6 @@ mod tests { #[tokio::test] async fn chunk_frame_parses_back_to_serde_response() { - // Payload lengths 0..=3 cover every base64 padding shape; the full - // byte range covers the whole alphabet. let payloads: Vec> = vec![ Vec::new(), b"a".to_vec(), @@ -675,9 +698,7 @@ mod tests { assert_eq!(got, want); } - /// Perf evidence, not a correctness gate. The serde side pays the - /// per-chunk `to_vec` the old producers paid. Run with: - /// cargo test --release -- --ignored --nocapture bench_chunk_render + /// Run with: cargo test --release -- --ignored --nocapture bench_chunk_render #[tokio::test] #[ignore] async fn bench_chunk_render_old_vs_new() { @@ -712,10 +733,8 @@ mod tests { assert_eq!(err.kind(), io::ErrorKind::InvalidData); } - /// Twin of the Go TestEnumValueSetsMatchFixture: pins each wire enum's - /// full value list (order-sensitive) to enums.json — frame fixtures carry - /// one representative value per enum, so a rename or an added variant on - /// only one side would otherwise drift silently. + /// Twin of the Go TestEnumValueSetsMatchFixture: pins every wire enum's + /// ordered value list to enums.json, which frame fixtures cannot. #[test] fn enum_value_sets_match_fixture() { fn rendered(variants: &[T]) -> Vec { @@ -781,8 +800,6 @@ mod tests { for entry in std::fs::read_dir(dir).expect("fixtures dir") { let path = entry.expect("entry").path(); let name = path.file_name().unwrap().to_string_lossy().into_owned(); - // Only the corpus itself; skip editor/OS cruft (.DS_Store, - // AppleDouble ._ sidecars) that can land in the dir. let Some(stem) = name.strip_suffix(".json") else { continue; }; @@ -799,7 +816,6 @@ mod tests { } seen += 1; } - // Exact, mirroring the Go guard: a new verb lands with its fixture. assert_eq!( seen, 60, "fixture corpus: adding a verb means adding its fixture" diff --git a/silkd/src/pty.rs b/silkd/src/pty.rs index 72f66321..37d0f10d 100644 --- a/silkd/src/pty.rs +++ b/silkd/src/pty.rs @@ -24,9 +24,9 @@ const POST_EXIT_DRAIN: Duration = Duration::from_secs(1); type Master = Arc>; -/// Opens a pty running the guest login shell, streams its output as `stdout` -/// frames and the exit as `exit`, and consumes client `stdin` frames as pty -/// input until the shell exits or the client disconnects. +/// Opens a pty running the guest shell ($SHELL, else bash), streams its output +/// as `stdout` frames and the exit as `exit`, and consumes client `stdin` +/// frames as pty input until the shell exits or the client disconnects. pub async fn open( table: &Table, now: u64, @@ -55,13 +55,15 @@ pub async fn open( { return crate::proto::error_frame(out, ErrorKind::BadRequest, e).await; } - match slave.try_clone().and_then(|i| Ok((i, slave.try_clone()?))) { - Ok((in_fd, out_fd)) => { + match (slave.try_clone(), slave.try_clone()) { + (Ok(in_fd), Ok(out_fd)) => { cmd.stdin(Stdio::from(in_fd)) .stdout(Stdio::from(out_fd)) .stderr(Stdio::from(slave)); } - Err(e) => return crate::proto::err_frame(out, &e, "dup pty slave").await, + (Err(e), _) | (_, Err(e)) => { + return crate::proto::err_frame(out, &e, "dup pty slave").await; + } } // SAFETY: make_controlling_tty runs only async-signal-safe syscalls, which // is the contract for a post-fork pre_exec hook. @@ -86,7 +88,7 @@ pub async fn open( let master: Master = match AsyncFd::new(master) { Ok(m) => Arc::new(m), Err(e) => { - child.start_kill().ok(); + let _ = child.start_kill(); table.remove_if(pid, &proc); return crate::proto::err_frame(out, &e, "watch pty master").await; } @@ -96,7 +98,7 @@ pub async fn open( .is_err() { // Client vanished before we streamed anything — tear down cleanly. - child.start_kill().ok(); + let _ = child.start_kill(); finish(&proc, -1); table.remove_if(pid, &proc); return Ok(()); @@ -163,7 +165,8 @@ async fn pump( Err(_) => { eof = true; continue } }; match guard.try_io(|fd| sysutil::read_fd(fd.get_ref().as_raw_fd(), &mut buf)) { - // 0 (BSD) and EIO (Linux) both mean the slave is fully closed. + // A read of 0 (BSD) or any error (EIO on Linux) means the + // slave is fully closed. Ok(Ok(0)) | Ok(Err(_)) => eof = true, Ok(Ok(n)) => { proc.emit_bytes(false, &buf[..n]); diff --git a/silkd/src/server.rs b/silkd/src/server.rs index eeb1ad22..b37236f5 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -1,7 +1,6 @@ //! Connection handling: read the leading request frame, dispatch to a //! handler, and for exec forward subsequent client frames over a channel. -use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufRead, AsyncWrite}; @@ -29,7 +28,7 @@ impl State { } } - pub async fn serve(self: &Arc, mut reader: R, mut writer: W) -> std::io::Result<()> + pub async fn serve(&self, mut reader: R, mut writer: W) -> std::io::Result<()> where R: AsyncBufRead + Unpin + Send + 'static, W: AsyncWrite + Unpin, diff --git a/silkd/src/session.rs b/silkd/src/session.rs index 0c7c792a..6466eced 100644 --- a/silkd/src/session.rs +++ b/silkd/src/session.rs @@ -74,6 +74,9 @@ impl Table { stdin, stdout, _child: child, + buf: vec![0u8; READ_CHUNK], + acc: Vec::new(), + frame: Vec::new(), }), last_active: Mutex::new(Instant::now()), }); @@ -202,10 +205,15 @@ impl Session { } } +/// The shell's pipes plus converse's scratch buffers, which are per-session +/// and reused across commands rather than reallocated per call. struct Io { stdin: ChildStdin, stdout: ChildStdout, _child: Child, + buf: Vec, + acc: Vec, + frame: Vec, } impl Io { @@ -219,13 +227,10 @@ impl Io { cmd: &str, mut out: Option<&mut W>, ) -> io::Result { - // Unpredictable marker: the shell runs untrusted code that must not be - // able to forge the sentinel and desync the stream. Redirect the - // command's stdin from /dev/null so a command that reads stdin (a REPL, - // `read`) can't swallow the sentinel printf and wedge the read. The - // `{ …; }` group runs in the current shell, so cd/export still persist. - // Trim so a trailing newline doesn't put `;` on its own line inside the - // group; empty becomes a no-op so `{ }` stays valid. + // The marker is unforgeable: the shell runs untrusted code that must + // not be able to fake the sentinel and desync the stream. `{ …; } + // ":", c => c, @@ -240,38 +245,40 @@ impl Io { let mb = marker.as_bytes(); let keep = mb.len().saturating_sub(1); - let mut acc: Vec = Vec::new(); - let mut buf = vec![0u8; READ_CHUNK]; - let mut frame = Vec::new(); + self.acc.clear(); loop { - let n = self.stdout.read(&mut buf).await?; + let n = self.stdout.read(&mut self.buf).await?; if n == 0 { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "shell exited")); } - acc.extend_from_slice(&buf[..n]); - if let Some(pos) = memchr::memmem::find(&acc, mb) { - emit(&mut out, &mut frame, &acc[..pos]).await; - acc.drain(..pos + mb.len()); - while !acc.contains(&b'\n') && acc.len() < EXIT_TAIL_MAX { - let m = self.stdout.read(&mut buf).await?; + self.acc.extend_from_slice(&self.buf[..n]); + if let Some(pos) = memchr::memmem::find(&self.acc, mb) { + emit(&mut out, &mut self.frame, &self.acc[..pos]).await; + self.acc.drain(..pos + mb.len()); + while !self.acc.contains(&b'\n') && self.acc.len() < EXIT_TAIL_MAX { + let m = self.stdout.read(&mut self.buf).await?; if m == 0 { break; } - acc.extend_from_slice(&buf[..m]); + self.acc.extend_from_slice(&self.buf[..m]); } // Parse only the exit-code line; a command that left a // background writer can land bytes after the newline, which // must not corrupt the code. - let end = acc.iter().position(|&b| b == b'\n').unwrap_or(acc.len()); - return Ok(String::from_utf8_lossy(&acc[..end]) + let end = self + .acc + .iter() + .position(|&b| b == b'\n') + .unwrap_or(self.acc.len()); + return Ok(String::from_utf8_lossy(&self.acc[..end]) .trim() .parse() .unwrap_or(-1)); } - if acc.len() > keep { - let upto = acc.len() - keep; - emit(&mut out, &mut frame, &acc[..upto]).await; - acc.drain(..upto); + if self.acc.len() > keep { + let upto = self.acc.len() - keep; + emit(&mut out, &mut self.frame, &self.acc[..upto]).await; + self.acc.drain(..upto); } } } diff --git a/silkd/src/sysutil.rs b/silkd/src/sysutil.rs index 234dddf5..4e5e8c35 100644 --- a/silkd/src/sysutil.rs +++ b/silkd/src/sysutil.rs @@ -3,12 +3,11 @@ //! lives here (pty.rs holds the one other unsafe block, its pre_exec //! registration). -use std::fmt::Write as _; use std::io::Read; use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; use std::process::ExitStatus; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Mutex, OnceLock}; +use std::sync::{LazyLock, Mutex}; use tokio::process::Command; @@ -29,6 +28,8 @@ const FORWARDED: [&str; 6] = [ "NO_PROXY", ]; +const HEX: [u8; 16] = *b"0123456789abcdef"; + /// A suffix unique within this process (pid + monotonic counter), enough to /// name a temp file a concurrent write in the same directory won't collide /// with. Not cryptographic — just unique. @@ -43,19 +44,33 @@ pub fn tmp_suffix() -> String { /// A 128-bit unpredictable hex token from the OS CSPRNG. Used where an /// in-sandbox command must not be able to guess or forge the value (the /// session-command sentinel). Falls back to the predictable tmp_suffix only if -/// /dev/urandom is unreadable, which never happens on a normal guest. +/// the CSPRNG is unreadable, which never happens on a normal guest. pub fn rand_token() -> String { let mut b = [0u8; 16]; - match std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut b)) { - Ok(()) => { - let mut s = String::with_capacity(32); - for x in b { - let _ = write!(s, "{x:02x}"); - } - s + if !fill_random(&mut b) { + return tmp_suffix(); + } + b.iter() + .flat_map(|byte| [HEX[(byte >> 4) as usize], HEX[(byte & 0xf) as usize]]) + .map(char::from) + .collect() +} + +/// Fills `b` from the OS CSPRNG, sparing an async caller the open/read/close a +/// tokio worker would block on. GRND_NONBLOCK never waits on an early-boot +/// pool; /dev/urandom is the same source and covers the fallback. +fn fill_random(b: &mut [u8]) -> bool { + #[cfg(target_os = "linux")] + { + // SAFETY: getrandom writes at most b.len() bytes into b's live buffer. + let n = unsafe { libc::getrandom(b.as_mut_ptr().cast(), b.len(), libc::GRND_NONBLOCK) }; + if n == b.len() as isize { + return true; } - Err(_) => tmp_suffix(), } + std::fs::File::open("/dev/urandom") + .and_then(|mut f| f.read_exact(b)) + .is_ok() } /// SIGKILLs the process group led by `pgid` — a session's shell plus its @@ -133,8 +148,9 @@ fn compose_env<'a>(nic: bool, proxy: &'a [(&'static str, String)]) -> Vec<(&'sta /// The FORWARDED values present in silkd's own environment, read once: the /// unit environment is fixed at service start and silkd never mutates it. fn proxy_vars() -> &'static [(&'static str, String)] { - static PROXY_VARS: OnceLock> = OnceLock::new(); - PROXY_VARS.get_or_init(|| snapshot_proxy(|key| std::env::var(key).ok())) + static PROXY_VARS: LazyLock> = + LazyLock::new(|| snapshot_proxy(|key| std::env::var(key).ok())); + &PROXY_VARS } fn snapshot_proxy(get: impl Fn(&str) -> Option) -> Vec<(&'static str, String)> { @@ -202,7 +218,7 @@ pub fn openpty(cols: u16, rows: u16) -> std::io::Result<(OwnedFd, OwnedFd)> { &mut slave, std::ptr::null_mut(), std::ptr::null_mut(), - std::ptr::addr_of_mut!(ws), + &raw mut ws, ) }; if rc != 0 { @@ -320,8 +336,6 @@ mod tests { assert!(env.iter().any(|(k, _)| *k == "TERM")); } - /// The no-NIC lane only reaches out if the proxy variables arrive without - /// anyone asking; nothing else of silkd's environment is the guest's. #[test] fn base_env_forwards_only_the_proxy_variables() { let proxy = snapshot_proxy(|key| match key { @@ -346,8 +360,6 @@ mod tests { ); } - /// A guest with a working NIC must not be steered into the loopback relay, - /// which is closed unless the host armed a proxy for this sandbox. #[test] fn base_env_forwards_nothing_when_a_nic_is_present() { let proxy = snapshot_proxy(|_| Some("http://127.0.0.1:3128".to_string())); @@ -358,8 +370,6 @@ mod tests { ); } - /// Env-inheriting children (git, language servers) follow the same lane - /// rule as cleared ones: scrubbed with a NIC, inherited without. #[test] fn align_proxy_env_scrubs_only_on_a_nic_lane() { let removed = |cmd: &Command| cmd.as_std().get_envs().filter(|(_, v)| v.is_none()).count(); diff --git a/silkd/src/tree.rs b/silkd/src/tree.rs index 76c9b384..5a5c5e33 100644 --- a/silkd/src/tree.rs +++ b/silkd/src/tree.rs @@ -31,7 +31,7 @@ where if let Err(e) = tokio::fs::create_dir_all(&dest).await { return err_frame(w, &e, "mkdir dest").await; } - let staging = match stage_dir(&dest) { + let staging = match stage_dir(&dest).await { Ok(s) => s, Err(e) => return err_frame(w, &e, "stage push").await, }; @@ -159,9 +159,9 @@ fn spawn_tar( /// Creates a unique staging dir inside `dest` — the same filesystem, so the /// post-extract merge is pure renames. -fn stage_dir(dest: &str) -> io::Result { +async fn stage_dir(dest: &str) -> io::Result { let staging = format!("{dest}/.silkd-push-{}", sysutil::tmp_suffix()); - std::fs::create_dir(&staging)?; + tokio::fs::create_dir(&staging).await?; Ok(staging) } @@ -189,8 +189,8 @@ fn merge_tree(src: &Path, dst: &Path) -> io::Result<()> { Ok(()) } -/// Reads a child's stderr to a string, capped so a pathological tar can't -/// balloon memory; the tail is what a human error message needs. +/// Reads a child's stderr to a string, keeping the first CAP bytes so a +/// pathological tar can't balloon memory — tar's first error is the useful one. async fn drain(mut stderr: tokio::process::ChildStderr) -> String { const CAP: usize = 16 * 1024; let mut out = Vec::new(); diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index e5b120f2..c618325d 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -10,6 +10,10 @@ use crate::proto::{self, ErrorKind, EventKind, Response}; const OVERFLOW_MESSAGE: &str = "watch event queue overflow"; +/// Events written per syscall. A build tool emits them in tight bursts, so +/// draining the queue in batches also keeps it far from its 256-slot cap. +const EVENT_BATCH: usize = 64; + /// Watches `path`, writing `ready`, ordered events, or a terminal error until disconnect. pub async fn watch( reader: &mut R, @@ -23,8 +27,10 @@ where { let (tx, mut rx) = mpsc::channel::(256); let mut tx = Some(tx); + let mut frames = Vec::new(); let mut watcher = match notify::recommended_watcher(move |res| { - forward_frames(&mut tx, to_frames(res)); + to_frames(res, &mut frames); + forward_frames(&mut tx, &mut frames); }) { Ok(watcher) => watcher, Err(e) => return proto::error_frame(w, ErrorKind::Internal, e.to_string()).await, @@ -38,18 +44,23 @@ where return proto::error_frame(w, map_kind(&e), e.to_string()).await; } proto::write_frame(w, &Response::Ready).await?; + let mut batch = Vec::new(); + let mut buf = Vec::new(); loop { tokio::select! { - frame = rx.recv() => match frame { - Some(frame) => { - let terminal = matches!(frame, Response::Error { .. }); - // A failed write is the disconnect the EOF arm can lose the select to. - if proto::write_frame(w, &frame).await.is_err() || terminal { - return Ok(()); - } - } + n = rx.recv_many(&mut batch, EVENT_BATCH) => { // Only overflow drops the sender mid-watch; the buffered prefix is already out. - None => return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await, + if n == 0 { + return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await; + } + let terminal = batch.iter().position(|f| matches!(f, Response::Error { .. })); + let upto = terminal.map_or(batch.len(), |i| i + 1); + // A failed write is the disconnect the EOF arm can lose the select to. + let sent = proto::write_frames(w, &mut buf, &batch[..upto]).await; + batch.clear(); + if sent.is_err() || terminal.is_some() { + return Ok(()); + } }, // The client sends nothing during a watch, so any readable state — // EOF (disconnect), a stray frame, or an error — ends the watch. @@ -58,9 +69,12 @@ where } } -fn forward_frames(tx: &mut Option>, frames: Vec) { - let Some(sender) = tx.as_ref() else { return }; - for frame in frames { +fn forward_frames(tx: &mut Option>, frames: &mut Vec) { + let Some(sender) = tx.as_ref() else { + frames.clear(); + return; + }; + for frame in frames.drain(..) { if sender.try_send(frame).is_err() { *tx = None; return; @@ -68,27 +82,28 @@ fn forward_frames(tx: &mut Option>, frames: Vec } } -fn to_frames(res: notify::Result) -> Vec { +/// Renders one notify event onto `out`, which the caller reuses — nearly every +/// event carries a single path. +fn to_frames(res: notify::Result, out: &mut Vec) { use notify::EventKind as N; let event = match res { Ok(event) => event, - Err(e) => return vec![Response::error(ErrorKind::Internal, e.to_string())], + Err(e) => { + out.push(Response::error(ErrorKind::Internal, e.to_string())); + return; + } }; let kind = match event.kind { N::Create(_) => EventKind::Created, N::Modify(notify::event::ModifyKind::Name(_)) => EventKind::Renamed, N::Modify(_) => EventKind::Modified, N::Remove(_) => EventKind::Deleted, - _ => return Vec::new(), + _ => return, }; - event - .paths - .into_iter() - .map(|p| Response::Event { - kind, - path: p.to_string_lossy().into_owned(), - }) - .collect() + out.extend(event.paths.into_iter().map(|p| Response::Event { + kind, + path: p.to_string_lossy().into_owned(), + })); } fn map_kind(e: ¬ify::Error) -> ErrorKind { @@ -104,7 +119,8 @@ mod tests { #[test] fn watcher_error_becomes_terminal_error_frame() { - let frames = to_frames(Err(notify::Error::generic("inotify overflow"))); + let mut frames = Vec::new(); + to_frames(Err(notify::Error::generic("inotify overflow")), &mut frames); assert_eq!(frames.len(), 1); match &frames[0] { Response::Error { message, .. } => assert!(message.contains("inotify overflow")), @@ -116,7 +132,8 @@ mod tests { fn create_event_becomes_event_frames() { let event = notify::Event::new(notify::EventKind::Create(notify::event::CreateKind::File)) .add_path("/x/y.txt".into()); - let frames = to_frames(Ok(event)); + let mut frames = Vec::new(); + to_frames(Ok(event), &mut frames); assert_eq!(frames.len(), 1); match &frames[0] { Response::Event { kind, path } => { @@ -132,9 +149,11 @@ mod tests { let (tx, mut rx) = mpsc::channel(1); let mut tx = Some(tx); - forward_frames(&mut tx, vec![Response::Ready, Response::Ready]); + let mut frames = vec![Response::Ready, Response::Ready]; + forward_frames(&mut tx, &mut frames); assert!(tx.is_none()); + assert!(frames.is_empty()); assert!(matches!(rx.try_recv(), Ok(Response::Ready))); assert!(matches!( rx.try_recv(), diff --git a/silkd/tests/common/mod.rs b/silkd/tests/common/mod.rs index 4536f449..c4eb9884 100644 --- a/silkd/tests/common/mod.rs +++ b/silkd/tests/common/mod.rs @@ -3,7 +3,7 @@ //! //! Compiled into each test binary separately, so not every helper is used by //! every binary — allow the resulting dead_code rather than fragment this. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] #![allow(dead_code)] use std::sync::Arc; @@ -19,9 +19,8 @@ use tokio::task::JoinHandle; pub type FrameWriter = WriteHalf; pub type FrameLines = Lines>>; -/// Opens a framed connection to a task serving `state`: the write half, a -/// line reader over the response frames, and the serve task handle. The split -/// halves let streaming verbs (pty, watch) interleave writes with reads. +/// Opens a framed connection to a task serving `state`: write half, response +/// lines, and the serve task handle, split so streaming verbs can interleave. pub fn connect(state: &Arc) -> (FrameWriter, FrameLines, JoinHandle>) { let (client, server) = tokio::io::duplex(1 << 20); let state = Arc::clone(state); @@ -36,9 +35,7 @@ pub async fn roundtrip(request_line: &str) -> Vec { request_on(&Arc::new(State::new()), &[request_line.to_string()]).await } -/// Runs one or more request lines on a single connection against `state` (so a -/// write/push can be followed by its data frames, and calls can be chained on -/// a shared state), returning the response frames. +/// Runs request lines on one connection against `state`, returning the frames. pub async fn request_on(state: &Arc, lines: &[String]) -> Vec { let (mut cw, mut out, handle) = connect(state); for line in lines { @@ -93,8 +90,7 @@ pub fn stdout_body(frames: &[Value]) -> String { String::from_utf8_lossy(&payload(frames, "stdout")).into_owned() } -/// A `data`-frame stream (16K chunks) terminated by `data_end`, for feeding -/// fs.write / fs.push. +/// A `data`-frame stream (16K chunks) terminated by `data_end`. pub fn data_frames(bytes: &[u8]) -> Vec { let mut lines: Vec = bytes .chunks(16 * 1024) diff --git a/silkd/tests/exec_e2e.rs b/silkd/tests/exec_e2e.rs index 6bfcc026..7606fb9d 100644 --- a/silkd/tests/exec_e2e.rs +++ b/silkd/tests/exec_e2e.rs @@ -2,7 +2,7 @@ //! (echo, sh), so it runs on any Unix — keep it that way so exec-path bugs //! surface on the dev host, not only on Linux CI. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -23,8 +23,6 @@ async fn exec_streams_stdout_then_exit() { #[tokio::test] async fn large_output_is_delivered_without_loss() { - // Exercises the backpressured foreground mpsc with far more chunks than - // its buffer; every line must arrive in order, none dropped. let frames = roundtrip(r#"{"op":"exec","argv":["/bin/sh","-c","seq 1 20000"]}"#).await; let body = stdout_body(&frames); let lines: Vec<&str> = body.lines().collect(); @@ -71,8 +69,6 @@ async fn kill_unknown_pid_is_not_found() { #[tokio::test] async fn daemonizing_child_does_not_wedge_the_exec() { - // sh forks a process that inherits stdout and outlives it. The exec must - // still return an exit frame promptly instead of blocking on pipe EOF. let frames = tokio::time::timeout( Duration::from_secs(8), roundtrip(r#"{"op":"exec","argv":["/bin/sh","-c","sleep 30 & exit 0"]}"#), @@ -86,9 +82,6 @@ async fn daemonizing_child_does_not_wedge_the_exec() { #[tokio::test] async fn daemonizer_exit_is_the_last_frame() { - // A child that leaves a grandchild holding stdout must still end its frame - // stream with exit and nothing after it — the post-exit drain aborts the - // pump, so no stray stdout can arrive once Exit is sent. let frames = tokio::time::timeout( Duration::from_secs(8), roundtrip(r#"{"op":"exec","argv":["/bin/sh","-c","(sleep 5; echo late) & exit 0"]}"#), @@ -124,8 +117,6 @@ async fn detached_pid(state: &Arc, script: &str) -> u64 { async fn attach_streams_live_output_then_exit() { let state = Arc::new(State::new()); let pid = detached_pid(&state, "sleep 0.3; echo live-chunk").await; - // Attach before the echo fires: the chunk must arrive live and the stream - // must end with exit. let frames = tokio::time::timeout( Duration::from_secs(8), one(&state, &format!(r#"{{"op":"attach","pid":{pid}}}"#)), @@ -171,8 +162,6 @@ async fn attach_unknown_pid_is_not_found() { async fn kill_actually_terminates_a_live_process() { let state = Arc::new(State::new()); let pid = detached_pid(&state, "sleep 30").await; - // Wait for it to register, then kill and confirm it reaches exited well - // before the 30s sleep would end. for _ in 0..50 { let ps = one(&state, r#"{"op":"ps"}"#).await; if ps[0]["procs"] @@ -217,7 +206,6 @@ async fn exec_missing_binary_is_an_internal_error() { #[tokio::test] async fn exec_stdin_is_piped_to_the_child() { - // cat echoes its stdin; feed a stdin frame, close it, expect the bytes back. let frames = exchange(&[ r#"{"op":"exec","argv":["/bin/cat"]}"#.to_string(), serde_json::json!({"op":"stdin","data":common::b64(b"piped-in\n")}).to_string(), @@ -292,9 +280,6 @@ async fn detached_exec_is_listed_then_logs_replay_output_and_exit() { #[tokio::test] async fn disconnect_during_drain_publishes_real_exit_code() { - // A grandchild holds stdout so the supervisor sits in the post-exit - // drain; the foreground client then vanishes mid-drain. The disconnect - // fallback must publish the child's real code to attachers, not -1. use tokio::io::AsyncWriteExt; let state = Arc::new(State::new()); @@ -312,9 +297,6 @@ async fn disconnect_during_drain_publishes_real_exit_code() { assert_eq!(type_of(&started), "started"); let pid = started["pid"].as_u64().unwrap(); - // Attach while the proc is live, then drop the exec client: writing the - // grandchild's "late" chunk fails and takes the disconnect path while - // the supervisor is still mid-drain. let st = Arc::clone(&state); let attach = tokio::spawn(async move { one( diff --git a/silkd/tests/find_e2e.rs b/silkd/tests/find_e2e.rs index a9b82b34..57f581ee 100644 --- a/silkd/tests/find_e2e.rs +++ b/silkd/tests/find_e2e.rs @@ -1,6 +1,6 @@ //! Search verb E2E: fs.find streams matches, fs.replace rewrites files. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use common::{exchange, type_of}; @@ -34,8 +34,8 @@ async fn find_streams_line_matches() { .iter() .all(|m| m["content"].as_str().unwrap().contains("TODO")) ); - assert!(matches.iter().any(|m| m["line"] == 2)); // a.rs - assert!(matches.iter().any(|m| m["line"] == 1)); // sub/c.rs + assert!(matches.iter().any(|m| m["line"] == 2)); + assert!(matches.iter().any(|m| m["line"] == 1)); assert_eq!(type_of(frames.last().unwrap()), "done"); } @@ -45,7 +45,6 @@ async fn find_glob_is_a_real_glob_not_a_substring() { write_file(dir.path(), "a.rs", "TODO\n").await; write_file(dir.path(), "a.rs.bak", "TODO\n").await; - // Anchored: "*.rs" must not match the .bak; "?" is a single character. let frames = exchange(&[json!({ "op": "fs_find", "path": dir.path().to_str().unwrap(), @@ -123,6 +122,55 @@ async fn replace_rewrites_and_counts() { ); } +#[tokio::test] +async fn replace_skips_a_file_over_the_size_bound() { + let dir = tempfile::tempdir().unwrap(); + let big = dir.path().join("big.log"); + let body = format!("foo\n{}", "x".repeat(8 * 1024 * 1024)); + tokio::fs::write(&big, &body).await.unwrap(); + + let frames = exchange(&[json!({ + "op": "fs_replace", + "files": [big.to_str().unwrap()], + "pattern": "foo", + "replacement": "bar" + }) + .to_string()]) + .await; + + let replaced: Vec<_> = frames.iter().filter(|f| type_of(f) == "replaced").collect(); + assert_eq!(replaced.len(), 1, "{frames:?}"); + assert_eq!(replaced[0]["replacements"], 0); + assert_eq!(type_of(frames.last().unwrap()), "done"); + assert_eq!( + tokio::fs::read_to_string(&big).await.unwrap().len(), + body.len(), + "an oversized file must not be rewritten" + ); +} + +#[tokio::test] +async fn find_skips_a_file_over_the_size_bound() { + let dir = tempfile::tempdir().unwrap(); + tokio::fs::write( + dir.path().join("big.log"), + format!("foo\n{}", "x".repeat(8 * 1024 * 1024)), + ) + .await + .unwrap(); + + let frames = exchange(&[json!({ + "op": "fs_find", + "path": dir.path().to_str().unwrap(), + "pattern": "foo" + }) + .to_string()]) + .await; + + assert_eq!(frames.iter().filter(|f| type_of(f) == "match").count(), 0); + assert_eq!(type_of(frames.last().unwrap()), "done"); +} + #[tokio::test] async fn replace_preserves_exec_bit() { use std::os::unix::fs::PermissionsExt; diff --git a/silkd/tests/forward_e2e.rs b/silkd/tests/forward_e2e.rs index 50e9ba48..cdc4979e 100644 --- a/silkd/tests/forward_e2e.rs +++ b/silkd/tests/forward_e2e.rs @@ -2,7 +2,9 @@ //! test runs under a deadline — the regressions this suite exists to catch //! (relay deadlocks) would otherwise hang CI instead of failing it. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] +mod common; + use std::sync::Arc; use std::time::Duration; @@ -14,8 +16,6 @@ use tokio::time::timeout; use silkd::server::State; -mod common; - const DEADLINE: Duration = Duration::from_secs(30); async fn echo_listener() -> u16 { @@ -54,7 +54,6 @@ async fn forward_round_trips_and_done_on_server_close() { let ready = lines.next_line().await.expect("read").expect("ready"); assert!(ready.contains("\"ready\""), "got {ready}"); - // "hi" → base64 aGk= ; the echo server must send it straight back. cw.write_all(b"{\"v\":1,\"op\":\"data\",\"data\":\"aGk=\"}\n") .await .expect("send data"); @@ -64,7 +63,6 @@ async fn forward_round_trips_and_done_on_server_close() { "got {echoed}" ); - // Half-close: the echo server sees EOF, closes, and the verb ends with done. cw.write_all(b"{\"v\":1,\"op\":\"data_end\"}\n") .await .expect("send data_end"); @@ -75,15 +73,10 @@ async fn forward_round_trips_and_done_on_server_close() { .expect("test deadline"); } -// Bidirectional bulk transfer must not head-of-line deadlock: the client -// streams > socket-buffer worth of input into an echo server that is -// simultaneously streaming it all back, so both directions have to make -// progress concurrently. A single-select relay (write_all stalling the read -// arm) hangs this test. #[tokio::test] async fn forward_bidirectional_bulk_no_deadlock() { - const CHUNK: usize = 3072; // multiple of 3: base64 with no padding - const FRAMES: usize = 400; // ~1.2 MiB, well past socket + duplex buffers + const CHUNK: usize = 3072; + const FRAMES: usize = 400; timeout(DEADLINE, async { let port = echo_listener().await; @@ -138,7 +131,6 @@ async fn forward_refused_port_is_not_found() { let state = Arc::new(State::new()); let (mut cw, mut lines, _) = common::connect(&state); - // Port 1 on loopback: nothing listens there in the test environment. cw.write_all(b"{\"v\":1,\"op\":\"port_forward\",\"port\":1}\n") .await .expect("send request"); @@ -165,7 +157,6 @@ async fn forward_stray_frame_is_bad_request() { let ready = lines.next_line().await.expect("read").expect("ready"); assert!(ready.contains("\"ready\""), "got {ready}"); - // A non-data frame mid-relay is a protocol violation, not a clean end. cw.write_all(b"{\"v\":1,\"op\":\"ps\"}\n") .await .expect("send stray"); diff --git a/silkd/tests/fs_e2e.rs b/silkd/tests/fs_e2e.rs index 7b12e2c1..ff660959 100644 --- a/silkd/tests/fs_e2e.rs +++ b/silkd/tests/fs_e2e.rs @@ -1,6 +1,6 @@ //! Filesystem verb E2E over the in-memory duplex, against real temp paths. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use common::{b64, exchange, type_of}; @@ -174,7 +174,6 @@ async fn overwrite_preserves_destination_mode() { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("script.sh"); - // Original file is executable (0755); overwrite without specifying a mode. std::fs::write(&path, b"old").unwrap(); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); let f = exchange(&[ diff --git a/silkd/tests/git_e2e.rs b/silkd/tests/git_e2e.rs index c4847afe..6c758db6 100644 --- a/silkd/tests/git_e2e.rs +++ b/silkd/tests/git_e2e.rs @@ -1,7 +1,7 @@ //! git verb E2E against a real git binary in temp repos. Local verbs run on //! any host; the none-lane guard is exercised via the test lane override. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use common::{exchange, type_of}; @@ -39,7 +39,6 @@ async fn add_commit_status_reports_structured_state() { let p = repo.path().to_str().unwrap(); std::fs::write(repo.path().join("a.txt"), "hello\n").unwrap(); - // Untracked file shows in status. let st = exchange(&[json!({"op":"git_status","path":p}).to_string()]).await; assert_eq!(type_of(&st[0]), "git_status_result"); assert_eq!(st[0]["branch"], "main"); @@ -50,7 +49,6 @@ async fn add_commit_status_reports_structured_state() { .any(|f| f["path"] == "a.txt" && f["staged"] == "?") ); - // Stage and commit; a hash comes back. let add = exchange(&[json!({"op":"git_add","path":p,"files":["a.txt"]}).to_string()]).await; assert_eq!(type_of(last(&add)), "done"); let commit = exchange(&[json!({ @@ -61,7 +59,6 @@ async fn add_commit_status_reports_structured_state() { assert_eq!(type_of(&commit[0]), "git_commit_result"); assert_eq!(commit[0]["hash"].as_str().unwrap().len(), 40); - // Clean tree now. let st2 = exchange(&[json!({"op":"git_status","path":p}).to_string()]).await; assert!( st2[0]["files"].as_array().unwrap().is_empty(), @@ -103,7 +100,7 @@ async fn branch_create_list_checkout() { #[tokio::test] async fn commit_failure_surfaces_git_error() { - let repo = init_repo(); // nothing staged + let repo = init_repo(); let p = repo.path().to_str().unwrap(); let frames = exchange(&[json!({ "op":"git_commit","path":p,"message":"empty","author":"Dev " @@ -113,10 +110,8 @@ async fn commit_failure_surfaces_git_error() { assert_eq!(type_of(&frames[0]), "error"); } -// One test owns the lane override so the two cases cannot fight over it. #[tokio::test] async fn clone_lane_behavior() { - // none lane refuses with a typed error pointing at fs.push. silkd::net::override_egress_for_tests(Some(false)); let none = exchange(&[json!({ "op":"git_clone","url":"https://example.com/x.git","path":"/tmp/silkd-x" @@ -127,7 +122,6 @@ async fn clone_lane_behavior() { assert_eq!(none[0]["kind"], "unimplemented"); assert!(none[0]["message"].as_str().unwrap().contains("fs.push")); - // egress lane clones (exercised against a local source repo). let src = init_repo(); std::fs::write(src.path().join("r.txt"), "body\n").unwrap(); git(src.path(), &["add", "r.txt"]); diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index 9a4fb90f..c84d1e49 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -4,7 +4,7 @@ //! real language server. Every test runs under a deadline so a relay deadlock //! fails CI instead of hanging it. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] use std::io::Write; use std::sync::Arc; use std::time::Duration; @@ -21,11 +21,8 @@ use common::{b64, connect, decode, one, type_of}; const DEADLINE: Duration = Duration::from_secs(30); -// SILKD_LSP_DIR is process-global; serialize the tests that set it. -static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); +static ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); -// A fake server that echoes each line it reads, prefixed — enough to prove -// the round trip without LSP semantics. const FAKE_SERVER: &str = r#"#!/bin/sh IFS= read -r line printf 'reply:%s\n' "$line" @@ -77,12 +74,8 @@ async fn lsp_start_language_name_cannot_escape() { } #[tokio::test] -#[allow(clippy::await_holding_lock)] // ENV_LOCK serializes SILKD_LSP_DIR across the whole test async fn lsp_broker_relays_to_the_server() { - // Point the broker at a manifest dir under our control: SILKD_LSP_DIR - // overrides /etc/silkd/lsp.d, and the planted manifest names the fake - // server as the language server for "faketest". - let _env_lock = ENV_LOCK.lock().unwrap(); + let _env_lock = ENV_LOCK.lock().await; let env = manifest_env(FAKE_SERVER); let bin = env.path().join("fake-lsp"); std::fs::write( @@ -105,7 +98,6 @@ async fn lsp_broker_relays_to_the_server() { .unwrap() .to_string(); - // Attach and send one JSON-RPC-ish line; expect the echoed reply back. let (mut cw, mut out, handle) = connect(&state); cw.write_all( json!({"op":"lsp_request","server_id":server_id}) @@ -136,8 +128,6 @@ async fn lsp_broker_relays_to_the_server() { assert_eq!(type_of(&frame), "data"); assert_eq!(decode(&frame), b"reply:ping\n"); - // The fake server exits after one reply: its stdout EOFs, so the request - // ends with Done and the broker reaps the server. let done = timeout(DEADLINE, out.next_line()) .await .expect("deadline") @@ -150,7 +140,6 @@ async fn lsp_broker_relays_to_the_server() { cw.shutdown().await.unwrap(); let _ = handle.await; - // Reaped: a later lsp_stop is not_found. let gone = one( &state, &json!({"op":"lsp_stop","server_id":server_id}).to_string(), @@ -160,9 +149,8 @@ async fn lsp_broker_relays_to_the_server() { } #[tokio::test] -#[allow(clippy::await_holding_lock)] // ENV_LOCK serializes SILKD_LSP_DIR across the whole test async fn lsp_stop_kills_an_idle_server() { - let _env_lock = ENV_LOCK.lock().unwrap(); + let _env_lock = ENV_LOCK.lock().await; let env = manifest_env("#!/bin/sh\nsleep 60\n"); std::fs::write( env.path().join("idletest"), @@ -197,12 +185,8 @@ async fn lsp_stop_kills_an_idle_server() { } #[tokio::test] -#[allow(clippy::await_holding_lock)] // ENV_LOCK serializes SILKD_LSP_DIR across the whole test async fn lsp_data_end_half_closes_stdin() { - // The fake server drains stdin to EOF before replying — it can only - // answer if data_end actually closes its stdin. - let _env_lock = ENV_LOCK.lock().unwrap(); - // unquoted $(...) word-splits away wc's platform-dependent padding + let _env_lock = ENV_LOCK.lock().await; let env = manifest_env("#!/bin/sh\nprintf 'ate:%s\\n' $(cat | wc -c)\n"); std::fs::write( env.path().join("eoftest"), diff --git a/silkd/tests/pty_e2e.rs b/silkd/tests/pty_e2e.rs index c8052503..571ca87d 100644 --- a/silkd/tests/pty_e2e.rs +++ b/silkd/tests/pty_e2e.rs @@ -1,7 +1,7 @@ //! pty.open E2E: a shell under a pseudo-terminal, driven over a duplex. //! openpty exists on macOS and Linux, so this runs on the dev host. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -42,7 +42,6 @@ async fn pty_runs_a_shell_and_echoes() { let pid = started["pid"].as_u64().unwrap(); assert!(pid > 0); - // A pty echoes input; run a command and look for its output. send( &mut cw, json!({"op":"stdin","data":common::b64(b"echo silk-pty-marker\n")}), @@ -72,7 +71,6 @@ async fn pty_appears_in_ps_and_resizes() { .as_u64() .unwrap(); - // ps (shared table) sees the pty as a process; resize succeeds on it. let ps = common::one(&state, r#"{"op":"ps"}"#).await; assert!( ps[0]["procs"] @@ -105,9 +103,6 @@ async fn resize_unknown_pid_is_not_found() { #[tokio::test] async fn pty_disconnect_tears_down_without_spin() { - // Client disconnects while the shell is idle. The pty must tear down (kill - // the shell, drop the table entry, end the serve task) instead of - // busy-spinning on the closed client channel. let state = Arc::new(State::new()); let (mut cw, mut lines, handle) = common::connect(&state); send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; @@ -115,13 +110,10 @@ async fn pty_disconnect_tears_down_without_spin() { .as_u64() .unwrap(); - // Let the shell settle, then disconnect. tokio::time::sleep(Duration::from_millis(200)).await; drop(lines); drop(cw); - // The serve task returns promptly (no spin); the final Exit-write to the - // gone client may error, which is fine — the point is it returns. let _ = tokio::time::timeout(Duration::from_secs(5), handle) .await .expect("pty did not tear down on disconnect"); diff --git a/silkd/tests/session_e2e.rs b/silkd/tests/session_e2e.rs index b16b4018..677ec2a7 100644 --- a/silkd/tests/session_e2e.rs +++ b/silkd/tests/session_e2e.rs @@ -2,7 +2,7 @@ //! exec calls. Spawns real bash, so Unix-only in practice; kept ungated so //! the persistence contract is checked on the dev host too. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -59,7 +59,6 @@ async fn session_applies_cwd_and_env_at_create() { let g = sh(&state, &id, &["sh", "-c", "echo $GREETING"]).await; assert_eq!(stdout_body(&g).trim(), "hello-silk"); - // cwd applied: a relative touch lands in the session's directory. let t = sh(&state, &id, &["touch", "marker"]).await; assert_eq!(type_of(t.last().unwrap()), "exit"); assert!(dir.path().join("marker").exists(), "cwd was not applied"); @@ -100,16 +99,12 @@ async fn session_list_and_rm() { #[tokio::test] async fn stdin_reading_command_does_not_wedge_the_session() { - // A command that reads stdin (here `cat`, which would block forever on the - // control pipe) must not hang the session — its stdin is isolated. Bound - // with a timeout: a regression would hang instead of returning. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let r = tokio::time::timeout(std::time::Duration::from_secs(8), sh(&state, &id, &["cat"])) .await .expect("session wedged on a stdin-reading command"); assert_eq!(type_of(r.last().unwrap()), "exit"); - // The session is still usable afterward. let after = sh(&state, &id, &["echo", "alive"]).await; assert_eq!(stdout_body(&after).trim(), "alive"); } @@ -118,7 +113,6 @@ async fn stdin_reading_command_does_not_wedge_the_session() { async fn shell_killing_command_removes_the_session() { let state = Arc::new(State::new()); let id = create(&state, json!({})).await; - // `exit` kills the shell; the session must then stop resolving. let _ = sh(&state, &id, &["exit"]).await; let again = sh(&state, &id, &["echo", "hi"]).await; assert_eq!(type_of(&again[0]), "error"); @@ -127,9 +121,6 @@ async fn shell_killing_command_removes_the_session() { #[tokio::test] async fn forged_sentinel_in_output_does_not_desync() { - // A command printing a plausible sentinel must not be mistaken for the - // real one (random token) — the real command output is preserved and the - // next command still frames correctly. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let r = sh( @@ -155,7 +146,6 @@ async fn reap_idle_removes_idle_sessions() { use std::time::Duration; let state = Arc::new(State::new()); let id = create(&state, json!({})).await; - // Everything is "idle" against a zero TTL. let reaped = state.sessions.reap_idle(Duration::ZERO); assert_eq!(reaped, 1); let gone = sh(&state, &id, &["echo", "hi"]).await; @@ -171,8 +161,6 @@ async fn reap_skips_a_session_running_a_command() { use std::time::Duration; let state = Arc::new(State::new()); let id = create(&state, json!({})).await; - // Hold the session busy with a slow command while a zero-TTL reap runs; - // an actively-running session must not be reaped. let busy = { let s = Arc::clone(&state); let sid = id.clone(); @@ -183,14 +171,11 @@ async fn reap_skips_a_session_running_a_command() { assert_eq!(reaped, 0, "a running session must not be reaped"); let done = busy.await.unwrap(); assert_eq!(type_of(done.last().unwrap()), "exit"); - // After it finishes it becomes reapable again. assert_eq!(state.sessions.reap_idle(Duration::ZERO), 1); } #[tokio::test] async fn session_argv_with_metacharacters_is_one_literal_token() { - // A single argv element containing shell metacharacters must be passed as - // one literal argument, not interpreted — shell_quote must neutralize it. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let evil = "a b; echo INJECTED $(id) `whoami` 'q'"; @@ -208,10 +193,6 @@ async fn session_argv_with_metacharacters_is_one_literal_token() { #[tokio::test] async fn session_rm_unwedges_a_running_external_command() { - // A session running a long external command blocks its run() until the - // command exits. session_rm must group-kill the shell AND the command so - // the blocked run() unwinds — killing only bash would leave the sleep - // holding the stdout pipe open. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let busy = { @@ -222,7 +203,6 @@ async fn session_rm_unwedges_a_running_external_command() { tokio::time::sleep(std::time::Duration::from_millis(200)).await; let rm = one(&state, &json!({"op":"session_rm","id":id}).to_string()).await; assert_eq!(type_of(&rm[0]), "done"); - // The blocked command must now return (shell group killed) within a bound. let done = tokio::time::timeout(std::time::Duration::from_secs(5), busy) .await .expect("session_rm did not unwedge the external command"); @@ -232,8 +212,6 @@ async fn session_rm_unwedges_a_running_external_command() { #[tokio::test] async fn empty_argv_in_session_is_a_bad_request() { - // The dispatcher validates argv ahead of the session/process split: an - // empty argv must not reach the shell as a successful no-op. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let f = one( @@ -247,8 +225,6 @@ async fn empty_argv_in_session_is_a_bad_request() { #[tokio::test] async fn detach_with_session_is_a_bad_request() { - // silkd's session path has no Started frame to answer a detached exec, so - // the combination must be rejected up front rather than silently run. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; let f = one( @@ -270,8 +246,6 @@ async fn exec_in_unknown_session_is_not_found() { #[tokio::test] async fn session_merges_stderr_into_stdout() { - // The shell's first init line is `exec 2>&1`: command stderr must arrive - // on the session's single output stream. let state = Arc::new(State::new()); let id = create(&state, json!({})).await; diff --git a/silkd/tests/tree_e2e.rs b/silkd/tests/tree_e2e.rs index aa27a6dc..6dd4ddc7 100644 --- a/silkd/tests/tree_e2e.rs +++ b/silkd/tests/tree_e2e.rs @@ -1,7 +1,7 @@ //! Whole-tree transfer E2E: push a tar stream in, pull a tar stream out, //! using the system tar to build/extract the reference archives. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::path::Path; @@ -76,7 +76,6 @@ async fn pull_streams_a_tar_of_the_path() { let out = tempfile::tempdir().unwrap(); sys_tar_extract(&archive, out.path()); - // the archive carries the entry under its own name (proj/…) assert_eq!( std::fs::read(out.path().join("proj/x.txt")).unwrap(), b"hello-pull" @@ -108,8 +107,6 @@ async fn push_then_pull_roundtrips_a_tree() { #[tokio::test] async fn push_truncated_stream_errors() { - // Send fs_push + a data frame but no data_end: tar gets a partial archive - // and the handler must report an error, not done. let dest = tempfile::tempdir().unwrap(); let frames = exchange(&[ json!({"op":"fs_push","dest":dest.path().join("p").to_str().unwrap()}).to_string(), @@ -125,7 +122,6 @@ async fn push_truncated_stream_errors() { #[tokio::test] async fn push_bad_archive_reports_tar_failure() { - // A complete stream that isn't a valid tar → tar exits nonzero → Internal. let dest = tempfile::tempdir().unwrap(); let frames = exchange(&[ json!({"op":"fs_push","dest":dest.path().join("p").to_str().unwrap()}).to_string(), @@ -139,7 +135,6 @@ async fn push_bad_archive_reports_tar_failure() { #[tokio::test] async fn pull_missing_path_errors() { - // Parent exists; the leaf itself is missing. let dir = tempfile::tempdir().unwrap(); let missing = dir.path().join("xyz"); let frames = @@ -195,8 +190,6 @@ async fn pull_dangling_symlink_archives_the_link() { #[tokio::test] async fn push_failure_leaves_dest_untouched() { - // Atomicity: a truncated stream must not mutate a populated dest at all — - // no partial files, no staging leftovers. let dest = tempfile::tempdir().unwrap(); std::fs::write(dest.path().join("keep.txt"), b"KEEP").unwrap(); std::fs::create_dir(dest.path().join("sub")).unwrap(); @@ -228,8 +221,6 @@ async fn push_failure_leaves_dest_untouched() { #[tokio::test] async fn push_overlays_existing_tree_and_cleans_staging() { - // tar semantics survive the staged merge: dirs merge recursively, an - // existing file is replaced, unrelated files survive, staging vanishes. let src = tempfile::tempdir().unwrap(); std::fs::create_dir(src.path().join("sub")).unwrap(); std::fs::write(src.path().join("sub/new.txt"), b"NEW").unwrap(); diff --git a/silkd/tests/watch_e2e.rs b/silkd/tests/watch_e2e.rs index 3bd5fb9b..dabd5a99 100644 --- a/silkd/tests/watch_e2e.rs +++ b/silkd/tests/watch_e2e.rs @@ -1,6 +1,6 @@ //! fs.watch E2E: a connection-bound event stream ended by client disconnect. -#![allow(clippy::unwrap_used, clippy::expect_used)] // test code, like #[cfg(test)] +#![allow(clippy::unwrap_used, clippy::expect_used)] mod common; use std::sync::Arc; @@ -24,7 +24,6 @@ async fn watch_streams_events_until_disconnect() { cw.write_all(req.as_bytes()).await.unwrap(); cw.write_all(b"\n").await.unwrap(); - // The ready frame guarantees the watch is armed — no arming sleep needed. let ready = tokio::time::timeout(Duration::from_secs(5), lines.next_line()) .await .expect("no ready within 5s") @@ -45,7 +44,6 @@ async fn watch_streams_events_until_disconnect() { assert_eq!(frame["type"], "event", "got {frame:?}"); assert!(frame["path"].as_str().unwrap().ends_with("new.txt")); - // Disconnect ends the watch: the server task returns promptly. drop(lines); drop(cw); tokio::time::timeout(Duration::from_secs(5), handle) From acee313bef51b1daa373a1e408e8dcc0d460549c Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 3 Sep 2026 03:39:36 +0800 Subject: [PATCH 3/6] =?UTF-8?q?review:=20Go=20SDK,=20wire,=20MCP,=20e2e=20?= =?UTF-8?q?=E2=80=94=20chunked=20pty=20writes,=20compile-checked=20decoder?= =?UTF-8?q?=20tables,=20comment=20budget;=20README=20and=20Makefile=20go.w?= =?UTF-8?q?ork=20wording?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pty.Write chunks like every other writer so a large paste cannot exceed the frame cap; the decoder tables use named pointer constraints so a mispaired entry fails to compile; frameTag returns bytes (one allocation off every decoded frame); apiError drains its body for connection reuse. README and Makefile now say why GOWORK=off is needed: the workspace includes these modules, CI has none. --- Makefile | 4 +-- README.md | 7 +++-- docs/sdk.md | 13 +++++---- e2e/cmd/androidsmoke/main.go | 4 +-- e2e/e2e_test.go | 28 ++++--------------- e2e/fakeengine_test.go | 10 +++---- mcp/server_test.go | 2 +- protocol/wire/frame.go | 32 +++++++++++++-------- protocol/wire/frame_test.go | 20 ++----------- sdk/go/checkpoint_test.go | 10 ------- sdk/go/claimopts_test.go | 2 -- sdk/go/client.go | 4 ++- sdk/go/client_test.go | 27 ++---------------- sdk/go/files_test.go | 4 +-- sdk/go/port_test.go | 2 +- sdk/go/proc_test.go | 10 +++---- sdk/go/pty.go | 14 +++++++--- sdk/go/pty_test.go | 51 ++++++++++++++++++++++++++++++++++ sdk/go/sandbox_test.go | 6 ---- sdk/go/silkd/conn_test.go | 3 -- sdk/go/silkd/silkdtest/fake.go | 7 ++++- sdk/go/utils.go | 20 +++++++------ sdk/go/volumes_test.go | 3 +- 23 files changed, 140 insertions(+), 143 deletions(-) diff --git a/Makefile b/Makefile index d7659622..ae7b48e6 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ SANDBOXD_VERSION ?= $(shell git describe --tags --match 'v*' --always --dirty) EXTRACT_IMAGE ?= $(BOOT_IMAGE) -# The parent workspace's go.work excludes these modules; GOWORK=off keeps -# local invocations identical to CI. +# The parent workspace's go.work includes these modules; GOWORK=off keeps +# local invocations identical to CI, which has no workspace. GO_MODULES := protocol/wire sandboxd sdk/go e2e mcp GO_OSES := linux darwin diff --git a/README.md b/README.md index f6651865..eb23013a 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ performance) — source in writable), signed preview URLs, the HTTP-upgrade byte relay to silkd, usage + audit journals, /metrics, reap + restart reconcile, memberlist mesh with redirect placement -- `sdk/go/` — Go SDK (stdlib-only): `Connect/New/Lookup`, `Exec/Run`, files, +- `sdk/go/` — Go SDK: `Connect/New/Lookup`, `Exec/Run`, files, `Push/Pull`, sessions, `Find/Replace`, `Watch`, git verbs, `OpenPty`, `Fork/Hibernate/Promote/Checkpoint`, `DialPort/ProxyPort/PreviewURL`, `StartLsp`, `Spawn/Ps/Kill/Logs/Attach`; `protocol/wire` carries the frame @@ -87,8 +87,9 @@ make silkd-image # silkd release binary in a scratch carrier image make images # base + python images against the local boot + silkd images ``` -The parent workspace's `go.work` excludes these modules; the Makefile forces -`GOWORK=off` so local runs match CI. silkd's integration tests spawn real +The parent workspace's `go.work` includes these modules, which is why the +Makefile forces `GOWORK=off`: CI has no workspace, and a workspace build +resolves sibling checkouts instead of the pinned versions. silkd's integration tests spawn real processes — run them in a Linux container too (`docker run rust:1 … cargo test`) before touching platform-sensitive paths; macOS green alone has hidden Linux-only breakage before. diff --git a/docs/sdk.md b/docs/sdk.md index ed0963b0..577aa31d 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -4,9 +4,10 @@ import sandbox "github.com/cocoonstack/sandbox/sdk/go" ``` -The SDK is stdlib-only. One `Client` talks to one entry node; sandbox -handles dial their owning node directly, so a client works unchanged against -a single node or a cluster. +The SDK has no third-party dependencies (only the sibling `protocol/wire` +module). One `Client` talks to one entry node; sandbox handles dial their +owning node directly, so a client works unchanged against a single node or a +cluster. ## A complete example @@ -529,5 +530,7 @@ if errors.As(err, &e) && e.Kind == wire.KindUnimplemented { } ``` -Context cancellation is honored on every call: canceling the ctx closes the -underlying connection and the call returns `ctx.Err()`. +Context cancellation is honored on every call that takes a ctx: canceling it +closes the underlying connection. Data-plane calls return `ctx.Err()` +directly; control-plane verbs return a `*url.Error` wrapping it, so test with +`errors.Is`, never `==`. `Close` is the exception: it takes no ctx. diff --git a/e2e/cmd/androidsmoke/main.go b/e2e/cmd/androidsmoke/main.go index 63448bd3..0456384e 100644 --- a/e2e/cmd/androidsmoke/main.go +++ b/e2e/cmd/androidsmoke/main.go @@ -96,7 +96,7 @@ func initTree(ctx context.Context, sb *sandbox.Sandbox) error { return nil } if time.Now().After(deadline) { - return fmt.Errorf("framework never came up:\n%s", tail(out)) + return fmt.Errorf("framework never came up:\n%s", head(out)) } time.Sleep(3 * time.Second) } @@ -151,7 +151,7 @@ func dialAdb(ctx context.Context, sb *sandbox.Sandbox) (string, error) { return reply, nil } -func tail(s string) string { +func head(s string) string { lines := strings.Split(strings.TrimSpace(s), "\n") if len(lines) > 12 { lines = lines[:12] diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 3a96f6ba..b4ae082a 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -49,8 +49,6 @@ func TestEndToEnd(t *testing.T) { }) t.Run("refill then warm claim", func(t *testing.T) { - // Warm-hit = zero VM ops is proven by the pool unit tests; asserting - // create counts here would race the background refill ticker. waitFor(t, func() bool { infos, _ := stack.mgr.Info() return len(infos) == 1 && infos[0].Warm >= 1 @@ -102,8 +100,7 @@ func TestForkEndToEnd(t *testing.T) { t.Fatalf("child %d exec: %q, %v", i, out, err) } } - // Children are independent claims: releasing one leaves the sibling and - // the parent alive. + if err := children[0].Close(); err != nil { t.Fatalf("close child 0: %v", err) } @@ -131,8 +128,7 @@ func TestPromoteEndToEnd(t *testing.T) { if tpl.ContentDigest == "" { t.Fatal("Promote returned an empty content digest") } - // Both claim surfaces must work: the owner-bound handle and the - // name-based Client call on the (single) node that holds the template. + child, err := tpl.New(t.Context()) if err != nil { t.Fatalf("claim via template handle: %v", err) @@ -206,9 +202,6 @@ func TestCheckpointEndToEnd(t *testing.T) { } } -// TestTwoTenantFlow drives two tenants through the real stack: each claims -// under its own token, quotas bind per tenant, checkpoint listings are -// isolated, and operator surfaces refuse tenant tokens with 403. func TestTwoTenantFlow(t *testing.T) { tenants := []config.TenantSpec{ {Name: "acme", Token: "acme-tok", MaxClaims: 1}, @@ -354,9 +347,6 @@ func TestVolumesEndToEnd(t *testing.T) { } } -// TestWritableVolumeEndToEnd drives one writable claim through the whole -// stack: the SDK's mode reaches the engine as a writable attach, a live writer -// refuses every other claim on the name, and release unmounts before removal. func TestWritableVolumeEndToEnd(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -389,9 +379,8 @@ func TestWritableVolumeEndToEnd(t *testing.T) { } } -// TestVolumeModeWireShape pins the claim reply's volume bytes independently of -// the SDK mirror. The read-only leg runs second on purpose: it is admitted -// only because the writer's release cleared the dirty marker. +// The read-only leg runs second: it is admitted only once the writer's +// release has cleared the dirty marker. func TestVolumeModeWireShape(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -448,10 +437,6 @@ func TestClaimRefRoundTrip(t *testing.T) { } } -// TestAttachOnlyVolumeEndToEnd drives one attach-only writable claim through -// the whole stack: the device is attached writable and nothing else happens — -// no mount, no marker, no unmount at release — while admission still excludes -// every other claim on the name, which is what protects third parties. func TestAttachOnlyVolumeEndToEnd(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -487,9 +472,6 @@ func TestAttachOnlyVolumeEndToEnd(t *testing.T) { assertNoDirtyMarker(t, scratch, "release") } -// TestAttachOnlyVolumeWireShape pins both claim replies' volume bytes: an -// attach-only entry echoes without a mount, the eager entry is unchanged, and -// the request flag never rides back in either. func TestAttachOnlyVolumeWireShape(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") stack := startTenantStack(t, "node-token", nil, @@ -570,7 +552,7 @@ func startStack(t *testing.T, apiToken string, pools ...config.PoolSpec) *stack func startTenantStack(t *testing.T, apiToken string, tenants []config.TenantSpec, volumes []config.VolumeSpec, pools ...config.PoolSpec) *stack { t.Helper() - // Short prefix: the sockets under it must fit darwin's 104-byte sun_path. + dir, err := os.MkdirTemp("", "sbx") if err != nil { t.Fatalf("setup: %v", err) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 7e612422..4ebb5cc8 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -16,9 +16,8 @@ import ( "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" ) -// fakeEngine replaces only the cocoon CLI: every "VM" it creates is a -// silkdtest daemon behind a real hybrid-vsock UDS, so probing and the data -// plane run the production code paths end to end. +// fakeEngine replaces only the cocoon CLI: every "VM" is a silkdtest daemon +// behind a real hybrid-vsock UDS, so the data plane runs production code. type fakeEngine struct { real *engine.Engine dir string @@ -82,9 +81,8 @@ func (f *fakeEngine) SnapshotRemove(_ context.Context, _ string) error { return func (f *fakeEngine) SnapshotList(_ context.Context) ([]string, error) { return nil, nil } -// Hibernate closes the VM's silkd listener (the "VMM died") and Restore -// brings a fresh one up, mirroring the real stop/resume observable from the -// control plane. +// Hibernate closes the VM's silkd listener and Restore brings a fresh one up, +// mirroring the stop/resume the control plane observes. func (f *fakeEngine) Hibernate(ctx context.Context, name, _ string) error { return f.Remove(ctx, name) } diff --git a/mcp/server_test.go b/mcp/server_test.go index b0495448..4860bef7 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -81,7 +81,7 @@ func TestServeSpeaksMCP(t *testing.T) { if got := toolText(t, replies[5]); got != "released" { t.Errorf("release: %q", got) } - // exec against the released sandbox: an in-band tool error, not a crash. + execReply := replies[6]["result"].(map[string]any) if execReply["isError"] != true { t.Errorf("exec after release: %+v, want isError", execReply) diff --git a/protocol/wire/frame.go b/protocol/wire/frame.go index dbcd7a5e..24dcd9e7 100644 --- a/protocol/wire/frame.go +++ b/protocol/wire/frame.go @@ -131,6 +131,17 @@ type Response interface{ RespType() string } type respDecoder func([]byte) (Response, error) +// reqPtr and respPtr turn a mistyped decoder table entry into a compile error. +type reqPtr[T any] interface { + *T + Request +} + +type respPtr[T any] interface { + *T + Response +} + // B64 carries request payload bytes. It exists because silkd's deserializer // requires a base64 string and rejects null — which is exactly what // encoding/json emits for a nil []byte. Decoding needs no counterpart: @@ -654,7 +665,7 @@ func DecodeRequest(line []byte) (Request, error) { if err != nil { return nil, fmt.Errorf("parse request frame: %w", err) } - dec, ok := requestDecoders[op] + dec, ok := requestDecoders[string(op)] if !ok { return nil, fmt.Errorf("unknown op %q", op) } @@ -676,7 +687,7 @@ func DecodeResponse(line []byte) (Response, error) { if err != nil { return nil, fmt.Errorf("parse response frame: %w", err) } - dec, ok := responseDecoders[typ] + dec, ok := responseDecoders[string(typ)] if !ok { return nil, fmt.Errorf("unknown response type %q", typ) } @@ -737,13 +748,14 @@ func encodeTagged(head string, v any) ([]byte, error) { // frameTag returns the frame's dispatch tag: a canonical head (both producers // emit the tag first, unescaped) yields it with a prefix cut; anything else // takes the full token walk. -func frameTag(line, head []byte, key string) (string, error) { +func frameTag(line, head []byte, key string) ([]byte, error) { if rest, ok := bytes.CutPrefix(line, head); ok { if end := bytes.IndexAny(rest, `"\`); end >= 0 && rest[end] == '"' { - return string(rest[:end]), nil + return rest[:end], nil } } - return scanTag(line, key) + tag, err := scanTag(line, key) + return []byte(tag), err } // scanTag extracts the string value of a top-level key without decoding the @@ -805,20 +817,18 @@ func decodeAs[T any](line []byte) (*T, error) { } // decodeReq / decodeResp adapt decodeAs to the interface-typed decoder maps. -// A table entry pairing an op with a non-Request type is caught by the shared -// fixture round-trip test, not the compiler. -func decodeReq[T any](line []byte) (Request, error) { +func decodeReq[T any, PT reqPtr[T]](line []byte) (Request, error) { v, err := decodeAs[T](line) if err != nil { return nil, err } - return any(v).(Request), nil + return PT(v), nil } -func decodeResp[T any](line []byte) (Response, error) { +func decodeResp[T any, PT respPtr[T]](line []byte) (Response, error) { v, err := decodeAs[T](line) if err != nil { return nil, err } - return any(v).(Response), nil + return PT(v), nil } diff --git a/protocol/wire/frame_test.go b/protocol/wire/frame_test.go index 8c89f8b3..50eefd1e 100644 --- a/protocol/wire/frame_test.go +++ b/protocol/wire/frame_test.go @@ -12,9 +12,6 @@ import ( const fixtureDir = "fixtures/v1" -// TestFixtureCorpusRoundTrips is the protocol/README contract: every golden -// frame must decode and re-encode to canonical-JSON equality, both request -// and response sides — a frame only one implementation round-trips is drift. func TestFixtureCorpusRoundTrips(t *testing.T) { entries, err := os.ReadDir(fixtureDir) if err != nil { @@ -60,17 +57,12 @@ func TestFixtureCorpusRoundTrips(t *testing.T) { } seen++ } - // Exact, deliberately: adding a verb means adding its fixture and - // bumping this — a new frame type outside the corpus is drift waiting - // to happen. + if seen != 60 { t.Fatalf("fixture corpus: %d frames, want exactly 60", seen) } } -// TestEveryVerbHasAFixture pins corpus completeness: every request and -// response type the SDK knows must appear in at least one golden frame, so -// no verb can exist outside the three-language drift guard. func TestEveryVerbHasAFixture(t *testing.T) { allRequests := []Request{ Exec{}, @@ -166,10 +158,7 @@ func TestEveryVerbHasAFixture(t *testing.T) { } } -// TestEnumValueSetsMatchFixture pins the full enum value lists -// (order-sensitive): frame fixtures carry one representative value per enum, -// so a variant renamed or added on only one side would otherwise drift -// silently. Twin of silkd's enum_value_sets_match_fixture. +// Twin of silkd's enum_value_sets_match_fixture; the lists are order-sensitive. func TestEnumValueSetsMatchFixture(t *testing.T) { raw, err := os.ReadFile(filepath.Join(fixtureDir, "enums.json")) if err != nil { @@ -209,8 +198,6 @@ func TestDataFieldsAreBase64(t *testing.T) { } func TestRequestDataNeverNull(t *testing.T) { - // silkd's deserializer wants a base64 string; a nil payload must encode - // as "" — encoding/json's null would abort the RPC as a protocol error. for _, req := range []Request{&Stdin{}, &Data{}} { frame, err := EncodeRequest(req) if err != nil { @@ -232,9 +219,6 @@ func TestUnknownFieldsIgnored(t *testing.T) { } } -// TestBulkDecodeStrictShape guards the fast bulk slicer: only the exact -// canonical frame takes the fast path, so a "data" match inside another -// field or any malformed JSON must never yield a silently wrong payload. func TestBulkDecodeStrictShape(t *testing.T) { resp, err := DecodeResponse([]byte(`{"type":"stdout","meta":{"data":"WFhY"},"data":"aGk="}`)) if err != nil { diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go index 4f3f0d2e..705ef5cf 100644 --- a/sdk/go/checkpoint_test.go +++ b/sdk/go/checkpoint_test.go @@ -9,8 +9,6 @@ import ( ) func TestCheckpointNewFollowsRedirect(t *testing.T) { - // The checkpoint's node no longer holds it and redirects; the retry - // must land on node B, bound there, exactly like Client.New's redirect. var nodeACalls int nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req checkpointClaimRequest @@ -48,8 +46,6 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { } func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { - // The probed owner transiently fails (500); once exhausted, New falls - // back to the origin, which this time fails definitively too. broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) @@ -87,8 +83,6 @@ func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { } func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { - // The only probed owner is mid-heal (503); once exhausted, New falls - // back to the origin, which heals (pulls the checkpoint) locally. busy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) @@ -124,8 +118,6 @@ func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { } } -// TestCheckpointNewRedirectNeverYieldsEmptyID: New must fail loudly, not -// return a Sandbox with an empty ID, when no redirect candidate answers. func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) @@ -144,8 +136,6 @@ func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { } func TestCheckpointNewSecondLevelRedirectFails(t *testing.T) { - // A compliant server never redirects a no_redirect retry; if one does - // anyway, it must be treated as a failed candidate rather than followed. nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) })) diff --git a/sdk/go/claimopts_test.go b/sdk/go/claimopts_test.go index c8570732..e224004a 100644 --- a/sdk/go/claimopts_test.go +++ b/sdk/go/claimopts_test.go @@ -5,8 +5,6 @@ import ( "testing" ) -// Checkpoint.New and Template.New pin the network lane and size to the -// snapshot, so WithNetwork/WithSize must fail loud rather than silently no-op. func TestSnapshotClaimRejectsPinnedAxes(t *testing.T) { ck := &Checkpoint{ID: "ck_1", c: &Client{}, addr: "node:1"} tpl := &Template{Name: "rt", c: &Client{}, addr: "node:1", net: "none", size: "small"} diff --git a/sdk/go/client.go b/sdk/go/client.go index 8d76f760..228084af 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -425,7 +425,9 @@ func (e *APIError) Error() string { // apiError surfaces the server's {"error": ...} body when present. func apiError(verb string, resp *http.Response) error { var er errorResponse - _ = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&er) + body := io.LimitReader(resp.Body, 4096) + _ = json.NewDecoder(body).Decode(&er) + _, _ = io.Copy(io.Discard, body) return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 9ad60447..87e55b2c 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -120,8 +120,6 @@ func TestCloseReleases(t *testing.T) { } func TestNewFollowsRedirect(t *testing.T) { - // The first node redirects; the second answers with the claim. The SDK - // must follow transparently and end up owning the sandbox at node B. var nodeB *httptest.Server nodeB = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{ @@ -150,8 +148,6 @@ func TestNewFollowsRedirect(t *testing.T) { } func TestDeleteTemplateFollowsRedirect(t *testing.T) { - // The entry node answers with the owner's address; the SDK retries the - // delete there, once. var deletedAtB, gotNoRedirect bool nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodDelete { @@ -182,9 +178,6 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { } } -// TestDeleteTemplateRetryPolicy pins the walk semantics shared with the -// Python SDK: a 404 moves to the next owner, a served error surfaces at -// once (walking on would let a later 404 mask it). func TestDeleteTemplateRetryPolicy(t *testing.T) { status := func(code int, hit *bool) *httptest.Server { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -232,8 +225,6 @@ func TestDeleteTemplateRetryPolicy(t *testing.T) { } func TestRedirectSetsNoRedirect(t *testing.T) { - // The retry at a redirect target must carry no_redirect so the target - // warm-or-provisions instead of bouncing the claim back. var gotNoRedirect bool target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req claimRequest @@ -263,7 +254,6 @@ func TestRedirectSetsNoRedirect(t *testing.T) { } func TestRedirectTriesAllCandidates(t *testing.T) { - // First candidate is dead; the SDK must fall through to the second. good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_4", Token: "t"}) })) @@ -286,9 +276,6 @@ func TestRedirectTriesAllCandidates(t *testing.T) { } func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { - // Candidate one is wrong for this claim (403, definitive) but candidate - // two would still succeed — the per-candidate walk retries broadly, so - // one ill-suited candidate must not cost a candidate that would answer. forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) })) @@ -316,8 +303,6 @@ func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { } func TestRedirectFallbackHeals(t *testing.T) { - // The only candidate holds the record but is full (429); once it's - // exhausted, New must fall back to the origin itself, which heals. var fullCalls int full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fullCalls++ @@ -360,8 +345,6 @@ func TestRedirectFallbackHeals(t *testing.T) { } func TestRedirectFallbackAlsoFails(t *testing.T) { - // Every candidate is full (429); the origin fallback answers too, but - // with a definitive error, which must surface as the final error. var fullCalls int full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fullCalls++ @@ -389,8 +372,7 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { if !strings.Contains(err.Error(), "origin also unavailable") { t.Errorf("err %v, want the origin's failure surfaced", err) } - // The candidate's failure must survive into the message too: it is what - // says why the claim left the origin in the first place. + if !strings.Contains(err.Error(), "candidate full") { t.Errorf("err %v, want the candidate's failure surfaced too", err) } @@ -416,9 +398,6 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // A definitive 4xx is not worth trying another candidate, and - // not worth falling back to the origin either — the origin - // would fail the exact same way. bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(tt.code) _ = json.NewEncoder(w).Encode(errorResponse{Error: tt.want}) @@ -481,8 +460,6 @@ func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { } func TestLookupScatter(t *testing.T) { - // The owner is node B; the entry node A doesn't own the sandbox but lists - // B as a peer. Lookup must scatter to B and bind the handle there. var owner *httptest.Server owner = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/sandboxes/sb_1/owner" && r.Header.Get("Authorization") == "Bearer tok" { @@ -496,7 +473,7 @@ func TestLookupScatter(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/sandboxes/sb_1/owner": - w.WriteHeader(http.StatusNotFound) // not owned here + w.WriteHeader(http.StatusNotFound) case "/v1/peers": _ = json.NewEncoder(w).Encode(map[string]any{"peers": []string{strings.TrimPrefix(owner.URL, "http://")}}) } diff --git a/sdk/go/files_test.go b/sdk/go/files_test.go index df56365a..e319c8ef 100644 --- a/sdk/go/files_test.go +++ b/sdk/go/files_test.go @@ -77,7 +77,7 @@ func TestSessionLifecycle(t *testing.T) { if len(ids) != 1 || ids[0] != sess.ID { t.Errorf("sessions %v, want [%s]", ids, sess.ID) } - // exec-in-session runs argv (the fake's echo handler). + out, err := sess.Exec(ctx, "echo", "hi") if err != nil { t.Fatalf("session exec: %v", err) @@ -109,8 +109,6 @@ func TestReadFileMultiChunk(t *testing.T) { } } -// fakeSandbox wires a Sandbox whose data plane is served by a temp-dir-backed -// silkd Fake behind a hijacking agent endpoint. func fakeSandbox(t *testing.T) *Sandbox { t.Helper() fake := silkdtest.NewFake(t.TempDir()) diff --git a/sdk/go/port_test.go b/sdk/go/port_test.go index 2adb57ba..cfd943a7 100644 --- a/sdk/go/port_test.go +++ b/sdk/go/port_test.go @@ -22,7 +22,7 @@ func TestDialPortEchoAndHalfClose(t *testing.T) { if _, err := io.ReadFull(pc, buf); err != nil || string(buf) != "ping" { t.Fatalf("read %q, %v — want the echo", buf, err) } - // Half-close: the fake's server closes, surfacing as EOF. + if err := pc.CloseWrite(); err != nil { t.Fatalf("close write: %v", err) } diff --git a/sdk/go/proc_test.go b/sdk/go/proc_test.go index ee6280e0..1a984a86 100644 --- a/sdk/go/proc_test.go +++ b/sdk/go/proc_test.go @@ -15,12 +15,12 @@ func TestProcVerbs(t *testing.T) { "ps": {`{"type":"procs","procs":[{"pid":41,"argv":["sleep","30"],"detached":true,"state":"running","started_at_epoch_secs":1}]}`}, "kill": {`{"type":"done"}`}, "logs": { - `{"type":"stdout","data":"aGVsbG8="}`, // "hello" - `{"type":"stderr","data":"b29wcw=="}`, // "oops" - `{"type":"done"}`, // still running: no exit + `{"type":"stdout","data":"aGVsbG8="}`, + `{"type":"stderr","data":"b29wcw=="}`, + `{"type":"done"}`, }, "attach": { - `{"type":"stdout","data":"bGF0ZQ=="}`, // "late" + `{"type":"stdout","data":"bGF0ZQ=="}`, `{"type":"exit","code":7}`, }, } @@ -75,8 +75,6 @@ func TestProcVerbsUnknownPid(t *testing.T) { } } -// procServe answers each RPC with a canned frame script keyed by op — the -// proc verbs are pure frame plumbing, so a scripted fake covers them. func procServe(script map[string][]string) func(net.Conn) { return func(conn net.Conn) { defer func() { _ = conn.Close() }() diff --git a/sdk/go/pty.go b/sdk/go/pty.go index a3a31624..8ebc7253 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -38,12 +38,18 @@ func (p *Pty) Read(b []byte) (int, error) { return p.out.Read(b) } -// Write feeds input to the terminal. +// Write feeds input to the terminal, chunked to stay under the frame cap. func (p *Pty) Write(b []byte) (int, error) { - if err := p.conn.Send(&wire.Stdin{Data: b}); err != nil { - return 0, err + sent := 0 + for len(b) > 0 { + chunk := b[:min(len(b), stdinChunk)] + if err := p.conn.Send(&wire.Stdin{Data: chunk}); err != nil { + return sent, err + } + sent += len(chunk) + b = b[len(chunk):] } - return len(b), nil + return sent, nil } // Resize adjusts the terminal window; it is a separate RPC keyed by pid. diff --git a/sdk/go/pty_test.go b/sdk/go/pty_test.go index b65e19b2..f7eb5547 100644 --- a/sdk/go/pty_test.go +++ b/sdk/go/pty_test.go @@ -2,10 +2,16 @@ package sandbox import ( "bufio" + "bytes" "context" "errors" + "fmt" "io" + "net" "testing" + + "github.com/cocoonstack/sandbox/protocol/wire" + "github.com/cocoonstack/sandbox/sdk/go/silkd" ) func TestPtyEchoAndExit(t *testing.T) { @@ -88,3 +94,48 @@ func TestPtyCloseUnblocksUnreadOutput(t *testing.T) { t.Errorf("stop called %d times, want 1", stops) } } + +func TestPtyWriteChunksOversizedInput(t *testing.T) { + client, server := net.Pipe() + defer func() { _ = client.Close() }() + p := &Pty{conn: silkd.NewConn(client)} + payload := bytes.Repeat([]byte("x"), stdinChunk*2+1) + + errc := make(chan error, 1) + go func() { + n, err := p.Write(payload) + if err == nil && n != len(payload) { + err = fmt.Errorf("Write returned %d, want %d", n, len(payload)) + } + errc <- err + }() + + sc := wire.NewFrameScanner(server) + var got []byte + for len(got) < len(payload) { + if !sc.Scan() { + t.Fatalf("scan stdin frame: %v", sc.Err()) + } + if len(sc.Bytes()) > wire.MaxFrame { + t.Fatalf("frame of %d bytes exceeds the %d cap", len(sc.Bytes()), wire.MaxFrame) + } + req, err := wire.DecodeRequest(sc.Bytes()) + if err != nil { + t.Fatalf("decode stdin frame: %v", err) + } + in, ok := req.(*wire.Stdin) + if !ok { + t.Fatalf("frame %q, want stdin", req.Op()) + } + if len(in.Data) > stdinChunk { + t.Fatalf("stdin frame carries %d bytes, want at most %d", len(in.Data), stdinChunk) + } + got = append(got, in.Data...) + } + if err := <-errc; err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Errorf("reassembled %d bytes, want the %d written", len(got), len(payload)) + } +} diff --git a/sdk/go/sandbox_test.go b/sdk/go/sandbox_test.go index 594b24a0..d879c9af 100644 --- a/sdk/go/sandbox_test.go +++ b/sdk/go/sandbox_test.go @@ -78,9 +78,6 @@ func TestRunContextCancel(t *testing.T) { } func TestUpgradeKeepsCoalescedBytes(t *testing.T) { - // The server flushes the 101 and every response frame in a single write - // before reading the request; the frames ride into the handshake reader - // and must survive. mux := http.NewServeMux() mux.HandleFunc("GET /v1/sandboxes/{id}/agent", func(w http.ResponseWriter, r *http.Request) { conn, _, err := http.NewResponseController(w).Hijack() @@ -139,9 +136,6 @@ func TestDialAgentRejectsControlChars(t *testing.T) { } } -// newAgentServer serves the agent endpoint by hijacking and handing the conn -// to serve (a fake silkd), mirroring sandboxd's relay from the SDK's -// perspective. func newAgentServer(t *testing.T, serve func(net.Conn)) *httptest.Server { t.Helper() mux := http.NewServeMux() diff --git a/sdk/go/silkd/conn_test.go b/sdk/go/silkd/conn_test.go index 808a5cd0..d934c814 100644 --- a/sdk/go/silkd/conn_test.go +++ b/sdk/go/silkd/conn_test.go @@ -13,9 +13,6 @@ import ( "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" ) -// TestSendBulkMatchesJSONEncoding pins the hand-built data/stdin envelope to -// byte-for-byte identity with the json.Marshal path — the fast path must not -// drift from the wire contract silkd parses. func TestSendBulkMatchesJSONEncoding(t *testing.T) { for _, req := range []wire.Request{ &wire.Data{Data: []byte("hello\x00\xff binary")}, diff --git a/sdk/go/silkd/silkdtest/fake.go b/sdk/go/silkd/silkdtest/fake.go index 4ff21542..eb424aa8 100644 --- a/sdk/go/silkd/silkdtest/fake.go +++ b/sdk/go/silkd/silkdtest/fake.go @@ -242,7 +242,12 @@ func (f *Fake) fsPull(conn net.Conn, path string) { return } _ = tw.Close() - send(conn, &wire.DataResp{Data: buf.Bytes()}) + data := buf.Bytes() + for len(data) > 0 { + n := min(readChunk, len(data)) + send(conn, &wire.DataResp{Data: data[:n]}) + data = data[n:] + } send(conn, wire.Done{}) } diff --git a/sdk/go/utils.go b/sdk/go/utils.go index 1fbd32d8..c334e78d 100644 --- a/sdk/go/utils.go +++ b/sdk/go/utils.go @@ -12,6 +12,12 @@ import ( // fsChunk matches silkd's BULK_CHUNK. const fsChunk = 256 * 1024 +// respPtr is a frame type's pointer form, so a non-Response T fails to compile. +type respPtr[T any] interface { + *T + wire.Response +} + // doneRPC sends a request that answers with Done or an error frame. func (s *Sandbox) doneRPC(ctx context.Context, req wire.Request) error { conn, done, err := s.call(ctx, req) @@ -46,17 +52,17 @@ func (s *Sandbox) downloadRPC(ctx context.Context, req wire.Request, sink func([ } // oneShotRPC sends req and returns its single typed reply frame. -func oneShotRPC[T any](ctx context.Context, s *Sandbox, req wire.Request) (*T, error) { +func oneShotRPC[T any, PT respPtr[T]](ctx context.Context, s *Sandbox, req wire.Request) (*T, error) { conn, done, err := s.call(ctx, req) if err != nil { return nil, err } defer done() - return expect[T](ctx, conn) + return expect[T, PT](ctx, conn) } // collectRPC sends req and gathers every streamed frame of type T until Done. -func collectRPC[T any](ctx context.Context, s *Sandbox, req wire.Request) ([]T, error) { +func collectRPC[T any, PT respPtr[T]](ctx context.Context, s *Sandbox, req wire.Request) ([]T, error) { conn, done, err := s.call(ctx, req) if err != nil { return nil, err @@ -68,9 +74,7 @@ func collectRPC[T any](ctx context.Context, s *Sandbox, req wire.Request) ([]T, if err != nil { return nil, err } - // Via any: a direct resp.(*T) assertion is rejected at compile time - // (*T is not known to implement Response). - if v, ok := any(resp).(*T); ok { + if v, ok := resp.(PT); ok { out = append(out, *v) continue } @@ -136,12 +140,12 @@ func terminalErr(ctx context.Context, conn *silkd.Conn) error { // expect reads one frame and requires it to be a *T, mapping an error frame // to a Go error. -func expect[T any](ctx context.Context, conn *silkd.Conn) (*T, error) { +func expect[T any, PT respPtr[T]](ctx context.Context, conn *silkd.Conn) (*T, error) { resp, err := recv(ctx, conn) if err != nil { return nil, err } - if v, ok := any(resp).(*T); ok { + if v, ok := resp.(PT); ok { return v, nil } if e, ok := resp.(*wire.ErrorResp); ok { diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index 64ad608e..483b8f1d 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -108,8 +108,7 @@ func TestTemplateNewVolumeClaimFollowsRedirect(t *testing.T) { if !got.NoRedirect || !got.RequirePromoted || !slices.Equal(got.Volumes, want) { t.Errorf("redirected claim = %+v, want promoted no_redirect with %v", got, want) } - // The redirect body carries no principal, so the bearer is what makes the - // peer resolve the same tenant — a dropped one would claim as the operator. + if originAuth != "Bearer tenant-token" || targetAuth != originAuth { t.Errorf("bearer at origin=%q at target=%q, want the caller's token on both hops", originAuth, targetAuth) } From 2d16b475330c8937fb4e39ecd076fd8ba02b451d Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 3 Sep 2026 03:47:15 +0800 Subject: [PATCH 4/6] review: keep the five constraint comments the condensation dropped Reconcile's ordering rule, recLock's shared mode, the recLocks eviction rule, the S3 lifecycle rule behind SweepStaging and the randHex suffix on the wake return path each carried a fact the code cannot show; they return as one-line comments. --- sandboxd/pool/hibernate.go | 2 +- sandboxd/pool/pool.go | 2 +- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/template.go | 2 +- sandboxd/store/s3/s3.go | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 62d7fa80..cae92656 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -162,7 +162,7 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, return "", fmt.Errorf("wake %s: persist claims: %w", sb.ID, err) } m.dropStale(ctx, sb) - // the resume consumed the memory image; reclaim its disk off the wake-return path + // the resume consumed the memory image; reclaim its disk off the wake-return path (the randHex suffix keeps a re-hibernate from reusing the name) go m.dropSnap(ctx, snap) return sock, nil } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 53985349..3a6c5136 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -294,7 +294,7 @@ type Manager struct { tplMu sync.Mutex tplSet map[string]string - // recLocks serializes same-id record mutations; an entry evicts only when no holder remains. + // recLocks serializes same-id record mutations; an entry evicts only when no holder remains, never on a bare delete, since a peer heal can republish the id. recLocks map[string]*sync.RWMutex recLocksMu sync.Mutex recRefs map[string]int diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 63199475..d67155b0 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -17,7 +17,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Reconcile aligns claims and VMs after a restart; it must run before the server starts. +// Reconcile aligns claims and VMs after a restart; it runs before the server because it swaps in fresh records that would bypass in-flight Transition locks. func (m *Manager) Reconcile(ctx context.Context) error { claims, err := m.store.load() if err != nil { diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index b05fed94..4fa6a334 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -175,7 +175,7 @@ func (m *Manager) pooledHash(hash string) bool { return false } -// recLock takes the per-record lock and a live reference; pair every call with recDone. +// recLock takes the per-record lock and a live reference (pair with recDone); clones and wakes hold it shared so a delete or re-publish never runs under an in-flight read. func (m *Manager) recLock(id string) *sync.RWMutex { m.recLocksMu.Lock() defer m.recLocksMu.Unlock() diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index ab0a4757..3df01115 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -187,6 +187,7 @@ func (s *Store) Delete(ctx context.Context, id string) error { return s.deleteKeys(ctx, []string{metaKey}) } +// SweepStaging clears local staging residue at startup; objects orphaned by a crash before meta.json are reclaimed by the bucket's S3 lifecycle rule (see deploy). func (s *Store) SweepStaging() error { return utils.RemoveDirEntries(s.staging, nil) } func (s *Store) SweepGenerations() error { return nil } From e4d61f0ffd6b4ab9da56c49ed46daf98c17b1553 Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 3 Sep 2026 04:10:50 +0800 Subject: [PATCH 5/6] fix: reap a language server whose Ready write fails; drain oversized API error bodies A client that vanishes between lsp_request taking the stdio halves and the Ready write left a child the attach TTL no longer covered; the write error now reaps it. The Go SDK's apiError drains resp.Body itself instead of the already exhausted LimitReader. The watch e2e no longer assumes the first event after ready is its own write: FSEvents replays the watched directory's creation first on macOS. --- sdk/go/client.go | 5 ++-- sdk/go/client_test.go | 15 +++++++++++ silkd/src/lsp.rs | 7 ++++- silkd/tests/lsp_e2e.rs | 56 +++++++++++++++++++++++++++++++++------- silkd/tests/watch_e2e.rs | 24 +++++++++++------ 5 files changed, 86 insertions(+), 21 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 228084af..aef31e58 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -425,9 +425,8 @@ func (e *APIError) Error() string { // apiError surfaces the server's {"error": ...} body when present. func apiError(verb string, resp *http.Response) error { var er errorResponse - body := io.LimitReader(resp.Body, 4096) - _ = json.NewDecoder(body).Decode(&er) - _, _ = io.Copy(io.Discard, body) + _ = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&er) + _, _ = io.Copy(io.Discard, resp.Body) return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 87e55b2c..e2232b58 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -2,6 +2,8 @@ package sandbox import ( "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" "reflect" @@ -489,6 +491,19 @@ func TestLookupScatter(t *testing.T) { } } +func TestAPIErrorDrainsOversizedBody(t *testing.T) { + body := strings.NewReader(`{"error":"` + strings.Repeat("x", 3*4096) + `"}`) + err := apiError("claim", &http.Response{StatusCode: http.StatusTooManyRequests, Body: io.NopCloser(body)}) + + var he *APIError + if !errors.As(err, &he) || he.Status != http.StatusTooManyRequests { + t.Fatalf("apiError = %v, want a 429 APIError", err) + } + if body.Len() != 0 { + t.Errorf("apiError left %d bytes unread; the transport cannot reuse the connection", body.Len()) + } +} + func testClient(t *testing.T, ts *httptest.Server, opts ...ClientOption) *Client { t.Helper() c, err := Connect(strings.TrimPrefix(ts.URL, "http://"), opts...) diff --git a/silkd/src/lsp.rs b/silkd/src/lsp.rs index 5080098d..3b94b1f5 100644 --- a/silkd/src/lsp.rs +++ b/silkd/src/lsp.rs @@ -136,7 +136,12 @@ impl Broker { } Some(Some(stdio)) => stdio, }; - proto::write_frame(w, &Response::Ready).await?; + // The stdio halves are out of the table, so the attach TTL no longer + // covers this server: a client already gone must reap it here. + if let Err(e) = proto::write_frame(w, &Response::Ready).await { + self.reap(server_id).await; + return Err(e); + } let feed = tokio::spawn(feed_stdin(client, stdin)); let res = pump_stdout(stdout, w).await; diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index c84d1e49..af0d5244 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -28,6 +28,7 @@ IFS= read -r line printf 'reply:%s\n' "$line" "#; +/// Creates a fake language server and points SILKD_LSP_DIR at its dir. fn manifest_env(server_body: &str) -> TempDir { let dir = tempfile::tempdir().unwrap(); let bin = dir.path().join("fake-lsp"); @@ -38,6 +39,9 @@ fn manifest_env(server_body: &str) -> TempDir { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); } + // SAFETY: ENV_LOCK, held by the caller for its whole test, serializes every + // writer and reader of SILKD_LSP_DIR in this binary. + unsafe { std::env::set_var("SILKD_LSP_DIR", dir.path()) }; dir } @@ -83,9 +87,6 @@ async fn lsp_broker_relays_to_the_server() { bin.to_string_lossy().as_bytes(), ) .unwrap(); - // SAFETY: ENV_LOCK (held for this test's whole body) serializes every - // writer/reader of SILKD_LSP_DIR in this binary. - unsafe { std::env::set_var("SILKD_LSP_DIR", env.path()) }; let state = Arc::new(State::new()); let start = one( @@ -157,9 +158,6 @@ async fn lsp_stop_kills_an_idle_server() { env.path().join("fake-lsp").to_string_lossy().as_bytes(), ) .unwrap(); - // SAFETY: ENV_LOCK (held for this test's whole body) serializes every - // writer/reader of SILKD_LSP_DIR in this binary. - unsafe { std::env::set_var("SILKD_LSP_DIR", env.path()) }; let state = Arc::new(State::new()); let start = one( &state, @@ -184,6 +182,49 @@ async fn lsp_stop_kills_an_idle_server() { assert_eq!(gone.last().unwrap()["kind"], "not_found"); } +#[tokio::test] +async fn lsp_request_reaps_when_the_client_vanishes_before_ready() { + let _env_lock = ENV_LOCK.lock().await; + let env = manifest_env("#!/bin/sh\nsleep 60\n"); + std::fs::write( + env.path().join("gonetest"), + env.path().join("fake-lsp").to_string_lossy().as_bytes(), + ) + .unwrap(); + let state = Arc::new(State::new()); + let start = one( + &state, + &json!({"op":"lsp_start","language":"gonetest"}).to_string(), + ) + .await; + let server_id = start.last().unwrap()["server_id"] + .as_str() + .unwrap() + .to_string(); + + let (mut client, server) = tokio::io::duplex(1 << 20); + let request = json!({"op":"lsp_request","server_id":server_id}).to_string(); + client.write_all(request.as_bytes()).await.unwrap(); + client.write_all(b"\n").await.unwrap(); + drop(client); + let (sr, sw) = tokio::io::split(server); + let served = timeout(DEADLINE, state.serve(tokio::io::BufReader::new(sr), sw)) + .await + .expect("deadline"); + assert!(served.is_err(), "the ready write must fail: {served:?}"); + + let gone = one( + &state, + &json!({"op":"lsp_stop","server_id":server_id}).to_string(), + ) + .await; + assert_eq!( + gone.last().unwrap()["kind"], + "not_found", + "a server whose client vanished before ready must be reaped: {gone:?}" + ); +} + #[tokio::test] async fn lsp_data_end_half_closes_stdin() { let _env_lock = ENV_LOCK.lock().await; @@ -193,9 +234,6 @@ async fn lsp_data_end_half_closes_stdin() { env.path().join("fake-lsp").to_string_lossy().as_bytes(), ) .unwrap(); - // SAFETY: ENV_LOCK (held for this test's whole body) serializes every - // writer/reader of SILKD_LSP_DIR in this binary. - unsafe { std::env::set_var("SILKD_LSP_DIR", env.path()) }; let state = Arc::new(State::new()); let start = one( &state, diff --git a/silkd/tests/watch_e2e.rs b/silkd/tests/watch_e2e.rs index dabd5a99..a7ddf41c 100644 --- a/silkd/tests/watch_e2e.rs +++ b/silkd/tests/watch_e2e.rs @@ -35,14 +35,22 @@ async fn watch_streams_events_until_disconnect() { .await .unwrap(); - let event = tokio::time::timeout(Duration::from_secs(5), lines.next_line()) - .await - .expect("no event within 5s") - .unwrap() - .expect("stream closed before an event"); - let frame: serde_json::Value = serde_json::from_str(&event).unwrap(); - assert_eq!(frame["type"], "event", "got {frame:?}"); - assert!(frame["path"].as_str().unwrap().ends_with("new.txt")); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let line = lines + .next_line() + .await + .unwrap() + .expect("stream closed before an event"); + let frame: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(frame["type"], "event", "got {frame:?}"); + if frame["path"].as_str().unwrap().ends_with("new.txt") { + return; + } + } + }) + .await + .expect("no new.txt event within 5s"); drop(lines); drop(cw); From 013c5694d6773c8071d836e6d497cc7531813735 Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 3 Sep 2026 04:17:47 +0800 Subject: [PATCH 6/6] test: the missing-manifest LSP test holds ENV_LOCK like every other reader of SILKD_LSP_DIR --- silkd/tests/lsp_e2e.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index af0d5244..e45c91d6 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -47,6 +47,7 @@ fn manifest_env(server_body: &str) -> TempDir { #[tokio::test] async fn lsp_start_missing_manifest_is_not_found() { + let _env_lock = ENV_LOCK.lock().await; let frames = timeout( DEADLINE, one(