From 2c2e02b742ed66a6318634a89bc63b959288db91 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 15:39:28 +0000 Subject: [PATCH 01/29] pool: persist runtime volume registrations Load root-token registrations before claim recovery and refuse unregister while claims, operations or VM cleanup retain a reference. Keep detached references protected until their removal is committed. --- docs/deploy.md | 11 +- docs/live-volumes.md | 6 + docs/sandboxd-api.md | 102 +++- docs/security.md | 25 +- e2e/volume_catalog_test.go | 112 ++++ sandboxd/config/config.go | 64 +- sandboxd/config/config_test.go | 29 + sandboxd/pool/checkpoint_references_test.go | 129 ++-- sandboxd/pool/claim.go | 6 +- sandboxd/pool/claims.go | 184 +++++- sandboxd/pool/claims_test.go | 12 +- sandboxd/pool/pool.go | 32 +- sandboxd/pool/pool_test.go | 21 +- sandboxd/pool/reconcile.go | 46 +- sandboxd/pool/reconcile_test.go | 4 +- sandboxd/pool/remove.go | 26 +- sandboxd/pool/replica.go | 3 +- sandboxd/pool/replica_test.go | 56 ++ sandboxd/pool/volume.go | 129 +++- sandboxd/pool/volume_acl_test.go | 152 +++++ sandboxd/pool/volume_bench_test.go | 81 +++ sandboxd/pool/volume_capture_test.go | 9 + sandboxd/pool/volume_catalog.go | 266 +++++++++ sandboxd/pool/volume_catalog_test.go | 617 ++++++++++++++++++++ sandboxd/pool/volume_cleanup_test.go | 543 +++++++++++++++++ sandboxd/pool/volume_convergence_test.go | 34 +- sandboxd/pool/volume_mutation.go | 34 +- sandboxd/pool/volume_mutation_test.go | 8 +- sandboxd/pool/volume_retirement_test.go | 416 +++++++++++++ sandboxd/pool/volume_rw_test.go | 4 +- sandboxd/pool/volume_static_acl_test.go | 42 ++ sandboxd/server/server.go | 6 + sandboxd/server/server_test.go | 21 +- sandboxd/server/volume_registration.go | 44 ++ sandboxd/server/volume_registration_test.go | 274 +++++++++ sandboxd/types/api.go | 8 + 36 files changed, 3346 insertions(+), 210 deletions(-) create mode 100644 e2e/volume_catalog_test.go create mode 100644 sandboxd/pool/volume_acl_test.go create mode 100644 sandboxd/pool/volume_bench_test.go create mode 100644 sandboxd/pool/volume_catalog.go create mode 100644 sandboxd/pool/volume_catalog_test.go create mode 100644 sandboxd/pool/volume_cleanup_test.go create mode 100644 sandboxd/pool/volume_retirement_test.go create mode 100644 sandboxd/pool/volume_static_acl_test.go create mode 100644 sandboxd/server/volume_registration.go create mode 100644 sandboxd/server/volume_registration_test.go diff --git a/docs/deploy.md b/docs/deploy.md index f8020f0e..51938d7b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -119,7 +119,7 @@ gets the intersection of the two. | `advertise_addr` | = `listen` | internal HTTP host:port for peer traffic and preview forwarding; also used by clients when `client_advertise` is unset. Must be routable when `listen` is a wildcard; a node with `mesh` set refuses to load while it names an unspecified host | | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | -| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, and a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later); root always has access. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, and a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later); root-owned claims have access; live additions use the claim owner's access list. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | @@ -247,6 +247,12 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes +Volume `tenants` entries use the tenant API name syntax. Static configuration +warns about invalid names and skips them during access checks. A nonempty list containing +only invalid names grants no tenant access. Runtime registration rejects invalid +tenant names. Valid names for tenants that have not yet been created remain +accepted. + Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes @@ -324,7 +330,8 @@ set, promoted-template intersection, and redirect follow the fleet-wide rule in [cluster](cluster.md#volumes-and-placement). An empty catalog `tenants` list allows every authenticated scope; a nonempty -list limits the image to those tenants, while root always bypasses it. A +list limits the image to those claim owners. Root-owned claims can use any +entry. Live additions use the claim owner's ACL, including requests sent by root. A listed tenant the set no longer holds grants nothing and logs a warning at boot. `GET /v1/volumes` reports the caller-visible fleet union and holder count, plus diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 790e0d21..66ff435f 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,6 +9,12 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. +A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) +without editing static config. Registration persists only node-local catalog +metadata and does not manage disk files. Omitted or empty runtime `tenants` +lists permit every authenticated scope. List tenants explicitly to restrict +access. Live changes remain limited to mounted read-only entries. + Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk before mounting the new one, so the path has a short unavailable interval. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 31a255cd..c103b4c8 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -8,7 +8,7 @@ All bodies are JSON. Three token kinds: preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces (the per-id sandbox reads, `PUT /v1/sandboxes/{id}/instance-metadata`, `GET /v1/info`, - `PUT /v1/pools`, `POST/DELETE /v1/drain`, `GET /metrics`, + `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, `GET /metrics`, `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. On a node with a shared @@ -255,6 +255,103 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. +## PUT /v1/volumes/{name} + +Auth: the configured root node API token. With no configured `api_token`, +registration and unregistration return 403. A tenant token returns 403. +Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. + +Registers a backing path on this node. The URL supplies the name; +the body has no `name` field. Unknown fields, duplicate fields, and trailing +JSON return 400. + +```json +{"path": "/srv/datasets/data.img", "directio": "off", + "writable": false, "tenants": ["acme"]} +``` + +- `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. +- `path` uses the same absolute-path validation as static volume config. + Registration does not open the backing or check its type or availability. + A missing backing retains its registration but cannot be attached until available. + Registration success does not establish that Cocoon supports the backing type. + Operators must keep the registered path and target stable. + Backing storage must be separate from daemon cleanup directories, including + `/readers`, `/goldens`, and `/fork-*`. + Backing paths must not resolve into cleanup directories. + Cleanup directories must not resolve into backing storage. + Startup and staging cleanup do not consult volume registrations and can + delete backing files placed in those locations. +- `directio` is `on`, `off`, or `auto`, and defaults to `off`. +- `writable` defaults to `false` and controls writable claim admission. +- `tenants` grants the named tenants self-service access. Omitted or empty + permits every authenticated scope, for both static and runtime entries. + Names use the tenant API name syntax. Runtime registration rejects invalid + names. Static config warns and skips invalid names. A nonempty static list with no + valid names remains restricted and grants no tenant access. A valid name may be + registered before that tenant exists; it grants no access until the tenant + is created. + +Success is 204 with an empty body. Repeating the same name, cleaned path, and +normalized registration fields succeeds. A different binding or different +registration fields return 409. Static and runtime registrations cannot share +a name or a cleaned backing path. Invalid registration fields return 400. +An uncertain write returns 503 with `Retry-After`. The requested registration +may still take effect through background retry or a later PUT or DELETE, +including a request for another name. A 503 can also reject a request before +acceptance, so it does not promise that the request will take effect. After +repairing storage failures, retry the exact request to establish completion. +Cancellation is not a rollback once a write has been accepted. +GET retains its existing projection without host paths or tenant access lists. +Registered entries support the existing mounted read-only live volume API. + +Only registration metadata is persisted locally in +`data_dir/runtime-volumes.json`, which loads before claim recovery. New records +use mode 0600. The daemon reads the record with the same filesystem permissions +as `claims.json` and replaces it durably. Registration creates, grows, and deletes +no disks. +Malformed or conflicting metadata prevents startup. A temporarily missing +backing file keeps its registration and refuses new attachment. +Catalog replacement and synchronization are independent of `sync_claims`, which +controls claim snapshot synchronization. Process-restart checks do not prove +power-loss recovery. The catalog is node-local with one sandboxd writer. +Startup reads only `runtime-volumes.json`. Crash leftovers named +`runtime-volumes.json.*.tmp` are neither loaded nor promoted. An operator can +remove those leftovers after stopping sandboxd and preserving any needed +evidence. + +## DELETE /v1/volumes/{name} + +Auth and name validation match PUT. Removes only the runtime registration and +never deletes or changes the backing file. Success, including an absent, +unreferenced runtime name, is 204 with an empty body. Static registrations and +names referenced by a claim, an in-flight operation, cleanup, or an +unacknowledged recovery snapshot return 409. Catalog persistence uncertainty +returns 503 with `Retry-After` and has the same asynchronous completion +semantics as PUT. A failed DELETE may therefore remove the registration later. +An empty GET volume view alone does not establish that a recovery reference +has retired. + +Downgrading to a daemon without runtime registration requires draining runtime +references or supplying equivalent static volume config. Pending live volume +changes must finish before downgrade. + +Stale VM cleanup inspects attachments before removal. A complete running +inventory retains holds only on the named volumes. An explicit empty disk +array permits unrelated volume admission and catalog changes while removal +retries continue. An omitted or null disk list, a stopped VM, an inspection +failure, or an active creating owner retains the node-wide fence until +inspection or confirmed removal resolves it. Cocoon v0.7.3 omits both empty +inventories and failed disk queries, and does not collect disks for stopped +VMs. Those results remain unknown and can block static read-only admission. +Allowing readers would conflict with an unknown writer. Cleanup retries +remain enabled. +A warm VM name is not evidence that no volume is attached; attachment can +succeed before claim finalization fails. Known inventories keep holds on their +reported names. Volume-free claims continue to work. Path uniqueness uses +cleaned path strings. The operator coordinates symlink and hardlink aliases +and cross-node ownership. + ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set @@ -277,7 +374,8 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed PUT or PATCH adds a top-level `error` naming the refusal, and the affected -state may carry its own `error`. `converging` is true only while a request worker is running. +state may carry its own `error`. `converging` is true while a request worker is running or a recovery-only +reference awaits an acknowledged claim write. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an diff --git a/docs/security.md b/docs/security.md index 6c519f49..841e9830 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,15 +94,26 @@ rather than confirming existence, and operator surfaces answer tenants 403. The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is -node-local. An empty entry `tenants` list permits every authenticated scope; a -nonempty list permits only those named tenants, while the root token always -has access. Config load rejects an access-list name that is not a configured -tenant. Claim lookup returns byte-identical errors for an unknown and a -forbidden name, and catalog discovery filters before replying, so a tenant +node-local. An empty `tenants` list permits every authenticated scope for +both static and runtime entries. A nonempty list grants the named tenants +self-service access. For live additions, the claim owner must satisfy the +entry's ACL, even when root sends the request. Runtime registration rejects +invalid tenant names. Static config warns and skips them. A nonempty static +list with no valid names permits no tenant access. A valid name that does not +yet identify a tenant grants no access until that tenant exists. +Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant cannot enumerate restricted entries by probing. Gossip carries names only. Neither gossip, discovery, persisted claims, usage events, nor the sandbox index exposes host image paths or access lists. +Runtime registration and unregistration require a configured root `api_token`; +these endpoints return 403 when the token is unset. New catalog records use +mode 0600. Reads and replacements use the daemon's filesystem permissions. +Registration uses the same path validation as static configuration. It does +not verify backing type +or pin backing identity. Operators must supply a backing supported by Cocoon, +prevent path or symlink target replacement while registered, and coordinate aliases. + Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path to every destination its tenant egress policy permits; review the volume access @@ -117,7 +128,9 @@ immutable and publish a new name/path for new content. Changing a running claim's volumes needs the root token or the owning tenant's API token. A sandbox token cannot, so a guest holding its own sandbox token cannot attach another dataset, and every addition passes the same access list -as a claim. +as a claim. Already-held references can finish repair or removal without a +new ACL grant. A recovery-only reference protects ownership and does not grant +permission for a fresh attachment. A writable entry (`writable: true`) adds a channel the read-only model does not have: whichever tenant is permitted to claim it `rw` changes what every diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go new file mode 100644 index 00000000..08c459fc --- /dev/null +++ b/e2e/volume_catalog_test.go @@ -0,0 +1,112 @@ +package e2e + +import ( + "encoding/json/v2" + "net/http" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + sandbox "github.com/cocoonstack/sandbox/sdk/go" +) + +func TestRuntimeVolumePublicAdmissionThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "shared bytes"), "writable": true}) + if err != nil { + t.Fatal(err) + } + for _, scope := range []struct { + token string + want int + }{{"acme-token", http.StatusForbidden}, {"", http.StatusUnauthorized}, {"node-token", http.StatusNoContent}} { + caller := *st + caller.token = scope.token + if status, body := rawJSON(t, &caller, http.MethodPut, "/v1/volumes/home", string(request)); status != scope.want { + t.Fatalf("registration auth: status=%d want=%d body=%s", status, scope.want, body) + } + } + volumes, err := acme.Volumes(t.Context()) + if err != nil || len(volumes) != 1 || volumes[0].Name != "home" { + t.Fatalf("empty runtime ACL is public: volumes=%v err=%v", volumes, err) + } + volume := sandbox.Volume{Name: "home", Mount: "/mnt/home"} + first, err := acme.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volume)) + if err != nil { + t.Fatalf("tenant claim with public runtime volume: %v", err) + } + defer first.Close() + if !slices.Equal(first.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("claim did not retain mounted volume: %+v", first.Volumes) + } + if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { + t.Fatal(err) + } + if view, attachErr := first.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("tenant PATCH of public runtime volume: view=%+v err=%v", view, attachErr) + } + if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { + t.Fatal(err) + } + if view, attachErr := first.SetVolumes(t.Context(), []sandbox.Volume{volume}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("tenant PUT of public runtime volume: view=%+v err=%v", view, attachErr) + } +} + +func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{ + {Name: "acme", Token: "acme-token"}, {Name: "beta", Token: "beta-token"}, + }, []config.VolumeSpec{{Name: "public", Path: writeVolumeImage(t, "public.img", "shared")}}) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/private", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + for _, tc := range []struct { + token string + want []string + allowed bool + }{{"acme-token", []string{"private", "public"}, true}, {"beta-token", []string{"public"}, false}} { + client, connectErr := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) + if connectErr != nil { + t.Fatal(connectErr) + } + volumes, listErr := client.Volumes(t.Context()) + var names []string + for _, volume := range volumes { + names = append(names, volume.Name) + } + if listErr != nil || !slices.Equal(names, tc.want) { + t.Fatalf("catalog ACL: names=%v want=%v err=%v", names, tc.want, listErr) + } + claim, claimErr := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(sandbox.Volume{Name: "public"})) + if claimErr != nil { + t.Fatal(claimErr) + } + defer claim.Close() + _, attachErr := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{{Name: "private"}}}) + if (attachErr == nil) != tc.allowed { + t.Fatalf("explicit ACL admission for %s: %v", tc.token, attachErr) + } + } + caller := *st + caller.token = "beta-token" + var responses [][]byte + for _, name := range []string{"private", "unknown"} { + status, body := rawJSON(t, &caller, http.MethodPost, "/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"`+name+`"}]}`) + if status != http.StatusBadRequest { + t.Fatalf("claim privacy for %s: status=%d body=%s", name, status, body) + } + responses = append(responses, body) + } + if string(responses[0]) != string(responses[1]) { + t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) + } +} diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fba3efba..6b2c9295 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -3,6 +3,7 @@ package config import ( "cmp" + "context" "encoding/base64" "encoding/json/jsontext" "encoding/json/v2" @@ -18,6 +19,7 @@ import ( "strings" "github.com/docker/go-units" + "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -398,28 +400,17 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { - names := make(map[string]struct{}, len(c.Volumes)) - paths := make(map[string]string, len(c.Volumes)) - for _, volume := range c.Volumes { - if !types.ValidVolumeName(volume.Name) { - return fmt.Errorf("volume name %q must match %s and not start with cocoon-", volume.Name, types.VolumeNameRe) - } - if _, ok := names[volume.Name]; ok { - return fmt.Errorf("duplicate volume name %q", volume.Name) - } - names[volume.Name] = struct{}{} - if !filepath.IsAbs(volume.Path) { - return fmt.Errorf("volume %q path must be absolute", volume.Name) - } - if other, ok := paths[filepath.Clean(volume.Path)]; ok { - return fmt.Errorf("volume %q shares its path with volume %q: admission is keyed by name", volume.Name, other) - } - paths[filepath.Clean(volume.Path)] = volume.Name - if !types.ValidDirectIO(volume.DirectIO) { - return fmt.Errorf("volume %q directio must be on, off, or auto, got %q", volume.Name, volume.DirectIO) - } + validation := slices.Clone(c.Volumes) + for i, volume := range c.Volumes { + validation[i].Tenants = slices.DeleteFunc(slices.Clone(volume.Tenants), func(tenant string) bool { + if types.NameRe.MatchString(tenant) { + return false + } + log.WithFunc("config.validateVolumes").Warnf(context.Background(), "volume %q: skipping invalid tenant %q", volume.Name, tenant) + return true + }) } - return nil + return ValidateVolumeCatalog(validation) } func (c *Config) validatePreview() error { @@ -637,6 +628,37 @@ func (c *Config) validateMetaStore() error { return nil } +// ValidateVolumeCatalog checks the shared static and runtime catalog rules without I/O. +func ValidateVolumeCatalog(volumes []VolumeSpec) error { + names := make(map[string]struct{}, len(volumes)) + paths := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if !types.ValidVolumeName(volume.Name) { + return fmt.Errorf("volume name %q must match %s and not start with cocoon-", volume.Name, types.VolumeNameRe) + } + if _, ok := names[volume.Name]; ok { + return fmt.Errorf("duplicate volume name %q", volume.Name) + } + names[volume.Name] = struct{}{} + if !filepath.IsAbs(volume.Path) { + return fmt.Errorf("volume %q path must be absolute", volume.Name) + } + if other, ok := paths[filepath.Clean(volume.Path)]; ok { + return fmt.Errorf("volume %q shares its path with volume %q: admission is keyed by name", volume.Name, other) + } + paths[filepath.Clean(volume.Path)] = volume.Name + if !types.ValidDirectIO(volume.DirectIO) { + return fmt.Errorf("volume %q directio must be on, off, or auto, got %q", volume.Name, volume.DirectIO) + } + for _, tenant := range volume.Tenants { + if !types.NameRe.MatchString(tenant) { + return fmt.Errorf("volume %q tenant %q must match %s", volume.Name, tenant, types.NameRe) + } + } + } + return nil +} + // Load reads a JSON config file, applies defaults, and validates. func Load(path string) (*Config, error) { raw, err := os.ReadFile(path) //nolint:gosec // path is the operator-supplied -config flag diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 06c4fe3c..ce5d5766 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -463,6 +463,35 @@ func TestLoadAcceptsAShardedNetworkList(t *testing.T) { } } +func TestValidateVolumeCatalogTenantNames(t *testing.T) { + for _, tenant := range []string{"", "has space", "_leading", strings.Repeat("a", 64)} { + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{tenant}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err == nil { + t.Errorf("invalid tenant %q accepted", tenant) + } + } + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{"future-tenant", "team/project:v1"}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err != nil { + t.Fatalf("syntactically valid future tenant rejected: %v", err) + } +} + +func TestLoadPreservesRestrictedACLWithInvalidVolumeTenants(t *testing.T) { + for _, tenants := range []string{`["acme", "has space", "", "beta"]`, `["has space"]`} { + cfg, err := Load(writeConfig(t, `{"pools":[],"volumes":[{"name":"data","path":"/images/data","tenants":`+tenants+`}]}`)) + if err != nil { + t.Fatal(err) + } + want := []string{"has space"} + if strings.Contains(tenants, "acme") { + want = []string{"acme", "has space", "", "beta"} + } + if !slices.Equal(cfg.Volumes[0].Tenants, want) { + t.Fatalf("tenants = %v, want %v", cfg.Volumes[0].Tenants, want) + } + } +} + func writeConfig(t *testing.T, body string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.json") diff --git a/sandboxd/pool/checkpoint_references_test.go b/sandboxd/pool/checkpoint_references_test.go index 3125c68a..94601f8d 100644 --- a/sandboxd/pool/checkpoint_references_test.go +++ b/sandboxd/pool/checkpoint_references_test.go @@ -6,6 +6,7 @@ import ( "io/fs" "maps" "os" + "path/filepath" "slices" "sync" "syscall" @@ -14,17 +15,11 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -const ( - failWrite = iota - captureWrite - failAfterRename -) - func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { recreate := func(s *claimStore) { s.set(&types.Sandbox{ID: "owner", ReplicaCk: "recreated"}) } for _, tc := range []struct { name string - fault int + fault func(*testing.T, *claimStore, claimSnapshot) func() acked, next *types.Sandbox startup func(claims map[string]*types.Sandbox) mutate func(s *claimStore) @@ -87,11 +82,8 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - if tc.fault == failAfterRename && os.Geteuid() == 0 { - t.Skip("requires non-root: root can read the write-only directory") - } dir := t.TempDir() - s := newClaimStore(dir, tc.fault == failAfterRename) + s := newClaimStore(dir, false) if tc.acked != nil { if err := s.commit(s.set(tc.acked)); err != nil { t.Fatal(err) @@ -109,51 +101,7 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { } else { snap = s.set(tc.next) } - var finish func() - switch tc.fault { - case failWrite: - finish = failClaimWrites(t, s) - if err := s.commit(snap); err == nil { - t.Fatal("write succeeded under the fault") - } - case failAfterRename: - if err := os.Chmod(dir, 0o300); err != nil { - t.Fatal(err) - } - finish = func() { _ = os.Chmod(dir, 0o700) } - t.Cleanup(finish) - if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { - t.Fatalf("want a failed directory sync after the rename: %v", err) - } - case captureWrite: - if err := syscall.Mkfifo(s.path+".tmp", 0o600); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - go func() { done <- s.commit(snap) }() - waitFor(t, func() bool { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.retained["owner"]) != 0 - }) - finish = func() { - r, err := os.Open(s.path + ".tmp") - if err != nil { - t.Fatal(err) - } - raw, err := io.ReadAll(r) - _ = r.Close() - if err = errors.Join(err, <-done); err != nil { - t.Fatal(err) - } - if err = os.Remove(s.path); err != nil { - t.Fatal(err) - } - if err = os.WriteFile(s.path, raw, 0o600); err != nil { - t.Fatal(err) - } - } - } + finish := tc.fault(t, s, snap) if tc.mutate != nil { tc.mutate(s) } @@ -208,3 +156,72 @@ func failClaimWrites(t *testing.T, s *claimStore) func() { t.Cleanup(restore) return restore } + +func captureClaimWrite(t *testing.T, s *claimStore, snap claimSnapshot, owner string) func() []byte { + t.Helper() + if err := syscall.Mkfifo(s.path+".tmp", 0o600); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- s.commit(snap) }() + waitFor(t, func() bool { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.retained[owner]) != 0 || len(s.retainedVolumes[owner]) != 0 + }) + return func() []byte { + r, err := os.Open(s.path + ".tmp") + if err != nil { + t.Fatal(err) + } + raw, err := io.ReadAll(r) + _ = r.Close() + if err = errors.Join(err, <-done); err != nil { + t.Fatal(err) + } + if err = os.Remove(s.path); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(s.path, raw, 0o600); err != nil { + t.Fatal(err) + } + return raw + } +} + +func failWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { + t.Helper() + restore := failClaimWrites(t, s) + if err := s.commit(snap); err == nil { + t.Fatal("write succeeded under the fault") + } + return restore +} + +func failAfterRename(t *testing.T, s *claimStore, snap claimSnapshot) func() { + t.Helper() + if os.Geteuid() == 0 { + t.Skip("requires non-root: root can read the write-only directory") + } + s.sync = true + dir := filepath.Dir(s.path) + if err := os.Chmod(dir, 0o300); err != nil { + t.Fatal(err) + } + restore := func() { + if err := os.Chmod(dir, 0o700); err != nil { + t.Error(err) + } + } + t.Cleanup(restore) + if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { + t.Fatalf("want a failed directory sync after the rename: %v", err) + } + return restore +} + +func captureWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { + t.Helper() + release := captureClaimWrite(t, s, snap, "owner") + return func() { release() } +} diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index f9427792..9d9110bb 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -371,7 +371,7 @@ func (m *Manager) overQuota(extra int, tenant string) error { // admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) - if err := m.admitClaim(tenant, applied); err != nil { + if err := m.admitClaim(tenant, volumes); err != nil { return nil, err } if err := m.confirmVolumesClean(volumes); err != nil { @@ -382,13 +382,13 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types } // admitClaim refuses a busy volume or an over-quota tenant before any VM is built. -func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { +func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveVolumes(volumes) + return m.reserveResolved(volumes, nil) } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 2f07ecaf..56f2bfe5 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -72,6 +72,8 @@ type claimRow struct { raw []byte } +type recoveryVolumeCount struct{ readers, writers int } + // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string @@ -81,16 +83,20 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 - refs map[string][4]string - retained map[string][]string - changed map[string]struct{} - checkpoints map[string]int - - recommitting atomic.Bool + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 + refs map[string][4]string + retained map[string][]string + changed map[string]struct{} + checkpoints map[string]int + currentVolumes map[string][]types.Volume + retainedVolumes map[string][]types.Volume + volumes map[string]recoveryVolumeCount + + recommitting atomic.Bool + recoveryVolumesPresent atomic.Bool } func newClaimStore(dataDir string, sync bool) *claimStore { @@ -98,6 +104,7 @@ func newClaimStore(dataDir string, sync bool) *claimStore { path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, refs: map[string][4]string{}, retained: map[string][]string{}, changed: map[string]struct{}{}, checkpoints: map[string]int{}, + currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]recoveryVolumeCount{}, } } @@ -119,6 +126,10 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } + if refs := volumeRefs(nil, slices.Clone(sb.Volumes), slices.Clone(sb.VolumesWant)); len(refs) != 0 { + s.changeRetainedVolumes(sb.ID, refs) + s.changed[sb.ID] = struct{}{} + } if refs := checkpointRefs(sb); refs != [4]string{} { s.changeRetainedRefs(sb.ID, appendRefs(nil, refs)) s.changed[sb.ID] = struct{}{} @@ -151,24 +162,34 @@ func (s *claimStore) del(ids ...string) claimSnapshot { func (s *claimStore) put(id string, sb *types.Sandbox) { var refs [4]string + var volumes []types.Volume if sb == nil { delete(s.rows, id) } else { + dto := dtoOf(sb) refs = checkpointRefs(sb) - s.rows[id] = &claimRow{dto: dtoOf(sb)} + volumes = volumeRefs(s.currentVolumes[id], dto.Volumes, dto.VolumesWant) + s.rows[id] = &claimRow{dto: dto} } before := s.refs[id] - if before == refs { - return + if before != refs { + s.changeCheckpointCounts(before[:], -1) + s.changeCheckpointCounts(refs[:], 1) + if refs == [4]string{} { + delete(s.refs, id) + } else { + s.refs[id] = refs + } + s.changed[id] = struct{}{} } - s.changeCheckpointCounts(before[:], -1) - s.changeCheckpointCounts(refs[:], 1) - if refs == [4]string{} { - delete(s.refs, id) - } else { - s.refs[id] = refs + if !slices.Equal(s.currentVolumes[id], volumes) { + if len(volumes) == 0 { + delete(s.currentVolumes, id) + } else { + s.currentVolumes[id] = volumes + } + s.changed[id] = struct{}{} } - s.changed[id] = struct{}{} } // mark sequences a persist of the projection as it already stands. @@ -211,9 +232,17 @@ func (s *claimStore) commit(snap claimSnapshot) error { } rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq captured := make(map[string][4]string, len(s.changed)) + var capturedVolumes map[string][]types.Volume for id := range s.changed { captured[id] = s.refs[id] - s.changeRetainedRefs(id, appendRefs(s.retained[id], s.refs[id])) + s.changeRetainedRefs(id, appendRefs(slices.Clone(s.retained[id]), s.refs[id])) + if volumes := s.currentVolumes[id]; len(volumes) != 0 { + if capturedVolumes == nil { + capturedVolumes = make(map[string][]types.Volume) + } + capturedVolumes[id] = volumes + } + s.changeRetainedVolumes(id, appendRecoveryRefs(slices.Clone(s.retainedVolumes[id]), s.currentVolumes[id])) } s.mu.Unlock() s.buf = append(s.buf[:0], '{') @@ -237,7 +266,8 @@ func (s *claimStore) commit(snap claimSnapshot) error { s.mu.Lock() for id, refs := range captured { s.changeRetainedRefs(id, appendRefs(nil, refs)) - if s.refs[id] == refs { + s.changeRetainedVolumes(id, capturedVolumes[id]) + if s.refs[id] == refs && slices.Equal(s.currentVolumes[id], capturedVolumes[id]) { delete(s.changed, id) } } @@ -340,6 +370,88 @@ func (s *claimStore) checkpointReferences() map[string]struct{} { return refs } +func (s *claimStore) changeVolumeCounts(refs []types.Volume, delta int) { + for _, v := range refs { + count := s.volumes[v.Name] + if v.RW() { + count.writers += delta + } else { + count.readers += delta + } + if count == (recoveryVolumeCount{}) { + delete(s.volumes, v.Name) + } else { + s.volumes[v.Name] = count + } + } +} + +func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { + s.changeVolumeCounts(s.retainedVolumes[id], -1) + s.changeVolumeCounts(after, 1) + if len(after) == 0 { + delete(s.retainedVolumes, id) + } else { + s.retainedVolumes[id] = after + } + s.recoveryVolumesPresent.Store(len(s.volumes) != 0) +} + +func (s *claimStore) referencesVolume(name string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.volumes[name] != (recoveryVolumeCount{}) +} + +func (s *claimStore) volumeConflicts(volume types.Volume, ownerID string) bool { + s.mu.Lock() + defer s.mu.Unlock() + count := s.volumes[volume.Name] + for _, ref := range s.retainedVolumes[ownerID] { + if ref == volume { + if ref.RW() { + count.writers-- + } else { + count.readers-- + } + } + } + return count.writers > 0 || volume.RW() && count.readers > 0 +} + +func (s *claimStore) recoveryVolumesFor(owner string) []types.Volume { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.retainedVolumes[owner]) +} + +func (s *claimStore) detachingOwners() map[string]bool { + if !s.recoveryVolumesPresent.Load() { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + var owners map[string]bool + for id := range s.changed { + if len(s.currentVolumes[id]) == 0 && len(s.retainedVolumes[id]) != 0 { + if owners == nil { + owners = map[string]bool{} + } + owners[id] = true + } + } + return owners +} + +func (s *claimStore) hasRecoveryVolumes(owner string) bool { + if !s.recoveryVolumesPresent.Load() { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return len(s.retainedVolumes[owner]) != 0 +} + func encodeRow(dto *claimDTO) ([]byte, error) { raw, err := json.Marshal(map[string]*claimDTO{dto.ID: dto}) if err != nil { @@ -381,3 +493,31 @@ func appendRefs(list []string, refs [4]string) []string { } return list } + +func appendRecoveryRefs(dst, src []types.Volume) []types.Volume { + for _, v := range src { + if !slices.Contains(dst, v) { + dst = append(dst, v) + } + } + return dst +} + +func volumeRefs(previous, applied, wanted []types.Volume) []types.Volume { + if len(applied) == 0 { + applied, wanted = wanted, nil + } + for _, refs := range [2][]types.Volume{applied, wanted} { + for _, v := range refs { + if !slices.Contains(previous, v) { + return appendRecoveryRefs(applied, wanted) + } + } + } + for _, v := range previous { + if !slices.Contains(applied, v) && !slices.Contains(wanted, v) { + return appendRecoveryRefs(applied, wanted) + } + } + return previous +} diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index f7e19311..e12ea0e7 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -88,10 +88,14 @@ func TestStoreRoundTrip(t *testing.T) { if err := s.save(claims); err != nil { t.Fatalf("save (sync=%v): %v", fsync, err) } - got, err := newClaimStore(filepath.Dir(s.path), false).load() + loadedStore := newClaimStore(filepath.Dir(s.path), false) + got, err := loadedStore.load() if err != nil { t.Fatalf("load (sync=%v): %v", fsync, err) } + if !loadedStore.hasRecoveryVolumes("sb_a") || loadedStore.hasRecoveryVolumes("sb_b") { + t.Fatal("loaded recovery volumes lost owner visibility") + } if !reflect.DeepEqual(got, claims) { t.Errorf("sync=%v: got %+v, want %+v", fsync, got, claims) } @@ -103,9 +107,10 @@ func TestStoreRoundTrip(t *testing.T) { func TestClaimStoreSnapshotDetachesVolumes(t *testing.T) { s := newClaimStore(t.TempDir(), false) - sb := &types.Sandbox{ID: "sb_a", Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}} + sb := &types.Sandbox{ID: "sb_a", Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VolumesWant: []types.Volume{{Name: "dataset-b", Mount: "/datasets/b"}}} snap := s.set(sb) sb.Volumes[0].Mount = "/mutated" + sb.VolumesWant[0].Mount = "/mutated-wanted" if err := s.commit(snap); err != nil { t.Fatalf("commit: %v", err) } @@ -120,6 +125,9 @@ func TestClaimStoreSnapshotDetachesVolumes(t *testing.T) { if !slices.Equal(loaded.Volumes, []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}) { t.Errorf("volumes %v, want detached snapshot", loaded.Volumes) } + if !slices.Equal(loaded.VolumesWant, []types.Volume{{Name: "dataset-b", Mount: "/datasets/b"}}) { + t.Errorf("wanted volumes %v, want detached snapshot", loaded.VolumesWant) + } } func TestClaimRowOmitsUnsetFlags(t *testing.T) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9e0124cb..e2cb2391 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -116,6 +116,9 @@ var ( errNotSleeping = errors.New("sandbox is no longer hibernated or archived") errWokeMeanwhile = errors.New("woke between sweep and hibernate") errNoEgressTap = errors.New("egress-lane claim has no lockable tap") + + ErrVolumeRegistrationConflict = errors.New("volume registration conflicts with the catalog") + ErrVolumeCatalogUnavailable = errors.New("volume catalog requires a successful persistence retry") ) // Engine is the slice of the cocoon driver the manager consumes. @@ -321,16 +324,14 @@ type PeerDeleteFunc func(ctx context.Context, id string) // Manager owns the node's pools, claims, and their persistence. type Manager struct { - eng Engine - dataDir string - egress bool - maxFork int - store *claimStore - volumes map[string]catalogVolume - // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]map[string]string + eng Engine + dataDir string + egress bool + maxFork int + store *claimStore + catalog volumeCatalog + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]map[string]string poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. @@ -448,8 +449,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - volumes: make(map[string]catalogVolume, len(cfg.Volumes)), - volumeAdmission: map[string]volumeHolders{}, + catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), holders: map[string]volumeHolders{}}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), @@ -478,12 +478,15 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg healSem: make(chan struct{}, maxConcurrentHeals), } for _, volume := range cfg.Volumes { - m.volumes[volume.Name] = catalogVolume{ - disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, + m.catalog.entries[volume.Name] = &catalogVolume{ + disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, tenants: slices.Clone(volume.Tenants), writable: volume.Writable, } } + if err := m.loadRuntimeVolumes(); err != nil { + return nil, err + } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) } @@ -600,6 +603,7 @@ func (m *Manager) Run(ctx context.Context) { case <-m.refillKick: m.refillOnce(ctx) case <-reap.C: + go m.retryVolumeCatalog(ctx) m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 56e4148f..d9fc8b46 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1255,6 +1255,9 @@ type fakeEngine struct { detachVolumeErr error liveVolumeHook func(string) error stopped map[string]bool + inspectHook func() error + inspectDisksUnknown bool + trackDisks bool disks map[string][]types.VMDisk creating map[string]bool staleOutcome engine.StaleCreateOutcome @@ -1526,14 +1529,22 @@ func (f *fakeEngine) List(ctx context.Context) ([]types.VMRecord, error) { func (f *fakeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { f.mu.Lock() f.inspects++ + hook := f.inspectHook f.mu.Unlock() + if hook != nil { + if err := hook(); err != nil { + return types.VMRecord{}, false, err + } + } vms, _ := f.list(ctx) i := slices.IndexFunc(vms, func(vm types.VMRecord) bool { return vm.Config.Name == name }) if i < 0 { return types.VMRecord{}, false, nil } f.mu.Lock() - vms[i].AttachedDevices.Disks = f.disks[name] + if !f.inspectDisksUnknown { + vms[i].AttachedDevices.Disks = f.disks[name] + } f.mu.Unlock() return vms[i], true, nil } @@ -1618,7 +1629,7 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } -func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { +func (f *fakeEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("attach"); err != nil { return err @@ -1635,6 +1646,12 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.Volum if spec.Name == f.diskAttachErrFor { err = errors.New("attach failed") } + if err == nil && f.trackDisks { + if f.disks == nil { + f.disks = map[string][]types.VMDisk{} + } + f.disks[name] = append(f.disks[name], types.VMDisk{Name: spec.Name, ReadOnly: !spec.RW}) + } f.mu.Unlock() if cancel != nil { cancel() diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index a116b580..056fdf81 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -70,13 +70,14 @@ func (m *Manager) Reconcile(ctx context.Context) error { owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } - saveErr := m.store.save(m.claimed) + js := m.store.reset(m.claimed) pooled := make(map[string]bool, len(m.pools)) for _, p := range m.pools { m.adoptGolden(p, imageIDs[p.key.Template]) pooled[p.hash] = true } m.mu.Unlock() + saveErr := m.store.commit(js) for _, sb := range adopted { m.recordHibernate(ctx, sb) } @@ -156,6 +157,10 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec // removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") + m.mu.Lock() + m.catalog.unknown++ + m.mu.Unlock() + td := volumeTeardown{unknown: true} if rec.State == vmStateCreating { // a canceled ctx must not skip the busy check guarding the forced remove switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); { @@ -163,41 +168,52 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe // verb missing (cocoon < v0.5.8) or failed: keep the old sweep. logger.Warnf(ctx, "reconcile stale create %s: %v; removing", name, err) case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: + m.finishVolumeTeardown(ctx, td) return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - m.queueStaleCreate(name, rec.TapDevice()) + m.queueStaleCreate(name, rec.TapDevice(), td) return false } // not-creating: the record moved on under the lock; remove normally. } + td = m.resolveStaleVolumes(ctx, name, td) if m.removeVM(ctx, name) { + m.finishVolumeTeardown(ctx, td) return true } - m.queueRemoval(name, "", rec.TapDevice(), volumeTeardown{holds: m.holdStaleVolumes(ctx, name, rec)}, time.Time{}) + m.queueRemoval(name, "", rec.TapDevice(), td, time.Time{}) return false } -func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.VMRecord) []types.Volume { - if rec.State != vmStateRunning { - return nil +func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volumeTeardown) volumeTeardown { + if !td.unknown { + return td } - inspected, _, err := m.eng.Inspect(ctx, name) + inspected, present, err := m.eng.Inspect(ctx, name) if err != nil { - log.WithFunc("pool.holdStaleVolumes").Warnf(ctx, "inspect stale vm %s, its volumes stay unheld: %v", name, err) - return nil + log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) + return td } - holds := make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW + if present && (inspected.State != vmStateRunning || inspected.AttachedDevices.Disks == nil) { + return td + } + var holds []types.Volume + if present { + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW + } } } m.mu.Lock() m.adoptVolumes(holds) + m.catalog.unknown-- m.mu.Unlock() - return holds + td.unknown, td.holds = false, holds + return td } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index b50d78dc..636555e7 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -181,8 +181,8 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { if !eng.removed("sbx-orphan-1") { t.Error("unowned running VM not removed") } - if eng.inspects != 0 { - t.Errorf("inspected a stale VM whose removal succeeded %d times", eng.inspects) + if eng.inspects != 1 { + t.Errorf("inspected stale VM %d times, want 1 before removal", eng.inspects) } } diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 75efec26..924e3f5d 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -45,7 +45,9 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { if m.removeVM(ctx, name) { - m.finishVolumeTeardown(ctx, td) + if !m.finishVolumeTeardown(ctx, td) { + m.queueRemoval(name, sandboxID, "", td, time.Time{}) + } return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -58,9 +60,9 @@ func (m *Manager) queueRemoval(name, sandboxID, tap string, td volumeTeardown, n m.mu.Unlock() } -func (m *Manager) queueStaleCreate(name, tap string) { +func (m *Manager) queueStaleCreate(name, tap string, td volumeTeardown) { m.mu.Lock() - m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true} + m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true, volumes: td} m.mu.Unlock() } @@ -77,7 +79,7 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } m.mu.Unlock() names := slices.Collect(maps.Keys(batch)) - return m.runBounded(ctx, len(names), func(ctx context.Context, i int) { + return m.runBounded(context.WithoutCancel(ctx), len(names), func(ctx context.Context, i int) { m.retryRemoval(ctx, names[i], batch[names[i]]) }) } @@ -87,25 +89,29 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending switch outcome, err := m.eng.ReconcileStaleCreate(ctx, name); { case err != nil: log.WithFunc("pool.retryRemoval").Warnf(ctx, "reconcile stale create %s: %v; retrying", name, err) - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) return } } + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if !m.removeVM(ctx, name) { m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) } -func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { - m.finishVolumeTeardown(ctx, pending.volumes) +func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { + if !m.finishVolumeTeardown(ctx, pending.volumes) { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + return + } if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 6524154d..93b666c9 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -27,6 +27,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { var withVolumes []string next := make(map[string]replicaState, len(m.replicas)) m.mu.Lock() + detaching := m.store.detachingOwners() for id, sb := range m.claimed { every, st := v.poolReplicate[sb.PolicyKey()], m.replicas[id] switch { @@ -35,7 +36,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { retire = append(retire, sb) } case !m.replicable(sb): - case m.hasVolumeHolds(sb): + case m.hasLiveVolumeHolds(sb) || detaching[id]: if sb.HasReplica() { retire = append(retire, sb) } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 0a157b61..b56103dc 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -4,6 +4,7 @@ import ( "encoding/json/v2" "errors" "net" + "os" "testing" "testing/synctest" "time" @@ -1002,6 +1003,61 @@ func TestReplicaAdoptionWakesTheClaimFromTheLastReplica(t *testing.T) { } } +func TestReplicaScanWaitsForDetachAcknowledgment(t *testing.T) { + c := newReplicaCluster(t) + m := c.node(t, "a", t.TempDir(), keepAnHour) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "replicated-data", "bytes")}); err != nil { + t.Fatal(err) + } + sb, attached, ordinary := mustClaim(t, m, testKey), mustClaim(t, m, testKey), mustClaim(t, m, testKey) + for _, owner := range []*types.Sandbox{sb, attached} { + if _, err := m.SetVolumes(t.Context(), owner.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + } + m.store.recommitting.Store(true) + eng := m.eng.(*fakeEngine) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "", nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detach did not fail after device removal: %+v, %v", view, err) + } + m.replicateOnce(t.Context()) + if m.replicating.Load() || !m.replicas[sb.ID].skipLogged || !m.replicas[attached.ID].skipLogged || m.replicas[ordinary.ID].tried.IsZero() || m.replicas[ordinary.ID].skipLogged { + t.Fatalf("scan failed to distinguish detached, attached, and ordinary claims: %+v", m.replicas) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture ignored retained attachment: %v", err) + } + if sb.HasReplica() || len(eng.snapSaves) != 0 { + t.Fatal("pending detach published a replica") + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.replicateOnce(t.Context()) + if st := m.replicas[sb.ID]; st.skipLogged || st.tried.IsZero() { + t.Fatalf("acknowledged detach stayed skipped: %+v", st) + } + m.replicas[sb.ID] = replicaState{tried: time.Now().Add(-time.Hour)} + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || rec.Consumed || rec.Checkpoint == "" || !sb.HasReplica() { + t.Fatalf("acknowledged detach did not permit replication: %+v, %t, %v", rec, ok, err) + } +} + func newReplicaCluster(t *testing.T) *failoverCluster { t.Helper() c := newFailoverCluster(t) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 5a1e8eed..0660e7c5 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -31,31 +31,40 @@ type volumeHolders struct { } type resolvedVolume struct { + entry *catalogVolume disk engine.VolumeSpec applied types.Volume } // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - holds []types.Volume - clears []string + done <-chan struct{} + unknown bool + holds []types.Volume + clears []string } type catalogVolume struct { + runtime bool disk engine.VolumeSpec tenants []string writable bool } -func (v catalogVolume) allowed(tenant string) bool { +func (v *catalogVolume) allowedFor(tenant string) bool { return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) } +func (v catalogVolume) registration() types.VolumeRegistration { + return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.tenants} +} + // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { - infos := make([]types.VolumeInfo, 0, len(m.volumes)) - for name, volume := range m.volumes { - if !volume.allowed(tenant) { + catalog := m.catalogSnapshot() + infos := make([]types.VolumeInfo, 0, len(catalog)) + for name, volume := range catalog { + if !volume.allowedFor(tenant) { continue } info := types.VolumeInfo{ @@ -74,7 +83,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { - if _, ok := m.volumes[name]; !ok { + if _, ok := catalog[name]; !ok { infos = append(infos, types.VolumeInfo{ Name: name, DefaultMount: types.DefaultVolumeMount(name), @@ -89,8 +98,9 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // VolumeNames returns the sorted names this node can currently serve. func (m *Manager) VolumeNames() []string { - names := make([]string, 0, len(m.volumes)) - for name, volume := range m.volumes { + catalog := m.catalogSnapshot() + names := make([]string, 0, len(catalog)) + for name, volume := range catalog { if _, err := os.Stat(volume.disk.Path); err == nil { names = append(names, name) } @@ -108,8 +118,9 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return true, nil } local := true + catalog := m.catalogSnapshot() for _, name := range names { - volume, ok := m.volumes[name] + volume, ok := catalog[name] if !ok { if tenant == "" { local = false @@ -117,7 +128,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri } return false, ErrVolumeUnavailable } - if !volume.allowed(tenant) { + if !volume.allowedFor(tenant) { return false, ErrVolumeUnavailable } if _, err := os.Stat(volume.disk.Path); err != nil { @@ -131,18 +142,24 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } + m.mu.Lock() + if err := m.catalog.admissionError(); err != nil { + m.mu.Unlock() + return nil, err + } + catalog := m.catalog.entries + m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := m.volumes[volume.Name] - if !ok || !entry.allowed(tenant) { + entry, ok := catalog[volume.Name] + if !ok || !entry.allowedFor(tenant) { return nil, ErrVolumeUnavailable } - resolvedEntry, resolveErr := m.resolveCatalogVolume(ctx, volume) + resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) if resolveErr != nil { return nil, resolveErr } @@ -151,9 +168,8 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return resolved, nil } -func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { - entry, ok := m.volumes[volume.Name] - if !ok { +func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, entry *catalogVolume) (resolvedVolume, error) { + if entry == nil { return resolvedVolume{}, ErrVolumeUnavailable } if volume.RW() && !entry.writable { @@ -161,13 +177,13 @@ func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) } if _, statErr := os.Stat(entry.disk.Path); statErr != nil { // the operator's path stays server-side; the refusal matches the unknown one - log.WithFunc("pool.resolveCatalogVolume").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) + log.WithFunc("pool.resolveCatalogEntry").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return resolvedVolume{}, ErrVolumeUnavailable } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false // past validation the empty mount carries it - return resolvedVolume{disk: disk, applied: volume}, nil + volume.AttachOnly = false + return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil } // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. @@ -227,7 +243,12 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { + m.mu.Lock() td := volumeTeardown{holds: heldVolumes(sb)} + if run := m.volumeRuns[sb]; run != nil { + td.done = run.done + } + m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { return td @@ -246,9 +267,11 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } - if entry, ok := m.volumes[volume.Name]; ok { + m.mu.Lock() + if entry, ok := m.catalog.entries[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } + m.mu.Unlock() } if stuck { // the unmounts may have spent the whole budget; the fallback flush gets its own @@ -262,9 +285,13 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT } // finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. -func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { - if len(td.holds) == 0 { - return +func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) bool { + if td.done != nil { + select { + case <-td.done: + default: + return false + } } logger := log.WithFunc("pool.finishVolumeTeardown") for _, path := range td.clears { @@ -272,13 +299,36 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { logger.Errorf(ctx, err, "clear dirty marker %s", path) } } - m.unreserveVolumes(td.holds) + m.mu.Lock() + m.releaseVolumes(td.holds) + if td.unknown { + m.catalog.unknown-- + } + m.mu.Unlock() + return true } // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { + return m.reserveVolumesFor(volumes, nil) +} + +func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { + if len(volumes) == 0 { + return nil + } + if err := m.catalog.admissionError(); err != nil { + return err + } + ownerID := "" + if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { + ownerID = owner.ID + } for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + if m.store.volumeConflicts(volume, ownerID) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } + holders := m.catalog.holders[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) } @@ -290,30 +340,30 @@ func (m *Manager) reserveVolumes(volumes []types.Volume) error { // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers++ } else { holders.readers++ } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } // releaseVolumes drops one claim's admission holds; callers hold m.mu. func (m *Manager) releaseVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers-- } else { holders.readers-- } if holders.writers < 1 && holders.readers < 1 { - delete(m.volumeAdmission, volume.Name) + delete(m.catalog.holders, volume.Name) continue } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } @@ -326,6 +376,21 @@ func (m *Manager) unreserveVolumes(volumes []types.Volume) { m.releaseVolumes(volumes) } +func (m *Manager) catalogSnapshot() map[string]*catalogVolume { + m.mu.Lock() + defer m.mu.Unlock() + return m.catalog.entries +} + +func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox) error { + for _, volume := range volumes { + if volume.entry == nil || m.catalog.entries[volume.applied.Name] != volume.entry { + return ErrVolumeUnavailable + } + } + return m.reserveVolumesFor(appliedVolumes(volumes), owner) +} + // quiesceBudget bounds one quiesce: a slice per writable mount plus one for the sync fallback. func quiesceBudget(mounts int) time.Duration { return min(time.Duration(mounts+1)*engine.VolumeCallTimeout, volumeQuiesceMax) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go new file mode 100644 index 00000000..2ebe99a2 --- /dev/null +++ b/sandboxd/pool/volume_acl_test.go @@ -0,0 +1,152 @@ +package pool + +import ( + "errors" + "os" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestRuntimeCatalogDefaultACL(t *testing.T) { + for _, tenants := range [][]string{nil, {}} { + t.Run("empty", func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m, _ := newTenantVolumeManager(t, eng, catalog) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: tenants} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { + t.Fatal(err) + } + for _, current := range []*Manager{m, restartACLManager(t, eng, m.dataDir, catalog)} { + if !slices.ContainsFunc(current.Volumes("acme", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("public runtime volume is hidden from a tenant") + } + if !slices.ContainsFunc(current.Volumes("", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("operator cannot see runtime volume") + } + if _, err := current.VolumePlacement(testKey, "acme", []string{"home"}); err != nil { + t.Errorf("tenant placement: %v", err) + } + if local, err := current.VolumePlacement(testKey, "", []string{"home"}); err != nil || !local { + t.Errorf("operator placement: %v %v", local, err) + } + tenantClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) + if err != nil { + t.Fatalf("tenant initial attachment: %v", err) + } + if err = current.Release(t.Context(), tenantClaim.ID, Cred{Token: tenantClaim.Token}); err != nil { + t.Fatal(err) + } + if _, err = current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Errorf("public static volume changed: %v", err) + } + rootClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) + if err != nil { + t.Fatalf("operator initial attachment: %v", err) + } + if err := current.Release(t.Context(), rootClaim.ID, Cred{Token: rootClaim.Token}); err != nil { + t.Fatalf("operator initial claim cleanup: %v", err) + } + } + }) + } +} + +func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { + m, sb := newTenantVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Tenants: []string{"other"}}); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "home"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("operator bypassed explicit claim-owner ACL: %v", err) + } +} + +func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { + for _, operation := range []string{"put", "patch"} { + t.Run(operation, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}}); err != nil { + t.Fatal(err) + } + want := []types.Volume{{Name: "home", Mount: "/home"}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatalf("still-held retry: %v", err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result: %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovery reference lost writer exclusion: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture ignored recovery reference: %v", err) + } + if err = m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister ignored recovery reference: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + m.mu.Lock() + entry := *m.catalog.entries["home"] + entry.tenants = []string{"other"} + m.catalog.entries["home"] = &entry + m.mu.Unlock() + before := len(eng.volumeSpecs) + switch operation { + case "put": + _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want) + case "patch": + _, err = m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + } + if !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("recovery-only reference authorized tenant attachment: %v", err) + } + if len(eng.volumeSpecs) != before { + t.Fatal("rejected tenant reached device attachment") + } + m.mu.Lock() + entry.tenants = []string{"acme"} + m.catalog.entries["home"] = &entry + m.mu.Unlock() + if _, err = m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatalf("operator reattach: %v", err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { + t.Fatalf("tenant cleanup after operator reattach: %v", err) + } + }) + } +} + +func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { + t.Helper() + cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + return m +} diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go new file mode 100644 index 00000000..4016f920 --- /dev/null +++ b/sandboxd/pool/volume_bench_test.go @@ -0,0 +1,81 @@ +package pool + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func BenchmarkVolumeAdmissionWithCheckpoints(b *testing.B) { + for _, checkpoints := range []bool{false, true} { + b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { + m := volumeBenchManager(b, checkpoints) + requested := []types.Volume{{Name: "bench-data", Mount: "/mnt/data"}} + b.ReportAllocs() + for b.Loop() { + resolved, err := m.resolveVolumes(b.Context(), "", requested) + if err != nil { + b.Fatal(err) + } + if len(resolved) != 1 || resolved[0].applied != requested[0] { + b.Fatalf("resolved volumes: %+v", resolved) + } + m.mu.Lock() + err = m.reserveResolved(resolved, nil) + if err == nil { + m.releaseVolumes(appliedVolumes(resolved)) + } + m.mu.Unlock() + if err != nil { + b.Fatal(err) + } + } + if len(m.catalog.holders) != 0 { + b.Fatalf("unreleased admission holds: %v", m.catalog.holders) + } + }) + } +} + +func BenchmarkVolumeHoldsWithCheckpoints(b *testing.B) { + for _, checkpoints := range []bool{false, true} { + b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { + m := volumeBenchManager(b, checkpoints) + b.ReportAllocs() + for b.Loop() { + m.mu.Lock() + held := 0 + for _, sb := range m.claimed { + if m.hasVolumeHolds(sb) { + held++ + } + } + m.mu.Unlock() + if held != 0 { + b.Fatalf("unrelated claims hold %d volumes", held) + } + } + }) + } +} + +func volumeBenchManager(b *testing.B, checkpoints bool) *Manager { + b.Helper() + m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) + m.claimed = checkpointBenchClaims(checkpoints) + if err := m.store.save(m.claimed); err != nil { + b.Fatal(err) + } + path := filepath.Join(b.TempDir(), "data.raw") + if err := os.WriteFile(path, []byte("benchmark backing"), 0o600); err != nil { + b.Fatal(err) + } + if err := m.RegisterVolume(b.Context(), "bench-data", types.VolumeRegistration{Path: path}); err != nil { + b.Fatal(err) + } + return m +} diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index 27c1b1dc..e0d0b8f4 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -15,7 +15,10 @@ func TestVolumeClaimRejectsDirectCapture(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) + m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} + m.store.set(sb) + m.mu.Unlock() tests := []struct { name string @@ -51,7 +54,10 @@ func TestVolumeClaimRunningWakeUnchanged(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) + m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} + m.store.set(sb) + m.mu.Unlock() sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { @@ -123,7 +129,10 @@ func TestIdleOnceSkipsVolumeClaim(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, IdleHibernateSeconds: 1}) sb := mustClaim(t, m, testKey) + m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} + m.store.set(sb) + m.mu.Unlock() backdate(m, sb, 2*time.Second) sweepIdle(t, m) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go new file mode 100644 index 00000000..f2ccc456 --- /dev/null +++ b/sandboxd/pool/volume_catalog.go @@ -0,0 +1,266 @@ +package pool + +import ( + "cmp" + "context" + "encoding/json/v2" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "sync" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +const runtimeVolumesFile = "runtime-volumes.json" + +type runtimeCatalog struct { + Version int `json:"version"` + Volumes map[string]types.VolumeRegistration `json:"volumes"` +} + +type catalogWrite struct { + entries map[string]*catalogVolume + raw []byte +} + +type volumeCatalog struct { + entries map[string]*catalogVolume + pending *catalogWrite + holders map[string]volumeHolders + unknown int + writeMu sync.Mutex +} + +func (c *volumeCatalog) admissionError() error { + if c.pending != nil || c.unknown > 0 { + return ErrVolumeCatalogUnavailable + } + return nil +} + +// RegisterVolume durably binds a root-managed name to a backing path. +func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + normalized, err := normalizeRegistration(name, spec) + if err != nil { + return err + } + m.mu.Lock() + entries := m.catalog.entries + err = m.catalog.admissionError() + m.mu.Unlock() + if err != nil { + return err + } + if entry := entries[name]; entry != nil { + if entry.runtime && sameRegistration(entry.registration(), normalized) { + return nil + } + return fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) + } + entries = maps.Clone(entries) + entries[name] = &catalogVolume{ + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + tenants: normalized.Tenants, + writable: normalized.Writable, + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) + } + return m.writeCatalog(ctx, name, entries) +} + +// UnregisterVolume retires an unreferenced runtime binding without touching its backing. +func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + if !types.ValidVolumeName(name) { + return fmt.Errorf("%w: invalid volume name", ErrBadVolume) + } + m.mu.Lock() + entries := m.catalog.entries + entry := entries[name] + err := m.catalog.admissionError() + if err == nil && ((entry != nil && !entry.runtime) || (entry == nil && m.volumeReferenced(name))) { + err = fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) + } + m.mu.Unlock() + if err != nil || entry == nil { + return err + } + entries = maps.Clone(entries) + delete(entries, name) + return m.writeCatalog(ctx, name, entries) +} + +func (m *Manager) loadRuntimeVolumes() error { + raw, err := os.ReadFile(filepath.Join(m.dataDir, runtimeVolumesFile)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read runtime volume catalog: %w", err) + } + var record runtimeCatalog + if err = json.Unmarshal(raw, &record, json.RejectUnknownMembers(true)); err != nil { + return fmt.Errorf("parse runtime volume catalog: %w", err) + } + if record.Version != 1 || record.Volumes == nil { + return errors.New("invalid runtime volume catalog version or volumes") + } + entries := maps.Clone(m.catalog.entries) + for name, spec := range record.Volumes { + normalized, normalizeErr := normalizeRegistration(name, spec) + if normalizeErr != nil { + return fmt.Errorf("runtime volume catalog: %w", normalizeErr) + } + if m.catalog.entries[name] != nil { + return fmt.Errorf("runtime volume catalog duplicates static name %q", name) + } + entries[name] = &catalogVolume{ + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + tenants: normalized.Tenants, + writable: normalized.Writable, + } + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } + m.catalog.entries = entries + return nil +} + +func (m *Manager) volumeReferenced(name string) bool { + if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { + return true + } + for _, run := range m.volumeRuns { + if volumeIndex(run.holds, name) >= 0 { + return true + } + } + for _, sb := range m.claimed { + if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { + return true + } + } + return m.store.referencesVolume(name) +} + +func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[string]*catalogVolume) error { + record := runtimeCatalog{Version: 1, Volumes: map[string]types.VolumeRegistration{}} + for key, entry := range entries { + if entry.runtime { + record.Volumes[key] = entry.registration() + } + } + raw, err := json.Marshal(record, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("%w: %w", ErrBadVolume, err) + } + proposal := &catalogWrite{entries: entries, raw: raw} + m.mu.Lock() + err = ctx.Err() + if err == nil { + err = m.catalog.admissionError() + } + if err == nil && m.volumeReferenced(name) { + err = fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + if err == nil { + m.catalog.pending = proposal + } + m.mu.Unlock() + if err != nil { + return err + } + return m.persistCatalog(proposal) +} + +func (m *Manager) commitPendingCatalog(ctx context.Context) error { + m.mu.Lock() + proposal, unknown := m.catalog.pending, m.catalog.unknown > 0 + m.mu.Unlock() + if proposal == nil { + return nil + } + if unknown { + return ErrVolumeCatalogUnavailable + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + return m.persistCatalog(proposal) +} + +func (m *Manager) persistCatalog(proposal *catalogWrite) error { + if err := utils.WriteFileSync(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw, 0o600); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + m.mu.Lock() + m.catalog.entries = proposal.entries + m.catalog.pending = nil + m.mu.Unlock() + return nil +} + +func (m *Manager) retryVolumeCatalog(ctx context.Context) { + if !m.catalog.writeMu.TryLock() { + return + } + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + log.WithFunc("pool.retryVolumeCatalog").Error(ctx, err, "persist runtime volume catalog") + } +} + +func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { + volumes := make([]config.VolumeSpec, 0, len(entries)) + for name, entry := range entries { + var tenants []string + if entry.runtime { + tenants = entry.tenants + } + volumes = append(volumes, config.VolumeSpec{ + Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, + Writable: entry.writable, Tenants: tenants, + }) + } + return volumes +} + +func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { + spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ + Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, + }}); err != nil { + return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) + } + spec.Path = filepath.Clean(spec.Path) + spec.Tenants = slices.Clone(spec.Tenants) + slices.Sort(spec.Tenants) + spec.Tenants = slices.Compact(spec.Tenants) + return spec, nil +} + +func sameRegistration(a, b types.VolumeRegistration) bool { + return a.Path == b.Path && a.DirectIO == b.DirectIO && a.Writable == b.Writable && slices.Equal(a.Tenants, b.Tenants) +} diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go new file mode 100644 index 00000000..002f7c5e --- /dev/null +++ b/sandboxd/pool/volume_catalog_test.go @@ -0,0 +1,617 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "sync" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "backing bytes stay intact") + spec := types.VolumeRegistration{Path: filepath.Dir(path) + "/./runtime", Writable: true, Tenants: []string{"beta", "acme", "acme"}} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + spec.Path = path + spec.DirectIO = types.DirectIOOff + spec.Tenants = []string{"acme", "beta"} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("normalized retry: %v", err) + } + st, statErr := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) + if statErr != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, statErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("recovered names: %v", got) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "forbidden", Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("ACL changed at restart: %v", err) + } + resolved, err := m2.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}) + if err != nil || len(resolved) != 1 || resolved[0].disk.DirectIO != "off" || !resolved[0].disk.RW { + t.Fatalf("policy roundtrip: %+v %v", resolved, err) + } + changed := spec + changed.DirectIO = "on" + if err := m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("changed semantics: %v", err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatalf("absent retry: %v", err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != "backing bytes stay intact" { + t.Fatalf("backing changed: %q %v", got, err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("unregister not durable: %v", got) + } +} + +func TestRuntimeCatalogLoadsOperatorManagedRecord(t *testing.T) { + for _, setup := range []struct { + name string + prepare func(*testing.T, string) + }{ + {"mode", func(t *testing.T, record string) { + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + }}, + {"symlink", func(t *testing.T, record string) { + if err := os.Rename(record, record+".target"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(record+".target", record); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(setup.name, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + setup.prepare(t, filepath.Join(m.dataDir, runtimeVolumesFile)) + restarted := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("operator-managed record not loaded: %v", got) + } + if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("replacement not loaded: %v", got) + } + }) + } +} + +func TestRuntimeCatalogUsesStaticPathValidation(t *testing.T) { + regular := writeVolumeImage(t, "backing", "unchanged") + link := filepath.Join(t.TempDir(), "backing-link") + if err := os.Symlink(regular, link); err != nil { + t.Fatal(err) + } + for _, path := range []string{regular, link, os.DevNull, t.TempDir(), filepath.Join(t.TempDir(), "missing")} { + t.Run(filepath.Base(path), func(t *testing.T) { + spec := config.VolumeSpec{Name: "runtime", Path: path, DirectIO: types.DirectIOOff, Writable: true} + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{spec}); err != nil { + t.Fatalf("static validation: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), nil) + registration := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("runtime validation differs from static: %v", err) + } + m2 := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) + if err := m2.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("idempotent retry after restart: %v", err) + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + if _, resolveErr := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(resolveErr, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", resolveErr) + } + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + }) + } + if got, err := os.ReadFile(regular); err != nil || string(got) != "unchanged" { + t.Fatalf("registration touched backing: %q %v", got, err) + } +} + +func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { + eng := newFakeEngine() + static := liveCatalog(t) + m := newVolumeManager(t, eng, static) + other := writeVolumeImage(t, "other-runtime", "bytes") + for _, tc := range []struct{ name, path string }{{"data", other}, {"runtime", static[0].Path}, {"runtime", filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path)}} { + if err := m.RegisterVolume(t.Context(), tc.name, types.VolumeRegistration{Path: tc.path}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("collision %+v: %v", tc, err) + } + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("static removal: %v", err) + } + sb := mustClaim(t, m, testKey) + sb.Volumes = []types.Volume{{Name: "unexplained", Mount: "/data"}} + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "unexplained", types.VolumeRegistration{Path: other}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebound unexplained recovery name: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("absent referenced removal: %v", err) + } + for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}} { + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid spec %+v: %v", spec, err) + } + } +} + +func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + m.store.path = filepath.Join(t.TempDir(), "claims.json") + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} + restore := blockCatalogRename(t, m) + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("write failure: %v", err) + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("pending admission: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + restore() + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry pending register: %v", err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("durable registration: %v", got) + } + restore = blockCatalogRename(t, m) + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("failed unregister: %v", err) + } + restore() + if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { + t.Fatalf("pending unregister did not converge before reuse: %v", err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"replacement"}) { + t.Fatalf("durable reuse: %v", got) + } +} + +func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} + restore := blockCatalogRename(t, m) + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("write failure: %v", err) + } + restore() + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("possible replacement not recovered: %v", got) + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain admission: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry pending register: %v", err) + } + restore = blockCatalogRename(t, m) + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain unregister: %v", err) + } + restore() + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("possible unregister replacement not recovered: %v", got) + } + if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { + t.Fatalf("pending unregister did not converge before reuse: %v", err) + } +} + +func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "original") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + spec.Path = writeVolumeImage(t, "replacement", "replacement") + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("stale binding admitted: %v", err) + } + if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { + t.Fatalf("stale binding leaked reservation: %+v", got) + } +} + +func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if os.Geteuid() == 0 { + if err := os.Chown(path, 65534, 65534); err != nil { + t.Fatal(err) + } + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("nonroot backing owner: %v", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + infos := m2.Volumes("", nil) + if len(infos) != 1 || infos[0].Available { + t.Fatalf("missing backing lost mapping: %+v", infos) + } + if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", err) + } + if err := m2.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("idempotent missing mapping: %v", err) + } +} + +func TestRuntimeRegistrationNormalization(t *testing.T) { + input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} + normalized, err := normalizeRegistration("data", input) + if err != nil || normalized.Path != "/images/data" || normalized.DirectIO != types.DirectIOOff || !slices.Equal(normalized.Tenants, []string{"acme", "beta"}) { + t.Fatalf("normalization: %+v %v", normalized, err) + } + normalized.Tenants[0] = "caller-copy" + if input.Tenants[1] != "acme" { + t.Fatal("normalization mutated caller input") + } + for _, tc := range []struct { + name string + spec types.VolumeRegistration + }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{""}}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{"has space"}}}} { + if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid registration %+v: %v", tc, err) + } + } +} + +func TestRuntimeCatalogRejectsInvalidMetadata(t *testing.T) { + static := liveCatalog(t) + for _, tc := range []struct{ name, raw string }{ + {"malformed", "{"}, + {"version", `{"version":2,"volumes":{}}`}, + {"null", `{"version":1,"volumes":null}`}, + {"name", `{"version":1,"volumes":{"bad/name":{"path":"/data"}}}`}, + {"relative", `{"version":1,"volumes":{"runtime":{"path":"relative"}}}`}, + {"directio", `{"version":1,"volumes":{"runtime":{"path":"/data","directio":"bad"}}}`}, + {"duplicate-name", `{"version":1,"volumes":{"runtime":{"path":"/one"},"runtime":{"path":"/two"}}}`}, + {"duplicate-path", `{"version":1,"volumes":{"runtime":{"path":"/one"},"second":{"path":"/one"}}}`}, + {"static-name", `{"version":1,"volumes":{"data":{"path":"/other"}}}`}, + {"static-path", fmt.Sprintf(`{"version":1,"volumes":{"runtime":{"path":%q}}}`, static[0].Path)}, + {"unknown-field", `{"version":1,"volumes":{},"unexpected":true}`}, + } { + t.Run(tc.name, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, static) + if err := os.WriteFile(filepath.Join(m.dataDir, runtimeVolumesFile), []byte(tc.raw), 0o600); err != nil { + t.Fatal(err) + } + cfg := &config.Config{DataDir: m.dataDir, Volumes: static} + if _, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)); err == nil { + t.Fatalf("accepted %s metadata", tc.name) + } + }) + } +} + +func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "unchanged") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "data", spec); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected failed result commit") + } + eng.liveVolumeHook = nil + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "different")}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebind unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("alias unretired path: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("same-file writer: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture: %v", err) + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("hibernate before retirement: %v", err) + } + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); err != nil { + t.Fatal(err) + } +} + +func TestCatalogSnapshotSurvivesRegistration(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + before := m.catalogSnapshot() + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/images/runtime"}); err != nil { + t.Fatal(err) + } + if before["runtime"] != nil { + t.Fatal("registration mutated captured catalog") + } + captured := m.catalogSnapshot() + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if captured["runtime"] == nil { + t.Fatal("unregister mutated captured catalog") + } +} + +func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) + if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("cancellation before acceptance: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled proposal left catalog: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("canceled registration fenced static claim: %v", err) + } +} + +func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + restore := blockCatalogRename(t, m) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("initial write failure: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("failed retry cleared pending write: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("pending replacement lost fence: %v", err) + } + restore() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := m.UnregisterVolume(ctx, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("canceled retry cleared pending write: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if !slices.Contains(m.VolumeNames(), "runtime") { + t.Fatal("retry lost accepted registration") + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatalf("repair did not restore static admission: %v", err) + } +} + +func TestResolvedCatalogBindingChangesOnlyForReusedName(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { + t.Fatal(err) + } + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatalf("unrelated registration invalidated resolution: %v", err) + } + m.unreserveVolumes(holds) + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("same-file reuse admitted stale resolution: %v", err) + } +} + +func TestRuntimeCatalogIgnoresAbandonedTemporaryFiles(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "bytes") + raw := fmt.Sprintf(`{"version":1,"volumes":{"abandoned":{"path":%q}}}`, path) + tmp := filepath.Join(m.dataDir, runtimeVolumesFile+".interrupted.tmp") + if err := os.WriteFile(tmp, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("temporary record promoted without canonical record: %v", got) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("temporary record overrode canonical record: %v", got) + } + if got, err := os.ReadFile(tmp); err != nil || string(got) != raw { + t.Fatalf("abandoned evidence changed: %q %v", got, err) + } +} + +func TestRuntimeCatalogAcceptsDaemonDataDirectorySymlink(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "state") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(parent, "linked-state") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + if err := os.Chmod(parent, 0o777); err != nil { + t.Fatal(err) + } + eng := newFakeEngine() + m := newVolumeManagerAt(t, eng, link, nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + restarted := newVolumeManagerAt(t, eng, link, nil) + if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("reloaded catalog: %v", got) + } + if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + entries := maps.Clone(m.catalogSnapshot()) + delete(entries, "runtime") + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatal(err) + } + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.writeCatalog(t.Context(), "runtime", entries); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("accepted removal prepared before reservation: %v", err) + } + m.unreserveVolumes(holds) + if err := m.writeCatalog(t.Context(), "runtime", entries); err != nil { + t.Fatalf("reservation retirement did not unblock removal: %v", err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("removal was not persisted: %v", got) + } +} + +func blockCatalogRename(t *testing.T, m *Manager) func() { + t.Helper() + record := filepath.Join(m.dataDir, runtimeVolumesFile) + raw, err := os.ReadFile(record) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + existed := err == nil + if existed { + if err := os.Remove(record); err != nil { + t.Fatal(err) + } + } + if err := os.Mkdir(record, 0o700); err != nil { + t.Fatal(err) + } + restore := sync.OnceFunc(func() { + if err := os.Remove(record); err != nil { + t.Fatal(err) + } + if existed { + if err := os.WriteFile(record, raw, 0o600); err != nil { + t.Fatal(err) + } + } + }) + t.Cleanup(restore) + return restore +} + +func installPendingCatalogRecord(t *testing.T, m *Manager) { + t.Helper() + m.mu.Lock() + proposal := m.catalog.pending + m.mu.Unlock() + if proposal == nil { + t.Fatal("possible replacement fixture requires an accepted pending write") + } + // Model the canonical bytes after a successful rename and uncertain directory sync. + if err := utils.WriteFileSync(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go new file mode 100644 index 00000000..c2ff00ae --- /dev/null +++ b/sandboxd/pool/volume_cleanup_test.go @@ -0,0 +1,543 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { + for _, state := range []string{vmStateRunning, vmStateCreating, "stopped"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "incomplete-inventory", "original") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-incomplete-inventory" + base.vms[name] = "/vsock/incomplete" + base.creating[name] = state == vmStateCreating + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + cli := filepath.Join(t.TempDir(), "inspect") + writeInspect := func(state, inventory string) { + t.Helper() + raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"%s\"%s}'\n", name, state, inventory) + if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { + t.Fatal(err) + } + } + writeInspect(state, "") + m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + rec := base.record(name) + rec.State = state + if m.removeStaleVM(t.Context(), name, rec) { + t.Fatal("stale VM unexpectedly disappeared") + } + for range 2 { + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unregister must refuse incomplete inventory: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("conflicting writer must refuse incomplete inventory: %v", err) + } + m.retryRemovals(t.Context()).Wait() + } + base.creating[name] = false + writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) + if state == vmStateCreating { + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy owner must retain the unknown fence: %v", err) + } + } + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("complete inventory lost the surviving writer: %v", err) + } + other := writeVolumeImage(t, "complete-inventory-other", "other") + if err := m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: other}); err != nil { + t.Fatalf("complete inventory retained the global fence: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained volume protection: %v", err) + } + }) + } +} + +func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true}); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + changed := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); changed <- err }() + <-eng.entered + resumed := false + defer func() { + if !resumed { + close(eng.resume) + <-changed + } + }() + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + select { + case err := <-released: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("release waited for paused worker") + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("removed VM freed active worker count: %+v", got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer overtook paused canceled worker: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Errorf("unregister overtook paused worker: %v", err) + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("maintenance freed active worker count: %+v", got) + } + close(eng.resume) + resumed = true + if err := <-changed; err == nil { + t.Fatal("removed claim accepted late attachment") + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("completed worker leaked count: %+v", got) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if err != nil { + t.Fatal(err) + } + base.removeErrFor = sb.VMName + if err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + m.mu.Lock() + queued := len(m.pendingRemovals) + m.mu.Unlock() + if queued != 0 { + t.Fatal("cleanup was not dispatched") + } + err = m.UnregisterVolume(t.Context(), "data") + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + if !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("dispatched cleanup lost guard: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { + for _, state := range []string{"running", "stopped", "creating"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-stale-unknown" + base.vms[name] = "/vsock/stale" + base.stopped[name] = state == "stopped" + base.creating[name] = state == "creating" + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + var inspectFailed atomic.Bool + inspectFailed.Store(true) + base.inspectHook = func() error { + if inspectFailed.Load() { + return errors.New("inspection unavailable") + } + return nil + } + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("unknown stale VM unexpectedly gone") + } + for range 2 { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment admitted reader: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment allowed unregister: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + if err := m.RegisterVolume(t.Context(), "new", types.VolumeRegistration{Path: writeVolumeImage(t, "new", "bytes")}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("dispatched unknown fence lost: %v", err) + } + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + } + inspectFailed.Store(false) + base.stopped[name], base.creating[name] = false, false + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { + t.Fatalf("inspection did not replace fence with known holds: %+v", got) + } + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated", "bytes")}); err != nil { + t.Fatalf("resolved inspection left global fence: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("known attachment lost exclusion: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained fence: %v", err) + } + }) + } +} + +func TestRuntimeCatalogLoadedBeforeClaimReconciliation(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true, Tenants: []string{"acme"}}); err != nil { + t.Fatal(err) + } + sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if claimErr != nil { + t.Fatal(claimErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "data" || !got[0].Available || !got[0].Writable { + t.Fatalf("catalog absent before reconcile: %+v", got) + } + if local, err := m2.VolumePlacement(testKey, "acme", []string{"data"}); err != nil || !local { + t.Fatalf("placement before reconcile: %v %v", local, err) + } + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + view, err := m2.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(view.Volumes, sb.Volumes) { + t.Fatalf("claim recovery: %+v %v", view, err) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovered holder missing: %v", err) + } +} + +func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-stale-before-inspect" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + entered, resume := make(chan struct{}), make(chan struct{}) + var gated atomic.Bool + base.inspectHook = func() error { + if gated.CompareAndSwap(false, true) { + close(entered) + <-resume + } + return errors.New("inspection unavailable") + } + done := make(chan bool, 1) + go func() { done <- m.removeStaleVM(t.Context(), name, base.record(name)) }() + <-entered + if !m.mu.TryLock() { + t.Fatal("stale inspection holds manager lock") + } + m.mu.Unlock() + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + close(resume) + if gone := <-done; gone { + t.Fatal("failed inspect/removal unexpectedly confirmed gone") + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("admitted volume while inspection was blocked: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("successful removal retained unknown fence: %v", err) + } +} + +func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-canceled-dispatch" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + base.inspectHook = func() error { return errors.New("inspection unavailable") } + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + ctx, cancel := context.WithCancel(t.Context()) + wg := m.retryRemovals(ctx) + cancel() + synctest.Wait() + base.removeErrFor = "" + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("canceled dispatch stranded unknown fence: %v", err) + } + }) +} + +func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { + base := newFakeEngine() + name := "sbx-warm-unfinalized" + base.vms[name] = "/vsock/warm" + base.diskAttachErrFor = "broken" + base.removeErrFor = name + rendezvousAttaches(base, 2) + catalog := []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "warm-data", "data"), Writable: true}, + {Name: "broken", Path: writeVolumeImage(t, "warm-broken", "broken")}, + } + dir := t.TempDir() + m := newVolumePoolManager(t, base, dir, catalog) + base.inspectDisksUnknown, base.trackDisks = true, true + warm := &types.Sandbox{VMName: name, Key: testKey, VsockSocket: "/vsock/warm"} + m.pools[testKey].warm = append(m.pools[testKey].warm, warm) + _, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{ + {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}, + {Name: "broken", AttachOnly: true}, + }}) + if err == nil { + t.Fatal("expected attach failure before finalization") + } + if warm.ID != "" { + t.Fatal("claim unexpectedly finalized") + } + base.mu.Lock() + attached := slices.Clone(base.disks[name]) + base.attachRendezvous = nil + base.mu.Unlock() + if !slices.ContainsFunc(attached, func(d types.VMDisk) bool { return d.Name == "data" && !d.ReadOnly }) { + t.Fatalf("warm VM has no attached writer: %+v", attached) + } + claims, err := newClaimStore(dir, false).load() + if err != nil || len(claims) != 0 { + t.Fatalf("pre-finalize journal: %+v, %v", claims, err) + } + restarted := newVolumeManagerAt(t, base, dir, catalog) + restarted.eng = base + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("ambiguous warm survivor admitted competing writer: %v", err) + } + base.removeErrFor = "" + restarted.retryRemovals(t.Context()).Wait() + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("confirmed removal failed to clear fence: %v", err) + } +} + +func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) { + base := newFakeEngine() + m := newTestManager(t, base, archivePool(3600)) + sb := mustClaim(t, m, testKey) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if err := m.archive(t.Context(), sb); err != nil { + t.Fatal(err) + } + if sb.VMName != "" { + t.Fatalf("archived VM name = %q", sb.VMName) + } + ctx, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + if err := m.checkVolumeRun(ctx, sb, run); !errors.Is(err, ErrArchived) { + t.Fatalf("archived mutation validation: %v", err) + } + td := m.quiesceVolumes(t.Context(), sb) + if td.done == nil || len(td.holds) != 0 || len(td.clears) != 0 || td.unknown { + t.Fatalf("archived teardown has cleanup obligations: %+v", td) + } + if m.finishVolumeTeardown(t.Context(), td) { + t.Fatal("open volume run unexpectedly completed") + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.finishVolumeRun(sb, run) + if len(m.pendingRemovals) != 0 { + t.Fatalf("empty teardown enqueued removal: %+v", m.pendingRemovals) + } + if slices.Contains(base.removedNames(), "") { + t.Fatal("release removed an empty VM name") + } + if m.store.referencesVolume("data") { + t.Fatal("release retained archived journal references") + } +} + +func TestStaleInventoryDistinguishesOmittedAndExplicitEmptyDisks(t *testing.T) { + for _, tc := range []struct { + name, inventory string + wantErr error + }{ + {"omitted devices", "", ErrVolumeCatalogUnavailable}, + {"omitted disks", `,"attached_devices":{}`, ErrVolumeCatalogUnavailable}, + {"null disks", `,"attached_devices":{"disks":null}`, ErrVolumeCatalogUnavailable}, + {"explicit empty disks", `,"attached_devices":{"disks":[]}`, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-inventory-shape" + base.vms[name], base.removeErrFor = "/vsock/stale", name + cli := filepath.Join(t.TempDir(), "inspect") + raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"running\"%s}'\n", name, tc.inventory) + if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { + t.Fatal(err) + } + m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("stale VM unexpectedly disappeared") + } + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("admission=%v, want=%v", err, tc.wantErr) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { + t.Fatalf("confirmed removal retained registration fence: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("confirmed removal retained admission fence: %v", err) + } + }) + } +} + +func TestStaleInventoryNarrowsFenceBeforeBlockedRemoval(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-known-before-remove" + eng.vms[name], eng.removeErrFor = "/vsock/stale", name + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + eng.removeStall = make(chan struct{}) + done := make(chan bool, 1) + go func() { done <- m.removeStaleVM(t.Context(), name, eng.record(name)) }() + synctest.Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("known stale writer: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { + t.Errorf("blocked removal retained global fence: %v", err) + } + close(eng.removeStall) + if <-done { + t.Fatal("failed removal reported gone") + } + }) +} + +func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { + for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound} { + t.Run(string(outcome), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-busy-create" + eng.vms[name], eng.creating[name] = "/vsock/creating", true + eng.staleOutcome = engine.StaleCreateBusy + eng.inspectHook = func() error { + t.Error("busy owner must finish before inventory can be trusted") + return errors.New("busy") + } + rec := eng.record(name) + rec.State = vmStateCreating + if m.removeStaleVM(t.Context(), name, rec) { + t.Fatal("busy create reported gone") + } + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy create admitted a volume: %v", err) + } + eng.staleOutcome = outcome + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("confirmed cleanup retained fence: %v", err) + } + }) + } +} + +type incompleteInventoryEngine struct { + *fakeEngine + cli *engine.Engine +} + +func (e *incompleteInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + return e.cli.Inspect(ctx, name) +} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 7c002ecc..ef8fb774 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -22,7 +22,7 @@ func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - delete(m.volumes, "data") + delete(m.catalog.entries, "data") before := len(eng.volumeSpecs) if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { t.Fatal(err) @@ -57,9 +57,9 @@ func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { t.Fatal("expected detaching state") } eng.detachVolumeErr = nil - entry := m.volumes["data"] + entry := m.catalog.entries["data"] entry.tenants = []string{"other-tenant"} - m.volumes["data"] = entry + m.catalog.entries["data"] = entry if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { t.Fatalf("held repair checked new authorization: %v", err) } @@ -313,7 +313,7 @@ func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { } func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { - for _, state := range []string{"applied", "desired", "admitted"} { + for _, state := range []string{"applied", "desired", "admitted", "recovery"} { t.Run(state, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}) @@ -321,9 +321,27 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { held := []types.Volume{{Name: "data", Mount: "/volumes/data"}} switch state { case "applied": + m.mu.Lock() sb.Volumes = held + m.store.set(sb) + m.mu.Unlock() case "desired": + m.mu.Lock() + sb.VolumesWant = held + m.store.set(sb) + m.mu.Unlock() + case "recovery": + m.mu.Lock() sb.VolumesWant = held + js := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(js); err != nil { + t.Fatal(err) + } + m.mu.Lock() + sb.VolumesWant = nil + m.store.set(sb) + m.mu.Unlock() case "admitted": _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { @@ -350,6 +368,14 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { if len(eng.trims) != 0 { t.Fatalf("capture trimmed held volume: %v", eng.trims) } + if state == "recovery" { + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retired", ""); err != nil { + t.Fatalf("checkpoint after recovery acknowledgment: %v", err) + } + } }) } } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 33dc50e2..596d047f 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -86,20 +86,21 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) + recovery := m.store.recoveryVolumesFor(sb.ID) m.mu.Unlock() target, err := change(current) if err != nil { return view, err } - target, err = normalizeLiveVolumes(target, held) + target, err = normalizeLiveVolumes(target, slices.Concat(held, recovery)) if err != nil { return view, err } - if err = validateLiveTarget(held, target); err != nil { + if err = validateLiveTarget(slices.Concat(held, recovery), target); err != nil { return view, err } additions := volumeDifference(target, held) - if len(additions) != 0 { + if len(additions) > 0 { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err } @@ -146,7 +147,10 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - repair, resolveErr := m.resolveCatalogVolume(ctx, v) + m.mu.Lock() + entry := m.catalog.entries[v.Name] + m.mu.Unlock() + repair, resolveErr := m.resolveCatalogEntry(ctx, v, entry) if resolveErr != nil { return view, resolveErr } @@ -257,23 +261,28 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, err } if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveVolumes(additions); err != nil { + if err := m.reserveResolved(resolved, sb); err != nil { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, err } run.holds = append(run.holds, additions...) m.mu.Unlock() + sb.Transition.Unlock() cleanErr := m.confirmVolumesClean(resolved) + sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -343,19 +352,26 @@ func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { return m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run } -func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { +func (m *Manager) hasLiveVolumeHolds(sb *types.Sandbox) bool { run := m.volumeRuns[sb] return hasVolumes(sb) || run != nil && len(run.holds) > 0 } +func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { + return m.hasLiveVolumeHolds(sb) || m.store.hasRecoveryVolumes(sb.ID) +} + func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { + held := heldVolumes(sb) + recovering := volumeDifference(m.store.recoveryVolumesFor(sb.ID), held) view := types.SandboxVolumesResponse{ Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, - Converging: m.volumeRuns[sb] != nil, + Converging: m.volumeRuns[sb] != nil || len(recovering) > 0, } - for _, v := range heldVolumes(sb) { + held = append(held, recovering...) + for _, v := range held { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { state = types.VolumeDetaching diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 4f94b749..d9de1c70 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -280,8 +280,11 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { } close(resume) attachErr := <-attached - if action == "reap" { - waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) + if action == "release" || action == "reap" { + waitFor(t, func() bool { + m.retryRemovals(t.Context()).Wait() + return volumeHoldersOf(m, "data") == (volumeHolders{}) + }) } if action == "release" || action == "reap" { if attachErr == nil { @@ -718,6 +721,7 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) + m.retryRemovals(t.Context()).Wait() waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go new file mode 100644 index 00000000..ae6b6ea5 --- /dev/null +++ b/sandboxd/pool/volume_retirement_test.go @@ -0,0 +1,416 @@ +package pool + +import ( + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result = %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || len(claims[sb.ID].Volumes) != 1 { + t.Fatalf("failed write did not leave old recovery reference: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer admitted while old claim still references detached volume: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Errorf("capture passed before recovery reference retired: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost old claim's reader exclusion: %v", err) + } + if _, err = m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatalf("retry after restart: %v", err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer still blocked after successful retirement: %v", err) + } +} + +func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { + for _, retry := range []string{"empty", "same-shape", "unrelated-snapshot"} { + t.Run(retry, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("detach result unexpectedly persisted") + } + eng.liveVolumeHook = nil + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("physical count retained after detach: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mount: "/different"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("recovery-only shape changed: %v", err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { + t.Fatalf("unretired view: %+v, %v", view, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + switch retry { + case "empty": + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "same-shape": + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got.readers != 1 || got.writers != 0 { + t.Fatalf("own retry counts: %+v", got) + } + } + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "unrelated-snapshot": + _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + } + if err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("retired count: %+v", got) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer after acknowledgment: %v", err) + } + }) + } +} + +func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil { + t.Fatal(err) + } + restoreWrites := failClaimWrites(t, m.store) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected failed intent write") + } + claims[sb.ID].VolumesWant = []types.Volume{{Name: "data", Mount: types.DefaultVolumeMount("data"), Mode: types.VolumeModeRO}} + raw, err := json.Marshal(claims) + if err != nil { + t.Fatal(err) + } + // Model an intent whose rename succeeded before an uncertain directory sync. + if err = utils.WriteFileSync(m.store.path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected empty snapshot error") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 { + t.Fatalf("memory not empty: %+v", sb) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("unexpected physical count: %+v", got) + } + claims, err = newClaimStore(m.dataDir, false).load() + if err != nil || len(claims[sb.ID].VolumesWant) != 1 { + t.Fatalf("actual recovery file: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer admitted after uncertain introduction: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain-introduction", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture admitted: %v", err) + } + restoreWrites() + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart admitted writer: %v", err) + } + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if len(m.store.recoveryVolumesFor(sb.ID)) != 0 { + t.Fatal("successful whole snapshot did not retire attempted row") + } +} + +func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s = newClaimStore(filepath.Dir(s.path), false) + if _, err := s.load(); err != nil { + t.Fatal(err) + } + s.reset(map[string]*types.Sandbox{}) + refs := s.recoveryVolumesFor(sb.ID) + refs[0].Name = "caller-copy" + if got := s.recoveryVolumesFor(sb.ID); len(got) != 1 || got[0].Name != "data" { + t.Fatalf("reset or caller erased actual file: %+v", got) + } + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if s.hasRecoveryVolumes(sb.ID) { + t.Fatal("empty snapshot did not retire file references") + } +} + +func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + restoreWrites := failClaimWrites(t, s) + for i := range 200 { + sb.Token = fmt.Sprint(i) + sb.Volumes[0].Mount = []string{"/data", "/other"}[i%2] + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("expected failed snapshot") + } + } + if refs := s.recoveryVolumesFor(sb.ID); len(refs) != 2 { + t.Fatalf("renewals retained duplicate history: %+v", refs) + } + refs := s.recoveryVolumesFor(sb.ID) + refs[0].Name = "caller-copy" + if got := s.recoveryVolumesFor(sb.ID); len(got) != 2 || got[0].Name != "data" { + t.Fatalf("recovery references not copied: %+v", got) + } + restoreWrites() + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.recoveryVolumesFor(sb.ID)) != 1 { + t.Fatal("successful snapshot did not replace possibility history") + } +} + +func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err == nil { + t.Fatal("expected failed detach result commit") + } + eng.liveVolumeHook = nil + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + m.mu.Lock() + entry := *m.catalog.entries["data"] + entry.tenants = []string{"different-tenant"} + m.catalog.entries["data"] = &entry + m.mu.Unlock() + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("recovery-only retry bypassed tenant removal: %v", err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("rejected recovery retry reserved a reader: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("recovery retirement after tenant removal: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("repair exemption admitted unrelated addition: %v", err) + } +} + +func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + applied := []types.Volume{{Name: "data", Mount: "/data"}} + wanted := []types.Volume{{Name: "wanted", Mount: "/wanted"}} + sb := &types.Sandbox{ID: "owner", Volumes: slices.Clone(applied), VolumesWant: slices.Clone(wanted)} + old := s.set(sb) + release := captureClaimWrite(t, s, old, sb.ID) + newer := s.del(sb.ID) + sb.Volumes[0].Name = "mutated-applied" + sb.VolumesWant[0].Name = "mutated-wanted" + raw := release() + if !s.pending(newer) { + t.Fatal("captured snapshot acknowledged newer deletion") + } + if refs := s.recoveryVolumesFor(sb.ID); !slices.Equal(refs, append(slices.Clone(applied), wanted...)) { + t.Fatalf("captured row not acknowledged: %+v", refs) + } + var persisted map[string]*types.Sandbox + err := json.Unmarshal(raw, &persisted) + if err != nil || persisted[sb.ID] == nil || !slices.Equal(persisted[sb.ID].Volumes, applied) || !slices.Equal(persisted[sb.ID].VolumesWant, wanted) { + t.Fatalf("captured write: %+v, %v", persisted, err) + } + if err = s.commit(newer); err != nil { + t.Fatal(err) + } + if s.hasRecoveryVolumes(sb.ID) { + t.Fatal("empty replacement retained old reference") + } + restoreWrites := failClaimWrites(t, s) + defer restoreWrites() + if err = s.commit(old); err != nil { + t.Fatal(err) + } + persisted, err = newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || len(persisted) != 0 { + t.Fatalf("old token resurrected snapshot: %+v, %v", persisted, err) + } +} + +func TestClaimSharedRecoveryVolumesRetirePerOwner(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + volume := types.Volume{Name: "shared", Mount: "/data"} + first := &types.Sandbox{ID: "first", Volumes: []types.Volume{volume}} + second := &types.Sandbox{ID: "second", Volumes: []types.Volume{volume}} + writer := volume + writer.Mode = types.VolumeModeRW + if err := s.commit(s.set(first, second)); err != nil { + t.Fatal(err) + } + if s.volumeConflicts(volume, "") || !s.volumeConflicts(writer, "") { + t.Fatal("shared readers have incorrect admission") + } + if err := s.commit(s.del(first.ID)); err != nil { + t.Fatal(err) + } + if !s.referencesVolume(volume.Name) || s.hasRecoveryVolumes(first.ID) || !s.hasRecoveryVolumes(second.ID) || !s.volumeConflicts(writer, first.ID) { + t.Fatal("retiring one reader released the other") + } + if err := s.commit(s.del(second.ID)); err != nil { + t.Fatal(err) + } + if s.referencesVolume(volume.Name) || s.volumeConflicts(writer, "") { + t.Fatal("last reader left recovery reservation") + } +} + +func TestClaimRecoveryVolumeOwnerExemptionRequiresExactAttachment(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + writer := types.Volume{Name: "data", Mount: "/data", Mode: types.VolumeModeRW} + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{writer}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + restore := failClaimWrites(t, s) + sb.VolumesWant = []types.Volume{{Name: "other", Mount: "/other"}} + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("expected write failure") + } + reader := writer + reader.Mode = types.VolumeModeRO + moved := writer + moved.Mount = "/elsewhere" + for _, tc := range []struct { + volume types.Volume + owner string + conflict bool + }{ + {writer, sb.ID, false}, {writer, "different", true}, {reader, sb.ID, true}, {moved, sb.ID, true}, + } { + if got := s.volumeConflicts(tc.volume, tc.owner); got != tc.conflict { + t.Errorf("volume %+v owner %q conflict=%t, want %t", tc.volume, tc.owner, got, tc.conflict) + } + } + s.set(&types.Sandbox{ID: sb.ID, Volumes: []types.Volume{writer}}) + restore() + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if s.referencesVolume("other") || s.volumeConflicts(writer, sb.ID) { + t.Fatal("successful return retained attempted attachment") + } +} + +func TestClaimCaptureAcknowledgesBothReferenceDomains(t *testing.T) { + for _, domain := range []string{"volumes", "checkpoints"} { + t.Run(domain, func(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ReplicaCk: "checkpoint", Volumes: []types.Volume{{Name: "data"}}} + release := captureClaimWrite(t, s, s.set(sb), sb.ID) + if domain == "volumes" { + sb.Volumes = nil + } else { + sb.ReplicaCk = "" + } + newer := s.set(sb) + release() + if !s.pending(newer) || !s.referencesCheckpoint("checkpoint") || !s.hasRecoveryVolumes(sb.ID) { + t.Fatal("older capture retired newer state or its recovery references") + } + if err := s.commit(newer); err != nil { + t.Fatal(err) + } + if got := s.referencesCheckpoint("checkpoint"); got != (domain == "volumes") { + t.Fatalf("checkpoint reference after newer commit = %t", got) + } + if got := s.hasRecoveryVolumes(sb.ID); got != (domain == "checkpoints") { + t.Fatalf("volume recovery after newer commit = %t", got) + } + }) + } +} diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index a81ff1bb..033bc8c3 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -229,7 +229,7 @@ func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { if err := m.reserveVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } - if holders := m.volumeAdmission["scratch"]; holders != (volumeHolders{writers: 1}) { + if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { t.Errorf("registry after the refusal=%+v, want the writer alone", holders) } } @@ -580,5 +580,5 @@ func newVolumeManagerAt(t *testing.T, eng *fakeEngine, dataDir string, volumes [ func volumeHoldersOf(m *Manager, name string) volumeHolders { m.mu.Lock() defer m.mu.Unlock() - return m.volumeAdmission[name] + return m.catalog.holders[name] } diff --git a/sandboxd/pool/volume_static_acl_test.go b/sandboxd/pool/volume_static_acl_test.go new file mode 100644 index 00000000..35a5af36 --- /dev/null +++ b/sandboxd/pool/volume_static_acl_test.go @@ -0,0 +1,42 @@ +package pool + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestInvalidStaticTenantListNeverBecomesPublic(t *testing.T) { + path := writeVolumeImage(t, "data", "private contents") + file := filepath.Join(t.TempDir(), "config.json") + raw := fmt.Sprintf(`{"pools":[],"volumes":[{"name":"data","path":%q,"tenants":["has space"]}]}`, path) + if err := os.WriteFile(file, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := config.Load(file) + if err != nil { + t.Fatalf("legacy config stopped booting: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), cfg.Volumes) + if got := m.Volumes("acme", nil); len(got) != 0 { + t.Fatalf("invalid restricted ACL became public: %+v", got) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: path + ".other"}); err != nil { + t.Fatalf("invalid legacy ACL blocked unrelated registration: %v", err) + } +} + +func TestRuntimeRegistrationRejectsInvalidTenantNames(t *testing.T) { + for _, tenant := range []string{"", "has space", "_leading"} { + t.Run(tenant, func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: "/images/data", Tenants: []string{tenant}}); err == nil { + t.Fatalf("runtime registration accepted invalid tenant %q", tenant) + } + }) + } +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 78b44fd6..70279f63 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -47,6 +47,8 @@ var poolErrHTTP = []struct { }{ {pool.ErrBadKey, http.StatusBadRequest, ""}, {pool.ErrBadVolume, http.StatusBadRequest, ""}, + {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, + {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrBadName, http.StatusBadRequest, ""}, {pool.ErrBadCount, http.StatusBadRequest, ""}, {pool.ErrBadEnv, http.StatusBadRequest, ""}, @@ -97,6 +99,8 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo + RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error + UnregisterVolume(ctx context.Context, name string) error GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) SetVolumes(ctx context.Context, id, tenant string, volumes []types.Volume) (types.SandboxVolumesResponse, error) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -269,6 +273,8 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) mux.HandleFunc("GET /v1/volumes", s.requireToken(s.handleVolumes)) + mux.HandleFunc("PUT /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) + mux.HandleFunc("DELETE /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 6debba66..0aab0cb7 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2482,6 +2482,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { + registerVolume func(context.Context, string, types.VolumeRegistration) error + unregisterVolume func(context.Context, string) error getVolumes func(string, string) (types.SandboxVolumesResponse, error) setVolumes func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) patchVolumes func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -2557,6 +2559,14 @@ type fakeManager struct { reader *fakeReader } +func (f *fakeManager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + return f.registerVolume(ctx, name, spec) +} + +func (f *fakeManager) UnregisterVolume(ctx context.Context, name string) error { + return f.unregisterVolume(ctx, name) +} + func (f *fakeManager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { return f.getVolumes(id, tenant) } @@ -2977,10 +2987,14 @@ func (f *fakePlacer) TemplateVolumeOwners(probe string, _ []string) []string { } return f.templateVolumeOwners } + func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } -func (f *fakePlacer) PeerAddrs() []string { return f.addrs } + +func (f *fakePlacer) PeerAddrs() []string { return f.addrs } + func (f *fakePlacer) ClientAddr(addr string) string { return addr } -func (f *fakePlacer) ConfigMismatches() int { return 0 } + +func (f *fakePlacer) ConfigMismatches() int { return 0 } type fakeProber struct { owners []string @@ -2988,4 +3002,5 @@ type fakeProber struct { } func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } -func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } + +func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go new file mode 100644 index 00000000..39bad6a6 --- /dev/null +++ b/sandboxd/server/volume_registration.go @@ -0,0 +1,44 @@ +package server + +import ( + "context" + "errors" + "net/http" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request) { + if s.apiToken == "" { + writeErr(w, http.StatusForbidden, "volume registration requires a configured api token") + return + } + name := r.PathValue("name") + if !types.ValidVolumeName(name) { + writeErr(w, http.StatusBadRequest, "invalid volume name") + return + } + var err error + op := "unregister volume" + if r.Method == http.MethodPut { + req, ok := decodeBodyStrict[types.VolumeRegistration](w, r) + if !ok { + return + } + op = "register volume" + err = s.mgr.RegisterVolume(r.Context(), name, req) + } else { + err = s.mgr.UnregisterVolume(r.Context(), name) + } + switch { + case errors.Is(err, pool.ErrVolumeCatalogUnavailable): + writePoolErr(w, err) + case errors.Is(err, context.DeadlineExceeded): + writeErr(w, http.StatusGatewayTimeout, op+" timed out") + case errors.Is(err, context.Canceled): + writeErr(w, http.StatusRequestTimeout, op+" canceled") + default: + writeResult(w, r, op, name, op+" failed", err, noContent(w)) + } +} diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go new file mode 100644 index 00000000..99ff6ac7 --- /dev/null +++ b/sandboxd/server/volume_registration_test.go @@ -0,0 +1,274 @@ +package server + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, apiToken, token string + code int + }{ + {"root", "root", "root", http.StatusNoContent}, + {"tenant", "root", "acme-token", http.StatusForbidden}, + {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, + {"unknown", "root", "unknown", http.StatusUnauthorized}, + {"missing", "root", "", http.StatusUnauthorized}, + {"open without token", "", "", http.StatusForbidden}, + {"open with tenant token", "", "acme-token", http.StatusForbidden}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + ts := newTenantTestServer(t, tt.apiToken, []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + body := `{"path":"/private/data.img"}` + if tt.code != http.StatusNoContent { + body = `{invalid` + } + resp := doReq(t, method, ts.URL+"/v1/volumes/data", tt.token, body) + defer resp.Body.Close() + got, _ := io.ReadAll(resp.Body) + if resp.StatusCode != tt.code || called != (tt.code == http.StatusNoContent) { + t.Fatalf("status=%d called=%v body=%s, want %d", resp.StatusCode, called, got, tt.code) + } + if tt.code == http.StatusNoContent && len(got) != 0 { + t.Fatalf("204 response body = %q", got) + } + }) + } + } +} + +func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { + path := filepath.Join(t.TempDir(), "data.img") + if err := os.WriteFile(path, []byte("existing backing"), 0o600); err != nil { + t.Fatal(err) + } + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, token := range []string{"", "acme-token", "unknown"} { + t.Run(method+"/"+token, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + handler := New("", "", mgr, nil, nil, nil, nil, nil).Handler() + body := bytes.NewBufferString(fmt.Sprintf(`{"path":%q,"writable":true}`, path)) + bodyLen := body.Len() + req := httptest.NewRequest(method, "/v1/volumes/data", body) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusForbidden || called || body.Len() != bodyLen { + t.Fatalf("status=%d called=%v unread=%d/%d, want 403 without reading body or calling manager", w.Code, called, body.Len(), bodyLen) + } + if w.Header().Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", w.Header()) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { + type requestKey struct{} + const body = `{"path":"/private/images/../data.img","directio":"auto","writable":true,"tenants":["beta","acme"]}` + want := types.VolumeRegistration{Path: "/private/images/../data.img", DirectIO: "auto", Writable: true, Tenants: []string{"beta", "acme"}} + var registered types.VolumeRegistration + var registeredName, removedName string + checkContext := func(ctx context.Context) { + if ctx.Value(requestKey{}) != "request-value" { + t.Error("manager did not receive the request context") + } + } + mgr := &fakeManager{ + registerVolume: func(ctx context.Context, name string, spec types.VolumeRegistration) error { + checkContext(ctx) + registeredName, registered = name, spec + return nil + }, + unregisterVolume: func(ctx context.Context, name string) error { + checkContext(ctx) + removedName = name + return nil + }, + } + handler := New("root", "", mgr, nil, nil, nil, nil, nil).Handler() + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for range 2 { + req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer root") + req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent || w.Body.Len() != 0 { + t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) + } + } + } + if registeredName != "data-v1" || removedName != "data-v1" || !reflect.DeepEqual(registered, want) { + t.Fatalf("registered %q %+v, removed %q, want data-v1 %+v", registeredName, registered, removedName, want) + } +} + +func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { + for _, tt := range []struct{ name, body string }{ + {"missing", ""}, + {"malformed", `{"path":`}, + {"array", `[]`}, + {"unknown field", `{"path":"/private/data.img","extra":true}`}, + {"body name", `{"name":"other","path":"/private/data.img"}`}, + {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, + {"trailing object", `{"path":"/private/data.img"}{}`}, + {"trailing text", `{"path":"/private/data.img"}oops`}, + {"wrong writable type", `{"path":"/private/data.img","writable":"true"}`}, + {"wrong tenants type", `{"path":"/private/data.img","tenants":"acme"}`}, + {"oversized", `{"path":"/` + strings.Repeat("a", maxBodyBytes) + `"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { + called = true + return nil + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", tt.body) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest || called { + t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) + } + }) + } +} + +func TestVolumeRegistrationRoutesValidateNames(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, encoded string + valid bool + }{ + {"data", "data", true}, + {"a_b-0", "a_b-0", true}, + {"abcdefghijklmnopqrst", "abcdefghijklmnopqrst", true}, + {"Data", "Data", false}, + {"1data", "1data", false}, + {"cocoon-data", "cocoon-data", false}, + {"abcdefghijklmnopqrstu", "abcdefghijklmnopqrstu", false}, + {"data/set", "data%2Fset", false}, + {"data set", "data%20set", false}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + var gotName string + mgr := &fakeManager{ + registerVolume: func(_ context.Context, name string, _ types.VolumeRegistration) error { gotName = name; return nil }, + unregisterVolume: func(_ context.Context, name string) error { gotName = name; return nil }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/"+tt.encoded, "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + wantCode, wantName := http.StatusBadRequest, "" + if tt.valid { + wantCode, wantName = http.StatusNoContent, tt.name + } + if resp.StatusCode != wantCode || gotName != wantName { + t.Fatalf("status=%d name=%q body=%s, want %d name=%q", resp.StatusCode, gotName, body, wantCode, wantName) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name string + err error + code int + }{ + {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, + {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, + {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, + {"uncertain cancellation", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.Canceled), http.StatusServiceUnavailable}, + {"uncertain timeout", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.DeadlineExceeded), http.StatusServiceUnavailable}, + {"timeout", fmt.Errorf("register: %w", context.DeadlineExceeded), http.StatusGatewayTimeout}, + {"canceled", fmt.Errorf("register: %w", context.Canceled), http.StatusRequestTimeout}, + {"unexpected failure", errors.New("open /private/data.img: permission denied"), http.StatusInternalServerError}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { return tt.err }, + unregisterVolume: func(context.Context, string) error { return tt.err }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/data", "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var got map[string]string + if err := json.Unmarshal(body, &got); err != nil || len(got) != 1 || got["error"] == "" { + t.Fatalf("error body = %s, decode error = %v", body, err) + } + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { + t.Fatalf("status=%d body=%s, want safe %d", resp.StatusCode, body, tt.code) + } + if tt.code == http.StatusServiceUnavailable { + seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")) + if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { + t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) + } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", resp.Header) + } + }) + } + } +} + +func TestVolumeRegistrationKeepsCatalogListingPrivate(t *testing.T) { + for _, token := range []string{"root", "acme-token"} { + t.Run(token, func(t *testing.T) { + mgr := &fakeManager{volumeCatalog: func(tenant string, _ map[string]int) []types.VolumeInfo { + wantTenant := "" + if token == "acme-token" { + wantTenant = "acme" + } + if tenant != wantTenant { + t.Errorf("listing tenant = %q, want %q", tenant, wantTenant) + } + return []types.VolumeInfo{{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1, Writable: true}} + }} + ts := newTenantTestServer(t, "root", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", token, "") + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + const want = `{"volumes":[{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1,"writable":true}]}` + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Fatalf("catalog = %d %s, want 200 %s", resp.StatusCode, body, want) + } + }) + } +} diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index ac0cd6b4..0d81b540 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -22,6 +22,14 @@ type VolumeInfo struct { Writable bool `json:"writable,omitzero"` } +// VolumeRegistration binds a backing path to a runtime catalog name. +type VolumeRegistration struct { + Path string `json:"path"` + DirectIO string `json:"directio,omitempty"` + Writable bool `json:"writable,omitzero"` + Tenants []string `json:"tenants,omitempty"` +} + // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { Add []Volume `json:"add,omitzero"` From 247227e5e9731054ea2d11d31b64ae125cb3fff0 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 12:52:41 +0000 Subject: [PATCH 02/29] test(pool): reproduce unrelated admission during catalog failure --- sandboxd/pool/volume_catalog_test.go | 62 ++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 002f7c5e..f44480b3 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -205,6 +205,68 @@ func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { } } +func TestRuntimeCatalogWriteFailureOnlyFencesAffectedName(t *testing.T) { + for _, unregister := range []bool{false, true} { + t.Run(fmt.Sprintf("unregister=%t", unregister), func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + for _, name := range []string{"healthy", "affected"} { + if name == "affected" && !unregister { + continue + } + if err := m.RegisterVolume(t.Context(), name, types.VolumeRegistration{Path: writeVolumeImage(t, name, "bytes")}); err != nil { + t.Fatal(err) + } + } + sb := mustClaim(t, m, testKey) + var before []resolvedVolume + if unregister { + var err error + before, err = m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "healthy"}, {Name: "affected"}}) + if err != nil { + t.Fatal(err) + } + } + restore := blockCatalogRename(t, m) + defer restore() + var err error + if unregister { + err = m.UnregisterVolume(t.Context(), "affected") + } else { + err = m.RegisterVolume(t.Context(), "affected", types.VolumeRegistration{Path: writeVolumeImage(t, "affected", "bytes")}) + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("obstructed catalog write: %v", err) + } + if unregister { + if _, err := m.admitVolumes("", before); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("resolved mixed admission: %v", err) + } + } + for _, volumes := range [][]types.Volume{{{Name: "affected"}}, {{Name: "healthy"}, {Name: "affected"}}} { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("affected claim: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("affected live addition: %v", err) + } + } + for _, name := range []string{"healthy", "affected"} { + if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { + t.Fatalf("refused request reserved %s: %+v", name, got) + } + } + for _, name := range []string{"data", "healthy"} { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: name}}}); err != nil { + t.Errorf("unrelated %s claim: %v", name, err) + } + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}, {Name: "healthy"}}); err != nil { + t.Errorf("unrelated live addition: %v", err) + } + }) + } +} + func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, nil) From debf72910742108d0e6a546e2cae95bc7fb10598 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:03:57 +0000 Subject: [PATCH 03/29] fix(pool): scope catalog uncertainty to affected volumes --- docs/sandboxd-api.md | 133 +++++++----------- sandboxd/engine/cocoond_test.go | 28 ++++ sandboxd/pool/archive_test.go | 3 +- sandboxd/pool/checkpoint_bench_test.go | 4 +- sandboxd/pool/claims.go | 19 +-- sandboxd/pool/claims_bench_test.go | 6 +- sandboxd/pool/claims_test.go | 2 +- sandboxd/pool/failover_test.go | 9 +- sandboxd/pool/hibernate_test.go | 12 +- sandboxd/pool/liveness_test.go | 3 +- sandboxd/pool/pool_test.go | 5 +- sandboxd/pool/reconcile.go | 25 ++-- sandboxd/pool/remove.go | 4 + sandboxd/pool/volume.go | 60 ++++----- sandboxd/pool/volume_bench_test.go | 2 +- sandboxd/pool/volume_catalog.go | 50 +++---- sandboxd/pool/volume_catalog_test.go | 2 +- sandboxd/pool/volume_cleanup_test.go | 164 +++++++++-------------- sandboxd/pool/volume_convergence_test.go | 9 +- sandboxd/pool/volume_mutation_test.go | 10 +- sandboxd/pool/volume_rw_test.go | 8 +- sandboxd/types/types.go | 1 + 22 files changed, 263 insertions(+), 296 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index c103b4c8..18b0238f 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -257,13 +257,10 @@ entry's response is byte-identical to v1. ## PUT /v1/volumes/{name} -Auth: the configured root node API token. With no configured `api_token`, -registration and unregistration return 403. A tenant token returns 403. -Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. - -Registers a backing path on this node. The URL supplies the name; -the body has no `name` field. Unknown fields, duplicate fields, and trailing -JSON return 400. +Requires the configured root API token. A tenant token or a node without an +`api_token` returns 403. Sandbox, unknown, and missing tokens return 401 when +`api_token` is configured. The URL supplies the name. Unknown body fields, +duplicate fields, and trailing JSON return 400. ```json {"path": "/srv/datasets/data.img", "directio": "off", @@ -271,86 +268,60 @@ JSON return 400. ``` - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. -- `path` uses the same absolute-path validation as static volume config. - Registration does not open the backing or check its type or availability. - A missing backing retains its registration but cannot be attached until available. - Registration success does not establish that Cocoon supports the backing type. - Operators must keep the registered path and target stable. - Backing storage must be separate from daemon cleanup directories, including +- `path` uses static config's absolute-path validation. Registration does not + open the backing or check its type or availability. A missing backing keeps + its registration but cannot be attached. Operators must keep the path and + target stable and separate from daemon cleanup directories, including `/readers`, `/goldens`, and `/fork-*`. - Backing paths must not resolve into cleanup directories. - Cleanup directories must not resolve into backing storage. - Startup and staging cleanup do not consult volume registrations and can - delete backing files placed in those locations. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. -- `tenants` grants the named tenants self-service access. Omitted or empty - permits every authenticated scope, for both static and runtime entries. - Names use the tenant API name syntax. Runtime registration rejects invalid - names. Static config warns and skips invalid names. A nonempty static list with no - valid names remains restricted and grants no tenant access. A valid name may be - registered before that tenant exists; it grants no access until the tenant - is created. - -Success is 204 with an empty body. Repeating the same name, cleaned path, and -normalized registration fields succeeds. A different binding or different -registration fields return 409. Static and runtime registrations cannot share -a name or a cleaned backing path. Invalid registration fields return 400. -An uncertain write returns 503 with `Retry-After`. The requested registration -may still take effect through background retry or a later PUT or DELETE, -including a request for another name. A 503 can also reject a request before -acceptance, so it does not promise that the request will take effect. After -repairing storage failures, retry the exact request to establish completion. -Cancellation is not a rollback once a write has been accepted. -GET retains its existing projection without host paths or tenant access lists. -Registered entries support the existing mounted read-only live volume API. - -Only registration metadata is persisted locally in -`data_dir/runtime-volumes.json`, which loads before claim recovery. New records -use mode 0600. The daemon reads the record with the same filesystem permissions -as `claims.json` and replaces it durably. Registration creates, grows, and deletes -no disks. -Malformed or conflicting metadata prevents startup. A temporarily missing -backing file keeps its registration and refuses new attachment. -Catalog replacement and synchronization are independent of `sync_claims`, which -controls claim snapshot synchronization. Process-restart checks do not prove -power-loss recovery. The catalog is node-local with one sandboxd writer. -Startup reads only `runtime-volumes.json`. Crash leftovers named -`runtime-volumes.json.*.tmp` are neither loaded nor promoted. An operator can -remove those leftovers after stopping sandboxd and preserving any needed -evidence. +- `tenants` grants named tenants self-service access. Omitted or empty permits + every authenticated scope for both static and runtime entries. Runtime + registration rejects invalid tenant names. Static config skips invalid names + and grants no tenant access if a nonempty list has no valid names. + A tenant name can be registered before that tenant exists. + +Success is 204. An identical normalized registration is idempotent. A different +binding, changed fields, or a static name or cleaned path collision returns 409. +Invalid fields return 400. GET keeps its existing projection without host paths +or tenant lists. Runtime entries support the existing live volume API. + +Persistence uncertainty returns 503 with `Retry-After`. An accepted change may +complete through background retry or a later PUT or DELETE for any name. +A 503 can also precede acceptance. Retry the exact request after repairing +storage to establish completion. Cancellation does not undo an accepted write. +Until persistence succeeds, new admission refuses the affected name while +unrelated published names remain usable. Every catalog writer retries the +accepted change before accepting another change. + +The node stores metadata in `data_dir/runtime-volumes.json` independently of +`sync_claims` and loads it before claim recovery. New records use mode 0600. Malformed +or conflicting metadata prevents startup. Temporary replacement files are not +loaded. Registration creates, grows, and deletes no backing disks. The catalog +is node-local with one sandboxd writer. The controller coordinates cross-node +ownership, symlink and hardlink aliases, backing availability, and storage +placement outside cleanup directories. ## DELETE /v1/volumes/{name} -Auth and name validation match PUT. Removes only the runtime registration and -never deletes or changes the backing file. Success, including an absent, -unreferenced runtime name, is 204 with an empty body. Static registrations and -names referenced by a claim, an in-flight operation, cleanup, or an -unacknowledged recovery snapshot return 409. Catalog persistence uncertainty -returns 503 with `Retry-After` and has the same asynchronous completion -semantics as PUT. A failed DELETE may therefore remove the registration later. -An empty GET volume view alone does not establish that a recovery reference -has retired. - -Downgrading to a daemon without runtime registration requires draining runtime -references or supplying equivalent static volume config. Pending live volume -changes must finish before downgrade. - -Stale VM cleanup inspects attachments before removal. A complete running -inventory retains holds only on the named volumes. An explicit empty disk -array permits unrelated volume admission and catalog changes while removal -retries continue. An omitted or null disk list, a stopped VM, an inspection -failure, or an active creating owner retains the node-wide fence until -inspection or confirmed removal resolves it. Cocoon v0.7.3 omits both empty -inventories and failed disk queries, and does not collect disks for stopped -VMs. Those results remain unknown and can block static read-only admission. -Allowing readers would conflict with an unknown writer. Cleanup retries -remain enabled. -A warm VM name is not evidence that no volume is attached; attachment can -succeed before claim finalization fails. Known inventories keep holds on their -reported names. Volume-free claims continue to work. Path uniqueness uses -cleaned path strings. The operator coordinates symlink and hardlink aliases -and cross-node ownership. +Auth and name validation match PUT. Deletes only runtime registration metadata. +Success, including an absent unreferenced name, is 204. Static names and names +referenced by a claim, an in-flight operation, cleanup, or an unacknowledged +recovery snapshot return 409. Persistence uncertainty returns 503 with +`Retry-After` and the same retry semantics as PUT. A failed DELETE can complete +later. An empty GET view does not establish that recovery references retired. + +Stale VM cleanup retains named holds after a complete running inventory. +An ordinary stopped VM needs no volume holds. An inferred stopped result with +`stale: true`, inspection errors, omitted or null running disk lists, and busy +creating owners retain a node-wide admission fence. Any present inventory +during busy cleanup retains that fence. Confirmed absence or removal clears it. +This assumes sandboxd controls starts and attachments in its VM namespace. +Volume-free claims remain available. Cleanup retries continue. + +Before downgrading to a daemon without runtime registration, drain runtime +references or supply equivalent static volume config and finish pending live +volume changes. ## GET /v1/sandboxes/{id}/volumes diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 053dc205..398241be 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -45,6 +45,8 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { fakeCocoon(t, "#!/bin/sh\n[ \"$1 $2\" = \"vm inspect\" ] || exit 2\n"+ "if [ \"$3\" = sbx-1 ]; then echo '{\"config\":{\"name\":\"sbx-1\"},\"state\":\"running\","+ "\"attached_devices\":{\"disks\":[{\"id\":\"disk-a\",\"name\":\"imagenet\",\"path\":\"/v/a.img\",\"readonly\":true},{\"id\":\"disk-b\",\"name\":\"scratch\",\"path\":\"/v/b.img\"}]}}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stale ]; then echo '{\"state\":\"stopped\",\"stale\":true}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stopped ]; then echo '{\"state\":\"stopped\"}'; exit 0; fi\n"+ "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") @@ -52,11 +54,37 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } + for _, name := range []string{"sbx-stale", "sbx-stopped"} { + rec, present, err := e.Inspect(t.Context(), name) + if err != nil || !present || rec.State != "stopped" || rec.Stale != (name == "sbx-stale") { + t.Errorf("Inspect %s lost stopped classification: %+v, %v, %v", name, rec, present, err) + } + } if _, ok, err := e.Inspect(t.Context(), "sbx-gone"); err != nil || ok { t.Errorf("Inspect of a missing VM = %v, %v; want not found without an error", ok, err) } } +func TestInspectPreservesMissingDiskInventory(t *testing.T) { + for _, tc := range []struct { + name, inventory string + unknown bool + }{ + {"omitted devices", "", true}, + {"omitted disks", `,"attached_devices":{}`, true}, + {"null disks", `,"attached_devices":{"disks":null}`, true}, + {"empty disks", `,"attached_devices":{"disks":[]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + fakeCocoon(t, "#!/bin/sh\necho '{\"state\":\"running\""+tc.inventory+"}'\n") + rec, present, err := New("cocoon", nil, nil, false, false, "").Inspect(t.Context(), "sbx-1") + if err != nil || !present || (rec.AttachedDevices.Disks == nil) != tc.unknown { + t.Fatalf("inventory classification: %+v, %v, %v", rec, present, err) + } + }) + } +} + func serveCocoond(t *testing.T, socket, stream string) { t.Helper() ln, err := net.Listen("unix", socket) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 59064b8b..5cd3aa11 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -155,7 +155,8 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_mine": {ID: "sb_mine", VMName: "sbx-gone-1", Key: testKey, Token: "tok"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) diff --git a/sandboxd/pool/checkpoint_bench_test.go b/sandboxd/pool/checkpoint_bench_test.go index e3ba1b6c..997f0e77 100644 --- a/sandboxd/pool/checkpoint_bench_test.go +++ b/sandboxd/pool/checkpoint_bench_test.go @@ -15,7 +15,7 @@ func BenchmarkClaimCommitWithCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { claims := checkpointBenchClaims(checkpoints) s := newClaimStore(b.TempDir(), false) - if err := s.save(claims); err != nil { + if err := s.commit(s.reset(claims)); err != nil { b.Fatal(err) } one := claims["sb_0000000000000000"] @@ -39,7 +39,7 @@ func BenchmarkPinnedArchiveCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.save(m.claimed); err != nil { + if err := m.store.commit(m.store.reset(m.claimed)); err != nil { b.Fatal(err) } want := 0 diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 56f2bfe5..c8876a73 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -72,8 +72,6 @@ type claimRow struct { raw []byte } -type recoveryVolumeCount struct{ readers, writers int } - // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string @@ -93,7 +91,7 @@ type claimStore struct { checkpoints map[string]int currentVolumes map[string][]types.Volume retainedVolumes map[string][]types.Volume - volumes map[string]recoveryVolumeCount + volumes map[string]volumeHolders recommitting atomic.Bool recoveryVolumesPresent atomic.Bool @@ -104,7 +102,7 @@ func newClaimStore(dataDir string, sync bool) *claimStore { path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, refs: map[string][4]string{}, retained: map[string][]string{}, changed: map[string]struct{}{}, checkpoints: map[string]int{}, - currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]recoveryVolumeCount{}, + currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]volumeHolders{}, } } @@ -235,14 +233,14 @@ func (s *claimStore) commit(snap claimSnapshot) error { var capturedVolumes map[string][]types.Volume for id := range s.changed { captured[id] = s.refs[id] - s.changeRetainedRefs(id, appendRefs(slices.Clone(s.retained[id]), s.refs[id])) + s.changeRetainedRefs(id, appendRefs(s.retained[id], s.refs[id])) if volumes := s.currentVolumes[id]; len(volumes) != 0 { if capturedVolumes == nil { capturedVolumes = make(map[string][]types.Volume) } capturedVolumes[id] = volumes } - s.changeRetainedVolumes(id, appendRecoveryRefs(slices.Clone(s.retainedVolumes[id]), s.currentVolumes[id])) + s.changeRetainedVolumes(id, appendRecoveryRefs(s.retainedVolumes[id], s.currentVolumes[id])) } s.mu.Unlock() s.buf = append(s.buf[:0], '{') @@ -321,11 +319,6 @@ func (s *claimStore) replace(raw []byte) error { return nil } -// save is the combined form for the startup Reconcile pass, before contention exists. -func (s *claimStore) save(claims map[string]*types.Sandbox) error { - return s.commit(s.reset(claims)) -} - func (s *claimStore) synced() bool { s.mu.Lock() defer s.mu.Unlock() @@ -378,7 +371,7 @@ func (s *claimStore) changeVolumeCounts(refs []types.Volume, delta int) { } else { count.readers += delta } - if count == (recoveryVolumeCount{}) { + if count == (volumeHolders{}) { delete(s.volumes, v.Name) } else { s.volumes[v.Name] = count @@ -400,7 +393,7 @@ func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { func (s *claimStore) referencesVolume(name string) bool { s.mu.Lock() defer s.mu.Unlock() - return s.volumes[name] != (recoveryVolumeCount{}) + return s.volumes[name] != (volumeHolders{}) } func (s *claimStore) volumeConflicts(volume types.Volume, ownerID string) bool { diff --git a/sandboxd/pool/claims_bench_test.go b/sandboxd/pool/claims_bench_test.go index b1ae79da..d1f059bc 100644 --- a/sandboxd/pool/claims_bench_test.go +++ b/sandboxd/pool/claims_bench_test.go @@ -18,7 +18,7 @@ func BenchmarkStoreSaveScaling(b *testing.B) { st := newClaimStore(b.TempDir(), false) claims := benchClaims(n) for b.Loop() { - if err := st.save(claims); err != nil { + if err := st.commit(st.reset(claims)); err != nil { b.Fatalf("save: %v", err) } } @@ -48,7 +48,7 @@ func BenchmarkStoreCommitEnv(b *testing.B) { sb.Env = envs[arm] } s := newClaimStore(b.TempDir(), durable) - if err := s.save(claims); err != nil { + if err := s.commit(s.reset(claims)); err != nil { b.Fatalf("save: %v", err) } one := claims[fmt.Sprintf("sb_%016x", 0)] @@ -105,7 +105,7 @@ func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, arm switch arm { case "under-lock": mu.Lock() - _ = s.save(claims) + _ = s.commit(s.reset(claims)) mu.Unlock() case "rebuild": mu.Lock() diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index e12ea0e7..d369f829 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -85,7 +85,7 @@ func TestStoreRoundTrip(t *testing.T) { } for _, fsync := range []bool{false, true} { s := newClaimStore(t.TempDir(), fsync) - if err := s.save(claims); err != nil { + if err := s.commit(s.reset(claims)); err != nil { t.Fatalf("save (sync=%v): %v", fsync, err) } loadedStore := newClaimStore(filepath.Dir(s.path), false) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 505dc55c..dd37a6bc 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -272,7 +272,8 @@ func TestFailoverRestartReclaimsAnInterruptedArchive(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - if err := newClaimStore(dirA, false).save(claims); err != nil { + claimStore := newClaimStore(dirA, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -295,7 +296,8 @@ func TestFailoverRestartReclaimSparesACheckpointAdoptedMidReconcile(t *testing.T claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - if err := newClaimStore(dirA, false).save(claims); err != nil { + claimStore := newClaimStore(dirA, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -320,7 +322,8 @@ func TestFailoverRestartReclaimsOnlyTheCheckpointsItPublished(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - if err := newClaimStore(dirA, false).save(claims); err != nil { + claimStore := newClaimStore(dirA, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index ffc4dd9e..7642b11a 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -528,7 +528,8 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { VsockSocket: "/vsock/lagged", PendingSnap: hibernatePrefix + "lagged-1-abc123", }, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -557,7 +558,8 @@ func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_orp": {ID: "sb_orp", VMName: "sbx-orphan-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/orphan"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -583,7 +585,8 @@ func TestReconcileKeepsIntentWithoutImageAsRunning(t *testing.T) { VsockSocket: "/vsock/noimg", PendingSnap: hibernatePrefix + "noimg-1-abc123", }, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -613,7 +616,8 @@ func TestReconcileAdoptsHibernated(t *testing.T) { HibernateSnap: hibernatePrefix + "hibernated-1", }, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 2864c615..ac557fb2 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -486,7 +486,8 @@ func TestReconcileKeepsRunningClaimsTheHostLost(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_lost": {ID: "sb_lost", VMName: "sbx-lost-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/lost"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } cfg := &config.Config{DataDir: dataDir, VMMRestart: tt.policy} diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index d9fc8b46..b66724ce 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -323,7 +323,8 @@ func TestReconcile(t *testing.T) { "sb_live": {ID: "sb_live", VMName: "sbx-live-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/live-old"}, "sb_dead": {ID: "sb_dead", VMName: "sbx-dead-1", Key: testKey, Token: "tok"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -1256,6 +1257,7 @@ type fakeEngine struct { liveVolumeHook func(string) error stopped map[string]bool inspectHook func() error + inspectStale bool inspectDisksUnknown bool trackDisks bool disks map[string][]types.VMDisk @@ -1542,6 +1544,7 @@ func (f *fakeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, return types.VMRecord{}, false, nil } f.mu.Lock() + vms[i].Stale = f.inspectStale if !f.inspectDisksUnknown { vms[i].AttachedDevices.Disks = f.disks[name] } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 056fdf81..9e103613 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -195,17 +195,26 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) return td } - if present && (inspected.State != vmStateRunning || inspected.AttachedDevices.Disks == nil) { - return td - } var holds []types.Volume if present { - holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW + switch inspected.State { + case "stopped": + if inspected.Stale { + return td + } + case vmStateRunning: + if inspected.AttachedDevices.Disks == nil { + return td } + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW + } + } + default: + return td } } m.mu.Lock() diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 924e3f5d..e901e0a5 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -92,6 +92,10 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: + if _, present, inspectErr := m.eng.Inspect(ctx, name); inspectErr == nil && !present { + m.finishRemoval(ctx, name, pending) + return + } m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 0660e7c5..49069b3c 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -147,9 +147,11 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } m.mu.Lock() - if err := m.catalog.admissionError(); err != nil { - m.mu.Unlock() - return nil, err + for _, volume := range applied { + if err := m.catalog.admissionError(volume.Name); err != nil { + m.mu.Unlock() + return nil, err + } } catalog := m.catalog.entries m.mu.Unlock() @@ -308,35 +310,6 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) b return true } -// reserveVolumes admits one claim's volumes atomically; callers hold m.mu. -func (m *Manager) reserveVolumes(volumes []types.Volume) error { - return m.reserveVolumesFor(volumes, nil) -} - -func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { - if len(volumes) == 0 { - return nil - } - if err := m.catalog.admissionError(); err != nil { - return err - } - ownerID := "" - if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { - ownerID = owner.ID - } - for _, volume := range volumes { - if m.store.volumeConflicts(volume, ownerID) { - return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) - } - holders := m.catalog.holders[volume.Name] - if holders.writers > 0 || (volume.RW() && holders.readers > 0) { - return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) - } - } - m.adoptVolumes(volumes) - return nil -} - // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { @@ -382,13 +355,30 @@ func (m *Manager) catalogSnapshot() map[string]*catalogVolume { return m.catalog.entries } +// reserveResolved admits the entire request while callers hold m.mu. func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox) error { - for _, volume := range volumes { - if volume.entry == nil || m.catalog.entries[volume.applied.Name] != volume.entry { + ownerID := "" + if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { + ownerID = owner.ID + } + for _, resolved := range volumes { + volume := resolved.applied + if err := m.catalog.admissionError(volume.Name); err != nil { + return err + } + if m.catalog.entries[volume.Name] != resolved.entry { return ErrVolumeUnavailable } + if m.store.volumeConflicts(volume, ownerID) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } + holders := m.catalog.holders[volume.Name] + if holders.writers > 0 || (volume.RW() && holders.readers > 0) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } } - return m.reserveVolumesFor(appliedVolumes(volumes), owner) + m.adoptVolumes(appliedVolumes(volumes)) + return nil } // quiesceBudget bounds one quiesce: a slice per writable mount plus one for the sync fallback. diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go index 4016f920..22242076 100644 --- a/sandboxd/pool/volume_bench_test.go +++ b/sandboxd/pool/volume_bench_test.go @@ -67,7 +67,7 @@ func volumeBenchManager(b *testing.B, checkpoints bool) *Manager { b.Helper() m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.save(m.claimed); err != nil { + if err := m.store.commit(m.store.reset(m.claimed)); err != nil { b.Fatal(err) } path := filepath.Join(b.TempDir(), "data.raw") diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index f2ccc456..4cd0c302 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -28,6 +28,7 @@ type runtimeCatalog struct { } type catalogWrite struct { + name string entries map[string]*catalogVolume raw []byte } @@ -40,8 +41,8 @@ type volumeCatalog struct { writeMu sync.Mutex } -func (c *volumeCatalog) admissionError() error { - if c.pending != nil || c.unknown > 0 { +func (c *volumeCatalog) admissionError(name string) error { + if c.unknown > 0 || c.pending != nil && c.pending.name == name { return ErrVolumeCatalogUnavailable } return nil @@ -60,7 +61,7 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo } m.mu.Lock() entries := m.catalog.entries - err = m.catalog.admissionError() + err = m.catalog.admissionError(name) m.mu.Unlock() if err != nil { return err @@ -71,6 +72,9 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo } return fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) } + if err := catalogPathConflict(entries, normalized.Path); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) + } entries = maps.Clone(entries) entries[name] = &catalogVolume{ runtime: true, @@ -78,9 +82,6 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo tenants: normalized.Tenants, writable: normalized.Writable, } - if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { - return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) - } return m.writeCatalog(ctx, name, entries) } @@ -97,7 +98,7 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { m.mu.Lock() entries := m.catalog.entries entry := entries[name] - err := m.catalog.admissionError() + err := m.catalog.admissionError(name) if err == nil && ((entry != nil && !entry.runtime) || (entry == nil && m.volumeReferenced(name))) { err = fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) } @@ -134,6 +135,9 @@ func (m *Manager) loadRuntimeVolumes() error { if m.catalog.entries[name] != nil { return fmt.Errorf("runtime volume catalog duplicates static name %q", name) } + if err := catalogPathConflict(entries, normalized.Path); err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } entries[name] = &catalogVolume{ runtime: true, disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, @@ -141,9 +145,6 @@ func (m *Manager) loadRuntimeVolumes() error { writable: normalized.Writable, } } - if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { - return fmt.Errorf("runtime volume catalog: %w", err) - } m.catalog.entries = entries return nil } @@ -152,11 +153,6 @@ func (m *Manager) volumeReferenced(name string) bool { if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { return true } - for _, run := range m.volumeRuns { - if volumeIndex(run.holds, name) >= 0 { - return true - } - } for _, sb := range m.claimed { if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { return true @@ -176,12 +172,9 @@ func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[str if err != nil { return fmt.Errorf("%w: %w", ErrBadVolume, err) } - proposal := &catalogWrite{entries: entries, raw: raw} + proposal := &catalogWrite{name: name, entries: entries, raw: raw} m.mu.Lock() err = ctx.Err() - if err == nil { - err = m.catalog.admissionError() - } if err == nil && m.volumeReferenced(name) { err = fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) } @@ -197,14 +190,11 @@ func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[str func (m *Manager) commitPendingCatalog(ctx context.Context) error { m.mu.Lock() - proposal, unknown := m.catalog.pending, m.catalog.unknown > 0 + proposal := m.catalog.pending m.mu.Unlock() if proposal == nil { return nil } - if unknown { - return ErrVolumeCatalogUnavailable - } if err := ctx.Err(); err != nil { return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) } @@ -232,19 +222,13 @@ func (m *Manager) retryVolumeCatalog(ctx context.Context) { } } -func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { - volumes := make([]config.VolumeSpec, 0, len(entries)) +func catalogPathConflict(entries map[string]*catalogVolume, path string) error { for name, entry := range entries { - var tenants []string - if entry.runtime { - tenants = entry.tenants + if filepath.Clean(entry.disk.Path) == path { + return fmt.Errorf("backing path already registered as %q", name) } - volumes = append(volumes, config.VolumeSpec{ - Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, - Writable: entry.writable, Tenants: tenants, - }) } - return volumes + return nil } func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index f44480b3..197405a9 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -508,7 +508,7 @@ func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) t.Fatalf("failed retry cleared pending write: %v", err) } m.retryVolumeCatalog(t.Context()) - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("pending replacement lost fence: %v", err) } restore() diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index c2ff00ae..a0f50798 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -3,9 +3,6 @@ package pool import ( "context" "errors" - "fmt" - "os" - "path/filepath" "slices" "sync/atomic" "testing" @@ -17,71 +14,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { - for _, state := range []string{vmStateRunning, vmStateCreating, "stopped"} { - t.Run(state, func(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, nil) - path := writeVolumeImage(t, "incomplete-inventory", "original") - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { - t.Fatal(err) - } - name := "sbx-incomplete-inventory" - base.vms[name] = "/vsock/incomplete" - base.creating[name] = state == vmStateCreating - base.staleOutcome = engine.StaleCreateBusy - base.removeErrFor = name - base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} - cli := filepath.Join(t.TempDir(), "inspect") - writeInspect := func(state, inventory string) { - t.Helper() - raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"%s\"%s}'\n", name, state, inventory) - if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { - t.Fatal(err) - } - } - writeInspect(state, "") - m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} - rec := base.record(name) - rec.State = state - if m.removeStaleVM(t.Context(), name, rec) { - t.Fatal("stale VM unexpectedly disappeared") - } - for range 2 { - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("unregister must refuse incomplete inventory: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("conflicting writer must refuse incomplete inventory: %v", err) - } - m.retryRemovals(t.Context()).Wait() - } - base.creating[name] = false - writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) - if state == vmStateCreating { - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("busy owner must retain the unknown fence: %v", err) - } - } - base.staleOutcome = engine.StaleCreateNotCreating - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("complete inventory lost the surviving writer: %v", err) - } - other := writeVolumeImage(t, "complete-inventory-other", "other") - if err := m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: other}); err != nil { - t.Fatalf("complete inventory retained the global fence: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if err := m.UnregisterVolume(t.Context(), "data"); err != nil { - t.Fatalf("confirmed removal retained volume protection: %v", err) - } - }) - } -} - func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, nil) @@ -178,8 +110,18 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { } func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { - for _, state := range []string{"running", "stopped", "creating"} { - t.Run(state, func(t *testing.T) { + for _, tc := range []struct { + name, state string + inspectError bool + }{ + {"running omitted", "running", false}, + {"running error", "running", true}, + {"inferred stopped omitted", "stopped", false}, + {"inferred stopped error", "stopped", true}, + {"creating omitted", "creating", false}, + {"creating error", "creating", true}, + } { + t.Run(tc.name, func(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, nil) path := writeVolumeImage(t, "runtime", "bytes") @@ -188,20 +130,24 @@ func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { } name := "sbx-stale-unknown" base.vms[name] = "/vsock/stale" - base.stopped[name] = state == "stopped" - base.creating[name] = state == "creating" + base.stopped[name] = tc.state == "stopped" + base.creating[name] = tc.state == "creating" + base.inspectStale = tc.state == "stopped" + base.inspectDisksUnknown = !tc.inspectError base.staleOutcome = engine.StaleCreateBusy base.removeErrFor = name base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} var inspectFailed atomic.Bool inspectFailed.Store(true) base.inspectHook = func() error { - if inspectFailed.Load() { + if tc.inspectError && inspectFailed.Load() { return errors.New("inspection unavailable") } return nil } - if m.removeStaleVM(t.Context(), name, base.record(name)) { + rec := base.record(name) + rec.State = tc.state + if m.removeStaleVM(t.Context(), name, rec) { t.Fatal("unknown stale VM unexpectedly gone") } for range 2 { @@ -228,6 +174,13 @@ func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { } inspectFailed.Store(false) base.stopped[name], base.creating[name] = false, false + base.inspectStale, base.inspectDisksUnknown = false, false + if tc.state == "creating" { + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy partial inventory cleared unknown: %v", err) + } + } base.staleOutcome = engine.StaleCreateNotCreating m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { @@ -439,27 +392,25 @@ func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) } } -func TestStaleInventoryDistinguishesOmittedAndExplicitEmptyDisks(t *testing.T) { +func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { for _, tc := range []struct { - name, inventory string - wantErr error + name string + stopped, stale bool + disks []types.VMDisk + wantErr error }{ - {"omitted devices", "", ErrVolumeCatalogUnavailable}, - {"omitted disks", `,"attached_devices":{}`, ErrVolumeCatalogUnavailable}, - {"null disks", `,"attached_devices":{"disks":null}`, ErrVolumeCatalogUnavailable}, - {"explicit empty disks", `,"attached_devices":{"disks":[]}`, nil}, + {"running unknown", false, false, nil, ErrVolumeCatalogUnavailable}, + {"running empty", false, false, []types.VMDisk{}, nil}, + {"ordinary stopped", true, false, nil, nil}, + {"inferred stopped", true, true, nil, ErrVolumeCatalogUnavailable}, } { t.Run(tc.name, func(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) name := "sbx-inventory-shape" base.vms[name], base.removeErrFor = "/vsock/stale", name - cli := filepath.Join(t.TempDir(), "inspect") - raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"running\"%s}'\n", name, tc.inventory) - if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { - t.Fatal(err) - } - m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + base.disks = map[string][]types.VMDisk{name: tc.disks} + base.stopped[name], base.inspectStale = tc.stopped, tc.stale if m.removeStaleVM(t.Context(), name, base.record(name)) { t.Fatal("stale VM unexpectedly disappeared") } @@ -467,6 +418,12 @@ func TestStaleInventoryDistinguishesOmittedAndExplicitEmptyDisks(t *testing.T) { if !errors.Is(err, tc.wantErr) { t.Fatalf("admission=%v, want=%v", err, tc.wantErr) } + if _, queued := m.pendingRemovals[name]; !queued { + t.Fatal("surviving VM lost pending removal") + } + if err := m.RegisterVolume(t.Context(), "before-removal", types.VolumeRegistration{Path: "/images/before-removal"}); !errors.Is(err, tc.wantErr) { + t.Fatalf("registration before removal: %v", err) + } base.removeErrFor = "" m.retryRemovals(t.Context()).Wait() if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { @@ -504,40 +461,43 @@ func TestStaleInventoryNarrowsFenceBeforeBlockedRemoval(t *testing.T) { } func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { - for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound} { + for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { t.Run(string(outcome), func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) name := "sbx-busy-create" eng.vms[name], eng.creating[name] = "/vsock/creating", true eng.staleOutcome = engine.StaleCreateBusy - eng.inspectHook = func() error { - t.Error("busy owner must finish before inventory can be trusted") - return errors.New("busy") - } + eng.disks = map[string][]types.VMDisk{name: {{Name: "other"}}} rec := eng.record(name) rec.State = vmStateCreating if m.removeStaleVM(t.Context(), name, rec) { t.Fatal("busy create reported gone") } - m.retryRemovals(t.Context()).Wait() + eng.creating[name] = false + for _, stopped := range []bool{false, true} { + eng.stopped[name] = stopped + before := eng.inspects + m.retryRemovals(t.Context()).Wait() + if eng.inspects != before+1 { + t.Fatal("busy retry did not inspect") + } + } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("busy create admitted a volume: %v", err) } eng.staleOutcome = outcome + if outcome == engine.StaleCreateBusy { + delete(eng.vms, name) + } + m.retryRemovals(t.Context()).Wait() m.retryRemovals(t.Context()).Wait() + if m.catalog.unknown != 0 || len(m.pendingRemovals) != 0 { + t.Fatalf("cleanup retained or double-released its fence: unknown=%d pending=%d", m.catalog.unknown, len(m.pendingRemovals)) + } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { t.Fatalf("confirmed cleanup retained fence: %v", err) } }) } } - -type incompleteInventoryEngine struct { - *fakeEngine - cli *engine.Engine -} - -func (e *incompleteInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { - return e.cli.Inspect(ctx, name) -} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index ef8fb774..4446957a 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -343,12 +343,19 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { m.store.set(sb) m.mu.Unlock() case "admitted": + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "data", "bytes")}); err != nil { + t.Fatal(err) + } _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { t.Fatal(err) } + resolved, err := m.resolveVolumes(t.Context(), "", held) + if err != nil { + t.Fatal(err) + } m.mu.Lock() - err = m.reserveVolumes(held) + err = m.reserveResolved(resolved, nil) run.holds = held m.mu.Unlock() if err != nil { diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index d9de1c70..66ebfd4e 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -403,7 +403,7 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { if got := claims["sb_1"].VolumesWant; !slices.Equal(got, tt.want) { t.Fatalf("loaded target %v, want %v", got, tt.want) } - if err = store.save(claims); err != nil { + if err = store.commit(store.reset(claims)); err != nil { t.Fatal(err) } rawBytes, err := os.ReadFile(store.path) @@ -854,10 +854,14 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume if err != nil { t.Fatal(err) } + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + resolved, err := m.resolveVolumes(t.Context(), "", transient) + if err != nil { + t.Fatal(err) + } m.mu.Lock() defer m.mu.Unlock() - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} - if err = m.reserveVolumes(transient); err != nil { + if err = m.reserveResolved(resolved, nil); err != nil { t.Fatal(err) } run.holds = append(run.holds, transient...) diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 033bc8c3..dadeda8c 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -153,7 +153,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { t.Fatalf("mark dirty: %v", markErr) } m.mu.Lock() - reserveErr := m.reserveVolumes(appliedVolumes(resolved)) + reserveErr := m.reserveResolved(resolved, nil) m.mu.Unlock() if reserveErr != nil { t.Fatalf("reserve once the writer released: %v", reserveErr) @@ -222,11 +222,15 @@ func TestVolumeAdmissionMatrix(t *testing.T) { func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") m := newVolumeManager(t, newFakeEngine(), []config.VolumeSpec{{Name: "scratch", Path: path, Writable: true}}) + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}) + if err != nil { + t.Fatal(err) + } m.mu.Lock() defer m.mu.Unlock() m.adoptVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch", Mode: types.VolumeModeRW}}) - if err := m.reserveVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}); !errors.Is(err, ErrVolumeBusy) { + if err := m.reserveResolved(resolved, nil); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..2a93ab78 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -731,6 +731,7 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { + Stale bool `json:"stale,omitzero"` State string `json:"state"` PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` From af6ee9a39fe1e1c2b993afa470cc1c992ea7d839 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:31:48 +0000 Subject: [PATCH 04/29] pool: reproduce private name disclosure during catalog writes --- sandboxd/pool/volume_acl_test.go | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index 2ebe99a2..a7b30825 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -65,6 +65,59 @@ func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { } } +func TestRuntimeCatalogPendingWritePreservesPrivateNameMasking(t *testing.T) { + for _, operation := range []string{"register", "unregister"} { + t.Run(operation, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "private", "bytes"), Tenants: []string{"other"}} + if operation == "unregister" { + if err := m.RegisterVolume(t.Context(), "private", spec); err != nil { + t.Fatal(err) + } + } + blockCatalogRename(t, m) + var err error + if operation == "unregister" { + err = m.UnregisterVolume(t.Context(), "private") + } else { + err = m.RegisterVolume(t.Context(), "private", spec) + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("obstructed catalog write: %v", err) + } + for _, requested := range [][]types.Volume{ + {{Name: "private"}}, + {{Name: "data"}, {Name: "private"}}, + {{Name: "private"}, {Name: "data"}}, + } { + missing := slices.Clone(requested) + for i := range missing { + if missing[i].Name == "private" { + missing[i].Name = "missing" + } + } + _, deniedErr := m.SetVolumes(t.Context(), sb.ID, "acme", requested) + _, missingErr := m.SetVolumes(t.Context(), sb.ID, "acme", missing) + if !errors.Is(deniedErr, ErrVolumeUnavailable) || !errors.Is(missingErr, ErrVolumeUnavailable) || deniedErr.Error() != missingErr.Error() { + t.Errorf("private name distinguishable for %+v: denied=%v missing=%v", requested, deniedErr, missingErr) + } + } + if view, err := m.GetVolumes(sb.ID, "acme"); err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("refused request changed claim volumes: %+v, %v", view, err) + } + if len(eng.volumeSpecs) != 0 { + t.Fatal("refused request reached device attachment") + } + for _, name := range []string{"data", "private", "missing"} { + if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { + t.Fatalf("refused request reserved %s: %+v", name, got) + } + } + }) + } +} + func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { for _, operation := range []string{"put", "patch"} { t.Run(operation, func(t *testing.T) { From 8303c4b9390a3cdbed36b853959addee8f0f54df Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:33:08 +0000 Subject: [PATCH 05/29] pool: check volume authorization before pending admission --- sandboxd/pool/volume.go | 13 +++++++------ sandboxd/pool/volume_catalog_test.go | 20 ++++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 49069b3c..9f840edf 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -147,21 +147,22 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } m.mu.Lock() + catalog := m.catalog.entries for _, volume := range applied { + entry := catalog[volume.Name] + if entry == nil || !entry.allowedFor(tenant) { + m.mu.Unlock() + return nil, ErrVolumeUnavailable + } if err := m.catalog.admissionError(volume.Name); err != nil { m.mu.Unlock() return nil, err } } - catalog := m.catalog.entries m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := catalog[volume.Name] - if !ok || !entry.allowedFor(tenant) { - return nil, ErrVolumeUnavailable - } - resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) + resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, catalog[volume.Name]) if resolveErr != nil { return nil, resolveErr } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 197405a9..928b6b18 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -179,8 +179,8 @@ func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("write failure: %v", err) } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("pending admission: %v", err) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { t.Fatalf("volume-free claim blocked: %v", err) @@ -242,11 +242,15 @@ func TestRuntimeCatalogWriteFailureOnlyFencesAffectedName(t *testing.T) { t.Fatalf("resolved mixed admission: %v", err) } } + wantErr := ErrVolumeUnavailable + if unregister { + wantErr = ErrVolumeCatalogUnavailable + } for _, volumes := range [][]types.Volume{{{Name: "affected"}}, {{Name: "healthy"}, {Name: "affected"}}} { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, wantErr) { t.Fatalf("affected claim: %v", err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, wantErr) { t.Fatalf("affected live addition: %v", err) } } @@ -280,8 +284,8 @@ func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { t.Fatalf("possible replacement not recovered: %v", got) } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("uncertain admission: %v", err) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { t.Fatalf("volume-free claim blocked: %v", err) @@ -508,8 +512,8 @@ func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) t.Fatalf("failed retry cleared pending write: %v", err) } m.retryVolumeCatalog(t.Context()) - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("pending replacement lost fence: %v", err) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) } restore() ctx, cancel := context.WithCancel(t.Context()) From e51545d4d7c882f292b8f5741f88aa5783e8371b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:38:56 +0000 Subject: [PATCH 06/29] pool: reproduce inventory changes during failed removal --- sandboxd/pool/volume_cleanup_test.go | 80 +++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index a0f50798..966b9400 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -436,30 +436,82 @@ func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { } } -func TestStaleInventoryNarrowsFenceBeforeBlockedRemoval(t *testing.T) { +func TestStaleInventoryRetainsFenceUntilRemovalFails(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-known-before-remove" + name := "sbx-attach-during-remove" eng.vms[name], eng.removeErrFor = "/vsock/stale", name - eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} eng.removeStall = make(chan struct{}) - done := make(chan bool, 1) - go func() { done <- m.removeStaleVM(t.Context(), name, eng.record(name)) }() - synctest.Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("known stale writer: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { - t.Errorf("blocked removal retained global fence: %v", err) - } - close(eng.removeStall) - if <-done { + done := make(chan struct{}) + go func() { + defer close(done) + if m.removeStaleVM(t.Context(), name, eng.record(name)) { + t.Error("failed removal reported gone") + } + }() + checkStaleRemovalInventory(t, m, eng, name, done) + }) +} + +func TestStaleInventoryRetainsFenceUntilRetryFails(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-attach-during-retry" + eng.vms[name], eng.removeErrFor = "/vsock/stale", name + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} + eng.inspectDisksUnknown = true + if m.removeStaleVM(t.Context(), name, eng.record(name)) { t.Fatal("failed removal reported gone") } + eng.inspectDisksUnknown = false + eng.removeStall = make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + m.retryRemovals(t.Context()).Wait() + }() + checkStaleRemovalInventory(t, m, eng, name, done) }) } +func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { + t.Helper() + synctest.Wait() + registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted a reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted registration: %v", err) + } + eng.mu.Lock() + eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) + eng.mu.Unlock() + close(eng.removeStall) + <-done + if _, queued := m.pendingRemovals[name]; !queued { + t.Fatal("failed removal lost its retry") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer attached during removal lost its hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("failed removal lost its reader hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Errorf("failed removal blocked a compatible reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { + t.Fatalf("failed removal blocked unrelated registration: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { + t.Errorf("failed removal blocked an unrelated reader: %v", err) + } +} + func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { t.Run(string(outcome), func(t *testing.T) { From aff97975ef48790cc783e1bbf19161fd7386bfe3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:39:21 +0000 Subject: [PATCH 07/29] pool: preserve stale inventory observation after failed removal --- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/remove.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 9e103613..a7255629 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -177,11 +177,11 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe } // not-creating: the record moved on under the lock; remove normally. } - td = m.resolveStaleVolumes(ctx, name, td) if m.removeVM(ctx, name) { m.finishVolumeTeardown(ctx, td) return true } + td = m.resolveStaleVolumes(ctx, name, td) m.queueRemoval(name, "", rec.TapDevice(), td, time.Time{}) return false } diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index e901e0a5..a3bebea6 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -103,8 +103,8 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending return } } - pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if !m.removeVM(ctx, name) { + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } From f49765ade315c2e9aa181a7ebde766b5e9649a41 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:43:27 +0000 Subject: [PATCH 08/29] pool: align cleanup tests with post-removal inspection --- sandboxd/pool/reconcile_test.go | 3 -- sandboxd/pool/volume_cleanup_test.go | 70 ++++++++++++++-------------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 636555e7..6177c19a 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -181,9 +181,6 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { if !eng.removed("sbx-orphan-1") { t.Error("unowned running VM not removed") } - if eng.inspects != 1 { - t.Errorf("inspected stale VM %d times, want 1 before removal", eng.inspects) - } } func TestReconcileSweepsInterruptedForkExports(t *testing.T) { diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 966b9400..46b7f946 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -477,41 +477,6 @@ func TestStaleInventoryRetainsFenceUntilRetryFails(t *testing.T) { }) } -func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { - t.Helper() - synctest.Wait() - registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted a reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted registration: %v", err) - } - eng.mu.Lock() - eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) - eng.mu.Unlock() - close(eng.removeStall) - <-done - if _, queued := m.pendingRemovals[name]; !queued { - t.Fatal("failed removal lost its retry") - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("writer attached during removal lost its hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("failed removal lost its reader hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Errorf("failed removal blocked a compatible reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { - t.Fatalf("failed removal blocked unrelated registration: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { - t.Errorf("failed removal blocked an unrelated reader: %v", err) - } -} - func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { t.Run(string(outcome), func(t *testing.T) { @@ -553,3 +518,38 @@ func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { }) } } + +func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { + t.Helper() + synctest.Wait() + registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted a reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted registration: %v", err) + } + eng.mu.Lock() + eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) + eng.mu.Unlock() + close(eng.removeStall) + <-done + if _, queued := m.pendingRemovals[name]; !queued { + t.Fatal("failed removal lost its retry") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer attached during removal lost its hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("failed removal lost its reader hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Errorf("failed removal blocked a compatible reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { + t.Fatalf("failed removal blocked unrelated registration: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { + t.Errorf("failed removal blocked an unrelated reader: %v", err) + } +} From df7c3082e91f8ee2858bcbcf11570be42379831a Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:12:50 +0000 Subject: [PATCH 09/29] pool: release taken-over volume holds when the run ends A teardown that takes over an in-flight volume run's holds now parks them on the run, and finishVolumeRun releases them once. This drops the done poll, the requeues that re-ran vm rm every reap tick, and the ignored bool result of finishVolumeTeardown. --- sandboxd/pool/remove.go | 17 ++++----- sandboxd/pool/volume.go | 31 ++++++++-------- sandboxd/pool/volume_cleanup_test.go | 51 +++------------------------ sandboxd/pool/volume_mutation.go | 8 +++-- sandboxd/pool/volume_mutation_test.go | 8 ++--- 5 files changed, 31 insertions(+), 84 deletions(-) diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index a3bebea6..5fc199a2 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -45,9 +45,7 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { if m.removeVM(ctx, name) { - if !m.finishVolumeTeardown(ctx, td) { - m.queueRemoval(name, sandboxID, "", td, time.Time{}) - } + m.finishVolumeTeardown(ctx, td) return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -93,13 +91,13 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending return case outcome == engine.StaleCreateBusy: if _, present, inspectErr := m.eng.Inspect(ctx, name); inspectErr == nil && !present { - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) return } m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) return } } @@ -108,14 +106,11 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) } -func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { - if !m.finishVolumeTeardown(ctx, pending.volumes) { - m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) - return - } +func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { + m.finishVolumeTeardown(ctx, pending.volumes) if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 9f840edf..0fb3270c 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -38,7 +38,7 @@ type resolvedVolume struct { // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - done <-chan struct{} + run *volumeRun unknown bool holds []types.Volume clears []string @@ -247,10 +247,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { m.mu.Lock() - td := volumeTeardown{holds: heldVolumes(sb)} - if run := m.volumeRuns[sb]; run != nil { - td.done = run.done - } + td := volumeTeardown{run: m.volumeRuns[sb], holds: heldVolumes(sb)} m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { @@ -287,15 +284,8 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT return td } -// finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. -func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) bool { - if td.done != nil { - select { - case <-td.done: - default: - return false - } - } +// finishVolumeTeardown runs once the VM is gone; an in-flight volume run releases the holds instead. +func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { logger := log.WithFunc("pool.finishVolumeTeardown") for _, path := range td.clears { if err := clearVolumeDirty(path); err != nil { @@ -303,12 +293,19 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) b } } m.mu.Lock() - m.releaseVolumes(td.holds) + defer m.mu.Unlock() if td.unknown { m.catalog.unknown-- } - m.mu.Unlock() - return true + if td.run != nil { + select { + case <-td.run.done: + default: + td.run.teardown = append(td.run.teardown, td.holds...) + return + } + } + m.releaseVolumes(td.holds) } // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 46b7f946..f3760304 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -43,6 +43,7 @@ func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { case <-time.After(time.Second): t.Fatal("release waited for paused worker") } + m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { t.Errorf("removed VM freed active worker count: %+v", got) } @@ -52,19 +53,17 @@ func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { t.Errorf("unregister overtook paused worker: %v", err) } - m.retryRemovals(t.Context()).Wait() - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { - t.Errorf("maintenance freed active worker count: %+v", got) - } close(eng.resume) resumed = true if err := <-changed; err == nil { t.Fatal("removed claim accepted late attachment") } - m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("completed worker leaked count: %+v", got) } + if n := len(slices.DeleteFunc(base.removedNames(), func(n string) bool { return n != sb.VMName })); n != 1 { + t.Fatalf("vm rm ran %d times, want once", n) + } if err := m.UnregisterVolume(t.Context(), "data"); err != nil { t.Fatal(err) } @@ -350,48 +349,6 @@ func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { } } -func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) { - base := newFakeEngine() - m := newTestManager(t, base, archivePool(3600)) - sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - if err := m.archive(t.Context(), sb); err != nil { - t.Fatal(err) - } - if sb.VMName != "" { - t.Fatalf("archived VM name = %q", sb.VMName) - } - ctx, run, err := m.beginVolumeRun(t.Context(), sb) - if err != nil { - t.Fatal(err) - } - if err := m.checkVolumeRun(ctx, sb, run); !errors.Is(err, ErrArchived) { - t.Fatalf("archived mutation validation: %v", err) - } - td := m.quiesceVolumes(t.Context(), sb) - if td.done == nil || len(td.holds) != 0 || len(td.clears) != 0 || td.unknown { - t.Fatalf("archived teardown has cleanup obligations: %+v", td) - } - if m.finishVolumeTeardown(t.Context(), td) { - t.Fatal("open volume run unexpectedly completed") - } - if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - m.finishVolumeRun(sb, run) - if len(m.pendingRemovals) != 0 { - t.Fatalf("empty teardown enqueued removal: %+v", m.pendingRemovals) - } - if slices.Contains(base.removedNames(), "") { - t.Fatal("release removed an empty VM name") - } - if m.store.referencesVolume("data") { - t.Fatal("release retained archived journal references") - } -} - func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { for _, tc := range []struct { name string diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 596d047f..24dfa517 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -19,9 +19,10 @@ const ( type volumeEdit func([]types.Volume) ([]types.Volume, error) type volumeRun struct { - cancel context.CancelFunc - done chan struct{} - holds []types.Volume + cancel context.CancelFunc + done chan struct{} + holds []types.Volume + teardown []types.Volume } // GetVolumes reads a claim's desired and applied volumes; a tenant reaches only its own claims. @@ -204,6 +205,7 @@ func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.Sandb if m.claimed[sb.ID] == sb { m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) } + m.releaseVolumes(run.teardown) delete(m.volumeRuns, sb) view := m.volumeView(sb) close(run.done) diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 66ebfd4e..86d9074c 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -280,11 +280,8 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { } close(resume) attachErr := <-attached - if action == "release" || action == "reap" { - waitFor(t, func() bool { - m.retryRemovals(t.Context()).Wait() - return volumeHoldersOf(m, "data") == (volumeHolders{}) - }) + if action == "reap" { + waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) } if action == "release" || action == "reap" { if attachErr == nil { @@ -721,7 +718,6 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) - m.retryRemovals(t.Context()).Wait() waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) From e942d310f12afd79dd91e2fe7645070b68dc7794 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:13:27 +0000 Subject: [PATCH 10/29] pool: classify stale VMs by inspection for the volume fence A stopped VM holds no volumes whether or not cocoon marks it stale, and a busy stale create now gets the same classification before it is queued and again when its retry inspects: running with a disk list takes per-volume holds, stopped or absent takes none. Only an Inspect error, a running VM without a disk list (cocoon v0.7.3 omits empty lists and swallows DiskList errors) or any other state keeps the node-wide fence. The table test covers each case by which claims and registrations are refused, with the disk list appearing only after the sweep so both retry paths classify. --- sandboxd/engine/cocoond_test.go | 8 - sandboxd/pool/pool.go | 1 + sandboxd/pool/pool_test.go | 18 +- sandboxd/pool/reconcile.go | 29 +- sandboxd/pool/reconcile_test.go | 3 + sandboxd/pool/remove.go | 6 +- sandboxd/pool/volume_cleanup_test.go | 470 +++++++-------------------- sandboxd/types/types.go | 1 - 8 files changed, 140 insertions(+), 396 deletions(-) diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 398241be..027b264c 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -45,8 +45,6 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { fakeCocoon(t, "#!/bin/sh\n[ \"$1 $2\" = \"vm inspect\" ] || exit 2\n"+ "if [ \"$3\" = sbx-1 ]; then echo '{\"config\":{\"name\":\"sbx-1\"},\"state\":\"running\","+ "\"attached_devices\":{\"disks\":[{\"id\":\"disk-a\",\"name\":\"imagenet\",\"path\":\"/v/a.img\",\"readonly\":true},{\"id\":\"disk-b\",\"name\":\"scratch\",\"path\":\"/v/b.img\"}]}}'; exit 0; fi\n"+ - "if [ \"$3\" = sbx-stale ]; then echo '{\"state\":\"stopped\",\"stale\":true}'; exit 0; fi\n"+ - "if [ \"$3\" = sbx-stopped ]; then echo '{\"state\":\"stopped\"}'; exit 0; fi\n"+ "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") @@ -54,12 +52,6 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } - for _, name := range []string{"sbx-stale", "sbx-stopped"} { - rec, present, err := e.Inspect(t.Context(), name) - if err != nil || !present || rec.State != "stopped" || rec.Stale != (name == "sbx-stale") { - t.Errorf("Inspect %s lost stopped classification: %+v, %v, %v", name, rec, present, err) - } - } if _, ok, err := e.Inspect(t.Context(), "sbx-gone"); err != nil || ok { t.Errorf("Inspect of a missing VM = %v, %v; want not found without an error", ok, err) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index e2cb2391..5c3b3525 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -72,6 +72,7 @@ const ( hibernatePrefix = "sbx-hib-" forkPrefix = "sbx-fork-" vmStateRunning = "running" + vmStateStopped = "stopped" vmStateCreating = "creating" goldenStampSuffix = ".stamp" diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index b66724ce..a372a55a 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1257,9 +1257,6 @@ type fakeEngine struct { liveVolumeHook func(string) error stopped map[string]bool inspectHook func() error - inspectStale bool - inspectDisksUnknown bool - trackDisks bool disks map[string][]types.VMDisk creating map[string]bool staleOutcome engine.StaleCreateOutcome @@ -1544,10 +1541,7 @@ func (f *fakeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, return types.VMRecord{}, false, nil } f.mu.Lock() - vms[i].Stale = f.inspectStale - if !f.inspectDisksUnknown { - vms[i].AttachedDevices.Disks = f.disks[name] - } + vms[i].AttachedDevices.Disks = f.disks[name] f.mu.Unlock() return vms[i], true, nil } @@ -1632,7 +1626,7 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } -func (f *fakeEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { +func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("attach"); err != nil { return err @@ -1649,12 +1643,6 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, name string, spec engine.Vo if spec.Name == f.diskAttachErrFor { err = errors.New("attach failed") } - if err == nil && f.trackDisks { - if f.disks == nil { - f.disks = map[string][]types.VMDisk{} - } - f.disks[name] = append(f.disks[name], types.VMDisk{Name: spec.Name, ReadOnly: !spec.RW}) - } f.mu.Unlock() if cancel != nil { cancel() @@ -1782,7 +1770,7 @@ func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { case f.creating[name]: state = vmStateCreating case f.stopped[name]: - state = "stopped" + state = vmStateStopped } rec := types.VMRecord{State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} if f.tap != "" { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index a7255629..9e3c641e 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -172,7 +172,7 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - m.queueStaleCreate(name, rec.TapDevice(), td) + m.queueStaleCreate(name, rec.TapDevice(), m.resolveStaleVolumes(ctx, name, td)) return false } // not-creating: the record moved on under the lock; remove normally. @@ -196,26 +196,17 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum return td } var holds []types.Volume - if present { - switch inspected.State { - case "stopped": - if inspected.Stale { - return td + switch { + case present && inspected.State == vmStateRunning && inspected.AttachedDevices.Disks != nil: + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW } - case vmStateRunning: - if inspected.AttachedDevices.Disks == nil { - return td - } - holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW - } - } - default: - return td } + case present && inspected.State != vmStateStopped: + return td } m.mu.Lock() m.adoptVolumes(holds) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 6177c19a..b50d78dc 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -181,6 +181,9 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { if !eng.removed("sbx-orphan-1") { t.Error("unowned running VM not removed") } + if eng.inspects != 0 { + t.Errorf("inspected a stale VM whose removal succeeded %d times", eng.inspects) + } } func TestReconcileSweepsInterruptedForkExports(t *testing.T) { diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 5fc199a2..9d104dac 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -90,11 +90,7 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: - if _, present, inspectErr := m.eng.Inspect(ctx, name); inspectErr == nil && !present { - m.finishRemoval(ctx, pending) - return - } - m.queueStaleCreate(name, pending.tap, pending.volumes) + m.queueStaleCreate(name, pending.tap, m.resolveStaleVolumes(ctx, name, pending.volumes)) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: m.finishRemoval(ctx, pending) diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index f3760304..9caf6edb 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -4,12 +4,12 @@ import ( "context" "errors" "slices" + "sync" "sync/atomic" "testing" "testing/synctest" "time" - "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -83,21 +83,9 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { if err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("expected survivor") } - for range cap(m.maintSem) { - m.maintSem <- struct{}{} - } - wg := m.retryRemovals(t.Context()) - m.mu.Lock() - queued := len(m.pendingRemovals) - m.mu.Unlock() - if queued != 0 { - t.Fatal("cleanup was not dispatched") - } + wg := dispatchBlocked(t.Context(), t, m) err = m.UnregisterVolume(t.Context(), "data") - for range cap(m.maintSem) { - <-m.maintSem - } - wg.Wait() + unblock(m, wg) if !errors.Is(err, ErrVolumeRegistrationConflict) { t.Fatalf("dispatched cleanup lost guard: %v", err) } @@ -108,134 +96,119 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { } } -func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { +func TestStaleVMFenceFollowsInspection(t *testing.T) { + writer, reader := []types.VMDisk{{Name: "data"}}, []types.VMDisk{{Name: "data", ReadOnly: true}} + busy, fenced, inspectErr := ErrVolumeBusy, ErrVolumeCatalogUnavailable, errors.New("inspection unavailable") for _, tc := range []struct { - name, state string - inspectError bool + name, swept, listed, state string + disks []types.VMDisk + inspectErr error + want [3]error // data, data rw, other + register error }{ - {"running omitted", "running", false}, - {"running error", "running", true}, - {"inferred stopped omitted", "stopped", false}, - {"inferred stopped error", "stopped", true}, - {"creating omitted", "creating", false}, - {"creating error", "creating", true}, + {"running writer", vmStateRunning, "sbx-stale", vmStateRunning, writer, nil, [3]error{busy, busy, nil}, nil}, + {"running reader", vmStateRunning, "sbx-stale", vmStateRunning, reader, nil, [3]error{nil, busy, nil}, nil}, + {"running empty", vmStateRunning, "sbx-stale", vmStateRunning, []types.VMDisk{}, nil, [3]error{}, nil}, + {"running omitted disks", vmStateRunning, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"stopped", vmStateRunning, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"inspected creating", vmStateRunning, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"inspect error", vmStateRunning, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, + {"busy creating", vmStateCreating, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"busy running", vmStateCreating, "sbx-stale", vmStateRunning, reader, nil, [3]error{nil, busy, nil}, nil}, + {"busy omitted disks", vmStateCreating, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"busy stopped", vmStateCreating, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"busy absent", vmStateCreating, "sbx-other", vmStateRunning, writer, nil, [3]error{}, nil}, + {"busy inspect error", vmStateCreating, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, } { t.Run(tc.name, func(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, nil) - path := writeVolumeImage(t, "runtime", "bytes") - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { - t.Fatal(err) + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-stale" + eng.vms[tc.listed], eng.removeErrFor, eng.staleOutcome = "/vsock/stale", name, engine.StaleCreateBusy + eng.stopped[tc.listed], eng.creating[tc.listed] = tc.state == vmStateStopped, tc.state == vmStateCreating + eng.inspectHook = func() error { return tc.inspectErr } + if m.removeStaleVM(t.Context(), name, types.VMRecord{State: tc.swept, Config: types.VMConfig{Name: name}}) { + t.Fatal("stale VM reported gone") } - name := "sbx-stale-unknown" - base.vms[name] = "/vsock/stale" - base.stopped[name] = tc.state == "stopped" - base.creating[name] = tc.state == "creating" - base.inspectStale = tc.state == "stopped" - base.inspectDisksUnknown = !tc.inspectError - base.staleOutcome = engine.StaleCreateBusy - base.removeErrFor = name - base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} - var inspectFailed atomic.Bool - inspectFailed.Store(true) - base.inspectHook = func() error { - if tc.inspectError && inspectFailed.Load() { - return errors.New("inspection unavailable") + // the disk list appears after the sweep, so running rows are classified by the retry + eng.disks = map[string][]types.VMDisk{name: tc.disks} + m.retryRemovals(t.Context()).Wait() + for i, volume := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "other"}} { + if err := claimAndRelease(t, m, volume); !errors.Is(err, tc.want[i]) { + t.Errorf("claim %+v = %v, want %v", volume, err, tc.want[i]) } - return nil } - rec := base.record(name) - rec.State = tc.state - if m.removeStaleVM(t.Context(), name, rec) { - t.Fatal("unknown stale VM unexpectedly gone") + if err := m.RegisterVolume(t.Context(), "fresh", types.VolumeRegistration{Path: writeVolumeImage(t, "fresh", "bytes")}); !errors.Is(err, tc.register) { + t.Errorf("unrelated registration = %v, want %v", err, tc.register) } - for range 2 { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("unknown stale attachment admitted reader: %v", err) - } - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("unknown stale attachment allowed unregister: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatalf("volume-free claim blocked: %v", err) - } - for range cap(m.maintSem) { - m.maintSem <- struct{}{} - } - wg := m.retryRemovals(t.Context()) - if err := m.RegisterVolume(t.Context(), "new", types.VolumeRegistration{Path: writeVolumeImage(t, "new", "bytes")}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("dispatched unknown fence lost: %v", err) - } - for range cap(m.maintSem) { - <-m.maintSem - } - wg.Wait() - } - inspectFailed.Store(false) - base.stopped[name], base.creating[name] = false, false - base.inspectStale, base.inspectDisksUnknown = false, false - if tc.state == "creating" { - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("busy partial inventory cleared unknown: %v", err) - } + if err := claimAndRelease(t, m); err != nil { + t.Errorf("volume-free claim: %v", err) } - base.staleOutcome = engine.StaleCreateNotCreating + eng.removeErrFor, eng.staleOutcome = "", engine.StaleCreateCollected m.retryRemovals(t.Context()).Wait() - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { - t.Fatalf("inspection did not replace fence with known holds: %+v", got) - } - if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated", "bytes")}); err != nil { - t.Fatalf("resolved inspection left global fence: %v", err) + for _, volume := range []types.Volume{{Name: "data", Mode: types.VolumeModeRW}, {Name: "other", Mode: types.VolumeModeRW}} { + if err := claimAndRelease(t, m, volume); err != nil { + t.Errorf("claim %+v after confirmed removal: %v", volume, err) + } } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("known attachment lost exclusion: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if err := m.UnregisterVolume(t.Context(), "data"); err != nil { - t.Fatalf("confirmed removal retained fence: %v", err) + if m.catalog.unknown != 0 { + t.Fatalf("cleanup retained or double-released its fence: unknown=%d", m.catalog.unknown) } }) } } -func TestRuntimeCatalogLoadedBeforeClaimReconciliation(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - path := writeVolumeImage(t, "runtime", "bytes") - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true, Tenants: []string{"acme"}}); err != nil { - t.Fatal(err) - } - sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) - if claimErr != nil { - t.Fatal(claimErr) +func TestStaleInventoryIsTakenAfterFailedRemoval(t *testing.T) { + sweep := func(ctx context.Context, m *Manager, name string) { + m.removeStaleVM(ctx, name, types.VMRecord{Config: types.VMConfig{Name: name}}) } - m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) - if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "data" || !got[0].Available || !got[0].Writable { - t.Fatalf("catalog absent before reconcile: %+v", got) - } - if local, err := m2.VolumePlacement(testKey, "acme", []string{"data"}); err != nil || !local { - t.Fatalf("placement before reconcile: %v %v", local, err) - } - if err := m2.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - view, err := m2.GetVolumes(sb.ID, "") - if err != nil || !slices.Equal(view.Volumes, sb.Volumes) { - t.Fatalf("claim recovery: %+v %v", view, err) - } - if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("recovered holder missing: %v", err) + retry := func(ctx context.Context, m *Manager, _ string) { m.retryRemovals(ctx).Wait() } + for _, tc := range []struct { + name string + before, after func(context.Context, *Manager, string) + }{{"sweep", func(context.Context, *Manager, string) {}, sweep}, {"retry", sweep, retry}} { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-attach-during-remove" + eng.vms[name], eng.removeErrFor = "/vsock/stale", name + tc.before(t.Context(), m, name) + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} + eng.removeStall = make(chan struct{}) + done := make(chan struct{}) + go func() { defer close(done); tc.after(t.Context(), m, name) }() + synctest.Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted a reader: %v", err) + } + eng.mu.Lock() + eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) + eng.mu.Unlock() + close(eng.removeStall) + <-done + for _, c := range []struct { + volume types.Volume + want error + }{ + {types.Volume{Name: "other", Mode: types.VolumeModeRW}, ErrVolumeBusy}, + {types.Volume{Name: "data", Mode: types.VolumeModeRW}, ErrVolumeBusy}, + {types.Volume{Name: "data"}, nil}, + } { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{c.volume}}); !errors.Is(err, c.want) { + t.Errorf("claim %+v = %v, want %v", c.volume, err, c.want) + } + } + }) + }) } } -func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing.T) { +func TestUnknownFencePrecedesInspection(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) name := "sbx-stale-before-inspect" - base.vms[name] = "/vsock/stale" - base.removeErrFor = name + base.vms[name], base.removeErrFor = "/vsock/stale", name entered, resume := make(chan struct{}), make(chan struct{}) var gated atomic.Bool base.inspectHook = func() error { @@ -248,22 +221,10 @@ func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing done := make(chan bool, 1) go func() { done <- m.removeStaleVM(t.Context(), name, base.record(name)) }() <-entered - if !m.mu.TryLock() { - t.Fatal("stale inspection holds manager lock") - } - m.mu.Unlock() _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) close(resume) - if gone := <-done; gone { - t.Fatal("failed inspect/removal unexpectedly confirmed gone") - } - if !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("admitted volume while inspection was blocked: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("successful removal retained unknown fence: %v", err) + if gone := <-done; gone || !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("admission while inspection was blocked: %v (gone=%t)", err, gone) } } @@ -272,24 +233,20 @@ func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) name := "sbx-canceled-dispatch" - base.vms[name] = "/vsock/stale" - base.removeErrFor = name + base.vms[name], base.removeErrFor = "/vsock/stale", name base.inspectHook = func() error { return errors.New("inspection unavailable") } if m.removeStaleVM(t.Context(), name, base.record(name)) { t.Fatal("expected survivor") } - for range cap(m.maintSem) { - m.maintSem <- struct{}{} - } ctx, cancel := context.WithCancel(t.Context()) - wg := m.retryRemovals(ctx) + wg := dispatchBlocked(ctx, t, m) cancel() synctest.Wait() - base.removeErrFor = "" - for range cap(m.maintSem) { - <-m.maintSem + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("dispatched unknown fence lost: %v", err) } - wg.Wait() + base.removeErrFor = "" + unblock(m, wg) m.retryRemovals(t.Context()).Wait() if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { t.Fatalf("canceled dispatch stranded unknown fence: %v", err) @@ -297,216 +254,33 @@ func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { }) } -func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { - base := newFakeEngine() - name := "sbx-warm-unfinalized" - base.vms[name] = "/vsock/warm" - base.diskAttachErrFor = "broken" - base.removeErrFor = name - rendezvousAttaches(base, 2) - catalog := []config.VolumeSpec{ - {Name: "data", Path: writeVolumeImage(t, "warm-data", "data"), Writable: true}, - {Name: "broken", Path: writeVolumeImage(t, "warm-broken", "broken")}, - } - dir := t.TempDir() - m := newVolumePoolManager(t, base, dir, catalog) - base.inspectDisksUnknown, base.trackDisks = true, true - warm := &types.Sandbox{VMName: name, Key: testKey, VsockSocket: "/vsock/warm"} - m.pools[testKey].warm = append(m.pools[testKey].warm, warm) - _, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{ - {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}, - {Name: "broken", AttachOnly: true}, - }}) - if err == nil { - t.Fatal("expected attach failure before finalization") - } - if warm.ID != "" { - t.Fatal("claim unexpectedly finalized") - } - base.mu.Lock() - attached := slices.Clone(base.disks[name]) - base.attachRendezvous = nil - base.mu.Unlock() - if !slices.ContainsFunc(attached, func(d types.VMDisk) bool { return d.Name == "data" && !d.ReadOnly }) { - t.Fatalf("warm VM has no attached writer: %+v", attached) - } - claims, err := newClaimStore(dir, false).load() - if err != nil || len(claims) != 0 { - t.Fatalf("pre-finalize journal: %+v, %v", claims, err) - } - restarted := newVolumeManagerAt(t, base, dir, catalog) - restarted.eng = base - if err := restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("ambiguous warm survivor admitted competing writer: %v", err) - } - base.removeErrFor = "" - restarted.retryRemovals(t.Context()).Wait() - if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("confirmed removal failed to clear fence: %v", err) +func claimAndRelease(t *testing.T, m *Manager, volumes ...types.Volume) error { + t.Helper() + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}) + if err != nil { + return err } + return m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) } -func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { - for _, tc := range []struct { - name string - stopped, stale bool - disks []types.VMDisk - wantErr error - }{ - {"running unknown", false, false, nil, ErrVolumeCatalogUnavailable}, - {"running empty", false, false, []types.VMDisk{}, nil}, - {"ordinary stopped", true, false, nil, nil}, - {"inferred stopped", true, true, nil, ErrVolumeCatalogUnavailable}, - } { - t.Run(tc.name, func(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, liveCatalog(t)) - name := "sbx-inventory-shape" - base.vms[name], base.removeErrFor = "/vsock/stale", name - base.disks = map[string][]types.VMDisk{name: tc.disks} - base.stopped[name], base.inspectStale = tc.stopped, tc.stale - if m.removeStaleVM(t.Context(), name, base.record(name)) { - t.Fatal("stale VM unexpectedly disappeared") - } - _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) - if !errors.Is(err, tc.wantErr) { - t.Fatalf("admission=%v, want=%v", err, tc.wantErr) - } - if _, queued := m.pendingRemovals[name]; !queued { - t.Fatal("surviving VM lost pending removal") - } - if err := m.RegisterVolume(t.Context(), "before-removal", types.VolumeRegistration{Path: "/images/before-removal"}); !errors.Is(err, tc.wantErr) { - t.Fatalf("registration before removal: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { - t.Fatalf("confirmed removal retained registration fence: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatalf("confirmed removal retained admission fence: %v", err) - } - }) +// dispatchBlocked moves the removal queue into workers parked on a full maintenance semaphore. +func dispatchBlocked(ctx context.Context, t *testing.T, m *Manager) *sync.WaitGroup { + t.Helper() + for range cap(m.maintSem) { + m.maintSem <- struct{}{} } -} - -func TestStaleInventoryRetainsFenceUntilRemovalFails(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-attach-during-remove" - eng.vms[name], eng.removeErrFor = "/vsock/stale", name - eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} - eng.removeStall = make(chan struct{}) - done := make(chan struct{}) - go func() { - defer close(done) - if m.removeStaleVM(t.Context(), name, eng.record(name)) { - t.Error("failed removal reported gone") - } - }() - checkStaleRemovalInventory(t, m, eng, name, done) - }) -} - -func TestStaleInventoryRetainsFenceUntilRetryFails(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-attach-during-retry" - eng.vms[name], eng.removeErrFor = "/vsock/stale", name - eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} - eng.inspectDisksUnknown = true - if m.removeStaleVM(t.Context(), name, eng.record(name)) { - t.Fatal("failed removal reported gone") - } - eng.inspectDisksUnknown = false - eng.removeStall = make(chan struct{}) - done := make(chan struct{}) - go func() { - defer close(done) - m.retryRemovals(t.Context()).Wait() - }() - checkStaleRemovalInventory(t, m, eng, name, done) - }) -} - -func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { - for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { - t.Run(string(outcome), func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-busy-create" - eng.vms[name], eng.creating[name] = "/vsock/creating", true - eng.staleOutcome = engine.StaleCreateBusy - eng.disks = map[string][]types.VMDisk{name: {{Name: "other"}}} - rec := eng.record(name) - rec.State = vmStateCreating - if m.removeStaleVM(t.Context(), name, rec) { - t.Fatal("busy create reported gone") - } - eng.creating[name] = false - for _, stopped := range []bool{false, true} { - eng.stopped[name] = stopped - before := eng.inspects - m.retryRemovals(t.Context()).Wait() - if eng.inspects != before+1 { - t.Fatal("busy retry did not inspect") - } - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("busy create admitted a volume: %v", err) - } - eng.staleOutcome = outcome - if outcome == engine.StaleCreateBusy { - delete(eng.vms, name) - } - m.retryRemovals(t.Context()).Wait() - m.retryRemovals(t.Context()).Wait() - if m.catalog.unknown != 0 || len(m.pendingRemovals) != 0 { - t.Fatalf("cleanup retained or double-released its fence: unknown=%d pending=%d", m.catalog.unknown, len(m.pendingRemovals)) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatalf("confirmed cleanup retained fence: %v", err) - } - }) + wg := m.retryRemovals(ctx) + m.mu.Lock() + defer m.mu.Unlock() + if len(m.pendingRemovals) != 0 { + t.Fatal("cleanup was not dispatched") } + return wg } -func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { - t.Helper() - synctest.Wait() - registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted a reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted registration: %v", err) - } - eng.mu.Lock() - eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) - eng.mu.Unlock() - close(eng.removeStall) - <-done - if _, queued := m.pendingRemovals[name]; !queued { - t.Fatal("failed removal lost its retry") - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("writer attached during removal lost its hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("failed removal lost its reader hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Errorf("failed removal blocked a compatible reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { - t.Fatalf("failed removal blocked unrelated registration: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { - t.Errorf("failed removal blocked an unrelated reader: %v", err) +func unblock(m *Manager, wg *sync.WaitGroup) { + for range cap(m.maintSem) { + <-m.maintSem } + wg.Wait() } diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 2a93ab78..4cc8d307 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -731,7 +731,6 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { - Stale bool `json:"stale,omitzero"` State string `json:"state"` PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` From de605a5bd82620c4298ab5e04f898bb1393d2892 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:14:10 +0000 Subject: [PATCH 11/29] pool: make the resolved volume check side-effect free The only owner caller checked its run under the same lock, so the check takes the owner ID instead of re-deriving it. It no longer adopts holds: each caller adopts the applied list it already holds and will store, under the same m.mu, so admission neither allocates nor takes a second list that must mirror the resolved one. --- sandboxd/pool/claim.go | 10 +++++++--- sandboxd/pool/volume.go | 9 ++------- sandboxd/pool/volume_bench_test.go | 6 ++++-- sandboxd/pool/volume_convergence_test.go | 4 +++- sandboxd/pool/volume_mutation.go | 3 ++- sandboxd/pool/volume_mutation_test.go | 3 ++- sandboxd/pool/volume_rw_test.go | 11 ++++------- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 9d9110bb..18ace6c0 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -371,7 +371,7 @@ func (m *Manager) overQuota(extra int, tenant string) error { // admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) - if err := m.admitClaim(tenant, volumes); err != nil { + if err := m.admitClaim(tenant, volumes, applied); err != nil { return nil, err } if err := m.confirmVolumesClean(volumes); err != nil { @@ -382,13 +382,17 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types } // admitClaim refuses a busy volume or an over-quota tenant before any VM is built. -func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { +func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume, applied []types.Volume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveResolved(volumes, nil) + if err := m.checkResolved(volumes, ""); err != nil { + return err + } + m.adoptVolumes(applied) + return nil } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 0fb3270c..659b54c6 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -353,12 +353,8 @@ func (m *Manager) catalogSnapshot() map[string]*catalogVolume { return m.catalog.entries } -// reserveResolved admits the entire request while callers hold m.mu. -func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox) error { - ownerID := "" - if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { - ownerID = owner.ID - } +// checkResolved refuses the request if any volume is fenced, rebound or busy; callers hold m.mu. +func (m *Manager) checkResolved(volumes []resolvedVolume, ownerID string) error { for _, resolved := range volumes { volume := resolved.applied if err := m.catalog.admissionError(volume.Name); err != nil { @@ -375,7 +371,6 @@ func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) } } - m.adoptVolumes(appliedVolumes(volumes)) return nil } diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go index 22242076..e717d96f 100644 --- a/sandboxd/pool/volume_bench_test.go +++ b/sandboxd/pool/volume_bench_test.go @@ -24,10 +24,12 @@ func BenchmarkVolumeAdmissionWithCheckpoints(b *testing.B) { if len(resolved) != 1 || resolved[0].applied != requested[0] { b.Fatalf("resolved volumes: %+v", resolved) } + applied := appliedVolumes(resolved) m.mu.Lock() - err = m.reserveResolved(resolved, nil) + err = m.checkResolved(resolved, "") if err == nil { - m.releaseVolumes(appliedVolumes(resolved)) + m.adoptVolumes(applied) + m.releaseVolumes(applied) } m.mu.Unlock() if err != nil { diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 4446957a..78b38b34 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -355,7 +355,9 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { t.Fatal(err) } m.mu.Lock() - err = m.reserveResolved(resolved, nil) + if err = m.checkResolved(resolved, ""); err == nil { + m.adoptVolumes(held) + } run.holds = held m.mu.Unlock() if err != nil { diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 24dfa517..a20b0b59 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -274,11 +274,12 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveResolved(resolved, sb); err != nil { + if err := m.checkResolved(resolved, sb.ID); err != nil { m.mu.Unlock() sb.Transition.Unlock() return claimSnapshot{}, false, err } + m.adoptVolumes(additions) run.holds = append(run.holds, additions...) m.mu.Unlock() sb.Transition.Unlock() diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 86d9074c..c25fbf27 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -857,9 +857,10 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume } m.mu.Lock() defer m.mu.Unlock() - if err = m.reserveResolved(resolved, nil); err != nil { + if err = m.checkResolved(resolved, ""); err != nil { t.Fatal(err) } + m.adoptVolumes(transient) run.holds = append(run.holds, transient...) return run } diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index dadeda8c..fda04c98 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -152,10 +152,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { if markErr := markVolumeDirty(path); markErr != nil { t.Fatalf("mark dirty: %v", markErr) } - m.mu.Lock() - reserveErr := m.reserveResolved(resolved, nil) - m.mu.Unlock() - if reserveErr != nil { + if reserveErr := m.admitClaim("", resolved, appliedVolumes(resolved)); reserveErr != nil { t.Fatalf("reserve once the writer released: %v", reserveErr) } @@ -227,13 +224,13 @@ func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { t.Fatal(err) } m.mu.Lock() - defer m.mu.Unlock() m.adoptVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch", Mode: types.VolumeModeRW}}) + m.mu.Unlock() - if err := m.reserveResolved(resolved, nil); !errors.Is(err, ErrVolumeBusy) { + if err := m.admitClaim("", resolved, appliedVolumes(resolved)); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } - if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { + if holders := volumeHoldersOf(m, "scratch"); holders != (volumeHolders{writers: 1}) { t.Errorf("registry after the refusal=%+v, want the writer alone", holders) } } From 969b7abbc8110ead9564ccaa82c8a0a2b3958996 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:14:43 +0000 Subject: [PATCH 12/29] pool: count catalog references from holders and the journal Every claimed volume is already counted in the admission holders, and the claim store keeps a per-name count for unacknowledged references, so the walk over every claim under m.mu adds nothing. --- sandboxd/pool/volume_catalog.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 4cd0c302..ee5ba8b8 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -150,15 +150,7 @@ func (m *Manager) loadRuntimeVolumes() error { } func (m *Manager) volumeReferenced(name string) bool { - if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { - return true - } - for _, sb := range m.claimed { - if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { - return true - } - } - return m.store.referencesVolume(name) + return m.catalog.holders[name] != (volumeHolders{}) || m.store.referencesVolume(name) } func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[string]*catalogVolume) error { From 1cee58e20db60756b72ac95ba08eeea53379880f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:15:05 +0000 Subject: [PATCH 13/29] pool: hold Transition across the volume clean check again This reverts the Transition release around confirmVolumesClean that "pool: persist runtime volume registrations" introduced earlier in this PR. The clean check is one marker stat; dropping Transition around it let a lifecycle change interleave between reservation and staging for no gain. --- sandboxd/pool/volume_mutation.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index a20b0b59..01d64821 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -263,29 +263,24 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() - sb.Transition.Unlock() return claimSnapshot{}, false, err } if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() - sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } if err := m.checkResolved(resolved, sb.ID); err != nil { m.mu.Unlock() - sb.Transition.Unlock() return claimSnapshot{}, false, err } m.adoptVolumes(additions) run.holds = append(run.holds, additions...) m.mu.Unlock() - sb.Transition.Unlock() cleanErr := m.confirmVolumesClean(resolved) - sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { From 14429edf104195f27b309478a727869e4d95a2ae Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:15:49 +0000 Subject: [PATCH 14/29] pool: drop duplicated volume registration checks The pool already rejects an invalid name with the same 400, so the handler check goes; the pool now validates before touching the pending catalog. The handler maps pool errors before context errors, so an uncertain catalog write still answers 503 without its own case. config.Load defaults static directio, and normalization builds the catalog entry directly instead of round-tripping through one-caller registration helpers. Marshaling the catalog record cannot fail: every string in it came through a UTF-8 validating JSON decode or a name regex. --- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume.go | 4 -- sandboxd/pool/volume_catalog.go | 66 ++++++++------------- sandboxd/pool/volume_catalog_test.go | 4 +- sandboxd/server/volume_registration.go | 8 +-- sandboxd/server/volume_registration_test.go | 38 ------------ 6 files changed, 29 insertions(+), 93 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 5c3b3525..94c10e83 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -480,7 +480,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } for _, volume := range cfg.Volumes { m.catalog.entries[volume.Name] = &catalogVolume{ - disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, + disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, tenants: slices.Clone(volume.Tenants), writable: volume.Writable, } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 659b54c6..0f04eb42 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -55,10 +55,6 @@ func (v *catalogVolume) allowedFor(tenant string) bool { return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) } -func (v catalogVolume) registration() types.VolumeRegistration { - return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.tenants} -} - // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { catalog := m.catalogSnapshot() diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index ee5ba8b8..938fa687 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -50,13 +50,13 @@ func (c *volumeCatalog) admissionError(name string) error { // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { - m.catalog.writeMu.Lock() - defer m.catalog.writeMu.Unlock() - if err := m.commitPendingCatalog(ctx); err != nil { + volume, err := normalizeRegistration(name, spec) + if err != nil { return err } - normalized, err := normalizeRegistration(name, spec) - if err != nil { + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err = m.commitPendingCatalog(ctx); err != nil { return err } m.mu.Lock() @@ -67,34 +67,29 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo return err } if entry := entries[name]; entry != nil { - if entry.runtime && sameRegistration(entry.registration(), normalized) { + if entry.runtime && entry.disk == volume.disk && entry.writable == volume.writable && slices.Equal(entry.tenants, volume.tenants) { return nil } return fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) } - if err := catalogPathConflict(entries, normalized.Path); err != nil { + if err := catalogPathConflict(entries, volume.disk.Path); err != nil { return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) } entries = maps.Clone(entries) - entries[name] = &catalogVolume{ - runtime: true, - disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, - tenants: normalized.Tenants, - writable: normalized.Writable, - } + entries[name] = volume return m.writeCatalog(ctx, name, entries) } // UnregisterVolume retires an unreferenced runtime binding without touching its backing. func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { + if !types.ValidVolumeName(name) { + return fmt.Errorf("%w: invalid volume name", ErrBadVolume) + } m.catalog.writeMu.Lock() defer m.catalog.writeMu.Unlock() if err := m.commitPendingCatalog(ctx); err != nil { return err } - if !types.ValidVolumeName(name) { - return fmt.Errorf("%w: invalid volume name", ErrBadVolume) - } m.mu.Lock() entries := m.catalog.entries entry := entries[name] @@ -128,22 +123,17 @@ func (m *Manager) loadRuntimeVolumes() error { } entries := maps.Clone(m.catalog.entries) for name, spec := range record.Volumes { - normalized, normalizeErr := normalizeRegistration(name, spec) + volume, normalizeErr := normalizeRegistration(name, spec) if normalizeErr != nil { return fmt.Errorf("runtime volume catalog: %w", normalizeErr) } if m.catalog.entries[name] != nil { return fmt.Errorf("runtime volume catalog duplicates static name %q", name) } - if err := catalogPathConflict(entries, normalized.Path); err != nil { + if err := catalogPathConflict(entries, volume.disk.Path); err != nil { return fmt.Errorf("runtime volume catalog: %w", err) } - entries[name] = &catalogVolume{ - runtime: true, - disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, - tenants: normalized.Tenants, - writable: normalized.Writable, - } + entries[name] = volume } m.catalog.entries = entries return nil @@ -157,16 +147,13 @@ func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[str record := runtimeCatalog{Version: 1, Volumes: map[string]types.VolumeRegistration{}} for key, entry := range entries { if entry.runtime { - record.Volumes[key] = entry.registration() + record.Volumes[key] = types.VolumeRegistration{Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, Writable: entry.writable, Tenants: entry.tenants} } } - raw, err := json.Marshal(record, json.Deterministic(true)) - if err != nil { - return fmt.Errorf("%w: %w", ErrBadVolume, err) - } + raw, _ := json.Marshal(record, json.Deterministic(true)) proposal := &catalogWrite{name: name, entries: entries, raw: raw} m.mu.Lock() - err = ctx.Err() + err := ctx.Err() if err == nil && m.volumeReferenced(name) { err = fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) } @@ -223,20 +210,17 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { +func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, }}); err != nil { - return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) + return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } - spec.Path = filepath.Clean(spec.Path) - spec.Tenants = slices.Clone(spec.Tenants) - slices.Sort(spec.Tenants) - spec.Tenants = slices.Compact(spec.Tenants) - return spec, nil -} - -func sameRegistration(a, b types.VolumeRegistration) bool { - return a.Path == b.Path && a.DirectIO == b.DirectIO && a.Writable == b.Writable && slices.Equal(a.Tenants, b.Tenants) + return &catalogVolume{ + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: filepath.Clean(spec.Path), DirectIO: spec.DirectIO}, + tenants: slices.Compact(slices.Sorted(slices.Values(spec.Tenants))), + writable: spec.Writable, + }, nil } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 928b6b18..de250141 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -364,10 +364,10 @@ func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { func TestRuntimeRegistrationNormalization(t *testing.T) { input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} normalized, err := normalizeRegistration("data", input) - if err != nil || normalized.Path != "/images/data" || normalized.DirectIO != types.DirectIOOff || !slices.Equal(normalized.Tenants, []string{"acme", "beta"}) { + if err != nil || normalized.disk.Path != "/images/data" || normalized.disk.DirectIO != types.DirectIOOff || !slices.Equal(normalized.tenants, []string{"acme", "beta"}) { t.Fatalf("normalization: %+v %v", normalized, err) } - normalized.Tenants[0] = "caller-copy" + normalized.tenants[0] = "caller-copy" if input.Tenants[1] != "acme" { t.Fatal("normalization mutated caller input") } diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go index 39bad6a6..29fae3ab 100644 --- a/sandboxd/server/volume_registration.go +++ b/sandboxd/server/volume_registration.go @@ -5,7 +5,6 @@ import ( "errors" "net/http" - "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -15,10 +14,6 @@ func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request return } name := r.PathValue("name") - if !types.ValidVolumeName(name) { - writeErr(w, http.StatusBadRequest, "invalid volume name") - return - } var err error op := "unregister volume" if r.Method == http.MethodPut { @@ -32,8 +27,7 @@ func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request err = s.mgr.UnregisterVolume(r.Context(), name) } switch { - case errors.Is(err, pool.ErrVolumeCatalogUnavailable): - writePoolErr(w, err) + case writePoolErr(w, err): case errors.Is(err, context.DeadlineExceeded): writeErr(w, http.StatusGatewayTimeout, op+" timed out") case errors.Is(err, context.Canceled): diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index 99ff6ac7..ef2c48f9 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -165,44 +165,6 @@ func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { } } -func TestVolumeRegistrationRoutesValidateNames(t *testing.T) { - for _, method := range []string{http.MethodPut, http.MethodDelete} { - for _, tt := range []struct { - name, encoded string - valid bool - }{ - {"data", "data", true}, - {"a_b-0", "a_b-0", true}, - {"abcdefghijklmnopqrst", "abcdefghijklmnopqrst", true}, - {"Data", "Data", false}, - {"1data", "1data", false}, - {"cocoon-data", "cocoon-data", false}, - {"abcdefghijklmnopqrstu", "abcdefghijklmnopqrstu", false}, - {"data/set", "data%2Fset", false}, - {"data set", "data%20set", false}, - } { - t.Run(method+"/"+tt.name, func(t *testing.T) { - var gotName string - mgr := &fakeManager{ - registerVolume: func(_ context.Context, name string, _ types.VolumeRegistration) error { gotName = name; return nil }, - unregisterVolume: func(_ context.Context, name string) error { gotName = name; return nil }, - } - ts := newTestServer(t, "root", mgr, nil) - resp := doReq(t, method, ts.URL+"/v1/volumes/"+tt.encoded, "root", `{"path":"/private/data.img"}`) - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - wantCode, wantName := http.StatusBadRequest, "" - if tt.valid { - wantCode, wantName = http.StatusNoContent, tt.name - } - if resp.StatusCode != wantCode || gotName != wantName { - t.Fatalf("status=%d name=%q body=%s, want %d name=%q", resp.StatusCode, gotName, body, wantCode, wantName) - } - }) - } - } -} - func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { for _, method := range []string{http.MethodPut, http.MethodDelete} { for _, tt := range []struct { From 9068fc035be56e40753d1234e5cdeac377491350 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:16:48 +0000 Subject: [PATCH 15/29] pool: compact runtime volume tests Merge overlapping catalog, ACL, retirement and server tests into tables without per-case branching, share one failed-detach helper, and restore the base claim save helper and capture-test fixtures instead of rewriting their call sites. Drop cases for record file guards that no longer exist, rows that re-test the shared strict body decoder or static path validation, and a config test the pool ACL test covers. Every refusal and recovery path keeps a behavioural assertion. --- e2e/volume_catalog_test.go | 106 ++-- sandboxd/config/config_test.go | 16 - sandboxd/pool/archive_test.go | 3 +- sandboxd/pool/checkpoint_bench_test.go | 4 +- sandboxd/pool/checkpoint_references_test.go | 72 +-- sandboxd/pool/claims_bench_test.go | 6 +- sandboxd/pool/claims_test.go | 6 +- sandboxd/pool/failover_test.go | 9 +- sandboxd/pool/hibernate_test.go | 12 +- sandboxd/pool/liveness_test.go | 3 +- sandboxd/pool/pool_test.go | 3 +- sandboxd/pool/replica_test.go | 23 +- sandboxd/pool/volume_acl_test.go | 222 ++----- sandboxd/pool/volume_bench_test.go | 2 +- sandboxd/pool/volume_capture_test.go | 9 - sandboxd/pool/volume_catalog_test.go | 618 ++++++-------------- sandboxd/pool/volume_convergence_test.go | 42 +- sandboxd/pool/volume_mutation_test.go | 11 +- sandboxd/pool/volume_retirement_test.go | 391 +++++-------- sandboxd/pool/volume_static_acl_test.go | 42 -- sandboxd/server/server_test.go | 11 +- sandboxd/server/volume_registration_test.go | 164 ++---- 22 files changed, 492 insertions(+), 1283 deletions(-) delete mode 100644 sandboxd/pool/volume_static_acl_test.go diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go index 08c459fc..3858f3e5 100644 --- a/e2e/volume_catalog_test.go +++ b/e2e/volume_catalog_test.go @@ -11,102 +11,64 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) -func TestRuntimeVolumePublicAdmissionThroughHTTP(t *testing.T) { - st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) - acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) - if err != nil { - t.Fatal(err) - } - request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "shared bytes"), "writable": true}) - if err != nil { - t.Fatal(err) - } - for _, scope := range []struct { - token string - want int - }{{"acme-token", http.StatusForbidden}, {"", http.StatusUnauthorized}, {"node-token", http.StatusNoContent}} { - caller := *st - caller.token = scope.token - if status, body := rawJSON(t, &caller, http.MethodPut, "/v1/volumes/home", string(request)); status != scope.want { - t.Fatalf("registration auth: status=%d want=%d body=%s", status, scope.want, body) - } - } - volumes, err := acme.Volumes(t.Context()) - if err != nil || len(volumes) != 1 || volumes[0].Name != "home" { - t.Fatalf("empty runtime ACL is public: volumes=%v err=%v", volumes, err) - } - volume := sandbox.Volume{Name: "home", Mount: "/mnt/home"} - first, err := acme.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volume)) - if err != nil { - t.Fatalf("tenant claim with public runtime volume: %v", err) - } - defer first.Close() - if !slices.Equal(first.Volumes, []sandbox.Volume{volume}) { - t.Fatalf("claim did not retain mounted volume: %+v", first.Volumes) - } - if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { - t.Fatal(err) - } - if view, attachErr := first.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { - t.Fatalf("tenant PATCH of public runtime volume: view=%+v err=%v", view, attachErr) - } - if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { - t.Fatal(err) - } - if view, attachErr := first.SetVolumes(t.Context(), []sandbox.Volume{volume}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { - t.Fatalf("tenant PUT of public runtime volume: view=%+v err=%v", view, attachErr) - } -} - -func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { +func TestRuntimeVolumeRegistrationThroughHTTP(t *testing.T) { st := startTenantStack(t, "node-token", []tenants.TenantSpec{ {Name: "acme", Token: "acme-token"}, {Name: "beta", Token: "beta-token"}, }, []config.VolumeSpec{{Name: "public", Path: writeVolumeImage(t, "public.img", "shared")}}) - request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}) - if err != nil { - t.Fatal(err) - } - if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/private", string(request)); status != http.StatusNoContent { - t.Fatalf("registration: %d %s", status, body) + for name, body := range map[string]map[string]any{ + "home": {"path": writeVolumeImage(t, "home.img", "shared"), "writable": true}, + "private": {"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}, + } { + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + if status, resp := rawJSON(t, st, http.MethodPut, "/v1/volumes/"+name, string(raw)); status != http.StatusNoContent { + t.Fatalf("register %s: status=%d body=%s", name, status, resp) + } } + home := sandbox.Volume{Name: "home", Mount: "/mnt/home"} for _, tc := range []struct { token string want []string - allowed bool - }{{"acme-token", []string{"private", "public"}, true}, {"beta-token", []string{"public"}, false}} { - client, connectErr := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) - if connectErr != nil { - t.Fatal(connectErr) + private bool + }{{"acme-token", []string{"home", "private", "public"}, true}, {"beta-token", []string{"home", "public"}, false}} { + client, err := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) + if err != nil { + t.Fatal(err) } - volumes, listErr := client.Volumes(t.Context()) - var names []string + volumes, err := client.Volumes(t.Context()) + names := make([]string, 0, len(volumes)) for _, volume := range volumes { names = append(names, volume.Name) } - if listErr != nil || !slices.Equal(names, tc.want) { - t.Fatalf("catalog ACL: names=%v want=%v err=%v", names, tc.want, listErr) + if err != nil || !slices.Equal(names, tc.want) { + t.Fatalf("catalog for %s: names=%v want=%v err=%v", tc.token, names, tc.want, err) } - claim, claimErr := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(sandbox.Volume{Name: "public"})) - if claimErr != nil { - t.Fatal(claimErr) + claim, err := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(home)) + if err != nil || !slices.Equal(claim.Volumes, []sandbox.Volume{home}) { + t.Fatalf("claim with public runtime volume: %+v, %v", claim, err) } defer claim.Close() - _, attachErr := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{{Name: "private"}}}) - if (attachErr == nil) != tc.allowed { - t.Fatalf("explicit ACL admission for %s: %v", tc.token, attachErr) + if _, err = claim.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { + t.Fatal(err) + } + view, err := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{home, {Name: "private"}}}) + if (err == nil) != tc.private || tc.private && !slices.Contains(view.Volumes, home) { + t.Fatalf("live attach for %s: view=%+v err=%v", tc.token, view, err) } } caller := *st caller.token = "beta-token" - var responses [][]byte + var responses []string for _, name := range []string{"private", "unknown"} { status, body := rawJSON(t, &caller, http.MethodPost, "/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"`+name+`"}]}`) if status != http.StatusBadRequest { t.Fatalf("claim privacy for %s: status=%d body=%s", name, status, body) } - responses = append(responses, body) + responses = append(responses, string(body)) } - if string(responses[0]) != string(responses[1]) { + if responses[0] != responses[1] { t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) } } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index ce5d5766..144c378a 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -476,22 +476,6 @@ func TestValidateVolumeCatalogTenantNames(t *testing.T) { } } -func TestLoadPreservesRestrictedACLWithInvalidVolumeTenants(t *testing.T) { - for _, tenants := range []string{`["acme", "has space", "", "beta"]`, `["has space"]`} { - cfg, err := Load(writeConfig(t, `{"pools":[],"volumes":[{"name":"data","path":"/images/data","tenants":`+tenants+`}]}`)) - if err != nil { - t.Fatal(err) - } - want := []string{"has space"} - if strings.Contains(tenants, "acme") { - want = []string{"acme", "has space", "", "beta"} - } - if !slices.Equal(cfg.Volumes[0].Tenants, want) { - t.Fatalf("tenants = %v, want %v", cfg.Volumes[0].Tenants, want) - } - } -} - func writeConfig(t *testing.T, body string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.json") diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 5cd3aa11..59064b8b 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -155,8 +155,7 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_mine": {ID: "sb_mine", VMName: "sbx-gone-1", Key: testKey, Token: "tok"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) diff --git a/sandboxd/pool/checkpoint_bench_test.go b/sandboxd/pool/checkpoint_bench_test.go index 997f0e77..e3ba1b6c 100644 --- a/sandboxd/pool/checkpoint_bench_test.go +++ b/sandboxd/pool/checkpoint_bench_test.go @@ -15,7 +15,7 @@ func BenchmarkClaimCommitWithCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { claims := checkpointBenchClaims(checkpoints) s := newClaimStore(b.TempDir(), false) - if err := s.commit(s.reset(claims)); err != nil { + if err := s.save(claims); err != nil { b.Fatal(err) } one := claims["sb_0000000000000000"] @@ -39,7 +39,7 @@ func BenchmarkPinnedArchiveCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.commit(m.store.reset(m.claimed)); err != nil { + if err := m.store.save(m.claimed); err != nil { b.Fatal(err) } want := 0 diff --git a/sandboxd/pool/checkpoint_references_test.go b/sandboxd/pool/checkpoint_references_test.go index 94601f8d..c3ec0ae3 100644 --- a/sandboxd/pool/checkpoint_references_test.go +++ b/sandboxd/pool/checkpoint_references_test.go @@ -6,7 +6,6 @@ import ( "io/fs" "maps" "os" - "path/filepath" "slices" "sync" "syscall" @@ -15,11 +14,17 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +const ( + failWrite = iota + captureWrite + failAfterRename +) + func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { recreate := func(s *claimStore) { s.set(&types.Sandbox{ID: "owner", ReplicaCk: "recreated"}) } for _, tc := range []struct { name string - fault func(*testing.T, *claimStore, claimSnapshot) func() + fault int acked, next *types.Sandbox startup func(claims map[string]*types.Sandbox) mutate func(s *claimStore) @@ -82,8 +87,11 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + if tc.fault == failAfterRename && os.Geteuid() == 0 { + t.Skip("requires non-root: root can read the write-only directory") + } dir := t.TempDir() - s := newClaimStore(dir, false) + s := newClaimStore(dir, tc.fault == failAfterRename) if tc.acked != nil { if err := s.commit(s.set(tc.acked)); err != nil { t.Fatal(err) @@ -101,7 +109,26 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { } else { snap = s.set(tc.next) } - finish := tc.fault(t, s, snap) + var finish func() + switch tc.fault { + case failWrite: + finish = failClaimWrites(t, s) + if err := s.commit(snap); err == nil { + t.Fatal("write succeeded under the fault") + } + case failAfterRename: + if err := os.Chmod(dir, 0o300); err != nil { + t.Fatal(err) + } + finish = func() { _ = os.Chmod(dir, 0o700) } + t.Cleanup(finish) + if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { + t.Fatalf("want a failed directory sync after the rename: %v", err) + } + case captureWrite: + release := captureClaimWrite(t, s, snap, "owner") + finish = func() { release() } + } if tc.mutate != nil { tc.mutate(s) } @@ -188,40 +215,3 @@ func captureClaimWrite(t *testing.T, s *claimStore, snap claimSnapshot, owner st return raw } } - -func failWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { - t.Helper() - restore := failClaimWrites(t, s) - if err := s.commit(snap); err == nil { - t.Fatal("write succeeded under the fault") - } - return restore -} - -func failAfterRename(t *testing.T, s *claimStore, snap claimSnapshot) func() { - t.Helper() - if os.Geteuid() == 0 { - t.Skip("requires non-root: root can read the write-only directory") - } - s.sync = true - dir := filepath.Dir(s.path) - if err := os.Chmod(dir, 0o300); err != nil { - t.Fatal(err) - } - restore := func() { - if err := os.Chmod(dir, 0o700); err != nil { - t.Error(err) - } - } - t.Cleanup(restore) - if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { - t.Fatalf("want a failed directory sync after the rename: %v", err) - } - return restore -} - -func captureWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { - t.Helper() - release := captureClaimWrite(t, s, snap, "owner") - return func() { release() } -} diff --git a/sandboxd/pool/claims_bench_test.go b/sandboxd/pool/claims_bench_test.go index d1f059bc..b1ae79da 100644 --- a/sandboxd/pool/claims_bench_test.go +++ b/sandboxd/pool/claims_bench_test.go @@ -18,7 +18,7 @@ func BenchmarkStoreSaveScaling(b *testing.B) { st := newClaimStore(b.TempDir(), false) claims := benchClaims(n) for b.Loop() { - if err := st.commit(st.reset(claims)); err != nil { + if err := st.save(claims); err != nil { b.Fatalf("save: %v", err) } } @@ -48,7 +48,7 @@ func BenchmarkStoreCommitEnv(b *testing.B) { sb.Env = envs[arm] } s := newClaimStore(b.TempDir(), durable) - if err := s.commit(s.reset(claims)); err != nil { + if err := s.save(claims); err != nil { b.Fatalf("save: %v", err) } one := claims[fmt.Sprintf("sb_%016x", 0)] @@ -105,7 +105,7 @@ func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, arm switch arm { case "under-lock": mu.Lock() - _ = s.commit(s.reset(claims)) + _ = s.save(claims) mu.Unlock() case "rebuild": mu.Lock() diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index d369f829..8ddae7aa 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -85,7 +85,7 @@ func TestStoreRoundTrip(t *testing.T) { } for _, fsync := range []bool{false, true} { s := newClaimStore(t.TempDir(), fsync) - if err := s.commit(s.reset(claims)); err != nil { + if err := s.save(claims); err != nil { t.Fatalf("save (sync=%v): %v", fsync, err) } loadedStore := newClaimStore(filepath.Dir(s.path), false) @@ -239,3 +239,7 @@ func TestClaimStoreConcurrentCommits(t *testing.T) { t.Fatalf("after concurrent commits: %+v, %v", got, err) } } + +func (s *claimStore) save(claims map[string]*types.Sandbox) error { + return s.commit(s.reset(claims)) +} diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index dd37a6bc..505dc55c 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -272,8 +272,7 @@ func TestFailoverRestartReclaimsAnInterruptedArchive(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - claimStore := newClaimStore(dirA, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dirA, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -296,8 +295,7 @@ func TestFailoverRestartReclaimSparesACheckpointAdoptedMidReconcile(t *testing.T claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - claimStore := newClaimStore(dirA, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dirA, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -322,8 +320,7 @@ func TestFailoverRestartReclaimsOnlyTheCheckpointsItPublished(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - claimStore := newClaimStore(dirA, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dirA, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index 7642b11a..ffc4dd9e 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -528,8 +528,7 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { VsockSocket: "/vsock/lagged", PendingSnap: hibernatePrefix + "lagged-1-abc123", }, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -558,8 +557,7 @@ func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_orp": {ID: "sb_orp", VMName: "sbx-orphan-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/orphan"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -585,8 +583,7 @@ func TestReconcileKeepsIntentWithoutImageAsRunning(t *testing.T) { VsockSocket: "/vsock/noimg", PendingSnap: hibernatePrefix + "noimg-1-abc123", }, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -616,8 +613,7 @@ func TestReconcileAdoptsHibernated(t *testing.T) { HibernateSnap: hibernatePrefix + "hibernated-1", }, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index ac557fb2..2864c615 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -486,8 +486,7 @@ func TestReconcileKeepsRunningClaimsTheHostLost(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_lost": {ID: "sb_lost", VMName: "sbx-lost-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/lost"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } cfg := &config.Config{DataDir: dataDir, VMMRestart: tt.policy} diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index a372a55a..b29c4e86 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -323,8 +323,7 @@ func TestReconcile(t *testing.T) { "sb_live": {ID: "sb_live", VMName: "sbx-live-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/live-old"}, "sb_dead": {ID: "sb_dead", VMName: "sbx-dead-1", Key: testKey, Token: "tok"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index b56103dc..f7bd658f 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -4,7 +4,6 @@ import ( "encoding/json/v2" "errors" "net" - "os" "testing" "testing/synctest" "time" @@ -1015,34 +1014,20 @@ func TestReplicaScanWaitsForDetachAcknowledgment(t *testing.T) { t.Fatal(err) } } - m.store.recommitting.Store(true) eng := m.eng.(*fakeEngine) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - view, err := m.SetVolumes(t.Context(), sb.ID, "", nil) - eng.liveVolumeHook = nil - if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("detach did not fail after device removal: %+v, %v", view, err) - } + unblock := detachWithoutCommit(t, m, eng, sb.ID, "") m.replicateOnce(t.Context()) if m.replicating.Load() || !m.replicas[sb.ID].skipLogged || !m.replicas[attached.ID].skipLogged || m.replicas[ordinary.ID].tried.IsZero() || m.replicas[ordinary.ID].skipLogged { t.Fatalf("scan failed to distinguish detached, attached, and ordinary claims: %+v", m.replicas) } - if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("capture ignored retained attachment: %v", err) } if sb.HasReplica() || len(eng.snapSaves) != 0 { t.Fatal("pending detach published a replica") } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) - } - if err = m.store.commit(m.store.mark()); err != nil { + unblock() + if err := m.store.commit(m.store.mark()); err != nil { t.Fatal(err) } m.replicateOnce(t.Context()) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index a7b30825..f4136667 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -1,205 +1,55 @@ package pool import ( - "errors" + "fmt" "os" - "slices" + "path/filepath" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestRuntimeCatalogDefaultACL(t *testing.T) { - for _, tenants := range [][]string{nil, {}} { - t.Run("empty", func(t *testing.T) { - eng := newFakeEngine() - catalog := liveCatalog(t) - m, _ := newTenantVolumeManager(t, eng, catalog) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: tenants} - if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { - t.Fatal(err) - } - for _, current := range []*Manager{m, restartACLManager(t, eng, m.dataDir, catalog)} { - if !slices.ContainsFunc(current.Volumes("acme", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { - t.Error("public runtime volume is hidden from a tenant") - } - if !slices.ContainsFunc(current.Volumes("", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { - t.Error("operator cannot see runtime volume") - } - if _, err := current.VolumePlacement(testKey, "acme", []string{"home"}); err != nil { - t.Errorf("tenant placement: %v", err) - } - if local, err := current.VolumePlacement(testKey, "", []string{"home"}); err != nil || !local { - t.Errorf("operator placement: %v %v", local, err) - } - tenantClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) - if err != nil { - t.Fatalf("tenant initial attachment: %v", err) - } - if err = current.Release(t.Context(), tenantClaim.ID, Cred{Token: tenantClaim.Token}); err != nil { - t.Fatal(err) - } - if _, err = current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { - t.Errorf("public static volume changed: %v", err) - } - rootClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) - if err != nil { - t.Fatalf("operator initial attachment: %v", err) - } - if err := current.Release(t.Context(), rootClaim.ID, Cred{Token: rootClaim.Token}); err != nil { - t.Fatalf("operator initial claim cleanup: %v", err) - } - } - }) - } -} - -func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { - m, sb := newTenantVolumeManager(t, newFakeEngine(), nil) - if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Tenants: []string{"other"}}); err != nil { +func TestRuntimeCatalogEmptyACLIsPublic(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{}} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "home"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("operator bypassed explicit claim-owner ACL: %v", err) - } -} - -func TestRuntimeCatalogPendingWritePreservesPrivateNameMasking(t *testing.T) { - for _, operation := range []string{"register", "unregister"} { - t.Run(operation, func(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "private", "bytes"), Tenants: []string{"other"}} - if operation == "unregister" { - if err := m.RegisterVolume(t.Context(), "private", spec); err != nil { - t.Fatal(err) - } - } - blockCatalogRename(t, m) - var err error - if operation == "unregister" { - err = m.UnregisterVolume(t.Context(), "private") - } else { - err = m.RegisterVolume(t.Context(), "private", spec) - } - if !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("obstructed catalog write: %v", err) - } - for _, requested := range [][]types.Volume{ - {{Name: "private"}}, - {{Name: "data"}, {Name: "private"}}, - {{Name: "private"}, {Name: "data"}}, - } { - missing := slices.Clone(requested) - for i := range missing { - if missing[i].Name == "private" { - missing[i].Name = "missing" - } - } - _, deniedErr := m.SetVolumes(t.Context(), sb.ID, "acme", requested) - _, missingErr := m.SetVolumes(t.Context(), sb.ID, "acme", missing) - if !errors.Is(deniedErr, ErrVolumeUnavailable) || !errors.Is(missingErr, ErrVolumeUnavailable) || deniedErr.Error() != missingErr.Error() { - t.Errorf("private name distinguishable for %+v: denied=%v missing=%v", requested, deniedErr, missingErr) - } - } - if view, err := m.GetVolumes(sb.ID, "acme"); err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("refused request changed claim volumes: %+v, %v", view, err) - } - if len(eng.volumeSpecs) != 0 { - t.Fatal("refused request reached device attachment") - } - for _, name := range []string{"data", "private", "missing"} { - if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { - t.Fatalf("refused request reserved %s: %+v", name, got) - } - } - }) + for _, current := range []*Manager{m, newVolumeManagerAt(t, eng, m.dataDir, nil)} { + if got := current.Volumes("acme", nil); len(got) != 1 || got[0].Name != "home" { + t.Fatalf("public runtime volume hidden from a tenant: %+v", got) + } + if local, err := current.VolumePlacement(testKey, "acme", []string{"home"}); err != nil || !local { + t.Fatalf("tenant placement: %v %v", local, err) + } + sb, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) + if err != nil { + t.Fatalf("tenant attachment: %v", err) + } + if err = current.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } } } -func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { - for _, operation := range []string{"put", "patch"} { - t.Run(operation, func(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, nil) - if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}}); err != nil { - t.Fatal(err) - } - want := []types.Volume{{Name: "home", Mount: "/home"}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { - t.Fatal(err) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { - t.Fatalf("still-held retry: %v", err) - } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - view, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil) - eng.liveVolumeHook = nil - if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("detached result: %+v, %v", view, err) - } - claims, err := newClaimStore(m.dataDir, false).load() - if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { - t.Fatalf("old durable attachment missing: %+v, %v", claims, err) - } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("recovery reference lost writer exclusion: %v", err) - } - if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("capture ignored recovery reference: %v", err) - } - if err = m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("unregister ignored recovery reference: %v", err) - } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) - } - m.mu.Lock() - entry := *m.catalog.entries["home"] - entry.tenants = []string{"other"} - m.catalog.entries["home"] = &entry - m.mu.Unlock() - before := len(eng.volumeSpecs) - switch operation { - case "put": - _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want) - case "patch": - _, err = m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) - } - if !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("recovery-only reference authorized tenant attachment: %v", err) - } - if len(eng.volumeSpecs) != before { - t.Fatal("rejected tenant reached device attachment") - } - m.mu.Lock() - entry.tenants = []string{"acme"} - m.catalog.entries["home"] = &entry - m.mu.Unlock() - if _, err = m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { - t.Fatalf("operator reattach: %v", err) - } - if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { - t.Fatalf("tenant cleanup after operator reattach: %v", err) - } - }) +func TestInvalidStaticTenantListNeverBecomesPublic(t *testing.T) { + path := writeVolumeImage(t, "data", "private contents") + file := filepath.Join(t.TempDir(), "config.json") + raw := fmt.Sprintf(`{"pools":[],"volumes":[{"name":"data","path":%q,"tenants":["has space"]}]}`, path) + if err := os.WriteFile(file, []byte(raw), 0o600); err != nil { + t.Fatal(err) } -} - -func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { - t.Helper() - cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} - m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + cfg, err := config.Load(file) if err != nil { - t.Fatal(err) + t.Fatalf("legacy config stopped booting: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), cfg.Volumes) + if got := m.Volumes("acme", nil); len(got) != 0 { + t.Fatalf("invalid restricted ACL became public: %+v", got) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: path + ".other"}); err != nil { + t.Fatalf("invalid legacy ACL blocked unrelated registration: %v", err) } - return m } diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go index e717d96f..a7c75e0b 100644 --- a/sandboxd/pool/volume_bench_test.go +++ b/sandboxd/pool/volume_bench_test.go @@ -69,7 +69,7 @@ func volumeBenchManager(b *testing.B, checkpoints bool) *Manager { b.Helper() m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.commit(m.store.reset(m.claimed)); err != nil { + if err := m.store.save(m.claimed); err != nil { b.Fatal(err) } path := filepath.Join(b.TempDir(), "data.raw") diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index e0d0b8f4..27c1b1dc 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -15,10 +15,7 @@ func TestVolumeClaimRejectsDirectCapture(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - m.store.set(sb) - m.mu.Unlock() tests := []struct { name string @@ -54,10 +51,7 @@ func TestVolumeClaimRunningWakeUnchanged(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - m.store.set(sb) - m.mu.Unlock() sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { @@ -129,10 +123,7 @@ func TestIdleOnceSkipsVolumeClaim(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, IdleHibernateSeconds: 1}) sb := mustClaim(t, m, testKey) - m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - m.store.set(sb) - m.mu.Unlock() backdate(m, sb, 2*time.Second) sweepIdle(t, m) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index de250141..f3281b35 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -21,40 +21,52 @@ func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { m := newVolumeManager(t, eng, nil) path := writeVolumeImage(t, "runtime", "backing bytes stay intact") spec := types.VolumeRegistration{Path: filepath.Dir(path) + "/./runtime", Writable: true, Tenants: []string{"beta", "acme", "acme"}} + before := m.catalogSnapshot() if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { t.Fatal(err) } - spec.Path = path - spec.DirectIO = types.DirectIOOff - spec.Tenants = []string{"acme", "beta"} - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + if before["runtime"] != nil || spec.Tenants[0] != "beta" { + t.Fatal("registration mutated a captured catalog or the caller's spec") + } + normalized := types.VolumeRegistration{Path: path, DirectIO: types.DirectIOOff, Writable: true, Tenants: []string{"acme", "beta"}} + if err := m.RegisterVolume(t.Context(), "runtime", normalized); err != nil { t.Fatalf("normalized retry: %v", err) } - st, statErr := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) - if statErr != nil || st.Mode().Perm() != 0o600 { - t.Fatalf("metadata mode: %v %v", st, statErr) + if st, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); err != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, err) + } + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "runtime"}}}) + if err != nil { + t.Fatal(err) } m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) - if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("recovered names: %v", got) + if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "runtime" || !got[0].Available || !got[0].Writable { + t.Fatalf("catalog absent before reconcile: %+v", got) + } + if got := m2.Volumes("forbidden", nil); len(got) != 0 { + t.Fatalf("ACL changed at restart: %+v", got) } - if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "forbidden", Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("ACL changed at restart: %v", err) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) } - resolved, err := m2.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}) - if err != nil || len(resolved) != 1 || resolved[0].disk.DirectIO != "off" || !resolved[0].disk.RW { - t.Fatalf("policy roundtrip: %+v %v", resolved, err) + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovered holder missing: %v", err) } - changed := spec + changed := normalized changed.DirectIO = "on" - if err := m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { + if err = m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { t.Fatalf("changed semantics: %v", err) } - if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + if err = m2.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister of a held volume: %v", err) + } + if err = m2.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } - if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatalf("absent retry: %v", err) + for range 2 { + if err = m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } } if got, err := os.ReadFile(path); err != nil || string(got) != "backing bytes stay intact" { t.Fatalf("backing changed: %q %v", got, err) @@ -64,253 +76,205 @@ func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { } } -func TestRuntimeCatalogLoadsOperatorManagedRecord(t *testing.T) { - for _, setup := range []struct { - name string - prepare func(*testing.T, string) - }{ - {"mode", func(t *testing.T, record string) { - if err := os.Chmod(record, 0o644); err != nil { - t.Fatal(err) - } - }}, - {"symlink", func(t *testing.T, record string) { - if err := os.Rename(record, record+".target"); err != nil { - t.Fatal(err) - } - if err := os.Symlink(record+".target", record); err != nil { - t.Fatal(err) - } - }}, - } { - t.Run(setup.name, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { - t.Fatal(err) - } - setup.prepare(t, filepath.Join(m.dataDir, runtimeVolumesFile)) - restarted := newVolumeManagerAt(t, eng, m.dataDir, nil) - if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("operator-managed record not loaded: %v", got) - } - if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("replacement not loaded: %v", got) - } - }) - } -} - -func TestRuntimeCatalogUsesStaticPathValidation(t *testing.T) { - regular := writeVolumeImage(t, "backing", "unchanged") +func TestRuntimeCatalogRegistersUncheckedBacking(t *testing.T) { link := filepath.Join(t.TempDir(), "backing-link") - if err := os.Symlink(regular, link); err != nil { + if err := os.Symlink(writeVolumeImage(t, "backing", "bytes"), link); err != nil { t.Fatal(err) } - for _, path := range []string{regular, link, os.DevNull, t.TempDir(), filepath.Join(t.TempDir(), "missing")} { - t.Run(filepath.Base(path), func(t *testing.T) { - spec := config.VolumeSpec{Name: "runtime", Path: path, DirectIO: types.DirectIOOff, Writable: true} - if err := config.ValidateVolumeCatalog([]config.VolumeSpec{spec}); err != nil { - t.Fatalf("static validation: %v", err) - } + for _, tc := range []struct { + name, path string + wantErr error + }{{"symlink", link, nil}, {"missing", filepath.Join(t.TempDir(), "missing"), ErrVolumeUnavailable}} { + t.Run(tc.name, func(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), nil) - registration := types.VolumeRegistration{Path: path, Writable: true} + registration := types.VolumeRegistration{Path: tc.path, Writable: true} if err := m.RegisterVolume(t.Context(), "runtime", registration); err != nil { - t.Fatalf("runtime validation differs from static: %v", err) + t.Fatalf("register: %v", err) } m2 := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) if err := m2.RegisterVolume(t.Context(), "runtime", registration); err != nil { t.Fatalf("idempotent retry after restart: %v", err) } - if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { - if _, resolveErr := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(resolveErr, ErrVolumeUnavailable) { - t.Fatalf("missing backing resolved: %v", resolveErr) - } + if infos := m2.Volumes("", nil); len(infos) != 1 || infos[0].Available != (tc.wantErr == nil) { + t.Fatalf("availability: %+v", infos) + } + if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, tc.wantErr) { + t.Fatalf("resolve = %v, want %v", err, tc.wantErr) } if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { t.Fatal(err) } }) } - if got, err := os.ReadFile(regular); err != nil || string(got) != "unchanged" { - t.Fatalf("registration touched backing: %q %v", got, err) - } } -func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { - eng := newFakeEngine() +func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { static := liveCatalog(t) - m := newVolumeManager(t, eng, static) + m := newVolumeManager(t, newFakeEngine(), static) other := writeVolumeImage(t, "other-runtime", "bytes") - for _, tc := range []struct{ name, path string }{{"data", other}, {"runtime", static[0].Path}, {"runtime", filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path)}} { - if err := m.RegisterVolume(t.Context(), tc.name, types.VolumeRegistration{Path: tc.path}); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("collision %+v: %v", tc, err) - } - } - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("static removal: %v", err) - } sb := mustClaim(t, m, testKey) sb.Volumes = []types.Volume{{Name: "unexplained", Mount: "/data"}} if err := m.store.commit(m.store.set(sb)); err != nil { t.Fatal(err) } - if err := m.RegisterVolume(t.Context(), "unexplained", types.VolumeRegistration{Path: other}); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("rebound unexplained recovery name: %v", err) - } - if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("absent referenced removal: %v", err) + alias := filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path) + for _, tc := range []struct { + name string + spec types.VolumeRegistration + want error + }{ + {"data", types.VolumeRegistration{Path: other}, ErrVolumeRegistrationConflict}, + {"runtime", types.VolumeRegistration{Path: static[0].Path}, ErrVolumeRegistrationConflict}, + {"runtime", types.VolumeRegistration{Path: alias}, ErrVolumeRegistrationConflict}, + {"unexplained", types.VolumeRegistration{Path: other}, ErrVolumeRegistrationConflict}, + {"cocoon-reserved", types.VolumeRegistration{Path: other}, ErrBadVolume}, + {"bad/name", types.VolumeRegistration{Path: other}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: "relative"}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: other, DirectIO: "invalid"}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: other, Tenants: []string{"has space"}}, ErrBadVolume}, + } { + if err := m.RegisterVolume(t.Context(), tc.name, tc.spec); !errors.Is(err, tc.want) { + t.Errorf("register %s %+v = %v, want %v", tc.name, tc.spec, err, tc.want) + } } - for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}} { - if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { - t.Fatalf("invalid spec %+v: %v", spec, err) + for _, name := range []string{"data", "unexplained"} { + if err := m.UnregisterVolume(t.Context(), name); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Errorf("unregister %s: %v", name, err) } } } -func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - m.store.path = filepath.Join(t.TempDir(), "claims.json") - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} +func TestRuntimeCatalogPendingWriteConverges(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + canceled, cancel := context.WithCancel(t.Context()) + cancel() + if err := m.RegisterVolume(canceled, "runtime", spec); !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("cancellation before acceptance: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled proposal left catalog: %v", err) + } restore := blockCatalogRename(t, m) if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("write failure: %v", err) } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("unpublished admission: %v", err) + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("failed retry cleared pending write: %v", err) } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatalf("volume-free claim blocked: %v", err) + m.retryVolumeCatalog(t.Context()) + for _, volumes := range [][]types.Volume{nil, {{Name: "data"}}} { + if err := claimAndRelease(t, m, volumes...); err != nil { + t.Fatalf("pending write blocked unrelated claim %v: %v", volumes, err) + } } restore() - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatalf("retry pending register: %v", err) + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("possible replacement not recovered: %v", got) } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("durable registration: %v", got) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) + } + if err := m.UnregisterVolume(canceled, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("canceled retry cleared pending write: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if !slices.Contains(m.VolumeNames(), "runtime") { + t.Fatal("retry lost accepted registration") } restore = blockCatalogRename(t, m) if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("failed unregister: %v", err) } restore() + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("possible unregister replacement not recovered: %v", got) + } if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { t.Fatalf("pending unregister did not converge before reuse: %v", err) } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"replacement"}) { + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"replacement"}) { t.Fatalf("durable reuse: %v", got) } } -func TestRuntimeCatalogWriteFailureOnlyFencesAffectedName(t *testing.T) { - for _, unregister := range []bool{false, true} { - t.Run(fmt.Sprintf("unregister=%t", unregister), func(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - for _, name := range []string{"healthy", "affected"} { - if name == "affected" && !unregister { - continue - } - if err := m.RegisterVolume(t.Context(), name, types.VolumeRegistration{Path: writeVolumeImage(t, name, "bytes")}); err != nil { - t.Fatal(err) - } +func TestRuntimeCatalogPendingWriteFencesOnlyAffectedName(t *testing.T) { + unregister := func(m *Manager, _ types.VolumeRegistration) error { return m.UnregisterVolume(t.Context(), "affected") } + register := func(m *Manager, spec types.VolumeRegistration) error { + return m.RegisterVolume(t.Context(), "affected", spec) + } + for _, tc := range []struct { + name string + before []string + write func(*Manager, types.VolumeRegistration) error + resolved []types.Volume + wantResolved, want error + }{ + {"register", []string{"healthy"}, register, []types.Volume{{Name: "healthy"}}, nil, ErrVolumeUnavailable}, + {"unregister", []string{"healthy", "affected"}, unregister, []types.Volume{{Name: "healthy"}, {Name: "affected"}}, ErrVolumeCatalogUnavailable, ErrVolumeCatalogUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + specs := map[string]types.VolumeRegistration{ + "healthy": {Path: writeVolumeImage(t, "healthy", "bytes")}, + "affected": {Path: writeVolumeImage(t, "affected", "bytes"), Tenants: []string{"other"}}, } - sb := mustClaim(t, m, testKey) - var before []resolvedVolume - if unregister { - var err error - before, err = m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "healthy"}, {Name: "affected"}}) - if err != nil { + for _, name := range tc.before { + if err := m.RegisterVolume(t.Context(), name, specs[name]); err != nil { t.Fatal(err) } } - restore := blockCatalogRename(t, m) - defer restore() - var err error - if unregister { - err = m.UnregisterVolume(t.Context(), "affected") - } else { - err = m.RegisterVolume(t.Context(), "affected", types.VolumeRegistration{Path: writeVolumeImage(t, "affected", "bytes")}) + resolved, err := m.resolveVolumes(t.Context(), "", tc.resolved) + if err != nil { + t.Fatal(err) } - if !errors.Is(err, ErrVolumeCatalogUnavailable) { + blockCatalogRename(t, m) + if err = tc.write(m, specs["affected"]); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("obstructed catalog write: %v", err) } - if unregister { - if _, err := m.admitVolumes("", before); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("resolved mixed admission: %v", err) - } - } - wantErr := ErrVolumeUnavailable - if unregister { - wantErr = ErrVolumeCatalogUnavailable + holds, err := m.admitVolumes("", resolved) + m.unreserveVolumes(holds) + if !errors.Is(err, tc.wantResolved) { + t.Fatalf("admission resolved before the write = %v, want %v", err, tc.wantResolved) } for _, volumes := range [][]types.Volume{{{Name: "affected"}}, {{Name: "healthy"}, {Name: "affected"}}} { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, wantErr) { - t.Fatalf("affected claim: %v", err) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, wantErr) { - t.Fatalf("affected live addition: %v", err) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, tc.want) { + t.Errorf("root claim %v = %v, want %v", volumes, err, tc.want) } } - for _, name := range []string{"healthy", "affected"} { + _, deniedErr := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "affected"}}) + _, missingErr := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "missing"}}) + if !errors.Is(deniedErr, ErrVolumeUnavailable) || !errors.Is(missingErr, ErrVolumeUnavailable) || deniedErr.Error() != missingErr.Error() { + t.Errorf("private name distinguishable: denied=%v missing=%v", deniedErr, missingErr) + } + if len(eng.volumeSpecs) != 0 { + t.Fatal("refused request reached device attachment") + } + for _, name := range []string{"data", "healthy", "affected", "missing"} { if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { t.Fatalf("refused request reserved %s: %+v", name, got) } } for _, name := range []string{"data", "healthy"} { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: name}}}); err != nil { + if err := claimAndRelease(t, m, types.Volume{Name: name}); err != nil { t.Errorf("unrelated %s claim: %v", name, err) } } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}, {Name: "healthy"}}); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "healthy"}}); err != nil { t.Errorf("unrelated live addition: %v", err) } }) } } -func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} - restore := blockCatalogRename(t, m) - if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("write failure: %v", err) - } - restore() - installPendingCatalogRecord(t, m) - if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("possible replacement not recovered: %v", got) - } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("unpublished admission: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatalf("volume-free claim blocked: %v", err) - } - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatalf("retry pending register: %v", err) - } - restore = blockCatalogRename(t, m) - if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("uncertain unregister: %v", err) - } - restore() - installPendingCatalogRecord(t, m) - if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("possible unregister replacement not recovered: %v", got) - } - if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { - t.Fatalf("pending unregister did not converge before reuse: %v", err) - } -} - -func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { +func TestResolvedCatalogBindingRequiresSameEntry(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), nil) - path := writeVolumeImage(t, "runtime", "original") - spec := types.VolumeRegistration{Path: path, Writable: true} + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { t.Fatal(err) } @@ -318,66 +282,25 @@ func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { if err != nil { t.Fatal(err) } - if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - spec.Path = writeVolumeImage(t, "replacement", "replacement") - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { t.Fatal(err) } - if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("stale binding admitted: %v", err) - } - if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { - t.Fatalf("stale binding leaked reservation: %+v", got) - } -} - -func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - path := writeVolumeImage(t, "runtime", "bytes") - if os.Geteuid() == 0 { - if err := os.Chown(path, 65534, 65534); err != nil { - t.Fatal(err) - } - } - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { - t.Fatalf("nonroot backing owner: %v", err) + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatalf("unrelated registration invalidated resolution: %v", err) } - if err := os.Remove(path); err != nil { + m.unreserveVolumes(holds) + if err = m.UnregisterVolume(t.Context(), "runtime"); err != nil { t.Fatal(err) } - m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) - infos := m2.Volumes("", nil) - if len(infos) != 1 || infos[0].Available { - t.Fatalf("missing backing lost mapping: %+v", infos) - } - if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("missing backing resolved: %v", err) - } - if err := m2.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { - t.Fatalf("idempotent missing mapping: %v", err) - } -} - -func TestRuntimeRegistrationNormalization(t *testing.T) { - input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} - normalized, err := normalizeRegistration("data", input) - if err != nil || normalized.disk.Path != "/images/data" || normalized.disk.DirectIO != types.DirectIOOff || !slices.Equal(normalized.tenants, []string{"acme", "beta"}) { - t.Fatalf("normalization: %+v %v", normalized, err) + if err = m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) } - normalized.tenants[0] = "caller-copy" - if input.Tenants[1] != "acme" { - t.Fatal("normalization mutated caller input") + if _, err = m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("same-file reuse admitted stale resolution: %v", err) } - for _, tc := range []struct { - name string - spec types.VolumeRegistration - }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{""}}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{"has space"}}}} { - if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { - t.Fatalf("invalid registration %+v: %v", tc, err) - } + if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { + t.Fatalf("stale binding leaked reservation: %+v", got) } } @@ -410,205 +333,6 @@ func TestRuntimeCatalogRejectsInvalidMetadata(t *testing.T) { } } -func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - path := writeVolumeImage(t, "runtime", "unchanged") - spec := types.VolumeRegistration{Path: path, Writable: true} - if err := m.RegisterVolume(t.Context(), "data", spec); err != nil { - t.Fatal(err) - } - sb := mustClaim(t, m, testKey) - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { - t.Fatal("expected failed result commit") - } - eng.liveVolumeHook = nil - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("unregister unretired name: %v", err) - } - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "different")}); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("rebind unretired name: %v", err) - } - if err := m.RegisterVolume(t.Context(), "same-file", spec); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("alias unretired path: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("same-file writer: %v", err) - } - if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("capture: %v", err) - } - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("hibernate before retirement: %v", err) - } - if err := os.Remove(tmp); err != nil { - t.Fatal(err) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { - t.Fatal(err) - } - if err := m.UnregisterVolume(t.Context(), "data"); err != nil { - t.Fatal(err) - } - if err := m.RegisterVolume(t.Context(), "same-file", spec); err != nil { - t.Fatal(err) - } -} - -func TestCatalogSnapshotSurvivesRegistration(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - before := m.catalogSnapshot() - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/images/runtime"}); err != nil { - t.Fatal(err) - } - if before["runtime"] != nil { - t.Fatal("registration mutated captured catalog") - } - captured := m.catalogSnapshot() - if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - if captured["runtime"] == nil { - t.Fatal("unregister mutated captured catalog") - } -} - -func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - ctx, cancel := context.WithCancel(t.Context()) - cancel() - err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) - if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("cancellation before acceptance: %v", err) - } - m.retryVolumeCatalog(t.Context()) - if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("canceled proposal left catalog: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatalf("canceled registration fenced static claim: %v", err) - } -} - -func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - restore := blockCatalogRename(t, m) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} - if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("initial write failure: %v", err) - } - if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("failed retry cleared pending write: %v", err) - } - m.retryVolumeCatalog(t.Context()) - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("unpublished admission: %v", err) - } - restore() - ctx, cancel := context.WithCancel(t.Context()) - cancel() - if err := m.UnregisterVolume(ctx, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("canceled retry cleared pending write: %v", err) - } - m.retryVolumeCatalog(t.Context()) - if !slices.Contains(m.VolumeNames(), "runtime") { - t.Fatal("retry lost accepted registration") - } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatalf("repair did not restore static admission: %v", err) - } -} - -func TestResolvedCatalogBindingChangesOnlyForReusedName(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatal(err) - } - resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) - if err != nil { - t.Fatal(err) - } - if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { - t.Fatal(err) - } - holds, err := m.admitVolumes("", resolved) - if err != nil { - t.Fatalf("unrelated registration invalidated resolution: %v", err) - } - m.unreserveVolumes(holds) - if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatal(err) - } - if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("same-file reuse admitted stale resolution: %v", err) - } -} - -func TestRuntimeCatalogIgnoresAbandonedTemporaryFiles(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - path := writeVolumeImage(t, "runtime", "bytes") - raw := fmt.Sprintf(`{"version":1,"volumes":{"abandoned":{"path":%q}}}`, path) - tmp := filepath.Join(m.dataDir, runtimeVolumesFile+".interrupted.tmp") - if err := os.WriteFile(tmp, []byte(raw), 0o600); err != nil { - t.Fatal(err) - } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("temporary record promoted without canonical record: %v", got) - } - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { - t.Fatal(err) - } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("temporary record overrode canonical record: %v", got) - } - if got, err := os.ReadFile(tmp); err != nil || string(got) != raw { - t.Fatalf("abandoned evidence changed: %q %v", got, err) - } -} - -func TestRuntimeCatalogAcceptsDaemonDataDirectorySymlink(t *testing.T) { - parent := t.TempDir() - dir := filepath.Join(parent, "state") - if err := os.Mkdir(dir, 0o700); err != nil { - t.Fatal(err) - } - link := filepath.Join(parent, "linked-state") - if err := os.Symlink(dir, link); err != nil { - t.Fatal(err) - } - if err := os.Chmod(parent, 0o777); err != nil { - t.Fatal(err) - } - eng := newFakeEngine() - m := newVolumeManagerAt(t, eng, link, nil) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatal(err) - } - restarted := newVolumeManagerAt(t, eng, link, nil) - if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("reloaded catalog: %v", got) - } - if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } -} - func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), nil) if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { @@ -633,9 +357,6 @@ func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { if err := m.writeCatalog(t.Context(), "runtime", entries); err != nil { t.Fatalf("reservation retirement did not unblock removal: %v", err) } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("removal was not persisted: %v", got) - } } func blockCatalogRename(t *testing.T, m *Manager) func() { @@ -646,12 +367,7 @@ func blockCatalogRename(t *testing.T, m *Manager) func() { t.Fatal(err) } existed := err == nil - if existed { - if err := os.Remove(record); err != nil { - t.Fatal(err) - } - } - if err := os.Mkdir(record, 0o700); err != nil { + if err := errors.Join(os.RemoveAll(record), os.Mkdir(record, 0o700)); err != nil { t.Fatal(err) } restore := sync.OnceFunc(func() { @@ -668,6 +384,7 @@ func blockCatalogRename(t *testing.T, m *Manager) func() { return restore } +// installPendingCatalogRecord models the canonical bytes after a successful rename and an uncertain directory sync. func installPendingCatalogRecord(t *testing.T, m *Manager) { t.Helper() m.mu.Lock() @@ -676,7 +393,6 @@ func installPendingCatalogRecord(t *testing.T, m *Manager) { if proposal == nil { t.Fatal("possible replacement fixture requires an accepted pending write") } - // Model the canonical bytes after a successful rename and uncertain directory sync. if err := utils.WriteFileSync(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw, 0o600); err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 78b38b34..2c71a21a 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -313,7 +313,7 @@ func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { } func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { - for _, state := range []string{"applied", "desired", "admitted", "recovery"} { + for _, state := range []string{"applied", "desired", "admitted"} { t.Run(state, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}) @@ -321,48 +321,18 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { held := []types.Volume{{Name: "data", Mount: "/volumes/data"}} switch state { case "applied": - m.mu.Lock() sb.Volumes = held - m.store.set(sb) - m.mu.Unlock() case "desired": - m.mu.Lock() - sb.VolumesWant = held - m.store.set(sb) - m.mu.Unlock() - case "recovery": - m.mu.Lock() sb.VolumesWant = held - js := m.store.set(sb) - m.mu.Unlock() - if err := m.store.commit(js); err != nil { - t.Fatal(err) - } - m.mu.Lock() - sb.VolumesWant = nil - m.store.set(sb) - m.mu.Unlock() case "admitted": - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "data", "bytes")}); err != nil { - t.Fatal(err) - } _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { t.Fatal(err) } - resolved, err := m.resolveVolumes(t.Context(), "", held) - if err != nil { - t.Fatal(err) - } m.mu.Lock() - if err = m.checkResolved(resolved, ""); err == nil { - m.adoptVolumes(held) - } + m.adoptVolumes(held) run.holds = held m.mu.Unlock() - if err != nil { - t.Fatal(err) - } defer m.finishVolumeRun(sb, run) } if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "blocked", ""); !errors.Is(err, ErrVolumeCapture) { @@ -377,14 +347,6 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { if len(eng.trims) != 0 { t.Fatalf("capture trimmed held volume: %v", eng.trims) } - if state == "recovery" { - if err := m.store.commit(m.store.mark()); err != nil { - t.Fatal(err) - } - if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retired", ""); err != nil { - t.Fatalf("checkpoint after recovery acknowledgment: %v", err) - } - } }) } } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index c25fbf27..87ef4c23 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -400,7 +400,7 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { if got := claims["sb_1"].VolumesWant; !slices.Equal(got, tt.want) { t.Fatalf("loaded target %v, want %v", got, tt.want) } - if err = store.commit(store.reset(claims)); err != nil { + if err = store.save(claims); err != nil { t.Fatal(err) } rawBytes, err := os.ReadFile(store.path) @@ -850,16 +850,9 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume if err != nil { t.Fatal(err) } - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} - resolved, err := m.resolveVolumes(t.Context(), "", transient) - if err != nil { - t.Fatal(err) - } m.mu.Lock() defer m.mu.Unlock() - if err = m.checkResolved(resolved, ""); err != nil { - t.Fatal(err) - } + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} m.adoptVolumes(transient) run.holds = append(run.holds, transient...) return run diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index ae6b6ea5..a334e5ef 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -13,118 +13,83 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { +func TestUnacknowledgedDetachKeepsRecoveryReference(t *testing.T) { eng := newFakeEngine() - catalog := liveCatalog(t) - m := newVolumeManager(t, eng, catalog) - sb := mustClaim(t, m, testKey) - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { t.Fatal(err) } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) + want := []types.Volume{{Name: "home", Mount: "/home"}} + for _, caller := range []string{"", "acme"} { + if _, err := m.SetVolumes(t.Context(), sb.ID, caller, want); err != nil { + t.Fatal(err) } - return nil - } - view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) - eng.liveVolumeHook = nil - if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("detached result = %+v, %v", view, err) } + unblock := detachWithoutCommit(t, m, eng, sb.ID, "acme") claims, err := newClaimStore(m.dataDir, false).load() - if err != nil || claims[sb.ID] == nil || len(claims[sb.ID].Volumes) != 1 { - t.Fatalf("failed write did not leave old recovery reference: %+v, %v", claims, err) + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("writer admitted while old claim still references detached volume: %v", err) + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching || !view.Converging { + t.Fatalf("unretired view: %+v, %v", view, err) } - if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { - t.Errorf("capture passed before recovery reference retired: %v", err) + writer := ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}} + for _, tc := range []struct { + name string + err error + want error + }{ + {"writer", errOf(m.ClaimProvision(t.Context(), testKey, writer)), ErrVolumeBusy}, + {"checkpoint", errOf(m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", "")), ErrVolumeCapture}, + {"hibernate", m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}), ErrVolumeCapture}, + {"unregister", m.UnregisterVolume(t.Context(), "home"), ErrVolumeRegistrationConflict}, + {"rebind", m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "bytes")}), ErrVolumeRegistrationConflict}, + {"alias", m.RegisterVolume(t.Context(), "same-file", spec), ErrVolumeRegistrationConflict}, + {"reshape", errOf(m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "home", Mount: "/different"}})), ErrBadVolume}, + } { + if !errors.Is(tc.err, tc.want) { + t.Errorf("%s with unretired reference = %v, want %v", tc.name, tc.err, tc.want) + } } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + unblock() + setVolumeTenants(m, "home", "other") + before := len(eng.volumeSpecs) + _, putErr := m.SetVolumes(t.Context(), sb.ID, "acme", want) + _, patchErr := m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + if !errors.Is(putErr, ErrVolumeUnavailable) || !errors.Is(patchErr, ErrVolumeUnavailable) || len(eng.volumeSpecs) != before { + t.Fatalf("recovery-only reference authorized a fresh attachment: put=%v patch=%v", putErr, patchErr) + } + setVolumeTenants(m, "home", "acme") + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "home"); got != (volumeHolders{readers: 1}) { + t.Fatalf("own retry counts: %+v", got) + } } - m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) - if err = m2.Reconcile(t.Context()); err != nil { + detachWithoutCommit(t, m, eng, sb.ID, "acme")() + if err = m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatal(err) } - if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("restart lost old claim's reader exclusion: %v", err) + _, putErr = m.SetVolumes(t.Context(), sb.ID, "", want) + _, patchErr = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + if !errors.Is(putErr, ErrTenantRemoved) || !errors.Is(patchErr, ErrTenantRemoved) || volumeHoldersOf(m, "home") != (volumeHolders{}) { + t.Fatalf("recovery-only retry bypassed tenant removal: put=%v patch=%v", putErr, patchErr) } - if _, err = m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { - t.Fatalf("retry after restart: %v", err) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("retirement after tenant removal: %v", err) } - if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("writer still blocked after successful retirement: %v", err) + if err = claimAndRelease(t, m, writer.Volumes...); err != nil { + t.Fatalf("acknowledged detach kept the reader exclusion: %v", err) } -} - -func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { - for _, retry := range []string{"empty", "same-shape", "unrelated-snapshot"} { - t.Run(retry, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - sb := mustClaim(t, m, testKey) - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { - t.Fatal("detach result unexpectedly persisted") - } - eng.liveVolumeHook = nil - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("physical count retained after detach: %+v", got) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { - t.Fatal("empty retry acknowledged an unretired file") - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mount: "/different"}}); !errors.Is(err, ErrBadVolume) { - t.Fatalf("recovery-only shape changed: %v", err) - } - view, err := m.GetVolumes(sb.ID, "") - if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { - t.Fatalf("unretired view: %+v, %v", view, err) - } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) - } - switch retry { - case "empty": - _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) - case "same-shape": - for range 2 { - if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - if got := volumeHoldersOf(m, "data"); got.readers != 1 || got.writers != 0 { - t.Fatalf("own retry counts: %+v", got) - } - } - _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) - case "unrelated-snapshot": - _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) - } - if err != nil { - t.Fatal(err) - } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("retired count: %+v", got) - } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("writer after acknowledgment: %v", err) - } - }) + if err = errors.Join(m.UnregisterVolume(t.Context(), "home"), m.RegisterVolume(t.Context(), "same-file", spec)); err != nil { + t.Fatalf("retired name or path stayed reserved: %v", err) } } @@ -146,24 +111,18 @@ func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { if err != nil { t.Fatal(err) } - // Model an intent whose rename succeeded before an uncertain directory sync. + // model an intent whose rename succeeded before an uncertain directory sync if err = utils.WriteFileSync(m.store.path, raw, 0o600); err != nil { t.Fatal(err) } if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { t.Fatal("expected empty snapshot error") } - if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 { + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { t.Fatalf("memory not empty: %+v", sb) } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("unexpected physical count: %+v", got) - } - claims, err = newClaimStore(m.dataDir, false).load() - if err != nil || len(claims[sb.ID].VolumesWant) != 1 { - t.Fatalf("actual recovery file: %+v, %v", claims, err) - } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + writer := ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}} + if _, err = m.ClaimProvision(t.Context(), testKey, writer); !errors.Is(err, ErrVolumeBusy) { t.Fatalf("writer admitted after uncertain introduction: %v", err) } if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain-introduction", ""); !errors.Is(err, ErrVolumeCapture) { @@ -174,42 +133,18 @@ func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { if err = m2.Reconcile(t.Context()); err != nil { t.Fatal(err) } - if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + if _, err = m2.ClaimProvision(t.Context(), testKey, writer); !errors.Is(err, ErrVolumeBusy) { t.Fatalf("restart admitted writer: %v", err) } if err = m.store.commit(m.store.mark()); err != nil { t.Fatal(err) } - if len(m.store.recoveryVolumesFor(sb.ID)) != 0 { - t.Fatal("successful whole snapshot did not retire attempted row") - } -} - -func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { - s := newClaimStore(t.TempDir(), false) - sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} - if err := s.commit(s.set(sb)); err != nil { - t.Fatal(err) - } - s = newClaimStore(filepath.Dir(s.path), false) - if _, err := s.load(); err != nil { - t.Fatal(err) - } - s.reset(map[string]*types.Sandbox{}) - refs := s.recoveryVolumesFor(sb.ID) - refs[0].Name = "caller-copy" - if got := s.recoveryVolumesFor(sb.ID); len(got) != 1 || got[0].Name != "data" { - t.Fatalf("reset or caller erased actual file: %+v", got) - } - if err := s.commit(s.mark()); err != nil { - t.Fatal(err) - } - if s.hasRecoveryVolumes(sb.ID) { - t.Fatal("empty snapshot did not retire file references") + if _, err = m.ClaimProvision(t.Context(), testKey, writer); err != nil { + t.Fatalf("successful whole snapshot did not retire attempted row: %v", err) } } -func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { +func TestClaimRecoveryReferencesRetireOnlyOnAcknowledgedWrite(t *testing.T) { s := newClaimStore(t.TempDir(), false) sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} if err := s.commit(s.set(sb)); err != nil { @@ -223,63 +158,25 @@ func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { t.Fatal("expected failed snapshot") } } - if refs := s.recoveryVolumesFor(sb.ID); len(refs) != 2 { - t.Fatalf("renewals retained duplicate history: %+v", refs) - } refs := s.recoveryVolumesFor(sb.ID) refs[0].Name = "caller-copy" if got := s.recoveryVolumesFor(sb.ID); len(got) != 2 || got[0].Name != "data" { - t.Fatalf("recovery references not copied: %+v", got) + t.Fatalf("recovery references duplicated or shared: %+v", got) } restoreWrites() - if err := s.commit(s.mark()); err != nil { - t.Fatal(err) - } - if len(s.recoveryVolumesFor(sb.ID)) != 1 { - t.Fatal("successful snapshot did not replace possibility history") - } -} - -func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { + s = newClaimStore(filepath.Dir(s.path), false) + if _, err := s.load(); err != nil { t.Fatal(err) } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err == nil { - t.Fatal("expected failed detach result commit") - } - eng.liveVolumeHook = nil - if err := m.DeleteTenant(t.Context(), "acme"); err != nil { - t.Fatal(err) + s.reset(map[string]*types.Sandbox{}) + if got := s.recoveryVolumesFor(sb.ID); len(got) != 1 || got[0].Name != "data" { + t.Fatalf("reset erased the file reference: %+v", got) } - m.mu.Lock() - entry := *m.catalog.entries["data"] - entry.tenants = []string{"different-tenant"} - m.catalog.entries["data"] = &entry - m.mu.Unlock() - if err := os.Remove(tmp); err != nil { + if err := s.commit(s.mark()); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { - t.Fatalf("recovery-only retry bypassed tenant removal: %v", err) - } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("rejected recovery retry reserved a reader: %+v", got) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { - t.Fatalf("recovery retirement after tenant removal: %v", err) - } - if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { - t.Fatalf("repair exemption admitted unrelated addition: %v", err) + if s.hasRecoveryVolumes(sb.ID) { + t.Fatal("empty snapshot did not retire file references") } } @@ -311,8 +208,7 @@ func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing. if s.hasRecoveryVolumes(sb.ID) { t.Fatal("empty replacement retained old reference") } - restoreWrites := failClaimWrites(t, s) - defer restoreWrites() + failClaimWrites(t, s) if err = s.commit(old); err != nil { t.Fatal(err) } @@ -322,81 +218,72 @@ func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing. } } -func TestClaimSharedRecoveryVolumesRetirePerOwner(t *testing.T) { +func TestClaimRecoveryVolumeConflicts(t *testing.T) { s := newClaimStore(t.TempDir(), false) - volume := types.Volume{Name: "shared", Mount: "/data"} - first := &types.Sandbox{ID: "first", Volumes: []types.Volume{volume}} - second := &types.Sandbox{ID: "second", Volumes: []types.Volume{volume}} - writer := volume - writer.Mode = types.VolumeModeRW - if err := s.commit(s.set(first, second)); err != nil { - t.Fatal(err) - } - if s.volumeConflicts(volume, "") || !s.volumeConflicts(writer, "") { - t.Fatal("shared readers have incorrect admission") - } - if err := s.commit(s.del(first.ID)); err != nil { - t.Fatal(err) - } - if !s.referencesVolume(volume.Name) || s.hasRecoveryVolumes(first.ID) || !s.hasRecoveryVolumes(second.ID) || !s.volumeConflicts(writer, first.ID) { - t.Fatal("retiring one reader released the other") - } - if err := s.commit(s.del(second.ID)); err != nil { - t.Fatal(err) - } - if s.referencesVolume(volume.Name) || s.volumeConflicts(writer, "") { - t.Fatal("last reader left recovery reservation") - } -} - -func TestClaimRecoveryVolumeOwnerExemptionRequiresExactAttachment(t *testing.T) { - s := newClaimStore(t.TempDir(), false) - writer := types.Volume{Name: "data", Mount: "/data", Mode: types.VolumeModeRW} - sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{writer}} - if err := s.commit(s.set(sb)); err != nil { + reader := types.Volume{Name: "shared", Mount: "/data"} + writer, moved := types.Volume{Name: "data", Mount: "/data", Mode: types.VolumeModeRW}, types.Volume{Name: "data", Mount: "/elsewhere", Mode: types.VolumeModeRW} + first, second := &types.Sandbox{ID: "first", Volumes: []types.Volume{reader}}, &types.Sandbox{ID: "second", Volumes: []types.Volume{reader}} + owner := &types.Sandbox{ID: "owner", Volumes: []types.Volume{writer}} + if err := s.commit(s.set(first, second, owner)); err != nil { t.Fatal(err) } restore := failClaimWrites(t, s) - sb.VolumesWant = []types.Volume{{Name: "other", Mount: "/other"}} - if err := s.commit(s.set(sb)); err == nil { + owner.VolumesWant = []types.Volume{{Name: "other", Mount: "/other"}} + if err := s.commit(s.set(owner)); err == nil { t.Fatal("expected write failure") } - reader := writer - reader.Mode = types.VolumeModeRO - moved := writer - moved.Mount = "/elsewhere" + sharedWriter, ownerReader := reader, writer + sharedWriter.Mode, ownerReader.Mode = types.VolumeModeRW, types.VolumeModeRO for _, tc := range []struct { volume types.Volume owner string conflict bool }{ - {writer, sb.ID, false}, {writer, "different", true}, {reader, sb.ID, true}, {moved, sb.ID, true}, + {reader, "", false}, + {sharedWriter, "", true}, + {sharedWriter, first.ID, true}, + {writer, owner.ID, false}, + {writer, "different", true}, + {ownerReader, owner.ID, true}, + {moved, owner.ID, true}, } { if got := s.volumeConflicts(tc.volume, tc.owner); got != tc.conflict { t.Errorf("volume %+v owner %q conflict=%t, want %t", tc.volume, tc.owner, got, tc.conflict) } } - s.set(&types.Sandbox{ID: sb.ID, Volumes: []types.Volume{writer}}) + s.set(&types.Sandbox{ID: owner.ID, Volumes: []types.Volume{writer}}) restore() - if err := s.commit(s.mark()); err != nil { + if err := s.commit(s.del(first.ID)); err != nil { t.Fatal(err) } - if s.referencesVolume("other") || s.volumeConflicts(writer, sb.ID) { + if !s.referencesVolume(reader.Name) || s.hasRecoveryVolumes(first.ID) || !s.volumeConflicts(sharedWriter, first.ID) { + t.Fatal("retiring one reader released the other") + } + if s.referencesVolume("other") || s.volumeConflicts(writer, owner.ID) { t.Fatal("successful return retained attempted attachment") } + if err := s.commit(s.del(second.ID)); err != nil { + t.Fatal(err) + } + if s.referencesVolume(reader.Name) || s.volumeConflicts(sharedWriter, "") { + t.Fatal("last reader left recovery reservation") + } } func TestClaimCaptureAcknowledgesBothReferenceDomains(t *testing.T) { - for _, domain := range []string{"volumes", "checkpoints"} { - t.Run(domain, func(t *testing.T) { + for _, tc := range []struct { + name string + drop func(*types.Sandbox) + checkpoint, volumes bool + }{ + {"volumes", func(sb *types.Sandbox) { sb.Volumes = nil }, true, false}, + {"checkpoints", func(sb *types.Sandbox) { sb.ReplicaCk = "" }, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { s := newClaimStore(t.TempDir(), false) sb := &types.Sandbox{ID: "owner", ReplicaCk: "checkpoint", Volumes: []types.Volume{{Name: "data"}}} release := captureClaimWrite(t, s, s.set(sb), sb.ID) - if domain == "volumes" { - sb.Volumes = nil - } else { - sb.ReplicaCk = "" - } + tc.drop(sb) newer := s.set(sb) release() if !s.pending(newer) || !s.referencesCheckpoint("checkpoint") || !s.hasRecoveryVolumes(sb.ID) { @@ -405,12 +292,42 @@ func TestClaimCaptureAcknowledgesBothReferenceDomains(t *testing.T) { if err := s.commit(newer); err != nil { t.Fatal(err) } - if got := s.referencesCheckpoint("checkpoint"); got != (domain == "volumes") { - t.Fatalf("checkpoint reference after newer commit = %t", got) - } - if got := s.hasRecoveryVolumes(sb.ID); got != (domain == "checkpoints") { - t.Fatalf("volume recovery after newer commit = %t", got) + if s.referencesCheckpoint("checkpoint") != tc.checkpoint || s.hasRecoveryVolumes(sb.ID) != tc.volumes { + t.Fatalf("references after newer commit: checkpoint=%t volumes=%t", s.referencesCheckpoint("checkpoint"), s.hasRecoveryVolumes(sb.ID)) } }) } } + +// detachWithoutCommit detaches every volume on the device while the claim write fails until unblock. +func detachWithoutCommit(t *testing.T, m *Manager, eng *fakeEngine, id, tenant string) (unblock func()) { + t.Helper() + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), id, tenant, nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detach did not fail after device removal: %+v, %v", view, err) + } + return func() { + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + } +} + +func errOf(_ any, err error) error { return err } + +func setVolumeTenants(m *Manager, name string, tenants ...string) { + m.mu.Lock() + defer m.mu.Unlock() + entry := *m.catalog.entries[name] + entry.tenants = tenants + m.catalog.entries[name] = &entry +} diff --git a/sandboxd/pool/volume_static_acl_test.go b/sandboxd/pool/volume_static_acl_test.go deleted file mode 100644 index 35a5af36..00000000 --- a/sandboxd/pool/volume_static_acl_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package pool - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/types" -) - -func TestInvalidStaticTenantListNeverBecomesPublic(t *testing.T) { - path := writeVolumeImage(t, "data", "private contents") - file := filepath.Join(t.TempDir(), "config.json") - raw := fmt.Sprintf(`{"pools":[],"volumes":[{"name":"data","path":%q,"tenants":["has space"]}]}`, path) - if err := os.WriteFile(file, []byte(raw), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := config.Load(file) - if err != nil { - t.Fatalf("legacy config stopped booting: %v", err) - } - m := newVolumeManager(t, newFakeEngine(), cfg.Volumes) - if got := m.Volumes("acme", nil); len(got) != 0 { - t.Fatalf("invalid restricted ACL became public: %+v", got) - } - if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: path + ".other"}); err != nil { - t.Fatalf("invalid legacy ACL blocked unrelated registration: %v", err) - } -} - -func TestRuntimeRegistrationRejectsInvalidTenantNames(t *testing.T) { - for _, tenant := range []string{"", "has space", "_leading"} { - t.Run(tenant, func(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: "/images/data", Tenants: []string{tenant}}); err == nil { - t.Fatalf("runtime registration accepted invalid tenant %q", tenant) - } - }) - } -} diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 0aab0cb7..f24769d1 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2987,14 +2987,10 @@ func (f *fakePlacer) TemplateVolumeOwners(probe string, _ []string) []string { } return f.templateVolumeOwners } - func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } - -func (f *fakePlacer) PeerAddrs() []string { return f.addrs } - +func (f *fakePlacer) PeerAddrs() []string { return f.addrs } func (f *fakePlacer) ClientAddr(addr string) string { return addr } - -func (f *fakePlacer) ConfigMismatches() int { return 0 } +func (f *fakePlacer) ConfigMismatches() int { return 0 } type fakeProber struct { owners []string @@ -3002,5 +2998,4 @@ type fakeProber struct { } func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } - -func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } +func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index ef2c48f9..037d225b 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -9,8 +9,6 @@ import ( "io" "net/http" "net/http/httptest" - "os" - "path/filepath" "reflect" "strconv" "strings" @@ -21,72 +19,35 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { +func TestVolumeRegistrationRoutesRequireRootBeforeReadingBody(t *testing.T) { for _, method := range []string{http.MethodPut, http.MethodDelete} { for _, tt := range []struct { name, apiToken, token string code int + wantCalled bool }{ - {"root", "root", "root", http.StatusNoContent}, - {"tenant", "root", "acme-token", http.StatusForbidden}, - {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, - {"unknown", "root", "unknown", http.StatusUnauthorized}, - {"missing", "root", "", http.StatusUnauthorized}, - {"open without token", "", "", http.StatusForbidden}, - {"open with tenant token", "", "acme-token", http.StatusForbidden}, + {"root", "root", "root", http.StatusNoContent, true}, + {"tenant", "root", "acme-token", http.StatusForbidden, false}, + {"sandbox", "root", "sandbox-token", http.StatusUnauthorized, false}, + {"unknown", "root", "unknown", http.StatusUnauthorized, false}, + {"missing", "root", "", http.StatusUnauthorized, false}, + {"no api token", "", "", http.StatusForbidden, false}, } { t.Run(method+"/"+tt.name, func(t *testing.T) { called := false mgr := &fakeManager{ registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, unregisterVolume: func(context.Context, string) error { called = true; return nil }, + tenants: []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, } - ts := newTenantTestServer(t, tt.apiToken, []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) - body := `{"path":"/private/data.img"}` - if tt.code != http.StatusNoContent { - body = `{invalid` - } - resp := doReq(t, method, ts.URL+"/v1/volumes/data", tt.token, body) - defer resp.Body.Close() - got, _ := io.ReadAll(resp.Body) - if resp.StatusCode != tt.code || called != (tt.code == http.StatusNoContent) { - t.Fatalf("status=%d called=%v body=%s, want %d", resp.StatusCode, called, got, tt.code) - } - if tt.code == http.StatusNoContent && len(got) != 0 { - t.Fatalf("204 response body = %q", got) - } - }) - } - } -} - -func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { - path := filepath.Join(t.TempDir(), "data.img") - if err := os.WriteFile(path, []byte("existing backing"), 0o600); err != nil { - t.Fatal(err) - } - for _, method := range []string{http.MethodPut, http.MethodDelete} { - for _, token := range []string{"", "acme-token", "unknown"} { - t.Run(method+"/"+token, func(t *testing.T) { - called := false - mgr := &fakeManager{ - registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, - unregisterVolume: func(context.Context, string) error { called = true; return nil }, - } - handler := New("", "", mgr, nil, nil, nil, nil, nil).Handler() - body := bytes.NewBufferString(fmt.Sprintf(`{"path":%q,"writable":true}`, path)) - bodyLen := body.Len() + body := bytes.NewBufferString(`{"path":"/private/data.img","writable":true}`) + unread := body.Len() req := httptest.NewRequest(method, "/v1/volumes/data", body) - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } + req.Header.Set("Authorization", "Bearer "+tt.token) w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusForbidden || called || body.Len() != bodyLen { - t.Fatalf("status=%d called=%v unread=%d/%d, want 403 without reading body or calling manager", w.Code, called, body.Len(), bodyLen) - } - if w.Header().Get("Retry-After") != "" { - t.Fatalf("permanent refusal advertises retry: %v", w.Header()) + New(tt.apiToken, "node:7777", mgr, &fakeDialer{}, nil, nil, nil, nil).Handler().ServeHTTP(w, req) + if w.Code != tt.code || called != tt.wantCalled || !tt.wantCalled && body.Len() != unread || w.Header().Get("Retry-After") != "" { + t.Fatalf("status=%d called=%v unread=%d/%d body=%s headers=%v, want %d", w.Code, called, body.Len(), unread, w.Body, w.Header(), tt.code) } }) } @@ -118,15 +79,13 @@ func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { } handler := New("root", "", mgr, nil, nil, nil, nil, nil).Handler() for _, method := range []string{http.MethodPut, http.MethodDelete} { - for range 2 { - req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) - req.Header.Set("Authorization", "Bearer root") - req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusNoContent || w.Body.Len() != 0 { - t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) - } + req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer root") + req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent || w.Body.Len() != 0 { + t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) } } if registeredName != "data-v1" || removedName != "data-v1" || !reflect.DeepEqual(registered, want) { @@ -134,34 +93,18 @@ func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { } } -func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { - for _, tt := range []struct{ name, body string }{ - {"missing", ""}, - {"malformed", `{"path":`}, - {"array", `[]`}, - {"unknown field", `{"path":"/private/data.img","extra":true}`}, - {"body name", `{"name":"other","path":"/private/data.img"}`}, - {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, - {"trailing object", `{"path":"/private/data.img"}{}`}, - {"trailing text", `{"path":"/private/data.img"}oops`}, - {"wrong writable type", `{"path":"/private/data.img","writable":"true"}`}, - {"wrong tenants type", `{"path":"/private/data.img","tenants":"acme"}`}, - {"oversized", `{"path":"/` + strings.Repeat("a", maxBodyBytes) + `"}`}, - } { - t.Run(tt.name, func(t *testing.T) { - called := false - mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { - called = true - return nil - }} - ts := newTestServer(t, "root", mgr, nil) - resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", tt.body) - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - if resp.StatusCode != http.StatusBadRequest || called { - t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) - } - }) +func TestVolumeRegistrationRejectsBodyName(t *testing.T) { + called := false + mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { + called = true + return nil + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", `{"name":"other","path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest || called { + t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) } } @@ -194,43 +137,12 @@ func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { if err := json.Unmarshal(body, &got); err != nil || len(got) != 1 || got["error"] == "" { t.Fatalf("error body = %s, decode error = %v", body, err) } - if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { - t.Fatalf("status=%d body=%s, want safe %d", resp.StatusCode, body, tt.code) - } - if tt.code == http.StatusServiceUnavailable { - seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")) - if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { - t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) - } - } else if resp.Header.Get("Retry-After") != "" { - t.Fatalf("permanent refusal advertises retry: %v", resp.Header) + retry := tt.code == http.StatusServiceUnavailable + seconds, _ := strconv.Atoi(resp.Header.Get("Retry-After")) + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") || (seconds >= 1) != retry || strings.Contains(got["error"], "retry") != retry { + t.Fatalf("status=%d headers=%v body=%s, want safe %d with retry=%t", resp.StatusCode, resp.Header, body, tt.code, retry) } }) } } } - -func TestVolumeRegistrationKeepsCatalogListingPrivate(t *testing.T) { - for _, token := range []string{"root", "acme-token"} { - t.Run(token, func(t *testing.T) { - mgr := &fakeManager{volumeCatalog: func(tenant string, _ map[string]int) []types.VolumeInfo { - wantTenant := "" - if token == "acme-token" { - wantTenant = "acme" - } - if tenant != wantTenant { - t.Errorf("listing tenant = %q, want %q", tenant, wantTenant) - } - return []types.VolumeInfo{{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1, Writable: true}} - }} - ts := newTenantTestServer(t, "root", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) - resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", token, "") - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - const want = `{"volumes":[{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1,"writable":true}]}` - if resp.StatusCode != http.StatusOK || string(body) != want { - t.Fatalf("catalog = %d %s, want 200 %s", resp.StatusCode, body, want) - } - }) - } -} From 960f7bce10a46cfd00243c1be28a5f104c7adf79 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:45:36 +0000 Subject: [PATCH 16/29] pool: read one catalog snapshot per quiesce and repair pass The catalog map is replaced on every write, never edited, and a name a claim holds cannot be unregistered or rebound while held, so the snapshot taken with the hold state serves the whole pass instead of taking m.mu again for every mount. --- sandboxd/pool/volume.go | 5 ++--- sandboxd/pool/volume_mutation.go | 7 ++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 0f04eb42..f5afb546 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -244,6 +244,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { m.mu.Lock() td := volumeTeardown{run: m.volumeRuns[sb], holds: heldVolumes(sb)} + catalog := m.catalog.entries m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { @@ -263,11 +264,9 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } - m.mu.Lock() - if entry, ok := m.catalog.entries[volume.Name]; ok { + if entry, ok := catalog[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } - m.mu.Unlock() } if stuck { // the unmounts may have spent the whole budget; the fallback flush gets its own diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 01d64821..8d36045a 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -141,17 +141,14 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.clearVolumeError(sb, run, v.Name) } m.mu.Lock() - pending := volumeDifference(sb.VolumesWant, sb.Volumes) + pending, catalog := volumeDifference(sb.VolumesWant, sb.Volumes), m.catalog.entries m.mu.Unlock() for _, v := range pending { failedName = v.Name if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - m.mu.Lock() - entry := m.catalog.entries[v.Name] - m.mu.Unlock() - repair, resolveErr := m.resolveCatalogEntry(ctx, v, entry) + repair, resolveErr := m.resolveCatalogEntry(ctx, v, catalog[v.Name]) if resolveErr != nil { return view, resolveErr } From 49b468a8d2ba3a553396ad77bb2b86655c45ce70 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:45:46 +0000 Subject: [PATCH 17/29] pool: restore the attach-only resolution comments Runtime registration rewrote resolveVolumes and the entry resolver but not the attach-only handling, so the two comments explaining why the flag is passed to validation and then cleared still apply. --- sandboxd/pool/volume.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index f5afb546..bfbf9a85 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -138,6 +138,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } + // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) @@ -181,7 +182,7 @@ func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false + volume.AttachOnly = false // past validation the empty mount carries it return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil } From 39d866671769784c75d037c1923562d8042901d9 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:46:29 +0000 Subject: [PATCH 18/29] pool: name the claim store volume counts for what they count The checkpoint count covers current and retained references, while the volume count covers retained references only, so the shared "volumes" names hid that difference. Name the count, its helper and the fast-path flag after the retained set they summarize, and seed a loaded claim's references with the same union commit uses instead of the change-aware helper with no previous list. --- sandboxd/pool/claims.go | 54 ++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index c8876a73..0f73ad60 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -81,20 +81,20 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 - refs map[string][4]string - retained map[string][]string - changed map[string]struct{} - checkpoints map[string]int - currentVolumes map[string][]types.Volume - retainedVolumes map[string][]types.Volume - volumes map[string]volumeHolders - - recommitting atomic.Bool - recoveryVolumesPresent atomic.Bool + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 + refs map[string][4]string + retained map[string][]string + changed map[string]struct{} + checkpoints map[string]int + currentVolumes map[string][]types.Volume + retainedVolumes map[string][]types.Volume + retainedVolumeCounts map[string]volumeHolders + + recommitting atomic.Bool + anyRetainedVolume atomic.Bool } func newClaimStore(dataDir string, sync bool) *claimStore { @@ -102,7 +102,7 @@ func newClaimStore(dataDir string, sync bool) *claimStore { path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, refs: map[string][4]string{}, retained: map[string][]string{}, changed: map[string]struct{}{}, checkpoints: map[string]int{}, - currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]volumeHolders{}, + currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, retainedVolumeCounts: map[string]volumeHolders{}, } } @@ -124,7 +124,7 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } - if refs := volumeRefs(nil, slices.Clone(sb.Volumes), slices.Clone(sb.VolumesWant)); len(refs) != 0 { + if refs := appendRecoveryRefs(slices.Clone(sb.Volumes), sb.VolumesWant); len(refs) != 0 { s.changeRetainedVolumes(sb.ID, refs) s.changed[sb.ID] = struct{}{} } @@ -363,43 +363,43 @@ func (s *claimStore) checkpointReferences() map[string]struct{} { return refs } -func (s *claimStore) changeVolumeCounts(refs []types.Volume, delta int) { +func (s *claimStore) changeRetainedVolumeCounts(refs []types.Volume, delta int) { for _, v := range refs { - count := s.volumes[v.Name] + count := s.retainedVolumeCounts[v.Name] if v.RW() { count.writers += delta } else { count.readers += delta } if count == (volumeHolders{}) { - delete(s.volumes, v.Name) + delete(s.retainedVolumeCounts, v.Name) } else { - s.volumes[v.Name] = count + s.retainedVolumeCounts[v.Name] = count } } } func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { - s.changeVolumeCounts(s.retainedVolumes[id], -1) - s.changeVolumeCounts(after, 1) + s.changeRetainedVolumeCounts(s.retainedVolumes[id], -1) + s.changeRetainedVolumeCounts(after, 1) if len(after) == 0 { delete(s.retainedVolumes, id) } else { s.retainedVolumes[id] = after } - s.recoveryVolumesPresent.Store(len(s.volumes) != 0) + s.anyRetainedVolume.Store(len(s.retainedVolumeCounts) != 0) } func (s *claimStore) referencesVolume(name string) bool { s.mu.Lock() defer s.mu.Unlock() - return s.volumes[name] != (volumeHolders{}) + return s.retainedVolumeCounts[name] != (volumeHolders{}) } func (s *claimStore) volumeConflicts(volume types.Volume, ownerID string) bool { s.mu.Lock() defer s.mu.Unlock() - count := s.volumes[volume.Name] + count := s.retainedVolumeCounts[volume.Name] for _, ref := range s.retainedVolumes[ownerID] { if ref == volume { if ref.RW() { @@ -419,7 +419,7 @@ func (s *claimStore) recoveryVolumesFor(owner string) []types.Volume { } func (s *claimStore) detachingOwners() map[string]bool { - if !s.recoveryVolumesPresent.Load() { + if !s.anyRetainedVolume.Load() { return nil } s.mu.Lock() @@ -437,7 +437,7 @@ func (s *claimStore) detachingOwners() map[string]bool { } func (s *claimStore) hasRecoveryVolumes(owner string) bool { - if !s.recoveryVolumesPresent.Load() { + if !s.anyRetainedVolume.Load() { return false } s.mu.Lock() From 920430d5421c0add32b05f8291f0a8af4558fdc3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:47:14 +0000 Subject: [PATCH 19/29] pool: cover invalid volume names and capture after acknowledgment Registration and unregistration validate the name before they retry a pending catalog write, so an invalid name answers ErrBadVolume (400) instead of the pending write's 503. Nothing pinned the unregister refusal or that ordering. The uncertain-introduction test refused capture but never showed it working again once a snapshot acknowledges the recovery reference. --- sandboxd/pool/volume_catalog_test.go | 14 +++++++++++--- sandboxd/pool/volume_retirement_test.go | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index f3281b35..17fd3c37 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -137,9 +137,12 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { t.Errorf("register %s %+v = %v, want %v", tc.name, tc.spec, err, tc.want) } } - for _, name := range []string{"data", "unexplained"} { - if err := m.UnregisterVolume(t.Context(), name); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Errorf("unregister %s: %v", name, err) + for _, tc := range []struct { + name string + want error + }{{"data", ErrVolumeRegistrationConflict}, {"unexplained", ErrVolumeRegistrationConflict}, {"bad/name", ErrBadVolume}} { + if err := m.UnregisterVolume(t.Context(), tc.name); !errors.Is(err, tc.want) { + t.Errorf("unregister %s = %v, want %v", tc.name, err, tc.want) } } } @@ -164,6 +167,11 @@ func TestRuntimeCatalogPendingWriteConverges(t *testing.T) { if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("failed retry cleared pending write: %v", err) } + for _, err := range []error{m.RegisterVolume(t.Context(), "bad/name", spec), m.UnregisterVolume(t.Context(), "bad/name")} { + if !errors.Is(err, ErrBadVolume) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("invalid name behind a pending write: %v", err) + } + } m.retryVolumeCatalog(t.Context()) for _, volumes := range [][]types.Volume{nil, {{Name: "data"}}} { if err := claimAndRelease(t, m, volumes...); err != nil { diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index a334e5ef..e72d1ed0 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -142,6 +142,9 @@ func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { if _, err = m.ClaimProvision(t.Context(), testKey, writer); err != nil { t.Fatalf("successful whole snapshot did not retire attempted row: %v", err) } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retired-introduction", ""); err != nil { + t.Fatalf("capture after acknowledgment: %v", err) + } } func TestClaimRecoveryReferencesRetireOnlyOnAcknowledgedWrite(t *testing.T) { From 70f98a92cb979e817772ab1f4baaa2e19a016b9e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:16:49 +0000 Subject: [PATCH 20/29] docs: trim volume fence prose Keep the observable 503 instead of fence internals, plus the one assumption a reader needs: a stopped VM holds no volumes only because sandboxd controls VM starts and disk attachments in its namespace. --- docs/sandboxd-api.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 18b0238f..ac8200ba 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -311,13 +311,10 @@ recovery snapshot return 409. Persistence uncertainty returns 503 with `Retry-After` and the same retry semantics as PUT. A failed DELETE can complete later. An empty GET view does not establish that recovery references retired. -Stale VM cleanup retains named holds after a complete running inventory. -An ordinary stopped VM needs no volume holds. An inferred stopped result with -`stale: true`, inspection errors, omitted or null running disk lists, and busy -creating owners retain a node-wide admission fence. Any present inventory -during busy cleanup retains that fence. Confirmed absence or removal clears it. -This assumes sandboxd controls starts and attachments in its VM namespace. -Volume-free claims remain available. Cleanup retries continue. +Volume claims and registrations also return 503 with `Retry-After` while the +node removes a leftover VM whose attached volumes it cannot determine. A +stopped leftover VM holds none, which assumes sandboxd controls VM starts and +disk attachments in its namespace. Volume-free claims are unaffected. Before downgrading to a daemon without runtime registration, drain runtime references or supply equivalent static volume config and finish pending live From 568a81af6b0c5668565828558702c3f4d2b621b0 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:49:42 +0000 Subject: [PATCH 21/29] docs: state the volume access-list rule and operator duties once The tenants rule and the operator's backing duties were restated with small differences in the API reference, deploy guide, security model and live-volume example. Keep each in the API reference and link to it from the others. Drop the claim that temporary replacement files are not loaded: the daemon reads only the one record path, so the sentence described no behaviour of its own. Rewrap the overlong lines. --- docs/deploy.md | 16 ++++----------- docs/live-volumes.md | 11 ++++++----- docs/sandboxd-api.md | 47 +++++++++++++++++++++++++++----------------- docs/security.md | 27 ++++++++++--------------- 4 files changed, 49 insertions(+), 52 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 51938d7b..ae34a812 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -119,7 +119,7 @@ gets the intersection of the two. | `advertise_addr` | = `listen` | internal HTTP host:port for peer traffic and preview forwarding; also used by clients when `client_advertise` is unset. Must be routable when `listen` is a wildcard; a node with `mesh` set refuses to load while it names an unspecified host | | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | -| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, and a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later); root-owned claims have access; live additions use the claim owner's access list. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | @@ -247,12 +247,6 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes -Volume `tenants` entries use the tenant API name syntax. Static configuration -warns about invalid names and skips them during access checks. A nonempty list containing -only invalid names grants no tenant access. Runtime registration rejects invalid -tenant names. Valid names for tenants that have not yet been created remain -accepted. - Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes @@ -329,11 +323,9 @@ Warm candidates retain their normal ranking, but a volume claim's candidate set, promoted-template intersection, and redirect follow the fleet-wide rule in [cluster](cluster.md#volumes-and-placement). -An empty catalog `tenants` list allows every authenticated scope; a nonempty -list limits the image to those claim owners. Root-owned claims can use any -entry. Live additions use the claim owner's ACL, including requests sent by root. A -listed tenant the set no longer holds grants nothing and logs a warning at -boot. +Catalog `tenants` lists follow the +[access-list rule](sandboxd-api.md#volume-access-lists). A listed tenant the +set no longer holds grants nothing and logs a warning at boot. `GET /v1/volumes` reports the caller-visible fleet union and holder count, plus the answering node's current local availability and whether the entry is `writable`, without exposing host paths or node addresses. Applied names, diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 66ff435f..43fa04cb 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,11 +9,12 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. -A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) -without editing static config. Registration persists only node-local catalog -metadata and does not manage disk files. Omitted or empty runtime `tenants` -lists permit every authenticated scope. List tenants explicitly to restrict -access. Live changes remain limited to mounted read-only entries. +A configured node root token can +[register backing paths](sandboxd-api.md#put-v1volumesname) without editing +static config. Registration persists only node-local catalog metadata and does +not manage disk files. Runtime `tenants` lists follow the +[access-list rule](sandboxd-api.md#volume-access-lists). Live changes remain +limited to mounted read-only entries. Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ac8200ba..948c15ae 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -8,8 +8,8 @@ All bodies are JSON. Three token kinds: preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces (the per-id sandbox reads, `PUT /v1/sandboxes/{id}/instance-metadata`, `GET /v1/info`, - `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, `GET /metrics`, - `GET /v1/checkpoints/{id}/blob`) + `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, + `GET /metrics`, `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. On a node with a shared [`meta_store`](deploy.md#shared-tenant-database), a tenant token the node @@ -270,16 +270,10 @@ duplicate fields, and trailing JSON return 400. - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. - `path` uses static config's absolute-path validation. Registration does not open the backing or check its type or availability. A missing backing keeps - its registration but cannot be attached. Operators must keep the path and - target stable and separate from daemon cleanup directories, including - `/readers`, `/goldens`, and `/fork-*`. + its registration but cannot be attached. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. -- `tenants` grants named tenants self-service access. Omitted or empty permits - every authenticated scope for both static and runtime entries. Runtime - registration rejects invalid tenant names. Static config skips invalid names - and grants no tenant access if a nonempty list has no valid names. - A tenant name can be registered before that tenant exists. +- `tenants` is the entry's [access list](#volume-access-lists). Success is 204. An identical normalized registration is idempotent. A different binding, changed fields, or a static name or cleaned path collision returns 409. @@ -295,12 +289,29 @@ unrelated published names remain usable. Every catalog writer retries the accepted change before accepting another change. The node stores metadata in `data_dir/runtime-volumes.json` independently of -`sync_claims` and loads it before claim recovery. New records use mode 0600. Malformed -or conflicting metadata prevents startup. Temporary replacement files are not -loaded. Registration creates, grows, and deletes no backing disks. The catalog -is node-local with one sandboxd writer. The controller coordinates cross-node -ownership, symlink and hardlink aliases, backing availability, and storage -placement outside cleanup directories. +`sync_claims` and loads it before claim recovery. New records use mode 0600. +Malformed or conflicting metadata prevents startup. Registration creates, +grows, and deletes no backing disks. The catalog is node-local with one +sandboxd writer. + +The operator or controller owns each backing: supply an image Cocoon supports, +keep the path and its symlink target stable while registered, coordinate +symlink and hardlink aliases, cross-node ownership, and availability, and keep +backings outside daemon cleanup directories such as `/readers`, +`/goldens`, and `/fork-*`. + +### Volume access lists + +Static and runtime entries share one `tenants` rule. An omitted or empty list +permits every authenticated scope. A nonempty list permits only claims owned +by the listed tenants, and root-owned claims can use any entry. Live additions +check the claim owner's list even when root sends the request; held names +finish repair or removal without a new grant, and a recovery-only reference +grants no fresh attachment. Names use the tenant API syntax: runtime +registration rejects an invalid name, while static config logs a warning and +skips it, so a nonempty static list with no valid names permits no tenant. A +valid name for a tenant that does not exist yet grants nothing until the +tenant is created. ## DELETE /v1/volumes/{name} @@ -342,8 +353,8 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed PUT or PATCH adds a top-level `error` naming the refusal, and the affected -state may carry its own `error`. `converging` is true while a request worker is running or a recovery-only -reference awaits an acknowledged claim write. +state may carry its own `error`. `converging` is true while a request worker +is running or a recovery-only reference awaits an acknowledged claim write. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an diff --git a/docs/security.md b/docs/security.md index 841e9830..df0b8a48 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,25 +94,20 @@ rather than confirming existence, and operator surfaces answer tenants 403. The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is -node-local. An empty `tenants` list permits every authenticated scope for -both static and runtime entries. A nonempty list grants the named tenants -self-service access. For live additions, the claim owner must satisfy the -entry's ACL, even when root sends the request. Runtime registration rejects -invalid tenant names. Static config warns and skips them. A nonempty static -list with no valid names permits no tenant access. A valid name that does not -yet identify a tenant grants no access until that tenant exists. -Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant -cannot enumerate restricted entries by probing. Gossip carries names only. +node-local. Entry `tenants` lists follow the [access-list +rule](sandboxd-api.md#volume-access-lists). Claim lookup returns byte-identical +errors for an unknown and a forbidden name, and catalog discovery filters before +replying, so a tenant cannot enumerate restricted entries by probing. Gossip +carries names only. Neither gossip, discovery, persisted claims, usage events, nor the sandbox index exposes host image paths or access lists. Runtime registration and unregistration require a configured root `api_token`; these endpoints return 403 when the token is unset. New catalog records use mode 0600. Reads and replacements use the daemon's filesystem permissions. -Registration uses the same path validation as static configuration. It does -not verify backing type -or pin backing identity. Operators must supply a backing supported by Cocoon, -prevent path or symlink target replacement while registered, and coordinate aliases. +Registration uses the same path validation as static configuration. It does not +verify backing type or pin backing identity; the operator duties this leaves are +listed under [PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path @@ -127,10 +122,8 @@ immutable and publish a new name/path for new content. Changing a running claim's volumes needs the root token or the owning tenant's API token. A sandbox token cannot, so a guest holding its own sandbox token -cannot attach another dataset, and every addition passes the same access list -as a claim. Already-held references can finish repair or removal without a -new ACL grant. A recovery-only reference protects ownership and does not grant -permission for a fresh attachment. +cannot attach another dataset, and every addition passes the same +[access list](sandboxd-api.md#volume-access-lists) as a claim. A writable entry (`writable: true`) adds a channel the read-only model does not have: whichever tenant is permitted to claim it `rw` changes what every From 30b65a399f9f0ec69300e195b260cbac1988ad85 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:09:44 +0000 Subject: [PATCH 22/29] pool: reject non-UTF-8 runtime volume paths writeCatalog ignores the json.Marshal error, so a Go caller that bypasses the HTTP JSON decode could register a path that is not valid UTF-8 and persist a runtime volume catalog the daemon cannot load on next start. Validate the path in normalizeRegistration and reject it as ErrBadVolume (400 at the HTTP layer) so the catalog file always stays loadable. The name, directio, and tenant fields are already restricted to ASCII. --- sandboxd/pool/volume_catalog.go | 4 ++++ sandboxd/pool/volume_catalog_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 938fa687..fef2bfa3 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "sync" + "unicode/utf8" "github.com/projecteru2/core/log" @@ -212,6 +213,9 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) + if !utf8.ValidString(spec.Path) { + return nil, fmt.Errorf("%w: volume %q path must be valid UTF-8", ErrBadVolume, name) + } if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, }}); err != nil { diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 17fd3c37..da2465dd 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -132,11 +132,15 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { {"runtime", types.VolumeRegistration{Path: "relative"}, ErrBadVolume}, {"runtime", types.VolumeRegistration{Path: other, DirectIO: "invalid"}, ErrBadVolume}, {"runtime", types.VolumeRegistration{Path: other, Tenants: []string{"has space"}}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: "/srv/\xff"}, ErrBadVolume}, } { if err := m.RegisterVolume(t.Context(), tc.name, tc.spec); !errors.Is(err, tc.want) { t.Errorf("register %s %+v = %v, want %v", tc.name, tc.spec, err, tc.want) } } + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) || m.catalog.entries["runtime"] != nil { + t.Fatalf("rejected registration persisted: stat=%v entry=%+v", err, m.catalog.entries["runtime"]) + } for _, tc := range []struct { name string want error From 00f62c778e26bd29b02e324858cc94950b035b16 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:38:00 +0000 Subject: [PATCH 23/29] pool: keep the volume fence for a stale stopped VM Cocoon reports state=stopped, stale=true, pid=0 when it has lost a VM's PID file. That stop is inferred, not observed: the VMM can still be running and writing its disks. Since e942d31 every stopped VM took no holds, so a failed removal of such a VM lifted the node-wide fence and let a second writer attach, or the name be unregistered, while the old VMM might still hold it. Restore VMRecord.Stale and keep the fence for a stale stop on both the sweep and the busy stale-create retry, which share the classification. An ordinary stopped VM still takes no holds. The fence table gains a stale row for each path, and the inspect test again checks that the flag parses. --- docs/sandboxd-api.md | 8 +++++--- sandboxd/engine/cocoond_test.go | 8 ++++++++ sandboxd/pool/pool_test.go | 5 +++-- sandboxd/pool/reconcile.go | 3 ++- sandboxd/pool/volume_cleanup_test.go | 6 +++++- sandboxd/types/types.go | 2 ++ 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 948c15ae..d6658f66 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -323,9 +323,11 @@ recovery snapshot return 409. Persistence uncertainty returns 503 with later. An empty GET view does not establish that recovery references retired. Volume claims and registrations also return 503 with `Retry-After` while the -node removes a leftover VM whose attached volumes it cannot determine. A -stopped leftover VM holds none, which assumes sandboxd controls VM starts and -disk attachments in its namespace. Volume-free claims are unaffected. +node removes a leftover VM whose attached volumes it cannot determine, +including one cocoon reports `stale` because it only inferred the stop from a +lost PID file. Any other stopped leftover VM holds none, which assumes sandboxd +controls VM starts and disk attachments in its namespace. Volume-free claims +are unaffected. Before downgrading to a daemon without runtime registration, drain runtime references or supply equivalent static volume config and finish pending live diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 027b264c..0db76951 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -45,6 +45,8 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { fakeCocoon(t, "#!/bin/sh\n[ \"$1 $2\" = \"vm inspect\" ] || exit 2\n"+ "if [ \"$3\" = sbx-1 ]; then echo '{\"config\":{\"name\":\"sbx-1\"},\"state\":\"running\","+ "\"attached_devices\":{\"disks\":[{\"id\":\"disk-a\",\"name\":\"imagenet\",\"path\":\"/v/a.img\",\"readonly\":true},{\"id\":\"disk-b\",\"name\":\"scratch\",\"path\":\"/v/b.img\"}]}}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stale ]; then echo '{\"state\":\"stopped\",\"stale\":true,\"pid\":0}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stopped ]; then echo '{\"state\":\"stopped\"}'; exit 0; fi\n"+ "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") @@ -52,6 +54,12 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } + for _, name := range []string{"sbx-stale", "sbx-stopped"} { + rec, present, err := e.Inspect(t.Context(), name) + if err != nil || !present || rec.State != "stopped" || rec.Stale != (name == "sbx-stale") { + t.Errorf("Inspect %s lost stopped classification: %+v, %v, %v", name, rec, present, err) + } + } if _, ok, err := e.Inspect(t.Context(), "sbx-gone"); err != nil || ok { t.Errorf("Inspect of a missing VM = %v, %v; want not found without an error", ok, err) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index b29c4e86..1318da41 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1255,6 +1255,7 @@ type fakeEngine struct { detachVolumeErr error liveVolumeHook func(string) error stopped map[string]bool + stale map[string]bool inspectHook func() error disks map[string][]types.VMDisk creating map[string]bool @@ -1296,7 +1297,7 @@ type fakeEngine struct { func newFakeEngine() *fakeEngine { return &fakeEngine{ - vms: map[string]string{}, stopped: map[string]bool{}, creating: map[string]bool{}, pids: map[string]int{}, + vms: map[string]string{}, stopped: map[string]bool{}, stale: map[string]bool{}, creating: map[string]bool{}, pids: map[string]int{}, attachDirty: map[string]bool{}, removeSeenOps: map[string][]string{}, removeSeenDirty: map[string][]string{}, metadataDocs: map[string]string{}, guestEnvs: map[string]string{}, sockRoot: "/vsock", } @@ -1771,7 +1772,7 @@ func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { case f.stopped[name]: state = vmStateStopped } - rec := types.VMRecord{State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} + rec := types.VMRecord{Stale: f.stale[name], State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} if f.tap != "" { rec.NetworkConfigs = []types.VMNetConfig{{TAP: f.tap}} } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 9e3c641e..3ccbfa06 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -205,7 +205,8 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum holds[i].Mode = types.VolumeModeRW } } - case present && inspected.State != vmStateStopped: + // a stale stop is inferred from a lost PID file; the VMM may still be writing its disks + case present && (inspected.State != vmStateStopped || inspected.Stale): return td } m.mu.Lock() diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 9caf6edb..6cd4aacb 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -99,6 +99,7 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { func TestStaleVMFenceFollowsInspection(t *testing.T) { writer, reader := []types.VMDisk{{Name: "data"}}, []types.VMDisk{{Name: "data", ReadOnly: true}} busy, fenced, inspectErr := ErrVolumeBusy, ErrVolumeCatalogUnavailable, errors.New("inspection unavailable") + staleStop := vmStateStopped + " stale" for _, tc := range []struct { name, swept, listed, state string disks []types.VMDisk @@ -111,12 +112,14 @@ func TestStaleVMFenceFollowsInspection(t *testing.T) { {"running empty", vmStateRunning, "sbx-stale", vmStateRunning, []types.VMDisk{}, nil, [3]error{}, nil}, {"running omitted disks", vmStateRunning, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, {"stopped", vmStateRunning, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"stopped stale (pid file lost)", vmStateRunning, "sbx-stale", staleStop, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"inspected creating", vmStateRunning, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"inspect error", vmStateRunning, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, {"busy creating", vmStateCreating, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"busy running", vmStateCreating, "sbx-stale", vmStateRunning, reader, nil, [3]error{nil, busy, nil}, nil}, {"busy omitted disks", vmStateCreating, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, {"busy stopped", vmStateCreating, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"busy stopped stale", vmStateCreating, "sbx-stale", staleStop, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"busy absent", vmStateCreating, "sbx-other", vmStateRunning, writer, nil, [3]error{}, nil}, {"busy inspect error", vmStateCreating, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, } { @@ -125,7 +128,8 @@ func TestStaleVMFenceFollowsInspection(t *testing.T) { m := newVolumeManager(t, eng, liveCatalog(t)) name := "sbx-stale" eng.vms[tc.listed], eng.removeErrFor, eng.staleOutcome = "/vsock/stale", name, engine.StaleCreateBusy - eng.stopped[tc.listed], eng.creating[tc.listed] = tc.state == vmStateStopped, tc.state == vmStateCreating + eng.stopped[tc.listed], eng.creating[tc.listed] = tc.state == vmStateStopped || tc.state == staleStop, tc.state == vmStateCreating + eng.stale[tc.listed] = tc.state == staleStop eng.inspectHook = func() error { return tc.inspectErr } if m.removeStaleVM(t.Context(), name, types.VMRecord{State: tc.swept, Config: types.VMConfig{Name: name}}) { t.Fatal("stale VM reported gone") diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..a92c2afd 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -731,6 +731,8 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { + // Stale marks a stop cocoon inferred from a lost PID file, not one it observed. + Stale bool `json:"stale,omitzero"` State string `json:"state"` PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` From 0df2bc5502635be00b3f6c2ebf69454e291d97e4 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:39:20 +0000 Subject: [PATCH 24/29] pool: scan published claims before retiring a volume registration 969b7ab dropped the claim walk from volumeReferenced on the premise that every claimed volume is counted in the admission holders or in the claim store's retained counts. AdoptArchived breaks that premise: it publishes the adopted claim in m.claimed without taking holders, and the store counts its volumes only once the claims.json commit starts. In that window unregister, or a re-registration under the same name, succeeded while a published claim still named the volume. Walk m.claimed under m.mu again, matching applied and wanted volumes by name without copying, ahead of the store check. The test adopts a peer's row that names a volume while an earlier claims.json write holds the adoption's own write back, and requires unregister to refuse. --- sandboxd/pool/volume_catalog.go | 11 ++++++- sandboxd/pool/volume_catalog_test.go | 49 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index fef2bfa3..0c5c4278 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -141,7 +141,16 @@ func (m *Manager) loadRuntimeVolumes() error { } func (m *Manager) volumeReferenced(name string) bool { - return m.catalog.holders[name] != (volumeHolders{}) || m.store.referencesVolume(name) + if m.catalog.holders[name] != (volumeHolders{}) { + return true + } + // an adopted claim is published with no holders before its claims.json write counts it + for _, sb := range m.claimed { + if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { + return true + } + } + return m.store.referencesVolume(name) } func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[string]*catalogVolume) error { diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index da2465dd..91ec9d83 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -2,6 +2,7 @@ package pool import ( "context" + "encoding/json/v2" "errors" "fmt" "maps" @@ -371,6 +372,54 @@ func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { } } +func TestAdoptedClaimReferencesVolumeBeforeItsClaimWrite(t *testing.T) { + c := newFailoverCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + if err := b.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "adopted", "bytes")}); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, a, testKey) + mustArchive(t, a, sb) + rec, _, err := c.reg.Get(t.Context(), sb.ID) + claim := &types.Sandbox{} + if err == nil { + err = json.Unmarshal(rec.Claim, claim) + } + // the row is a peer's record, so the adopter cannot assume it names no volumes + claim.Volumes = []types.Volume{{Name: "data", Mount: "/data"}} + if err == nil { + rec.Claim, err = json.Marshal(claim) + } + if err == nil { + _, err = c.reg.Put(t.Context(), rec) + } + if err != nil { + t.Fatal(err) + } + c.expire("a") + // an earlier claims.json write in flight keeps the adoption's write from starting + b.store.writeMu.Lock() + unlock := sync.OnceFunc(b.store.writeMu.Unlock) + t.Cleanup(unlock) + adopted := make(chan error, 1) + go func() { + ok, adoptErr := b.AdoptArchived(t.Context(), sb.ID, sb.Token) + if adoptErr == nil && !ok { + adoptErr = errors.New("not adopted") + } + adopted <- adoptErr + }() + waitFor(t, func() bool { _, held := b.byID(sb.ID); return held }) + err = b.UnregisterVolume(t.Context(), "data") + unlock() + if adoptErr := <-adopted; adoptErr != nil { + t.Fatal(adoptErr) + } + if !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister while a published adoption references the volume = %v", err) + } +} + func blockCatalogRename(t *testing.T, m *Manager) func() { t.Helper() record := filepath.Join(m.dataDir, runtimeVolumesFile) From 7c513c9e4b899abb457d4d14ef637791c1d359fa Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:51:50 +0000 Subject: [PATCH 25/29] pool: bound runtime volume registration by volume_roots The root API token can register any absolute host path as a volume, writable and open to every scope if the request asks for it, so the token reaches host files such as data_dir/claims.json over the network. Static config has the same reach but needs host access to change. An optional volume_roots list limits runtime registration to paths that, once cleaned, equal a root or sit under one on a separator boundary, so /srv/data2 does not match /srv/data and "/" admits everything. Anything else is the existing bad-volume error, a 400. The check is lexical, like static path validation; symlinks are not resolved. Config load rejects a relative or empty root and stores cleaned roots. Persisted registrations go through the same check at startup, so a narrowed list cannot leave an out-of-root registration claimable; like other invalid catalog metadata it prevents startup until the operator unregisters it or widens the roots. Static volumes are not checked, and an empty list keeps today's behaviour. --- sandboxd/config/config.go | 12 ++++++-- sandboxd/config/config_test.go | 7 ++++- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume_catalog.go | 16 ++++++++--- sandboxd/pool/volume_catalog_test.go | 41 ++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 6b2c9295..03a8a694 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -277,8 +277,10 @@ type Config struct { DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` Volumes []VolumeSpec `json:"volumes,omitempty"` - Mesh *MeshConfig `json:"mesh,omitempty"` - Pools []PoolSpec `json:"pools"` + // VolumeRoots bounds runtime registration paths only; static volumes stay unchecked. + VolumeRoots []string `json:"volume_roots,omitempty"` + Mesh *MeshConfig `json:"mesh,omitempty"` + Pools []PoolSpec `json:"pools"` } // HasEgress reports whether the node can attach egress-lane VMs. @@ -400,6 +402,12 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { + for i, root := range c.VolumeRoots { + if !filepath.IsAbs(root) { + return fmt.Errorf("volume_roots %q must be an absolute path", root) + } + c.VolumeRoots[i] = filepath.Clean(root) + } validation := slices.Clone(c.Volumes) for i, volume := range c.Volumes { validation[i].Tenants = slices.DeleteFunc(slices.Clone(volume.Tenants), func(tenant string) bool { diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 144c378a..b1e51586 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -167,6 +167,8 @@ func TestLoadRejectsInvalid(t *testing.T) { {"duplicate volume name", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img"},{"name":"data","path":"/srv/datasets/b.img"}]}`, "duplicate volume"}, {"relative volume path", `{"pools":[],"volumes":[{"name":"data","path":"datasets/a.img"}]}`, "path must be absolute"}, {"bad volume directio", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img","directio":"yes"}]}`, "directio must be"}, + {"relative volume root", `{"pools":[],"volume_roots":["srv/datasets"]}`, "volume_roots"}, + {"empty volume root", `{"pools":[],"volume_roots":[""]}`, "volume_roots"}, {"reader without warm", `{"pools":[{"template":"rt:24.04","warm":0,"reader":true}]}`, "reader needs warm"}, } { t.Run(tt.name, func(t *testing.T) { @@ -292,7 +294,7 @@ func TestLoadReadsReader(t *testing.T) { } func TestLoadAcceptsVolumes(t *testing.T) { - path := writeConfig(t, `{"pools":[],"volumes":[ + path := writeConfig(t, `{"pools":[],"volume_roots":["/srv/models/"],"volumes":[ {"name":"imagenet","path":"/srv/datasets/imagenet.img"}, {"name":"weights-llama","path":"/srv/datasets/llama.img","directio":"on"}]}`) cfg, err := Load(path) @@ -303,6 +305,9 @@ func TestLoadAcceptsVolumes(t *testing.T) { cfg.Volumes[1].DirectIO != types.DirectIOOn { t.Errorf("volumes = %+v", cfg.Volumes) } + if !slices.Equal(cfg.VolumeRoots, []string{"/srv/models"}) { + t.Errorf("volume_roots = %q, want the cleaned root with static volumes outside it accepted", cfg.VolumeRoots) + } } func TestLoadAcceptsVolumeTenantAccessList(t *testing.T) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 94c10e83..112f1ba6 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -450,7 +450,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), holders: map[string]volumeHolders{}}, + catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), roots: cfg.VolumeRoots, holders: map[string]volumeHolders{}}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 0c5c4278..06263410 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "slices" + "strings" "sync" "unicode/utf8" @@ -36,6 +37,7 @@ type catalogWrite struct { type volumeCatalog struct { entries map[string]*catalogVolume + roots []string pending *catalogWrite holders map[string]volumeHolders unknown int @@ -51,7 +53,7 @@ func (c *volumeCatalog) admissionError(name string) error { // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { - volume, err := normalizeRegistration(name, spec) + volume, err := normalizeRegistration(name, spec, m.catalog.roots) if err != nil { return err } @@ -124,7 +126,7 @@ func (m *Manager) loadRuntimeVolumes() error { } entries := maps.Clone(m.catalog.entries) for name, spec := range record.Volumes { - volume, normalizeErr := normalizeRegistration(name, spec) + volume, normalizeErr := normalizeRegistration(name, spec, m.catalog.roots) if normalizeErr != nil { return fmt.Errorf("runtime volume catalog: %w", normalizeErr) } @@ -220,7 +222,7 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { +func normalizeRegistration(name string, spec types.VolumeRegistration, roots []string) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if !utf8.ValidString(spec.Path) { return nil, fmt.Errorf("%w: volume %q path must be valid UTF-8", ErrBadVolume, name) @@ -230,9 +232,15 @@ func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalog }}); err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } + path := filepath.Clean(spec.Path) + if len(roots) > 0 && !slices.ContainsFunc(roots, func(root string) bool { + return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) + }) { + return nil, fmt.Errorf("%w: volume %q path is outside volume_roots", ErrBadVolume, name) + } return &catalogVolume{ runtime: true, - disk: engine.VolumeSpec{Name: name, Path: filepath.Clean(spec.Path), DirectIO: spec.DirectIO}, + disk: engine.VolumeSpec{Name: name, Path: path, DirectIO: spec.DirectIO}, tenants: slices.Compact(slices.Sorted(slices.Values(spec.Tenants))), writable: spec.Writable, }, nil diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 91ec9d83..42a57495 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -152,6 +152,47 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { } } +func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { + newRootedManager := func(t *testing.T, dataDir string, roots ...string) (*Manager, error) { + t.Helper() + cfg := &config.Config{DataDir: dataDir, VolumeRoots: roots} + return NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), testBackends(t, cfg)) + } + for _, tc := range []struct { + name, path string + roots []string + want error + }{ + {"inside a root", "/srv/data/a.img", []string{"/mnt/other", "/srv/data"}, nil}, + {"equal to a root", "/srv/data/", []string{"/srv/data"}, nil}, + {"sibling sharing a prefix", "/srv/data2/a.img", []string{"/srv/data"}, ErrBadVolume}, + {"dot-dot out of a root", "/srv/data/../etc/a.img", []string{"/srv/data"}, ErrBadVolume}, + {"outside every root", "/etc/a.img", []string{"/srv/data"}, ErrBadVolume}, + {"filesystem root", "/etc/a.img", []string{"/"}, nil}, + {"no roots", "/etc/a.img", nil, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + m, err := newRootedManager(t, t.TempDir(), tc.roots...) + if err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: tc.path}); !errors.Is(err, tc.want) { + t.Fatalf("register %s under %v = %v, want %v", tc.path, tc.roots, err, tc.want) + } + }) + } + m := newVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/etc/a.img"}); err != nil { + t.Fatal(err) + } + if _, err := newRootedManager(t, m.dataDir, "/etc"); err != nil { + t.Fatalf("restart with the registration inside volume_roots: %v", err) + } + if _, err := newRootedManager(t, m.dataDir, "/srv/data"); err == nil { + t.Fatal("restart accepted a persisted registration outside volume_roots") + } +} + func TestRuntimeCatalogPendingWriteConverges(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) From 42ef827a0d5048cdf6fe222cf5435437b42fd9bd Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:51:50 +0000 Subject: [PATCH 26/29] docs: document volume_roots and the root token's path reach Operators need to know that the root token can register any absolute host path, and that volume_roots is the way to bound it. The config reference states the lexical rule, its symlink limit, the startup refusal and that static entries are exempt; security.md says to protect the token like host root or set the list, and links to the access-list rule instead of restating it. --- docs/deploy.md | 1 + docs/sandboxd-api.md | 8 +++++--- docs/security.md | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index ae34a812..714bb6eb 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,6 +120,7 @@ gets the intersection of the two. | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. A registration already in `runtime-volumes.json` outside the roots prevents startup. Keep `data_dir` outside every root. Unset accepts any absolute path | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index d6658f66..518cb71f 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -268,9 +268,11 @@ duplicate fields, and trailing JSON return 400. ``` - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. -- `path` uses static config's absolute-path validation. Registration does not - open the backing or check its type or availability. A missing backing keeps - its registration but cannot be attached. +- `path` uses static config's absolute-path validation. When the node sets + [`volume_roots`](deploy.md#configuration), the cleaned path must equal or sit + under one of them. Registration does not open the backing or check its type + or availability. A missing backing keeps its registration but cannot be + attached. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. - `tenants` is the entry's [access list](#volume-access-lists). diff --git a/docs/security.md b/docs/security.md index df0b8a48..a3973e2a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -108,6 +108,10 @@ mode 0600. Reads and replacements use the daemon's filesystem permissions. Registration uses the same path validation as static configuration. It does not verify backing type or pin backing identity; the operator duties this leaves are listed under [PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). +Unless [`volume_roots`](deploy.md#configuration) restricts it, the root token can +register any absolute host path, writable if requested, for every scope its +[access list](sandboxd-api.md#volume-access-lists) permits. Protect the root +token like host root, or set `volume_roots`. Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path From de8300bf2613feb417c2511f594544fdbe6ee19e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 01:01:01 +0000 Subject: [PATCH 27/29] pool: skip saved registrations outside volume_roots at startup Narrowing volume_roots left the daemon unable to start while any saved registration sat outside the new roots, so tightening the bound took the node down until the operator widened it again or edited runtime-volumes.json by hand. Startup now logs a warning naming the volume and leaves it out of the loaded catalog, so it cannot be claimed and the daemon serves the rest. The skip runs after the whole file validates, so malformed or conflicting metadata still refuses startup as before. A recovered claim that names the skipped volume keeps its holders and gets the same refusal as any name missing from the catalog. The entry stays on disk until the next registration change rewrites the file from the loaded catalog, which drops it. --- docs/deploy.md | 2 +- docs/security.md | 14 ++++---- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume_catalog.go | 32 +++++++++++------ sandboxd/pool/volume_catalog_test.go | 51 +++++++++++++++++++++++----- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 714bb6eb..f67686a8 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,7 +120,7 @@ gets the intersection of the two. | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | -| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. A registration already in `runtime-volumes.json` outside the roots prevents startup. Keep `data_dir` outside every root. Unset accepts any absolute path | +| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. After narrowing, a registration already in `runtime-volumes.json` outside the roots is skipped with a warning at startup, so it cannot be claimed, and the next registration change drops it from the file. Keep `data_dir` outside every root. Unset accepts any absolute path | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | diff --git a/docs/security.md b/docs/security.md index a3973e2a..107bd7cc 100644 --- a/docs/security.md +++ b/docs/security.md @@ -105,13 +105,13 @@ exposes host image paths or access lists. Runtime registration and unregistration require a configured root `api_token`; these endpoints return 403 when the token is unset. New catalog records use mode 0600. Reads and replacements use the daemon's filesystem permissions. -Registration uses the same path validation as static configuration. It does not -verify backing type or pin backing identity; the operator duties this leaves are -listed under [PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). -Unless [`volume_roots`](deploy.md#configuration) restricts it, the root token can -register any absolute host path, writable if requested, for every scope its -[access list](sandboxd-api.md#volume-access-lists) permits. Protect the root -token like host root, or set `volume_roots`. +Registration uses static configuration's path validation plus, when set, the +[`volume_roots`](deploy.md#configuration) bound. It does not verify backing type +or pin backing identity; the operator duties this leaves are listed under +[PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). Without +`volume_roots` the root token can register any absolute host path, writable if +requested, for every scope its [access list](sandboxd-api.md#volume-access-lists) +permits. Protect the root token like host root, or set `volume_roots`. Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 112f1ba6..f385c897 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -485,7 +485,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg writable: volume.Writable, } } - if err := m.loadRuntimeVolumes(); err != nil { + if err := m.loadRuntimeVolumes(ctx); err != nil { return nil, err } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 06263410..fb5efe67 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -53,10 +53,13 @@ func (c *volumeCatalog) admissionError(name string) error { // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { - volume, err := normalizeRegistration(name, spec, m.catalog.roots) + volume, err := normalizeRegistration(name, spec) if err != nil { return err } + if !m.catalog.withinRoots(volume.disk.Path) { + return fmt.Errorf("%w: volume %q path is outside volume_roots", ErrBadVolume, name) + } m.catalog.writeMu.Lock() defer m.catalog.writeMu.Unlock() if err = m.commitPendingCatalog(ctx); err != nil { @@ -109,7 +112,7 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { return m.writeCatalog(ctx, name, entries) } -func (m *Manager) loadRuntimeVolumes() error { +func (m *Manager) loadRuntimeVolumes(ctx context.Context) error { raw, err := os.ReadFile(filepath.Join(m.dataDir, runtimeVolumesFile)) if errors.Is(err, os.ErrNotExist) { return nil @@ -126,7 +129,7 @@ func (m *Manager) loadRuntimeVolumes() error { } entries := maps.Clone(m.catalog.entries) for name, spec := range record.Volumes { - volume, normalizeErr := normalizeRegistration(name, spec, m.catalog.roots) + volume, normalizeErr := normalizeRegistration(name, spec) if normalizeErr != nil { return fmt.Errorf("runtime volume catalog: %w", normalizeErr) } @@ -138,6 +141,13 @@ func (m *Manager) loadRuntimeVolumes() error { } entries[name] = volume } + // skip only once every entry validated, so other bad metadata still refuses startup + for name := range record.Volumes { + if path := entries[name].disk.Path; !m.catalog.withinRoots(path) { + log.WithFunc("pool.loadRuntimeVolumes").Warnf(ctx, "runtime volume %q: skipping image %s outside volume_roots", name, path) + delete(entries, name) + } + } m.catalog.entries = entries return nil } @@ -222,7 +232,13 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func normalizeRegistration(name string, spec types.VolumeRegistration, roots []string) (*catalogVolume, error) { +func (c *volumeCatalog) withinRoots(path string) bool { + return len(c.roots) == 0 || slices.ContainsFunc(c.roots, func(root string) bool { + return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) + }) +} + +func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if !utf8.ValidString(spec.Path) { return nil, fmt.Errorf("%w: volume %q path must be valid UTF-8", ErrBadVolume, name) @@ -232,15 +248,9 @@ func normalizeRegistration(name string, spec types.VolumeRegistration, roots []s }}); err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } - path := filepath.Clean(spec.Path) - if len(roots) > 0 && !slices.ContainsFunc(roots, func(root string) bool { - return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) - }) { - return nil, fmt.Errorf("%w: volume %q path is outside volume_roots", ErrBadVolume, name) - } return &catalogVolume{ runtime: true, - disk: engine.VolumeSpec{Name: name, Path: path, DirectIO: spec.DirectIO}, + disk: engine.VolumeSpec{Name: name, Path: filepath.Clean(spec.Path), DirectIO: spec.DirectIO}, tenants: slices.Compact(slices.Sorted(slices.Values(spec.Tenants))), writable: spec.Writable, }, nil diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 42a57495..34d7697e 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -153,10 +153,10 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { } func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { - newRootedManager := func(t *testing.T, dataDir string, roots ...string) (*Manager, error) { + newRootedManager := func(t *testing.T, eng *fakeEngine, dataDir string, roots ...string) (*Manager, error) { t.Helper() cfg := &config.Config{DataDir: dataDir, VolumeRoots: roots} - return NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), testBackends(t, cfg)) + return NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) } for _, tc := range []struct { name, path string @@ -172,7 +172,7 @@ func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { {"no roots", "/etc/a.img", nil, nil}, } { t.Run(tc.name, func(t *testing.T) { - m, err := newRootedManager(t, t.TempDir(), tc.roots...) + m, err := newRootedManager(t, newFakeEngine(), t.TempDir(), tc.roots...) if err != nil { t.Fatal(err) } @@ -181,15 +181,50 @@ func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { } }) } - m := newVolumeManager(t, newFakeEngine(), nil) - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/etc/a.img"}); err != nil { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + images := map[string]string{"outside": writeVolumeImage(t, "outside", "bytes"), "inside": writeVolumeImage(t, "inside", "bytes")} + for name, path := range images { + if err := m.RegisterVolume(t.Context(), name, types.VolumeRegistration{Path: path}); err != nil { + t.Fatal(err) + } + } + outside, inside := filepath.Dir(images["outside"]), filepath.Dir(images["inside"]) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "outside"}}}) + if err != nil { t.Fatal(err) } - if _, err := newRootedManager(t, m.dataDir, "/etc"); err != nil { + wide, err := newRootedManager(t, eng, m.dataDir, outside, inside) + if err != nil || wide.catalogSnapshot()["outside"] == nil { t.Fatalf("restart with the registration inside volume_roots: %v", err) } - if _, err := newRootedManager(t, m.dataDir, "/srv/data"); err == nil { - t.Fatal("restart accepted a persisted registration outside volume_roots") + m2, err := newRootedManager(t, eng, m.dataDir, inside) + if err != nil { + t.Fatalf("restart refused a persisted registration outside volume_roots: %v", err) + } + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if got := m2.catalogSnapshot(); got["outside"] != nil || got["inside"] == nil { + t.Fatalf("narrowed catalog = %v, want only the in-root registration", slices.Sorted(maps.Keys(got))) + } + if got := volumeHoldersOf(m2, "outside"); got.readers != 1 { + t.Fatalf("recovered claim holders of the skipped registration = %+v", got) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "outside"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("claim of the skipped registration = %v, want ErrVolumeUnavailable", err) + } + if err = claimAndRelease(t, m2, types.Volume{Name: "inside"}); err != nil { + t.Fatalf("in-root registration: %v", err) + } + if err = m2.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("release of a recovered claim holding the skipped registration: %v", err) + } + if err = m2.RegisterVolume(t.Context(), "later", types.VolumeRegistration{Path: filepath.Join(inside, "later")}); err != nil { + t.Fatal(err) + } + if wide, err = newRootedManager(t, eng, m.dataDir, outside, inside); err != nil || wide.catalogSnapshot()["outside"] != nil { + t.Fatalf("a later catalog write kept the skipped registration: %v", err) } } From c5fa28ffffec1b8712e511307839c8f8ce584797 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 02:12:08 +0000 Subject: [PATCH 28/29] pool: group volume catalog methods --- sandboxd/pool/volume_catalog.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index fb5efe67..c5622f4d 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -51,6 +51,12 @@ func (c *volumeCatalog) admissionError(name string) error { return nil } +func (c *volumeCatalog) withinRoots(path string) bool { + return len(c.roots) == 0 || slices.ContainsFunc(c.roots, func(root string) bool { + return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) + }) +} + // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { volume, err := normalizeRegistration(name, spec) @@ -232,12 +238,6 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func (c *volumeCatalog) withinRoots(path string) bool { - return len(c.roots) == 0 || slices.ContainsFunc(c.roots, func(root string) bool { - return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) - }) -} - func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if !utf8.ValidString(spec.Path) { From 09bb47b30ae04207d815a05156c77580a6418a5f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 02:18:52 +0000 Subject: [PATCH 29/29] docs: note empty volume_roots and DELETE of a skipped registration An explicit empty list behaves like unset, and DELETE of a registration skipped at startup is a no-op until the next catalog write. Say both so operators are not surprised. --- docs/deploy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index f67686a8..a704a01b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,7 +120,7 @@ gets the intersection of the two. | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | -| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. After narrowing, a registration already in `runtime-volumes.json` outside the roots is skipped with a warning at startup, so it cannot be claimed, and the next registration change drops it from the file. Keep `data_dir` outside every root. Unset accepts any absolute path | +| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. After narrowing, a registration already in `runtime-volumes.json` outside the roots is skipped with a warning at startup, so it cannot be claimed, and the next registration change drops it from the file. A DELETE of a skipped name that nothing references returns 204 without rewriting the file. Keep `data_dir` outside every root. Unset or an empty list accepts any absolute path | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) |