From 2c2e02b742ed66a6318634a89bc63b959288db91 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 15:39:28 +0000 Subject: [PATCH 01/39] pool: persist runtime volume registrations Load root-token registrations before claim recovery and refuse unregister while claims, operations or VM cleanup retain a reference. Keep detached references protected until their removal is committed. --- docs/deploy.md | 11 +- docs/live-volumes.md | 6 + docs/sandboxd-api.md | 102 +++- docs/security.md | 25 +- e2e/volume_catalog_test.go | 112 ++++ sandboxd/config/config.go | 64 +- sandboxd/config/config_test.go | 29 + sandboxd/pool/checkpoint_references_test.go | 129 ++-- sandboxd/pool/claim.go | 6 +- sandboxd/pool/claims.go | 184 +++++- sandboxd/pool/claims_test.go | 12 +- sandboxd/pool/pool.go | 32 +- sandboxd/pool/pool_test.go | 21 +- sandboxd/pool/reconcile.go | 46 +- sandboxd/pool/reconcile_test.go | 4 +- sandboxd/pool/remove.go | 26 +- sandboxd/pool/replica.go | 3 +- sandboxd/pool/replica_test.go | 56 ++ sandboxd/pool/volume.go | 129 +++- sandboxd/pool/volume_acl_test.go | 152 +++++ sandboxd/pool/volume_bench_test.go | 81 +++ sandboxd/pool/volume_capture_test.go | 9 + sandboxd/pool/volume_catalog.go | 266 +++++++++ sandboxd/pool/volume_catalog_test.go | 617 ++++++++++++++++++++ sandboxd/pool/volume_cleanup_test.go | 543 +++++++++++++++++ sandboxd/pool/volume_convergence_test.go | 34 +- sandboxd/pool/volume_mutation.go | 34 +- sandboxd/pool/volume_mutation_test.go | 8 +- sandboxd/pool/volume_retirement_test.go | 416 +++++++++++++ sandboxd/pool/volume_rw_test.go | 4 +- sandboxd/pool/volume_static_acl_test.go | 42 ++ sandboxd/server/server.go | 6 + sandboxd/server/server_test.go | 21 +- sandboxd/server/volume_registration.go | 44 ++ sandboxd/server/volume_registration_test.go | 274 +++++++++ sandboxd/types/api.go | 8 + 36 files changed, 3346 insertions(+), 210 deletions(-) create mode 100644 e2e/volume_catalog_test.go create mode 100644 sandboxd/pool/volume_acl_test.go create mode 100644 sandboxd/pool/volume_bench_test.go create mode 100644 sandboxd/pool/volume_catalog.go create mode 100644 sandboxd/pool/volume_catalog_test.go create mode 100644 sandboxd/pool/volume_cleanup_test.go create mode 100644 sandboxd/pool/volume_retirement_test.go create mode 100644 sandboxd/pool/volume_static_acl_test.go create mode 100644 sandboxd/server/volume_registration.go create mode 100644 sandboxd/server/volume_registration_test.go diff --git a/docs/deploy.md b/docs/deploy.md index f8020f0e..51938d7b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -119,7 +119,7 @@ gets the intersection of the two. | `advertise_addr` | = `listen` | internal HTTP host:port for peer traffic and preview forwarding; also used by clients when `client_advertise` is unset. Must be routable when `listen` is a wildcard; a node with `mesh` set refuses to load while it names an unspecified host | | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | -| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, and a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later); root always has access. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, and a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later); root-owned claims have access; live additions use the claim owner's access list. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | @@ -247,6 +247,12 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes +Volume `tenants` entries use the tenant API name syntax. Static configuration +warns about invalid names and skips them during access checks. A nonempty list containing +only invalid names grants no tenant access. Runtime registration rejects invalid +tenant names. Valid names for tenants that have not yet been created remain +accepted. + Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes @@ -324,7 +330,8 @@ set, promoted-template intersection, and redirect follow the fleet-wide rule in [cluster](cluster.md#volumes-and-placement). An empty catalog `tenants` list allows every authenticated scope; a nonempty -list limits the image to those tenants, while root always bypasses it. A +list limits the image to those claim owners. Root-owned claims can use any +entry. Live additions use the claim owner's ACL, including requests sent by root. A listed tenant the set no longer holds grants nothing and logs a warning at boot. `GET /v1/volumes` reports the caller-visible fleet union and holder count, plus diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 790e0d21..66ff435f 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,6 +9,12 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. +A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) +without editing static config. Registration persists only node-local catalog +metadata and does not manage disk files. Omitted or empty runtime `tenants` +lists permit every authenticated scope. List tenants explicitly to restrict +access. Live changes remain limited to mounted read-only entries. + Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk before mounting the new one, so the path has a short unavailable interval. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 31a255cd..c103b4c8 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -8,7 +8,7 @@ All bodies are JSON. Three token kinds: preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces (the per-id sandbox reads, `PUT /v1/sandboxes/{id}/instance-metadata`, `GET /v1/info`, - `PUT /v1/pools`, `POST/DELETE /v1/drain`, `GET /metrics`, + `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, `GET /metrics`, `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. On a node with a shared @@ -255,6 +255,103 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. +## PUT /v1/volumes/{name} + +Auth: the configured root node API token. With no configured `api_token`, +registration and unregistration return 403. A tenant token returns 403. +Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. + +Registers a backing path on this node. The URL supplies the name; +the body has no `name` field. Unknown fields, duplicate fields, and trailing +JSON return 400. + +```json +{"path": "/srv/datasets/data.img", "directio": "off", + "writable": false, "tenants": ["acme"]} +``` + +- `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. +- `path` uses the same absolute-path validation as static volume config. + Registration does not open the backing or check its type or availability. + A missing backing retains its registration but cannot be attached until available. + Registration success does not establish that Cocoon supports the backing type. + Operators must keep the registered path and target stable. + Backing storage must be separate from daemon cleanup directories, including + `/readers`, `/goldens`, and `/fork-*`. + Backing paths must not resolve into cleanup directories. + Cleanup directories must not resolve into backing storage. + Startup and staging cleanup do not consult volume registrations and can + delete backing files placed in those locations. +- `directio` is `on`, `off`, or `auto`, and defaults to `off`. +- `writable` defaults to `false` and controls writable claim admission. +- `tenants` grants the named tenants self-service access. Omitted or empty + permits every authenticated scope, for both static and runtime entries. + Names use the tenant API name syntax. Runtime registration rejects invalid + names. Static config warns and skips invalid names. A nonempty static list with no + valid names remains restricted and grants no tenant access. A valid name may be + registered before that tenant exists; it grants no access until the tenant + is created. + +Success is 204 with an empty body. Repeating the same name, cleaned path, and +normalized registration fields succeeds. A different binding or different +registration fields return 409. Static and runtime registrations cannot share +a name or a cleaned backing path. Invalid registration fields return 400. +An uncertain write returns 503 with `Retry-After`. The requested registration +may still take effect through background retry or a later PUT or DELETE, +including a request for another name. A 503 can also reject a request before +acceptance, so it does not promise that the request will take effect. After +repairing storage failures, retry the exact request to establish completion. +Cancellation is not a rollback once a write has been accepted. +GET retains its existing projection without host paths or tenant access lists. +Registered entries support the existing mounted read-only live volume API. + +Only registration metadata is persisted locally in +`data_dir/runtime-volumes.json`, which loads before claim recovery. New records +use mode 0600. The daemon reads the record with the same filesystem permissions +as `claims.json` and replaces it durably. Registration creates, grows, and deletes +no disks. +Malformed or conflicting metadata prevents startup. A temporarily missing +backing file keeps its registration and refuses new attachment. +Catalog replacement and synchronization are independent of `sync_claims`, which +controls claim snapshot synchronization. Process-restart checks do not prove +power-loss recovery. The catalog is node-local with one sandboxd writer. +Startup reads only `runtime-volumes.json`. Crash leftovers named +`runtime-volumes.json.*.tmp` are neither loaded nor promoted. An operator can +remove those leftovers after stopping sandboxd and preserving any needed +evidence. + +## DELETE /v1/volumes/{name} + +Auth and name validation match PUT. Removes only the runtime registration and +never deletes or changes the backing file. Success, including an absent, +unreferenced runtime name, is 204 with an empty body. Static registrations and +names referenced by a claim, an in-flight operation, cleanup, or an +unacknowledged recovery snapshot return 409. Catalog persistence uncertainty +returns 503 with `Retry-After` and has the same asynchronous completion +semantics as PUT. A failed DELETE may therefore remove the registration later. +An empty GET volume view alone does not establish that a recovery reference +has retired. + +Downgrading to a daemon without runtime registration requires draining runtime +references or supplying equivalent static volume config. Pending live volume +changes must finish before downgrade. + +Stale VM cleanup inspects attachments before removal. A complete running +inventory retains holds only on the named volumes. An explicit empty disk +array permits unrelated volume admission and catalog changes while removal +retries continue. An omitted or null disk list, a stopped VM, an inspection +failure, or an active creating owner retains the node-wide fence until +inspection or confirmed removal resolves it. Cocoon v0.7.3 omits both empty +inventories and failed disk queries, and does not collect disks for stopped +VMs. Those results remain unknown and can block static read-only admission. +Allowing readers would conflict with an unknown writer. Cleanup retries +remain enabled. +A warm VM name is not evidence that no volume is attached; attachment can +succeed before claim finalization fails. Known inventories keep holds on their +reported names. Volume-free claims continue to work. Path uniqueness uses +cleaned path strings. The operator coordinates symlink and hardlink aliases +and cross-node ownership. + ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set @@ -277,7 +374,8 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed PUT or PATCH adds a top-level `error` naming the refusal, and the affected -state may carry its own `error`. `converging` is true only while a request worker is running. +state may carry its own `error`. `converging` is true while a request worker is running or a recovery-only +reference awaits an acknowledged claim write. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an diff --git a/docs/security.md b/docs/security.md index 6c519f49..841e9830 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,15 +94,26 @@ rather than confirming existence, and operator surfaces answer tenants 403. The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is -node-local. An empty entry `tenants` list permits every authenticated scope; a -nonempty list permits only those named tenants, while the root token always -has access. Config load rejects an access-list name that is not a configured -tenant. Claim lookup returns byte-identical errors for an unknown and a -forbidden name, and catalog discovery filters before replying, so a tenant +node-local. An empty `tenants` list permits every authenticated scope for +both static and runtime entries. A nonempty list grants the named tenants +self-service access. For live additions, the claim owner must satisfy the +entry's ACL, even when root sends the request. Runtime registration rejects +invalid tenant names. Static config warns and skips them. A nonempty static +list with no valid names permits no tenant access. A valid name that does not +yet identify a tenant grants no access until that tenant exists. +Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant cannot enumerate restricted entries by probing. Gossip carries names only. Neither gossip, discovery, persisted claims, usage events, nor the sandbox index exposes host image paths or access lists. +Runtime registration and unregistration require a configured root `api_token`; +these endpoints return 403 when the token is unset. New catalog records use +mode 0600. Reads and replacements use the daemon's filesystem permissions. +Registration uses the same path validation as static configuration. It does +not verify backing type +or pin backing identity. Operators must supply a backing supported by Cocoon, +prevent path or symlink target replacement while registered, and coordinate aliases. + Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path to every destination its tenant egress policy permits; review the volume access @@ -117,7 +128,9 @@ immutable and publish a new name/path for new content. Changing a running claim's volumes needs the root token or the owning tenant's API token. A sandbox token cannot, so a guest holding its own sandbox token cannot attach another dataset, and every addition passes the same access list -as a claim. +as a claim. Already-held references can finish repair or removal without a +new ACL grant. A recovery-only reference protects ownership and does not grant +permission for a fresh attachment. A writable entry (`writable: true`) adds a channel the read-only model does not have: whichever tenant is permitted to claim it `rw` changes what every diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go new file mode 100644 index 00000000..08c459fc --- /dev/null +++ b/e2e/volume_catalog_test.go @@ -0,0 +1,112 @@ +package e2e + +import ( + "encoding/json/v2" + "net/http" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + sandbox "github.com/cocoonstack/sandbox/sdk/go" +) + +func TestRuntimeVolumePublicAdmissionThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "shared bytes"), "writable": true}) + if err != nil { + t.Fatal(err) + } + for _, scope := range []struct { + token string + want int + }{{"acme-token", http.StatusForbidden}, {"", http.StatusUnauthorized}, {"node-token", http.StatusNoContent}} { + caller := *st + caller.token = scope.token + if status, body := rawJSON(t, &caller, http.MethodPut, "/v1/volumes/home", string(request)); status != scope.want { + t.Fatalf("registration auth: status=%d want=%d body=%s", status, scope.want, body) + } + } + volumes, err := acme.Volumes(t.Context()) + if err != nil || len(volumes) != 1 || volumes[0].Name != "home" { + t.Fatalf("empty runtime ACL is public: volumes=%v err=%v", volumes, err) + } + volume := sandbox.Volume{Name: "home", Mount: "/mnt/home"} + first, err := acme.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volume)) + if err != nil { + t.Fatalf("tenant claim with public runtime volume: %v", err) + } + defer first.Close() + if !slices.Equal(first.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("claim did not retain mounted volume: %+v", first.Volumes) + } + if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { + t.Fatal(err) + } + if view, attachErr := first.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("tenant PATCH of public runtime volume: view=%+v err=%v", view, attachErr) + } + if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { + t.Fatal(err) + } + if view, attachErr := first.SetVolumes(t.Context(), []sandbox.Volume{volume}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("tenant PUT of public runtime volume: view=%+v err=%v", view, attachErr) + } +} + +func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { + st := startTenantStack(t, "node-token", []tenants.TenantSpec{ + {Name: "acme", Token: "acme-token"}, {Name: "beta", Token: "beta-token"}, + }, []config.VolumeSpec{{Name: "public", Path: writeVolumeImage(t, "public.img", "shared")}}) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/private", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + for _, tc := range []struct { + token string + want []string + allowed bool + }{{"acme-token", []string{"private", "public"}, true}, {"beta-token", []string{"public"}, false}} { + client, connectErr := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) + if connectErr != nil { + t.Fatal(connectErr) + } + volumes, listErr := client.Volumes(t.Context()) + var names []string + for _, volume := range volumes { + names = append(names, volume.Name) + } + if listErr != nil || !slices.Equal(names, tc.want) { + t.Fatalf("catalog ACL: names=%v want=%v err=%v", names, tc.want, listErr) + } + claim, claimErr := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(sandbox.Volume{Name: "public"})) + if claimErr != nil { + t.Fatal(claimErr) + } + defer claim.Close() + _, attachErr := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{{Name: "private"}}}) + if (attachErr == nil) != tc.allowed { + t.Fatalf("explicit ACL admission for %s: %v", tc.token, attachErr) + } + } + caller := *st + caller.token = "beta-token" + var responses [][]byte + for _, name := range []string{"private", "unknown"} { + status, body := rawJSON(t, &caller, http.MethodPost, "/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"`+name+`"}]}`) + if status != http.StatusBadRequest { + t.Fatalf("claim privacy for %s: status=%d body=%s", name, status, body) + } + responses = append(responses, body) + } + if string(responses[0]) != string(responses[1]) { + t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) + } +} diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fba3efba..6b2c9295 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -3,6 +3,7 @@ package config import ( "cmp" + "context" "encoding/base64" "encoding/json/jsontext" "encoding/json/v2" @@ -18,6 +19,7 @@ import ( "strings" "github.com/docker/go-units" + "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -398,28 +400,17 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { - names := make(map[string]struct{}, len(c.Volumes)) - paths := make(map[string]string, len(c.Volumes)) - for _, volume := range c.Volumes { - if !types.ValidVolumeName(volume.Name) { - return fmt.Errorf("volume name %q must match %s and not start with cocoon-", volume.Name, types.VolumeNameRe) - } - if _, ok := names[volume.Name]; ok { - return fmt.Errorf("duplicate volume name %q", volume.Name) - } - names[volume.Name] = struct{}{} - if !filepath.IsAbs(volume.Path) { - return fmt.Errorf("volume %q path must be absolute", volume.Name) - } - if other, ok := paths[filepath.Clean(volume.Path)]; ok { - return fmt.Errorf("volume %q shares its path with volume %q: admission is keyed by name", volume.Name, other) - } - paths[filepath.Clean(volume.Path)] = volume.Name - if !types.ValidDirectIO(volume.DirectIO) { - return fmt.Errorf("volume %q directio must be on, off, or auto, got %q", volume.Name, volume.DirectIO) - } + validation := slices.Clone(c.Volumes) + for i, volume := range c.Volumes { + validation[i].Tenants = slices.DeleteFunc(slices.Clone(volume.Tenants), func(tenant string) bool { + if types.NameRe.MatchString(tenant) { + return false + } + log.WithFunc("config.validateVolumes").Warnf(context.Background(), "volume %q: skipping invalid tenant %q", volume.Name, tenant) + return true + }) } - return nil + return ValidateVolumeCatalog(validation) } func (c *Config) validatePreview() error { @@ -637,6 +628,37 @@ func (c *Config) validateMetaStore() error { return nil } +// ValidateVolumeCatalog checks the shared static and runtime catalog rules without I/O. +func ValidateVolumeCatalog(volumes []VolumeSpec) error { + names := make(map[string]struct{}, len(volumes)) + paths := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if !types.ValidVolumeName(volume.Name) { + return fmt.Errorf("volume name %q must match %s and not start with cocoon-", volume.Name, types.VolumeNameRe) + } + if _, ok := names[volume.Name]; ok { + return fmt.Errorf("duplicate volume name %q", volume.Name) + } + names[volume.Name] = struct{}{} + if !filepath.IsAbs(volume.Path) { + return fmt.Errorf("volume %q path must be absolute", volume.Name) + } + if other, ok := paths[filepath.Clean(volume.Path)]; ok { + return fmt.Errorf("volume %q shares its path with volume %q: admission is keyed by name", volume.Name, other) + } + paths[filepath.Clean(volume.Path)] = volume.Name + if !types.ValidDirectIO(volume.DirectIO) { + return fmt.Errorf("volume %q directio must be on, off, or auto, got %q", volume.Name, volume.DirectIO) + } + for _, tenant := range volume.Tenants { + if !types.NameRe.MatchString(tenant) { + return fmt.Errorf("volume %q tenant %q must match %s", volume.Name, tenant, types.NameRe) + } + } + } + return nil +} + // Load reads a JSON config file, applies defaults, and validates. func Load(path string) (*Config, error) { raw, err := os.ReadFile(path) //nolint:gosec // path is the operator-supplied -config flag diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 06c4fe3c..ce5d5766 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -463,6 +463,35 @@ func TestLoadAcceptsAShardedNetworkList(t *testing.T) { } } +func TestValidateVolumeCatalogTenantNames(t *testing.T) { + for _, tenant := range []string{"", "has space", "_leading", strings.Repeat("a", 64)} { + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{tenant}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err == nil { + t.Errorf("invalid tenant %q accepted", tenant) + } + } + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{"future-tenant", "team/project:v1"}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err != nil { + t.Fatalf("syntactically valid future tenant rejected: %v", err) + } +} + +func TestLoadPreservesRestrictedACLWithInvalidVolumeTenants(t *testing.T) { + for _, tenants := range []string{`["acme", "has space", "", "beta"]`, `["has space"]`} { + cfg, err := Load(writeConfig(t, `{"pools":[],"volumes":[{"name":"data","path":"/images/data","tenants":`+tenants+`}]}`)) + if err != nil { + t.Fatal(err) + } + want := []string{"has space"} + if strings.Contains(tenants, "acme") { + want = []string{"acme", "has space", "", "beta"} + } + if !slices.Equal(cfg.Volumes[0].Tenants, want) { + t.Fatalf("tenants = %v, want %v", cfg.Volumes[0].Tenants, want) + } + } +} + func writeConfig(t *testing.T, body string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.json") diff --git a/sandboxd/pool/checkpoint_references_test.go b/sandboxd/pool/checkpoint_references_test.go index 3125c68a..94601f8d 100644 --- a/sandboxd/pool/checkpoint_references_test.go +++ b/sandboxd/pool/checkpoint_references_test.go @@ -6,6 +6,7 @@ import ( "io/fs" "maps" "os" + "path/filepath" "slices" "sync" "syscall" @@ -14,17 +15,11 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -const ( - failWrite = iota - captureWrite - failAfterRename -) - func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { recreate := func(s *claimStore) { s.set(&types.Sandbox{ID: "owner", ReplicaCk: "recreated"}) } for _, tc := range []struct { name string - fault int + fault func(*testing.T, *claimStore, claimSnapshot) func() acked, next *types.Sandbox startup func(claims map[string]*types.Sandbox) mutate func(s *claimStore) @@ -87,11 +82,8 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - if tc.fault == failAfterRename && os.Geteuid() == 0 { - t.Skip("requires non-root: root can read the write-only directory") - } dir := t.TempDir() - s := newClaimStore(dir, tc.fault == failAfterRename) + s := newClaimStore(dir, false) if tc.acked != nil { if err := s.commit(s.set(tc.acked)); err != nil { t.Fatal(err) @@ -109,51 +101,7 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { } else { snap = s.set(tc.next) } - var finish func() - switch tc.fault { - case failWrite: - finish = failClaimWrites(t, s) - if err := s.commit(snap); err == nil { - t.Fatal("write succeeded under the fault") - } - case failAfterRename: - if err := os.Chmod(dir, 0o300); err != nil { - t.Fatal(err) - } - finish = func() { _ = os.Chmod(dir, 0o700) } - t.Cleanup(finish) - if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { - t.Fatalf("want a failed directory sync after the rename: %v", err) - } - case captureWrite: - if err := syscall.Mkfifo(s.path+".tmp", 0o600); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - go func() { done <- s.commit(snap) }() - waitFor(t, func() bool { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.retained["owner"]) != 0 - }) - finish = func() { - r, err := os.Open(s.path + ".tmp") - if err != nil { - t.Fatal(err) - } - raw, err := io.ReadAll(r) - _ = r.Close() - if err = errors.Join(err, <-done); err != nil { - t.Fatal(err) - } - if err = os.Remove(s.path); err != nil { - t.Fatal(err) - } - if err = os.WriteFile(s.path, raw, 0o600); err != nil { - t.Fatal(err) - } - } - } + finish := tc.fault(t, s, snap) if tc.mutate != nil { tc.mutate(s) } @@ -208,3 +156,72 @@ func failClaimWrites(t *testing.T, s *claimStore) func() { t.Cleanup(restore) return restore } + +func captureClaimWrite(t *testing.T, s *claimStore, snap claimSnapshot, owner string) func() []byte { + t.Helper() + if err := syscall.Mkfifo(s.path+".tmp", 0o600); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- s.commit(snap) }() + waitFor(t, func() bool { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.retained[owner]) != 0 || len(s.retainedVolumes[owner]) != 0 + }) + return func() []byte { + r, err := os.Open(s.path + ".tmp") + if err != nil { + t.Fatal(err) + } + raw, err := io.ReadAll(r) + _ = r.Close() + if err = errors.Join(err, <-done); err != nil { + t.Fatal(err) + } + if err = os.Remove(s.path); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(s.path, raw, 0o600); err != nil { + t.Fatal(err) + } + return raw + } +} + +func failWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { + t.Helper() + restore := failClaimWrites(t, s) + if err := s.commit(snap); err == nil { + t.Fatal("write succeeded under the fault") + } + return restore +} + +func failAfterRename(t *testing.T, s *claimStore, snap claimSnapshot) func() { + t.Helper() + if os.Geteuid() == 0 { + t.Skip("requires non-root: root can read the write-only directory") + } + s.sync = true + dir := filepath.Dir(s.path) + if err := os.Chmod(dir, 0o300); err != nil { + t.Fatal(err) + } + restore := func() { + if err := os.Chmod(dir, 0o700); err != nil { + t.Error(err) + } + } + t.Cleanup(restore) + if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { + t.Fatalf("want a failed directory sync after the rename: %v", err) + } + return restore +} + +func captureWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { + t.Helper() + release := captureClaimWrite(t, s, snap, "owner") + return func() { release() } +} diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index f9427792..9d9110bb 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -371,7 +371,7 @@ func (m *Manager) overQuota(extra int, tenant string) error { // admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) - if err := m.admitClaim(tenant, applied); err != nil { + if err := m.admitClaim(tenant, volumes); err != nil { return nil, err } if err := m.confirmVolumesClean(volumes); err != nil { @@ -382,13 +382,13 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types } // admitClaim refuses a busy volume or an over-quota tenant before any VM is built. -func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { +func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveVolumes(volumes) + return m.reserveResolved(volumes, nil) } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 2f07ecaf..56f2bfe5 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -72,6 +72,8 @@ type claimRow struct { raw []byte } +type recoveryVolumeCount struct{ readers, writers int } + // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string @@ -81,16 +83,20 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 - refs map[string][4]string - retained map[string][]string - changed map[string]struct{} - checkpoints map[string]int - - recommitting atomic.Bool + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 + refs map[string][4]string + retained map[string][]string + changed map[string]struct{} + checkpoints map[string]int + currentVolumes map[string][]types.Volume + retainedVolumes map[string][]types.Volume + volumes map[string]recoveryVolumeCount + + recommitting atomic.Bool + recoveryVolumesPresent atomic.Bool } func newClaimStore(dataDir string, sync bool) *claimStore { @@ -98,6 +104,7 @@ func newClaimStore(dataDir string, sync bool) *claimStore { path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, refs: map[string][4]string{}, retained: map[string][]string{}, changed: map[string]struct{}{}, checkpoints: map[string]int{}, + currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]recoveryVolumeCount{}, } } @@ -119,6 +126,10 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } + if refs := volumeRefs(nil, slices.Clone(sb.Volumes), slices.Clone(sb.VolumesWant)); len(refs) != 0 { + s.changeRetainedVolumes(sb.ID, refs) + s.changed[sb.ID] = struct{}{} + } if refs := checkpointRefs(sb); refs != [4]string{} { s.changeRetainedRefs(sb.ID, appendRefs(nil, refs)) s.changed[sb.ID] = struct{}{} @@ -151,24 +162,34 @@ func (s *claimStore) del(ids ...string) claimSnapshot { func (s *claimStore) put(id string, sb *types.Sandbox) { var refs [4]string + var volumes []types.Volume if sb == nil { delete(s.rows, id) } else { + dto := dtoOf(sb) refs = checkpointRefs(sb) - s.rows[id] = &claimRow{dto: dtoOf(sb)} + volumes = volumeRefs(s.currentVolumes[id], dto.Volumes, dto.VolumesWant) + s.rows[id] = &claimRow{dto: dto} } before := s.refs[id] - if before == refs { - return + if before != refs { + s.changeCheckpointCounts(before[:], -1) + s.changeCheckpointCounts(refs[:], 1) + if refs == [4]string{} { + delete(s.refs, id) + } else { + s.refs[id] = refs + } + s.changed[id] = struct{}{} } - s.changeCheckpointCounts(before[:], -1) - s.changeCheckpointCounts(refs[:], 1) - if refs == [4]string{} { - delete(s.refs, id) - } else { - s.refs[id] = refs + if !slices.Equal(s.currentVolumes[id], volumes) { + if len(volumes) == 0 { + delete(s.currentVolumes, id) + } else { + s.currentVolumes[id] = volumes + } + s.changed[id] = struct{}{} } - s.changed[id] = struct{}{} } // mark sequences a persist of the projection as it already stands. @@ -211,9 +232,17 @@ func (s *claimStore) commit(snap claimSnapshot) error { } rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq captured := make(map[string][4]string, len(s.changed)) + var capturedVolumes map[string][]types.Volume for id := range s.changed { captured[id] = s.refs[id] - s.changeRetainedRefs(id, appendRefs(s.retained[id], s.refs[id])) + s.changeRetainedRefs(id, appendRefs(slices.Clone(s.retained[id]), s.refs[id])) + if volumes := s.currentVolumes[id]; len(volumes) != 0 { + if capturedVolumes == nil { + capturedVolumes = make(map[string][]types.Volume) + } + capturedVolumes[id] = volumes + } + s.changeRetainedVolumes(id, appendRecoveryRefs(slices.Clone(s.retainedVolumes[id]), s.currentVolumes[id])) } s.mu.Unlock() s.buf = append(s.buf[:0], '{') @@ -237,7 +266,8 @@ func (s *claimStore) commit(snap claimSnapshot) error { s.mu.Lock() for id, refs := range captured { s.changeRetainedRefs(id, appendRefs(nil, refs)) - if s.refs[id] == refs { + s.changeRetainedVolumes(id, capturedVolumes[id]) + if s.refs[id] == refs && slices.Equal(s.currentVolumes[id], capturedVolumes[id]) { delete(s.changed, id) } } @@ -340,6 +370,88 @@ func (s *claimStore) checkpointReferences() map[string]struct{} { return refs } +func (s *claimStore) changeVolumeCounts(refs []types.Volume, delta int) { + for _, v := range refs { + count := s.volumes[v.Name] + if v.RW() { + count.writers += delta + } else { + count.readers += delta + } + if count == (recoveryVolumeCount{}) { + delete(s.volumes, v.Name) + } else { + s.volumes[v.Name] = count + } + } +} + +func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { + s.changeVolumeCounts(s.retainedVolumes[id], -1) + s.changeVolumeCounts(after, 1) + if len(after) == 0 { + delete(s.retainedVolumes, id) + } else { + s.retainedVolumes[id] = after + } + s.recoveryVolumesPresent.Store(len(s.volumes) != 0) +} + +func (s *claimStore) referencesVolume(name string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.volumes[name] != (recoveryVolumeCount{}) +} + +func (s *claimStore) volumeConflicts(volume types.Volume, ownerID string) bool { + s.mu.Lock() + defer s.mu.Unlock() + count := s.volumes[volume.Name] + for _, ref := range s.retainedVolumes[ownerID] { + if ref == volume { + if ref.RW() { + count.writers-- + } else { + count.readers-- + } + } + } + return count.writers > 0 || volume.RW() && count.readers > 0 +} + +func (s *claimStore) recoveryVolumesFor(owner string) []types.Volume { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.retainedVolumes[owner]) +} + +func (s *claimStore) detachingOwners() map[string]bool { + if !s.recoveryVolumesPresent.Load() { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + var owners map[string]bool + for id := range s.changed { + if len(s.currentVolumes[id]) == 0 && len(s.retainedVolumes[id]) != 0 { + if owners == nil { + owners = map[string]bool{} + } + owners[id] = true + } + } + return owners +} + +func (s *claimStore) hasRecoveryVolumes(owner string) bool { + if !s.recoveryVolumesPresent.Load() { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return len(s.retainedVolumes[owner]) != 0 +} + func encodeRow(dto *claimDTO) ([]byte, error) { raw, err := json.Marshal(map[string]*claimDTO{dto.ID: dto}) if err != nil { @@ -381,3 +493,31 @@ func appendRefs(list []string, refs [4]string) []string { } return list } + +func appendRecoveryRefs(dst, src []types.Volume) []types.Volume { + for _, v := range src { + if !slices.Contains(dst, v) { + dst = append(dst, v) + } + } + return dst +} + +func volumeRefs(previous, applied, wanted []types.Volume) []types.Volume { + if len(applied) == 0 { + applied, wanted = wanted, nil + } + for _, refs := range [2][]types.Volume{applied, wanted} { + for _, v := range refs { + if !slices.Contains(previous, v) { + return appendRecoveryRefs(applied, wanted) + } + } + } + for _, v := range previous { + if !slices.Contains(applied, v) && !slices.Contains(wanted, v) { + return appendRecoveryRefs(applied, wanted) + } + } + return previous +} diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index f7e19311..e12ea0e7 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -88,10 +88,14 @@ func TestStoreRoundTrip(t *testing.T) { if err := s.save(claims); err != nil { t.Fatalf("save (sync=%v): %v", fsync, err) } - got, err := newClaimStore(filepath.Dir(s.path), false).load() + loadedStore := newClaimStore(filepath.Dir(s.path), false) + got, err := loadedStore.load() if err != nil { t.Fatalf("load (sync=%v): %v", fsync, err) } + if !loadedStore.hasRecoveryVolumes("sb_a") || loadedStore.hasRecoveryVolumes("sb_b") { + t.Fatal("loaded recovery volumes lost owner visibility") + } if !reflect.DeepEqual(got, claims) { t.Errorf("sync=%v: got %+v, want %+v", fsync, got, claims) } @@ -103,9 +107,10 @@ func TestStoreRoundTrip(t *testing.T) { func TestClaimStoreSnapshotDetachesVolumes(t *testing.T) { s := newClaimStore(t.TempDir(), false) - sb := &types.Sandbox{ID: "sb_a", Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}} + sb := &types.Sandbox{ID: "sb_a", Volumes: []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}, VolumesWant: []types.Volume{{Name: "dataset-b", Mount: "/datasets/b"}}} snap := s.set(sb) sb.Volumes[0].Mount = "/mutated" + sb.VolumesWant[0].Mount = "/mutated-wanted" if err := s.commit(snap); err != nil { t.Fatalf("commit: %v", err) } @@ -120,6 +125,9 @@ func TestClaimStoreSnapshotDetachesVolumes(t *testing.T) { if !slices.Equal(loaded.Volumes, []types.Volume{{Name: "dataset-a", Mount: "/datasets/a"}}) { t.Errorf("volumes %v, want detached snapshot", loaded.Volumes) } + if !slices.Equal(loaded.VolumesWant, []types.Volume{{Name: "dataset-b", Mount: "/datasets/b"}}) { + t.Errorf("wanted volumes %v, want detached snapshot", loaded.VolumesWant) + } } func TestClaimRowOmitsUnsetFlags(t *testing.T) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9e0124cb..e2cb2391 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -116,6 +116,9 @@ var ( errNotSleeping = errors.New("sandbox is no longer hibernated or archived") errWokeMeanwhile = errors.New("woke between sweep and hibernate") errNoEgressTap = errors.New("egress-lane claim has no lockable tap") + + ErrVolumeRegistrationConflict = errors.New("volume registration conflicts with the catalog") + ErrVolumeCatalogUnavailable = errors.New("volume catalog requires a successful persistence retry") ) // Engine is the slice of the cocoon driver the manager consumes. @@ -321,16 +324,14 @@ type PeerDeleteFunc func(ctx context.Context, id string) // Manager owns the node's pools, claims, and their persistence. type Manager struct { - eng Engine - dataDir string - egress bool - maxFork int - store *claimStore - volumes map[string]catalogVolume - // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]map[string]string + eng Engine + dataDir string + egress bool + maxFork int + store *claimStore + catalog volumeCatalog + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]map[string]string poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. @@ -448,8 +449,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - volumes: make(map[string]catalogVolume, len(cfg.Volumes)), - volumeAdmission: map[string]volumeHolders{}, + catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), holders: map[string]volumeHolders{}}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), @@ -478,12 +478,15 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg healSem: make(chan struct{}, maxConcurrentHeals), } for _, volume := range cfg.Volumes { - m.volumes[volume.Name] = catalogVolume{ - disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, + m.catalog.entries[volume.Name] = &catalogVolume{ + disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, tenants: slices.Clone(volume.Tenants), writable: volume.Writable, } } + if err := m.loadRuntimeVolumes(); err != nil { + return nil, err + } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) } @@ -600,6 +603,7 @@ func (m *Manager) Run(ctx context.Context) { case <-m.refillKick: m.refillOnce(ctx) case <-reap.C: + go m.retryVolumeCatalog(ctx) m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 56e4148f..d9fc8b46 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1255,6 +1255,9 @@ type fakeEngine struct { detachVolumeErr error liveVolumeHook func(string) error stopped map[string]bool + inspectHook func() error + inspectDisksUnknown bool + trackDisks bool disks map[string][]types.VMDisk creating map[string]bool staleOutcome engine.StaleCreateOutcome @@ -1526,14 +1529,22 @@ func (f *fakeEngine) List(ctx context.Context) ([]types.VMRecord, error) { func (f *fakeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { f.mu.Lock() f.inspects++ + hook := f.inspectHook f.mu.Unlock() + if hook != nil { + if err := hook(); err != nil { + return types.VMRecord{}, false, err + } + } vms, _ := f.list(ctx) i := slices.IndexFunc(vms, func(vm types.VMRecord) bool { return vm.Config.Name == name }) if i < 0 { return types.VMRecord{}, false, nil } f.mu.Lock() - vms[i].AttachedDevices.Disks = f.disks[name] + if !f.inspectDisksUnknown { + vms[i].AttachedDevices.Disks = f.disks[name] + } f.mu.Unlock() return vms[i], true, nil } @@ -1618,7 +1629,7 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } -func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { +func (f *fakeEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("attach"); err != nil { return err @@ -1635,6 +1646,12 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.Volum if spec.Name == f.diskAttachErrFor { err = errors.New("attach failed") } + if err == nil && f.trackDisks { + if f.disks == nil { + f.disks = map[string][]types.VMDisk{} + } + f.disks[name] = append(f.disks[name], types.VMDisk{Name: spec.Name, ReadOnly: !spec.RW}) + } f.mu.Unlock() if cancel != nil { cancel() diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index a116b580..056fdf81 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -70,13 +70,14 @@ func (m *Manager) Reconcile(ctx context.Context) error { owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } - saveErr := m.store.save(m.claimed) + js := m.store.reset(m.claimed) pooled := make(map[string]bool, len(m.pools)) for _, p := range m.pools { m.adoptGolden(p, imageIDs[p.key.Template]) pooled[p.hash] = true } m.mu.Unlock() + saveErr := m.store.commit(js) for _, sb := range adopted { m.recordHibernate(ctx, sb) } @@ -156,6 +157,10 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec // removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") + m.mu.Lock() + m.catalog.unknown++ + m.mu.Unlock() + td := volumeTeardown{unknown: true} if rec.State == vmStateCreating { // a canceled ctx must not skip the busy check guarding the forced remove switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); { @@ -163,41 +168,52 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe // verb missing (cocoon < v0.5.8) or failed: keep the old sweep. logger.Warnf(ctx, "reconcile stale create %s: %v; removing", name, err) case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: + m.finishVolumeTeardown(ctx, td) return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - m.queueStaleCreate(name, rec.TapDevice()) + m.queueStaleCreate(name, rec.TapDevice(), td) return false } // not-creating: the record moved on under the lock; remove normally. } + td = m.resolveStaleVolumes(ctx, name, td) if m.removeVM(ctx, name) { + m.finishVolumeTeardown(ctx, td) return true } - m.queueRemoval(name, "", rec.TapDevice(), volumeTeardown{holds: m.holdStaleVolumes(ctx, name, rec)}, time.Time{}) + m.queueRemoval(name, "", rec.TapDevice(), td, time.Time{}) return false } -func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.VMRecord) []types.Volume { - if rec.State != vmStateRunning { - return nil +func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volumeTeardown) volumeTeardown { + if !td.unknown { + return td } - inspected, _, err := m.eng.Inspect(ctx, name) + inspected, present, err := m.eng.Inspect(ctx, name) if err != nil { - log.WithFunc("pool.holdStaleVolumes").Warnf(ctx, "inspect stale vm %s, its volumes stay unheld: %v", name, err) - return nil + log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) + return td } - holds := make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW + if present && (inspected.State != vmStateRunning || inspected.AttachedDevices.Disks == nil) { + return td + } + var holds []types.Volume + if present { + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW + } } } m.mu.Lock() m.adoptVolumes(holds) + m.catalog.unknown-- m.mu.Unlock() - return holds + td.unknown, td.holds = false, holds + return td } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index b50d78dc..636555e7 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -181,8 +181,8 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { if !eng.removed("sbx-orphan-1") { t.Error("unowned running VM not removed") } - if eng.inspects != 0 { - t.Errorf("inspected a stale VM whose removal succeeded %d times", eng.inspects) + if eng.inspects != 1 { + t.Errorf("inspected stale VM %d times, want 1 before removal", eng.inspects) } } diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 75efec26..924e3f5d 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -45,7 +45,9 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { if m.removeVM(ctx, name) { - m.finishVolumeTeardown(ctx, td) + if !m.finishVolumeTeardown(ctx, td) { + m.queueRemoval(name, sandboxID, "", td, time.Time{}) + } return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -58,9 +60,9 @@ func (m *Manager) queueRemoval(name, sandboxID, tap string, td volumeTeardown, n m.mu.Unlock() } -func (m *Manager) queueStaleCreate(name, tap string) { +func (m *Manager) queueStaleCreate(name, tap string, td volumeTeardown) { m.mu.Lock() - m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true} + m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true, volumes: td} m.mu.Unlock() } @@ -77,7 +79,7 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } m.mu.Unlock() names := slices.Collect(maps.Keys(batch)) - return m.runBounded(ctx, len(names), func(ctx context.Context, i int) { + return m.runBounded(context.WithoutCancel(ctx), len(names), func(ctx context.Context, i int) { m.retryRemoval(ctx, names[i], batch[names[i]]) }) } @@ -87,25 +89,29 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending switch outcome, err := m.eng.ReconcileStaleCreate(ctx, name); { case err != nil: log.WithFunc("pool.retryRemoval").Warnf(ctx, "reconcile stale create %s: %v; retrying", name, err) - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) return } } + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if !m.removeVM(ctx, name) { m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) } -func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { - m.finishVolumeTeardown(ctx, pending.volumes) +func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { + if !m.finishVolumeTeardown(ctx, pending.volumes) { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + return + } if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 6524154d..93b666c9 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -27,6 +27,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { var withVolumes []string next := make(map[string]replicaState, len(m.replicas)) m.mu.Lock() + detaching := m.store.detachingOwners() for id, sb := range m.claimed { every, st := v.poolReplicate[sb.PolicyKey()], m.replicas[id] switch { @@ -35,7 +36,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { retire = append(retire, sb) } case !m.replicable(sb): - case m.hasVolumeHolds(sb): + case m.hasLiveVolumeHolds(sb) || detaching[id]: if sb.HasReplica() { retire = append(retire, sb) } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 0a157b61..b56103dc 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -4,6 +4,7 @@ import ( "encoding/json/v2" "errors" "net" + "os" "testing" "testing/synctest" "time" @@ -1002,6 +1003,61 @@ func TestReplicaAdoptionWakesTheClaimFromTheLastReplica(t *testing.T) { } } +func TestReplicaScanWaitsForDetachAcknowledgment(t *testing.T) { + c := newReplicaCluster(t) + m := c.node(t, "a", t.TempDir(), keepAnHour) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "replicated-data", "bytes")}); err != nil { + t.Fatal(err) + } + sb, attached, ordinary := mustClaim(t, m, testKey), mustClaim(t, m, testKey), mustClaim(t, m, testKey) + for _, owner := range []*types.Sandbox{sb, attached} { + if _, err := m.SetVolumes(t.Context(), owner.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + } + m.store.recommitting.Store(true) + eng := m.eng.(*fakeEngine) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "", nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detach did not fail after device removal: %+v, %v", view, err) + } + m.replicateOnce(t.Context()) + if m.replicating.Load() || !m.replicas[sb.ID].skipLogged || !m.replicas[attached.ID].skipLogged || m.replicas[ordinary.ID].tried.IsZero() || m.replicas[ordinary.ID].skipLogged { + t.Fatalf("scan failed to distinguish detached, attached, and ordinary claims: %+v", m.replicas) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture ignored retained attachment: %v", err) + } + if sb.HasReplica() || len(eng.snapSaves) != 0 { + t.Fatal("pending detach published a replica") + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.replicateOnce(t.Context()) + if st := m.replicas[sb.ID]; st.skipLogged || st.tried.IsZero() { + t.Fatalf("acknowledged detach stayed skipped: %+v", st) + } + m.replicas[sb.ID] = replicaState{tried: time.Now().Add(-time.Hour)} + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || rec.Consumed || rec.Checkpoint == "" || !sb.HasReplica() { + t.Fatalf("acknowledged detach did not permit replication: %+v, %t, %v", rec, ok, err) + } +} + func newReplicaCluster(t *testing.T) *failoverCluster { t.Helper() c := newFailoverCluster(t) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 5a1e8eed..0660e7c5 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -31,31 +31,40 @@ type volumeHolders struct { } type resolvedVolume struct { + entry *catalogVolume disk engine.VolumeSpec applied types.Volume } // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - holds []types.Volume - clears []string + done <-chan struct{} + unknown bool + holds []types.Volume + clears []string } type catalogVolume struct { + runtime bool disk engine.VolumeSpec tenants []string writable bool } -func (v catalogVolume) allowed(tenant string) bool { +func (v *catalogVolume) allowedFor(tenant string) bool { return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) } +func (v catalogVolume) registration() types.VolumeRegistration { + return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.tenants} +} + // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { - infos := make([]types.VolumeInfo, 0, len(m.volumes)) - for name, volume := range m.volumes { - if !volume.allowed(tenant) { + catalog := m.catalogSnapshot() + infos := make([]types.VolumeInfo, 0, len(catalog)) + for name, volume := range catalog { + if !volume.allowedFor(tenant) { continue } info := types.VolumeInfo{ @@ -74,7 +83,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { - if _, ok := m.volumes[name]; !ok { + if _, ok := catalog[name]; !ok { infos = append(infos, types.VolumeInfo{ Name: name, DefaultMount: types.DefaultVolumeMount(name), @@ -89,8 +98,9 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // VolumeNames returns the sorted names this node can currently serve. func (m *Manager) VolumeNames() []string { - names := make([]string, 0, len(m.volumes)) - for name, volume := range m.volumes { + catalog := m.catalogSnapshot() + names := make([]string, 0, len(catalog)) + for name, volume := range catalog { if _, err := os.Stat(volume.disk.Path); err == nil { names = append(names, name) } @@ -108,8 +118,9 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return true, nil } local := true + catalog := m.catalogSnapshot() for _, name := range names { - volume, ok := m.volumes[name] + volume, ok := catalog[name] if !ok { if tenant == "" { local = false @@ -117,7 +128,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri } return false, ErrVolumeUnavailable } - if !volume.allowed(tenant) { + if !volume.allowedFor(tenant) { return false, ErrVolumeUnavailable } if _, err := os.Stat(volume.disk.Path); err != nil { @@ -131,18 +142,24 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } + m.mu.Lock() + if err := m.catalog.admissionError(); err != nil { + m.mu.Unlock() + return nil, err + } + catalog := m.catalog.entries + m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := m.volumes[volume.Name] - if !ok || !entry.allowed(tenant) { + entry, ok := catalog[volume.Name] + if !ok || !entry.allowedFor(tenant) { return nil, ErrVolumeUnavailable } - resolvedEntry, resolveErr := m.resolveCatalogVolume(ctx, volume) + resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) if resolveErr != nil { return nil, resolveErr } @@ -151,9 +168,8 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return resolved, nil } -func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { - entry, ok := m.volumes[volume.Name] - if !ok { +func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, entry *catalogVolume) (resolvedVolume, error) { + if entry == nil { return resolvedVolume{}, ErrVolumeUnavailable } if volume.RW() && !entry.writable { @@ -161,13 +177,13 @@ func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) } if _, statErr := os.Stat(entry.disk.Path); statErr != nil { // the operator's path stays server-side; the refusal matches the unknown one - log.WithFunc("pool.resolveCatalogVolume").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) + log.WithFunc("pool.resolveCatalogEntry").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return resolvedVolume{}, ErrVolumeUnavailable } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false // past validation the empty mount carries it - return resolvedVolume{disk: disk, applied: volume}, nil + volume.AttachOnly = false + return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil } // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. @@ -227,7 +243,12 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { + m.mu.Lock() td := volumeTeardown{holds: heldVolumes(sb)} + if run := m.volumeRuns[sb]; run != nil { + td.done = run.done + } + m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { return td @@ -246,9 +267,11 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } - if entry, ok := m.volumes[volume.Name]; ok { + m.mu.Lock() + if entry, ok := m.catalog.entries[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } + m.mu.Unlock() } if stuck { // the unmounts may have spent the whole budget; the fallback flush gets its own @@ -262,9 +285,13 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT } // finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. -func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { - if len(td.holds) == 0 { - return +func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) bool { + if td.done != nil { + select { + case <-td.done: + default: + return false + } } logger := log.WithFunc("pool.finishVolumeTeardown") for _, path := range td.clears { @@ -272,13 +299,36 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { logger.Errorf(ctx, err, "clear dirty marker %s", path) } } - m.unreserveVolumes(td.holds) + m.mu.Lock() + m.releaseVolumes(td.holds) + if td.unknown { + m.catalog.unknown-- + } + m.mu.Unlock() + return true } // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { + return m.reserveVolumesFor(volumes, nil) +} + +func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { + if len(volumes) == 0 { + return nil + } + if err := m.catalog.admissionError(); err != nil { + return err + } + ownerID := "" + if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { + ownerID = owner.ID + } for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + if m.store.volumeConflicts(volume, ownerID) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } + holders := m.catalog.holders[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) } @@ -290,30 +340,30 @@ func (m *Manager) reserveVolumes(volumes []types.Volume) error { // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers++ } else { holders.readers++ } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } // releaseVolumes drops one claim's admission holds; callers hold m.mu. func (m *Manager) releaseVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers-- } else { holders.readers-- } if holders.writers < 1 && holders.readers < 1 { - delete(m.volumeAdmission, volume.Name) + delete(m.catalog.holders, volume.Name) continue } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } @@ -326,6 +376,21 @@ func (m *Manager) unreserveVolumes(volumes []types.Volume) { m.releaseVolumes(volumes) } +func (m *Manager) catalogSnapshot() map[string]*catalogVolume { + m.mu.Lock() + defer m.mu.Unlock() + return m.catalog.entries +} + +func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox) error { + for _, volume := range volumes { + if volume.entry == nil || m.catalog.entries[volume.applied.Name] != volume.entry { + return ErrVolumeUnavailable + } + } + return m.reserveVolumesFor(appliedVolumes(volumes), owner) +} + // quiesceBudget bounds one quiesce: a slice per writable mount plus one for the sync fallback. func quiesceBudget(mounts int) time.Duration { return min(time.Duration(mounts+1)*engine.VolumeCallTimeout, volumeQuiesceMax) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go new file mode 100644 index 00000000..2ebe99a2 --- /dev/null +++ b/sandboxd/pool/volume_acl_test.go @@ -0,0 +1,152 @@ +package pool + +import ( + "errors" + "os" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestRuntimeCatalogDefaultACL(t *testing.T) { + for _, tenants := range [][]string{nil, {}} { + t.Run("empty", func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m, _ := newTenantVolumeManager(t, eng, catalog) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: tenants} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { + t.Fatal(err) + } + for _, current := range []*Manager{m, restartACLManager(t, eng, m.dataDir, catalog)} { + if !slices.ContainsFunc(current.Volumes("acme", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("public runtime volume is hidden from a tenant") + } + if !slices.ContainsFunc(current.Volumes("", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("operator cannot see runtime volume") + } + if _, err := current.VolumePlacement(testKey, "acme", []string{"home"}); err != nil { + t.Errorf("tenant placement: %v", err) + } + if local, err := current.VolumePlacement(testKey, "", []string{"home"}); err != nil || !local { + t.Errorf("operator placement: %v %v", local, err) + } + tenantClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) + if err != nil { + t.Fatalf("tenant initial attachment: %v", err) + } + if err = current.Release(t.Context(), tenantClaim.ID, Cred{Token: tenantClaim.Token}); err != nil { + t.Fatal(err) + } + if _, err = current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Errorf("public static volume changed: %v", err) + } + rootClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) + if err != nil { + t.Fatalf("operator initial attachment: %v", err) + } + if err := current.Release(t.Context(), rootClaim.ID, Cred{Token: rootClaim.Token}); err != nil { + t.Fatalf("operator initial claim cleanup: %v", err) + } + } + }) + } +} + +func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { + m, sb := newTenantVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Tenants: []string{"other"}}); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "home"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("operator bypassed explicit claim-owner ACL: %v", err) + } +} + +func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { + for _, operation := range []string{"put", "patch"} { + t.Run(operation, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}}); err != nil { + t.Fatal(err) + } + want := []types.Volume{{Name: "home", Mount: "/home"}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatalf("still-held retry: %v", err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result: %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovery reference lost writer exclusion: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture ignored recovery reference: %v", err) + } + if err = m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister ignored recovery reference: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + m.mu.Lock() + entry := *m.catalog.entries["home"] + entry.tenants = []string{"other"} + m.catalog.entries["home"] = &entry + m.mu.Unlock() + before := len(eng.volumeSpecs) + switch operation { + case "put": + _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want) + case "patch": + _, err = m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + } + if !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("recovery-only reference authorized tenant attachment: %v", err) + } + if len(eng.volumeSpecs) != before { + t.Fatal("rejected tenant reached device attachment") + } + m.mu.Lock() + entry.tenants = []string{"acme"} + m.catalog.entries["home"] = &entry + m.mu.Unlock() + if _, err = m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatalf("operator reattach: %v", err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { + t.Fatalf("tenant cleanup after operator reattach: %v", err) + } + }) + } +} + +func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { + t.Helper() + cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + return m +} diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go new file mode 100644 index 00000000..4016f920 --- /dev/null +++ b/sandboxd/pool/volume_bench_test.go @@ -0,0 +1,81 @@ +package pool + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func BenchmarkVolumeAdmissionWithCheckpoints(b *testing.B) { + for _, checkpoints := range []bool{false, true} { + b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { + m := volumeBenchManager(b, checkpoints) + requested := []types.Volume{{Name: "bench-data", Mount: "/mnt/data"}} + b.ReportAllocs() + for b.Loop() { + resolved, err := m.resolveVolumes(b.Context(), "", requested) + if err != nil { + b.Fatal(err) + } + if len(resolved) != 1 || resolved[0].applied != requested[0] { + b.Fatalf("resolved volumes: %+v", resolved) + } + m.mu.Lock() + err = m.reserveResolved(resolved, nil) + if err == nil { + m.releaseVolumes(appliedVolumes(resolved)) + } + m.mu.Unlock() + if err != nil { + b.Fatal(err) + } + } + if len(m.catalog.holders) != 0 { + b.Fatalf("unreleased admission holds: %v", m.catalog.holders) + } + }) + } +} + +func BenchmarkVolumeHoldsWithCheckpoints(b *testing.B) { + for _, checkpoints := range []bool{false, true} { + b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { + m := volumeBenchManager(b, checkpoints) + b.ReportAllocs() + for b.Loop() { + m.mu.Lock() + held := 0 + for _, sb := range m.claimed { + if m.hasVolumeHolds(sb) { + held++ + } + } + m.mu.Unlock() + if held != 0 { + b.Fatalf("unrelated claims hold %d volumes", held) + } + } + }) + } +} + +func volumeBenchManager(b *testing.B, checkpoints bool) *Manager { + b.Helper() + m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) + m.claimed = checkpointBenchClaims(checkpoints) + if err := m.store.save(m.claimed); err != nil { + b.Fatal(err) + } + path := filepath.Join(b.TempDir(), "data.raw") + if err := os.WriteFile(path, []byte("benchmark backing"), 0o600); err != nil { + b.Fatal(err) + } + if err := m.RegisterVolume(b.Context(), "bench-data", types.VolumeRegistration{Path: path}); err != nil { + b.Fatal(err) + } + return m +} diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index 27c1b1dc..e0d0b8f4 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -15,7 +15,10 @@ func TestVolumeClaimRejectsDirectCapture(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) + m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} + m.store.set(sb) + m.mu.Unlock() tests := []struct { name string @@ -51,7 +54,10 @@ func TestVolumeClaimRunningWakeUnchanged(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) + m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} + m.store.set(sb) + m.mu.Unlock() sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { @@ -123,7 +129,10 @@ func TestIdleOnceSkipsVolumeClaim(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, IdleHibernateSeconds: 1}) sb := mustClaim(t, m, testKey) + m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} + m.store.set(sb) + m.mu.Unlock() backdate(m, sb, 2*time.Second) sweepIdle(t, m) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go new file mode 100644 index 00000000..f2ccc456 --- /dev/null +++ b/sandboxd/pool/volume_catalog.go @@ -0,0 +1,266 @@ +package pool + +import ( + "cmp" + "context" + "encoding/json/v2" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "sync" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +const runtimeVolumesFile = "runtime-volumes.json" + +type runtimeCatalog struct { + Version int `json:"version"` + Volumes map[string]types.VolumeRegistration `json:"volumes"` +} + +type catalogWrite struct { + entries map[string]*catalogVolume + raw []byte +} + +type volumeCatalog struct { + entries map[string]*catalogVolume + pending *catalogWrite + holders map[string]volumeHolders + unknown int + writeMu sync.Mutex +} + +func (c *volumeCatalog) admissionError() error { + if c.pending != nil || c.unknown > 0 { + return ErrVolumeCatalogUnavailable + } + return nil +} + +// RegisterVolume durably binds a root-managed name to a backing path. +func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + normalized, err := normalizeRegistration(name, spec) + if err != nil { + return err + } + m.mu.Lock() + entries := m.catalog.entries + err = m.catalog.admissionError() + m.mu.Unlock() + if err != nil { + return err + } + if entry := entries[name]; entry != nil { + if entry.runtime && sameRegistration(entry.registration(), normalized) { + return nil + } + return fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) + } + entries = maps.Clone(entries) + entries[name] = &catalogVolume{ + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + tenants: normalized.Tenants, + writable: normalized.Writable, + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) + } + return m.writeCatalog(ctx, name, entries) +} + +// UnregisterVolume retires an unreferenced runtime binding without touching its backing. +func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + if !types.ValidVolumeName(name) { + return fmt.Errorf("%w: invalid volume name", ErrBadVolume) + } + m.mu.Lock() + entries := m.catalog.entries + entry := entries[name] + err := m.catalog.admissionError() + if err == nil && ((entry != nil && !entry.runtime) || (entry == nil && m.volumeReferenced(name))) { + err = fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) + } + m.mu.Unlock() + if err != nil || entry == nil { + return err + } + entries = maps.Clone(entries) + delete(entries, name) + return m.writeCatalog(ctx, name, entries) +} + +func (m *Manager) loadRuntimeVolumes() error { + raw, err := os.ReadFile(filepath.Join(m.dataDir, runtimeVolumesFile)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read runtime volume catalog: %w", err) + } + var record runtimeCatalog + if err = json.Unmarshal(raw, &record, json.RejectUnknownMembers(true)); err != nil { + return fmt.Errorf("parse runtime volume catalog: %w", err) + } + if record.Version != 1 || record.Volumes == nil { + return errors.New("invalid runtime volume catalog version or volumes") + } + entries := maps.Clone(m.catalog.entries) + for name, spec := range record.Volumes { + normalized, normalizeErr := normalizeRegistration(name, spec) + if normalizeErr != nil { + return fmt.Errorf("runtime volume catalog: %w", normalizeErr) + } + if m.catalog.entries[name] != nil { + return fmt.Errorf("runtime volume catalog duplicates static name %q", name) + } + entries[name] = &catalogVolume{ + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + tenants: normalized.Tenants, + writable: normalized.Writable, + } + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } + m.catalog.entries = entries + return nil +} + +func (m *Manager) volumeReferenced(name string) bool { + if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { + return true + } + for _, run := range m.volumeRuns { + if volumeIndex(run.holds, name) >= 0 { + return true + } + } + for _, sb := range m.claimed { + if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { + return true + } + } + return m.store.referencesVolume(name) +} + +func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[string]*catalogVolume) error { + record := runtimeCatalog{Version: 1, Volumes: map[string]types.VolumeRegistration{}} + for key, entry := range entries { + if entry.runtime { + record.Volumes[key] = entry.registration() + } + } + raw, err := json.Marshal(record, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("%w: %w", ErrBadVolume, err) + } + proposal := &catalogWrite{entries: entries, raw: raw} + m.mu.Lock() + err = ctx.Err() + if err == nil { + err = m.catalog.admissionError() + } + if err == nil && m.volumeReferenced(name) { + err = fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + if err == nil { + m.catalog.pending = proposal + } + m.mu.Unlock() + if err != nil { + return err + } + return m.persistCatalog(proposal) +} + +func (m *Manager) commitPendingCatalog(ctx context.Context) error { + m.mu.Lock() + proposal, unknown := m.catalog.pending, m.catalog.unknown > 0 + m.mu.Unlock() + if proposal == nil { + return nil + } + if unknown { + return ErrVolumeCatalogUnavailable + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + return m.persistCatalog(proposal) +} + +func (m *Manager) persistCatalog(proposal *catalogWrite) error { + if err := utils.WriteFileSync(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw, 0o600); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + m.mu.Lock() + m.catalog.entries = proposal.entries + m.catalog.pending = nil + m.mu.Unlock() + return nil +} + +func (m *Manager) retryVolumeCatalog(ctx context.Context) { + if !m.catalog.writeMu.TryLock() { + return + } + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + log.WithFunc("pool.retryVolumeCatalog").Error(ctx, err, "persist runtime volume catalog") + } +} + +func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { + volumes := make([]config.VolumeSpec, 0, len(entries)) + for name, entry := range entries { + var tenants []string + if entry.runtime { + tenants = entry.tenants + } + volumes = append(volumes, config.VolumeSpec{ + Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, + Writable: entry.writable, Tenants: tenants, + }) + } + return volumes +} + +func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { + spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ + Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, + }}); err != nil { + return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) + } + spec.Path = filepath.Clean(spec.Path) + spec.Tenants = slices.Clone(spec.Tenants) + slices.Sort(spec.Tenants) + spec.Tenants = slices.Compact(spec.Tenants) + return spec, nil +} + +func sameRegistration(a, b types.VolumeRegistration) bool { + return a.Path == b.Path && a.DirectIO == b.DirectIO && a.Writable == b.Writable && slices.Equal(a.Tenants, b.Tenants) +} diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go new file mode 100644 index 00000000..002f7c5e --- /dev/null +++ b/sandboxd/pool/volume_catalog_test.go @@ -0,0 +1,617 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "sync" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "backing bytes stay intact") + spec := types.VolumeRegistration{Path: filepath.Dir(path) + "/./runtime", Writable: true, Tenants: []string{"beta", "acme", "acme"}} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + spec.Path = path + spec.DirectIO = types.DirectIOOff + spec.Tenants = []string{"acme", "beta"} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("normalized retry: %v", err) + } + st, statErr := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) + if statErr != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, statErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("recovered names: %v", got) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "forbidden", Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("ACL changed at restart: %v", err) + } + resolved, err := m2.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}) + if err != nil || len(resolved) != 1 || resolved[0].disk.DirectIO != "off" || !resolved[0].disk.RW { + t.Fatalf("policy roundtrip: %+v %v", resolved, err) + } + changed := spec + changed.DirectIO = "on" + if err := m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("changed semantics: %v", err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatalf("absent retry: %v", err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != "backing bytes stay intact" { + t.Fatalf("backing changed: %q %v", got, err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("unregister not durable: %v", got) + } +} + +func TestRuntimeCatalogLoadsOperatorManagedRecord(t *testing.T) { + for _, setup := range []struct { + name string + prepare func(*testing.T, string) + }{ + {"mode", func(t *testing.T, record string) { + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + }}, + {"symlink", func(t *testing.T, record string) { + if err := os.Rename(record, record+".target"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(record+".target", record); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(setup.name, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + setup.prepare(t, filepath.Join(m.dataDir, runtimeVolumesFile)) + restarted := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("operator-managed record not loaded: %v", got) + } + if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("replacement not loaded: %v", got) + } + }) + } +} + +func TestRuntimeCatalogUsesStaticPathValidation(t *testing.T) { + regular := writeVolumeImage(t, "backing", "unchanged") + link := filepath.Join(t.TempDir(), "backing-link") + if err := os.Symlink(regular, link); err != nil { + t.Fatal(err) + } + for _, path := range []string{regular, link, os.DevNull, t.TempDir(), filepath.Join(t.TempDir(), "missing")} { + t.Run(filepath.Base(path), func(t *testing.T) { + spec := config.VolumeSpec{Name: "runtime", Path: path, DirectIO: types.DirectIOOff, Writable: true} + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{spec}); err != nil { + t.Fatalf("static validation: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), nil) + registration := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("runtime validation differs from static: %v", err) + } + m2 := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) + if err := m2.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("idempotent retry after restart: %v", err) + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + if _, resolveErr := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(resolveErr, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", resolveErr) + } + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + }) + } + if got, err := os.ReadFile(regular); err != nil || string(got) != "unchanged" { + t.Fatalf("registration touched backing: %q %v", got, err) + } +} + +func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { + eng := newFakeEngine() + static := liveCatalog(t) + m := newVolumeManager(t, eng, static) + other := writeVolumeImage(t, "other-runtime", "bytes") + for _, tc := range []struct{ name, path string }{{"data", other}, {"runtime", static[0].Path}, {"runtime", filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path)}} { + if err := m.RegisterVolume(t.Context(), tc.name, types.VolumeRegistration{Path: tc.path}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("collision %+v: %v", tc, err) + } + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("static removal: %v", err) + } + sb := mustClaim(t, m, testKey) + sb.Volumes = []types.Volume{{Name: "unexplained", Mount: "/data"}} + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "unexplained", types.VolumeRegistration{Path: other}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebound unexplained recovery name: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("absent referenced removal: %v", err) + } + for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}} { + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid spec %+v: %v", spec, err) + } + } +} + +func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + m.store.path = filepath.Join(t.TempDir(), "claims.json") + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} + restore := blockCatalogRename(t, m) + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("write failure: %v", err) + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("pending admission: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + restore() + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry pending register: %v", err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("durable registration: %v", got) + } + restore = blockCatalogRename(t, m) + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("failed unregister: %v", err) + } + restore() + if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { + t.Fatalf("pending unregister did not converge before reuse: %v", err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"replacement"}) { + t.Fatalf("durable reuse: %v", got) + } +} + +func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} + restore := blockCatalogRename(t, m) + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("write failure: %v", err) + } + restore() + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("possible replacement not recovered: %v", got) + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain admission: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry pending register: %v", err) + } + restore = blockCatalogRename(t, m) + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain unregister: %v", err) + } + restore() + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("possible unregister replacement not recovered: %v", got) + } + if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { + t.Fatalf("pending unregister did not converge before reuse: %v", err) + } +} + +func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "original") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + spec.Path = writeVolumeImage(t, "replacement", "replacement") + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("stale binding admitted: %v", err) + } + if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { + t.Fatalf("stale binding leaked reservation: %+v", got) + } +} + +func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if os.Geteuid() == 0 { + if err := os.Chown(path, 65534, 65534); err != nil { + t.Fatal(err) + } + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("nonroot backing owner: %v", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + infos := m2.Volumes("", nil) + if len(infos) != 1 || infos[0].Available { + t.Fatalf("missing backing lost mapping: %+v", infos) + } + if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", err) + } + if err := m2.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("idempotent missing mapping: %v", err) + } +} + +func TestRuntimeRegistrationNormalization(t *testing.T) { + input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} + normalized, err := normalizeRegistration("data", input) + if err != nil || normalized.Path != "/images/data" || normalized.DirectIO != types.DirectIOOff || !slices.Equal(normalized.Tenants, []string{"acme", "beta"}) { + t.Fatalf("normalization: %+v %v", normalized, err) + } + normalized.Tenants[0] = "caller-copy" + if input.Tenants[1] != "acme" { + t.Fatal("normalization mutated caller input") + } + for _, tc := range []struct { + name string + spec types.VolumeRegistration + }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{""}}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{"has space"}}}} { + if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid registration %+v: %v", tc, err) + } + } +} + +func TestRuntimeCatalogRejectsInvalidMetadata(t *testing.T) { + static := liveCatalog(t) + for _, tc := range []struct{ name, raw string }{ + {"malformed", "{"}, + {"version", `{"version":2,"volumes":{}}`}, + {"null", `{"version":1,"volumes":null}`}, + {"name", `{"version":1,"volumes":{"bad/name":{"path":"/data"}}}`}, + {"relative", `{"version":1,"volumes":{"runtime":{"path":"relative"}}}`}, + {"directio", `{"version":1,"volumes":{"runtime":{"path":"/data","directio":"bad"}}}`}, + {"duplicate-name", `{"version":1,"volumes":{"runtime":{"path":"/one"},"runtime":{"path":"/two"}}}`}, + {"duplicate-path", `{"version":1,"volumes":{"runtime":{"path":"/one"},"second":{"path":"/one"}}}`}, + {"static-name", `{"version":1,"volumes":{"data":{"path":"/other"}}}`}, + {"static-path", fmt.Sprintf(`{"version":1,"volumes":{"runtime":{"path":%q}}}`, static[0].Path)}, + {"unknown-field", `{"version":1,"volumes":{},"unexpected":true}`}, + } { + t.Run(tc.name, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, static) + if err := os.WriteFile(filepath.Join(m.dataDir, runtimeVolumesFile), []byte(tc.raw), 0o600); err != nil { + t.Fatal(err) + } + cfg := &config.Config{DataDir: m.dataDir, Volumes: static} + if _, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)); err == nil { + t.Fatalf("accepted %s metadata", tc.name) + } + }) + } +} + +func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "unchanged") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "data", spec); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected failed result commit") + } + eng.liveVolumeHook = nil + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "different")}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebind unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("alias unretired path: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("same-file writer: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture: %v", err) + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("hibernate before retirement: %v", err) + } + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); err != nil { + t.Fatal(err) + } +} + +func TestCatalogSnapshotSurvivesRegistration(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + before := m.catalogSnapshot() + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/images/runtime"}); err != nil { + t.Fatal(err) + } + if before["runtime"] != nil { + t.Fatal("registration mutated captured catalog") + } + captured := m.catalogSnapshot() + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if captured["runtime"] == nil { + t.Fatal("unregister mutated captured catalog") + } +} + +func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) + if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("cancellation before acceptance: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled proposal left catalog: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("canceled registration fenced static claim: %v", err) + } +} + +func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + restore := blockCatalogRename(t, m) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("initial write failure: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("failed retry cleared pending write: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("pending replacement lost fence: %v", err) + } + restore() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := m.UnregisterVolume(ctx, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("canceled retry cleared pending write: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if !slices.Contains(m.VolumeNames(), "runtime") { + t.Fatal("retry lost accepted registration") + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatalf("repair did not restore static admission: %v", err) + } +} + +func TestResolvedCatalogBindingChangesOnlyForReusedName(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { + t.Fatal(err) + } + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatalf("unrelated registration invalidated resolution: %v", err) + } + m.unreserveVolumes(holds) + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("same-file reuse admitted stale resolution: %v", err) + } +} + +func TestRuntimeCatalogIgnoresAbandonedTemporaryFiles(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "bytes") + raw := fmt.Sprintf(`{"version":1,"volumes":{"abandoned":{"path":%q}}}`, path) + tmp := filepath.Join(m.dataDir, runtimeVolumesFile+".interrupted.tmp") + if err := os.WriteFile(tmp, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("temporary record promoted without canonical record: %v", got) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("temporary record overrode canonical record: %v", got) + } + if got, err := os.ReadFile(tmp); err != nil || string(got) != raw { + t.Fatalf("abandoned evidence changed: %q %v", got, err) + } +} + +func TestRuntimeCatalogAcceptsDaemonDataDirectorySymlink(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "state") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(parent, "linked-state") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + if err := os.Chmod(parent, 0o777); err != nil { + t.Fatal(err) + } + eng := newFakeEngine() + m := newVolumeManagerAt(t, eng, link, nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + restarted := newVolumeManagerAt(t, eng, link, nil) + if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("reloaded catalog: %v", got) + } + if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + entries := maps.Clone(m.catalogSnapshot()) + delete(entries, "runtime") + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatal(err) + } + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.writeCatalog(t.Context(), "runtime", entries); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("accepted removal prepared before reservation: %v", err) + } + m.unreserveVolumes(holds) + if err := m.writeCatalog(t.Context(), "runtime", entries); err != nil { + t.Fatalf("reservation retirement did not unblock removal: %v", err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("removal was not persisted: %v", got) + } +} + +func blockCatalogRename(t *testing.T, m *Manager) func() { + t.Helper() + record := filepath.Join(m.dataDir, runtimeVolumesFile) + raw, err := os.ReadFile(record) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + existed := err == nil + if existed { + if err := os.Remove(record); err != nil { + t.Fatal(err) + } + } + if err := os.Mkdir(record, 0o700); err != nil { + t.Fatal(err) + } + restore := sync.OnceFunc(func() { + if err := os.Remove(record); err != nil { + t.Fatal(err) + } + if existed { + if err := os.WriteFile(record, raw, 0o600); err != nil { + t.Fatal(err) + } + } + }) + t.Cleanup(restore) + return restore +} + +func installPendingCatalogRecord(t *testing.T, m *Manager) { + t.Helper() + m.mu.Lock() + proposal := m.catalog.pending + m.mu.Unlock() + if proposal == nil { + t.Fatal("possible replacement fixture requires an accepted pending write") + } + // Model the canonical bytes after a successful rename and uncertain directory sync. + if err := utils.WriteFileSync(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go new file mode 100644 index 00000000..c2ff00ae --- /dev/null +++ b/sandboxd/pool/volume_cleanup_test.go @@ -0,0 +1,543 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { + for _, state := range []string{vmStateRunning, vmStateCreating, "stopped"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "incomplete-inventory", "original") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-incomplete-inventory" + base.vms[name] = "/vsock/incomplete" + base.creating[name] = state == vmStateCreating + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + cli := filepath.Join(t.TempDir(), "inspect") + writeInspect := func(state, inventory string) { + t.Helper() + raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"%s\"%s}'\n", name, state, inventory) + if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { + t.Fatal(err) + } + } + writeInspect(state, "") + m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + rec := base.record(name) + rec.State = state + if m.removeStaleVM(t.Context(), name, rec) { + t.Fatal("stale VM unexpectedly disappeared") + } + for range 2 { + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unregister must refuse incomplete inventory: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("conflicting writer must refuse incomplete inventory: %v", err) + } + m.retryRemovals(t.Context()).Wait() + } + base.creating[name] = false + writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) + if state == vmStateCreating { + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy owner must retain the unknown fence: %v", err) + } + } + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("complete inventory lost the surviving writer: %v", err) + } + other := writeVolumeImage(t, "complete-inventory-other", "other") + if err := m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: other}); err != nil { + t.Fatalf("complete inventory retained the global fence: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained volume protection: %v", err) + } + }) + } +} + +func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true}); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + changed := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); changed <- err }() + <-eng.entered + resumed := false + defer func() { + if !resumed { + close(eng.resume) + <-changed + } + }() + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + select { + case err := <-released: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("release waited for paused worker") + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("removed VM freed active worker count: %+v", got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer overtook paused canceled worker: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Errorf("unregister overtook paused worker: %v", err) + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("maintenance freed active worker count: %+v", got) + } + close(eng.resume) + resumed = true + if err := <-changed; err == nil { + t.Fatal("removed claim accepted late attachment") + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("completed worker leaked count: %+v", got) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if err != nil { + t.Fatal(err) + } + base.removeErrFor = sb.VMName + if err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + m.mu.Lock() + queued := len(m.pendingRemovals) + m.mu.Unlock() + if queued != 0 { + t.Fatal("cleanup was not dispatched") + } + err = m.UnregisterVolume(t.Context(), "data") + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + if !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("dispatched cleanup lost guard: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { + for _, state := range []string{"running", "stopped", "creating"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-stale-unknown" + base.vms[name] = "/vsock/stale" + base.stopped[name] = state == "stopped" + base.creating[name] = state == "creating" + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + var inspectFailed atomic.Bool + inspectFailed.Store(true) + base.inspectHook = func() error { + if inspectFailed.Load() { + return errors.New("inspection unavailable") + } + return nil + } + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("unknown stale VM unexpectedly gone") + } + for range 2 { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment admitted reader: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment allowed unregister: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + if err := m.RegisterVolume(t.Context(), "new", types.VolumeRegistration{Path: writeVolumeImage(t, "new", "bytes")}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("dispatched unknown fence lost: %v", err) + } + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + } + inspectFailed.Store(false) + base.stopped[name], base.creating[name] = false, false + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { + t.Fatalf("inspection did not replace fence with known holds: %+v", got) + } + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated", "bytes")}); err != nil { + t.Fatalf("resolved inspection left global fence: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("known attachment lost exclusion: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained fence: %v", err) + } + }) + } +} + +func TestRuntimeCatalogLoadedBeforeClaimReconciliation(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true, Tenants: []string{"acme"}}); err != nil { + t.Fatal(err) + } + sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if claimErr != nil { + t.Fatal(claimErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "data" || !got[0].Available || !got[0].Writable { + t.Fatalf("catalog absent before reconcile: %+v", got) + } + if local, err := m2.VolumePlacement(testKey, "acme", []string{"data"}); err != nil || !local { + t.Fatalf("placement before reconcile: %v %v", local, err) + } + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + view, err := m2.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(view.Volumes, sb.Volumes) { + t.Fatalf("claim recovery: %+v %v", view, err) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovered holder missing: %v", err) + } +} + +func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-stale-before-inspect" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + entered, resume := make(chan struct{}), make(chan struct{}) + var gated atomic.Bool + base.inspectHook = func() error { + if gated.CompareAndSwap(false, true) { + close(entered) + <-resume + } + return errors.New("inspection unavailable") + } + done := make(chan bool, 1) + go func() { done <- m.removeStaleVM(t.Context(), name, base.record(name)) }() + <-entered + if !m.mu.TryLock() { + t.Fatal("stale inspection holds manager lock") + } + m.mu.Unlock() + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + close(resume) + if gone := <-done; gone { + t.Fatal("failed inspect/removal unexpectedly confirmed gone") + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("admitted volume while inspection was blocked: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("successful removal retained unknown fence: %v", err) + } +} + +func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-canceled-dispatch" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + base.inspectHook = func() error { return errors.New("inspection unavailable") } + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + ctx, cancel := context.WithCancel(t.Context()) + wg := m.retryRemovals(ctx) + cancel() + synctest.Wait() + base.removeErrFor = "" + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("canceled dispatch stranded unknown fence: %v", err) + } + }) +} + +func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { + base := newFakeEngine() + name := "sbx-warm-unfinalized" + base.vms[name] = "/vsock/warm" + base.diskAttachErrFor = "broken" + base.removeErrFor = name + rendezvousAttaches(base, 2) + catalog := []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "warm-data", "data"), Writable: true}, + {Name: "broken", Path: writeVolumeImage(t, "warm-broken", "broken")}, + } + dir := t.TempDir() + m := newVolumePoolManager(t, base, dir, catalog) + base.inspectDisksUnknown, base.trackDisks = true, true + warm := &types.Sandbox{VMName: name, Key: testKey, VsockSocket: "/vsock/warm"} + m.pools[testKey].warm = append(m.pools[testKey].warm, warm) + _, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{ + {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}, + {Name: "broken", AttachOnly: true}, + }}) + if err == nil { + t.Fatal("expected attach failure before finalization") + } + if warm.ID != "" { + t.Fatal("claim unexpectedly finalized") + } + base.mu.Lock() + attached := slices.Clone(base.disks[name]) + base.attachRendezvous = nil + base.mu.Unlock() + if !slices.ContainsFunc(attached, func(d types.VMDisk) bool { return d.Name == "data" && !d.ReadOnly }) { + t.Fatalf("warm VM has no attached writer: %+v", attached) + } + claims, err := newClaimStore(dir, false).load() + if err != nil || len(claims) != 0 { + t.Fatalf("pre-finalize journal: %+v, %v", claims, err) + } + restarted := newVolumeManagerAt(t, base, dir, catalog) + restarted.eng = base + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("ambiguous warm survivor admitted competing writer: %v", err) + } + base.removeErrFor = "" + restarted.retryRemovals(t.Context()).Wait() + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("confirmed removal failed to clear fence: %v", err) + } +} + +func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) { + base := newFakeEngine() + m := newTestManager(t, base, archivePool(3600)) + sb := mustClaim(t, m, testKey) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if err := m.archive(t.Context(), sb); err != nil { + t.Fatal(err) + } + if sb.VMName != "" { + t.Fatalf("archived VM name = %q", sb.VMName) + } + ctx, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + if err := m.checkVolumeRun(ctx, sb, run); !errors.Is(err, ErrArchived) { + t.Fatalf("archived mutation validation: %v", err) + } + td := m.quiesceVolumes(t.Context(), sb) + if td.done == nil || len(td.holds) != 0 || len(td.clears) != 0 || td.unknown { + t.Fatalf("archived teardown has cleanup obligations: %+v", td) + } + if m.finishVolumeTeardown(t.Context(), td) { + t.Fatal("open volume run unexpectedly completed") + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.finishVolumeRun(sb, run) + if len(m.pendingRemovals) != 0 { + t.Fatalf("empty teardown enqueued removal: %+v", m.pendingRemovals) + } + if slices.Contains(base.removedNames(), "") { + t.Fatal("release removed an empty VM name") + } + if m.store.referencesVolume("data") { + t.Fatal("release retained archived journal references") + } +} + +func TestStaleInventoryDistinguishesOmittedAndExplicitEmptyDisks(t *testing.T) { + for _, tc := range []struct { + name, inventory string + wantErr error + }{ + {"omitted devices", "", ErrVolumeCatalogUnavailable}, + {"omitted disks", `,"attached_devices":{}`, ErrVolumeCatalogUnavailable}, + {"null disks", `,"attached_devices":{"disks":null}`, ErrVolumeCatalogUnavailable}, + {"explicit empty disks", `,"attached_devices":{"disks":[]}`, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-inventory-shape" + base.vms[name], base.removeErrFor = "/vsock/stale", name + cli := filepath.Join(t.TempDir(), "inspect") + raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"running\"%s}'\n", name, tc.inventory) + if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { + t.Fatal(err) + } + m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("stale VM unexpectedly disappeared") + } + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("admission=%v, want=%v", err, tc.wantErr) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { + t.Fatalf("confirmed removal retained registration fence: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("confirmed removal retained admission fence: %v", err) + } + }) + } +} + +func TestStaleInventoryNarrowsFenceBeforeBlockedRemoval(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-known-before-remove" + eng.vms[name], eng.removeErrFor = "/vsock/stale", name + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + eng.removeStall = make(chan struct{}) + done := make(chan bool, 1) + go func() { done <- m.removeStaleVM(t.Context(), name, eng.record(name)) }() + synctest.Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("known stale writer: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { + t.Errorf("blocked removal retained global fence: %v", err) + } + close(eng.removeStall) + if <-done { + t.Fatal("failed removal reported gone") + } + }) +} + +func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { + for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound} { + t.Run(string(outcome), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-busy-create" + eng.vms[name], eng.creating[name] = "/vsock/creating", true + eng.staleOutcome = engine.StaleCreateBusy + eng.inspectHook = func() error { + t.Error("busy owner must finish before inventory can be trusted") + return errors.New("busy") + } + rec := eng.record(name) + rec.State = vmStateCreating + if m.removeStaleVM(t.Context(), name, rec) { + t.Fatal("busy create reported gone") + } + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy create admitted a volume: %v", err) + } + eng.staleOutcome = outcome + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("confirmed cleanup retained fence: %v", err) + } + }) + } +} + +type incompleteInventoryEngine struct { + *fakeEngine + cli *engine.Engine +} + +func (e *incompleteInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + return e.cli.Inspect(ctx, name) +} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 7c002ecc..ef8fb774 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -22,7 +22,7 @@ func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - delete(m.volumes, "data") + delete(m.catalog.entries, "data") before := len(eng.volumeSpecs) if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { t.Fatal(err) @@ -57,9 +57,9 @@ func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { t.Fatal("expected detaching state") } eng.detachVolumeErr = nil - entry := m.volumes["data"] + entry := m.catalog.entries["data"] entry.tenants = []string{"other-tenant"} - m.volumes["data"] = entry + m.catalog.entries["data"] = entry if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { t.Fatalf("held repair checked new authorization: %v", err) } @@ -313,7 +313,7 @@ func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { } func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { - for _, state := range []string{"applied", "desired", "admitted"} { + for _, state := range []string{"applied", "desired", "admitted", "recovery"} { t.Run(state, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}) @@ -321,9 +321,27 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { held := []types.Volume{{Name: "data", Mount: "/volumes/data"}} switch state { case "applied": + m.mu.Lock() sb.Volumes = held + m.store.set(sb) + m.mu.Unlock() case "desired": + m.mu.Lock() + sb.VolumesWant = held + m.store.set(sb) + m.mu.Unlock() + case "recovery": + m.mu.Lock() sb.VolumesWant = held + js := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(js); err != nil { + t.Fatal(err) + } + m.mu.Lock() + sb.VolumesWant = nil + m.store.set(sb) + m.mu.Unlock() case "admitted": _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { @@ -350,6 +368,14 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { if len(eng.trims) != 0 { t.Fatalf("capture trimmed held volume: %v", eng.trims) } + if state == "recovery" { + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retired", ""); err != nil { + t.Fatalf("checkpoint after recovery acknowledgment: %v", err) + } + } }) } } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 33dc50e2..596d047f 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -86,20 +86,21 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) + recovery := m.store.recoveryVolumesFor(sb.ID) m.mu.Unlock() target, err := change(current) if err != nil { return view, err } - target, err = normalizeLiveVolumes(target, held) + target, err = normalizeLiveVolumes(target, slices.Concat(held, recovery)) if err != nil { return view, err } - if err = validateLiveTarget(held, target); err != nil { + if err = validateLiveTarget(slices.Concat(held, recovery), target); err != nil { return view, err } additions := volumeDifference(target, held) - if len(additions) != 0 { + if len(additions) > 0 { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err } @@ -146,7 +147,10 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - repair, resolveErr := m.resolveCatalogVolume(ctx, v) + m.mu.Lock() + entry := m.catalog.entries[v.Name] + m.mu.Unlock() + repair, resolveErr := m.resolveCatalogEntry(ctx, v, entry) if resolveErr != nil { return view, resolveErr } @@ -257,23 +261,28 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, err } if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveVolumes(additions); err != nil { + if err := m.reserveResolved(resolved, sb); err != nil { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, err } run.holds = append(run.holds, additions...) m.mu.Unlock() + sb.Transition.Unlock() cleanErr := m.confirmVolumesClean(resolved) + sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -343,19 +352,26 @@ func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { return m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run } -func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { +func (m *Manager) hasLiveVolumeHolds(sb *types.Sandbox) bool { run := m.volumeRuns[sb] return hasVolumes(sb) || run != nil && len(run.holds) > 0 } +func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { + return m.hasLiveVolumeHolds(sb) || m.store.hasRecoveryVolumes(sb.ID) +} + func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { + held := heldVolumes(sb) + recovering := volumeDifference(m.store.recoveryVolumesFor(sb.ID), held) view := types.SandboxVolumesResponse{ Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, - Converging: m.volumeRuns[sb] != nil, + Converging: m.volumeRuns[sb] != nil || len(recovering) > 0, } - for _, v := range heldVolumes(sb) { + held = append(held, recovering...) + for _, v := range held { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { state = types.VolumeDetaching diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 4f94b749..d9de1c70 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -280,8 +280,11 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { } close(resume) attachErr := <-attached - if action == "reap" { - waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) + if action == "release" || action == "reap" { + waitFor(t, func() bool { + m.retryRemovals(t.Context()).Wait() + return volumeHoldersOf(m, "data") == (volumeHolders{}) + }) } if action == "release" || action == "reap" { if attachErr == nil { @@ -718,6 +721,7 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) + m.retryRemovals(t.Context()).Wait() waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go new file mode 100644 index 00000000..ae6b6ea5 --- /dev/null +++ b/sandboxd/pool/volume_retirement_test.go @@ -0,0 +1,416 @@ +package pool + +import ( + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result = %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || len(claims[sb.ID].Volumes) != 1 { + t.Fatalf("failed write did not leave old recovery reference: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer admitted while old claim still references detached volume: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Errorf("capture passed before recovery reference retired: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost old claim's reader exclusion: %v", err) + } + if _, err = m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatalf("retry after restart: %v", err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer still blocked after successful retirement: %v", err) + } +} + +func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { + for _, retry := range []string{"empty", "same-shape", "unrelated-snapshot"} { + t.Run(retry, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("detach result unexpectedly persisted") + } + eng.liveVolumeHook = nil + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("physical count retained after detach: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mount: "/different"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("recovery-only shape changed: %v", err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { + t.Fatalf("unretired view: %+v, %v", view, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + switch retry { + case "empty": + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "same-shape": + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got.readers != 1 || got.writers != 0 { + t.Fatalf("own retry counts: %+v", got) + } + } + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "unrelated-snapshot": + _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + } + if err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("retired count: %+v", got) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer after acknowledgment: %v", err) + } + }) + } +} + +func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil { + t.Fatal(err) + } + restoreWrites := failClaimWrites(t, m.store) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected failed intent write") + } + claims[sb.ID].VolumesWant = []types.Volume{{Name: "data", Mount: types.DefaultVolumeMount("data"), Mode: types.VolumeModeRO}} + raw, err := json.Marshal(claims) + if err != nil { + t.Fatal(err) + } + // Model an intent whose rename succeeded before an uncertain directory sync. + if err = utils.WriteFileSync(m.store.path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected empty snapshot error") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 { + t.Fatalf("memory not empty: %+v", sb) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("unexpected physical count: %+v", got) + } + claims, err = newClaimStore(m.dataDir, false).load() + if err != nil || len(claims[sb.ID].VolumesWant) != 1 { + t.Fatalf("actual recovery file: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer admitted after uncertain introduction: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain-introduction", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture admitted: %v", err) + } + restoreWrites() + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart admitted writer: %v", err) + } + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if len(m.store.recoveryVolumesFor(sb.ID)) != 0 { + t.Fatal("successful whole snapshot did not retire attempted row") + } +} + +func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s = newClaimStore(filepath.Dir(s.path), false) + if _, err := s.load(); err != nil { + t.Fatal(err) + } + s.reset(map[string]*types.Sandbox{}) + refs := s.recoveryVolumesFor(sb.ID) + refs[0].Name = "caller-copy" + if got := s.recoveryVolumesFor(sb.ID); len(got) != 1 || got[0].Name != "data" { + t.Fatalf("reset or caller erased actual file: %+v", got) + } + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if s.hasRecoveryVolumes(sb.ID) { + t.Fatal("empty snapshot did not retire file references") + } +} + +func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + restoreWrites := failClaimWrites(t, s) + for i := range 200 { + sb.Token = fmt.Sprint(i) + sb.Volumes[0].Mount = []string{"/data", "/other"}[i%2] + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("expected failed snapshot") + } + } + if refs := s.recoveryVolumesFor(sb.ID); len(refs) != 2 { + t.Fatalf("renewals retained duplicate history: %+v", refs) + } + refs := s.recoveryVolumesFor(sb.ID) + refs[0].Name = "caller-copy" + if got := s.recoveryVolumesFor(sb.ID); len(got) != 2 || got[0].Name != "data" { + t.Fatalf("recovery references not copied: %+v", got) + } + restoreWrites() + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.recoveryVolumesFor(sb.ID)) != 1 { + t.Fatal("successful snapshot did not replace possibility history") + } +} + +func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err == nil { + t.Fatal("expected failed detach result commit") + } + eng.liveVolumeHook = nil + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + m.mu.Lock() + entry := *m.catalog.entries["data"] + entry.tenants = []string{"different-tenant"} + m.catalog.entries["data"] = &entry + m.mu.Unlock() + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("recovery-only retry bypassed tenant removal: %v", err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("rejected recovery retry reserved a reader: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("recovery retirement after tenant removal: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("repair exemption admitted unrelated addition: %v", err) + } +} + +func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + applied := []types.Volume{{Name: "data", Mount: "/data"}} + wanted := []types.Volume{{Name: "wanted", Mount: "/wanted"}} + sb := &types.Sandbox{ID: "owner", Volumes: slices.Clone(applied), VolumesWant: slices.Clone(wanted)} + old := s.set(sb) + release := captureClaimWrite(t, s, old, sb.ID) + newer := s.del(sb.ID) + sb.Volumes[0].Name = "mutated-applied" + sb.VolumesWant[0].Name = "mutated-wanted" + raw := release() + if !s.pending(newer) { + t.Fatal("captured snapshot acknowledged newer deletion") + } + if refs := s.recoveryVolumesFor(sb.ID); !slices.Equal(refs, append(slices.Clone(applied), wanted...)) { + t.Fatalf("captured row not acknowledged: %+v", refs) + } + var persisted map[string]*types.Sandbox + err := json.Unmarshal(raw, &persisted) + if err != nil || persisted[sb.ID] == nil || !slices.Equal(persisted[sb.ID].Volumes, applied) || !slices.Equal(persisted[sb.ID].VolumesWant, wanted) { + t.Fatalf("captured write: %+v, %v", persisted, err) + } + if err = s.commit(newer); err != nil { + t.Fatal(err) + } + if s.hasRecoveryVolumes(sb.ID) { + t.Fatal("empty replacement retained old reference") + } + restoreWrites := failClaimWrites(t, s) + defer restoreWrites() + if err = s.commit(old); err != nil { + t.Fatal(err) + } + persisted, err = newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || len(persisted) != 0 { + t.Fatalf("old token resurrected snapshot: %+v, %v", persisted, err) + } +} + +func TestClaimSharedRecoveryVolumesRetirePerOwner(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + volume := types.Volume{Name: "shared", Mount: "/data"} + first := &types.Sandbox{ID: "first", Volumes: []types.Volume{volume}} + second := &types.Sandbox{ID: "second", Volumes: []types.Volume{volume}} + writer := volume + writer.Mode = types.VolumeModeRW + if err := s.commit(s.set(first, second)); err != nil { + t.Fatal(err) + } + if s.volumeConflicts(volume, "") || !s.volumeConflicts(writer, "") { + t.Fatal("shared readers have incorrect admission") + } + if err := s.commit(s.del(first.ID)); err != nil { + t.Fatal(err) + } + if !s.referencesVolume(volume.Name) || s.hasRecoveryVolumes(first.ID) || !s.hasRecoveryVolumes(second.ID) || !s.volumeConflicts(writer, first.ID) { + t.Fatal("retiring one reader released the other") + } + if err := s.commit(s.del(second.ID)); err != nil { + t.Fatal(err) + } + if s.referencesVolume(volume.Name) || s.volumeConflicts(writer, "") { + t.Fatal("last reader left recovery reservation") + } +} + +func TestClaimRecoveryVolumeOwnerExemptionRequiresExactAttachment(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + writer := types.Volume{Name: "data", Mount: "/data", Mode: types.VolumeModeRW} + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{writer}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + restore := failClaimWrites(t, s) + sb.VolumesWant = []types.Volume{{Name: "other", Mount: "/other"}} + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("expected write failure") + } + reader := writer + reader.Mode = types.VolumeModeRO + moved := writer + moved.Mount = "/elsewhere" + for _, tc := range []struct { + volume types.Volume + owner string + conflict bool + }{ + {writer, sb.ID, false}, {writer, "different", true}, {reader, sb.ID, true}, {moved, sb.ID, true}, + } { + if got := s.volumeConflicts(tc.volume, tc.owner); got != tc.conflict { + t.Errorf("volume %+v owner %q conflict=%t, want %t", tc.volume, tc.owner, got, tc.conflict) + } + } + s.set(&types.Sandbox{ID: sb.ID, Volumes: []types.Volume{writer}}) + restore() + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if s.referencesVolume("other") || s.volumeConflicts(writer, sb.ID) { + t.Fatal("successful return retained attempted attachment") + } +} + +func TestClaimCaptureAcknowledgesBothReferenceDomains(t *testing.T) { + for _, domain := range []string{"volumes", "checkpoints"} { + t.Run(domain, func(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ReplicaCk: "checkpoint", Volumes: []types.Volume{{Name: "data"}}} + release := captureClaimWrite(t, s, s.set(sb), sb.ID) + if domain == "volumes" { + sb.Volumes = nil + } else { + sb.ReplicaCk = "" + } + newer := s.set(sb) + release() + if !s.pending(newer) || !s.referencesCheckpoint("checkpoint") || !s.hasRecoveryVolumes(sb.ID) { + t.Fatal("older capture retired newer state or its recovery references") + } + if err := s.commit(newer); err != nil { + t.Fatal(err) + } + if got := s.referencesCheckpoint("checkpoint"); got != (domain == "volumes") { + t.Fatalf("checkpoint reference after newer commit = %t", got) + } + if got := s.hasRecoveryVolumes(sb.ID); got != (domain == "checkpoints") { + t.Fatalf("volume recovery after newer commit = %t", got) + } + }) + } +} diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index a81ff1bb..033bc8c3 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -229,7 +229,7 @@ func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { if err := m.reserveVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } - if holders := m.volumeAdmission["scratch"]; holders != (volumeHolders{writers: 1}) { + if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { t.Errorf("registry after the refusal=%+v, want the writer alone", holders) } } @@ -580,5 +580,5 @@ func newVolumeManagerAt(t *testing.T, eng *fakeEngine, dataDir string, volumes [ func volumeHoldersOf(m *Manager, name string) volumeHolders { m.mu.Lock() defer m.mu.Unlock() - return m.volumeAdmission[name] + return m.catalog.holders[name] } diff --git a/sandboxd/pool/volume_static_acl_test.go b/sandboxd/pool/volume_static_acl_test.go new file mode 100644 index 00000000..35a5af36 --- /dev/null +++ b/sandboxd/pool/volume_static_acl_test.go @@ -0,0 +1,42 @@ +package pool + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestInvalidStaticTenantListNeverBecomesPublic(t *testing.T) { + path := writeVolumeImage(t, "data", "private contents") + file := filepath.Join(t.TempDir(), "config.json") + raw := fmt.Sprintf(`{"pools":[],"volumes":[{"name":"data","path":%q,"tenants":["has space"]}]}`, path) + if err := os.WriteFile(file, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := config.Load(file) + if err != nil { + t.Fatalf("legacy config stopped booting: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), cfg.Volumes) + if got := m.Volumes("acme", nil); len(got) != 0 { + t.Fatalf("invalid restricted ACL became public: %+v", got) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: path + ".other"}); err != nil { + t.Fatalf("invalid legacy ACL blocked unrelated registration: %v", err) + } +} + +func TestRuntimeRegistrationRejectsInvalidTenantNames(t *testing.T) { + for _, tenant := range []string{"", "has space", "_leading"} { + t.Run(tenant, func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: "/images/data", Tenants: []string{tenant}}); err == nil { + t.Fatalf("runtime registration accepted invalid tenant %q", tenant) + } + }) + } +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 78b44fd6..70279f63 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -47,6 +47,8 @@ var poolErrHTTP = []struct { }{ {pool.ErrBadKey, http.StatusBadRequest, ""}, {pool.ErrBadVolume, http.StatusBadRequest, ""}, + {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, + {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrBadName, http.StatusBadRequest, ""}, {pool.ErrBadCount, http.StatusBadRequest, ""}, {pool.ErrBadEnv, http.StatusBadRequest, ""}, @@ -97,6 +99,8 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo + RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error + UnregisterVolume(ctx context.Context, name string) error GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) SetVolumes(ctx context.Context, id, tenant string, volumes []types.Volume) (types.SandboxVolumesResponse, error) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -269,6 +273,8 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) mux.HandleFunc("GET /v1/volumes", s.requireToken(s.handleVolumes)) + mux.HandleFunc("PUT /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) + mux.HandleFunc("DELETE /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 6debba66..0aab0cb7 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2482,6 +2482,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { + registerVolume func(context.Context, string, types.VolumeRegistration) error + unregisterVolume func(context.Context, string) error getVolumes func(string, string) (types.SandboxVolumesResponse, error) setVolumes func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) patchVolumes func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -2557,6 +2559,14 @@ type fakeManager struct { reader *fakeReader } +func (f *fakeManager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + return f.registerVolume(ctx, name, spec) +} + +func (f *fakeManager) UnregisterVolume(ctx context.Context, name string) error { + return f.unregisterVolume(ctx, name) +} + func (f *fakeManager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { return f.getVolumes(id, tenant) } @@ -2977,10 +2987,14 @@ func (f *fakePlacer) TemplateVolumeOwners(probe string, _ []string) []string { } return f.templateVolumeOwners } + func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } -func (f *fakePlacer) PeerAddrs() []string { return f.addrs } + +func (f *fakePlacer) PeerAddrs() []string { return f.addrs } + func (f *fakePlacer) ClientAddr(addr string) string { return addr } -func (f *fakePlacer) ConfigMismatches() int { return 0 } + +func (f *fakePlacer) ConfigMismatches() int { return 0 } type fakeProber struct { owners []string @@ -2988,4 +3002,5 @@ type fakeProber struct { } func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } -func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } + +func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go new file mode 100644 index 00000000..39bad6a6 --- /dev/null +++ b/sandboxd/server/volume_registration.go @@ -0,0 +1,44 @@ +package server + +import ( + "context" + "errors" + "net/http" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request) { + if s.apiToken == "" { + writeErr(w, http.StatusForbidden, "volume registration requires a configured api token") + return + } + name := r.PathValue("name") + if !types.ValidVolumeName(name) { + writeErr(w, http.StatusBadRequest, "invalid volume name") + return + } + var err error + op := "unregister volume" + if r.Method == http.MethodPut { + req, ok := decodeBodyStrict[types.VolumeRegistration](w, r) + if !ok { + return + } + op = "register volume" + err = s.mgr.RegisterVolume(r.Context(), name, req) + } else { + err = s.mgr.UnregisterVolume(r.Context(), name) + } + switch { + case errors.Is(err, pool.ErrVolumeCatalogUnavailable): + writePoolErr(w, err) + case errors.Is(err, context.DeadlineExceeded): + writeErr(w, http.StatusGatewayTimeout, op+" timed out") + case errors.Is(err, context.Canceled): + writeErr(w, http.StatusRequestTimeout, op+" canceled") + default: + writeResult(w, r, op, name, op+" failed", err, noContent(w)) + } +} diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go new file mode 100644 index 00000000..99ff6ac7 --- /dev/null +++ b/sandboxd/server/volume_registration_test.go @@ -0,0 +1,274 @@ +package server + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, apiToken, token string + code int + }{ + {"root", "root", "root", http.StatusNoContent}, + {"tenant", "root", "acme-token", http.StatusForbidden}, + {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, + {"unknown", "root", "unknown", http.StatusUnauthorized}, + {"missing", "root", "", http.StatusUnauthorized}, + {"open without token", "", "", http.StatusForbidden}, + {"open with tenant token", "", "acme-token", http.StatusForbidden}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + ts := newTenantTestServer(t, tt.apiToken, []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + body := `{"path":"/private/data.img"}` + if tt.code != http.StatusNoContent { + body = `{invalid` + } + resp := doReq(t, method, ts.URL+"/v1/volumes/data", tt.token, body) + defer resp.Body.Close() + got, _ := io.ReadAll(resp.Body) + if resp.StatusCode != tt.code || called != (tt.code == http.StatusNoContent) { + t.Fatalf("status=%d called=%v body=%s, want %d", resp.StatusCode, called, got, tt.code) + } + if tt.code == http.StatusNoContent && len(got) != 0 { + t.Fatalf("204 response body = %q", got) + } + }) + } + } +} + +func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { + path := filepath.Join(t.TempDir(), "data.img") + if err := os.WriteFile(path, []byte("existing backing"), 0o600); err != nil { + t.Fatal(err) + } + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, token := range []string{"", "acme-token", "unknown"} { + t.Run(method+"/"+token, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + handler := New("", "", mgr, nil, nil, nil, nil, nil).Handler() + body := bytes.NewBufferString(fmt.Sprintf(`{"path":%q,"writable":true}`, path)) + bodyLen := body.Len() + req := httptest.NewRequest(method, "/v1/volumes/data", body) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusForbidden || called || body.Len() != bodyLen { + t.Fatalf("status=%d called=%v unread=%d/%d, want 403 without reading body or calling manager", w.Code, called, body.Len(), bodyLen) + } + if w.Header().Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", w.Header()) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { + type requestKey struct{} + const body = `{"path":"/private/images/../data.img","directio":"auto","writable":true,"tenants":["beta","acme"]}` + want := types.VolumeRegistration{Path: "/private/images/../data.img", DirectIO: "auto", Writable: true, Tenants: []string{"beta", "acme"}} + var registered types.VolumeRegistration + var registeredName, removedName string + checkContext := func(ctx context.Context) { + if ctx.Value(requestKey{}) != "request-value" { + t.Error("manager did not receive the request context") + } + } + mgr := &fakeManager{ + registerVolume: func(ctx context.Context, name string, spec types.VolumeRegistration) error { + checkContext(ctx) + registeredName, registered = name, spec + return nil + }, + unregisterVolume: func(ctx context.Context, name string) error { + checkContext(ctx) + removedName = name + return nil + }, + } + handler := New("root", "", mgr, nil, nil, nil, nil, nil).Handler() + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for range 2 { + req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer root") + req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent || w.Body.Len() != 0 { + t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) + } + } + } + if registeredName != "data-v1" || removedName != "data-v1" || !reflect.DeepEqual(registered, want) { + t.Fatalf("registered %q %+v, removed %q, want data-v1 %+v", registeredName, registered, removedName, want) + } +} + +func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { + for _, tt := range []struct{ name, body string }{ + {"missing", ""}, + {"malformed", `{"path":`}, + {"array", `[]`}, + {"unknown field", `{"path":"/private/data.img","extra":true}`}, + {"body name", `{"name":"other","path":"/private/data.img"}`}, + {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, + {"trailing object", `{"path":"/private/data.img"}{}`}, + {"trailing text", `{"path":"/private/data.img"}oops`}, + {"wrong writable type", `{"path":"/private/data.img","writable":"true"}`}, + {"wrong tenants type", `{"path":"/private/data.img","tenants":"acme"}`}, + {"oversized", `{"path":"/` + strings.Repeat("a", maxBodyBytes) + `"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { + called = true + return nil + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", tt.body) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest || called { + t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) + } + }) + } +} + +func TestVolumeRegistrationRoutesValidateNames(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, encoded string + valid bool + }{ + {"data", "data", true}, + {"a_b-0", "a_b-0", true}, + {"abcdefghijklmnopqrst", "abcdefghijklmnopqrst", true}, + {"Data", "Data", false}, + {"1data", "1data", false}, + {"cocoon-data", "cocoon-data", false}, + {"abcdefghijklmnopqrstu", "abcdefghijklmnopqrstu", false}, + {"data/set", "data%2Fset", false}, + {"data set", "data%20set", false}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + var gotName string + mgr := &fakeManager{ + registerVolume: func(_ context.Context, name string, _ types.VolumeRegistration) error { gotName = name; return nil }, + unregisterVolume: func(_ context.Context, name string) error { gotName = name; return nil }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/"+tt.encoded, "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + wantCode, wantName := http.StatusBadRequest, "" + if tt.valid { + wantCode, wantName = http.StatusNoContent, tt.name + } + if resp.StatusCode != wantCode || gotName != wantName { + t.Fatalf("status=%d name=%q body=%s, want %d name=%q", resp.StatusCode, gotName, body, wantCode, wantName) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name string + err error + code int + }{ + {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, + {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, + {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, + {"uncertain cancellation", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.Canceled), http.StatusServiceUnavailable}, + {"uncertain timeout", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.DeadlineExceeded), http.StatusServiceUnavailable}, + {"timeout", fmt.Errorf("register: %w", context.DeadlineExceeded), http.StatusGatewayTimeout}, + {"canceled", fmt.Errorf("register: %w", context.Canceled), http.StatusRequestTimeout}, + {"unexpected failure", errors.New("open /private/data.img: permission denied"), http.StatusInternalServerError}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { return tt.err }, + unregisterVolume: func(context.Context, string) error { return tt.err }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/data", "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var got map[string]string + if err := json.Unmarshal(body, &got); err != nil || len(got) != 1 || got["error"] == "" { + t.Fatalf("error body = %s, decode error = %v", body, err) + } + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { + t.Fatalf("status=%d body=%s, want safe %d", resp.StatusCode, body, tt.code) + } + if tt.code == http.StatusServiceUnavailable { + seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")) + if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { + t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) + } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", resp.Header) + } + }) + } + } +} + +func TestVolumeRegistrationKeepsCatalogListingPrivate(t *testing.T) { + for _, token := range []string{"root", "acme-token"} { + t.Run(token, func(t *testing.T) { + mgr := &fakeManager{volumeCatalog: func(tenant string, _ map[string]int) []types.VolumeInfo { + wantTenant := "" + if token == "acme-token" { + wantTenant = "acme" + } + if tenant != wantTenant { + t.Errorf("listing tenant = %q, want %q", tenant, wantTenant) + } + return []types.VolumeInfo{{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1, Writable: true}} + }} + ts := newTenantTestServer(t, "root", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", token, "") + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + const want = `{"volumes":[{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1,"writable":true}]}` + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Fatalf("catalog = %d %s, want 200 %s", resp.StatusCode, body, want) + } + }) + } +} diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index ac0cd6b4..0d81b540 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -22,6 +22,14 @@ type VolumeInfo struct { Writable bool `json:"writable,omitzero"` } +// VolumeRegistration binds a backing path to a runtime catalog name. +type VolumeRegistration struct { + Path string `json:"path"` + DirectIO string `json:"directio,omitempty"` + Writable bool `json:"writable,omitzero"` + Tenants []string `json:"tenants,omitempty"` +} + // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { Add []Volume `json:"add,omitzero"` From 247227e5e9731054ea2d11d31b64ae125cb3fff0 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 12:52:41 +0000 Subject: [PATCH 02/39] test(pool): reproduce unrelated admission during catalog failure --- sandboxd/pool/volume_catalog_test.go | 62 ++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 002f7c5e..f44480b3 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -205,6 +205,68 @@ func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { } } +func TestRuntimeCatalogWriteFailureOnlyFencesAffectedName(t *testing.T) { + for _, unregister := range []bool{false, true} { + t.Run(fmt.Sprintf("unregister=%t", unregister), func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + for _, name := range []string{"healthy", "affected"} { + if name == "affected" && !unregister { + continue + } + if err := m.RegisterVolume(t.Context(), name, types.VolumeRegistration{Path: writeVolumeImage(t, name, "bytes")}); err != nil { + t.Fatal(err) + } + } + sb := mustClaim(t, m, testKey) + var before []resolvedVolume + if unregister { + var err error + before, err = m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "healthy"}, {Name: "affected"}}) + if err != nil { + t.Fatal(err) + } + } + restore := blockCatalogRename(t, m) + defer restore() + var err error + if unregister { + err = m.UnregisterVolume(t.Context(), "affected") + } else { + err = m.RegisterVolume(t.Context(), "affected", types.VolumeRegistration{Path: writeVolumeImage(t, "affected", "bytes")}) + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("obstructed catalog write: %v", err) + } + if unregister { + if _, err := m.admitVolumes("", before); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("resolved mixed admission: %v", err) + } + } + for _, volumes := range [][]types.Volume{{{Name: "affected"}}, {{Name: "healthy"}, {Name: "affected"}}} { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("affected claim: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("affected live addition: %v", err) + } + } + for _, name := range []string{"healthy", "affected"} { + if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { + t.Fatalf("refused request reserved %s: %+v", name, got) + } + } + for _, name := range []string{"data", "healthy"} { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: name}}}); err != nil { + t.Errorf("unrelated %s claim: %v", name, err) + } + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}, {Name: "healthy"}}); err != nil { + t.Errorf("unrelated live addition: %v", err) + } + }) + } +} + func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, nil) From debf72910742108d0e6a546e2cae95bc7fb10598 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:03:57 +0000 Subject: [PATCH 03/39] fix(pool): scope catalog uncertainty to affected volumes --- docs/sandboxd-api.md | 133 +++++++----------- sandboxd/engine/cocoond_test.go | 28 ++++ sandboxd/pool/archive_test.go | 3 +- sandboxd/pool/checkpoint_bench_test.go | 4 +- sandboxd/pool/claims.go | 19 +-- sandboxd/pool/claims_bench_test.go | 6 +- sandboxd/pool/claims_test.go | 2 +- sandboxd/pool/failover_test.go | 9 +- sandboxd/pool/hibernate_test.go | 12 +- sandboxd/pool/liveness_test.go | 3 +- sandboxd/pool/pool_test.go | 5 +- sandboxd/pool/reconcile.go | 25 ++-- sandboxd/pool/remove.go | 4 + sandboxd/pool/volume.go | 60 ++++----- sandboxd/pool/volume_bench_test.go | 2 +- sandboxd/pool/volume_catalog.go | 50 +++---- sandboxd/pool/volume_catalog_test.go | 2 +- sandboxd/pool/volume_cleanup_test.go | 164 +++++++++-------------- sandboxd/pool/volume_convergence_test.go | 9 +- sandboxd/pool/volume_mutation_test.go | 10 +- sandboxd/pool/volume_rw_test.go | 8 +- sandboxd/types/types.go | 1 + 22 files changed, 263 insertions(+), 296 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index c103b4c8..18b0238f 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -257,13 +257,10 @@ entry's response is byte-identical to v1. ## PUT /v1/volumes/{name} -Auth: the configured root node API token. With no configured `api_token`, -registration and unregistration return 403. A tenant token returns 403. -Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. - -Registers a backing path on this node. The URL supplies the name; -the body has no `name` field. Unknown fields, duplicate fields, and trailing -JSON return 400. +Requires the configured root API token. A tenant token or a node without an +`api_token` returns 403. Sandbox, unknown, and missing tokens return 401 when +`api_token` is configured. The URL supplies the name. Unknown body fields, +duplicate fields, and trailing JSON return 400. ```json {"path": "/srv/datasets/data.img", "directio": "off", @@ -271,86 +268,60 @@ JSON return 400. ``` - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. -- `path` uses the same absolute-path validation as static volume config. - Registration does not open the backing or check its type or availability. - A missing backing retains its registration but cannot be attached until available. - Registration success does not establish that Cocoon supports the backing type. - Operators must keep the registered path and target stable. - Backing storage must be separate from daemon cleanup directories, including +- `path` uses static config's absolute-path validation. Registration does not + open the backing or check its type or availability. A missing backing keeps + its registration but cannot be attached. Operators must keep the path and + target stable and separate from daemon cleanup directories, including `/readers`, `/goldens`, and `/fork-*`. - Backing paths must not resolve into cleanup directories. - Cleanup directories must not resolve into backing storage. - Startup and staging cleanup do not consult volume registrations and can - delete backing files placed in those locations. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. -- `tenants` grants the named tenants self-service access. Omitted or empty - permits every authenticated scope, for both static and runtime entries. - Names use the tenant API name syntax. Runtime registration rejects invalid - names. Static config warns and skips invalid names. A nonempty static list with no - valid names remains restricted and grants no tenant access. A valid name may be - registered before that tenant exists; it grants no access until the tenant - is created. - -Success is 204 with an empty body. Repeating the same name, cleaned path, and -normalized registration fields succeeds. A different binding or different -registration fields return 409. Static and runtime registrations cannot share -a name or a cleaned backing path. Invalid registration fields return 400. -An uncertain write returns 503 with `Retry-After`. The requested registration -may still take effect through background retry or a later PUT or DELETE, -including a request for another name. A 503 can also reject a request before -acceptance, so it does not promise that the request will take effect. After -repairing storage failures, retry the exact request to establish completion. -Cancellation is not a rollback once a write has been accepted. -GET retains its existing projection without host paths or tenant access lists. -Registered entries support the existing mounted read-only live volume API. - -Only registration metadata is persisted locally in -`data_dir/runtime-volumes.json`, which loads before claim recovery. New records -use mode 0600. The daemon reads the record with the same filesystem permissions -as `claims.json` and replaces it durably. Registration creates, grows, and deletes -no disks. -Malformed or conflicting metadata prevents startup. A temporarily missing -backing file keeps its registration and refuses new attachment. -Catalog replacement and synchronization are independent of `sync_claims`, which -controls claim snapshot synchronization. Process-restart checks do not prove -power-loss recovery. The catalog is node-local with one sandboxd writer. -Startup reads only `runtime-volumes.json`. Crash leftovers named -`runtime-volumes.json.*.tmp` are neither loaded nor promoted. An operator can -remove those leftovers after stopping sandboxd and preserving any needed -evidence. +- `tenants` grants named tenants self-service access. Omitted or empty permits + every authenticated scope for both static and runtime entries. Runtime + registration rejects invalid tenant names. Static config skips invalid names + and grants no tenant access if a nonempty list has no valid names. + A tenant name can be registered before that tenant exists. + +Success is 204. An identical normalized registration is idempotent. A different +binding, changed fields, or a static name or cleaned path collision returns 409. +Invalid fields return 400. GET keeps its existing projection without host paths +or tenant lists. Runtime entries support the existing live volume API. + +Persistence uncertainty returns 503 with `Retry-After`. An accepted change may +complete through background retry or a later PUT or DELETE for any name. +A 503 can also precede acceptance. Retry the exact request after repairing +storage to establish completion. Cancellation does not undo an accepted write. +Until persistence succeeds, new admission refuses the affected name while +unrelated published names remain usable. Every catalog writer retries the +accepted change before accepting another change. + +The node stores metadata in `data_dir/runtime-volumes.json` independently of +`sync_claims` and loads it before claim recovery. New records use mode 0600. Malformed +or conflicting metadata prevents startup. Temporary replacement files are not +loaded. Registration creates, grows, and deletes no backing disks. The catalog +is node-local with one sandboxd writer. The controller coordinates cross-node +ownership, symlink and hardlink aliases, backing availability, and storage +placement outside cleanup directories. ## DELETE /v1/volumes/{name} -Auth and name validation match PUT. Removes only the runtime registration and -never deletes or changes the backing file. Success, including an absent, -unreferenced runtime name, is 204 with an empty body. Static registrations and -names referenced by a claim, an in-flight operation, cleanup, or an -unacknowledged recovery snapshot return 409. Catalog persistence uncertainty -returns 503 with `Retry-After` and has the same asynchronous completion -semantics as PUT. A failed DELETE may therefore remove the registration later. -An empty GET volume view alone does not establish that a recovery reference -has retired. - -Downgrading to a daemon without runtime registration requires draining runtime -references or supplying equivalent static volume config. Pending live volume -changes must finish before downgrade. - -Stale VM cleanup inspects attachments before removal. A complete running -inventory retains holds only on the named volumes. An explicit empty disk -array permits unrelated volume admission and catalog changes while removal -retries continue. An omitted or null disk list, a stopped VM, an inspection -failure, or an active creating owner retains the node-wide fence until -inspection or confirmed removal resolves it. Cocoon v0.7.3 omits both empty -inventories and failed disk queries, and does not collect disks for stopped -VMs. Those results remain unknown and can block static read-only admission. -Allowing readers would conflict with an unknown writer. Cleanup retries -remain enabled. -A warm VM name is not evidence that no volume is attached; attachment can -succeed before claim finalization fails. Known inventories keep holds on their -reported names. Volume-free claims continue to work. Path uniqueness uses -cleaned path strings. The operator coordinates symlink and hardlink aliases -and cross-node ownership. +Auth and name validation match PUT. Deletes only runtime registration metadata. +Success, including an absent unreferenced name, is 204. Static names and names +referenced by a claim, an in-flight operation, cleanup, or an unacknowledged +recovery snapshot return 409. Persistence uncertainty returns 503 with +`Retry-After` and the same retry semantics as PUT. A failed DELETE can complete +later. An empty GET view does not establish that recovery references retired. + +Stale VM cleanup retains named holds after a complete running inventory. +An ordinary stopped VM needs no volume holds. An inferred stopped result with +`stale: true`, inspection errors, omitted or null running disk lists, and busy +creating owners retain a node-wide admission fence. Any present inventory +during busy cleanup retains that fence. Confirmed absence or removal clears it. +This assumes sandboxd controls starts and attachments in its VM namespace. +Volume-free claims remain available. Cleanup retries continue. + +Before downgrading to a daemon without runtime registration, drain runtime +references or supply equivalent static volume config and finish pending live +volume changes. ## GET /v1/sandboxes/{id}/volumes diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 053dc205..398241be 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -45,6 +45,8 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { fakeCocoon(t, "#!/bin/sh\n[ \"$1 $2\" = \"vm inspect\" ] || exit 2\n"+ "if [ \"$3\" = sbx-1 ]; then echo '{\"config\":{\"name\":\"sbx-1\"},\"state\":\"running\","+ "\"attached_devices\":{\"disks\":[{\"id\":\"disk-a\",\"name\":\"imagenet\",\"path\":\"/v/a.img\",\"readonly\":true},{\"id\":\"disk-b\",\"name\":\"scratch\",\"path\":\"/v/b.img\"}]}}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stale ]; then echo '{\"state\":\"stopped\",\"stale\":true}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stopped ]; then echo '{\"state\":\"stopped\"}'; exit 0; fi\n"+ "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") @@ -52,11 +54,37 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } + for _, name := range []string{"sbx-stale", "sbx-stopped"} { + rec, present, err := e.Inspect(t.Context(), name) + if err != nil || !present || rec.State != "stopped" || rec.Stale != (name == "sbx-stale") { + t.Errorf("Inspect %s lost stopped classification: %+v, %v, %v", name, rec, present, err) + } + } if _, ok, err := e.Inspect(t.Context(), "sbx-gone"); err != nil || ok { t.Errorf("Inspect of a missing VM = %v, %v; want not found without an error", ok, err) } } +func TestInspectPreservesMissingDiskInventory(t *testing.T) { + for _, tc := range []struct { + name, inventory string + unknown bool + }{ + {"omitted devices", "", true}, + {"omitted disks", `,"attached_devices":{}`, true}, + {"null disks", `,"attached_devices":{"disks":null}`, true}, + {"empty disks", `,"attached_devices":{"disks":[]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + fakeCocoon(t, "#!/bin/sh\necho '{\"state\":\"running\""+tc.inventory+"}'\n") + rec, present, err := New("cocoon", nil, nil, false, false, "").Inspect(t.Context(), "sbx-1") + if err != nil || !present || (rec.AttachedDevices.Disks == nil) != tc.unknown { + t.Fatalf("inventory classification: %+v, %v, %v", rec, present, err) + } + }) + } +} + func serveCocoond(t *testing.T, socket, stream string) { t.Helper() ln, err := net.Listen("unix", socket) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 59064b8b..5cd3aa11 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -155,7 +155,8 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_mine": {ID: "sb_mine", VMName: "sbx-gone-1", Key: testKey, Token: "tok"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) diff --git a/sandboxd/pool/checkpoint_bench_test.go b/sandboxd/pool/checkpoint_bench_test.go index e3ba1b6c..997f0e77 100644 --- a/sandboxd/pool/checkpoint_bench_test.go +++ b/sandboxd/pool/checkpoint_bench_test.go @@ -15,7 +15,7 @@ func BenchmarkClaimCommitWithCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { claims := checkpointBenchClaims(checkpoints) s := newClaimStore(b.TempDir(), false) - if err := s.save(claims); err != nil { + if err := s.commit(s.reset(claims)); err != nil { b.Fatal(err) } one := claims["sb_0000000000000000"] @@ -39,7 +39,7 @@ func BenchmarkPinnedArchiveCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.save(m.claimed); err != nil { + if err := m.store.commit(m.store.reset(m.claimed)); err != nil { b.Fatal(err) } want := 0 diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 56f2bfe5..c8876a73 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -72,8 +72,6 @@ type claimRow struct { raw []byte } -type recoveryVolumeCount struct{ readers, writers int } - // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string @@ -93,7 +91,7 @@ type claimStore struct { checkpoints map[string]int currentVolumes map[string][]types.Volume retainedVolumes map[string][]types.Volume - volumes map[string]recoveryVolumeCount + volumes map[string]volumeHolders recommitting atomic.Bool recoveryVolumesPresent atomic.Bool @@ -104,7 +102,7 @@ func newClaimStore(dataDir string, sync bool) *claimStore { path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, refs: map[string][4]string{}, retained: map[string][]string{}, changed: map[string]struct{}{}, checkpoints: map[string]int{}, - currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]recoveryVolumeCount{}, + currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]volumeHolders{}, } } @@ -235,14 +233,14 @@ func (s *claimStore) commit(snap claimSnapshot) error { var capturedVolumes map[string][]types.Volume for id := range s.changed { captured[id] = s.refs[id] - s.changeRetainedRefs(id, appendRefs(slices.Clone(s.retained[id]), s.refs[id])) + s.changeRetainedRefs(id, appendRefs(s.retained[id], s.refs[id])) if volumes := s.currentVolumes[id]; len(volumes) != 0 { if capturedVolumes == nil { capturedVolumes = make(map[string][]types.Volume) } capturedVolumes[id] = volumes } - s.changeRetainedVolumes(id, appendRecoveryRefs(slices.Clone(s.retainedVolumes[id]), s.currentVolumes[id])) + s.changeRetainedVolumes(id, appendRecoveryRefs(s.retainedVolumes[id], s.currentVolumes[id])) } s.mu.Unlock() s.buf = append(s.buf[:0], '{') @@ -321,11 +319,6 @@ func (s *claimStore) replace(raw []byte) error { return nil } -// save is the combined form for the startup Reconcile pass, before contention exists. -func (s *claimStore) save(claims map[string]*types.Sandbox) error { - return s.commit(s.reset(claims)) -} - func (s *claimStore) synced() bool { s.mu.Lock() defer s.mu.Unlock() @@ -378,7 +371,7 @@ func (s *claimStore) changeVolumeCounts(refs []types.Volume, delta int) { } else { count.readers += delta } - if count == (recoveryVolumeCount{}) { + if count == (volumeHolders{}) { delete(s.volumes, v.Name) } else { s.volumes[v.Name] = count @@ -400,7 +393,7 @@ func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { func (s *claimStore) referencesVolume(name string) bool { s.mu.Lock() defer s.mu.Unlock() - return s.volumes[name] != (recoveryVolumeCount{}) + return s.volumes[name] != (volumeHolders{}) } func (s *claimStore) volumeConflicts(volume types.Volume, ownerID string) bool { diff --git a/sandboxd/pool/claims_bench_test.go b/sandboxd/pool/claims_bench_test.go index b1ae79da..d1f059bc 100644 --- a/sandboxd/pool/claims_bench_test.go +++ b/sandboxd/pool/claims_bench_test.go @@ -18,7 +18,7 @@ func BenchmarkStoreSaveScaling(b *testing.B) { st := newClaimStore(b.TempDir(), false) claims := benchClaims(n) for b.Loop() { - if err := st.save(claims); err != nil { + if err := st.commit(st.reset(claims)); err != nil { b.Fatalf("save: %v", err) } } @@ -48,7 +48,7 @@ func BenchmarkStoreCommitEnv(b *testing.B) { sb.Env = envs[arm] } s := newClaimStore(b.TempDir(), durable) - if err := s.save(claims); err != nil { + if err := s.commit(s.reset(claims)); err != nil { b.Fatalf("save: %v", err) } one := claims[fmt.Sprintf("sb_%016x", 0)] @@ -105,7 +105,7 @@ func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, arm switch arm { case "under-lock": mu.Lock() - _ = s.save(claims) + _ = s.commit(s.reset(claims)) mu.Unlock() case "rebuild": mu.Lock() diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index e12ea0e7..d369f829 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -85,7 +85,7 @@ func TestStoreRoundTrip(t *testing.T) { } for _, fsync := range []bool{false, true} { s := newClaimStore(t.TempDir(), fsync) - if err := s.save(claims); err != nil { + if err := s.commit(s.reset(claims)); err != nil { t.Fatalf("save (sync=%v): %v", fsync, err) } loadedStore := newClaimStore(filepath.Dir(s.path), false) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 505dc55c..dd37a6bc 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -272,7 +272,8 @@ func TestFailoverRestartReclaimsAnInterruptedArchive(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - if err := newClaimStore(dirA, false).save(claims); err != nil { + claimStore := newClaimStore(dirA, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -295,7 +296,8 @@ func TestFailoverRestartReclaimSparesACheckpointAdoptedMidReconcile(t *testing.T claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - if err := newClaimStore(dirA, false).save(claims); err != nil { + claimStore := newClaimStore(dirA, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -320,7 +322,8 @@ func TestFailoverRestartReclaimsOnlyTheCheckpointsItPublished(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - if err := newClaimStore(dirA, false).save(claims); err != nil { + claimStore := newClaimStore(dirA, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index ffc4dd9e..7642b11a 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -528,7 +528,8 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { VsockSocket: "/vsock/lagged", PendingSnap: hibernatePrefix + "lagged-1-abc123", }, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -557,7 +558,8 @@ func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_orp": {ID: "sb_orp", VMName: "sbx-orphan-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/orphan"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -583,7 +585,8 @@ func TestReconcileKeepsIntentWithoutImageAsRunning(t *testing.T) { VsockSocket: "/vsock/noimg", PendingSnap: hibernatePrefix + "noimg-1-abc123", }, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -613,7 +616,8 @@ func TestReconcileAdoptsHibernated(t *testing.T) { HibernateSnap: hibernatePrefix + "hibernated-1", }, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 2864c615..ac557fb2 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -486,7 +486,8 @@ func TestReconcileKeepsRunningClaimsTheHostLost(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_lost": {ID: "sb_lost", VMName: "sbx-lost-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/lost"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } cfg := &config.Config{DataDir: dataDir, VMMRestart: tt.policy} diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index d9fc8b46..b66724ce 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -323,7 +323,8 @@ func TestReconcile(t *testing.T) { "sb_live": {ID: "sb_live", VMName: "sbx-live-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/live-old"}, "sb_dead": {ID: "sb_dead", VMName: "sbx-dead-1", Key: testKey, Token: "tok"}, } - if err := newClaimStore(dataDir, false).save(claims); err != nil { + claimStore := newClaimStore(dataDir, false) + if err := claimStore.commit(claimStore.reset(claims)); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -1256,6 +1257,7 @@ type fakeEngine struct { liveVolumeHook func(string) error stopped map[string]bool inspectHook func() error + inspectStale bool inspectDisksUnknown bool trackDisks bool disks map[string][]types.VMDisk @@ -1542,6 +1544,7 @@ func (f *fakeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, return types.VMRecord{}, false, nil } f.mu.Lock() + vms[i].Stale = f.inspectStale if !f.inspectDisksUnknown { vms[i].AttachedDevices.Disks = f.disks[name] } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 056fdf81..9e103613 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -195,17 +195,26 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) return td } - if present && (inspected.State != vmStateRunning || inspected.AttachedDevices.Disks == nil) { - return td - } var holds []types.Volume if present { - holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW + switch inspected.State { + case "stopped": + if inspected.Stale { + return td + } + case vmStateRunning: + if inspected.AttachedDevices.Disks == nil { + return td } + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW + } + } + default: + return td } } m.mu.Lock() diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 924e3f5d..e901e0a5 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -92,6 +92,10 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: + if _, present, inspectErr := m.eng.Inspect(ctx, name); inspectErr == nil && !present { + m.finishRemoval(ctx, name, pending) + return + } m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 0660e7c5..49069b3c 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -147,9 +147,11 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } m.mu.Lock() - if err := m.catalog.admissionError(); err != nil { - m.mu.Unlock() - return nil, err + for _, volume := range applied { + if err := m.catalog.admissionError(volume.Name); err != nil { + m.mu.Unlock() + return nil, err + } } catalog := m.catalog.entries m.mu.Unlock() @@ -308,35 +310,6 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) b return true } -// reserveVolumes admits one claim's volumes atomically; callers hold m.mu. -func (m *Manager) reserveVolumes(volumes []types.Volume) error { - return m.reserveVolumesFor(volumes, nil) -} - -func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { - if len(volumes) == 0 { - return nil - } - if err := m.catalog.admissionError(); err != nil { - return err - } - ownerID := "" - if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { - ownerID = owner.ID - } - for _, volume := range volumes { - if m.store.volumeConflicts(volume, ownerID) { - return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) - } - holders := m.catalog.holders[volume.Name] - if holders.writers > 0 || (volume.RW() && holders.readers > 0) { - return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) - } - } - m.adoptVolumes(volumes) - return nil -} - // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { @@ -382,13 +355,30 @@ func (m *Manager) catalogSnapshot() map[string]*catalogVolume { return m.catalog.entries } +// reserveResolved admits the entire request while callers hold m.mu. func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox) error { - for _, volume := range volumes { - if volume.entry == nil || m.catalog.entries[volume.applied.Name] != volume.entry { + ownerID := "" + if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { + ownerID = owner.ID + } + for _, resolved := range volumes { + volume := resolved.applied + if err := m.catalog.admissionError(volume.Name); err != nil { + return err + } + if m.catalog.entries[volume.Name] != resolved.entry { return ErrVolumeUnavailable } + if m.store.volumeConflicts(volume, ownerID) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } + holders := m.catalog.holders[volume.Name] + if holders.writers > 0 || (volume.RW() && holders.readers > 0) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } } - return m.reserveVolumesFor(appliedVolumes(volumes), owner) + m.adoptVolumes(appliedVolumes(volumes)) + return nil } // quiesceBudget bounds one quiesce: a slice per writable mount plus one for the sync fallback. diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go index 4016f920..22242076 100644 --- a/sandboxd/pool/volume_bench_test.go +++ b/sandboxd/pool/volume_bench_test.go @@ -67,7 +67,7 @@ func volumeBenchManager(b *testing.B, checkpoints bool) *Manager { b.Helper() m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.save(m.claimed); err != nil { + if err := m.store.commit(m.store.reset(m.claimed)); err != nil { b.Fatal(err) } path := filepath.Join(b.TempDir(), "data.raw") diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index f2ccc456..4cd0c302 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -28,6 +28,7 @@ type runtimeCatalog struct { } type catalogWrite struct { + name string entries map[string]*catalogVolume raw []byte } @@ -40,8 +41,8 @@ type volumeCatalog struct { writeMu sync.Mutex } -func (c *volumeCatalog) admissionError() error { - if c.pending != nil || c.unknown > 0 { +func (c *volumeCatalog) admissionError(name string) error { + if c.unknown > 0 || c.pending != nil && c.pending.name == name { return ErrVolumeCatalogUnavailable } return nil @@ -60,7 +61,7 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo } m.mu.Lock() entries := m.catalog.entries - err = m.catalog.admissionError() + err = m.catalog.admissionError(name) m.mu.Unlock() if err != nil { return err @@ -71,6 +72,9 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo } return fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) } + if err := catalogPathConflict(entries, normalized.Path); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) + } entries = maps.Clone(entries) entries[name] = &catalogVolume{ runtime: true, @@ -78,9 +82,6 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo tenants: normalized.Tenants, writable: normalized.Writable, } - if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { - return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) - } return m.writeCatalog(ctx, name, entries) } @@ -97,7 +98,7 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { m.mu.Lock() entries := m.catalog.entries entry := entries[name] - err := m.catalog.admissionError() + err := m.catalog.admissionError(name) if err == nil && ((entry != nil && !entry.runtime) || (entry == nil && m.volumeReferenced(name))) { err = fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) } @@ -134,6 +135,9 @@ func (m *Manager) loadRuntimeVolumes() error { if m.catalog.entries[name] != nil { return fmt.Errorf("runtime volume catalog duplicates static name %q", name) } + if err := catalogPathConflict(entries, normalized.Path); err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } entries[name] = &catalogVolume{ runtime: true, disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, @@ -141,9 +145,6 @@ func (m *Manager) loadRuntimeVolumes() error { writable: normalized.Writable, } } - if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { - return fmt.Errorf("runtime volume catalog: %w", err) - } m.catalog.entries = entries return nil } @@ -152,11 +153,6 @@ func (m *Manager) volumeReferenced(name string) bool { if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { return true } - for _, run := range m.volumeRuns { - if volumeIndex(run.holds, name) >= 0 { - return true - } - } for _, sb := range m.claimed { if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { return true @@ -176,12 +172,9 @@ func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[str if err != nil { return fmt.Errorf("%w: %w", ErrBadVolume, err) } - proposal := &catalogWrite{entries: entries, raw: raw} + proposal := &catalogWrite{name: name, entries: entries, raw: raw} m.mu.Lock() err = ctx.Err() - if err == nil { - err = m.catalog.admissionError() - } if err == nil && m.volumeReferenced(name) { err = fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) } @@ -197,14 +190,11 @@ func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[str func (m *Manager) commitPendingCatalog(ctx context.Context) error { m.mu.Lock() - proposal, unknown := m.catalog.pending, m.catalog.unknown > 0 + proposal := m.catalog.pending m.mu.Unlock() if proposal == nil { return nil } - if unknown { - return ErrVolumeCatalogUnavailable - } if err := ctx.Err(); err != nil { return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) } @@ -232,19 +222,13 @@ func (m *Manager) retryVolumeCatalog(ctx context.Context) { } } -func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { - volumes := make([]config.VolumeSpec, 0, len(entries)) +func catalogPathConflict(entries map[string]*catalogVolume, path string) error { for name, entry := range entries { - var tenants []string - if entry.runtime { - tenants = entry.tenants + if filepath.Clean(entry.disk.Path) == path { + return fmt.Errorf("backing path already registered as %q", name) } - volumes = append(volumes, config.VolumeSpec{ - Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, - Writable: entry.writable, Tenants: tenants, - }) } - return volumes + return nil } func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index f44480b3..197405a9 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -508,7 +508,7 @@ func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) t.Fatalf("failed retry cleared pending write: %v", err) } m.retryVolumeCatalog(t.Context()) - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("pending replacement lost fence: %v", err) } restore() diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index c2ff00ae..a0f50798 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -3,9 +3,6 @@ package pool import ( "context" "errors" - "fmt" - "os" - "path/filepath" "slices" "sync/atomic" "testing" @@ -17,71 +14,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { - for _, state := range []string{vmStateRunning, vmStateCreating, "stopped"} { - t.Run(state, func(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, nil) - path := writeVolumeImage(t, "incomplete-inventory", "original") - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { - t.Fatal(err) - } - name := "sbx-incomplete-inventory" - base.vms[name] = "/vsock/incomplete" - base.creating[name] = state == vmStateCreating - base.staleOutcome = engine.StaleCreateBusy - base.removeErrFor = name - base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} - cli := filepath.Join(t.TempDir(), "inspect") - writeInspect := func(state, inventory string) { - t.Helper() - raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"%s\"%s}'\n", name, state, inventory) - if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { - t.Fatal(err) - } - } - writeInspect(state, "") - m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} - rec := base.record(name) - rec.State = state - if m.removeStaleVM(t.Context(), name, rec) { - t.Fatal("stale VM unexpectedly disappeared") - } - for range 2 { - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("unregister must refuse incomplete inventory: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("conflicting writer must refuse incomplete inventory: %v", err) - } - m.retryRemovals(t.Context()).Wait() - } - base.creating[name] = false - writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) - if state == vmStateCreating { - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("busy owner must retain the unknown fence: %v", err) - } - } - base.staleOutcome = engine.StaleCreateNotCreating - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("complete inventory lost the surviving writer: %v", err) - } - other := writeVolumeImage(t, "complete-inventory-other", "other") - if err := m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: other}); err != nil { - t.Fatalf("complete inventory retained the global fence: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if err := m.UnregisterVolume(t.Context(), "data"); err != nil { - t.Fatalf("confirmed removal retained volume protection: %v", err) - } - }) - } -} - func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, nil) @@ -178,8 +110,18 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { } func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { - for _, state := range []string{"running", "stopped", "creating"} { - t.Run(state, func(t *testing.T) { + for _, tc := range []struct { + name, state string + inspectError bool + }{ + {"running omitted", "running", false}, + {"running error", "running", true}, + {"inferred stopped omitted", "stopped", false}, + {"inferred stopped error", "stopped", true}, + {"creating omitted", "creating", false}, + {"creating error", "creating", true}, + } { + t.Run(tc.name, func(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, nil) path := writeVolumeImage(t, "runtime", "bytes") @@ -188,20 +130,24 @@ func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { } name := "sbx-stale-unknown" base.vms[name] = "/vsock/stale" - base.stopped[name] = state == "stopped" - base.creating[name] = state == "creating" + base.stopped[name] = tc.state == "stopped" + base.creating[name] = tc.state == "creating" + base.inspectStale = tc.state == "stopped" + base.inspectDisksUnknown = !tc.inspectError base.staleOutcome = engine.StaleCreateBusy base.removeErrFor = name base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} var inspectFailed atomic.Bool inspectFailed.Store(true) base.inspectHook = func() error { - if inspectFailed.Load() { + if tc.inspectError && inspectFailed.Load() { return errors.New("inspection unavailable") } return nil } - if m.removeStaleVM(t.Context(), name, base.record(name)) { + rec := base.record(name) + rec.State = tc.state + if m.removeStaleVM(t.Context(), name, rec) { t.Fatal("unknown stale VM unexpectedly gone") } for range 2 { @@ -228,6 +174,13 @@ func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { } inspectFailed.Store(false) base.stopped[name], base.creating[name] = false, false + base.inspectStale, base.inspectDisksUnknown = false, false + if tc.state == "creating" { + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy partial inventory cleared unknown: %v", err) + } + } base.staleOutcome = engine.StaleCreateNotCreating m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { @@ -439,27 +392,25 @@ func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) } } -func TestStaleInventoryDistinguishesOmittedAndExplicitEmptyDisks(t *testing.T) { +func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { for _, tc := range []struct { - name, inventory string - wantErr error + name string + stopped, stale bool + disks []types.VMDisk + wantErr error }{ - {"omitted devices", "", ErrVolumeCatalogUnavailable}, - {"omitted disks", `,"attached_devices":{}`, ErrVolumeCatalogUnavailable}, - {"null disks", `,"attached_devices":{"disks":null}`, ErrVolumeCatalogUnavailable}, - {"explicit empty disks", `,"attached_devices":{"disks":[]}`, nil}, + {"running unknown", false, false, nil, ErrVolumeCatalogUnavailable}, + {"running empty", false, false, []types.VMDisk{}, nil}, + {"ordinary stopped", true, false, nil, nil}, + {"inferred stopped", true, true, nil, ErrVolumeCatalogUnavailable}, } { t.Run(tc.name, func(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) name := "sbx-inventory-shape" base.vms[name], base.removeErrFor = "/vsock/stale", name - cli := filepath.Join(t.TempDir(), "inspect") - raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"running\"%s}'\n", name, tc.inventory) - if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { - t.Fatal(err) - } - m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + base.disks = map[string][]types.VMDisk{name: tc.disks} + base.stopped[name], base.inspectStale = tc.stopped, tc.stale if m.removeStaleVM(t.Context(), name, base.record(name)) { t.Fatal("stale VM unexpectedly disappeared") } @@ -467,6 +418,12 @@ func TestStaleInventoryDistinguishesOmittedAndExplicitEmptyDisks(t *testing.T) { if !errors.Is(err, tc.wantErr) { t.Fatalf("admission=%v, want=%v", err, tc.wantErr) } + if _, queued := m.pendingRemovals[name]; !queued { + t.Fatal("surviving VM lost pending removal") + } + if err := m.RegisterVolume(t.Context(), "before-removal", types.VolumeRegistration{Path: "/images/before-removal"}); !errors.Is(err, tc.wantErr) { + t.Fatalf("registration before removal: %v", err) + } base.removeErrFor = "" m.retryRemovals(t.Context()).Wait() if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { @@ -504,40 +461,43 @@ func TestStaleInventoryNarrowsFenceBeforeBlockedRemoval(t *testing.T) { } func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { - for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound} { + for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { t.Run(string(outcome), func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) name := "sbx-busy-create" eng.vms[name], eng.creating[name] = "/vsock/creating", true eng.staleOutcome = engine.StaleCreateBusy - eng.inspectHook = func() error { - t.Error("busy owner must finish before inventory can be trusted") - return errors.New("busy") - } + eng.disks = map[string][]types.VMDisk{name: {{Name: "other"}}} rec := eng.record(name) rec.State = vmStateCreating if m.removeStaleVM(t.Context(), name, rec) { t.Fatal("busy create reported gone") } - m.retryRemovals(t.Context()).Wait() + eng.creating[name] = false + for _, stopped := range []bool{false, true} { + eng.stopped[name] = stopped + before := eng.inspects + m.retryRemovals(t.Context()).Wait() + if eng.inspects != before+1 { + t.Fatal("busy retry did not inspect") + } + } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("busy create admitted a volume: %v", err) } eng.staleOutcome = outcome + if outcome == engine.StaleCreateBusy { + delete(eng.vms, name) + } + m.retryRemovals(t.Context()).Wait() m.retryRemovals(t.Context()).Wait() + if m.catalog.unknown != 0 || len(m.pendingRemovals) != 0 { + t.Fatalf("cleanup retained or double-released its fence: unknown=%d pending=%d", m.catalog.unknown, len(m.pendingRemovals)) + } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { t.Fatalf("confirmed cleanup retained fence: %v", err) } }) } } - -type incompleteInventoryEngine struct { - *fakeEngine - cli *engine.Engine -} - -func (e *incompleteInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { - return e.cli.Inspect(ctx, name) -} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index ef8fb774..4446957a 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -343,12 +343,19 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { m.store.set(sb) m.mu.Unlock() case "admitted": + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "data", "bytes")}); err != nil { + t.Fatal(err) + } _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { t.Fatal(err) } + resolved, err := m.resolveVolumes(t.Context(), "", held) + if err != nil { + t.Fatal(err) + } m.mu.Lock() - err = m.reserveVolumes(held) + err = m.reserveResolved(resolved, nil) run.holds = held m.mu.Unlock() if err != nil { diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index d9de1c70..66ebfd4e 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -403,7 +403,7 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { if got := claims["sb_1"].VolumesWant; !slices.Equal(got, tt.want) { t.Fatalf("loaded target %v, want %v", got, tt.want) } - if err = store.save(claims); err != nil { + if err = store.commit(store.reset(claims)); err != nil { t.Fatal(err) } rawBytes, err := os.ReadFile(store.path) @@ -854,10 +854,14 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume if err != nil { t.Fatal(err) } + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} + resolved, err := m.resolveVolumes(t.Context(), "", transient) + if err != nil { + t.Fatal(err) + } m.mu.Lock() defer m.mu.Unlock() - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} - if err = m.reserveVolumes(transient); err != nil { + if err = m.reserveResolved(resolved, nil); err != nil { t.Fatal(err) } run.holds = append(run.holds, transient...) diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 033bc8c3..dadeda8c 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -153,7 +153,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { t.Fatalf("mark dirty: %v", markErr) } m.mu.Lock() - reserveErr := m.reserveVolumes(appliedVolumes(resolved)) + reserveErr := m.reserveResolved(resolved, nil) m.mu.Unlock() if reserveErr != nil { t.Fatalf("reserve once the writer released: %v", reserveErr) @@ -222,11 +222,15 @@ func TestVolumeAdmissionMatrix(t *testing.T) { func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") m := newVolumeManager(t, newFakeEngine(), []config.VolumeSpec{{Name: "scratch", Path: path, Writable: true}}) + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}) + if err != nil { + t.Fatal(err) + } m.mu.Lock() defer m.mu.Unlock() m.adoptVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch", Mode: types.VolumeModeRW}}) - if err := m.reserveVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}); !errors.Is(err, ErrVolumeBusy) { + if err := m.reserveResolved(resolved, nil); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..2a93ab78 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -731,6 +731,7 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { + Stale bool `json:"stale,omitzero"` State string `json:"state"` PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` From af6ee9a39fe1e1c2b993afa470cc1c992ea7d839 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:31:48 +0000 Subject: [PATCH 04/39] pool: reproduce private name disclosure during catalog writes --- sandboxd/pool/volume_acl_test.go | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index 2ebe99a2..a7b30825 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -65,6 +65,59 @@ func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { } } +func TestRuntimeCatalogPendingWritePreservesPrivateNameMasking(t *testing.T) { + for _, operation := range []string{"register", "unregister"} { + t.Run(operation, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "private", "bytes"), Tenants: []string{"other"}} + if operation == "unregister" { + if err := m.RegisterVolume(t.Context(), "private", spec); err != nil { + t.Fatal(err) + } + } + blockCatalogRename(t, m) + var err error + if operation == "unregister" { + err = m.UnregisterVolume(t.Context(), "private") + } else { + err = m.RegisterVolume(t.Context(), "private", spec) + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("obstructed catalog write: %v", err) + } + for _, requested := range [][]types.Volume{ + {{Name: "private"}}, + {{Name: "data"}, {Name: "private"}}, + {{Name: "private"}, {Name: "data"}}, + } { + missing := slices.Clone(requested) + for i := range missing { + if missing[i].Name == "private" { + missing[i].Name = "missing" + } + } + _, deniedErr := m.SetVolumes(t.Context(), sb.ID, "acme", requested) + _, missingErr := m.SetVolumes(t.Context(), sb.ID, "acme", missing) + if !errors.Is(deniedErr, ErrVolumeUnavailable) || !errors.Is(missingErr, ErrVolumeUnavailable) || deniedErr.Error() != missingErr.Error() { + t.Errorf("private name distinguishable for %+v: denied=%v missing=%v", requested, deniedErr, missingErr) + } + } + if view, err := m.GetVolumes(sb.ID, "acme"); err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("refused request changed claim volumes: %+v, %v", view, err) + } + if len(eng.volumeSpecs) != 0 { + t.Fatal("refused request reached device attachment") + } + for _, name := range []string{"data", "private", "missing"} { + if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { + t.Fatalf("refused request reserved %s: %+v", name, got) + } + } + }) + } +} + func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { for _, operation := range []string{"put", "patch"} { t.Run(operation, func(t *testing.T) { From 8303c4b9390a3cdbed36b853959addee8f0f54df Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:33:08 +0000 Subject: [PATCH 05/39] pool: check volume authorization before pending admission --- sandboxd/pool/volume.go | 13 +++++++------ sandboxd/pool/volume_catalog_test.go | 20 ++++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 49069b3c..9f840edf 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -147,21 +147,22 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } m.mu.Lock() + catalog := m.catalog.entries for _, volume := range applied { + entry := catalog[volume.Name] + if entry == nil || !entry.allowedFor(tenant) { + m.mu.Unlock() + return nil, ErrVolumeUnavailable + } if err := m.catalog.admissionError(volume.Name); err != nil { m.mu.Unlock() return nil, err } } - catalog := m.catalog.entries m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := catalog[volume.Name] - if !ok || !entry.allowedFor(tenant) { - return nil, ErrVolumeUnavailable - } - resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) + resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, catalog[volume.Name]) if resolveErr != nil { return nil, resolveErr } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 197405a9..928b6b18 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -179,8 +179,8 @@ func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("write failure: %v", err) } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("pending admission: %v", err) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { t.Fatalf("volume-free claim blocked: %v", err) @@ -242,11 +242,15 @@ func TestRuntimeCatalogWriteFailureOnlyFencesAffectedName(t *testing.T) { t.Fatalf("resolved mixed admission: %v", err) } } + wantErr := ErrVolumeUnavailable + if unregister { + wantErr = ErrVolumeCatalogUnavailable + } for _, volumes := range [][]types.Volume{{{Name: "affected"}}, {{Name: "healthy"}, {Name: "affected"}}} { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, wantErr) { t.Fatalf("affected claim: %v", err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, wantErr) { t.Fatalf("affected live addition: %v", err) } } @@ -280,8 +284,8 @@ func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { t.Fatalf("possible replacement not recovered: %v", got) } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("uncertain admission: %v", err) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) } if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { t.Fatalf("volume-free claim blocked: %v", err) @@ -508,8 +512,8 @@ func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) t.Fatalf("failed retry cleared pending write: %v", err) } m.retryVolumeCatalog(t.Context()) - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("pending replacement lost fence: %v", err) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) } restore() ctx, cancel := context.WithCancel(t.Context()) From e51545d4d7c882f292b8f5741f88aa5783e8371b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:38:56 +0000 Subject: [PATCH 06/39] pool: reproduce inventory changes during failed removal --- sandboxd/pool/volume_cleanup_test.go | 80 +++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index a0f50798..966b9400 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -436,30 +436,82 @@ func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { } } -func TestStaleInventoryNarrowsFenceBeforeBlockedRemoval(t *testing.T) { +func TestStaleInventoryRetainsFenceUntilRemovalFails(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-known-before-remove" + name := "sbx-attach-during-remove" eng.vms[name], eng.removeErrFor = "/vsock/stale", name - eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} eng.removeStall = make(chan struct{}) - done := make(chan bool, 1) - go func() { done <- m.removeStaleVM(t.Context(), name, eng.record(name)) }() - synctest.Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("known stale writer: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { - t.Errorf("blocked removal retained global fence: %v", err) - } - close(eng.removeStall) - if <-done { + done := make(chan struct{}) + go func() { + defer close(done) + if m.removeStaleVM(t.Context(), name, eng.record(name)) { + t.Error("failed removal reported gone") + } + }() + checkStaleRemovalInventory(t, m, eng, name, done) + }) +} + +func TestStaleInventoryRetainsFenceUntilRetryFails(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-attach-during-retry" + eng.vms[name], eng.removeErrFor = "/vsock/stale", name + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} + eng.inspectDisksUnknown = true + if m.removeStaleVM(t.Context(), name, eng.record(name)) { t.Fatal("failed removal reported gone") } + eng.inspectDisksUnknown = false + eng.removeStall = make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + m.retryRemovals(t.Context()).Wait() + }() + checkStaleRemovalInventory(t, m, eng, name, done) }) } +func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { + t.Helper() + synctest.Wait() + registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted a reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted registration: %v", err) + } + eng.mu.Lock() + eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) + eng.mu.Unlock() + close(eng.removeStall) + <-done + if _, queued := m.pendingRemovals[name]; !queued { + t.Fatal("failed removal lost its retry") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer attached during removal lost its hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("failed removal lost its reader hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Errorf("failed removal blocked a compatible reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { + t.Fatalf("failed removal blocked unrelated registration: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { + t.Errorf("failed removal blocked an unrelated reader: %v", err) + } +} + func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { t.Run(string(outcome), func(t *testing.T) { From aff97975ef48790cc783e1bbf19161fd7386bfe3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:39:21 +0000 Subject: [PATCH 07/39] pool: preserve stale inventory observation after failed removal --- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/remove.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 9e103613..a7255629 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -177,11 +177,11 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe } // not-creating: the record moved on under the lock; remove normally. } - td = m.resolveStaleVolumes(ctx, name, td) if m.removeVM(ctx, name) { m.finishVolumeTeardown(ctx, td) return true } + td = m.resolveStaleVolumes(ctx, name, td) m.queueRemoval(name, "", rec.TapDevice(), td, time.Time{}) return false } diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index e901e0a5..a3bebea6 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -103,8 +103,8 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending return } } - pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if !m.removeVM(ctx, name) { + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } From f49765ade315c2e9aa181a7ebde766b5e9649a41 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 13:43:27 +0000 Subject: [PATCH 08/39] pool: align cleanup tests with post-removal inspection --- sandboxd/pool/reconcile_test.go | 3 -- sandboxd/pool/volume_cleanup_test.go | 70 ++++++++++++++-------------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 636555e7..6177c19a 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -181,9 +181,6 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { if !eng.removed("sbx-orphan-1") { t.Error("unowned running VM not removed") } - if eng.inspects != 1 { - t.Errorf("inspected stale VM %d times, want 1 before removal", eng.inspects) - } } func TestReconcileSweepsInterruptedForkExports(t *testing.T) { diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 966b9400..46b7f946 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -477,41 +477,6 @@ func TestStaleInventoryRetainsFenceUntilRetryFails(t *testing.T) { }) } -func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { - t.Helper() - synctest.Wait() - registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted a reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted registration: %v", err) - } - eng.mu.Lock() - eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) - eng.mu.Unlock() - close(eng.removeStall) - <-done - if _, queued := m.pendingRemovals[name]; !queued { - t.Fatal("failed removal lost its retry") - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("writer attached during removal lost its hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("failed removal lost its reader hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Errorf("failed removal blocked a compatible reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { - t.Fatalf("failed removal blocked unrelated registration: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { - t.Errorf("failed removal blocked an unrelated reader: %v", err) - } -} - func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { t.Run(string(outcome), func(t *testing.T) { @@ -553,3 +518,38 @@ func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { }) } } + +func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { + t.Helper() + synctest.Wait() + registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted a reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted registration: %v", err) + } + eng.mu.Lock() + eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) + eng.mu.Unlock() + close(eng.removeStall) + <-done + if _, queued := m.pendingRemovals[name]; !queued { + t.Fatal("failed removal lost its retry") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer attached during removal lost its hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("failed removal lost its reader hold: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Errorf("failed removal blocked a compatible reader: %v", err) + } + if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { + t.Fatalf("failed removal blocked unrelated registration: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { + t.Errorf("failed removal blocked an unrelated reader: %v", err) + } +} From df7c3082e91f8ee2858bcbcf11570be42379831a Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:12:50 +0000 Subject: [PATCH 09/39] pool: release taken-over volume holds when the run ends A teardown that takes over an in-flight volume run's holds now parks them on the run, and finishVolumeRun releases them once. This drops the done poll, the requeues that re-ran vm rm every reap tick, and the ignored bool result of finishVolumeTeardown. --- sandboxd/pool/remove.go | 17 ++++----- sandboxd/pool/volume.go | 31 ++++++++-------- sandboxd/pool/volume_cleanup_test.go | 51 +++------------------------ sandboxd/pool/volume_mutation.go | 8 +++-- sandboxd/pool/volume_mutation_test.go | 8 ++--- 5 files changed, 31 insertions(+), 84 deletions(-) diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index a3bebea6..5fc199a2 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -45,9 +45,7 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { if m.removeVM(ctx, name) { - if !m.finishVolumeTeardown(ctx, td) { - m.queueRemoval(name, sandboxID, "", td, time.Time{}) - } + m.finishVolumeTeardown(ctx, td) return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -93,13 +91,13 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending return case outcome == engine.StaleCreateBusy: if _, present, inspectErr := m.eng.Inspect(ctx, name); inspectErr == nil && !present { - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) return } m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) return } } @@ -108,14 +106,11 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, name, pending) + m.finishRemoval(ctx, pending) } -func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { - if !m.finishVolumeTeardown(ctx, pending.volumes) { - m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) - return - } +func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { + m.finishVolumeTeardown(ctx, pending.volumes) if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 9f840edf..0fb3270c 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -38,7 +38,7 @@ type resolvedVolume struct { // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - done <-chan struct{} + run *volumeRun unknown bool holds []types.Volume clears []string @@ -247,10 +247,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { m.mu.Lock() - td := volumeTeardown{holds: heldVolumes(sb)} - if run := m.volumeRuns[sb]; run != nil { - td.done = run.done - } + td := volumeTeardown{run: m.volumeRuns[sb], holds: heldVolumes(sb)} m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { @@ -287,15 +284,8 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT return td } -// finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. -func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) bool { - if td.done != nil { - select { - case <-td.done: - default: - return false - } - } +// finishVolumeTeardown runs once the VM is gone; an in-flight volume run releases the holds instead. +func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { logger := log.WithFunc("pool.finishVolumeTeardown") for _, path := range td.clears { if err := clearVolumeDirty(path); err != nil { @@ -303,12 +293,19 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) b } } m.mu.Lock() - m.releaseVolumes(td.holds) + defer m.mu.Unlock() if td.unknown { m.catalog.unknown-- } - m.mu.Unlock() - return true + if td.run != nil { + select { + case <-td.run.done: + default: + td.run.teardown = append(td.run.teardown, td.holds...) + return + } + } + m.releaseVolumes(td.holds) } // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 46b7f946..f3760304 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -43,6 +43,7 @@ func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { case <-time.After(time.Second): t.Fatal("release waited for paused worker") } + m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { t.Errorf("removed VM freed active worker count: %+v", got) } @@ -52,19 +53,17 @@ func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { t.Errorf("unregister overtook paused worker: %v", err) } - m.retryRemovals(t.Context()).Wait() - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { - t.Errorf("maintenance freed active worker count: %+v", got) - } close(eng.resume) resumed = true if err := <-changed; err == nil { t.Fatal("removed claim accepted late attachment") } - m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("completed worker leaked count: %+v", got) } + if n := len(slices.DeleteFunc(base.removedNames(), func(n string) bool { return n != sb.VMName })); n != 1 { + t.Fatalf("vm rm ran %d times, want once", n) + } if err := m.UnregisterVolume(t.Context(), "data"); err != nil { t.Fatal(err) } @@ -350,48 +349,6 @@ func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { } } -func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) { - base := newFakeEngine() - m := newTestManager(t, base, archivePool(3600)) - sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - if err := m.archive(t.Context(), sb); err != nil { - t.Fatal(err) - } - if sb.VMName != "" { - t.Fatalf("archived VM name = %q", sb.VMName) - } - ctx, run, err := m.beginVolumeRun(t.Context(), sb) - if err != nil { - t.Fatal(err) - } - if err := m.checkVolumeRun(ctx, sb, run); !errors.Is(err, ErrArchived) { - t.Fatalf("archived mutation validation: %v", err) - } - td := m.quiesceVolumes(t.Context(), sb) - if td.done == nil || len(td.holds) != 0 || len(td.clears) != 0 || td.unknown { - t.Fatalf("archived teardown has cleanup obligations: %+v", td) - } - if m.finishVolumeTeardown(t.Context(), td) { - t.Fatal("open volume run unexpectedly completed") - } - if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - m.finishVolumeRun(sb, run) - if len(m.pendingRemovals) != 0 { - t.Fatalf("empty teardown enqueued removal: %+v", m.pendingRemovals) - } - if slices.Contains(base.removedNames(), "") { - t.Fatal("release removed an empty VM name") - } - if m.store.referencesVolume("data") { - t.Fatal("release retained archived journal references") - } -} - func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { for _, tc := range []struct { name string diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 596d047f..24dfa517 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -19,9 +19,10 @@ const ( type volumeEdit func([]types.Volume) ([]types.Volume, error) type volumeRun struct { - cancel context.CancelFunc - done chan struct{} - holds []types.Volume + cancel context.CancelFunc + done chan struct{} + holds []types.Volume + teardown []types.Volume } // GetVolumes reads a claim's desired and applied volumes; a tenant reaches only its own claims. @@ -204,6 +205,7 @@ func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.Sandb if m.claimed[sb.ID] == sb { m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) } + m.releaseVolumes(run.teardown) delete(m.volumeRuns, sb) view := m.volumeView(sb) close(run.done) diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 66ebfd4e..86d9074c 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -280,11 +280,8 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { } close(resume) attachErr := <-attached - if action == "release" || action == "reap" { - waitFor(t, func() bool { - m.retryRemovals(t.Context()).Wait() - return volumeHoldersOf(m, "data") == (volumeHolders{}) - }) + if action == "reap" { + waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) } if action == "release" || action == "reap" { if attachErr == nil { @@ -721,7 +718,6 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) - m.retryRemovals(t.Context()).Wait() waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) From e942d310f12afd79dd91e2fe7645070b68dc7794 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:13:27 +0000 Subject: [PATCH 10/39] pool: classify stale VMs by inspection for the volume fence A stopped VM holds no volumes whether or not cocoon marks it stale, and a busy stale create now gets the same classification before it is queued and again when its retry inspects: running with a disk list takes per-volume holds, stopped or absent takes none. Only an Inspect error, a running VM without a disk list (cocoon v0.7.3 omits empty lists and swallows DiskList errors) or any other state keeps the node-wide fence. The table test covers each case by which claims and registrations are refused, with the disk list appearing only after the sweep so both retry paths classify. --- sandboxd/engine/cocoond_test.go | 8 - sandboxd/pool/pool.go | 1 + sandboxd/pool/pool_test.go | 18 +- sandboxd/pool/reconcile.go | 29 +- sandboxd/pool/reconcile_test.go | 3 + sandboxd/pool/remove.go | 6 +- sandboxd/pool/volume_cleanup_test.go | 470 +++++++-------------------- sandboxd/types/types.go | 1 - 8 files changed, 140 insertions(+), 396 deletions(-) diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 398241be..027b264c 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -45,8 +45,6 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { fakeCocoon(t, "#!/bin/sh\n[ \"$1 $2\" = \"vm inspect\" ] || exit 2\n"+ "if [ \"$3\" = sbx-1 ]; then echo '{\"config\":{\"name\":\"sbx-1\"},\"state\":\"running\","+ "\"attached_devices\":{\"disks\":[{\"id\":\"disk-a\",\"name\":\"imagenet\",\"path\":\"/v/a.img\",\"readonly\":true},{\"id\":\"disk-b\",\"name\":\"scratch\",\"path\":\"/v/b.img\"}]}}'; exit 0; fi\n"+ - "if [ \"$3\" = sbx-stale ]; then echo '{\"state\":\"stopped\",\"stale\":true}'; exit 0; fi\n"+ - "if [ \"$3\" = sbx-stopped ]; then echo '{\"state\":\"stopped\"}'; exit 0; fi\n"+ "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") @@ -54,12 +52,6 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } - for _, name := range []string{"sbx-stale", "sbx-stopped"} { - rec, present, err := e.Inspect(t.Context(), name) - if err != nil || !present || rec.State != "stopped" || rec.Stale != (name == "sbx-stale") { - t.Errorf("Inspect %s lost stopped classification: %+v, %v, %v", name, rec, present, err) - } - } if _, ok, err := e.Inspect(t.Context(), "sbx-gone"); err != nil || ok { t.Errorf("Inspect of a missing VM = %v, %v; want not found without an error", ok, err) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index e2cb2391..5c3b3525 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -72,6 +72,7 @@ const ( hibernatePrefix = "sbx-hib-" forkPrefix = "sbx-fork-" vmStateRunning = "running" + vmStateStopped = "stopped" vmStateCreating = "creating" goldenStampSuffix = ".stamp" diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index b66724ce..a372a55a 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1257,9 +1257,6 @@ type fakeEngine struct { liveVolumeHook func(string) error stopped map[string]bool inspectHook func() error - inspectStale bool - inspectDisksUnknown bool - trackDisks bool disks map[string][]types.VMDisk creating map[string]bool staleOutcome engine.StaleCreateOutcome @@ -1544,10 +1541,7 @@ func (f *fakeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, return types.VMRecord{}, false, nil } f.mu.Lock() - vms[i].Stale = f.inspectStale - if !f.inspectDisksUnknown { - vms[i].AttachedDevices.Disks = f.disks[name] - } + vms[i].AttachedDevices.Disks = f.disks[name] f.mu.Unlock() return vms[i], true, nil } @@ -1632,7 +1626,7 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } -func (f *fakeEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { +func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("attach"); err != nil { return err @@ -1649,12 +1643,6 @@ func (f *fakeEngine) DiskAttach(ctx context.Context, name string, spec engine.Vo if spec.Name == f.diskAttachErrFor { err = errors.New("attach failed") } - if err == nil && f.trackDisks { - if f.disks == nil { - f.disks = map[string][]types.VMDisk{} - } - f.disks[name] = append(f.disks[name], types.VMDisk{Name: spec.Name, ReadOnly: !spec.RW}) - } f.mu.Unlock() if cancel != nil { cancel() @@ -1782,7 +1770,7 @@ func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { case f.creating[name]: state = vmStateCreating case f.stopped[name]: - state = "stopped" + state = vmStateStopped } rec := types.VMRecord{State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} if f.tap != "" { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index a7255629..9e3c641e 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -172,7 +172,7 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - m.queueStaleCreate(name, rec.TapDevice(), td) + m.queueStaleCreate(name, rec.TapDevice(), m.resolveStaleVolumes(ctx, name, td)) return false } // not-creating: the record moved on under the lock; remove normally. @@ -196,26 +196,17 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum return td } var holds []types.Volume - if present { - switch inspected.State { - case "stopped": - if inspected.Stale { - return td + switch { + case present && inspected.State == vmStateRunning && inspected.AttachedDevices.Disks != nil: + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW } - case vmStateRunning: - if inspected.AttachedDevices.Disks == nil { - return td - } - holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW - } - } - default: - return td } + case present && inspected.State != vmStateStopped: + return td } m.mu.Lock() m.adoptVolumes(holds) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 6177c19a..b50d78dc 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -181,6 +181,9 @@ func TestReconcileStaleRunningVMSkipsVerb(t *testing.T) { if !eng.removed("sbx-orphan-1") { t.Error("unowned running VM not removed") } + if eng.inspects != 0 { + t.Errorf("inspected a stale VM whose removal succeeded %d times", eng.inspects) + } } func TestReconcileSweepsInterruptedForkExports(t *testing.T) { diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 5fc199a2..9d104dac 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -90,11 +90,7 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: - if _, present, inspectErr := m.eng.Inspect(ctx, name); inspectErr == nil && !present { - m.finishRemoval(ctx, pending) - return - } - m.queueStaleCreate(name, pending.tap, pending.volumes) + m.queueStaleCreate(name, pending.tap, m.resolveStaleVolumes(ctx, name, pending.volumes)) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: m.finishRemoval(ctx, pending) diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index f3760304..9caf6edb 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -4,12 +4,12 @@ import ( "context" "errors" "slices" + "sync" "sync/atomic" "testing" "testing/synctest" "time" - "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -83,21 +83,9 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { if err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("expected survivor") } - for range cap(m.maintSem) { - m.maintSem <- struct{}{} - } - wg := m.retryRemovals(t.Context()) - m.mu.Lock() - queued := len(m.pendingRemovals) - m.mu.Unlock() - if queued != 0 { - t.Fatal("cleanup was not dispatched") - } + wg := dispatchBlocked(t.Context(), t, m) err = m.UnregisterVolume(t.Context(), "data") - for range cap(m.maintSem) { - <-m.maintSem - } - wg.Wait() + unblock(m, wg) if !errors.Is(err, ErrVolumeRegistrationConflict) { t.Fatalf("dispatched cleanup lost guard: %v", err) } @@ -108,134 +96,119 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { } } -func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { +func TestStaleVMFenceFollowsInspection(t *testing.T) { + writer, reader := []types.VMDisk{{Name: "data"}}, []types.VMDisk{{Name: "data", ReadOnly: true}} + busy, fenced, inspectErr := ErrVolumeBusy, ErrVolumeCatalogUnavailable, errors.New("inspection unavailable") for _, tc := range []struct { - name, state string - inspectError bool + name, swept, listed, state string + disks []types.VMDisk + inspectErr error + want [3]error // data, data rw, other + register error }{ - {"running omitted", "running", false}, - {"running error", "running", true}, - {"inferred stopped omitted", "stopped", false}, - {"inferred stopped error", "stopped", true}, - {"creating omitted", "creating", false}, - {"creating error", "creating", true}, + {"running writer", vmStateRunning, "sbx-stale", vmStateRunning, writer, nil, [3]error{busy, busy, nil}, nil}, + {"running reader", vmStateRunning, "sbx-stale", vmStateRunning, reader, nil, [3]error{nil, busy, nil}, nil}, + {"running empty", vmStateRunning, "sbx-stale", vmStateRunning, []types.VMDisk{}, nil, [3]error{}, nil}, + {"running omitted disks", vmStateRunning, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"stopped", vmStateRunning, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"inspected creating", vmStateRunning, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"inspect error", vmStateRunning, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, + {"busy creating", vmStateCreating, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"busy running", vmStateCreating, "sbx-stale", vmStateRunning, reader, nil, [3]error{nil, busy, nil}, nil}, + {"busy omitted disks", vmStateCreating, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, + {"busy stopped", vmStateCreating, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"busy absent", vmStateCreating, "sbx-other", vmStateRunning, writer, nil, [3]error{}, nil}, + {"busy inspect error", vmStateCreating, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, } { t.Run(tc.name, func(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, nil) - path := writeVolumeImage(t, "runtime", "bytes") - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { - t.Fatal(err) + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-stale" + eng.vms[tc.listed], eng.removeErrFor, eng.staleOutcome = "/vsock/stale", name, engine.StaleCreateBusy + eng.stopped[tc.listed], eng.creating[tc.listed] = tc.state == vmStateStopped, tc.state == vmStateCreating + eng.inspectHook = func() error { return tc.inspectErr } + if m.removeStaleVM(t.Context(), name, types.VMRecord{State: tc.swept, Config: types.VMConfig{Name: name}}) { + t.Fatal("stale VM reported gone") } - name := "sbx-stale-unknown" - base.vms[name] = "/vsock/stale" - base.stopped[name] = tc.state == "stopped" - base.creating[name] = tc.state == "creating" - base.inspectStale = tc.state == "stopped" - base.inspectDisksUnknown = !tc.inspectError - base.staleOutcome = engine.StaleCreateBusy - base.removeErrFor = name - base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} - var inspectFailed atomic.Bool - inspectFailed.Store(true) - base.inspectHook = func() error { - if tc.inspectError && inspectFailed.Load() { - return errors.New("inspection unavailable") + // the disk list appears after the sweep, so running rows are classified by the retry + eng.disks = map[string][]types.VMDisk{name: tc.disks} + m.retryRemovals(t.Context()).Wait() + for i, volume := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "other"}} { + if err := claimAndRelease(t, m, volume); !errors.Is(err, tc.want[i]) { + t.Errorf("claim %+v = %v, want %v", volume, err, tc.want[i]) } - return nil } - rec := base.record(name) - rec.State = tc.state - if m.removeStaleVM(t.Context(), name, rec) { - t.Fatal("unknown stale VM unexpectedly gone") + if err := m.RegisterVolume(t.Context(), "fresh", types.VolumeRegistration{Path: writeVolumeImage(t, "fresh", "bytes")}); !errors.Is(err, tc.register) { + t.Errorf("unrelated registration = %v, want %v", err, tc.register) } - for range 2 { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("unknown stale attachment admitted reader: %v", err) - } - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("unknown stale attachment allowed unregister: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatalf("volume-free claim blocked: %v", err) - } - for range cap(m.maintSem) { - m.maintSem <- struct{}{} - } - wg := m.retryRemovals(t.Context()) - if err := m.RegisterVolume(t.Context(), "new", types.VolumeRegistration{Path: writeVolumeImage(t, "new", "bytes")}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("dispatched unknown fence lost: %v", err) - } - for range cap(m.maintSem) { - <-m.maintSem - } - wg.Wait() - } - inspectFailed.Store(false) - base.stopped[name], base.creating[name] = false, false - base.inspectStale, base.inspectDisksUnknown = false, false - if tc.state == "creating" { - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("busy partial inventory cleared unknown: %v", err) - } + if err := claimAndRelease(t, m); err != nil { + t.Errorf("volume-free claim: %v", err) } - base.staleOutcome = engine.StaleCreateNotCreating + eng.removeErrFor, eng.staleOutcome = "", engine.StaleCreateCollected m.retryRemovals(t.Context()).Wait() - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { - t.Fatalf("inspection did not replace fence with known holds: %+v", got) - } - if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated", "bytes")}); err != nil { - t.Fatalf("resolved inspection left global fence: %v", err) + for _, volume := range []types.Volume{{Name: "data", Mode: types.VolumeModeRW}, {Name: "other", Mode: types.VolumeModeRW}} { + if err := claimAndRelease(t, m, volume); err != nil { + t.Errorf("claim %+v after confirmed removal: %v", volume, err) + } } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("known attachment lost exclusion: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if err := m.UnregisterVolume(t.Context(), "data"); err != nil { - t.Fatalf("confirmed removal retained fence: %v", err) + if m.catalog.unknown != 0 { + t.Fatalf("cleanup retained or double-released its fence: unknown=%d", m.catalog.unknown) } }) } } -func TestRuntimeCatalogLoadedBeforeClaimReconciliation(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - path := writeVolumeImage(t, "runtime", "bytes") - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true, Tenants: []string{"acme"}}); err != nil { - t.Fatal(err) - } - sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) - if claimErr != nil { - t.Fatal(claimErr) +func TestStaleInventoryIsTakenAfterFailedRemoval(t *testing.T) { + sweep := func(ctx context.Context, m *Manager, name string) { + m.removeStaleVM(ctx, name, types.VMRecord{Config: types.VMConfig{Name: name}}) } - m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) - if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "data" || !got[0].Available || !got[0].Writable { - t.Fatalf("catalog absent before reconcile: %+v", got) - } - if local, err := m2.VolumePlacement(testKey, "acme", []string{"data"}); err != nil || !local { - t.Fatalf("placement before reconcile: %v %v", local, err) - } - if err := m2.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - view, err := m2.GetVolumes(sb.ID, "") - if err != nil || !slices.Equal(view.Volumes, sb.Volumes) { - t.Fatalf("claim recovery: %+v %v", view, err) - } - if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("recovered holder missing: %v", err) + retry := func(ctx context.Context, m *Manager, _ string) { m.retryRemovals(ctx).Wait() } + for _, tc := range []struct { + name string + before, after func(context.Context, *Manager, string) + }{{"sweep", func(context.Context, *Manager, string) {}, sweep}, {"retry", sweep, retry}} { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + name := "sbx-attach-during-remove" + eng.vms[name], eng.removeErrFor = "/vsock/stale", name + tc.before(t.Context(), m, name) + eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} + eng.removeStall = make(chan struct{}) + done := make(chan struct{}) + go func() { defer close(done); tc.after(t.Context(), m, name) }() + synctest.Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("in-flight removal admitted a reader: %v", err) + } + eng.mu.Lock() + eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) + eng.mu.Unlock() + close(eng.removeStall) + <-done + for _, c := range []struct { + volume types.Volume + want error + }{ + {types.Volume{Name: "other", Mode: types.VolumeModeRW}, ErrVolumeBusy}, + {types.Volume{Name: "data", Mode: types.VolumeModeRW}, ErrVolumeBusy}, + {types.Volume{Name: "data"}, nil}, + } { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{c.volume}}); !errors.Is(err, c.want) { + t.Errorf("claim %+v = %v, want %v", c.volume, err, c.want) + } + } + }) + }) } } -func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing.T) { +func TestUnknownFencePrecedesInspection(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) name := "sbx-stale-before-inspect" - base.vms[name] = "/vsock/stale" - base.removeErrFor = name + base.vms[name], base.removeErrFor = "/vsock/stale", name entered, resume := make(chan struct{}), make(chan struct{}) var gated atomic.Bool base.inspectHook = func() error { @@ -248,22 +221,10 @@ func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing done := make(chan bool, 1) go func() { done <- m.removeStaleVM(t.Context(), name, base.record(name)) }() <-entered - if !m.mu.TryLock() { - t.Fatal("stale inspection holds manager lock") - } - m.mu.Unlock() _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) close(resume) - if gone := <-done; gone { - t.Fatal("failed inspect/removal unexpectedly confirmed gone") - } - if !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("admitted volume while inspection was blocked: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("successful removal retained unknown fence: %v", err) + if gone := <-done; gone || !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("admission while inspection was blocked: %v (gone=%t)", err, gone) } } @@ -272,24 +233,20 @@ func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { base := newFakeEngine() m := newVolumeManager(t, base, liveCatalog(t)) name := "sbx-canceled-dispatch" - base.vms[name] = "/vsock/stale" - base.removeErrFor = name + base.vms[name], base.removeErrFor = "/vsock/stale", name base.inspectHook = func() error { return errors.New("inspection unavailable") } if m.removeStaleVM(t.Context(), name, base.record(name)) { t.Fatal("expected survivor") } - for range cap(m.maintSem) { - m.maintSem <- struct{}{} - } ctx, cancel := context.WithCancel(t.Context()) - wg := m.retryRemovals(ctx) + wg := dispatchBlocked(ctx, t, m) cancel() synctest.Wait() - base.removeErrFor = "" - for range cap(m.maintSem) { - <-m.maintSem + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("dispatched unknown fence lost: %v", err) } - wg.Wait() + base.removeErrFor = "" + unblock(m, wg) m.retryRemovals(t.Context()).Wait() if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { t.Fatalf("canceled dispatch stranded unknown fence: %v", err) @@ -297,216 +254,33 @@ func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { }) } -func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { - base := newFakeEngine() - name := "sbx-warm-unfinalized" - base.vms[name] = "/vsock/warm" - base.diskAttachErrFor = "broken" - base.removeErrFor = name - rendezvousAttaches(base, 2) - catalog := []config.VolumeSpec{ - {Name: "data", Path: writeVolumeImage(t, "warm-data", "data"), Writable: true}, - {Name: "broken", Path: writeVolumeImage(t, "warm-broken", "broken")}, - } - dir := t.TempDir() - m := newVolumePoolManager(t, base, dir, catalog) - base.inspectDisksUnknown, base.trackDisks = true, true - warm := &types.Sandbox{VMName: name, Key: testKey, VsockSocket: "/vsock/warm"} - m.pools[testKey].warm = append(m.pools[testKey].warm, warm) - _, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{ - {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}, - {Name: "broken", AttachOnly: true}, - }}) - if err == nil { - t.Fatal("expected attach failure before finalization") - } - if warm.ID != "" { - t.Fatal("claim unexpectedly finalized") - } - base.mu.Lock() - attached := slices.Clone(base.disks[name]) - base.attachRendezvous = nil - base.mu.Unlock() - if !slices.ContainsFunc(attached, func(d types.VMDisk) bool { return d.Name == "data" && !d.ReadOnly }) { - t.Fatalf("warm VM has no attached writer: %+v", attached) - } - claims, err := newClaimStore(dir, false).load() - if err != nil || len(claims) != 0 { - t.Fatalf("pre-finalize journal: %+v, %v", claims, err) - } - restarted := newVolumeManagerAt(t, base, dir, catalog) - restarted.eng = base - if err := restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("ambiguous warm survivor admitted competing writer: %v", err) - } - base.removeErrFor = "" - restarted.retryRemovals(t.Context()).Wait() - if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("confirmed removal failed to clear fence: %v", err) +func claimAndRelease(t *testing.T, m *Manager, volumes ...types.Volume) error { + t.Helper() + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}) + if err != nil { + return err } + return m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) } -func TestStaleInventoryClassifiesStoppedAndRunning(t *testing.T) { - for _, tc := range []struct { - name string - stopped, stale bool - disks []types.VMDisk - wantErr error - }{ - {"running unknown", false, false, nil, ErrVolumeCatalogUnavailable}, - {"running empty", false, false, []types.VMDisk{}, nil}, - {"ordinary stopped", true, false, nil, nil}, - {"inferred stopped", true, true, nil, ErrVolumeCatalogUnavailable}, - } { - t.Run(tc.name, func(t *testing.T) { - base := newFakeEngine() - m := newVolumeManager(t, base, liveCatalog(t)) - name := "sbx-inventory-shape" - base.vms[name], base.removeErrFor = "/vsock/stale", name - base.disks = map[string][]types.VMDisk{name: tc.disks} - base.stopped[name], base.inspectStale = tc.stopped, tc.stale - if m.removeStaleVM(t.Context(), name, base.record(name)) { - t.Fatal("stale VM unexpectedly disappeared") - } - _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) - if !errors.Is(err, tc.wantErr) { - t.Fatalf("admission=%v, want=%v", err, tc.wantErr) - } - if _, queued := m.pendingRemovals[name]; !queued { - t.Fatal("surviving VM lost pending removal") - } - if err := m.RegisterVolume(t.Context(), "before-removal", types.VolumeRegistration{Path: "/images/before-removal"}); !errors.Is(err, tc.wantErr) { - t.Fatalf("registration before removal: %v", err) - } - base.removeErrFor = "" - m.retryRemovals(t.Context()).Wait() - if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: "/images/unrelated"}); err != nil { - t.Fatalf("confirmed removal retained registration fence: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatalf("confirmed removal retained admission fence: %v", err) - } - }) +// dispatchBlocked moves the removal queue into workers parked on a full maintenance semaphore. +func dispatchBlocked(ctx context.Context, t *testing.T, m *Manager) *sync.WaitGroup { + t.Helper() + for range cap(m.maintSem) { + m.maintSem <- struct{}{} } -} - -func TestStaleInventoryRetainsFenceUntilRemovalFails(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-attach-during-remove" - eng.vms[name], eng.removeErrFor = "/vsock/stale", name - eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} - eng.removeStall = make(chan struct{}) - done := make(chan struct{}) - go func() { - defer close(done) - if m.removeStaleVM(t.Context(), name, eng.record(name)) { - t.Error("failed removal reported gone") - } - }() - checkStaleRemovalInventory(t, m, eng, name, done) - }) -} - -func TestStaleInventoryRetainsFenceUntilRetryFails(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-attach-during-retry" - eng.vms[name], eng.removeErrFor = "/vsock/stale", name - eng.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: true}}} - eng.inspectDisksUnknown = true - if m.removeStaleVM(t.Context(), name, eng.record(name)) { - t.Fatal("failed removal reported gone") - } - eng.inspectDisksUnknown = false - eng.removeStall = make(chan struct{}) - done := make(chan struct{}) - go func() { - defer close(done) - m.retryRemovals(t.Context()).Wait() - }() - checkStaleRemovalInventory(t, m, eng, name, done) - }) -} - -func TestBusyStaleCreateKeepsVolumeFenceUntilCleanup(t *testing.T) { - for _, outcome := range []engine.StaleCreateOutcome{engine.StaleCreateCollected, engine.StaleCreateNotFound, engine.StaleCreateBusy} { - t.Run(string(outcome), func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - name := "sbx-busy-create" - eng.vms[name], eng.creating[name] = "/vsock/creating", true - eng.staleOutcome = engine.StaleCreateBusy - eng.disks = map[string][]types.VMDisk{name: {{Name: "other"}}} - rec := eng.record(name) - rec.State = vmStateCreating - if m.removeStaleVM(t.Context(), name, rec) { - t.Fatal("busy create reported gone") - } - eng.creating[name] = false - for _, stopped := range []bool{false, true} { - eng.stopped[name] = stopped - before := eng.inspects - m.retryRemovals(t.Context()).Wait() - if eng.inspects != before+1 { - t.Fatal("busy retry did not inspect") - } - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("busy create admitted a volume: %v", err) - } - eng.staleOutcome = outcome - if outcome == engine.StaleCreateBusy { - delete(eng.vms, name) - } - m.retryRemovals(t.Context()).Wait() - m.retryRemovals(t.Context()).Wait() - if m.catalog.unknown != 0 || len(m.pendingRemovals) != 0 { - t.Fatalf("cleanup retained or double-released its fence: unknown=%d pending=%d", m.catalog.unknown, len(m.pendingRemovals)) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatalf("confirmed cleanup retained fence: %v", err) - } - }) + wg := m.retryRemovals(ctx) + m.mu.Lock() + defer m.mu.Unlock() + if len(m.pendingRemovals) != 0 { + t.Fatal("cleanup was not dispatched") } + return wg } -func checkStaleRemovalInventory(t *testing.T, m *Manager, eng *fakeEngine, name string, done <-chan struct{}) { - t.Helper() - synctest.Wait() - registration := types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated.img", "unrelated")} - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted a reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Errorf("in-flight removal admitted registration: %v", err) - } - eng.mu.Lock() - eng.disks[name] = append(eng.disks[name], types.VMDisk{Name: "other"}) - eng.mu.Unlock() - close(eng.removeStall) - <-done - if _, queued := m.pendingRemovals[name]; !queued { - t.Fatal("failed removal lost its retry") - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "other", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("writer attached during removal lost its hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("failed removal lost its reader hold: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Errorf("failed removal blocked a compatible reader: %v", err) - } - if err := m.RegisterVolume(t.Context(), "unrelated", registration); err != nil { - t.Fatalf("failed removal blocked unrelated registration: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "unrelated"}}}); err != nil { - t.Errorf("failed removal blocked an unrelated reader: %v", err) +func unblock(m *Manager, wg *sync.WaitGroup) { + for range cap(m.maintSem) { + <-m.maintSem } + wg.Wait() } diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 2a93ab78..4cc8d307 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -731,7 +731,6 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { - Stale bool `json:"stale,omitzero"` State string `json:"state"` PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` From de605a5bd82620c4298ab5e04f898bb1393d2892 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:14:10 +0000 Subject: [PATCH 11/39] pool: make the resolved volume check side-effect free The only owner caller checked its run under the same lock, so the check takes the owner ID instead of re-deriving it. It no longer adopts holds: each caller adopts the applied list it already holds and will store, under the same m.mu, so admission neither allocates nor takes a second list that must mirror the resolved one. --- sandboxd/pool/claim.go | 10 +++++++--- sandboxd/pool/volume.go | 9 ++------- sandboxd/pool/volume_bench_test.go | 6 ++++-- sandboxd/pool/volume_convergence_test.go | 4 +++- sandboxd/pool/volume_mutation.go | 3 ++- sandboxd/pool/volume_mutation_test.go | 3 ++- sandboxd/pool/volume_rw_test.go | 11 ++++------- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 9d9110bb..18ace6c0 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -371,7 +371,7 @@ func (m *Manager) overQuota(extra int, tenant string) error { // admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) - if err := m.admitClaim(tenant, volumes); err != nil { + if err := m.admitClaim(tenant, volumes, applied); err != nil { return nil, err } if err := m.confirmVolumesClean(volumes); err != nil { @@ -382,13 +382,17 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types } // admitClaim refuses a busy volume or an over-quota tenant before any VM is built. -func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { +func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume, applied []types.Volume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveResolved(volumes, nil) + if err := m.checkResolved(volumes, ""); err != nil { + return err + } + m.adoptVolumes(applied) + return nil } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 0fb3270c..659b54c6 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -353,12 +353,8 @@ func (m *Manager) catalogSnapshot() map[string]*catalogVolume { return m.catalog.entries } -// reserveResolved admits the entire request while callers hold m.mu. -func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox) error { - ownerID := "" - if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { - ownerID = owner.ID - } +// checkResolved refuses the request if any volume is fenced, rebound or busy; callers hold m.mu. +func (m *Manager) checkResolved(volumes []resolvedVolume, ownerID string) error { for _, resolved := range volumes { volume := resolved.applied if err := m.catalog.admissionError(volume.Name); err != nil { @@ -375,7 +371,6 @@ func (m *Manager) reserveResolved(volumes []resolvedVolume, owner *types.Sandbox return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) } } - m.adoptVolumes(appliedVolumes(volumes)) return nil } diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go index 22242076..e717d96f 100644 --- a/sandboxd/pool/volume_bench_test.go +++ b/sandboxd/pool/volume_bench_test.go @@ -24,10 +24,12 @@ func BenchmarkVolumeAdmissionWithCheckpoints(b *testing.B) { if len(resolved) != 1 || resolved[0].applied != requested[0] { b.Fatalf("resolved volumes: %+v", resolved) } + applied := appliedVolumes(resolved) m.mu.Lock() - err = m.reserveResolved(resolved, nil) + err = m.checkResolved(resolved, "") if err == nil { - m.releaseVolumes(appliedVolumes(resolved)) + m.adoptVolumes(applied) + m.releaseVolumes(applied) } m.mu.Unlock() if err != nil { diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 4446957a..78b38b34 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -355,7 +355,9 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { t.Fatal(err) } m.mu.Lock() - err = m.reserveResolved(resolved, nil) + if err = m.checkResolved(resolved, ""); err == nil { + m.adoptVolumes(held) + } run.holds = held m.mu.Unlock() if err != nil { diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 24dfa517..a20b0b59 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -274,11 +274,12 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveResolved(resolved, sb); err != nil { + if err := m.checkResolved(resolved, sb.ID); err != nil { m.mu.Unlock() sb.Transition.Unlock() return claimSnapshot{}, false, err } + m.adoptVolumes(additions) run.holds = append(run.holds, additions...) m.mu.Unlock() sb.Transition.Unlock() diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 86d9074c..c25fbf27 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -857,9 +857,10 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume } m.mu.Lock() defer m.mu.Unlock() - if err = m.reserveResolved(resolved, nil); err != nil { + if err = m.checkResolved(resolved, ""); err != nil { t.Fatal(err) } + m.adoptVolumes(transient) run.holds = append(run.holds, transient...) return run } diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index dadeda8c..fda04c98 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -152,10 +152,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { if markErr := markVolumeDirty(path); markErr != nil { t.Fatalf("mark dirty: %v", markErr) } - m.mu.Lock() - reserveErr := m.reserveResolved(resolved, nil) - m.mu.Unlock() - if reserveErr != nil { + if reserveErr := m.admitClaim("", resolved, appliedVolumes(resolved)); reserveErr != nil { t.Fatalf("reserve once the writer released: %v", reserveErr) } @@ -227,13 +224,13 @@ func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { t.Fatal(err) } m.mu.Lock() - defer m.mu.Unlock() m.adoptVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch", Mode: types.VolumeModeRW}}) + m.mu.Unlock() - if err := m.reserveResolved(resolved, nil); !errors.Is(err, ErrVolumeBusy) { + if err := m.admitClaim("", resolved, appliedVolumes(resolved)); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } - if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { + if holders := volumeHoldersOf(m, "scratch"); holders != (volumeHolders{writers: 1}) { t.Errorf("registry after the refusal=%+v, want the writer alone", holders) } } From 969b7abbc8110ead9564ccaa82c8a0a2b3958996 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:14:43 +0000 Subject: [PATCH 12/39] pool: count catalog references from holders and the journal Every claimed volume is already counted in the admission holders, and the claim store keeps a per-name count for unacknowledged references, so the walk over every claim under m.mu adds nothing. --- sandboxd/pool/volume_catalog.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 4cd0c302..ee5ba8b8 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -150,15 +150,7 @@ func (m *Manager) loadRuntimeVolumes() error { } func (m *Manager) volumeReferenced(name string) bool { - if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { - return true - } - for _, sb := range m.claimed { - if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { - return true - } - } - return m.store.referencesVolume(name) + return m.catalog.holders[name] != (volumeHolders{}) || m.store.referencesVolume(name) } func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[string]*catalogVolume) error { From 1cee58e20db60756b72ac95ba08eeea53379880f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:15:05 +0000 Subject: [PATCH 13/39] pool: hold Transition across the volume clean check again This reverts the Transition release around confirmVolumesClean that "pool: persist runtime volume registrations" introduced earlier in this PR. The clean check is one marker stat; dropping Transition around it let a lifecycle change interleave between reservation and staging for no gain. --- sandboxd/pool/volume_mutation.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index a20b0b59..01d64821 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -263,29 +263,24 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() - sb.Transition.Unlock() return claimSnapshot{}, false, err } if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() - sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } if err := m.checkResolved(resolved, sb.ID); err != nil { m.mu.Unlock() - sb.Transition.Unlock() return claimSnapshot{}, false, err } m.adoptVolumes(additions) run.holds = append(run.holds, additions...) m.mu.Unlock() - sb.Transition.Unlock() cleanErr := m.confirmVolumesClean(resolved) - sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { From 14429edf104195f27b309478a727869e4d95a2ae Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:15:49 +0000 Subject: [PATCH 14/39] pool: drop duplicated volume registration checks The pool already rejects an invalid name with the same 400, so the handler check goes; the pool now validates before touching the pending catalog. The handler maps pool errors before context errors, so an uncertain catalog write still answers 503 without its own case. config.Load defaults static directio, and normalization builds the catalog entry directly instead of round-tripping through one-caller registration helpers. Marshaling the catalog record cannot fail: every string in it came through a UTF-8 validating JSON decode or a name regex. --- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume.go | 4 -- sandboxd/pool/volume_catalog.go | 66 ++++++++------------- sandboxd/pool/volume_catalog_test.go | 4 +- sandboxd/server/volume_registration.go | 8 +-- sandboxd/server/volume_registration_test.go | 38 ------------ 6 files changed, 29 insertions(+), 93 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 5c3b3525..94c10e83 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -480,7 +480,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } for _, volume := range cfg.Volumes { m.catalog.entries[volume.Name] = &catalogVolume{ - disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, + disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, tenants: slices.Clone(volume.Tenants), writable: volume.Writable, } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 659b54c6..0f04eb42 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -55,10 +55,6 @@ func (v *catalogVolume) allowedFor(tenant string) bool { return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) } -func (v catalogVolume) registration() types.VolumeRegistration { - return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.tenants} -} - // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { catalog := m.catalogSnapshot() diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index ee5ba8b8..938fa687 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -50,13 +50,13 @@ func (c *volumeCatalog) admissionError(name string) error { // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { - m.catalog.writeMu.Lock() - defer m.catalog.writeMu.Unlock() - if err := m.commitPendingCatalog(ctx); err != nil { + volume, err := normalizeRegistration(name, spec) + if err != nil { return err } - normalized, err := normalizeRegistration(name, spec) - if err != nil { + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err = m.commitPendingCatalog(ctx); err != nil { return err } m.mu.Lock() @@ -67,34 +67,29 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo return err } if entry := entries[name]; entry != nil { - if entry.runtime && sameRegistration(entry.registration(), normalized) { + if entry.runtime && entry.disk == volume.disk && entry.writable == volume.writable && slices.Equal(entry.tenants, volume.tenants) { return nil } return fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) } - if err := catalogPathConflict(entries, normalized.Path); err != nil { + if err := catalogPathConflict(entries, volume.disk.Path); err != nil { return fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) } entries = maps.Clone(entries) - entries[name] = &catalogVolume{ - runtime: true, - disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, - tenants: normalized.Tenants, - writable: normalized.Writable, - } + entries[name] = volume return m.writeCatalog(ctx, name, entries) } // UnregisterVolume retires an unreferenced runtime binding without touching its backing. func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { + if !types.ValidVolumeName(name) { + return fmt.Errorf("%w: invalid volume name", ErrBadVolume) + } m.catalog.writeMu.Lock() defer m.catalog.writeMu.Unlock() if err := m.commitPendingCatalog(ctx); err != nil { return err } - if !types.ValidVolumeName(name) { - return fmt.Errorf("%w: invalid volume name", ErrBadVolume) - } m.mu.Lock() entries := m.catalog.entries entry := entries[name] @@ -128,22 +123,17 @@ func (m *Manager) loadRuntimeVolumes() error { } entries := maps.Clone(m.catalog.entries) for name, spec := range record.Volumes { - normalized, normalizeErr := normalizeRegistration(name, spec) + volume, normalizeErr := normalizeRegistration(name, spec) if normalizeErr != nil { return fmt.Errorf("runtime volume catalog: %w", normalizeErr) } if m.catalog.entries[name] != nil { return fmt.Errorf("runtime volume catalog duplicates static name %q", name) } - if err := catalogPathConflict(entries, normalized.Path); err != nil { + if err := catalogPathConflict(entries, volume.disk.Path); err != nil { return fmt.Errorf("runtime volume catalog: %w", err) } - entries[name] = &catalogVolume{ - runtime: true, - disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, - tenants: normalized.Tenants, - writable: normalized.Writable, - } + entries[name] = volume } m.catalog.entries = entries return nil @@ -157,16 +147,13 @@ func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[str record := runtimeCatalog{Version: 1, Volumes: map[string]types.VolumeRegistration{}} for key, entry := range entries { if entry.runtime { - record.Volumes[key] = entry.registration() + record.Volumes[key] = types.VolumeRegistration{Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, Writable: entry.writable, Tenants: entry.tenants} } } - raw, err := json.Marshal(record, json.Deterministic(true)) - if err != nil { - return fmt.Errorf("%w: %w", ErrBadVolume, err) - } + raw, _ := json.Marshal(record, json.Deterministic(true)) proposal := &catalogWrite{name: name, entries: entries, raw: raw} m.mu.Lock() - err = ctx.Err() + err := ctx.Err() if err == nil && m.volumeReferenced(name) { err = fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) } @@ -223,20 +210,17 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { +func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, }}); err != nil { - return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) + return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } - spec.Path = filepath.Clean(spec.Path) - spec.Tenants = slices.Clone(spec.Tenants) - slices.Sort(spec.Tenants) - spec.Tenants = slices.Compact(spec.Tenants) - return spec, nil -} - -func sameRegistration(a, b types.VolumeRegistration) bool { - return a.Path == b.Path && a.DirectIO == b.DirectIO && a.Writable == b.Writable && slices.Equal(a.Tenants, b.Tenants) + return &catalogVolume{ + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: filepath.Clean(spec.Path), DirectIO: spec.DirectIO}, + tenants: slices.Compact(slices.Sorted(slices.Values(spec.Tenants))), + writable: spec.Writable, + }, nil } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 928b6b18..de250141 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -364,10 +364,10 @@ func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { func TestRuntimeRegistrationNormalization(t *testing.T) { input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} normalized, err := normalizeRegistration("data", input) - if err != nil || normalized.Path != "/images/data" || normalized.DirectIO != types.DirectIOOff || !slices.Equal(normalized.Tenants, []string{"acme", "beta"}) { + if err != nil || normalized.disk.Path != "/images/data" || normalized.disk.DirectIO != types.DirectIOOff || !slices.Equal(normalized.tenants, []string{"acme", "beta"}) { t.Fatalf("normalization: %+v %v", normalized, err) } - normalized.Tenants[0] = "caller-copy" + normalized.tenants[0] = "caller-copy" if input.Tenants[1] != "acme" { t.Fatal("normalization mutated caller input") } diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go index 39bad6a6..29fae3ab 100644 --- a/sandboxd/server/volume_registration.go +++ b/sandboxd/server/volume_registration.go @@ -5,7 +5,6 @@ import ( "errors" "net/http" - "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -15,10 +14,6 @@ func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request return } name := r.PathValue("name") - if !types.ValidVolumeName(name) { - writeErr(w, http.StatusBadRequest, "invalid volume name") - return - } var err error op := "unregister volume" if r.Method == http.MethodPut { @@ -32,8 +27,7 @@ func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request err = s.mgr.UnregisterVolume(r.Context(), name) } switch { - case errors.Is(err, pool.ErrVolumeCatalogUnavailable): - writePoolErr(w, err) + case writePoolErr(w, err): case errors.Is(err, context.DeadlineExceeded): writeErr(w, http.StatusGatewayTimeout, op+" timed out") case errors.Is(err, context.Canceled): diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index 99ff6ac7..ef2c48f9 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -165,44 +165,6 @@ func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { } } -func TestVolumeRegistrationRoutesValidateNames(t *testing.T) { - for _, method := range []string{http.MethodPut, http.MethodDelete} { - for _, tt := range []struct { - name, encoded string - valid bool - }{ - {"data", "data", true}, - {"a_b-0", "a_b-0", true}, - {"abcdefghijklmnopqrst", "abcdefghijklmnopqrst", true}, - {"Data", "Data", false}, - {"1data", "1data", false}, - {"cocoon-data", "cocoon-data", false}, - {"abcdefghijklmnopqrstu", "abcdefghijklmnopqrstu", false}, - {"data/set", "data%2Fset", false}, - {"data set", "data%20set", false}, - } { - t.Run(method+"/"+tt.name, func(t *testing.T) { - var gotName string - mgr := &fakeManager{ - registerVolume: func(_ context.Context, name string, _ types.VolumeRegistration) error { gotName = name; return nil }, - unregisterVolume: func(_ context.Context, name string) error { gotName = name; return nil }, - } - ts := newTestServer(t, "root", mgr, nil) - resp := doReq(t, method, ts.URL+"/v1/volumes/"+tt.encoded, "root", `{"path":"/private/data.img"}`) - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - wantCode, wantName := http.StatusBadRequest, "" - if tt.valid { - wantCode, wantName = http.StatusNoContent, tt.name - } - if resp.StatusCode != wantCode || gotName != wantName { - t.Fatalf("status=%d name=%q body=%s, want %d name=%q", resp.StatusCode, gotName, body, wantCode, wantName) - } - }) - } - } -} - func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { for _, method := range []string{http.MethodPut, http.MethodDelete} { for _, tt := range []struct { From 9068fc035be56e40753d1234e5cdeac377491350 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:16:48 +0000 Subject: [PATCH 15/39] pool: compact runtime volume tests Merge overlapping catalog, ACL, retirement and server tests into tables without per-case branching, share one failed-detach helper, and restore the base claim save helper and capture-test fixtures instead of rewriting their call sites. Drop cases for record file guards that no longer exist, rows that re-test the shared strict body decoder or static path validation, and a config test the pool ACL test covers. Every refusal and recovery path keeps a behavioural assertion. --- e2e/volume_catalog_test.go | 106 ++-- sandboxd/config/config_test.go | 16 - sandboxd/pool/archive_test.go | 3 +- sandboxd/pool/checkpoint_bench_test.go | 4 +- sandboxd/pool/checkpoint_references_test.go | 72 +-- sandboxd/pool/claims_bench_test.go | 6 +- sandboxd/pool/claims_test.go | 6 +- sandboxd/pool/failover_test.go | 9 +- sandboxd/pool/hibernate_test.go | 12 +- sandboxd/pool/liveness_test.go | 3 +- sandboxd/pool/pool_test.go | 3 +- sandboxd/pool/replica_test.go | 23 +- sandboxd/pool/volume_acl_test.go | 222 ++----- sandboxd/pool/volume_bench_test.go | 2 +- sandboxd/pool/volume_capture_test.go | 9 - sandboxd/pool/volume_catalog_test.go | 618 ++++++-------------- sandboxd/pool/volume_convergence_test.go | 42 +- sandboxd/pool/volume_mutation_test.go | 11 +- sandboxd/pool/volume_retirement_test.go | 391 +++++-------- sandboxd/pool/volume_static_acl_test.go | 42 -- sandboxd/server/server_test.go | 11 +- sandboxd/server/volume_registration_test.go | 164 ++---- 22 files changed, 492 insertions(+), 1283 deletions(-) delete mode 100644 sandboxd/pool/volume_static_acl_test.go diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go index 08c459fc..3858f3e5 100644 --- a/e2e/volume_catalog_test.go +++ b/e2e/volume_catalog_test.go @@ -11,102 +11,64 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) -func TestRuntimeVolumePublicAdmissionThroughHTTP(t *testing.T) { - st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) - acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) - if err != nil { - t.Fatal(err) - } - request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "shared bytes"), "writable": true}) - if err != nil { - t.Fatal(err) - } - for _, scope := range []struct { - token string - want int - }{{"acme-token", http.StatusForbidden}, {"", http.StatusUnauthorized}, {"node-token", http.StatusNoContent}} { - caller := *st - caller.token = scope.token - if status, body := rawJSON(t, &caller, http.MethodPut, "/v1/volumes/home", string(request)); status != scope.want { - t.Fatalf("registration auth: status=%d want=%d body=%s", status, scope.want, body) - } - } - volumes, err := acme.Volumes(t.Context()) - if err != nil || len(volumes) != 1 || volumes[0].Name != "home" { - t.Fatalf("empty runtime ACL is public: volumes=%v err=%v", volumes, err) - } - volume := sandbox.Volume{Name: "home", Mount: "/mnt/home"} - first, err := acme.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volume)) - if err != nil { - t.Fatalf("tenant claim with public runtime volume: %v", err) - } - defer first.Close() - if !slices.Equal(first.Volumes, []sandbox.Volume{volume}) { - t.Fatalf("claim did not retain mounted volume: %+v", first.Volumes) - } - if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { - t.Fatal(err) - } - if view, attachErr := first.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { - t.Fatalf("tenant PATCH of public runtime volume: view=%+v err=%v", view, attachErr) - } - if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { - t.Fatal(err) - } - if view, attachErr := first.SetVolumes(t.Context(), []sandbox.Volume{volume}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { - t.Fatalf("tenant PUT of public runtime volume: view=%+v err=%v", view, attachErr) - } -} - -func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { +func TestRuntimeVolumeRegistrationThroughHTTP(t *testing.T) { st := startTenantStack(t, "node-token", []tenants.TenantSpec{ {Name: "acme", Token: "acme-token"}, {Name: "beta", Token: "beta-token"}, }, []config.VolumeSpec{{Name: "public", Path: writeVolumeImage(t, "public.img", "shared")}}) - request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}) - if err != nil { - t.Fatal(err) - } - if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/private", string(request)); status != http.StatusNoContent { - t.Fatalf("registration: %d %s", status, body) + for name, body := range map[string]map[string]any{ + "home": {"path": writeVolumeImage(t, "home.img", "shared"), "writable": true}, + "private": {"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}, + } { + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + if status, resp := rawJSON(t, st, http.MethodPut, "/v1/volumes/"+name, string(raw)); status != http.StatusNoContent { + t.Fatalf("register %s: status=%d body=%s", name, status, resp) + } } + home := sandbox.Volume{Name: "home", Mount: "/mnt/home"} for _, tc := range []struct { token string want []string - allowed bool - }{{"acme-token", []string{"private", "public"}, true}, {"beta-token", []string{"public"}, false}} { - client, connectErr := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) - if connectErr != nil { - t.Fatal(connectErr) + private bool + }{{"acme-token", []string{"home", "private", "public"}, true}, {"beta-token", []string{"home", "public"}, false}} { + client, err := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) + if err != nil { + t.Fatal(err) } - volumes, listErr := client.Volumes(t.Context()) - var names []string + volumes, err := client.Volumes(t.Context()) + names := make([]string, 0, len(volumes)) for _, volume := range volumes { names = append(names, volume.Name) } - if listErr != nil || !slices.Equal(names, tc.want) { - t.Fatalf("catalog ACL: names=%v want=%v err=%v", names, tc.want, listErr) + if err != nil || !slices.Equal(names, tc.want) { + t.Fatalf("catalog for %s: names=%v want=%v err=%v", tc.token, names, tc.want, err) } - claim, claimErr := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(sandbox.Volume{Name: "public"})) - if claimErr != nil { - t.Fatal(claimErr) + claim, err := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(home)) + if err != nil || !slices.Equal(claim.Volumes, []sandbox.Volume{home}) { + t.Fatalf("claim with public runtime volume: %+v, %v", claim, err) } defer claim.Close() - _, attachErr := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{{Name: "private"}}}) - if (attachErr == nil) != tc.allowed { - t.Fatalf("explicit ACL admission for %s: %v", tc.token, attachErr) + if _, err = claim.SetVolumes(t.Context(), []sandbox.Volume{}); err != nil { + t.Fatal(err) + } + view, err := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{home, {Name: "private"}}}) + if (err == nil) != tc.private || tc.private && !slices.Contains(view.Volumes, home) { + t.Fatalf("live attach for %s: view=%+v err=%v", tc.token, view, err) } } caller := *st caller.token = "beta-token" - var responses [][]byte + var responses []string for _, name := range []string{"private", "unknown"} { status, body := rawJSON(t, &caller, http.MethodPost, "/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"`+name+`"}]}`) if status != http.StatusBadRequest { t.Fatalf("claim privacy for %s: status=%d body=%s", name, status, body) } - responses = append(responses, body) + responses = append(responses, string(body)) } - if string(responses[0]) != string(responses[1]) { + if responses[0] != responses[1] { t.Fatalf("forbidden and unknown responses differ: %s versus %s", responses[0], responses[1]) } } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index ce5d5766..144c378a 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -476,22 +476,6 @@ func TestValidateVolumeCatalogTenantNames(t *testing.T) { } } -func TestLoadPreservesRestrictedACLWithInvalidVolumeTenants(t *testing.T) { - for _, tenants := range []string{`["acme", "has space", "", "beta"]`, `["has space"]`} { - cfg, err := Load(writeConfig(t, `{"pools":[],"volumes":[{"name":"data","path":"/images/data","tenants":`+tenants+`}]}`)) - if err != nil { - t.Fatal(err) - } - want := []string{"has space"} - if strings.Contains(tenants, "acme") { - want = []string{"acme", "has space", "", "beta"} - } - if !slices.Equal(cfg.Volumes[0].Tenants, want) { - t.Fatalf("tenants = %v, want %v", cfg.Volumes[0].Tenants, want) - } - } -} - func writeConfig(t *testing.T, body string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.json") diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 5cd3aa11..59064b8b 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -155,8 +155,7 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_mine": {ID: "sb_mine", VMName: "sbx-gone-1", Key: testKey, Token: "tok"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) diff --git a/sandboxd/pool/checkpoint_bench_test.go b/sandboxd/pool/checkpoint_bench_test.go index 997f0e77..e3ba1b6c 100644 --- a/sandboxd/pool/checkpoint_bench_test.go +++ b/sandboxd/pool/checkpoint_bench_test.go @@ -15,7 +15,7 @@ func BenchmarkClaimCommitWithCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { claims := checkpointBenchClaims(checkpoints) s := newClaimStore(b.TempDir(), false) - if err := s.commit(s.reset(claims)); err != nil { + if err := s.save(claims); err != nil { b.Fatal(err) } one := claims["sb_0000000000000000"] @@ -39,7 +39,7 @@ func BenchmarkPinnedArchiveCheckpoints(b *testing.B) { b.Run(fmt.Sprintf("claims=5000/checkpoints=%t", checkpoints), func(b *testing.B) { m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.commit(m.store.reset(m.claimed)); err != nil { + if err := m.store.save(m.claimed); err != nil { b.Fatal(err) } want := 0 diff --git a/sandboxd/pool/checkpoint_references_test.go b/sandboxd/pool/checkpoint_references_test.go index 94601f8d..c3ec0ae3 100644 --- a/sandboxd/pool/checkpoint_references_test.go +++ b/sandboxd/pool/checkpoint_references_test.go @@ -6,7 +6,6 @@ import ( "io/fs" "maps" "os" - "path/filepath" "slices" "sync" "syscall" @@ -15,11 +14,17 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +const ( + failWrite = iota + captureWrite + failAfterRename +) + func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { recreate := func(s *claimStore) { s.set(&types.Sandbox{ID: "owner", ReplicaCk: "recreated"}) } for _, tc := range []struct { name string - fault func(*testing.T, *claimStore, claimSnapshot) func() + fault int acked, next *types.Sandbox startup func(claims map[string]*types.Sandbox) mutate func(s *claimStore) @@ -82,8 +87,11 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + if tc.fault == failAfterRename && os.Geteuid() == 0 { + t.Skip("requires non-root: root can read the write-only directory") + } dir := t.TempDir() - s := newClaimStore(dir, false) + s := newClaimStore(dir, tc.fault == failAfterRename) if tc.acked != nil { if err := s.commit(s.set(tc.acked)); err != nil { t.Fatal(err) @@ -101,7 +109,26 @@ func TestClaimCheckpointReferencesOutliveUnacknowledgedWrites(t *testing.T) { } else { snap = s.set(tc.next) } - finish := tc.fault(t, s, snap) + var finish func() + switch tc.fault { + case failWrite: + finish = failClaimWrites(t, s) + if err := s.commit(snap); err == nil { + t.Fatal("write succeeded under the fault") + } + case failAfterRename: + if err := os.Chmod(dir, 0o300); err != nil { + t.Fatal(err) + } + finish = func() { _ = os.Chmod(dir, 0o700) } + t.Cleanup(finish) + if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { + t.Fatalf("want a failed directory sync after the rename: %v", err) + } + case captureWrite: + release := captureClaimWrite(t, s, snap, "owner") + finish = func() { release() } + } if tc.mutate != nil { tc.mutate(s) } @@ -188,40 +215,3 @@ func captureClaimWrite(t *testing.T, s *claimStore, snap claimSnapshot, owner st return raw } } - -func failWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { - t.Helper() - restore := failClaimWrites(t, s) - if err := s.commit(snap); err == nil { - t.Fatal("write succeeded under the fault") - } - return restore -} - -func failAfterRename(t *testing.T, s *claimStore, snap claimSnapshot) func() { - t.Helper() - if os.Geteuid() == 0 { - t.Skip("requires non-root: root can read the write-only directory") - } - s.sync = true - dir := filepath.Dir(s.path) - if err := os.Chmod(dir, 0o300); err != nil { - t.Fatal(err) - } - restore := func() { - if err := os.Chmod(dir, 0o700); err != nil { - t.Error(err) - } - } - t.Cleanup(restore) - if err := s.commit(snap); !errors.Is(err, fs.ErrPermission) { - t.Fatalf("want a failed directory sync after the rename: %v", err) - } - return restore -} - -func captureWrite(t *testing.T, s *claimStore, snap claimSnapshot) func() { - t.Helper() - release := captureClaimWrite(t, s, snap, "owner") - return func() { release() } -} diff --git a/sandboxd/pool/claims_bench_test.go b/sandboxd/pool/claims_bench_test.go index d1f059bc..b1ae79da 100644 --- a/sandboxd/pool/claims_bench_test.go +++ b/sandboxd/pool/claims_bench_test.go @@ -18,7 +18,7 @@ func BenchmarkStoreSaveScaling(b *testing.B) { st := newClaimStore(b.TempDir(), false) claims := benchClaims(n) for b.Loop() { - if err := st.commit(st.reset(claims)); err != nil { + if err := st.save(claims); err != nil { b.Fatalf("save: %v", err) } } @@ -48,7 +48,7 @@ func BenchmarkStoreCommitEnv(b *testing.B) { sb.Env = envs[arm] } s := newClaimStore(b.TempDir(), durable) - if err := s.commit(s.reset(claims)); err != nil { + if err := s.save(claims); err != nil { b.Fatalf("save: %v", err) } one := claims[fmt.Sprintf("sb_%016x", 0)] @@ -105,7 +105,7 @@ func benchPersistContention(b *testing.B, claims map[string]*types.Sandbox, arm switch arm { case "under-lock": mu.Lock() - _ = s.commit(s.reset(claims)) + _ = s.save(claims) mu.Unlock() case "rebuild": mu.Lock() diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index d369f829..8ddae7aa 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -85,7 +85,7 @@ func TestStoreRoundTrip(t *testing.T) { } for _, fsync := range []bool{false, true} { s := newClaimStore(t.TempDir(), fsync) - if err := s.commit(s.reset(claims)); err != nil { + if err := s.save(claims); err != nil { t.Fatalf("save (sync=%v): %v", fsync, err) } loadedStore := newClaimStore(filepath.Dir(s.path), false) @@ -239,3 +239,7 @@ func TestClaimStoreConcurrentCommits(t *testing.T) { t.Fatalf("after concurrent commits: %+v, %v", got, err) } } + +func (s *claimStore) save(claims map[string]*types.Sandbox) error { + return s.commit(s.reset(claims)) +} diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index dd37a6bc..505dc55c 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -272,8 +272,7 @@ func TestFailoverRestartReclaimsAnInterruptedArchive(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - claimStore := newClaimStore(dirA, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dirA, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -296,8 +295,7 @@ func TestFailoverRestartReclaimSparesACheckpointAdoptedMidReconcile(t *testing.T claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - claimStore := newClaimStore(dirA, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dirA, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) @@ -322,8 +320,7 @@ func TestFailoverRestartReclaimsOnlyTheCheckpointsItPublished(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, } - claimStore := newClaimStore(dirA, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dirA, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } a := c.node(t, "a", dirA, keepAnHour) diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index 7642b11a..ffc4dd9e 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -528,8 +528,7 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { VsockSocket: "/vsock/lagged", PendingSnap: hibernatePrefix + "lagged-1-abc123", }, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -558,8 +557,7 @@ func TestReconcileIgnoresStaleOrphanSnapshot(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_orp": {ID: "sb_orp", VMName: "sbx-orphan-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/orphan"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -585,8 +583,7 @@ func TestReconcileKeepsIntentWithoutImageAsRunning(t *testing.T) { VsockSocket: "/vsock/noimg", PendingSnap: hibernatePrefix + "noimg-1-abc123", }, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir) @@ -616,8 +613,7 @@ func TestReconcileAdoptsHibernated(t *testing.T) { HibernateSnap: hibernatePrefix + "hibernated-1", }, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index ac557fb2..2864c615 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -486,8 +486,7 @@ func TestReconcileKeepsRunningClaimsTheHostLost(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_lost": {ID: "sb_lost", VMName: "sbx-lost-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/lost"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } cfg := &config.Config{DataDir: dataDir, VMMRestart: tt.policy} diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index a372a55a..b29c4e86 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -323,8 +323,7 @@ func TestReconcile(t *testing.T) { "sb_live": {ID: "sb_live", VMName: "sbx-live-1", Key: testKey, Token: "tok", VsockSocket: "/vsock/live-old"}, "sb_dead": {ID: "sb_dead", VMName: "sbx-dead-1", Key: testKey, Token: "tok"}, } - claimStore := newClaimStore(dataDir, false) - if err := claimStore.commit(claimStore.reset(claims)); err != nil { + if err := newClaimStore(dataDir, false).save(claims); err != nil { t.Fatalf("setup: %v", err) } m := newTestManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index b56103dc..f7bd658f 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -4,7 +4,6 @@ import ( "encoding/json/v2" "errors" "net" - "os" "testing" "testing/synctest" "time" @@ -1015,34 +1014,20 @@ func TestReplicaScanWaitsForDetachAcknowledgment(t *testing.T) { t.Fatal(err) } } - m.store.recommitting.Store(true) eng := m.eng.(*fakeEngine) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - view, err := m.SetVolumes(t.Context(), sb.ID, "", nil) - eng.liveVolumeHook = nil - if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("detach did not fail after device removal: %+v, %v", view, err) - } + unblock := detachWithoutCommit(t, m, eng, sb.ID, "") m.replicateOnce(t.Context()) if m.replicating.Load() || !m.replicas[sb.ID].skipLogged || !m.replicas[attached.ID].skipLogged || m.replicas[ordinary.ID].tried.IsZero() || m.replicas[ordinary.ID].skipLogged { t.Fatalf("scan failed to distinguish detached, attached, and ordinary claims: %+v", m.replicas) } - if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("capture ignored retained attachment: %v", err) } if sb.HasReplica() || len(eng.snapSaves) != 0 { t.Fatal("pending detach published a replica") } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) - } - if err = m.store.commit(m.store.mark()); err != nil { + unblock() + if err := m.store.commit(m.store.mark()); err != nil { t.Fatal(err) } m.replicateOnce(t.Context()) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index a7b30825..f4136667 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -1,205 +1,55 @@ package pool import ( - "errors" + "fmt" "os" - "slices" + "path/filepath" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestRuntimeCatalogDefaultACL(t *testing.T) { - for _, tenants := range [][]string{nil, {}} { - t.Run("empty", func(t *testing.T) { - eng := newFakeEngine() - catalog := liveCatalog(t) - m, _ := newTenantVolumeManager(t, eng, catalog) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: tenants} - if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { - t.Fatal(err) - } - for _, current := range []*Manager{m, restartACLManager(t, eng, m.dataDir, catalog)} { - if !slices.ContainsFunc(current.Volumes("acme", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { - t.Error("public runtime volume is hidden from a tenant") - } - if !slices.ContainsFunc(current.Volumes("", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { - t.Error("operator cannot see runtime volume") - } - if _, err := current.VolumePlacement(testKey, "acme", []string{"home"}); err != nil { - t.Errorf("tenant placement: %v", err) - } - if local, err := current.VolumePlacement(testKey, "", []string{"home"}); err != nil || !local { - t.Errorf("operator placement: %v %v", local, err) - } - tenantClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) - if err != nil { - t.Fatalf("tenant initial attachment: %v", err) - } - if err = current.Release(t.Context(), tenantClaim.ID, Cred{Token: tenantClaim.Token}); err != nil { - t.Fatal(err) - } - if _, err = current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { - t.Errorf("public static volume changed: %v", err) - } - rootClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) - if err != nil { - t.Fatalf("operator initial attachment: %v", err) - } - if err := current.Release(t.Context(), rootClaim.ID, Cred{Token: rootClaim.Token}); err != nil { - t.Fatalf("operator initial claim cleanup: %v", err) - } - } - }) - } -} - -func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { - m, sb := newTenantVolumeManager(t, newFakeEngine(), nil) - if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Tenants: []string{"other"}}); err != nil { +func TestRuntimeCatalogEmptyACLIsPublic(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{}} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "home"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("operator bypassed explicit claim-owner ACL: %v", err) - } -} - -func TestRuntimeCatalogPendingWritePreservesPrivateNameMasking(t *testing.T) { - for _, operation := range []string{"register", "unregister"} { - t.Run(operation, func(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "private", "bytes"), Tenants: []string{"other"}} - if operation == "unregister" { - if err := m.RegisterVolume(t.Context(), "private", spec); err != nil { - t.Fatal(err) - } - } - blockCatalogRename(t, m) - var err error - if operation == "unregister" { - err = m.UnregisterVolume(t.Context(), "private") - } else { - err = m.RegisterVolume(t.Context(), "private", spec) - } - if !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("obstructed catalog write: %v", err) - } - for _, requested := range [][]types.Volume{ - {{Name: "private"}}, - {{Name: "data"}, {Name: "private"}}, - {{Name: "private"}, {Name: "data"}}, - } { - missing := slices.Clone(requested) - for i := range missing { - if missing[i].Name == "private" { - missing[i].Name = "missing" - } - } - _, deniedErr := m.SetVolumes(t.Context(), sb.ID, "acme", requested) - _, missingErr := m.SetVolumes(t.Context(), sb.ID, "acme", missing) - if !errors.Is(deniedErr, ErrVolumeUnavailable) || !errors.Is(missingErr, ErrVolumeUnavailable) || deniedErr.Error() != missingErr.Error() { - t.Errorf("private name distinguishable for %+v: denied=%v missing=%v", requested, deniedErr, missingErr) - } - } - if view, err := m.GetVolumes(sb.ID, "acme"); err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("refused request changed claim volumes: %+v, %v", view, err) - } - if len(eng.volumeSpecs) != 0 { - t.Fatal("refused request reached device attachment") - } - for _, name := range []string{"data", "private", "missing"} { - if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { - t.Fatalf("refused request reserved %s: %+v", name, got) - } - } - }) + for _, current := range []*Manager{m, newVolumeManagerAt(t, eng, m.dataDir, nil)} { + if got := current.Volumes("acme", nil); len(got) != 1 || got[0].Name != "home" { + t.Fatalf("public runtime volume hidden from a tenant: %+v", got) + } + if local, err := current.VolumePlacement(testKey, "acme", []string{"home"}); err != nil || !local { + t.Fatalf("tenant placement: %v %v", local, err) + } + sb, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mount: "/home"}}}) + if err != nil { + t.Fatalf("tenant attachment: %v", err) + } + if err = current.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } } } -func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { - for _, operation := range []string{"put", "patch"} { - t.Run(operation, func(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, nil) - if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}}); err != nil { - t.Fatal(err) - } - want := []types.Volume{{Name: "home", Mount: "/home"}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { - t.Fatal(err) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { - t.Fatalf("still-held retry: %v", err) - } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - view, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil) - eng.liveVolumeHook = nil - if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("detached result: %+v, %v", view, err) - } - claims, err := newClaimStore(m.dataDir, false).load() - if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { - t.Fatalf("old durable attachment missing: %+v, %v", claims, err) - } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("recovery reference lost writer exclusion: %v", err) - } - if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("capture ignored recovery reference: %v", err) - } - if err = m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("unregister ignored recovery reference: %v", err) - } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) - } - m.mu.Lock() - entry := *m.catalog.entries["home"] - entry.tenants = []string{"other"} - m.catalog.entries["home"] = &entry - m.mu.Unlock() - before := len(eng.volumeSpecs) - switch operation { - case "put": - _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want) - case "patch": - _, err = m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) - } - if !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("recovery-only reference authorized tenant attachment: %v", err) - } - if len(eng.volumeSpecs) != before { - t.Fatal("rejected tenant reached device attachment") - } - m.mu.Lock() - entry.tenants = []string{"acme"} - m.catalog.entries["home"] = &entry - m.mu.Unlock() - if _, err = m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { - t.Fatalf("operator reattach: %v", err) - } - if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { - t.Fatalf("tenant cleanup after operator reattach: %v", err) - } - }) +func TestInvalidStaticTenantListNeverBecomesPublic(t *testing.T) { + path := writeVolumeImage(t, "data", "private contents") + file := filepath.Join(t.TempDir(), "config.json") + raw := fmt.Sprintf(`{"pools":[],"volumes":[{"name":"data","path":%q,"tenants":["has space"]}]}`, path) + if err := os.WriteFile(file, []byte(raw), 0o600); err != nil { + t.Fatal(err) } -} - -func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { - t.Helper() - cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} - m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + cfg, err := config.Load(file) if err != nil { - t.Fatal(err) + t.Fatalf("legacy config stopped booting: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), cfg.Volumes) + if got := m.Volumes("acme", nil); len(got) != 0 { + t.Fatalf("invalid restricted ACL became public: %+v", got) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: path + ".other"}); err != nil { + t.Fatalf("invalid legacy ACL blocked unrelated registration: %v", err) } - return m } diff --git a/sandboxd/pool/volume_bench_test.go b/sandboxd/pool/volume_bench_test.go index e717d96f..a7c75e0b 100644 --- a/sandboxd/pool/volume_bench_test.go +++ b/sandboxd/pool/volume_bench_test.go @@ -69,7 +69,7 @@ func volumeBenchManager(b *testing.B, checkpoints bool) *Manager { b.Helper() m := newTestManager(b, newFakeEngine(), config.PoolSpec{PoolKey: testKey}) m.claimed = checkpointBenchClaims(checkpoints) - if err := m.store.commit(m.store.reset(m.claimed)); err != nil { + if err := m.store.save(m.claimed); err != nil { b.Fatal(err) } path := filepath.Join(b.TempDir(), "data.raw") diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index e0d0b8f4..27c1b1dc 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -15,10 +15,7 @@ func TestVolumeClaimRejectsDirectCapture(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - m.store.set(sb) - m.mu.Unlock() tests := []struct { name string @@ -54,10 +51,7 @@ func TestVolumeClaimRunningWakeUnchanged(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - m.store.set(sb) - m.mu.Unlock() sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { @@ -129,10 +123,7 @@ func TestIdleOnceSkipsVolumeClaim(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, IdleHibernateSeconds: 1}) sb := mustClaim(t, m, testKey) - m.mu.Lock() sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - m.store.set(sb) - m.mu.Unlock() backdate(m, sb, 2*time.Second) sweepIdle(t, m) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index de250141..f3281b35 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -21,40 +21,52 @@ func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { m := newVolumeManager(t, eng, nil) path := writeVolumeImage(t, "runtime", "backing bytes stay intact") spec := types.VolumeRegistration{Path: filepath.Dir(path) + "/./runtime", Writable: true, Tenants: []string{"beta", "acme", "acme"}} + before := m.catalogSnapshot() if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { t.Fatal(err) } - spec.Path = path - spec.DirectIO = types.DirectIOOff - spec.Tenants = []string{"acme", "beta"} - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + if before["runtime"] != nil || spec.Tenants[0] != "beta" { + t.Fatal("registration mutated a captured catalog or the caller's spec") + } + normalized := types.VolumeRegistration{Path: path, DirectIO: types.DirectIOOff, Writable: true, Tenants: []string{"acme", "beta"}} + if err := m.RegisterVolume(t.Context(), "runtime", normalized); err != nil { t.Fatalf("normalized retry: %v", err) } - st, statErr := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) - if statErr != nil || st.Mode().Perm() != 0o600 { - t.Fatalf("metadata mode: %v %v", st, statErr) + if st, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); err != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, err) + } + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "runtime"}}}) + if err != nil { + t.Fatal(err) } m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) - if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("recovered names: %v", got) + if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "runtime" || !got[0].Available || !got[0].Writable { + t.Fatalf("catalog absent before reconcile: %+v", got) + } + if got := m2.Volumes("forbidden", nil); len(got) != 0 { + t.Fatalf("ACL changed at restart: %+v", got) } - if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "forbidden", Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("ACL changed at restart: %v", err) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) } - resolved, err := m2.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}) - if err != nil || len(resolved) != 1 || resolved[0].disk.DirectIO != "off" || !resolved[0].disk.RW { - t.Fatalf("policy roundtrip: %+v %v", resolved, err) + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovered holder missing: %v", err) } - changed := spec + changed := normalized changed.DirectIO = "on" - if err := m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { + if err = m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { t.Fatalf("changed semantics: %v", err) } - if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + if err = m2.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister of a held volume: %v", err) + } + if err = m2.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } - if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatalf("absent retry: %v", err) + for range 2 { + if err = m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } } if got, err := os.ReadFile(path); err != nil || string(got) != "backing bytes stay intact" { t.Fatalf("backing changed: %q %v", got, err) @@ -64,253 +76,205 @@ func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { } } -func TestRuntimeCatalogLoadsOperatorManagedRecord(t *testing.T) { - for _, setup := range []struct { - name string - prepare func(*testing.T, string) - }{ - {"mode", func(t *testing.T, record string) { - if err := os.Chmod(record, 0o644); err != nil { - t.Fatal(err) - } - }}, - {"symlink", func(t *testing.T, record string) { - if err := os.Rename(record, record+".target"); err != nil { - t.Fatal(err) - } - if err := os.Symlink(record+".target", record); err != nil { - t.Fatal(err) - } - }}, - } { - t.Run(setup.name, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { - t.Fatal(err) - } - setup.prepare(t, filepath.Join(m.dataDir, runtimeVolumesFile)) - restarted := newVolumeManagerAt(t, eng, m.dataDir, nil) - if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("operator-managed record not loaded: %v", got) - } - if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("replacement not loaded: %v", got) - } - }) - } -} - -func TestRuntimeCatalogUsesStaticPathValidation(t *testing.T) { - regular := writeVolumeImage(t, "backing", "unchanged") +func TestRuntimeCatalogRegistersUncheckedBacking(t *testing.T) { link := filepath.Join(t.TempDir(), "backing-link") - if err := os.Symlink(regular, link); err != nil { + if err := os.Symlink(writeVolumeImage(t, "backing", "bytes"), link); err != nil { t.Fatal(err) } - for _, path := range []string{regular, link, os.DevNull, t.TempDir(), filepath.Join(t.TempDir(), "missing")} { - t.Run(filepath.Base(path), func(t *testing.T) { - spec := config.VolumeSpec{Name: "runtime", Path: path, DirectIO: types.DirectIOOff, Writable: true} - if err := config.ValidateVolumeCatalog([]config.VolumeSpec{spec}); err != nil { - t.Fatalf("static validation: %v", err) - } + for _, tc := range []struct { + name, path string + wantErr error + }{{"symlink", link, nil}, {"missing", filepath.Join(t.TempDir(), "missing"), ErrVolumeUnavailable}} { + t.Run(tc.name, func(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), nil) - registration := types.VolumeRegistration{Path: path, Writable: true} + registration := types.VolumeRegistration{Path: tc.path, Writable: true} if err := m.RegisterVolume(t.Context(), "runtime", registration); err != nil { - t.Fatalf("runtime validation differs from static: %v", err) + t.Fatalf("register: %v", err) } m2 := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) if err := m2.RegisterVolume(t.Context(), "runtime", registration); err != nil { t.Fatalf("idempotent retry after restart: %v", err) } - if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { - if _, resolveErr := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(resolveErr, ErrVolumeUnavailable) { - t.Fatalf("missing backing resolved: %v", resolveErr) - } + if infos := m2.Volumes("", nil); len(infos) != 1 || infos[0].Available != (tc.wantErr == nil) { + t.Fatalf("availability: %+v", infos) + } + if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, tc.wantErr) { + t.Fatalf("resolve = %v, want %v", err, tc.wantErr) } if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { t.Fatal(err) } }) } - if got, err := os.ReadFile(regular); err != nil || string(got) != "unchanged" { - t.Fatalf("registration touched backing: %q %v", got, err) - } } -func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { - eng := newFakeEngine() +func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { static := liveCatalog(t) - m := newVolumeManager(t, eng, static) + m := newVolumeManager(t, newFakeEngine(), static) other := writeVolumeImage(t, "other-runtime", "bytes") - for _, tc := range []struct{ name, path string }{{"data", other}, {"runtime", static[0].Path}, {"runtime", filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path)}} { - if err := m.RegisterVolume(t.Context(), tc.name, types.VolumeRegistration{Path: tc.path}); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("collision %+v: %v", tc, err) - } - } - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("static removal: %v", err) - } sb := mustClaim(t, m, testKey) sb.Volumes = []types.Volume{{Name: "unexplained", Mount: "/data"}} if err := m.store.commit(m.store.set(sb)); err != nil { t.Fatal(err) } - if err := m.RegisterVolume(t.Context(), "unexplained", types.VolumeRegistration{Path: other}); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("rebound unexplained recovery name: %v", err) - } - if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("absent referenced removal: %v", err) + alias := filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path) + for _, tc := range []struct { + name string + spec types.VolumeRegistration + want error + }{ + {"data", types.VolumeRegistration{Path: other}, ErrVolumeRegistrationConflict}, + {"runtime", types.VolumeRegistration{Path: static[0].Path}, ErrVolumeRegistrationConflict}, + {"runtime", types.VolumeRegistration{Path: alias}, ErrVolumeRegistrationConflict}, + {"unexplained", types.VolumeRegistration{Path: other}, ErrVolumeRegistrationConflict}, + {"cocoon-reserved", types.VolumeRegistration{Path: other}, ErrBadVolume}, + {"bad/name", types.VolumeRegistration{Path: other}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: "relative"}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: other, DirectIO: "invalid"}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: other, Tenants: []string{"has space"}}, ErrBadVolume}, + } { + if err := m.RegisterVolume(t.Context(), tc.name, tc.spec); !errors.Is(err, tc.want) { + t.Errorf("register %s %+v = %v, want %v", tc.name, tc.spec, err, tc.want) + } } - for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}} { - if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { - t.Fatalf("invalid spec %+v: %v", spec, err) + for _, name := range []string{"data", "unexplained"} { + if err := m.UnregisterVolume(t.Context(), name); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Errorf("unregister %s: %v", name, err) } } } -func TestRuntimeCatalogWriteFailureFencesAndConverges(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - m.store.path = filepath.Join(t.TempDir(), "claims.json") - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} +func TestRuntimeCatalogPendingWriteConverges(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + canceled, cancel := context.WithCancel(t.Context()) + cancel() + if err := m.RegisterVolume(canceled, "runtime", spec); !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("cancellation before acceptance: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled proposal left catalog: %v", err) + } restore := blockCatalogRename(t, m) if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("write failure: %v", err) } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("unpublished admission: %v", err) + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("failed retry cleared pending write: %v", err) } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatalf("volume-free claim blocked: %v", err) + m.retryVolumeCatalog(t.Context()) + for _, volumes := range [][]types.Volume{nil, {{Name: "data"}}} { + if err := claimAndRelease(t, m, volumes...); err != nil { + t.Fatalf("pending write blocked unrelated claim %v: %v", volumes, err) + } } restore() - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatalf("retry pending register: %v", err) + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("possible replacement not recovered: %v", got) } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("durable registration: %v", got) + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("unpublished admission: %v", err) + } + if err := m.UnregisterVolume(canceled, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("canceled retry cleared pending write: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if !slices.Contains(m.VolumeNames(), "runtime") { + t.Fatal("retry lost accepted registration") } restore = blockCatalogRename(t, m) if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("failed unregister: %v", err) } restore() + installPendingCatalogRecord(t, m) + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("possible unregister replacement not recovered: %v", got) + } if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { t.Fatalf("pending unregister did not converge before reuse: %v", err) } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"replacement"}) { + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"replacement"}) { t.Fatalf("durable reuse: %v", got) } } -func TestRuntimeCatalogWriteFailureOnlyFencesAffectedName(t *testing.T) { - for _, unregister := range []bool{false, true} { - t.Run(fmt.Sprintf("unregister=%t", unregister), func(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - for _, name := range []string{"healthy", "affected"} { - if name == "affected" && !unregister { - continue - } - if err := m.RegisterVolume(t.Context(), name, types.VolumeRegistration{Path: writeVolumeImage(t, name, "bytes")}); err != nil { - t.Fatal(err) - } +func TestRuntimeCatalogPendingWriteFencesOnlyAffectedName(t *testing.T) { + unregister := func(m *Manager, _ types.VolumeRegistration) error { return m.UnregisterVolume(t.Context(), "affected") } + register := func(m *Manager, spec types.VolumeRegistration) error { + return m.RegisterVolume(t.Context(), "affected", spec) + } + for _, tc := range []struct { + name string + before []string + write func(*Manager, types.VolumeRegistration) error + resolved []types.Volume + wantResolved, want error + }{ + {"register", []string{"healthy"}, register, []types.Volume{{Name: "healthy"}}, nil, ErrVolumeUnavailable}, + {"unregister", []string{"healthy", "affected"}, unregister, []types.Volume{{Name: "healthy"}, {Name: "affected"}}, ErrVolumeCatalogUnavailable, ErrVolumeCatalogUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + specs := map[string]types.VolumeRegistration{ + "healthy": {Path: writeVolumeImage(t, "healthy", "bytes")}, + "affected": {Path: writeVolumeImage(t, "affected", "bytes"), Tenants: []string{"other"}}, } - sb := mustClaim(t, m, testKey) - var before []resolvedVolume - if unregister { - var err error - before, err = m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "healthy"}, {Name: "affected"}}) - if err != nil { + for _, name := range tc.before { + if err := m.RegisterVolume(t.Context(), name, specs[name]); err != nil { t.Fatal(err) } } - restore := blockCatalogRename(t, m) - defer restore() - var err error - if unregister { - err = m.UnregisterVolume(t.Context(), "affected") - } else { - err = m.RegisterVolume(t.Context(), "affected", types.VolumeRegistration{Path: writeVolumeImage(t, "affected", "bytes")}) + resolved, err := m.resolveVolumes(t.Context(), "", tc.resolved) + if err != nil { + t.Fatal(err) } - if !errors.Is(err, ErrVolumeCatalogUnavailable) { + blockCatalogRename(t, m) + if err = tc.write(m, specs["affected"]); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("obstructed catalog write: %v", err) } - if unregister { - if _, err := m.admitVolumes("", before); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("resolved mixed admission: %v", err) - } - } - wantErr := ErrVolumeUnavailable - if unregister { - wantErr = ErrVolumeCatalogUnavailable + holds, err := m.admitVolumes("", resolved) + m.unreserveVolumes(holds) + if !errors.Is(err, tc.wantResolved) { + t.Fatalf("admission resolved before the write = %v, want %v", err, tc.wantResolved) } for _, volumes := range [][]types.Volume{{{Name: "affected"}}, {{Name: "healthy"}, {Name: "affected"}}} { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, wantErr) { - t.Fatalf("affected claim: %v", err) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", volumes); !errors.Is(err, wantErr) { - t.Fatalf("affected live addition: %v", err) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: volumes}); !errors.Is(err, tc.want) { + t.Errorf("root claim %v = %v, want %v", volumes, err, tc.want) } } - for _, name := range []string{"healthy", "affected"} { + _, deniedErr := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "affected"}}) + _, missingErr := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "missing"}}) + if !errors.Is(deniedErr, ErrVolumeUnavailable) || !errors.Is(missingErr, ErrVolumeUnavailable) || deniedErr.Error() != missingErr.Error() { + t.Errorf("private name distinguishable: denied=%v missing=%v", deniedErr, missingErr) + } + if len(eng.volumeSpecs) != 0 { + t.Fatal("refused request reached device attachment") + } + for _, name := range []string{"data", "healthy", "affected", "missing"} { if got := volumeHoldersOf(m, name); got != (volumeHolders{}) { t.Fatalf("refused request reserved %s: %+v", name, got) } } for _, name := range []string{"data", "healthy"} { - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: name}}}); err != nil { + if err := claimAndRelease(t, m, types.Volume{Name: name}); err != nil { t.Errorf("unrelated %s claim: %v", name, err) } } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}, {Name: "healthy"}}); err != nil { + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}, {Name: "healthy"}}); err != nil { t.Errorf("unrelated live addition: %v", err) } }) } } -func TestRuntimeCatalogPossibleReplacementFencesAndConverges(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true} - restore := blockCatalogRename(t, m) - if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("write failure: %v", err) - } - restore() - installPendingCatalogRecord(t, m) - if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("possible replacement not recovered: %v", got) - } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("unpublished admission: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatalf("volume-free claim blocked: %v", err) - } - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatalf("retry pending register: %v", err) - } - restore = blockCatalogRename(t, m) - if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("uncertain unregister: %v", err) - } - restore() - installPendingCatalogRecord(t, m) - if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("possible unregister replacement not recovered: %v", got) - } - if err := m.RegisterVolume(t.Context(), "replacement", spec); err != nil { - t.Fatalf("pending unregister did not converge before reuse: %v", err) - } -} - -func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { +func TestResolvedCatalogBindingRequiresSameEntry(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), nil) - path := writeVolumeImage(t, "runtime", "original") - spec := types.VolumeRegistration{Path: path, Writable: true} + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { t.Fatal(err) } @@ -318,66 +282,25 @@ func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { if err != nil { t.Fatal(err) } - if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - spec.Path = writeVolumeImage(t, "replacement", "replacement") - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { t.Fatal(err) } - if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("stale binding admitted: %v", err) - } - if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { - t.Fatalf("stale binding leaked reservation: %+v", got) - } -} - -func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - path := writeVolumeImage(t, "runtime", "bytes") - if os.Geteuid() == 0 { - if err := os.Chown(path, 65534, 65534); err != nil { - t.Fatal(err) - } - } - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { - t.Fatalf("nonroot backing owner: %v", err) + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatalf("unrelated registration invalidated resolution: %v", err) } - if err := os.Remove(path); err != nil { + m.unreserveVolumes(holds) + if err = m.UnregisterVolume(t.Context(), "runtime"); err != nil { t.Fatal(err) } - m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) - infos := m2.Volumes("", nil) - if len(infos) != 1 || infos[0].Available { - t.Fatalf("missing backing lost mapping: %+v", infos) - } - if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("missing backing resolved: %v", err) - } - if err := m2.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { - t.Fatalf("idempotent missing mapping: %v", err) - } -} - -func TestRuntimeRegistrationNormalization(t *testing.T) { - input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} - normalized, err := normalizeRegistration("data", input) - if err != nil || normalized.disk.Path != "/images/data" || normalized.disk.DirectIO != types.DirectIOOff || !slices.Equal(normalized.tenants, []string{"acme", "beta"}) { - t.Fatalf("normalization: %+v %v", normalized, err) + if err = m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) } - normalized.tenants[0] = "caller-copy" - if input.Tenants[1] != "acme" { - t.Fatal("normalization mutated caller input") + if _, err = m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("same-file reuse admitted stale resolution: %v", err) } - for _, tc := range []struct { - name string - spec types.VolumeRegistration - }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{""}}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{"has space"}}}} { - if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { - t.Fatalf("invalid registration %+v: %v", tc, err) - } + if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { + t.Fatalf("stale binding leaked reservation: %+v", got) } } @@ -410,205 +333,6 @@ func TestRuntimeCatalogRejectsInvalidMetadata(t *testing.T) { } } -func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, nil) - path := writeVolumeImage(t, "runtime", "unchanged") - spec := types.VolumeRegistration{Path: path, Writable: true} - if err := m.RegisterVolume(t.Context(), "data", spec); err != nil { - t.Fatal(err) - } - sb := mustClaim(t, m, testKey) - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { - t.Fatal("expected failed result commit") - } - eng.liveVolumeHook = nil - if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("unregister unretired name: %v", err) - } - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "different")}); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("rebind unretired name: %v", err) - } - if err := m.RegisterVolume(t.Context(), "same-file", spec); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Fatalf("alias unretired path: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("same-file writer: %v", err) - } - if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("capture: %v", err) - } - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { - t.Fatalf("hibernate before retirement: %v", err) - } - if err := os.Remove(tmp); err != nil { - t.Fatal(err) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { - t.Fatal(err) - } - if err := m.UnregisterVolume(t.Context(), "data"); err != nil { - t.Fatal(err) - } - if err := m.RegisterVolume(t.Context(), "same-file", spec); err != nil { - t.Fatal(err) - } -} - -func TestCatalogSnapshotSurvivesRegistration(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - before := m.catalogSnapshot() - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/images/runtime"}); err != nil { - t.Fatal(err) - } - if before["runtime"] != nil { - t.Fatal("registration mutated captured catalog") - } - captured := m.catalogSnapshot() - if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - if captured["runtime"] == nil { - t.Fatal("unregister mutated captured catalog") - } -} - -func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - ctx, cancel := context.WithCancel(t.Context()) - cancel() - err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) - if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("cancellation before acceptance: %v", err) - } - m.retryVolumeCatalog(t.Context()) - if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("canceled proposal left catalog: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { - t.Fatalf("canceled registration fenced static claim: %v", err) - } -} - -func TestRuntimeCatalogPendingWriteSurvivesFailedAndCanceledRetry(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) - restore := blockCatalogRename(t, m) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} - if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("initial write failure: %v", err) - } - if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("failed retry cleared pending write: %v", err) - } - m.retryVolumeCatalog(t.Context()) - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("unpublished admission: %v", err) - } - restore() - ctx, cancel := context.WithCancel(t.Context()) - cancel() - if err := m.UnregisterVolume(ctx, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { - t.Fatalf("canceled retry cleared pending write: %v", err) - } - m.retryVolumeCatalog(t.Context()) - if !slices.Contains(m.VolumeNames(), "runtime") { - t.Fatal("retry lost accepted registration") - } - if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatalf("repair did not restore static admission: %v", err) - } -} - -func TestResolvedCatalogBindingChangesOnlyForReusedName(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatal(err) - } - resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) - if err != nil { - t.Fatal(err) - } - if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { - t.Fatal(err) - } - holds, err := m.admitVolumes("", resolved) - if err != nil { - t.Fatalf("unrelated registration invalidated resolution: %v", err) - } - m.unreserveVolumes(holds) - if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatal(err) - } - if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { - t.Fatalf("same-file reuse admitted stale resolution: %v", err) - } -} - -func TestRuntimeCatalogIgnoresAbandonedTemporaryFiles(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - path := writeVolumeImage(t, "runtime", "bytes") - raw := fmt.Sprintf(`{"version":1,"volumes":{"abandoned":{"path":%q}}}`, path) - tmp := filepath.Join(m.dataDir, runtimeVolumesFile+".interrupted.tmp") - if err := os.WriteFile(tmp, []byte(raw), 0o600); err != nil { - t.Fatal(err) - } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("temporary record promoted without canonical record: %v", got) - } - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { - t.Fatal(err) - } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("temporary record overrode canonical record: %v", got) - } - if got, err := os.ReadFile(tmp); err != nil || string(got) != raw { - t.Fatalf("abandoned evidence changed: %q %v", got, err) - } -} - -func TestRuntimeCatalogAcceptsDaemonDataDirectorySymlink(t *testing.T) { - parent := t.TempDir() - dir := filepath.Join(parent, "state") - if err := os.Mkdir(dir, 0o700); err != nil { - t.Fatal(err) - } - link := filepath.Join(parent, "linked-state") - if err := os.Symlink(dir, link); err != nil { - t.Fatal(err) - } - if err := os.Chmod(parent, 0o777); err != nil { - t.Fatal(err) - } - eng := newFakeEngine() - m := newVolumeManagerAt(t, eng, link, nil) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} - if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { - t.Fatal(err) - } - restarted := newVolumeManagerAt(t, eng, link, nil) - if got := restarted.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { - t.Fatalf("reloaded catalog: %v", got) - } - if err := restarted.UnregisterVolume(t.Context(), "runtime"); err != nil { - t.Fatal(err) - } -} - func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), nil) if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { @@ -633,9 +357,6 @@ func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { if err := m.writeCatalog(t.Context(), "runtime", entries); err != nil { t.Fatalf("reservation retirement did not unblock removal: %v", err) } - if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { - t.Fatalf("removal was not persisted: %v", got) - } } func blockCatalogRename(t *testing.T, m *Manager) func() { @@ -646,12 +367,7 @@ func blockCatalogRename(t *testing.T, m *Manager) func() { t.Fatal(err) } existed := err == nil - if existed { - if err := os.Remove(record); err != nil { - t.Fatal(err) - } - } - if err := os.Mkdir(record, 0o700); err != nil { + if err := errors.Join(os.RemoveAll(record), os.Mkdir(record, 0o700)); err != nil { t.Fatal(err) } restore := sync.OnceFunc(func() { @@ -668,6 +384,7 @@ func blockCatalogRename(t *testing.T, m *Manager) func() { return restore } +// installPendingCatalogRecord models the canonical bytes after a successful rename and an uncertain directory sync. func installPendingCatalogRecord(t *testing.T, m *Manager) { t.Helper() m.mu.Lock() @@ -676,7 +393,6 @@ func installPendingCatalogRecord(t *testing.T, m *Manager) { if proposal == nil { t.Fatal("possible replacement fixture requires an accepted pending write") } - // Model the canonical bytes after a successful rename and uncertain directory sync. if err := utils.WriteFileSync(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw, 0o600); err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 78b38b34..2c71a21a 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -313,7 +313,7 @@ func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { } func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { - for _, state := range []string{"applied", "desired", "admitted", "recovery"} { + for _, state := range []string{"applied", "desired", "admitted"} { t.Run(state, func(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}) @@ -321,48 +321,18 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { held := []types.Volume{{Name: "data", Mount: "/volumes/data"}} switch state { case "applied": - m.mu.Lock() sb.Volumes = held - m.store.set(sb) - m.mu.Unlock() case "desired": - m.mu.Lock() - sb.VolumesWant = held - m.store.set(sb) - m.mu.Unlock() - case "recovery": - m.mu.Lock() sb.VolumesWant = held - js := m.store.set(sb) - m.mu.Unlock() - if err := m.store.commit(js); err != nil { - t.Fatal(err) - } - m.mu.Lock() - sb.VolumesWant = nil - m.store.set(sb) - m.mu.Unlock() case "admitted": - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "data", "bytes")}); err != nil { - t.Fatal(err) - } _, run, err := m.beginVolumeRun(t.Context(), sb) if err != nil { t.Fatal(err) } - resolved, err := m.resolveVolumes(t.Context(), "", held) - if err != nil { - t.Fatal(err) - } m.mu.Lock() - if err = m.checkResolved(resolved, ""); err == nil { - m.adoptVolumes(held) - } + m.adoptVolumes(held) run.holds = held m.mu.Unlock() - if err != nil { - t.Fatal(err) - } defer m.finishVolumeRun(sb, run) } if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "blocked", ""); !errors.Is(err, ErrVolumeCapture) { @@ -377,14 +347,6 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { if len(eng.trims) != 0 { t.Fatalf("capture trimmed held volume: %v", eng.trims) } - if state == "recovery" { - if err := m.store.commit(m.store.mark()); err != nil { - t.Fatal(err) - } - if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retired", ""); err != nil { - t.Fatalf("checkpoint after recovery acknowledgment: %v", err) - } - } }) } } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index c25fbf27..87ef4c23 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -400,7 +400,7 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { if got := claims["sb_1"].VolumesWant; !slices.Equal(got, tt.want) { t.Fatalf("loaded target %v, want %v", got, tt.want) } - if err = store.commit(store.reset(claims)); err != nil { + if err = store.save(claims); err != nil { t.Fatal(err) } rawBytes, err := os.ReadFile(store.path) @@ -850,16 +850,9 @@ func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volume if err != nil { t.Fatal(err) } - transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} - resolved, err := m.resolveVolumes(t.Context(), "", transient) - if err != nil { - t.Fatal(err) - } m.mu.Lock() defer m.mu.Unlock() - if err = m.checkResolved(resolved, ""); err != nil { - t.Fatal(err) - } + transient := []types.Volume{{Name: "data", Mount: "/volumes/data"}} m.adoptVolumes(transient) run.holds = append(run.holds, transient...) return run diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index ae6b6ea5..a334e5ef 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -13,118 +13,83 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) -func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { +func TestUnacknowledgedDetachKeepsRecoveryReference(t *testing.T) { eng := newFakeEngine() - catalog := liveCatalog(t) - m := newVolumeManager(t, eng, catalog) - sb := mustClaim(t, m, testKey) - if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { t.Fatal(err) } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) + want := []types.Volume{{Name: "home", Mount: "/home"}} + for _, caller := range []string{"", "acme"} { + if _, err := m.SetVolumes(t.Context(), sb.ID, caller, want); err != nil { + t.Fatal(err) } - return nil - } - view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) - eng.liveVolumeHook = nil - if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("detached result = %+v, %v", view, err) } + unblock := detachWithoutCommit(t, m, eng, sb.ID, "acme") claims, err := newClaimStore(m.dataDir, false).load() - if err != nil || claims[sb.ID] == nil || len(claims[sb.ID].Volumes) != 1 { - t.Fatalf("failed write did not leave old recovery reference: %+v, %v", claims, err) + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Errorf("writer admitted while old claim still references detached volume: %v", err) + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching || !view.Converging { + t.Fatalf("unretired view: %+v, %v", view, err) } - if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { - t.Errorf("capture passed before recovery reference retired: %v", err) + writer := ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}} + for _, tc := range []struct { + name string + err error + want error + }{ + {"writer", errOf(m.ClaimProvision(t.Context(), testKey, writer)), ErrVolumeBusy}, + {"checkpoint", errOf(m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", "")), ErrVolumeCapture}, + {"hibernate", m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}), ErrVolumeCapture}, + {"unregister", m.UnregisterVolume(t.Context(), "home"), ErrVolumeRegistrationConflict}, + {"rebind", m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "bytes")}), ErrVolumeRegistrationConflict}, + {"alias", m.RegisterVolume(t.Context(), "same-file", spec), ErrVolumeRegistrationConflict}, + {"reshape", errOf(m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "home", Mount: "/different"}})), ErrBadVolume}, + } { + if !errors.Is(tc.err, tc.want) { + t.Errorf("%s with unretired reference = %v, want %v", tc.name, tc.err, tc.want) + } } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + unblock() + setVolumeTenants(m, "home", "other") + before := len(eng.volumeSpecs) + _, putErr := m.SetVolumes(t.Context(), sb.ID, "acme", want) + _, patchErr := m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + if !errors.Is(putErr, ErrVolumeUnavailable) || !errors.Is(patchErr, ErrVolumeUnavailable) || len(eng.volumeSpecs) != before { + t.Fatalf("recovery-only reference authorized a fresh attachment: put=%v patch=%v", putErr, patchErr) + } + setVolumeTenants(m, "home", "acme") + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "home"); got != (volumeHolders{readers: 1}) { + t.Fatalf("own retry counts: %+v", got) + } } - m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) - if err = m2.Reconcile(t.Context()); err != nil { + detachWithoutCommit(t, m, eng, sb.ID, "acme")() + if err = m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatal(err) } - if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("restart lost old claim's reader exclusion: %v", err) + _, putErr = m.SetVolumes(t.Context(), sb.ID, "", want) + _, patchErr = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + if !errors.Is(putErr, ErrTenantRemoved) || !errors.Is(patchErr, ErrTenantRemoved) || volumeHoldersOf(m, "home") != (volumeHolders{}) { + t.Fatalf("recovery-only retry bypassed tenant removal: put=%v patch=%v", putErr, patchErr) } - if _, err = m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { - t.Fatalf("retry after restart: %v", err) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("retirement after tenant removal: %v", err) } - if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("writer still blocked after successful retirement: %v", err) + if err = claimAndRelease(t, m, writer.Volumes...); err != nil { + t.Fatalf("acknowledged detach kept the reader exclusion: %v", err) } -} - -func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { - for _, retry := range []string{"empty", "same-shape", "unrelated-snapshot"} { - t.Run(retry, func(t *testing.T) { - eng := newFakeEngine() - m := newVolumeManager(t, eng, liveCatalog(t)) - sb := mustClaim(t, m, testKey) - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { - t.Fatal("detach result unexpectedly persisted") - } - eng.liveVolumeHook = nil - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("physical count retained after detach: %+v", got) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { - t.Fatal("empty retry acknowledged an unretired file") - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mount: "/different"}}); !errors.Is(err, ErrBadVolume) { - t.Fatalf("recovery-only shape changed: %v", err) - } - view, err := m.GetVolumes(sb.ID, "") - if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { - t.Fatalf("unretired view: %+v, %v", view, err) - } - if err = os.Remove(tmp); err != nil { - t.Fatal(err) - } - switch retry { - case "empty": - _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) - case "same-shape": - for range 2 { - if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatal(err) - } - if got := volumeHoldersOf(m, "data"); got.readers != 1 || got.writers != 0 { - t.Fatalf("own retry counts: %+v", got) - } - } - _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) - case "unrelated-snapshot": - _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) - } - if err != nil { - t.Fatal(err) - } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("retired count: %+v", got) - } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { - t.Fatalf("writer after acknowledgment: %v", err) - } - }) + if err = errors.Join(m.UnregisterVolume(t.Context(), "home"), m.RegisterVolume(t.Context(), "same-file", spec)); err != nil { + t.Fatalf("retired name or path stayed reserved: %v", err) } } @@ -146,24 +111,18 @@ func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { if err != nil { t.Fatal(err) } - // Model an intent whose rename succeeded before an uncertain directory sync. + // model an intent whose rename succeeded before an uncertain directory sync if err = utils.WriteFileSync(m.store.path, raw, 0o600); err != nil { t.Fatal(err) } if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { t.Fatal("expected empty snapshot error") } - if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 { + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 || volumeHoldersOf(m, "data") != (volumeHolders{}) { t.Fatalf("memory not empty: %+v", sb) } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("unexpected physical count: %+v", got) - } - claims, err = newClaimStore(m.dataDir, false).load() - if err != nil || len(claims[sb.ID].VolumesWant) != 1 { - t.Fatalf("actual recovery file: %+v, %v", claims, err) - } - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + writer := ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}} + if _, err = m.ClaimProvision(t.Context(), testKey, writer); !errors.Is(err, ErrVolumeBusy) { t.Fatalf("writer admitted after uncertain introduction: %v", err) } if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain-introduction", ""); !errors.Is(err, ErrVolumeCapture) { @@ -174,42 +133,18 @@ func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { if err = m2.Reconcile(t.Context()); err != nil { t.Fatal(err) } - if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + if _, err = m2.ClaimProvision(t.Context(), testKey, writer); !errors.Is(err, ErrVolumeBusy) { t.Fatalf("restart admitted writer: %v", err) } if err = m.store.commit(m.store.mark()); err != nil { t.Fatal(err) } - if len(m.store.recoveryVolumesFor(sb.ID)) != 0 { - t.Fatal("successful whole snapshot did not retire attempted row") - } -} - -func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { - s := newClaimStore(t.TempDir(), false) - sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} - if err := s.commit(s.set(sb)); err != nil { - t.Fatal(err) - } - s = newClaimStore(filepath.Dir(s.path), false) - if _, err := s.load(); err != nil { - t.Fatal(err) - } - s.reset(map[string]*types.Sandbox{}) - refs := s.recoveryVolumesFor(sb.ID) - refs[0].Name = "caller-copy" - if got := s.recoveryVolumesFor(sb.ID); len(got) != 1 || got[0].Name != "data" { - t.Fatalf("reset or caller erased actual file: %+v", got) - } - if err := s.commit(s.mark()); err != nil { - t.Fatal(err) - } - if s.hasRecoveryVolumes(sb.ID) { - t.Fatal("empty snapshot did not retire file references") + if _, err = m.ClaimProvision(t.Context(), testKey, writer); err != nil { + t.Fatalf("successful whole snapshot did not retire attempted row: %v", err) } } -func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { +func TestClaimRecoveryReferencesRetireOnlyOnAcknowledgedWrite(t *testing.T) { s := newClaimStore(t.TempDir(), false) sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} if err := s.commit(s.set(sb)); err != nil { @@ -223,63 +158,25 @@ func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { t.Fatal("expected failed snapshot") } } - if refs := s.recoveryVolumesFor(sb.ID); len(refs) != 2 { - t.Fatalf("renewals retained duplicate history: %+v", refs) - } refs := s.recoveryVolumesFor(sb.ID) refs[0].Name = "caller-copy" if got := s.recoveryVolumesFor(sb.ID); len(got) != 2 || got[0].Name != "data" { - t.Fatalf("recovery references not copied: %+v", got) + t.Fatalf("recovery references duplicated or shared: %+v", got) } restoreWrites() - if err := s.commit(s.mark()); err != nil { - t.Fatal(err) - } - if len(s.recoveryVolumesFor(sb.ID)) != 1 { - t.Fatal("successful snapshot did not replace possibility history") - } -} - -func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { + s = newClaimStore(filepath.Dir(s.path), false) + if _, err := s.load(); err != nil { t.Fatal(err) } - m.store.recommitting.Store(true) - tmp := m.store.path + ".tmp" - eng.liveVolumeHook = func(verb string) error { - if verb == "detach" { - return os.Mkdir(tmp, 0o700) - } - return nil - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err == nil { - t.Fatal("expected failed detach result commit") - } - eng.liveVolumeHook = nil - if err := m.DeleteTenant(t.Context(), "acme"); err != nil { - t.Fatal(err) + s.reset(map[string]*types.Sandbox{}) + if got := s.recoveryVolumesFor(sb.ID); len(got) != 1 || got[0].Name != "data" { + t.Fatalf("reset erased the file reference: %+v", got) } - m.mu.Lock() - entry := *m.catalog.entries["data"] - entry.tenants = []string{"different-tenant"} - m.catalog.entries["data"] = &entry - m.mu.Unlock() - if err := os.Remove(tmp); err != nil { + if err := s.commit(s.mark()); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { - t.Fatalf("recovery-only retry bypassed tenant removal: %v", err) - } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("rejected recovery retry reserved a reader: %+v", got) - } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { - t.Fatalf("recovery retirement after tenant removal: %v", err) - } - if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { - t.Fatalf("repair exemption admitted unrelated addition: %v", err) + if s.hasRecoveryVolumes(sb.ID) { + t.Fatal("empty snapshot did not retire file references") } } @@ -311,8 +208,7 @@ func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing. if s.hasRecoveryVolumes(sb.ID) { t.Fatal("empty replacement retained old reference") } - restoreWrites := failClaimWrites(t, s) - defer restoreWrites() + failClaimWrites(t, s) if err = s.commit(old); err != nil { t.Fatal(err) } @@ -322,81 +218,72 @@ func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing. } } -func TestClaimSharedRecoveryVolumesRetirePerOwner(t *testing.T) { +func TestClaimRecoveryVolumeConflicts(t *testing.T) { s := newClaimStore(t.TempDir(), false) - volume := types.Volume{Name: "shared", Mount: "/data"} - first := &types.Sandbox{ID: "first", Volumes: []types.Volume{volume}} - second := &types.Sandbox{ID: "second", Volumes: []types.Volume{volume}} - writer := volume - writer.Mode = types.VolumeModeRW - if err := s.commit(s.set(first, second)); err != nil { - t.Fatal(err) - } - if s.volumeConflicts(volume, "") || !s.volumeConflicts(writer, "") { - t.Fatal("shared readers have incorrect admission") - } - if err := s.commit(s.del(first.ID)); err != nil { - t.Fatal(err) - } - if !s.referencesVolume(volume.Name) || s.hasRecoveryVolumes(first.ID) || !s.hasRecoveryVolumes(second.ID) || !s.volumeConflicts(writer, first.ID) { - t.Fatal("retiring one reader released the other") - } - if err := s.commit(s.del(second.ID)); err != nil { - t.Fatal(err) - } - if s.referencesVolume(volume.Name) || s.volumeConflicts(writer, "") { - t.Fatal("last reader left recovery reservation") - } -} - -func TestClaimRecoveryVolumeOwnerExemptionRequiresExactAttachment(t *testing.T) { - s := newClaimStore(t.TempDir(), false) - writer := types.Volume{Name: "data", Mount: "/data", Mode: types.VolumeModeRW} - sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{writer}} - if err := s.commit(s.set(sb)); err != nil { + reader := types.Volume{Name: "shared", Mount: "/data"} + writer, moved := types.Volume{Name: "data", Mount: "/data", Mode: types.VolumeModeRW}, types.Volume{Name: "data", Mount: "/elsewhere", Mode: types.VolumeModeRW} + first, second := &types.Sandbox{ID: "first", Volumes: []types.Volume{reader}}, &types.Sandbox{ID: "second", Volumes: []types.Volume{reader}} + owner := &types.Sandbox{ID: "owner", Volumes: []types.Volume{writer}} + if err := s.commit(s.set(first, second, owner)); err != nil { t.Fatal(err) } restore := failClaimWrites(t, s) - sb.VolumesWant = []types.Volume{{Name: "other", Mount: "/other"}} - if err := s.commit(s.set(sb)); err == nil { + owner.VolumesWant = []types.Volume{{Name: "other", Mount: "/other"}} + if err := s.commit(s.set(owner)); err == nil { t.Fatal("expected write failure") } - reader := writer - reader.Mode = types.VolumeModeRO - moved := writer - moved.Mount = "/elsewhere" + sharedWriter, ownerReader := reader, writer + sharedWriter.Mode, ownerReader.Mode = types.VolumeModeRW, types.VolumeModeRO for _, tc := range []struct { volume types.Volume owner string conflict bool }{ - {writer, sb.ID, false}, {writer, "different", true}, {reader, sb.ID, true}, {moved, sb.ID, true}, + {reader, "", false}, + {sharedWriter, "", true}, + {sharedWriter, first.ID, true}, + {writer, owner.ID, false}, + {writer, "different", true}, + {ownerReader, owner.ID, true}, + {moved, owner.ID, true}, } { if got := s.volumeConflicts(tc.volume, tc.owner); got != tc.conflict { t.Errorf("volume %+v owner %q conflict=%t, want %t", tc.volume, tc.owner, got, tc.conflict) } } - s.set(&types.Sandbox{ID: sb.ID, Volumes: []types.Volume{writer}}) + s.set(&types.Sandbox{ID: owner.ID, Volumes: []types.Volume{writer}}) restore() - if err := s.commit(s.mark()); err != nil { + if err := s.commit(s.del(first.ID)); err != nil { t.Fatal(err) } - if s.referencesVolume("other") || s.volumeConflicts(writer, sb.ID) { + if !s.referencesVolume(reader.Name) || s.hasRecoveryVolumes(first.ID) || !s.volumeConflicts(sharedWriter, first.ID) { + t.Fatal("retiring one reader released the other") + } + if s.referencesVolume("other") || s.volumeConflicts(writer, owner.ID) { t.Fatal("successful return retained attempted attachment") } + if err := s.commit(s.del(second.ID)); err != nil { + t.Fatal(err) + } + if s.referencesVolume(reader.Name) || s.volumeConflicts(sharedWriter, "") { + t.Fatal("last reader left recovery reservation") + } } func TestClaimCaptureAcknowledgesBothReferenceDomains(t *testing.T) { - for _, domain := range []string{"volumes", "checkpoints"} { - t.Run(domain, func(t *testing.T) { + for _, tc := range []struct { + name string + drop func(*types.Sandbox) + checkpoint, volumes bool + }{ + {"volumes", func(sb *types.Sandbox) { sb.Volumes = nil }, true, false}, + {"checkpoints", func(sb *types.Sandbox) { sb.ReplicaCk = "" }, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { s := newClaimStore(t.TempDir(), false) sb := &types.Sandbox{ID: "owner", ReplicaCk: "checkpoint", Volumes: []types.Volume{{Name: "data"}}} release := captureClaimWrite(t, s, s.set(sb), sb.ID) - if domain == "volumes" { - sb.Volumes = nil - } else { - sb.ReplicaCk = "" - } + tc.drop(sb) newer := s.set(sb) release() if !s.pending(newer) || !s.referencesCheckpoint("checkpoint") || !s.hasRecoveryVolumes(sb.ID) { @@ -405,12 +292,42 @@ func TestClaimCaptureAcknowledgesBothReferenceDomains(t *testing.T) { if err := s.commit(newer); err != nil { t.Fatal(err) } - if got := s.referencesCheckpoint("checkpoint"); got != (domain == "volumes") { - t.Fatalf("checkpoint reference after newer commit = %t", got) - } - if got := s.hasRecoveryVolumes(sb.ID); got != (domain == "checkpoints") { - t.Fatalf("volume recovery after newer commit = %t", got) + if s.referencesCheckpoint("checkpoint") != tc.checkpoint || s.hasRecoveryVolumes(sb.ID) != tc.volumes { + t.Fatalf("references after newer commit: checkpoint=%t volumes=%t", s.referencesCheckpoint("checkpoint"), s.hasRecoveryVolumes(sb.ID)) } }) } } + +// detachWithoutCommit detaches every volume on the device while the claim write fails until unblock. +func detachWithoutCommit(t *testing.T, m *Manager, eng *fakeEngine, id, tenant string) (unblock func()) { + t.Helper() + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), id, tenant, nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detach did not fail after device removal: %+v, %v", view, err) + } + return func() { + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + } +} + +func errOf(_ any, err error) error { return err } + +func setVolumeTenants(m *Manager, name string, tenants ...string) { + m.mu.Lock() + defer m.mu.Unlock() + entry := *m.catalog.entries[name] + entry.tenants = tenants + m.catalog.entries[name] = &entry +} diff --git a/sandboxd/pool/volume_static_acl_test.go b/sandboxd/pool/volume_static_acl_test.go deleted file mode 100644 index 35a5af36..00000000 --- a/sandboxd/pool/volume_static_acl_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package pool - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/types" -) - -func TestInvalidStaticTenantListNeverBecomesPublic(t *testing.T) { - path := writeVolumeImage(t, "data", "private contents") - file := filepath.Join(t.TempDir(), "config.json") - raw := fmt.Sprintf(`{"pools":[],"volumes":[{"name":"data","path":%q,"tenants":["has space"]}]}`, path) - if err := os.WriteFile(file, []byte(raw), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := config.Load(file) - if err != nil { - t.Fatalf("legacy config stopped booting: %v", err) - } - m := newVolumeManager(t, newFakeEngine(), cfg.Volumes) - if got := m.Volumes("acme", nil); len(got) != 0 { - t.Fatalf("invalid restricted ACL became public: %+v", got) - } - if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: path + ".other"}); err != nil { - t.Fatalf("invalid legacy ACL blocked unrelated registration: %v", err) - } -} - -func TestRuntimeRegistrationRejectsInvalidTenantNames(t *testing.T) { - for _, tenant := range []string{"", "has space", "_leading"} { - t.Run(tenant, func(t *testing.T) { - m := newVolumeManager(t, newFakeEngine(), nil) - if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: "/images/data", Tenants: []string{tenant}}); err == nil { - t.Fatalf("runtime registration accepted invalid tenant %q", tenant) - } - }) - } -} diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 0aab0cb7..f24769d1 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2987,14 +2987,10 @@ func (f *fakePlacer) TemplateVolumeOwners(probe string, _ []string) []string { } return f.templateVolumeOwners } - func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } - -func (f *fakePlacer) PeerAddrs() []string { return f.addrs } - +func (f *fakePlacer) PeerAddrs() []string { return f.addrs } func (f *fakePlacer) ClientAddr(addr string) string { return addr } - -func (f *fakePlacer) ConfigMismatches() int { return 0 } +func (f *fakePlacer) ConfigMismatches() int { return 0 } type fakeProber struct { owners []string @@ -3002,5 +2998,4 @@ type fakeProber struct { } func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } - -func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } +func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index ef2c48f9..037d225b 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -9,8 +9,6 @@ import ( "io" "net/http" "net/http/httptest" - "os" - "path/filepath" "reflect" "strconv" "strings" @@ -21,72 +19,35 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { +func TestVolumeRegistrationRoutesRequireRootBeforeReadingBody(t *testing.T) { for _, method := range []string{http.MethodPut, http.MethodDelete} { for _, tt := range []struct { name, apiToken, token string code int + wantCalled bool }{ - {"root", "root", "root", http.StatusNoContent}, - {"tenant", "root", "acme-token", http.StatusForbidden}, - {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, - {"unknown", "root", "unknown", http.StatusUnauthorized}, - {"missing", "root", "", http.StatusUnauthorized}, - {"open without token", "", "", http.StatusForbidden}, - {"open with tenant token", "", "acme-token", http.StatusForbidden}, + {"root", "root", "root", http.StatusNoContent, true}, + {"tenant", "root", "acme-token", http.StatusForbidden, false}, + {"sandbox", "root", "sandbox-token", http.StatusUnauthorized, false}, + {"unknown", "root", "unknown", http.StatusUnauthorized, false}, + {"missing", "root", "", http.StatusUnauthorized, false}, + {"no api token", "", "", http.StatusForbidden, false}, } { t.Run(method+"/"+tt.name, func(t *testing.T) { called := false mgr := &fakeManager{ registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, unregisterVolume: func(context.Context, string) error { called = true; return nil }, + tenants: []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, } - ts := newTenantTestServer(t, tt.apiToken, []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) - body := `{"path":"/private/data.img"}` - if tt.code != http.StatusNoContent { - body = `{invalid` - } - resp := doReq(t, method, ts.URL+"/v1/volumes/data", tt.token, body) - defer resp.Body.Close() - got, _ := io.ReadAll(resp.Body) - if resp.StatusCode != tt.code || called != (tt.code == http.StatusNoContent) { - t.Fatalf("status=%d called=%v body=%s, want %d", resp.StatusCode, called, got, tt.code) - } - if tt.code == http.StatusNoContent && len(got) != 0 { - t.Fatalf("204 response body = %q", got) - } - }) - } - } -} - -func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { - path := filepath.Join(t.TempDir(), "data.img") - if err := os.WriteFile(path, []byte("existing backing"), 0o600); err != nil { - t.Fatal(err) - } - for _, method := range []string{http.MethodPut, http.MethodDelete} { - for _, token := range []string{"", "acme-token", "unknown"} { - t.Run(method+"/"+token, func(t *testing.T) { - called := false - mgr := &fakeManager{ - registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, - unregisterVolume: func(context.Context, string) error { called = true; return nil }, - } - handler := New("", "", mgr, nil, nil, nil, nil, nil).Handler() - body := bytes.NewBufferString(fmt.Sprintf(`{"path":%q,"writable":true}`, path)) - bodyLen := body.Len() + body := bytes.NewBufferString(`{"path":"/private/data.img","writable":true}`) + unread := body.Len() req := httptest.NewRequest(method, "/v1/volumes/data", body) - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } + req.Header.Set("Authorization", "Bearer "+tt.token) w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusForbidden || called || body.Len() != bodyLen { - t.Fatalf("status=%d called=%v unread=%d/%d, want 403 without reading body or calling manager", w.Code, called, body.Len(), bodyLen) - } - if w.Header().Get("Retry-After") != "" { - t.Fatalf("permanent refusal advertises retry: %v", w.Header()) + New(tt.apiToken, "node:7777", mgr, &fakeDialer{}, nil, nil, nil, nil).Handler().ServeHTTP(w, req) + if w.Code != tt.code || called != tt.wantCalled || !tt.wantCalled && body.Len() != unread || w.Header().Get("Retry-After") != "" { + t.Fatalf("status=%d called=%v unread=%d/%d body=%s headers=%v, want %d", w.Code, called, body.Len(), unread, w.Body, w.Header(), tt.code) } }) } @@ -118,15 +79,13 @@ func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { } handler := New("root", "", mgr, nil, nil, nil, nil, nil).Handler() for _, method := range []string{http.MethodPut, http.MethodDelete} { - for range 2 { - req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) - req.Header.Set("Authorization", "Bearer root") - req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - if w.Code != http.StatusNoContent || w.Body.Len() != 0 { - t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) - } + req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer root") + req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent || w.Body.Len() != 0 { + t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) } } if registeredName != "data-v1" || removedName != "data-v1" || !reflect.DeepEqual(registered, want) { @@ -134,34 +93,18 @@ func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { } } -func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { - for _, tt := range []struct{ name, body string }{ - {"missing", ""}, - {"malformed", `{"path":`}, - {"array", `[]`}, - {"unknown field", `{"path":"/private/data.img","extra":true}`}, - {"body name", `{"name":"other","path":"/private/data.img"}`}, - {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, - {"trailing object", `{"path":"/private/data.img"}{}`}, - {"trailing text", `{"path":"/private/data.img"}oops`}, - {"wrong writable type", `{"path":"/private/data.img","writable":"true"}`}, - {"wrong tenants type", `{"path":"/private/data.img","tenants":"acme"}`}, - {"oversized", `{"path":"/` + strings.Repeat("a", maxBodyBytes) + `"}`}, - } { - t.Run(tt.name, func(t *testing.T) { - called := false - mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { - called = true - return nil - }} - ts := newTestServer(t, "root", mgr, nil) - resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", tt.body) - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - if resp.StatusCode != http.StatusBadRequest || called { - t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) - } - }) +func TestVolumeRegistrationRejectsBodyName(t *testing.T) { + called := false + mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { + called = true + return nil + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", `{"name":"other","path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest || called { + t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) } } @@ -194,43 +137,12 @@ func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { if err := json.Unmarshal(body, &got); err != nil || len(got) != 1 || got["error"] == "" { t.Fatalf("error body = %s, decode error = %v", body, err) } - if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { - t.Fatalf("status=%d body=%s, want safe %d", resp.StatusCode, body, tt.code) - } - if tt.code == http.StatusServiceUnavailable { - seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")) - if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { - t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) - } - } else if resp.Header.Get("Retry-After") != "" { - t.Fatalf("permanent refusal advertises retry: %v", resp.Header) + retry := tt.code == http.StatusServiceUnavailable + seconds, _ := strconv.Atoi(resp.Header.Get("Retry-After")) + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") || (seconds >= 1) != retry || strings.Contains(got["error"], "retry") != retry { + t.Fatalf("status=%d headers=%v body=%s, want safe %d with retry=%t", resp.StatusCode, resp.Header, body, tt.code, retry) } }) } } } - -func TestVolumeRegistrationKeepsCatalogListingPrivate(t *testing.T) { - for _, token := range []string{"root", "acme-token"} { - t.Run(token, func(t *testing.T) { - mgr := &fakeManager{volumeCatalog: func(tenant string, _ map[string]int) []types.VolumeInfo { - wantTenant := "" - if token == "acme-token" { - wantTenant = "acme" - } - if tenant != wantTenant { - t.Errorf("listing tenant = %q, want %q", tenant, wantTenant) - } - return []types.VolumeInfo{{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1, Writable: true}} - }} - ts := newTenantTestServer(t, "root", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) - resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", token, "") - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - const want = `{"volumes":[{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1,"writable":true}]}` - if resp.StatusCode != http.StatusOK || string(body) != want { - t.Fatalf("catalog = %d %s, want 200 %s", resp.StatusCode, body, want) - } - }) - } -} From 960f7bce10a46cfd00243c1be28a5f104c7adf79 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:45:36 +0000 Subject: [PATCH 16/39] pool: read one catalog snapshot per quiesce and repair pass The catalog map is replaced on every write, never edited, and a name a claim holds cannot be unregistered or rebound while held, so the snapshot taken with the hold state serves the whole pass instead of taking m.mu again for every mount. --- sandboxd/pool/volume.go | 5 ++--- sandboxd/pool/volume_mutation.go | 7 ++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 0f04eb42..f5afb546 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -244,6 +244,7 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { m.mu.Lock() td := volumeTeardown{run: m.volumeRuns[sb], holds: heldVolumes(sb)} + catalog := m.catalog.entries m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { @@ -263,11 +264,9 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } - m.mu.Lock() - if entry, ok := m.catalog.entries[volume.Name]; ok { + if entry, ok := catalog[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } - m.mu.Unlock() } if stuck { // the unmounts may have spent the whole budget; the fallback flush gets its own diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 01d64821..8d36045a 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -141,17 +141,14 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.clearVolumeError(sb, run, v.Name) } m.mu.Lock() - pending := volumeDifference(sb.VolumesWant, sb.Volumes) + pending, catalog := volumeDifference(sb.VolumesWant, sb.Volumes), m.catalog.entries m.mu.Unlock() for _, v := range pending { failedName = v.Name if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } - m.mu.Lock() - entry := m.catalog.entries[v.Name] - m.mu.Unlock() - repair, resolveErr := m.resolveCatalogEntry(ctx, v, entry) + repair, resolveErr := m.resolveCatalogEntry(ctx, v, catalog[v.Name]) if resolveErr != nil { return view, resolveErr } From 49b468a8d2ba3a553396ad77bb2b86655c45ce70 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:45:46 +0000 Subject: [PATCH 17/39] pool: restore the attach-only resolution comments Runtime registration rewrote resolveVolumes and the entry resolver but not the attach-only handling, so the two comments explaining why the flag is passed to validation and then cleared still apply. --- sandboxd/pool/volume.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index f5afb546..bfbf9a85 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -138,6 +138,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } + // an attach-only claim has no mounts to default; re-defaulting here would mount them applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) @@ -181,7 +182,7 @@ func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false + volume.AttachOnly = false // past validation the empty mount carries it return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil } From 39d866671769784c75d037c1923562d8042901d9 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:46:29 +0000 Subject: [PATCH 18/39] pool: name the claim store volume counts for what they count The checkpoint count covers current and retained references, while the volume count covers retained references only, so the shared "volumes" names hid that difference. Name the count, its helper and the fast-path flag after the retained set they summarize, and seed a loaded claim's references with the same union commit uses instead of the change-aware helper with no previous list. --- sandboxd/pool/claims.go | 54 ++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index c8876a73..0f73ad60 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -81,20 +81,20 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 - refs map[string][4]string - retained map[string][]string - changed map[string]struct{} - checkpoints map[string]int - currentVolumes map[string][]types.Volume - retainedVolumes map[string][]types.Volume - volumes map[string]volumeHolders - - recommitting atomic.Bool - recoveryVolumesPresent atomic.Bool + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 + refs map[string][4]string + retained map[string][]string + changed map[string]struct{} + checkpoints map[string]int + currentVolumes map[string][]types.Volume + retainedVolumes map[string][]types.Volume + retainedVolumeCounts map[string]volumeHolders + + recommitting atomic.Bool + anyRetainedVolume atomic.Bool } func newClaimStore(dataDir string, sync bool) *claimStore { @@ -102,7 +102,7 @@ func newClaimStore(dataDir string, sync bool) *claimStore { path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}, refs: map[string][4]string{}, retained: map[string][]string{}, changed: map[string]struct{}{}, checkpoints: map[string]int{}, - currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, volumes: map[string]volumeHolders{}, + currentVolumes: map[string][]types.Volume{}, retainedVolumes: map[string][]types.Volume{}, retainedVolumeCounts: map[string]volumeHolders{}, } } @@ -124,7 +124,7 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } - if refs := volumeRefs(nil, slices.Clone(sb.Volumes), slices.Clone(sb.VolumesWant)); len(refs) != 0 { + if refs := appendRecoveryRefs(slices.Clone(sb.Volumes), sb.VolumesWant); len(refs) != 0 { s.changeRetainedVolumes(sb.ID, refs) s.changed[sb.ID] = struct{}{} } @@ -363,43 +363,43 @@ func (s *claimStore) checkpointReferences() map[string]struct{} { return refs } -func (s *claimStore) changeVolumeCounts(refs []types.Volume, delta int) { +func (s *claimStore) changeRetainedVolumeCounts(refs []types.Volume, delta int) { for _, v := range refs { - count := s.volumes[v.Name] + count := s.retainedVolumeCounts[v.Name] if v.RW() { count.writers += delta } else { count.readers += delta } if count == (volumeHolders{}) { - delete(s.volumes, v.Name) + delete(s.retainedVolumeCounts, v.Name) } else { - s.volumes[v.Name] = count + s.retainedVolumeCounts[v.Name] = count } } } func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { - s.changeVolumeCounts(s.retainedVolumes[id], -1) - s.changeVolumeCounts(after, 1) + s.changeRetainedVolumeCounts(s.retainedVolumes[id], -1) + s.changeRetainedVolumeCounts(after, 1) if len(after) == 0 { delete(s.retainedVolumes, id) } else { s.retainedVolumes[id] = after } - s.recoveryVolumesPresent.Store(len(s.volumes) != 0) + s.anyRetainedVolume.Store(len(s.retainedVolumeCounts) != 0) } func (s *claimStore) referencesVolume(name string) bool { s.mu.Lock() defer s.mu.Unlock() - return s.volumes[name] != (volumeHolders{}) + return s.retainedVolumeCounts[name] != (volumeHolders{}) } func (s *claimStore) volumeConflicts(volume types.Volume, ownerID string) bool { s.mu.Lock() defer s.mu.Unlock() - count := s.volumes[volume.Name] + count := s.retainedVolumeCounts[volume.Name] for _, ref := range s.retainedVolumes[ownerID] { if ref == volume { if ref.RW() { @@ -419,7 +419,7 @@ func (s *claimStore) recoveryVolumesFor(owner string) []types.Volume { } func (s *claimStore) detachingOwners() map[string]bool { - if !s.recoveryVolumesPresent.Load() { + if !s.anyRetainedVolume.Load() { return nil } s.mu.Lock() @@ -437,7 +437,7 @@ func (s *claimStore) detachingOwners() map[string]bool { } func (s *claimStore) hasRecoveryVolumes(owner string) bool { - if !s.recoveryVolumesPresent.Load() { + if !s.anyRetainedVolume.Load() { return false } s.mu.Lock() From 920430d5421c0add32b05f8291f0a8af4558fdc3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:47:14 +0000 Subject: [PATCH 19/39] pool: cover invalid volume names and capture after acknowledgment Registration and unregistration validate the name before they retry a pending catalog write, so an invalid name answers ErrBadVolume (400) instead of the pending write's 503. Nothing pinned the unregister refusal or that ordering. The uncertain-introduction test refused capture but never showed it working again once a snapshot acknowledges the recovery reference. --- sandboxd/pool/volume_catalog_test.go | 14 +++++++++++--- sandboxd/pool/volume_retirement_test.go | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index f3281b35..17fd3c37 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -137,9 +137,12 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { t.Errorf("register %s %+v = %v, want %v", tc.name, tc.spec, err, tc.want) } } - for _, name := range []string{"data", "unexplained"} { - if err := m.UnregisterVolume(t.Context(), name); !errors.Is(err, ErrVolumeRegistrationConflict) { - t.Errorf("unregister %s: %v", name, err) + for _, tc := range []struct { + name string + want error + }{{"data", ErrVolumeRegistrationConflict}, {"unexplained", ErrVolumeRegistrationConflict}, {"bad/name", ErrBadVolume}} { + if err := m.UnregisterVolume(t.Context(), tc.name); !errors.Is(err, tc.want) { + t.Errorf("unregister %s = %v, want %v", tc.name, err, tc.want) } } } @@ -164,6 +167,11 @@ func TestRuntimeCatalogPendingWriteConverges(t *testing.T) { if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("failed retry cleared pending write: %v", err) } + for _, err := range []error{m.RegisterVolume(t.Context(), "bad/name", spec), m.UnregisterVolume(t.Context(), "bad/name")} { + if !errors.Is(err, ErrBadVolume) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("invalid name behind a pending write: %v", err) + } + } m.retryVolumeCatalog(t.Context()) for _, volumes := range [][]types.Volume{nil, {{Name: "data"}}} { if err := claimAndRelease(t, m, volumes...); err != nil { diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index a334e5ef..e72d1ed0 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -142,6 +142,9 @@ func TestPossibleClaimIntroductionRemainsProtected(t *testing.T) { if _, err = m.ClaimProvision(t.Context(), testKey, writer); err != nil { t.Fatalf("successful whole snapshot did not retire attempted row: %v", err) } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retired-introduction", ""); err != nil { + t.Fatalf("capture after acknowledgment: %v", err) + } } func TestClaimRecoveryReferencesRetireOnlyOnAcknowledgedWrite(t *testing.T) { From 70f98a92cb979e817772ab1f4baaa2e19a016b9e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 15:16:49 +0000 Subject: [PATCH 20/39] docs: trim volume fence prose Keep the observable 503 instead of fence internals, plus the one assumption a reader needs: a stopped VM holds no volumes only because sandboxd controls VM starts and disk attachments in its namespace. --- docs/sandboxd-api.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 18b0238f..ac8200ba 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -311,13 +311,10 @@ recovery snapshot return 409. Persistence uncertainty returns 503 with `Retry-After` and the same retry semantics as PUT. A failed DELETE can complete later. An empty GET view does not establish that recovery references retired. -Stale VM cleanup retains named holds after a complete running inventory. -An ordinary stopped VM needs no volume holds. An inferred stopped result with -`stale: true`, inspection errors, omitted or null running disk lists, and busy -creating owners retain a node-wide admission fence. Any present inventory -during busy cleanup retains that fence. Confirmed absence or removal clears it. -This assumes sandboxd controls starts and attachments in its VM namespace. -Volume-free claims remain available. Cleanup retries continue. +Volume claims and registrations also return 503 with `Retry-After` while the +node removes a leftover VM whose attached volumes it cannot determine. A +stopped leftover VM holds none, which assumes sandboxd controls VM starts and +disk attachments in its namespace. Volume-free claims are unaffected. Before downgrading to a daemon without runtime registration, drain runtime references or supply equivalent static volume config and finish pending live From 568a81af6b0c5668565828558702c3f4d2b621b0 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 26 Oct 2026 23:49:42 +0000 Subject: [PATCH 21/39] docs: state the volume access-list rule and operator duties once The tenants rule and the operator's backing duties were restated with small differences in the API reference, deploy guide, security model and live-volume example. Keep each in the API reference and link to it from the others. Drop the claim that temporary replacement files are not loaded: the daemon reads only the one record path, so the sentence described no behaviour of its own. Rewrap the overlong lines. --- docs/deploy.md | 16 ++++----------- docs/live-volumes.md | 11 ++++++----- docs/sandboxd-api.md | 47 +++++++++++++++++++++++++++----------------- docs/security.md | 27 ++++++++++--------------- 4 files changed, 49 insertions(+), 52 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 51938d7b..ae34a812 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -119,7 +119,7 @@ gets the intersection of the two. | `advertise_addr` | = `listen` | internal HTTP host:port for peer traffic and preview forwarding; also used by clients when `client_advertise` is unset. Must be routable when `listen` is a wildcard; a node with `mesh` set refuses to load while it names an unspecified host | | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | -| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, and a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later); root-owned claims have access; live additions use the claim owner's access list. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | @@ -247,12 +247,6 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes -Volume `tenants` entries use the tenant API name syntax. Static configuration -warns about invalid names and skips them during access checks. A nonempty list containing -only invalid names grants no tenant access. Runtime registration rejects invalid -tenant names. Valid names for tenants that have not yet been created remain -accepted. - Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes @@ -329,11 +323,9 @@ Warm candidates retain their normal ranking, but a volume claim's candidate set, promoted-template intersection, and redirect follow the fleet-wide rule in [cluster](cluster.md#volumes-and-placement). -An empty catalog `tenants` list allows every authenticated scope; a nonempty -list limits the image to those claim owners. Root-owned claims can use any -entry. Live additions use the claim owner's ACL, including requests sent by root. A -listed tenant the set no longer holds grants nothing and logs a warning at -boot. +Catalog `tenants` lists follow the +[access-list rule](sandboxd-api.md#volume-access-lists). A listed tenant the +set no longer holds grants nothing and logs a warning at boot. `GET /v1/volumes` reports the caller-visible fleet union and holder count, plus the answering node's current local availability and whether the entry is `writable`, without exposing host paths or node addresses. Applied names, diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 66ff435f..43fa04cb 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,11 +9,12 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. -A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) -without editing static config. Registration persists only node-local catalog -metadata and does not manage disk files. Omitted or empty runtime `tenants` -lists permit every authenticated scope. List tenants explicitly to restrict -access. Live changes remain limited to mounted read-only entries. +A configured node root token can +[register backing paths](sandboxd-api.md#put-v1volumesname) without editing +static config. Registration persists only node-local catalog metadata and does +not manage disk files. Runtime `tenants` lists follow the +[access-list rule](sandboxd-api.md#volume-access-lists). Live changes remain +limited to mounted read-only entries. Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ac8200ba..948c15ae 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -8,8 +8,8 @@ All bodies are JSON. Three token kinds: preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces (the per-id sandbox reads, `PUT /v1/sandboxes/{id}/instance-metadata`, `GET /v1/info`, - `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, `GET /metrics`, - `GET /v1/checkpoints/{id}/blob`) + `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, + `GET /metrics`, `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. On a node with a shared [`meta_store`](deploy.md#shared-tenant-database), a tenant token the node @@ -270,16 +270,10 @@ duplicate fields, and trailing JSON return 400. - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. - `path` uses static config's absolute-path validation. Registration does not open the backing or check its type or availability. A missing backing keeps - its registration but cannot be attached. Operators must keep the path and - target stable and separate from daemon cleanup directories, including - `/readers`, `/goldens`, and `/fork-*`. + its registration but cannot be attached. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. -- `tenants` grants named tenants self-service access. Omitted or empty permits - every authenticated scope for both static and runtime entries. Runtime - registration rejects invalid tenant names. Static config skips invalid names - and grants no tenant access if a nonempty list has no valid names. - A tenant name can be registered before that tenant exists. +- `tenants` is the entry's [access list](#volume-access-lists). Success is 204. An identical normalized registration is idempotent. A different binding, changed fields, or a static name or cleaned path collision returns 409. @@ -295,12 +289,29 @@ unrelated published names remain usable. Every catalog writer retries the accepted change before accepting another change. The node stores metadata in `data_dir/runtime-volumes.json` independently of -`sync_claims` and loads it before claim recovery. New records use mode 0600. Malformed -or conflicting metadata prevents startup. Temporary replacement files are not -loaded. Registration creates, grows, and deletes no backing disks. The catalog -is node-local with one sandboxd writer. The controller coordinates cross-node -ownership, symlink and hardlink aliases, backing availability, and storage -placement outside cleanup directories. +`sync_claims` and loads it before claim recovery. New records use mode 0600. +Malformed or conflicting metadata prevents startup. Registration creates, +grows, and deletes no backing disks. The catalog is node-local with one +sandboxd writer. + +The operator or controller owns each backing: supply an image Cocoon supports, +keep the path and its symlink target stable while registered, coordinate +symlink and hardlink aliases, cross-node ownership, and availability, and keep +backings outside daemon cleanup directories such as `/readers`, +`/goldens`, and `/fork-*`. + +### Volume access lists + +Static and runtime entries share one `tenants` rule. An omitted or empty list +permits every authenticated scope. A nonempty list permits only claims owned +by the listed tenants, and root-owned claims can use any entry. Live additions +check the claim owner's list even when root sends the request; held names +finish repair or removal without a new grant, and a recovery-only reference +grants no fresh attachment. Names use the tenant API syntax: runtime +registration rejects an invalid name, while static config logs a warning and +skips it, so a nonempty static list with no valid names permits no tenant. A +valid name for a tenant that does not exist yet grants nothing until the +tenant is created. ## DELETE /v1/volumes/{name} @@ -342,8 +353,8 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed PUT or PATCH adds a top-level `error` naming the refusal, and the affected -state may carry its own `error`. `converging` is true while a request worker is running or a recovery-only -reference awaits an acknowledged claim write. +state may carry its own `error`. `converging` is true while a request worker +is running or a recovery-only reference awaits an acknowledged claim write. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an diff --git a/docs/security.md b/docs/security.md index 841e9830..df0b8a48 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,25 +94,20 @@ rather than confirming existence, and operator surfaces answer tenants 403. The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is -node-local. An empty `tenants` list permits every authenticated scope for -both static and runtime entries. A nonempty list grants the named tenants -self-service access. For live additions, the claim owner must satisfy the -entry's ACL, even when root sends the request. Runtime registration rejects -invalid tenant names. Static config warns and skips them. A nonempty static -list with no valid names permits no tenant access. A valid name that does not -yet identify a tenant grants no access until that tenant exists. -Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant -cannot enumerate restricted entries by probing. Gossip carries names only. +node-local. Entry `tenants` lists follow the [access-list +rule](sandboxd-api.md#volume-access-lists). Claim lookup returns byte-identical +errors for an unknown and a forbidden name, and catalog discovery filters before +replying, so a tenant cannot enumerate restricted entries by probing. Gossip +carries names only. Neither gossip, discovery, persisted claims, usage events, nor the sandbox index exposes host image paths or access lists. Runtime registration and unregistration require a configured root `api_token`; these endpoints return 403 when the token is unset. New catalog records use mode 0600. Reads and replacements use the daemon's filesystem permissions. -Registration uses the same path validation as static configuration. It does -not verify backing type -or pin backing identity. Operators must supply a backing supported by Cocoon, -prevent path or symlink target replacement while registered, and coordinate aliases. +Registration uses the same path validation as static configuration. It does not +verify backing type or pin backing identity; the operator duties this leaves are +listed under [PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path @@ -127,10 +122,8 @@ immutable and publish a new name/path for new content. Changing a running claim's volumes needs the root token or the owning tenant's API token. A sandbox token cannot, so a guest holding its own sandbox token -cannot attach another dataset, and every addition passes the same access list -as a claim. Already-held references can finish repair or removal without a -new ACL grant. A recovery-only reference protects ownership and does not grant -permission for a fresh attachment. +cannot attach another dataset, and every addition passes the same +[access list](sandboxd-api.md#volume-access-lists) as a claim. A writable entry (`writable: true`) adds a channel the read-only model does not have: whichever tenant is permitted to claim it `rw` changes what every From 30b65a399f9f0ec69300e195b260cbac1988ad85 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:09:44 +0000 Subject: [PATCH 22/39] pool: reject non-UTF-8 runtime volume paths writeCatalog ignores the json.Marshal error, so a Go caller that bypasses the HTTP JSON decode could register a path that is not valid UTF-8 and persist a runtime volume catalog the daemon cannot load on next start. Validate the path in normalizeRegistration and reject it as ErrBadVolume (400 at the HTTP layer) so the catalog file always stays loadable. The name, directio, and tenant fields are already restricted to ASCII. --- sandboxd/pool/volume_catalog.go | 4 ++++ sandboxd/pool/volume_catalog_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 938fa687..fef2bfa3 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "sync" + "unicode/utf8" "github.com/projecteru2/core/log" @@ -212,6 +213,9 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) + if !utf8.ValidString(spec.Path) { + return nil, fmt.Errorf("%w: volume %q path must be valid UTF-8", ErrBadVolume, name) + } if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, }}); err != nil { diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 17fd3c37..da2465dd 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -132,11 +132,15 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { {"runtime", types.VolumeRegistration{Path: "relative"}, ErrBadVolume}, {"runtime", types.VolumeRegistration{Path: other, DirectIO: "invalid"}, ErrBadVolume}, {"runtime", types.VolumeRegistration{Path: other, Tenants: []string{"has space"}}, ErrBadVolume}, + {"runtime", types.VolumeRegistration{Path: "/srv/\xff"}, ErrBadVolume}, } { if err := m.RegisterVolume(t.Context(), tc.name, tc.spec); !errors.Is(err, tc.want) { t.Errorf("register %s %+v = %v, want %v", tc.name, tc.spec, err, tc.want) } } + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) || m.catalog.entries["runtime"] != nil { + t.Fatalf("rejected registration persisted: stat=%v entry=%+v", err, m.catalog.entries["runtime"]) + } for _, tc := range []struct { name string want error From 00f62c778e26bd29b02e324858cc94950b035b16 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:38:00 +0000 Subject: [PATCH 23/39] pool: keep the volume fence for a stale stopped VM Cocoon reports state=stopped, stale=true, pid=0 when it has lost a VM's PID file. That stop is inferred, not observed: the VMM can still be running and writing its disks. Since e942d31 every stopped VM took no holds, so a failed removal of such a VM lifted the node-wide fence and let a second writer attach, or the name be unregistered, while the old VMM might still hold it. Restore VMRecord.Stale and keep the fence for a stale stop on both the sweep and the busy stale-create retry, which share the classification. An ordinary stopped VM still takes no holds. The fence table gains a stale row for each path, and the inspect test again checks that the flag parses. --- docs/sandboxd-api.md | 8 +++++--- sandboxd/engine/cocoond_test.go | 8 ++++++++ sandboxd/pool/pool_test.go | 5 +++-- sandboxd/pool/reconcile.go | 3 ++- sandboxd/pool/volume_cleanup_test.go | 6 +++++- sandboxd/types/types.go | 2 ++ 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 948c15ae..d6658f66 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -323,9 +323,11 @@ recovery snapshot return 409. Persistence uncertainty returns 503 with later. An empty GET view does not establish that recovery references retired. Volume claims and registrations also return 503 with `Retry-After` while the -node removes a leftover VM whose attached volumes it cannot determine. A -stopped leftover VM holds none, which assumes sandboxd controls VM starts and -disk attachments in its namespace. Volume-free claims are unaffected. +node removes a leftover VM whose attached volumes it cannot determine, +including one cocoon reports `stale` because it only inferred the stop from a +lost PID file. Any other stopped leftover VM holds none, which assumes sandboxd +controls VM starts and disk attachments in its namespace. Volume-free claims +are unaffected. Before downgrading to a daemon without runtime registration, drain runtime references or supply equivalent static volume config and finish pending live diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 027b264c..0db76951 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -45,6 +45,8 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { fakeCocoon(t, "#!/bin/sh\n[ \"$1 $2\" = \"vm inspect\" ] || exit 2\n"+ "if [ \"$3\" = sbx-1 ]; then echo '{\"config\":{\"name\":\"sbx-1\"},\"state\":\"running\","+ "\"attached_devices\":{\"disks\":[{\"id\":\"disk-a\",\"name\":\"imagenet\",\"path\":\"/v/a.img\",\"readonly\":true},{\"id\":\"disk-b\",\"name\":\"scratch\",\"path\":\"/v/b.img\"}]}}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stale ]; then echo '{\"state\":\"stopped\",\"stale\":true,\"pid\":0}'; exit 0; fi\n"+ + "if [ \"$3\" = sbx-stopped ]; then echo '{\"state\":\"stopped\"}'; exit 0; fi\n"+ "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") @@ -52,6 +54,12 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } + for _, name := range []string{"sbx-stale", "sbx-stopped"} { + rec, present, err := e.Inspect(t.Context(), name) + if err != nil || !present || rec.State != "stopped" || rec.Stale != (name == "sbx-stale") { + t.Errorf("Inspect %s lost stopped classification: %+v, %v, %v", name, rec, present, err) + } + } if _, ok, err := e.Inspect(t.Context(), "sbx-gone"); err != nil || ok { t.Errorf("Inspect of a missing VM = %v, %v; want not found without an error", ok, err) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index b29c4e86..1318da41 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1255,6 +1255,7 @@ type fakeEngine struct { detachVolumeErr error liveVolumeHook func(string) error stopped map[string]bool + stale map[string]bool inspectHook func() error disks map[string][]types.VMDisk creating map[string]bool @@ -1296,7 +1297,7 @@ type fakeEngine struct { func newFakeEngine() *fakeEngine { return &fakeEngine{ - vms: map[string]string{}, stopped: map[string]bool{}, creating: map[string]bool{}, pids: map[string]int{}, + vms: map[string]string{}, stopped: map[string]bool{}, stale: map[string]bool{}, creating: map[string]bool{}, pids: map[string]int{}, attachDirty: map[string]bool{}, removeSeenOps: map[string][]string{}, removeSeenDirty: map[string][]string{}, metadataDocs: map[string]string{}, guestEnvs: map[string]string{}, sockRoot: "/vsock", } @@ -1771,7 +1772,7 @@ func (f *fakeEngine) list(_ context.Context) ([]types.VMRecord, error) { case f.stopped[name]: state = vmStateStopped } - rec := types.VMRecord{State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} + rec := types.VMRecord{Stale: f.stale[name], State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} if f.tap != "" { rec.NetworkConfigs = []types.VMNetConfig{{TAP: f.tap}} } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 9e3c641e..3ccbfa06 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -205,7 +205,8 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum holds[i].Mode = types.VolumeModeRW } } - case present && inspected.State != vmStateStopped: + // a stale stop is inferred from a lost PID file; the VMM may still be writing its disks + case present && (inspected.State != vmStateStopped || inspected.Stale): return td } m.mu.Lock() diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 9caf6edb..6cd4aacb 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -99,6 +99,7 @@ func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { func TestStaleVMFenceFollowsInspection(t *testing.T) { writer, reader := []types.VMDisk{{Name: "data"}}, []types.VMDisk{{Name: "data", ReadOnly: true}} busy, fenced, inspectErr := ErrVolumeBusy, ErrVolumeCatalogUnavailable, errors.New("inspection unavailable") + staleStop := vmStateStopped + " stale" for _, tc := range []struct { name, swept, listed, state string disks []types.VMDisk @@ -111,12 +112,14 @@ func TestStaleVMFenceFollowsInspection(t *testing.T) { {"running empty", vmStateRunning, "sbx-stale", vmStateRunning, []types.VMDisk{}, nil, [3]error{}, nil}, {"running omitted disks", vmStateRunning, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, {"stopped", vmStateRunning, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"stopped stale (pid file lost)", vmStateRunning, "sbx-stale", staleStop, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"inspected creating", vmStateRunning, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"inspect error", vmStateRunning, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, {"busy creating", vmStateCreating, "sbx-stale", vmStateCreating, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"busy running", vmStateCreating, "sbx-stale", vmStateRunning, reader, nil, [3]error{nil, busy, nil}, nil}, {"busy omitted disks", vmStateCreating, "sbx-stale", vmStateRunning, nil, nil, [3]error{fenced, fenced, fenced}, fenced}, {"busy stopped", vmStateCreating, "sbx-stale", vmStateStopped, writer, nil, [3]error{}, nil}, + {"busy stopped stale", vmStateCreating, "sbx-stale", staleStop, writer, nil, [3]error{fenced, fenced, fenced}, fenced}, {"busy absent", vmStateCreating, "sbx-other", vmStateRunning, writer, nil, [3]error{}, nil}, {"busy inspect error", vmStateCreating, "sbx-stale", vmStateRunning, writer, inspectErr, [3]error{fenced, fenced, fenced}, fenced}, } { @@ -125,7 +128,8 @@ func TestStaleVMFenceFollowsInspection(t *testing.T) { m := newVolumeManager(t, eng, liveCatalog(t)) name := "sbx-stale" eng.vms[tc.listed], eng.removeErrFor, eng.staleOutcome = "/vsock/stale", name, engine.StaleCreateBusy - eng.stopped[tc.listed], eng.creating[tc.listed] = tc.state == vmStateStopped, tc.state == vmStateCreating + eng.stopped[tc.listed], eng.creating[tc.listed] = tc.state == vmStateStopped || tc.state == staleStop, tc.state == vmStateCreating + eng.stale[tc.listed] = tc.state == staleStop eng.inspectHook = func() error { return tc.inspectErr } if m.removeStaleVM(t.Context(), name, types.VMRecord{State: tc.swept, Config: types.VMConfig{Name: name}}) { t.Fatal("stale VM reported gone") diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..a92c2afd 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -731,6 +731,8 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { + // Stale marks a stop cocoon inferred from a lost PID file, not one it observed. + Stale bool `json:"stale,omitzero"` State string `json:"state"` PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` From 0df2bc5502635be00b3f6c2ebf69454e291d97e4 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:39:20 +0000 Subject: [PATCH 24/39] pool: scan published claims before retiring a volume registration 969b7ab dropped the claim walk from volumeReferenced on the premise that every claimed volume is counted in the admission holders or in the claim store's retained counts. AdoptArchived breaks that premise: it publishes the adopted claim in m.claimed without taking holders, and the store counts its volumes only once the claims.json commit starts. In that window unregister, or a re-registration under the same name, succeeded while a published claim still named the volume. Walk m.claimed under m.mu again, matching applied and wanted volumes by name without copying, ahead of the store check. The test adopts a peer's row that names a volume while an earlier claims.json write holds the adoption's own write back, and requires unregister to refuse. --- sandboxd/pool/volume_catalog.go | 11 ++++++- sandboxd/pool/volume_catalog_test.go | 49 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index fef2bfa3..0c5c4278 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -141,7 +141,16 @@ func (m *Manager) loadRuntimeVolumes() error { } func (m *Manager) volumeReferenced(name string) bool { - return m.catalog.holders[name] != (volumeHolders{}) || m.store.referencesVolume(name) + if m.catalog.holders[name] != (volumeHolders{}) { + return true + } + // an adopted claim is published with no holders before its claims.json write counts it + for _, sb := range m.claimed { + if volumeIndex(sb.Volumes, name) >= 0 || volumeIndex(sb.VolumesWant, name) >= 0 { + return true + } + } + return m.store.referencesVolume(name) } func (m *Manager) writeCatalog(ctx context.Context, name string, entries map[string]*catalogVolume) error { diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index da2465dd..91ec9d83 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -2,6 +2,7 @@ package pool import ( "context" + "encoding/json/v2" "errors" "fmt" "maps" @@ -371,6 +372,54 @@ func TestRuntimeCatalogAcceptanceRechecksReferences(t *testing.T) { } } +func TestAdoptedClaimReferencesVolumeBeforeItsClaimWrite(t *testing.T) { + c := newFailoverCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + if err := b.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "adopted", "bytes")}); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, a, testKey) + mustArchive(t, a, sb) + rec, _, err := c.reg.Get(t.Context(), sb.ID) + claim := &types.Sandbox{} + if err == nil { + err = json.Unmarshal(rec.Claim, claim) + } + // the row is a peer's record, so the adopter cannot assume it names no volumes + claim.Volumes = []types.Volume{{Name: "data", Mount: "/data"}} + if err == nil { + rec.Claim, err = json.Marshal(claim) + } + if err == nil { + _, err = c.reg.Put(t.Context(), rec) + } + if err != nil { + t.Fatal(err) + } + c.expire("a") + // an earlier claims.json write in flight keeps the adoption's write from starting + b.store.writeMu.Lock() + unlock := sync.OnceFunc(b.store.writeMu.Unlock) + t.Cleanup(unlock) + adopted := make(chan error, 1) + go func() { + ok, adoptErr := b.AdoptArchived(t.Context(), sb.ID, sb.Token) + if adoptErr == nil && !ok { + adoptErr = errors.New("not adopted") + } + adopted <- adoptErr + }() + waitFor(t, func() bool { _, held := b.byID(sb.ID); return held }) + err = b.UnregisterVolume(t.Context(), "data") + unlock() + if adoptErr := <-adopted; adoptErr != nil { + t.Fatal(adoptErr) + } + if !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister while a published adoption references the volume = %v", err) + } +} + func blockCatalogRename(t *testing.T, m *Manager) func() { t.Helper() record := filepath.Join(m.dataDir, runtimeVolumesFile) From 7c513c9e4b899abb457d4d14ef637791c1d359fa Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:51:50 +0000 Subject: [PATCH 25/39] pool: bound runtime volume registration by volume_roots The root API token can register any absolute host path as a volume, writable and open to every scope if the request asks for it, so the token reaches host files such as data_dir/claims.json over the network. Static config has the same reach but needs host access to change. An optional volume_roots list limits runtime registration to paths that, once cleaned, equal a root or sit under one on a separator boundary, so /srv/data2 does not match /srv/data and "/" admits everything. Anything else is the existing bad-volume error, a 400. The check is lexical, like static path validation; symlinks are not resolved. Config load rejects a relative or empty root and stores cleaned roots. Persisted registrations go through the same check at startup, so a narrowed list cannot leave an out-of-root registration claimable; like other invalid catalog metadata it prevents startup until the operator unregisters it or widens the roots. Static volumes are not checked, and an empty list keeps today's behaviour. --- sandboxd/config/config.go | 12 ++++++-- sandboxd/config/config_test.go | 7 ++++- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume_catalog.go | 16 ++++++++--- sandboxd/pool/volume_catalog_test.go | 41 ++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 6b2c9295..03a8a694 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -277,8 +277,10 @@ type Config struct { DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` Volumes []VolumeSpec `json:"volumes,omitempty"` - Mesh *MeshConfig `json:"mesh,omitempty"` - Pools []PoolSpec `json:"pools"` + // VolumeRoots bounds runtime registration paths only; static volumes stay unchecked. + VolumeRoots []string `json:"volume_roots,omitempty"` + Mesh *MeshConfig `json:"mesh,omitempty"` + Pools []PoolSpec `json:"pools"` } // HasEgress reports whether the node can attach egress-lane VMs. @@ -400,6 +402,12 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { + for i, root := range c.VolumeRoots { + if !filepath.IsAbs(root) { + return fmt.Errorf("volume_roots %q must be an absolute path", root) + } + c.VolumeRoots[i] = filepath.Clean(root) + } validation := slices.Clone(c.Volumes) for i, volume := range c.Volumes { validation[i].Tenants = slices.DeleteFunc(slices.Clone(volume.Tenants), func(tenant string) bool { diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 144c378a..b1e51586 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -167,6 +167,8 @@ func TestLoadRejectsInvalid(t *testing.T) { {"duplicate volume name", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img"},{"name":"data","path":"/srv/datasets/b.img"}]}`, "duplicate volume"}, {"relative volume path", `{"pools":[],"volumes":[{"name":"data","path":"datasets/a.img"}]}`, "path must be absolute"}, {"bad volume directio", `{"pools":[],"volumes":[{"name":"data","path":"/srv/datasets/a.img","directio":"yes"}]}`, "directio must be"}, + {"relative volume root", `{"pools":[],"volume_roots":["srv/datasets"]}`, "volume_roots"}, + {"empty volume root", `{"pools":[],"volume_roots":[""]}`, "volume_roots"}, {"reader without warm", `{"pools":[{"template":"rt:24.04","warm":0,"reader":true}]}`, "reader needs warm"}, } { t.Run(tt.name, func(t *testing.T) { @@ -292,7 +294,7 @@ func TestLoadReadsReader(t *testing.T) { } func TestLoadAcceptsVolumes(t *testing.T) { - path := writeConfig(t, `{"pools":[],"volumes":[ + path := writeConfig(t, `{"pools":[],"volume_roots":["/srv/models/"],"volumes":[ {"name":"imagenet","path":"/srv/datasets/imagenet.img"}, {"name":"weights-llama","path":"/srv/datasets/llama.img","directio":"on"}]}`) cfg, err := Load(path) @@ -303,6 +305,9 @@ func TestLoadAcceptsVolumes(t *testing.T) { cfg.Volumes[1].DirectIO != types.DirectIOOn { t.Errorf("volumes = %+v", cfg.Volumes) } + if !slices.Equal(cfg.VolumeRoots, []string{"/srv/models"}) { + t.Errorf("volume_roots = %q, want the cleaned root with static volumes outside it accepted", cfg.VolumeRoots) + } } func TestLoadAcceptsVolumeTenantAccessList(t *testing.T) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 94c10e83..112f1ba6 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -450,7 +450,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), holders: map[string]volumeHolders{}}, + catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), roots: cfg.VolumeRoots, holders: map[string]volumeHolders{}}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 0c5c4278..06263410 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "slices" + "strings" "sync" "unicode/utf8" @@ -36,6 +37,7 @@ type catalogWrite struct { type volumeCatalog struct { entries map[string]*catalogVolume + roots []string pending *catalogWrite holders map[string]volumeHolders unknown int @@ -51,7 +53,7 @@ func (c *volumeCatalog) admissionError(name string) error { // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { - volume, err := normalizeRegistration(name, spec) + volume, err := normalizeRegistration(name, spec, m.catalog.roots) if err != nil { return err } @@ -124,7 +126,7 @@ func (m *Manager) loadRuntimeVolumes() error { } entries := maps.Clone(m.catalog.entries) for name, spec := range record.Volumes { - volume, normalizeErr := normalizeRegistration(name, spec) + volume, normalizeErr := normalizeRegistration(name, spec, m.catalog.roots) if normalizeErr != nil { return fmt.Errorf("runtime volume catalog: %w", normalizeErr) } @@ -220,7 +222,7 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { +func normalizeRegistration(name string, spec types.VolumeRegistration, roots []string) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if !utf8.ValidString(spec.Path) { return nil, fmt.Errorf("%w: volume %q path must be valid UTF-8", ErrBadVolume, name) @@ -230,9 +232,15 @@ func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalog }}); err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } + path := filepath.Clean(spec.Path) + if len(roots) > 0 && !slices.ContainsFunc(roots, func(root string) bool { + return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) + }) { + return nil, fmt.Errorf("%w: volume %q path is outside volume_roots", ErrBadVolume, name) + } return &catalogVolume{ runtime: true, - disk: engine.VolumeSpec{Name: name, Path: filepath.Clean(spec.Path), DirectIO: spec.DirectIO}, + disk: engine.VolumeSpec{Name: name, Path: path, DirectIO: spec.DirectIO}, tenants: slices.Compact(slices.Sorted(slices.Values(spec.Tenants))), writable: spec.Writable, }, nil diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 91ec9d83..42a57495 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -152,6 +152,47 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { } } +func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { + newRootedManager := func(t *testing.T, dataDir string, roots ...string) (*Manager, error) { + t.Helper() + cfg := &config.Config{DataDir: dataDir, VolumeRoots: roots} + return NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), testBackends(t, cfg)) + } + for _, tc := range []struct { + name, path string + roots []string + want error + }{ + {"inside a root", "/srv/data/a.img", []string{"/mnt/other", "/srv/data"}, nil}, + {"equal to a root", "/srv/data/", []string{"/srv/data"}, nil}, + {"sibling sharing a prefix", "/srv/data2/a.img", []string{"/srv/data"}, ErrBadVolume}, + {"dot-dot out of a root", "/srv/data/../etc/a.img", []string{"/srv/data"}, ErrBadVolume}, + {"outside every root", "/etc/a.img", []string{"/srv/data"}, ErrBadVolume}, + {"filesystem root", "/etc/a.img", []string{"/"}, nil}, + {"no roots", "/etc/a.img", nil, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + m, err := newRootedManager(t, t.TempDir(), tc.roots...) + if err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: tc.path}); !errors.Is(err, tc.want) { + t.Fatalf("register %s under %v = %v, want %v", tc.path, tc.roots, err, tc.want) + } + }) + } + m := newVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/etc/a.img"}); err != nil { + t.Fatal(err) + } + if _, err := newRootedManager(t, m.dataDir, "/etc"); err != nil { + t.Fatalf("restart with the registration inside volume_roots: %v", err) + } + if _, err := newRootedManager(t, m.dataDir, "/srv/data"); err == nil { + t.Fatal("restart accepted a persisted registration outside volume_roots") + } +} + func TestRuntimeCatalogPendingWriteConverges(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) From 42ef827a0d5048cdf6fe222cf5435437b42fd9bd Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 00:51:50 +0000 Subject: [PATCH 26/39] docs: document volume_roots and the root token's path reach Operators need to know that the root token can register any absolute host path, and that volume_roots is the way to bound it. The config reference states the lexical rule, its symlink limit, the startup refusal and that static entries are exempt; security.md says to protect the token like host root or set the list, and links to the access-list rule instead of restating it. --- docs/deploy.md | 1 + docs/sandboxd-api.md | 8 +++++--- docs/security.md | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index ae34a812..714bb6eb 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,6 +120,7 @@ gets the intersection of the two. | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. A registration already in `runtime-volumes.json` outside the roots prevents startup. Keep `data_dir` outside every root. Unset accepts any absolute path | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index d6658f66..518cb71f 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -268,9 +268,11 @@ duplicate fields, and trailing JSON return 400. ``` - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. -- `path` uses static config's absolute-path validation. Registration does not - open the backing or check its type or availability. A missing backing keeps - its registration but cannot be attached. +- `path` uses static config's absolute-path validation. When the node sets + [`volume_roots`](deploy.md#configuration), the cleaned path must equal or sit + under one of them. Registration does not open the backing or check its type + or availability. A missing backing keeps its registration but cannot be + attached. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. - `tenants` is the entry's [access list](#volume-access-lists). diff --git a/docs/security.md b/docs/security.md index df0b8a48..a3973e2a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -108,6 +108,10 @@ mode 0600. Reads and replacements use the daemon's filesystem permissions. Registration uses the same path validation as static configuration. It does not verify backing type or pin backing identity; the operator duties this leaves are listed under [PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). +Unless [`volume_roots`](deploy.md#configuration) restricts it, the root token can +register any absolute host path, writable if requested, for every scope its +[access list](sandboxd-api.md#volume-access-lists) permits. Protect the root +token like host root, or set `volume_roots`. Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path From de8300bf2613feb417c2511f594544fdbe6ee19e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 01:01:01 +0000 Subject: [PATCH 27/39] pool: skip saved registrations outside volume_roots at startup Narrowing volume_roots left the daemon unable to start while any saved registration sat outside the new roots, so tightening the bound took the node down until the operator widened it again or edited runtime-volumes.json by hand. Startup now logs a warning naming the volume and leaves it out of the loaded catalog, so it cannot be claimed and the daemon serves the rest. The skip runs after the whole file validates, so malformed or conflicting metadata still refuses startup as before. A recovered claim that names the skipped volume keeps its holders and gets the same refusal as any name missing from the catalog. The entry stays on disk until the next registration change rewrites the file from the loaded catalog, which drops it. --- docs/deploy.md | 2 +- docs/security.md | 14 ++++---- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume_catalog.go | 32 +++++++++++------ sandboxd/pool/volume_catalog_test.go | 51 +++++++++++++++++++++++----- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 714bb6eb..f67686a8 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,7 +120,7 @@ gets the intersection of the two. | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | -| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. A registration already in `runtime-volumes.json` outside the roots prevents startup. Keep `data_dir` outside every root. Unset accepts any absolute path | +| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. After narrowing, a registration already in `runtime-volumes.json` outside the roots is skipped with a warning at startup, so it cannot be claimed, and the next registration change drops it from the file. Keep `data_dir` outside every root. Unset accepts any absolute path | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | diff --git a/docs/security.md b/docs/security.md index a3973e2a..107bd7cc 100644 --- a/docs/security.md +++ b/docs/security.md @@ -105,13 +105,13 @@ exposes host image paths or access lists. Runtime registration and unregistration require a configured root `api_token`; these endpoints return 403 when the token is unset. New catalog records use mode 0600. Reads and replacements use the daemon's filesystem permissions. -Registration uses the same path validation as static configuration. It does not -verify backing type or pin backing identity; the operator duties this leaves are -listed under [PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). -Unless [`volume_roots`](deploy.md#configuration) restricts it, the root token can -register any absolute host path, writable if requested, for every scope its -[access list](sandboxd-api.md#volume-access-lists) permits. Protect the root -token like host root, or set `volume_roots`. +Registration uses static configuration's path validation plus, when set, the +[`volume_roots`](deploy.md#configuration) bound. It does not verify backing type +or pin backing identity; the operator duties this leaves are listed under +[PUT /v1/volumes/{name}](sandboxd-api.md#put-v1volumesname). Without +`volume_roots` the root token can register any absolute host path, writable if +requested, for every scope its [access list](sandboxd-api.md#volume-access-lists) +permits. Protect the root token like host root, or set `volume_roots`. Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 112f1ba6..f385c897 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -485,7 +485,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg writable: volume.Writable, } } - if err := m.loadRuntimeVolumes(); err != nil { + if err := m.loadRuntimeVolumes(ctx); err != nil { return nil, err } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 06263410..fb5efe67 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -53,10 +53,13 @@ func (c *volumeCatalog) admissionError(name string) error { // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { - volume, err := normalizeRegistration(name, spec, m.catalog.roots) + volume, err := normalizeRegistration(name, spec) if err != nil { return err } + if !m.catalog.withinRoots(volume.disk.Path) { + return fmt.Errorf("%w: volume %q path is outside volume_roots", ErrBadVolume, name) + } m.catalog.writeMu.Lock() defer m.catalog.writeMu.Unlock() if err = m.commitPendingCatalog(ctx); err != nil { @@ -109,7 +112,7 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { return m.writeCatalog(ctx, name, entries) } -func (m *Manager) loadRuntimeVolumes() error { +func (m *Manager) loadRuntimeVolumes(ctx context.Context) error { raw, err := os.ReadFile(filepath.Join(m.dataDir, runtimeVolumesFile)) if errors.Is(err, os.ErrNotExist) { return nil @@ -126,7 +129,7 @@ func (m *Manager) loadRuntimeVolumes() error { } entries := maps.Clone(m.catalog.entries) for name, spec := range record.Volumes { - volume, normalizeErr := normalizeRegistration(name, spec, m.catalog.roots) + volume, normalizeErr := normalizeRegistration(name, spec) if normalizeErr != nil { return fmt.Errorf("runtime volume catalog: %w", normalizeErr) } @@ -138,6 +141,13 @@ func (m *Manager) loadRuntimeVolumes() error { } entries[name] = volume } + // skip only once every entry validated, so other bad metadata still refuses startup + for name := range record.Volumes { + if path := entries[name].disk.Path; !m.catalog.withinRoots(path) { + log.WithFunc("pool.loadRuntimeVolumes").Warnf(ctx, "runtime volume %q: skipping image %s outside volume_roots", name, path) + delete(entries, name) + } + } m.catalog.entries = entries return nil } @@ -222,7 +232,13 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func normalizeRegistration(name string, spec types.VolumeRegistration, roots []string) (*catalogVolume, error) { +func (c *volumeCatalog) withinRoots(path string) bool { + return len(c.roots) == 0 || slices.ContainsFunc(c.roots, func(root string) bool { + return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) + }) +} + +func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if !utf8.ValidString(spec.Path) { return nil, fmt.Errorf("%w: volume %q path must be valid UTF-8", ErrBadVolume, name) @@ -232,15 +248,9 @@ func normalizeRegistration(name string, spec types.VolumeRegistration, roots []s }}); err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } - path := filepath.Clean(spec.Path) - if len(roots) > 0 && !slices.ContainsFunc(roots, func(root string) bool { - return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) - }) { - return nil, fmt.Errorf("%w: volume %q path is outside volume_roots", ErrBadVolume, name) - } return &catalogVolume{ runtime: true, - disk: engine.VolumeSpec{Name: name, Path: path, DirectIO: spec.DirectIO}, + disk: engine.VolumeSpec{Name: name, Path: filepath.Clean(spec.Path), DirectIO: spec.DirectIO}, tenants: slices.Compact(slices.Sorted(slices.Values(spec.Tenants))), writable: spec.Writable, }, nil diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 42a57495..34d7697e 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -153,10 +153,10 @@ func TestRuntimeCatalogRejectsCollisionsAndInvalidRegistrations(t *testing.T) { } func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { - newRootedManager := func(t *testing.T, dataDir string, roots ...string) (*Manager, error) { + newRootedManager := func(t *testing.T, eng *fakeEngine, dataDir string, roots ...string) (*Manager, error) { t.Helper() cfg := &config.Config{DataDir: dataDir, VolumeRoots: roots} - return NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), testBackends(t, cfg)) + return NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) } for _, tc := range []struct { name, path string @@ -172,7 +172,7 @@ func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { {"no roots", "/etc/a.img", nil, nil}, } { t.Run(tc.name, func(t *testing.T) { - m, err := newRootedManager(t, t.TempDir(), tc.roots...) + m, err := newRootedManager(t, newFakeEngine(), t.TempDir(), tc.roots...) if err != nil { t.Fatal(err) } @@ -181,15 +181,50 @@ func TestRuntimeCatalogVolumeRootsBoundRegistration(t *testing.T) { } }) } - m := newVolumeManager(t, newFakeEngine(), nil) - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/etc/a.img"}); err != nil { + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + images := map[string]string{"outside": writeVolumeImage(t, "outside", "bytes"), "inside": writeVolumeImage(t, "inside", "bytes")} + for name, path := range images { + if err := m.RegisterVolume(t.Context(), name, types.VolumeRegistration{Path: path}); err != nil { + t.Fatal(err) + } + } + outside, inside := filepath.Dir(images["outside"]), filepath.Dir(images["inside"]) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "outside"}}}) + if err != nil { t.Fatal(err) } - if _, err := newRootedManager(t, m.dataDir, "/etc"); err != nil { + wide, err := newRootedManager(t, eng, m.dataDir, outside, inside) + if err != nil || wide.catalogSnapshot()["outside"] == nil { t.Fatalf("restart with the registration inside volume_roots: %v", err) } - if _, err := newRootedManager(t, m.dataDir, "/srv/data"); err == nil { - t.Fatal("restart accepted a persisted registration outside volume_roots") + m2, err := newRootedManager(t, eng, m.dataDir, inside) + if err != nil { + t.Fatalf("restart refused a persisted registration outside volume_roots: %v", err) + } + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if got := m2.catalogSnapshot(); got["outside"] != nil || got["inside"] == nil { + t.Fatalf("narrowed catalog = %v, want only the in-root registration", slices.Sorted(maps.Keys(got))) + } + if got := volumeHoldersOf(m2, "outside"); got.readers != 1 { + t.Fatalf("recovered claim holders of the skipped registration = %+v", got) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "outside"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("claim of the skipped registration = %v, want ErrVolumeUnavailable", err) + } + if err = claimAndRelease(t, m2, types.Volume{Name: "inside"}); err != nil { + t.Fatalf("in-root registration: %v", err) + } + if err = m2.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("release of a recovered claim holding the skipped registration: %v", err) + } + if err = m2.RegisterVolume(t.Context(), "later", types.VolumeRegistration{Path: filepath.Join(inside, "later")}); err != nil { + t.Fatal(err) + } + if wide, err = newRootedManager(t, eng, m.dataDir, outside, inside); err != nil || wide.catalogSnapshot()["outside"] != nil { + t.Fatalf("a later catalog write kept the skipped registration: %v", err) } } From c5fa28ffffec1b8712e511307839c8f8ce584797 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 02:12:08 +0000 Subject: [PATCH 28/39] pool: group volume catalog methods --- sandboxd/pool/volume_catalog.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index fb5efe67..c5622f4d 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -51,6 +51,12 @@ func (c *volumeCatalog) admissionError(name string) error { return nil } +func (c *volumeCatalog) withinRoots(path string) bool { + return len(c.roots) == 0 || slices.ContainsFunc(c.roots, func(root string) bool { + return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) + }) +} + // RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { volume, err := normalizeRegistration(name, spec) @@ -232,12 +238,6 @@ func catalogPathConflict(entries map[string]*catalogVolume, path string) error { return nil } -func (c *volumeCatalog) withinRoots(path string) bool { - return len(c.roots) == 0 || slices.ContainsFunc(c.roots, func(root string) bool { - return path == root || strings.HasPrefix(path, strings.TrimSuffix(root, string(filepath.Separator))+string(filepath.Separator)) - }) -} - func normalizeRegistration(name string, spec types.VolumeRegistration) (*catalogVolume, error) { spec.DirectIO = cmp.Or(spec.DirectIO, types.DirectIOOff) if !utf8.ValidString(spec.Path) { From 09bb47b30ae04207d815a05156c77580a6418a5f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 02:18:52 +0000 Subject: [PATCH 29/39] docs: note empty volume_roots and DELETE of a skipped registration An explicit empty list behaves like unset, and DELETE of a registration skipped at startup is a no-op until the next catalog write. Say both so operators are not surprised. --- docs/deploy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index f67686a8..a704a01b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,7 +120,7 @@ gets the intersection of the two. | `client_advertise` | unset | client-facing HTTP(S) origin for this node, e.g. `https://node-a.sandbox.example.com`; published in owner, redirect, and peer responses. No path, query, fragment, or userinfo | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) on the egress lane (an egress-lane pool policy or any tenant policy) needs `bridges` and rejects a CNI network at load; none-lane pool policies ride the proxy on either | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional [access list](sandboxd-api.md#volume-access-lists); a listed name the tenant set does not hold grants nothing and logs a warning at boot (the [tenant API](sandboxd-api.md#tenants-v1tenants) may add it later). `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | -| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. After narrowing, a registration already in `runtime-volumes.json` outside the roots is skipped with a warning at startup, so it cannot be claimed, and the next registration change drops it from the file. Keep `data_dir` outside every root. Unset accepts any absolute path | +| `volume_roots` | unset | absolute directories that bound [runtime registration](sandboxd-api.md#put-v1volumesname): the registered path, once cleaned, must equal a root or sit under one, or the PUT returns 400. The check is lexical: `..` is resolved but symlinks are not followed, so a symlink under a root can point outside it. Static `volumes` entries are not checked. After narrowing, a registration already in `runtime-volumes.json` outside the roots is skipped with a warning at startup, so it cannot be claimed, and the next registration change drops it from the file. A DELETE of a skipped name that nothing references returns 204 without rewriting the file. Keep `data_dir` outside every root. Unset or an empty list accepts any absolute path | | `secrets` | unset | credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or egress-class rule references the name (on a claim where both apply, only the pool rule's secret is injected); the value comes from the claim's `guest: false` env, then the node's environment, never this file. Without `value_env` the secret's name is the env to read, and only claims supply it. See [egress](egress.md#claim-env-and-secrets) | | `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant); `prefix:port,port` scopes an entry to those ports, a bare prefix admits every port. Prefer the port form. Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_upstream` | unset | per-claim upstream proxies: `{"claim_env": "EGRESS_UPSTREAM", "allow": ["res.example.com", "10.1.0.0/16"]}`. A claim's `guest: false` entry under `claim_env` (`http://` or `socks5://`, optional `user:pass@`, or `direct`) routes its egress through that upstream, which must be on `allow` (host names, IPs, CIDR prefixes). See [upstream proxies](egress.md#upstream-proxies) | From e6391c7cf02032e4a9d684ff96069fd2f1b661e6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 03:12:03 +0000 Subject: [PATCH 30/39] volumes: support live writable and attach-only attachments Controllers need explicit device-only attachment and live writable mounts. Preserve writer exclusion and failed-write references while extending the existing convergence API and SDKs. --- e2e/e2e_test.go | 4 +- e2e/fakeengine_test.go | 10 +- e2e/volume_mutation_test.go | 2 +- sandboxd/engine/volume_live.go | 67 ++++--- sandboxd/engine/volume_live_test.go | 110 ++++++++++- sandboxd/pool/claims.go | 11 ++ sandboxd/pool/failover.go | 8 +- sandboxd/pool/pool.go | 7 +- sandboxd/pool/pool_test.go | 21 ++- sandboxd/pool/replica.go | 13 +- sandboxd/pool/volume.go | 8 +- sandboxd/pool/volume_attach_test.go | 14 +- sandboxd/pool/volume_convergence_test.go | 8 +- sandboxd/pool/volume_mutation.go | 122 +++++++------ sandboxd/pool/volume_mutation_test.go | 87 ++++++++- sandboxd/pool/volume_replica_test.go | 222 +++++++++++++++++++++++ sandboxd/pool/volume_retirement_test.go | 161 ++++++++-------- sandboxd/pool/volume_rw_test.go | 79 ++++++++ sandboxd/server/api.go | 15 ++ sandboxd/server/server.go | 5 +- sandboxd/server/volume_mutation.go | 2 +- sandboxd/server/volume_mutation_test.go | 82 ++++++++- sandboxd/types/types.go | 19 +- sandboxd/types/volume_test.go | 53 ++++-- sdk/go/client.go | 15 -- sdk/go/options.go | 17 +- sdk/go/template.go | 3 - sdk/go/volume_mutation_test.go | 39 +++- sdk/go/volumes.go | 4 +- sdk/go/volumes_test.go | 32 +--- sdk/python/cocoonsandbox/client.py | 5 +- sdk/python/cocoonsandbox/sandbox.py | 30 ++- sdk/python/cocoonsandbox/template.py | 11 +- sdk/python/cocoonsandbox/volumes.py | 23 +-- sdk/python/tests/test_client.py | 20 +- sdk/python/tests/test_volumes.py | 57 +++++- 36 files changed, 1039 insertions(+), 347 deletions(-) create mode 100644 sandboxd/pool/volume_replica_test.go diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index cf305baa..a97279b6 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -764,7 +764,7 @@ func TestAttachOnlyVolumeEndToEnd(t *testing.T) { if err != nil { t.Fatalf("attach-only claim: %v", err) } - if want := []sandbox.Volume{{Name: "scratch", Mode: "rw"}}; !slices.Equal(sb.Volumes, want) { + if want := []sandbox.Volume{{Name: "scratch", Mode: "rw", AttachOnly: true}}; !slices.Equal(sb.Volumes, want) { t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) } applied := []string{"attach:scratch:rw"} @@ -800,7 +800,7 @@ func TestAttachOnlyVolumeWireShape(t *testing.T) { { "attach-only omits the mount", `{"template":"rt:24.04","volumes_attach_only":true,"volumes":[{"name":"scratch","mode":"rw"}]}`, - map[string]any{"name": "scratch", "mode": "rw"}, + map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, }, { "eager claim is unchanged", diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 9bbc9113..39aa4951 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -215,14 +215,16 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { - if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return err +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (bool, error) { + if mount != "" { + if err := f.UnmountVolume(ctx, sock, mount); err != nil { + return false, err + } } f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "detach:"+name) - return nil + return false, nil } func (f *fakeEngine) volumeOpsLog() []string { diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 5fcedef8..ccd6d01d 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -52,7 +52,7 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { t.Fatal(checkErr) } _, err = sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}, {Name: "dataset"}}) - if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicate volume") { + if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicates name") { t.Errorf("invalid target explanation: %v", err) } view, err := sb.GetVolumes(t.Context()) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index e1033a84..c69ff92e 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -31,7 +31,7 @@ func (e *LiveVolumeUnsupported) Error() string { } type volumeMount struct { - devNum, root, path, options string + devNum, root, path, options, superOptions string } // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. @@ -63,38 +63,51 @@ func (e *Engine) MountLiveVolume(ctx context.Context, vsockSocket, name, mount s } // DetachVolume unmounts a guest volume before confirming host removal. -func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) error { +// cleanRW requires an exact whole-device mount with writable mount and superblock +// options, a normal unmount, no residual device mounts, and successful host detach. +func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) (cleanRW bool, err error) { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() device, found, err := e.findVolumeDevice(ctx, vsockSocket, name) if err != nil { - return err + return false, err } + normal := false if found { - if err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { - return err + if normal, err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { + return false, err } } - _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name) - return err + if _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name); err != nil { + return false, err + } + return normal, nil } -func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) error { +func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) (bool, error) { + if mount == "" { + return false, e.refuseOtherMounts(ctx, vsockSocket, device) + } devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { - return err + return false, err } target := e.resolveMount(ctx, vsockSocket, mount, mounts) atTarget := slices.DeleteFunc(mounts, func(m volumeMount) bool { return m.path != target }) + normal := false if len(atTarget) == 1 && atTarget[0].devNum == devNum && atTarget[0].root == "/" { if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { - return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} + return false, &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} } - return err + return false, err } + normal = writableMountOptions(atTarget[0].options) && writableMountOptions(atTarget[0].superOptions) } - return e.refuseOtherMounts(ctx, vsockSocket, device) + if err = e.refuseOtherMounts(ctx, vsockSocket, device); err != nil { + return false, err + } + return normal, nil } func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) (string, []volumeMount, error) { @@ -106,15 +119,8 @@ func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) ( if err != nil { return "", nil, err } - var mounts []volumeMount - for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { - fields := strings.Fields(line) - if len(fields) < 10 || !slices.Contains(fields[6:], "-") { - return "", nil, errors.New("invalid guest mountinfo") - } - mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5]}) - } - return strings.TrimSpace(string(devNum)), mounts, nil + mounts, err := parseVolumeMounts(string(raw)) + return strings.TrimSpace(string(devNum)), mounts, err } func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount string, rw bool) (bool, error) { @@ -164,6 +170,25 @@ func (e *Engine) refuseOtherMounts(ctx context.Context, vsockSocket, device stri return nil } +func parseVolumeMounts(raw string) ([]volumeMount, error) { + var mounts []volumeMount + for line := range strings.SplitSeq(strings.TrimSpace(raw), "\n") { + mount, filesystem, ok := strings.Cut(line, " - ") + fields := strings.Fields(mount) + fsFields := strings.Split(filesystem, " ") + if !ok || len(fields) < 6 || len(fsFields) != 3 || fsFields[0] == "" || fsFields[2] == "" { + return nil, errors.New("invalid guest mountinfo") + } + mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5], fsFields[2]}) + } + return mounts, nil +} + +func writableMountOptions(options string) bool { + parts := strings.Split(options, ",") + return slices.Contains(parts, "rw") && !slices.Contains(parts, "ro") && !slices.Contains(parts, "") +} + func bindConflict(path string) *VolumeMountConflict { return &VolumeMountConflict{Message: fmt.Sprintf("volume device has another mount at %q; remove guest mount references before retrying", path)} } diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index a731f2e1..f7c0c3b4 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -54,7 +54,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { t.Fatal(err) } } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { + if _, err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { t.Fatal(err) } fake.mu.Lock() @@ -74,6 +74,48 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { } } +func TestLiveVolumeAcceptsEmptyMountSource(t *testing.T) { + for _, tt := range []struct { + name, mount string + rw bool + }{ + {name: "mounted read-only", mount: "/data"}, + {name: "mounted writable", mount: "/data", rw: true}, + {name: "attach-only"}, + } { + t.Run(tt.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + const unrelated = "501 84 0:62 / /empty-source rw,relatime - tmpfs rw,size=1024k,inode64\n" + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + mode := mountMode(tt.rw) + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated + "2 1 8:32 / /data " + mode + " - ext4 /dev/vdc " + mode + "\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if tt.mount != "" { + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, tt.mount, tt.rw); err != nil { + t.Fatalf("mount with unrelated empty source: %v", err) + } + } + result, err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount) + if err != nil || result != tt.rw { + t.Fatalf("detach with unrelated empty source: result=%+v err=%v", result, err) + } + }) + } +} + func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -191,7 +233,7 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n3 2 8:32 /sub /data ro - ext4 /dev/vdc ro\n") } fake.mu.Unlock() - err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") if err == nil { t.Fatal("detach must fail closed") } @@ -222,7 +264,7 @@ func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { sock := sockPath(t) serveFakeSilkd(t, sock) spec := liveDiskCLI(t, hostErr) - err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") wantErr := hostErr != "" if (err != nil) != wantErr { t.Fatalf("got %v, want error=%v", err, wantErr) @@ -240,7 +282,7 @@ func TestLiveDetachIgnoresAnotherDeviceAtThePath(t *testing.T) { fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:48 / /data ro - ext4 /dev/vdd ro\n") fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { + if _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { t.Fatalf("detach of an unmounted device blocked by another device's mount: %v", err) } fake.mu.Lock() @@ -250,6 +292,66 @@ func TestLiveDetachIgnoresAnotherDeviceAtThePath(t *testing.T) { } } +func TestLiveDetachCleanEvidenceRequiresWritableUnmountAndHostSuccess(t *testing.T) { + for _, tc := range []struct { + name, options, superOptions, mount, hostErr string + missing, unmounted, busy, residual, wantClean, wantErr bool + }{ + {name: "normal", options: "rw", superOptions: "rw", mount: "/data", wantClean: true}, + {name: "unordered options", options: "relatime,rw", superOptions: "errors=remount-ro,rw", mount: "/data", wantClean: true}, + {name: "readonly", options: "ro", superOptions: "ro", mount: "/data"}, + {name: "mount readonly", options: "ro", superOptions: "rw", mount: "/data"}, + {name: "contradictory mount", options: "rw,ro", superOptions: "rw", mount: "/data"}, + {name: "superblock readonly", options: "rw", superOptions: "ro", mount: "/data"}, + {name: "contradictory superblock", options: "rw", superOptions: "rw,ro", mount: "/data"}, + {name: "empty option", options: "rw,,relatime", superOptions: "rw", mount: "/data"}, + {name: "missing superblock options", options: "rw", mount: "/data", wantErr: true}, + {name: "extra superblock fields", options: "rw", superOptions: "rw extra", mount: "/data", wantErr: true}, + {name: "pre-unmounted", unmounted: true, mount: "/data"}, + {name: "missing device", missing: true, unmounted: true, mount: "/data"}, + {name: "host failure", options: "rw", superOptions: "rw", mount: "/data", hostErr: "host removal failed", wantErr: true}, + {name: "busy", options: "rw", superOptions: "rw", mount: "/data", busy: true, wantErr: true}, + {name: "residual mount", options: "rw", superOptions: "rw", mount: "/data", residual: true, wantErr: true}, + {name: "attach-only mounted", options: "rw", superOptions: "rw", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + if !tc.missing { + configureVolumeDevices(fake) + } + spec := liveDiskCLI(t, tc.hostErr) + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + raw := rootMountInfo + if !tc.unmounted { + raw += "2 1 8:32 / /data " + tc.options + " shared:4 - ext4 /dev/vdc " + tc.superOptions + "\n" + } + fake.serial["/proc/self/mountinfo"] = []byte(raw) + if tc.busy { + fake.execCode = 32 + } + fake.execHook = func(argv []string) { + if argv[0] == "umount" && !tc.residual && !tc.busy { + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + clean, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tc.mount) + if (err != nil) != tc.wantErr || clean != tc.wantClean { + t.Fatalf("clean=%t err=%v", clean, err) + } + if tc.mount == "" { + fake.mu.Lock() + defer fake.mu.Unlock() + if len(guestChanges(fake.execCalls)) != 0 { + t.Fatal("unmounted a caller-managed filesystem") + } + } + }) + } +} + func liveDiskCLI(t *testing.T, detachErr string) VolumeSpec { t.Helper() dir := t.TempDir() diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 0f73ad60..5d6967d5 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -121,6 +121,11 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { s.mu.Lock() defer s.mu.Unlock() for _, sb := range claims { + for _, volumes := range [][]types.Volume{sb.Volumes, sb.VolumesWant} { + for i := range volumes { + volumes[i].AttachOnly = volumes[i].AttachOnly || volumes[i].Mount == "" + } + } if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } @@ -390,6 +395,12 @@ func (s *claimStore) changeRetainedVolumes(id string, after []types.Volume) { s.anyRetainedVolume.Store(len(s.retainedVolumeCounts) != 0) } +func (s *claimStore) hasCheckpointReferences(owner string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.refs[owner] != [4]string{} || len(s.retained[owner]) != 0 +} + func (s *claimStore) referencesVolume(name string) bool { s.mu.Lock() defer s.mu.Unlock() diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index b02b20dd..7246e222 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -130,6 +130,10 @@ func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, repl return 0, nil } m.mu.Lock() + if replica && m.hasVolumeHolds(sb) { + m.mu.Unlock() + return 0, ErrVolumeCapture + } dto, deadline, held := dtoOf(sb), sb.Deadline, sb.ArchiveEpoch switch { case replica: @@ -299,7 +303,9 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox, held int6 epoch, err = m.putReplica(ctx, sb, sb.ReplicaCk) } if err != nil { - log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) + if !errors.Is(err, ErrVolumeCapture) { + log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) + } return } _, _ = m.commitIfLive(ctx, sb, func() { sb.ArchiveEpoch = epoch }) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index f385c897..c156a329 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -103,6 +103,11 @@ var ( ErrNoEgressFork = errors.New("egress-lane sandboxes cannot fork, checkpoint, or promote: a resumed guest egresses before its fresh tap can be locked") ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") + + ErrReplicaPresent = errors.New("live writable attachment requires replica and recovery checkpoint retirement") + ErrReplicatingPool = errors.New("live writable volumes require a non-replicating pool") + ErrReplicaStranded = errors.New("live writable attachment is blocked by retained replica or archive state with archive_failover disabled") + // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") ErrArchived = errors.New("sandbox is archived; an exec or file call wakes it first") @@ -149,7 +154,7 @@ type Engine interface { TrimCow(ctx context.Context, vsockSocket string) error CheckLiveVolumeSupport(ctx context.Context) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error - DetachVolume(ctx context.Context, vmName, sock, name, mount string) error + DetachVolume(ctx context.Context, vmName, sock, name, mount string) (cleanRW bool, err error) MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 1318da41..fa51d1e5 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1253,6 +1253,8 @@ type fakeEngine struct { mountVolumeErr error unmountVolumeErr error detachVolumeErr error + detachNormalUnmount *bool + liveVolumeSupportErr error liveVolumeHook func(string) error stopped map[string]bool stale map[string]bool @@ -1624,7 +1626,7 @@ func (f *fakeEngine) TrimCow(_ context.Context, sock string) error { return f.trimErr } -func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } +func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return f.liveVolumeSupportErr } func (f *fakeEngine) DiskAttach(ctx context.Context, _ string, spec engine.VolumeSpec) error { if f.liveVolumeHook != nil { @@ -1670,24 +1672,31 @@ func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount stri return f.MountVolume(ctx, sock, name, mount, rw) } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (cleanRW bool, err error) { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("detach"); err != nil { - return err + return false, err } } if err := ctx.Err(); err != nil { - return err + return false, err } if mount != "" { if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return err + return false, err } } f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "detach:"+name) - return f.detachVolumeErr + if f.detachVolumeErr != nil { + return false, f.detachVolumeErr + } + normal := slices.ContainsFunc(f.volumeMounts, func(v types.Volume) bool { return v.Name == name && v.Mount == mount && v.RW() }) + if f.detachNormalUnmount != nil { + normal = *f.detachNormalUnmount + } + return normal, nil } func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 93b666c9..8d6ff141 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -113,16 +113,21 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - ok := m.claimed[sb.ID] == sb && sb.ArchiveCk == "" && sb.ArchivePending == "" var intent claimSnapshot - if ok { + var refusal error + switch { + case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "": + refusal = errWokeMeanwhile + case m.hasVolumeHolds(sb): + refusal = ErrVolumeCapture + default: sb.ReplicaPending = ckID intent = m.store.set(sb) } m.mu.Unlock() - if !ok { + if refusal != nil { m.deleteOrphanArchiveCk(ctx, ckID) - return errWokeMeanwhile + return refusal } if err := m.store.commit(intent); err != nil { _ = m.clearReplicaIntent(ctx, sb, ckID) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index bfbf9a85..1eea410a 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -138,8 +138,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - // an attach-only claim has no mounts to default; re-defaulting here would mount them - applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) + applied, err := types.ValidateVolumes(requested) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } @@ -182,7 +181,6 @@ func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false // past validation the empty mount carries it return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil } @@ -215,7 +213,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume, live bool) error { // attach-only mounts nothing, so there is no umount to verify and no marker. - attachOnly := volume.applied.Mount == "" + attachOnly := volume.applied.AttachOnly // write-ahead: the marker must be durable before any guest write can be. if volume.disk.RW && !attachOnly { if err := markVolumeDirty(volume.disk.Path); err != nil { @@ -386,7 +384,7 @@ func writableMounts(volumes []types.Volume) int { } func needsUnmount(volume types.Volume) bool { - return volume.RW() && volume.Mount != "" + return volume.RW() && !volume.AttachOnly } func appliedVolumes(volumes []resolvedVolume) []types.Volume { diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index 7fc14e75..519f4aff 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -21,7 +21,7 @@ func TestAttachOnlyClaimAttachesWithoutMounting(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - want := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}} + want := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW, AttachOnly: true}} if !slices.Equal(sb.Volumes, want) { t.Errorf("applied volumes=%v, want %v", sb.Volumes, want) } @@ -78,11 +78,7 @@ func TestAttachOnlyClaimAttachesEveryVolumeConcurrently(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - want := []types.Volume{ - {Name: "scratch", Mode: types.VolumeModeRW}, - {Name: "dataset"}, - {Name: "cache", Mode: types.VolumeModeRW}, - } + want := slices.Clone(requested) if !slices.Equal(sb.Volumes, want) { t.Errorf("volumes=%v, want the request order %v", sb.Volumes, want) } @@ -380,7 +376,7 @@ func TestQuiesceVolumesSkipsAttachOnlyEntries(t *testing.T) { sb := &types.Sandbox{ ID: "sb_mixed", VMName: "sbx-mixed", Key: testKey, VsockSocket: "/vsock/mixed", Volumes: []types.Volume{ - {Name: "scratch", Mode: types.VolumeModeRW}, + {Name: "scratch", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "mounted", Mount: "/volumes/mounted", Mode: types.VolumeModeRW}, }, } @@ -406,11 +402,11 @@ func TestWritableMountsCountsOnlyQuiescableEntries(t *testing.T) { }{ {"none", nil, 0}, {"read-only mount", []types.Volume{{Name: "a", Mount: "/a"}}, 0}, - {"attach-only writer", []types.Volume{{Name: "a", Mode: types.VolumeModeRW}}, 0}, + {"attach-only writer", []types.Volume{{Name: "a", Mode: types.VolumeModeRW, AttachOnly: true}}, 0}, {"writable mount", []types.Volume{{Name: "a", Mount: "/a", Mode: types.VolumeModeRW}}, 1}, {"mixed", []types.Volume{ {Name: "a", Mount: "/a", Mode: types.VolumeModeRW}, - {Name: "b", Mode: types.VolumeModeRW}, + {Name: "b", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "c", Mount: "/c"}, }, 1}, } { diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 2c71a21a..8ca48522 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -288,7 +288,7 @@ func TestUnsupportedRuntimeDoesNotChangeVolumeIntent(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - m.eng = &unsupportedVolumeEngine{eng} + eng.liveVolumeSupportErr = &engine.LiveVolumeUnsupported{} before, err := os.ReadFile(m.store.path) if err != nil { t.Fatal(err) @@ -421,12 +421,6 @@ func TestUncertainTargetCommitReleasesOnlyConfirmedPrecleanedHolds(t *testing.T) } } -type unsupportedVolumeEngine struct{ *fakeEngine } - -func (e *unsupportedVolumeEngine) CheckLiveVolumeSupport(context.Context) error { - return &engine.LiveVolumeUnsupported{} -} - func unmountConflict() *engine.VolumeMountConflict { return &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 8d36045a..0573087d 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -41,13 +41,13 @@ func (m *Manager) SetVolumes(ctx context.Context, id, tenant string, want []type return m.changeVolumes(ctx, id, tenant, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) } -// PatchVolumes adds and removes a running claim's read-only mounted volumes by name. +// PatchVolumes adds and removes a running claim's volumes by name. func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { return m.changeVolumes(ctx, id, tenant, func(current []types.Volume) ([]types.Volume, error) { if err := validateDetachNames(patch.Remove); err != nil { return nil, err } - additions, err := normalizeLiveVolumes(patch.Add, current) + additions, err := normalizeLiveVolumes(patch.Add) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err != nil { return view, err } - target, err = normalizeLiveVolumes(target, slices.Concat(held, recovery)) + target, err = normalizeLiveVolumes(target) if err != nil { return view, err } @@ -110,7 +110,13 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err != nil { return view, err } - if err = m.checkVolumeRun(ctx, sb, run); err != nil { + m.mu.Lock() + err = m.checkVolumeRunLocked(ctx, sb, run) + if err == nil && slices.ContainsFunc(additions, types.Volume.RW) { + err = m.liveWriterErrorLocked(sb) + } + m.mu.Unlock() + if err != nil { return view, err } sb.Touch() @@ -259,25 +265,34 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru } func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { - sb.Transition.Lock() - defer sb.Transition.Unlock() - m.mu.Lock() - if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { - m.mu.Unlock() - return claimSnapshot{}, false, err - } - if sb.OnExpire == types.ExpireArchive && len(target) > 0 { - m.mu.Unlock() - return claimSnapshot{}, false, ErrVolumeCapture - } - if err := m.checkResolved(resolved, sb.ID); err != nil { - m.mu.Unlock() + if err := func() error { + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { + return err + } + if sb.OnExpire == types.ExpireArchive && len(target) > 0 { + return ErrVolumeCapture + } + if slices.ContainsFunc(additions, types.Volume.RW) { + if err := m.liveWriterErrorLocked(sb); err != nil { + return err + } + } + if err := m.checkResolved(resolved, sb.ID); err != nil { + return err + } + m.adoptVolumes(additions) + run.holds = append(run.holds, additions...) + return nil + }(); err != nil { return claimSnapshot{}, false, err } - m.adoptVolumes(additions) - run.holds = append(run.holds, additions...) - m.mu.Unlock() cleanErr := m.confirmVolumesClean(resolved) + sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -299,14 +314,40 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return m.store.set(sb), true, nil } +func (m *Manager) liveWriterErrorLocked(sb *types.Sandbox) error { + if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { + return ErrReplicatingPool + } + if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "" || sb.ArchiveCk != "") { + return ErrReplicaStranded + } + if sb.HasReplica() || sb.ArchivePending != "" || sb.ArchiveCk != "" || m.store.hasCheckpointReferences(sb.ID) { + return ErrReplicaPresent + } + return nil +} + func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { - if err := m.checkVolumeRun(ctx, sb, run); err != nil { + m.mu.Lock() + err := m.checkVolumeRunLocked(ctx, sb, run) + entry := m.catalog.entries[v.Name] + m.mu.Unlock() + if err != nil { return err } - if err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount); err != nil { + cleanRW, err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount) + if err != nil { return fmt.Errorf("detach volume %q: %w", v.Name, err) } - return m.checkVolumeRun(ctx, sb, run) + if err = m.checkVolumeRun(ctx, sb, run); err != nil { + return err + } + if needsUnmount(v) && cleanRW && entry != nil { + if err = clearVolumeDirty(entry.disk.Path); err != nil { + return fmt.Errorf("clear dirty marker for volume %q: %w", v.Name, err) + } + } + return nil } func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume, attached bool) error { @@ -423,30 +464,9 @@ func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name strin } } -func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) { - if len(volumes) > types.MaxClaimVolumes { - return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) - } - out := make([]types.Volume, 0, len(volumes)) - for _, v := range volumes { - attachOnly := v.AttachOnly - if i := volumeIndex(held, v.Name); i >= 0 && held[i].Mount == "" && v.Mount == "" { - attachOnly = true - } - normalized, err := types.ValidateVolumes([]types.Volume{v}, attachOnly) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) - } - if volumeIndex(out, v.Name) >= 0 { - return nil, fmt.Errorf("%w: duplicate volume %q", ErrBadVolume, v.Name) - } - if attachOnly { - normalized[0].AttachOnly = false - } - out = append(out, normalized[0]) - } - mounted := slices.DeleteFunc(slices.Clone(out), func(v types.Volume) bool { return v.Mount == "" }) - if _, err := types.ValidateVolumes(mounted, false); err != nil { +func normalizeLiveVolumes(volumes []types.Volume) ([]types.Volume, error) { + out, err := types.ValidateVolumes(volumes) + if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } return out, nil @@ -458,18 +478,10 @@ func validateLiveTarget(held, target []types.Volume) error { if i >= 0 && target[i] != v { return fmt.Errorf("%w: remove volume %q before changing its mount or mode", ErrBadVolume, v.Name) } - if immutableVolume(v) && i < 0 { - return fmt.Errorf("%w: writable and attach-only volumes cannot change live", ErrBadVolume) - } - } - if slices.ContainsFunc(volumeDifference(target, held), immutableVolume) { - return fmt.Errorf("%w: live additions require read-only mounts", ErrBadVolume) } return nil } -func immutableVolume(v types.Volume) bool { return v.RW() || v.AttachOnly || v.Mount == "" } - func volumeIndex(volumes []types.Volume, name string) int { return slices.IndexFunc(volumes, func(v types.Volume) bool { return v.Name == name }) } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 87ef4c23..22a2b9e0 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -18,6 +18,75 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestLiveVolumeFourModesAndMixedRoundtrip(t *testing.T) { + eng := newFakeEngine() + catalog := []config.VolumeSpec{} + request := []types.Volume{{Name: "mounted"}, {Name: "writer", Mode: types.VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: types.VolumeModeRW, AttachOnly: true}} + for _, v := range request { + catalog = append(catalog, config.VolumeSpec{Name: v.Name, Path: writeVolumeImage(t, v.Name, "seed"), Writable: true}) + } + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + view, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: request}) + if err != nil { + t.Fatal(err) + } + if len(view.Volumes) != 4 || len(eng.volumeMounts) != 2 { + t.Fatalf("mixed modes: view=%+v mounts=%+v", view, eng.volumeMounts) + } + for _, v := range view.Volumes { + if v.AttachOnly != (v.Mount == "") { + t.Fatalf("lost canonical shape: %+v", v) + } + } + before := slices.Clone(eng.volumeOpsLog()) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", view.VolumesWant); err != nil { + t.Fatal(err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatal("GET-to-PUT roundtrip changed devices") + } + for _, change := range []types.Volume{{Name: "raw", Mount: "/new"}, {Name: "raw-writer", AttachOnly: true}, {Name: "writer", Mount: "/other", Mode: types.VolumeModeRW}} { + if _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{change}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("held shape change accepted: %+v, %v", change, err) + } + } + legacy := slices.Clone(view.VolumesWant) + for i := range legacy { + legacy[i].AttachOnly = false + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", legacy); !errors.Is(err, ErrBadVolume) { + t.Fatalf("explicit false inherited attach-only: %v", err) + } + assertLiveJournal(t, m, sb) + raw, err := os.ReadFile(m.store.path) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(m.store.path, []byte(strings.ReplaceAll(string(raw), `,"attach_only":true`, "")), 0o600); err != nil { + t.Fatal(err) + } + restarted := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + recovered, err := restarted.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(recovered.Volumes, view.Volumes) { + t.Fatalf("mixed restart: %+v, %v", recovered, err) + } + if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "raw-writer", AttachOnly: true}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost writer exclusion: %v", err) + } + if _, err = restarted.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + for _, spec := range catalog { + if volumeDirty(spec.Path) || volumeHoldersOf(restarted, spec.Name) != (volumeHolders{}) { + t.Fatalf("detach leaked marker or hold: %s", spec.Name) + } + } +} + func TestLiveVolumesCommitAndRetry(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) @@ -68,8 +137,8 @@ func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("accepted mode change: %v", err) } - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { - t.Fatalf("writable detach changed the batch: %v", err) + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); err != nil || len(sb.Volumes) != 0 { + t.Fatalf("writable detach failed: %v", err) } }) } @@ -80,8 +149,8 @@ func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) for _, request := range [][]types.Volume{ - {{Name: "data", Mode: "rw"}}, - {{Name: "data", AttachOnly: true}}, + {{Name: "data", Mode: "unknown"}}, + {{Name: "data", AttachOnly: true, Mount: "/data"}}, {{Name: "data", Mount: "/etc"}}, {{Name: "data"}, {Name: "data"}}, {{Name: "data", Mount: "/a"}, {Name: "other", Mount: "/a/b"}}, @@ -226,7 +295,7 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { eng.liveVolumeHook = func(string) error { close(entered); <-resume; return nil } attached := make(chan error, 1) go func() { - _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}) attached <- err }() <-entered @@ -307,7 +376,7 @@ func TestLiveVolumeCanceledRequestRetainsIntent(t *testing.T) { sb := mustClaim(t, m, testKey) ctx, cancel := context.WithCancel(t.Context()) eng.liveVolumeHook = func(string) error { cancel(); return nil } - _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}) + _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}) if !errors.Is(err, context.Canceled) || slices.Equal(sb.VolumesWant, sb.Volumes) { t.Fatalf("canceled attach: %v, pending=%v", err, sb.VolumesWant) } @@ -384,6 +453,7 @@ func TestDesiredVolumesJournalMigration(t *testing.T) { want []types.Volume }{ {"legacy", "", []types.Volume{{Name: "data", Mount: "/data"}}}, + {"legacy attach-only desired", `,"volumes_want":[{"name":"raw","mode":"rw"}]`, []types.Volume{{Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}}}, {"explicit empty", `,"volumes_want":[]`, nil}, {"desired", `,"volumes_want":[{"name":"other","mount":"/other"}]`, []types.Volume{{Name: "other", Mount: "/other"}}}, } { @@ -489,7 +559,10 @@ func TestCanceledVolumeWriterOwnsHardwareUntilReturn(t *testing.T) { m.eng = eng ctx, cancel := context.WithCancel(t.Context()) first := make(chan error, 1) - go func() { _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); first <- err }() + go func() { + _, err := m.SetVolumes(ctx, sb.ID, sb.Tenant, []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}) + first <- err + }() <-eng.entered cancel() waitingCtx, stopWaiting := context.WithTimeout(t.Context(), 25*time.Millisecond) diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go new file mode 100644 index 00000000..0d6c4386 --- /dev/null +++ b/sandboxd/pool/volume_replica_test.go @@ -0,0 +1,222 @@ +package pool + +import ( + "errors" + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" + "slices" + "sync" + "testing" + "time" +) + +func TestReplicaRecoveryDoesNotRepublishReadOnlyHold(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + if view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil || len(view.Volumes) != 1 { + t.Fatalf("read-only live attachment: %+v, %v", view, err) + } + if _, err := c.reg.Consume(t.Context(), sb.ID, sb.ArchiveEpoch); err != nil { + t.Fatal(err) + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed { + t.Fatalf("setup: %+v, %t, %v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if rec, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok || !rec.Consumed { + t.Fatalf("recovery republished read-only hold: %+v, %t, %v", rec, ok, err) + } +} + +func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { + m, c := newReplicaVolumeManager(t) + disableReplication(m) + sb := mustClaim(t, m, testKey) + stalled := &stallingStore{Store: m.ckpts, published: make(chan string), release: make(chan struct{})} + m.ckpts = stalled + result := make(chan error, 1) + done := make(chan struct{}) + go func() { defer close(done); result <- m.replicate(t.Context(), sb) }() + ck := <-stalled.published + release := sync.OnceFunc(func() { close(stalled.release) }) + t.Cleanup(func() { release(); <-done }) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := m.SetVolumes(t.Context(), sb.ID, "", writer) + if err != nil || len(view.Volumes) != 1 { + t.Fatalf("writer during checkpoint publication: %+v, %v", view, err) + } + release() + if err = <-result; !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("stale replica registered after writer admission: %v", err) + } + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || ok || sb.HasReplica() || ckExists(t, m, ck) { + t.Fatalf("stale replica remains recoverable: row=%t, claim=%t, checkpoint=%t, err=%v", ok, sb.HasReplica(), ckExists(t, m, ck), getErr) + } + if got := volumeHoldersOf(m, "data"); got.writers != 1 { + t.Fatalf("refusing replica lost writer hold: %+v", got) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("snapshot admission while writer is held: %v", err) + } +} + +func TestReplicaCaptureRefusesPendingLiveWriter(t *testing.T) { + m, c := newReplicaVolumeManager(t) + disableReplication(m) + sb := mustClaim(t, m, testKey) + eng := m.eng.(*fakeEngine) + entered, resume := make(chan struct{}), make(chan struct{}) + release := sync.OnceFunc(func() { close(resume) }) + eng.liveVolumeHook = func(verb string) error { + if verb == "attach" { + close(entered) + <-resume + } + return nil + } + result := make(chan error, 1) + done := make(chan struct{}) + go func() { + defer close(done) + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}) + result <- err + }() + t.Cleanup(func() { release(); <-done }) + <-entered + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 1 || !view.Converging { + t.Fatalf("writer did not reach pending attachment: %+v, %v", view, err) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("captured pending writer: %v", err) + } + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || ok || len(eng.snapSaves) != 0 { + t.Fatalf("pending writer produced snapshot or row: snapshots=%v, row=%t, err=%v", eng.snapSaves, ok, getErr) + } + release() + if err = <-result; err != nil { + t.Fatalf("pending writer did not complete: %v", err) + } +} + +func TestLiveWriterPolicyAndCheckpointRefusals(t *testing.T) { + for _, tc := range []struct { + name string + prepare func(*Manager, *types.Sandbox) + want error + }{ + {"replicating pool", func(*Manager, *types.Sandbox) {}, ErrReplicatingPool}, + {"source pool replicates", func(_ *Manager, sb *types.Sandbox) { + sb.PolicySource = testKey + sb.Key.Template = "promoted:template" + }, ErrReplicatingPool}, + {"source pool does not replicate", func(_ *Manager, sb *types.Sandbox) { + sb.PolicySource = testKey + sb.PolicySource.Template = "other:template" + }, nil}, + {"published replica", func(m *Manager, sb *types.Sandbox) { disableReplication(m); replicateNow(t, m, sb) }, ErrReplicaPresent}, + {"pending replica", func(m *Manager, sb *types.Sandbox) { + disableReplication(m) + sb.ReplicaPending = "pending" + m.store.set(sb) + }, ErrReplicaPresent}, + {"registry disabled", func(m *Manager, sb *types.Sandbox) { disableReplication(m); replicateNow(t, m, sb); m.archives = nil }, ErrReplicaStranded}, + {"archive pending without registry", func(m *Manager, sb *types.Sandbox) { + disableReplication(m) + sb.ArchivePending = "pending" + m.archives = nil + }, ErrReplicaStranded}, + {"retained only without registry", func(m *Manager, sb *types.Sandbox) { + disableReplication(m) + sb.ReplicaCk = "retained" + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + sb.ReplicaCk = "" + m.store.set(sb) + m.archives = nil + }, ErrReplicaPresent}, + {"unrelated checkpoint", func(m *Manager, sb *types.Sandbox) { + disableReplication(m) + other := mustClaim(t, m, testKey) + other.ReplicaCk = "unrelated" + m.store.set(other) + }, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + tc.prepare(m, sb) + before, ops := m.store.mark(), m.eng.(*fakeEngine).volumeOpsLog() + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}) + if !errors.Is(err, tc.want) { + t.Fatalf("writer refusal = %v, want %v", err, tc.want) + } + if tc.want != nil && (len(view.VolumesWant) != 0 || len(view.Volumes) != 0 || view.Converging || m.store.mark() != before || volumeHoldersOf(m, "data") != (volumeHolders{}) || !slices.Equal(ops, m.eng.(*fakeEngine).volumeOpsLog())) { + t.Fatalf("refusal changed claim or device: %+v", view) + } + }) + } +} + +func TestInitialWriterAndRetiredReplicaAllowLiveWork(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + if err != nil { + t.Fatal(err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", sb.Volumes); err != nil { + t.Fatalf("held writer retry: %v", err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("writer cleanup: %v", err) + } + replicateNow(t, m, sb) + disableReplication(m) + m.store.recommitting.Store(true) + restore := failClaimWrites(t, m.store) + m.retireReplica(t.Context(), sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW}} + if sb.HasReplica() || !m.store.hasCheckpointReferences(sb.ID) { + t.Fatal("failed clear lost checkpoint uncertainty") + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatalf("unacknowledged retirement: %v", err) + } + restore() + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil { + t.Fatalf("acknowledged retirement: %v", err) + } +} + +func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { + t.Helper() + c := newReplicaCluster(t) + spec := archivePool(keepAnHour) + spec.ReplicateSeconds = c.replicate + cfg := &config.Config{ + APIToken: "root", DataDir: t.TempDir(), CheckpointDir: c.ckDir, + Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: "a"}, + Volumes: []config.VolumeSpec{{Name: "data", Path: writeVolumeImage(t, "data", "original"), Writable: true}}, + } + backends := testBackends(t, cfg) + backends.Archives = c.reg + m, err := NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + c.renew(t, "a") + m.liftFence() + return m, c +} + +func disableReplication(m *Manager) { + view := *m.view.Load() + view.poolReplicate = nil + m.view.Store(&view) +} diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index e72d1ed0..c6a224bc 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -14,82 +14,91 @@ import ( ) func TestUnacknowledgedDetachKeepsRecoveryReference(t *testing.T) { - eng := newFakeEngine() - m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) - spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}} - if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { - t.Fatal(err) - } - want := []types.Volume{{Name: "home", Mount: "/home"}} - for _, caller := range []string{"", "acme"} { - if _, err := m.SetVolumes(t.Context(), sb.ID, caller, want); err != nil { - t.Fatal(err) - } - } - unblock := detachWithoutCommit(t, m, eng, sb.ID, "acme") - claims, err := newClaimStore(m.dataDir, false).load() - if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { - t.Fatalf("old durable attachment missing: %+v, %v", claims, err) - } - view, err := m.GetVolumes(sb.ID, "") - if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching || !view.Converging { - t.Fatalf("unretired view: %+v, %v", view, err) - } - writer := ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}} - for _, tc := range []struct { - name string - err error - want error - }{ - {"writer", errOf(m.ClaimProvision(t.Context(), testKey, writer)), ErrVolumeBusy}, - {"checkpoint", errOf(m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", "")), ErrVolumeCapture}, - {"hibernate", m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}), ErrVolumeCapture}, - {"unregister", m.UnregisterVolume(t.Context(), "home"), ErrVolumeRegistrationConflict}, - {"rebind", m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "bytes")}), ErrVolumeRegistrationConflict}, - {"alias", m.RegisterVolume(t.Context(), "same-file", spec), ErrVolumeRegistrationConflict}, - {"reshape", errOf(m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "home", Mount: "/different"}})), ErrBadVolume}, - } { - if !errors.Is(tc.err, tc.want) { - t.Errorf("%s with unretired reference = %v, want %v", tc.name, tc.err, tc.want) - } - } - if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err == nil { - t.Fatal("empty retry acknowledged an unretired file") - } - unblock() - setVolumeTenants(m, "home", "other") - before := len(eng.volumeSpecs) - _, putErr := m.SetVolumes(t.Context(), sb.ID, "acme", want) - _, patchErr := m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) - if !errors.Is(putErr, ErrVolumeUnavailable) || !errors.Is(patchErr, ErrVolumeUnavailable) || len(eng.volumeSpecs) != before { - t.Fatalf("recovery-only reference authorized a fresh attachment: put=%v patch=%v", putErr, patchErr) - } - setVolumeTenants(m, "home", "acme") - for range 2 { - if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { - t.Fatal(err) - } - if got := volumeHoldersOf(m, "home"); got != (volumeHolders{readers: 1}) { - t.Fatalf("own retry counts: %+v", got) - } - } - detachWithoutCommit(t, m, eng, sb.ID, "acme")() - if err = m.DeleteTenant(t.Context(), "acme"); err != nil { - t.Fatal(err) - } - _, putErr = m.SetVolumes(t.Context(), sb.ID, "", want) - _, patchErr = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) - if !errors.Is(putErr, ErrTenantRemoved) || !errors.Is(patchErr, ErrTenantRemoved) || volumeHoldersOf(m, "home") != (volumeHolders{}) { - t.Fatalf("recovery-only retry bypassed tenant removal: put=%v patch=%v", putErr, patchErr) - } - if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { - t.Fatalf("retirement after tenant removal: %v", err) - } - if err = claimAndRelease(t, m, writer.Volumes...); err != nil { - t.Fatalf("acknowledged detach kept the reader exclusion: %v", err) - } - if err = errors.Join(m.UnregisterVolume(t.Context(), "home"), m.RegisterVolume(t.Context(), "same-file", spec)); err != nil { - t.Fatalf("retired name or path stayed reserved: %v", err) + for _, volume := range []types.Volume{{Name: "home", Mount: "/home"}, {Name: "home", Mount: "/home", Mode: types.VolumeModeRW}, {Name: "home", AttachOnly: true}, {Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("rw=%t/raw=%t", volume.RW(), volume.AttachOnly), func(t *testing.T) { + expectedHolds := volumeHolders{readers: 1} + if volume.RW() { + expectedHolds = volumeHolders{writers: 1} + } + + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: []string{"acme"}} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { + t.Fatal(err) + } + want := []types.Volume{volume} + for _, caller := range []string{"", "acme"} { + if _, err := m.SetVolumes(t.Context(), sb.ID, caller, want); err != nil { + t.Fatal(err) + } + } + unblock := detachWithoutCommit(t, m, eng, sb.ID, "acme") + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching || !view.Converging { + t.Fatalf("unretired view: %+v, %v", view, err) + } + writer := ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW}}} + for _, tc := range []struct { + name string + err error + want error + }{ + {"writer", errOf(m.ClaimProvision(t.Context(), testKey, writer)), ErrVolumeBusy}, + {"checkpoint", errOf(m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", "")), ErrVolumeCapture}, + {"hibernate", m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}), ErrVolumeCapture}, + {"unregister", m.UnregisterVolume(t.Context(), "home"), ErrVolumeRegistrationConflict}, + {"rebind", m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "bytes")}), ErrVolumeRegistrationConflict}, + {"alias", m.RegisterVolume(t.Context(), "same-file", spec), ErrVolumeRegistrationConflict}, + {"reshape", errOf(m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "home", Mount: "/different"}})), ErrBadVolume}, + } { + if !errors.Is(tc.err, tc.want) { + t.Errorf("%s with unretired reference = %v, want %v", tc.name, tc.err, tc.want) + } + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + unblock() + setVolumeTenants(m, "home", "other") + before := len(eng.volumeSpecs) + _, putErr := m.SetVolumes(t.Context(), sb.ID, "acme", want) + _, patchErr := m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + if !errors.Is(putErr, ErrVolumeUnavailable) || !errors.Is(patchErr, ErrVolumeUnavailable) || len(eng.volumeSpecs) != before { + t.Fatalf("recovery-only reference authorized a fresh attachment: put=%v patch=%v", putErr, patchErr) + } + setVolumeTenants(m, "home", "acme") + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "home"); got != expectedHolds { + t.Fatalf("own retry counts: %+v", got) + } + } + detachWithoutCommit(t, m, eng, sb.ID, "acme")() + if err = m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + _, putErr = m.SetVolumes(t.Context(), sb.ID, "", want) + _, patchErr = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "data"}}}) + if !errors.Is(putErr, ErrTenantRemoved) || !errors.Is(patchErr, ErrTenantRemoved) || volumeHoldersOf(m, "home") != (volumeHolders{}) { + t.Fatalf("recovery-only retry bypassed tenant removal: put=%v patch=%v", putErr, patchErr) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("retirement after tenant removal: %v", err) + } + if err = claimAndRelease(t, m, writer.Volumes...); err != nil { + t.Fatalf("acknowledged detach kept the reader exclusion: %v", err) + } + if err = errors.Join(m.UnregisterVolume(t.Context(), "home"), m.RegisterVolume(t.Context(), "same-file", spec)); err != nil { + t.Fatalf("retired name or path stayed reserved: %v", err) + } + }) } } diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index fda04c98..0369858d 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -3,6 +3,7 @@ package pool import ( "encoding/json/v2" "errors" + "fmt" "os" "path/filepath" "slices" @@ -16,6 +17,84 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestLiveWritableDetachRequiresCleanEvidence(t *testing.T) { + for _, proof := range []bool{false, true} { + t.Run(fmt.Sprint(proof), func(t *testing.T) { + base := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, base, catalog) + base.detachNormalUnmount = &proof + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + path := catalog[0].Path + if !volumeDirty(path) || !base.attachDirty["data"] { + t.Fatal("writer was attached before dirty marker") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if volumeDirty(path) == proof { + t.Fatalf("marker disagrees with clean evidence: proof=%t dirty=%t", proof, volumeDirty(path)) + } + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if proof && err != nil || !proof && !errors.Is(err, ErrVolumeNeedsRecovery) { + t.Fatalf("reader admission after detach: %v", err) + } + }) + } +} + +func TestLiveWritableFailedHostDetachKeepsMarkerAndHold(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("host removal uncertain") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("failed host detach reported success") + } + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("failed host detach lost dirty marker or writer") + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture after uncertain detach: %v", err) + } + eng.detachVolumeErr = nil + eng.detachNormalUnmount = new(bool) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if !volumeDirty(catalog[0].Path) { + t.Fatal("absent-device retry invented clean evidence") + } +} + +func TestLiveWritableUncertainAdditionClearsOnlyAfterDetach(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + eng.mountVolumeErr = errors.New("mount response lost") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err == nil { + t.Fatal("expected unknown mount result") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 1 || !volumeDirty(catalog[0].Path) { + t.Fatal("pending writable attachment lost intent or marker") + } + eng.mountVolumeErr = nil + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("uncertain addition cleanup retained marker or hold") + } +} + func TestWritableClaimMarksDirtyBeforeAttachAndClearsOnRelease(t *testing.T) { path := writeVolumeImage(t, "scratch.img", "data") eng := newFakeEngine() diff --git a/sandboxd/server/api.go b/sandboxd/server/api.go index 6e26e0c9..9a26c5df 100644 --- a/sandboxd/server/api.go +++ b/sandboxd/server/api.go @@ -1,6 +1,8 @@ package server import ( + "errors" + "slices" "time" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -42,6 +44,19 @@ func (r ClaimRequest) Key() types.PoolKey { return types.PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } +func (r ClaimRequest) normalizedVolumes() ([]types.Volume, error) { + volumes := slices.Clone(r.Volumes) + if r.VolumesAttachOnly { + if len(volumes) == 0 { + return nil, errors.New("volumes_attach_only requires at least one volume") + } + for i := range volumes { + volumes[i].AttachOnly = true + } + } + return types.ValidateVolumes(volumes) +} + // ClaimResponse is the wire reply of POST /v1/claim; success and Redirect are exclusive. type ClaimResponse struct { ID string `json:"id,omitempty"` diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 70279f63..270c2ad0 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -65,6 +65,9 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, + {pool.ErrReplicaPresent, http.StatusServiceUnavailable, ""}, + {pool.ErrReplicatingPool, http.StatusConflict, ""}, + {pool.ErrReplicaStranded, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, @@ -356,7 +359,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req ClaimRequest, key types.PoolKey, hash, tenant string) { - volumes, err := types.ValidateVolumes(req.Volumes, req.VolumesAttachOnly) + volumes, err := req.normalizedVolumes() if err != nil { writeErr(w, http.StatusBadRequest, fmt.Sprintf("%v: %v", pool.ErrBadVolume, err)) return diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index 1e4f86f9..86dd6398 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -60,7 +60,7 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { code, view.Error = mapped, msg } } - if code >= http.StatusInternalServerError { + if code >= http.StatusInternalServerError && !errors.Is(err, pool.ErrReplicaPresent) { log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) } } diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 48ff46b6..baca0330 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -153,7 +153,7 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { const cause = "cannot unmount /data; close guest references and retry" - for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture} { + for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent, pool.ErrReplicatingPool, pool.ErrReplicaStranded} { t.Run(refusal.Error(), func(t *testing.T) { mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: "volume change failed; retry the desired target"}, fmt.Errorf("%w: volume %q", refusal, "data") @@ -165,9 +165,87 @@ func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatal(err) } - if resp.StatusCode != http.StatusConflict || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { + wantStatus := http.StatusConflict + if errors.Is(refusal, pool.ErrReplicaPresent) { + wantStatus = http.StatusServiceUnavailable + if resp.Header.Get("Retry-After") == "" { + t.Fatal("temporary replica retirement refusal has no Retry-After") + } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatal("permanent conflict advertises a retry delay") + } + if resp.StatusCode != wantStatus || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { t.Fatalf("status=%d view=%+v", resp.StatusCode, got) } }) } } + +func TestLiveVolumeRoutesPreserveMixedAttachOnlyRoundtrip(t *testing.T) { + want := []types.Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} + view := types.SandboxVolumesResponse{VolumesWant: want, Volumes: want} + mgr := &fakeManager{ + getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, + setVolumes: func(_, _ string, got []types.Volume) (types.SandboxVolumesResponse, error) { + if !slices.Equal(got, want) { + t.Errorf("GET-to-PUT changed mixed intent: %+v", got) + } + return view, nil + }, + patchVolumes: func(_, _ string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { + if !slices.Equal(patch.Add, want) { + t.Errorf("PATCH lost per-entry intent: %+v", patch.Add) + } + return view, nil + }, + } + ts := newTenantTestServer(t, "root", nil, mgr, nil) + get := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", "") + defer get.Body.Close() + var got types.SandboxVolumesResponse + if err := json.UnmarshalRead(get.Body, &got); err != nil || !slices.Equal(got.VolumesWant, want) { + t.Fatalf("GET lost per-entry attachment intent: %+v, %v", got, err) + } + for _, method := range []string{http.MethodPut, http.MethodPatch} { + key := "volumes" + if method == http.MethodPatch { + key = "add" + } + body, err := json.Marshal(map[string][]types.Volume{key: got.VolumesWant}) + if err != nil { + t.Fatal(err) + } + resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", string(body)) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s status=%d", method, resp.StatusCode) + } + } +} + +func TestClaimVolumeCompatibilityExpansion(t *testing.T) { + for _, tt := range []struct { + name string + req ClaimRequest + want []types.Volume + err bool + }{ + {"default mount", ClaimRequest{Volumes: []types.Volume{{Name: "data"}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}}, false}, + {"legacy all devices", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data"}, {Name: "work", Mode: types.VolumeModeRW}}}, []types.Volume{{Name: "data", AttachOnly: true}, {Name: "work", Mode: types.VolumeModeRW, AttachOnly: true}}, false}, + {"mixed", ClaimRequest{Volumes: []types.Volume{{Name: "data"}, {Name: "work", AttachOnly: true}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}, {Name: "work", AttachOnly: true}}, false}, + {"mixed reversed", ClaimRequest{Volumes: []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data"}}}, []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data", Mount: "/volumes/data"}}, false}, + {"empty legacy", ClaimRequest{VolumesAttachOnly: true}, nil, true}, + {"legacy mount conflict", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data", Mount: "/data"}}}, nil, true}, + } { + t.Run(tt.name, func(t *testing.T) { + before := slices.Clone(tt.req.Volumes) + got, err := tt.req.normalizedVolumes() + if (err != nil) != tt.err || !slices.Equal(got, tt.want) { + t.Fatalf("normalized volumes=%+v err=%v, want %+v error=%t", got, err, tt.want, tt.err) + } + if !slices.Equal(tt.req.Volumes, before) { + t.Fatal("legacy expansion mutated caller input") + } + }) + } +} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index a92c2afd..be49f2ef 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -754,8 +754,8 @@ type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` Mode string `json:"mode,omitempty"` - // AttachOnly carries the claim's flag into validation; past it the empty Mount is the signal. - AttachOnly bool `json:"-"` + // AttachOnly leaves mounting and unmounting to the caller. + AttachOnly bool `json:"attach_only,omitzero"` } // RW reports whether the entry asks for write access. @@ -805,19 +805,11 @@ func VolumeRWNames(volumes []Volume) []string { return names } -// VolumesAttachOnly reports whether the caller mounts the devices itself. -func VolumesAttachOnly(volumes []Volume) bool { - return slices.ContainsFunc(volumes, func(v Volume) bool { return v.AttachOnly }) -} - // ValidateVolumes returns detached entries with defaults filled and modes normalized. -func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { +func ValidateVolumes(volumes []Volume) ([]Volume, error) { if len(volumes) > MaxClaimVolumes { return nil, fmt.Errorf("volumes must contain at most %d entries, got %d", MaxClaimVolumes, len(volumes)) } - if attachOnly && len(volumes) == 0 { - return nil, errors.New("volumes_attach_only requires at least one volume") - } applied := make([]Volume, len(volumes)) names := make(map[string]struct{}, len(volumes)) for i, volume := range volumes { @@ -836,7 +828,7 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { default: return nil, fmt.Errorf("volumes[%d] mode %q must be %s or %s", i, mode, VolumeModeRO, VolumeModeRW) } - if attachOnly { + if volume.AttachOnly { if volume.Mount != "" { return nil, fmt.Errorf("volumes[%d] mount %q is meaningless when the caller mounts the device itself", i, volume.Mount) } @@ -849,6 +841,9 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { } for j := range i { other := applied[j].Mount + if other == "" { + continue + } switch { case mount == other: return nil, fmt.Errorf("volumes[%d] mount %q duplicates volumes[%d]", i, mount, j) diff --git a/sandboxd/types/volume_test.go b/sandboxd/types/volume_test.go index 9549af49..c7f428f1 100644 --- a/sandboxd/types/volume_test.go +++ b/sandboxd/types/volume_test.go @@ -31,7 +31,7 @@ func TestValidVolumeName(t *testing.T) { func TestValidateVolumes(t *testing.T) { volumes := []Volume{{Name: "dataset"}, {Name: "weights-1", Mount: "/models"}} wantInput := slices.Clone(volumes) - got, err := ValidateVolumes(volumes, false) + got, err := ValidateVolumes(volumes) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -54,7 +54,7 @@ func TestValidateVolumes(t *testing.T) { {Name: "f"}, {Name: "g"}, {Name: "h"}, - }, false); err != nil || len(got) != MaxClaimVolumes { + }); err != nil || len(got) != MaxClaimVolumes { t.Errorf("ValidateVolumes at limit: got=%v err=%v", got, err) } @@ -75,7 +75,7 @@ func TestValidateVolumes(t *testing.T) { {"uppercase-mode", []Volume{{Name: "dataset", Mode: "RW"}}}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := ValidateVolumes(tt.volumes, false); err == nil { + if _, err := ValidateVolumes(tt.volumes); err == nil { t.Fatal("ValidateVolumes succeeded") } }) @@ -87,7 +87,7 @@ func TestValidateVolumesNormalizesMode(t *testing.T) { {Name: "shared"}, {Name: "explicit", Mode: VolumeModeRO}, {Name: "writable", Mode: VolumeModeRW}, - }, false) + }) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -112,10 +112,10 @@ func TestValidateVolumesNormalizesMode(t *testing.T) { func TestValidateVolumesAttachOnly(t *testing.T) { got, err := ValidateVolumes([]Volume{ - {Name: "dataset"}, - {Name: "scratch", Mode: VolumeModeRW}, - {Name: "explicit", Mode: VolumeModeRO}, - }, true) + {Name: "dataset", AttachOnly: true}, + {Name: "scratch", Mode: VolumeModeRW, AttachOnly: true}, + {Name: "explicit", Mode: VolumeModeRO, AttachOnly: true}, + }) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -127,18 +127,11 @@ func TestValidateVolumesAttachOnly(t *testing.T) { if !slices.Equal(got, want) { t.Errorf("volumes %v, want %v", got, want) } - if !VolumesAttachOnly(got) { - t.Error("validated attach-only set does not report itself") - } - if VolumesAttachOnly(nil) || VolumesAttachOnly([]Volume{{Name: "dataset", Mount: "/volumes/dataset"}}) { - t.Error("mounted set reports itself attach-only") - } for _, tt := range []struct { name string volumes []Volume }{ - {"no volumes", nil}, {"default mount spelled out", []Volume{{Name: "dataset", Mount: "/volumes/dataset"}}}, {"custom mount", []Volume{{Name: "dataset", Mount: "/datasets"}}}, {"one entry of many", []Volume{{Name: "dataset"}, {Name: "scratch", Mount: "/scratch"}}}, @@ -147,7 +140,10 @@ func TestValidateVolumesAttachOnly(t *testing.T) { {"unknown mode", []Volume{{Name: "dataset", Mode: "readwrite"}}}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := ValidateVolumes(tt.volumes, true); err == nil { + for i := range tt.volumes { + tt.volumes[i].AttachOnly = true + } + if _, err := ValidateVolumes(tt.volumes); err == nil { t.Fatal("ValidateVolumes succeeded") } }) @@ -157,19 +153,38 @@ func TestValidateVolumesAttachOnly(t *testing.T) { func TestValidateVolumesRejectsGuestOSMounts(t *testing.T) { for _, root := range append([]string{"/"}, guestOSMountRoots...) { t.Run(root, func(t *testing.T) { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root}}, false); err == nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root}}); err == nil { t.Fatalf("accepted OS mount %q", root) } if root != "/" { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root + "/child"}}, false); err == nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root + "/child"}}); err == nil { t.Fatalf("accepted mount under OS root %q", root) } } }) } for _, mount := range []string{"/data", "/home/dataset", "/opt-data", "/usrdata"} { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: mount}}, false); err != nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: mount}}); err != nil { t.Errorf("rejected allowed mount %q: %v", mount, err) } } } + +func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { + request := []Volume{{Name: "mounted"}, {Name: "writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} + got, err := ValidateVolumes(request) + if err != nil { + t.Fatal(err) + } + want := []Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "writer", Mount: "/volumes/writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} + if !slices.Equal(got, want) { + t.Fatalf("mixed intent changed: got %+v, want %+v", got, want) + } + again, err := ValidateVolumes(got) + if err != nil || !slices.Equal(again, want) { + t.Fatalf("second validation changed mixed intent: %+v, %v", again, err) + } + if _, err = ValidateVolumes([]Volume{{Name: "raw", AttachOnly: true, Mount: "/data"}}); err == nil { + t.Fatal("accepted attach-only with a mount") + } +} diff --git a/sdk/go/client.go b/sdk/go/client.go index ccb932fd..587f2422 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -110,18 +110,6 @@ func (r claimRequest) rejectPinnedAxes() error { return nil } -func (r claimRequest) validateVolumes() error { - for _, v := range r.Volumes { - if v.Mode != "" && v.Mode != volumeModeRW { - return fmt.Errorf("volume %q: mode must be \"\", %q, or %q, got %q", v.Name, volumeModeRO, volumeModeRW, v.Mode) - } - if r.VolumesAttachOnly && v.Mount != "" { - return fmt.Errorf("volume %q: mount %q is meaningless with WithVolumesAttachOnly, which leaves mounting to the caller", v.Name, v.Mount) - } - } - return nil -} - type claimResponse struct { ID string `json:"id"` Token string `json:"token"` @@ -172,9 +160,6 @@ func (c *Client) New(ctx context.Context, template string, opts ...Option) (*San for _, opt := range opts { opt(&claim) } - if err := claim.validateVolumes(); err != nil { - return nil, err - } addr, cr, err := claimFollow(c.addr, "claim", func(noRedirect, requirePromoted bool) ([]byte, error) { claim.NoRedirect, claim.RequirePromoted = noRedirect, requirePromoted return encodeBody("claim", claim) diff --git a/sdk/go/options.go b/sdk/go/options.go index 20b75401..036d3539 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -17,9 +17,6 @@ const ( Large Size = "large" XLarge Size = "xlarge" XXLarge Size = "2xlarge" - - volumeModeRO = "ro" - volumeModeRW = "rw" ) // NetShape selects the sandbox network lane. @@ -35,6 +32,8 @@ type Volume struct { Mount string `json:"mount,omitempty"` // Mode is "rw" for a writable mount; empty (or "ro") means read-only. Mode string `json:"mode,omitempty"` + // AttachOnly leaves mounting and unmounting to the caller. + AttachOnly bool `json:"attach_only,omitzero"` } // VolumeInfo describes one visible fleet catalog entry. @@ -80,18 +79,12 @@ func WithSize(s Size) Option { // read-only, set Volume.Mode to "rw" for a writable mount. func WithVolumes(volumes ...Volume) Option { volumes = slices.Clone(volumes) - for i, v := range volumes { - if v.Mode == volumeModeRO { - volumes[i].Mode = "" - } - } return func(r *claimRequest) { r.Volumes = volumes } } -// WithVolumesAttachOnly attaches the claim's volumes without mounting them; -// the workload finds each device by its serial and owns the mount — and its -// flush — from there: release without a clean self-umount discards unsynced -// guest pages, since the sandbox performs no sync of its own. +// WithVolumesAttachOnly is the legacy claim-wide form of Volume.AttachOnly. +// The caller owns mounting, flushing, and unmounting every attached device. +// Sandbox release does not sync these devices. func WithVolumesAttachOnly() Option { return func(r *claimRequest) { r.VolumesAttachOnly = true } } diff --git a/sdk/go/template.go b/sdk/go/template.go index 34ee31f7..c7f45c7b 100644 --- a/sdk/go/template.go +++ b/sdk/go/template.go @@ -29,9 +29,6 @@ func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { if err := claim.rejectPinnedAxes(); err != nil { return nil, err } - if err := claim.validateVolumes(); err != nil { - return nil, err - } claim.Net, claim.Size = t.net, t.size noVolumes := len(claim.Volumes) == 0 addr, cr, err := claimFollow(t.addr, "claim", func(noRedirect, _ bool) ([]byte, error) { diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 7f65427c..a49e2890 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -51,7 +51,7 @@ func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mount":"/new"}],"volumes":[{"name":"old","mount":"/old"}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mode":"rw","attach_only":true}],"volumes":[{"name":"old","attach_only":true}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -177,6 +177,43 @@ func TestLiveVolumeIncompleteViewPreservesHandle(t *testing.T) { } } +func TestLiveVolumeAttachOnlyRoundTrip(t *testing.T) { + const entries = `[{"name":"mounted-ro","mount":"/volumes/mounted-ro"},{"name":"mounted-rw","mount":"/work","mode":"rw"},{"name":"device-ro","attach_only":true},{"name":"device-rw","mode":"rw","attach_only":true}]` + var forwarded []map[string]any + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + var body struct { + Volumes []map[string]any `json:"volumes"` + } + if err := json.UnmarshalRead(r.Body, &body); err != nil { + t.Errorf("decode replacement: %v", err) + } + forwarded = body.Volumes + } + _, _ = io.WriteString(w, `{"volumes_want":`+entries+`,"volumes":`+entries+`,"states":[],"converging":false}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", owner: c.addr, c: c} + view, err := sb.GetVolumes(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err = sb.SetVolumes(t.Context(), view.VolumesWant); err != nil { + t.Fatal(err) + } + var want []map[string]any + if err = json.Unmarshal([]byte(entries), &want); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(forwarded, want) { + t.Fatalf("GET-to-PUT changed volume intent: got=%v want=%v", forwarded, want) + } + if !slices.Equal(sb.Volumes, view.Volumes) || !slices.Equal(sb.VolumesWant, view.VolumesWant) { + t.Fatal("roundtrip changed handle state") + } +} + func callVolumes(t *testing.T, sb *Sandbox, method string) (SandboxVolumesResponse, error) { t.Helper() switch method { diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 69e1dea4..4d7e3910 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -55,7 +55,7 @@ func (s *Sandbox) GetVolumes(ctx context.Context) (SandboxVolumesResponse, error return s.requestVolumes(ctx, http.MethodGet, nil, "get volumes") } -// SetVolumes converges the full desired set; an explicit empty slice clears mutable mounts, and nil is rejected. +// SetVolumes converges the full desired set; an explicit empty slice detaches all volumes, and nil is rejected. func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolumesResponse, error) { if volumes == nil { return SandboxVolumesResponse{}, errors.New("set volumes: use an explicit empty slice to clear volumes") @@ -65,7 +65,7 @@ func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolu }{Volumes: volumes}, "set volumes") } -// PatchVolumes adds or removes read-only mounted entries by name, retaining immutable entries. +// PatchVolumes adds or removes volumes by name. func (s *Sandbox) PatchVolumes(ctx context.Context, patch VolumePatch) (SandboxVolumesResponse, error) { return s.requestVolumes(ctx, http.MethodPatch, patch, "patch volumes") } diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index d8a2c14e..f35563bf 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -196,7 +196,7 @@ func TestWithVolumesEncodesMode(t *testing.T) { mode string }{ {"empty stays omitted", ""}, - {"ro normalizes to omitted", "ro"}, + {"ro stays explicit", "ro"}, {"rw rides the wire", "rw"}, } for _, tt := range tests { @@ -215,9 +215,9 @@ func TestWithVolumesEncodesMode(t *testing.T) { t.Fatalf("New: %v", err) } got, present := raw.Volumes[0]["mode"] - if tt.mode == volumeModeRW { - if !present || got != volumeModeRW { - t.Errorf("wire mode = %v (present=%v), want %q", got, present, volumeModeRW) + if tt.mode != "" { + if !present || got != tt.mode { + t.Errorf("wire mode = %v (present=%v), want %q", got, present, tt.mode) } } else if present { t.Errorf("wire mode = %v, want omitted", got) @@ -231,7 +231,7 @@ func TestWithVolumesAttachOnlySendsFlagAndDecodesMountlessEcho(t *testing.T) { Volumes []map[string]any `json:"volumes"` VolumesAttachOnly bool `json:"volumes_attach_only"` } - echo := []Volume{{Name: "imagenet"}, {Name: "scratch", Mode: "rw"}} + echo := []Volume{{Name: "imagenet", AttachOnly: true}, {Name: "scratch", Mode: "rw", AttachOnly: true}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := json.UnmarshalRead(r.Body, &raw); err != nil { t.Errorf("decode body: %v", err) @@ -259,20 +259,7 @@ func TestWithVolumesAttachOnlySendsFlagAndDecodesMountlessEcho(t *testing.T) { } } -func TestWithVolumesAttachOnlyRejectsMountLocally(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - t.Error("server should not be contacted for a locally invalid claim") - })) - t.Cleanup(ts.Close) - - _, err := testClient(t, ts).New(t.Context(), "rt:24.04", - WithVolumes(Volume{Name: "imagenet", Mount: "/datasets"}), WithVolumesAttachOnly()) - if err == nil || !strings.Contains(err.Error(), "meaningless with WithVolumesAttachOnly") { - t.Errorf("err = %v, want local mount rejection", err) - } -} - -func TestRejectsInvalidVolumeModeLocally(t *testing.T) { +func TestVolumePolicyErrorsComeFromServer(t *testing.T) { tests := []struct { name string claim func(ctx context.Context, c *Client) (*Sandbox, error) @@ -287,14 +274,15 @@ func TestRejectsInvalidVolumeModeLocally(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - t.Error("server should not be contacted for a locally invalid mode") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"mode must be ro or rw"}`)) })) t.Cleanup(ts.Close) _, err := tt.claim(t.Context(), testClient(t, ts)) if err == nil || !strings.Contains(err.Error(), "mode must be") { - t.Errorf("err = %v, want local mode rejection", err) + t.Errorf("err = %v, want server mode rejection", err) } }) } diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index aa3c8fd1..cdf8587e 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -9,7 +9,7 @@ import threading import time import urllib.parse -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Sequence from typing import Any, TypeVar, cast from .checkpoint import Checkpoint @@ -18,6 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query +from .volumes import Volume T = TypeVar("T") @@ -60,7 +61,7 @@ def new( size: str = "", ttl_seconds: int = 0, claim_ref: str = "", - volumes: list[str | Mapping[str, str]] | None = None, + volumes: list[str | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 1a83fa86..b3368d44 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -7,7 +7,7 @@ import threading import time from collections import deque -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any, cast from .checkpoint import Checkpoint @@ -45,7 +45,7 @@ def __init__( deadline: str = "", from_checkpoint: str = "", template_digest: str = "", - volumes: list[dict[str, str]] | None = None, + volumes: list[Volume] | None = None, ) -> None: self._client = client self.id = id @@ -54,8 +54,8 @@ def __init__( self.deadline = deadline self.from_checkpoint = from_checkpoint self.template_digest = template_digest - self.volumes = [dict(volume) for volume in volumes or []] - self.volumes_want = [dict(volume) for volume in self.volumes] + self.volumes = [volume.copy() for volume in volumes or []] + self.volumes_want = [volume.copy() for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 @@ -73,29 +73,27 @@ def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse """Reads the desired and applied volumes from the owning node and records them on this handle.""" return self._volumes_request("GET", None, "get volumes", timeout) - def set_volumes( - self, volumes: list[str | Mapping[str, str] | Volume], *, timeout: float | None = None - ) -> SandboxVolumesResponse: - """Replaces the desired set and converges; an explicit empty list clears mutable mounts.""" - if volumes is None or isinstance(volumes, str): + def set_volumes(self, volumes: list[str | Volume], *, timeout: float | None = None) -> SandboxVolumesResponse: + """Replaces the desired set and converges; an explicit empty list detaches all volumes.""" + if not isinstance(volumes, list): raise TypeError("set volumes requires an explicit list; use [] to clear volumes") return self._volumes_request( - "PUT", {"volumes": [_volume_body(volume, True) for volume in volumes]}, "set volumes", timeout + "PUT", {"volumes": [_volume_body(volume) for volume in volumes]}, "set volumes", timeout ) def patch_volumes( self, *, - add: list[str | Mapping[str, str] | Volume] | None = None, + add: list[str | Volume] | None = None, remove: list[str] | None = None, timeout: float | None = None, ) -> SandboxVolumesResponse: - """Adds or removes mounted read-only entries by name; omitted arguments leave the set unchanged.""" - if isinstance(add, str) or isinstance(remove, str): + """Adds or removes volumes by name; omitted arguments leave the set unchanged.""" + if (add is not None and not isinstance(add, list)) or (remove is not None and not isinstance(remove, list)): raise TypeError("patch volumes takes lists of volumes and names") body: dict[str, Any] = {} if add is not None: - body["add"] = [_volume_body(volume, True) for volume in add] + body["add"] = [_volume_body(volume) for volume in add] if remove is not None: body["remove"] = list(remove) return self._volumes_request("PATCH", body, "patch volumes", timeout) @@ -445,8 +443,8 @@ def _volumes_request( return view def _record_volumes(self, view: SandboxVolumesResponse) -> None: - self.volumes_want = [cast(dict[str, str], volume.copy()) for volume in view["volumes_want"]] - self.volumes = [cast(dict[str, str], volume.copy()) for volume in view["volumes"]] + self.volumes_want = [volume.copy() for volume in view["volumes_want"]] + self.volumes = [volume.copy() for volume in view["volumes"]] def _dial(self, deadline: float | None = None) -> Conn: return self._client._dial(self.owner, self.id, self.token, deadline) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 5e468aa0..94db0da6 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -3,10 +3,9 @@ from __future__ import annotations import urllib.parse -from collections.abc import Mapping from typing import TYPE_CHECKING, Any -from .volumes import _volume_body +from .volumes import Volume, _volume_body if TYPE_CHECKING: from .client import Client @@ -27,7 +26,7 @@ def __init__(self, client: Client, addr: str, name: str, net: str, size: str, co def new( self, ttl_seconds: int = 0, - volumes: list[str | Mapping[str, str]] | None = None, + volumes: list[str | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, @@ -55,7 +54,7 @@ def _claim_body( net: str, size: str, ttl_seconds: int, - volumes: list[str | Mapping[str, str]] | None = None, + volumes: list[str | Volume] | None = None, mount: bool = True, claim_ref: str = "", ) -> dict[str, Any]: @@ -66,8 +65,10 @@ def _claim_body( claim["size"] = size if ttl_seconds: claim["ttl_seconds"] = ttl_seconds + if volumes is not None and not isinstance(volumes, list): + raise TypeError("volumes must be a list") if volumes: - claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] + claim["volumes"] = [_volume_body(volume) for volume in volumes] if not mount: claim["volumes_attach_only"] = True if claim_ref: diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index ce156554..586fd999 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -16,10 +16,11 @@ class _VolumeName(TypedDict): class Volume(_VolumeName, total=False): - """A catalog volume by name, with its guest mount point and an `rw` mode for writable claims.""" + """A catalog volume with a guest mount or caller-managed device attachment.""" mount: str mode: str + attach_only: bool class _VolumeState(TypedDict): @@ -51,29 +52,15 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N reply is None or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")) or not isinstance(reply.get("converging"), bool) + or any(not isinstance(entry, dict) for key in ("volumes_want", "volumes", "states") for entry in reply[key]) ): return None return cast(SandboxVolumesResponse, reply) -def _volume_body(volume: str | Mapping[str, str] | Volume, mount: bool) -> dict[str, str]: +def _volume_body(volume: str | Volume) -> Volume: if isinstance(volume, str): return {"name": volume} if not isinstance(volume, Mapping): raise TypeError("volume must be a name string or mapping") - unknown = sorted(set(volume) - {"name", "mount", "mode"}) - if unknown: - raise TypeError( - f"volume mapping accepts only name, mount, and mode, got unexpected key(s): {', '.join(unknown)}" - ) - if not mount and "mount" in volume: - raise TypeError("volume mount is meaningless with mount=False, which leaves mounting to the caller") - body = dict(cast(Mapping[str, str], volume)) - mode = body.get("mode") - if mode in (None, "", "ro"): - body.pop("mode", None) - elif mode != "rw": - name = volume.get("name") - prefix = f"volume {name!r}: " if name is not None else "" - raise TypeError(f"{prefix}mode must be 'rw' or 'ro', got {mode!r}") - return body + return volume.copy() diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 90b799dc..de1e43dc 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -75,8 +75,6 @@ def test_claim_sends_volumes(node): ("volumes", "match"), [ ([("imagenet", "/datasets/imagenet")], "name string or mapping"), - ([{"name": "imagenet", "mode": "rwx"}], "volume 'imagenet': mode must be 'rw' or 'ro', got 'rwx'"), - ([{"name": "imagenet", "bogus": "x"}], r"unexpected key\(s\): bogus"), ], ) def test_claim_rejects_invalid_volumes(node, volumes, match): @@ -100,11 +98,12 @@ def test_claim_sends_volume_mode_rw(node): assert sb.volumes == [{"name": "scratch", "mount": "/data", "mode": "rw"}] -def test_claim_omits_volume_mode_ro(node): +def test_claim_preserves_explicit_volume_mode(node): seen = recording_claim({"id": "sb_1", "token": "tok"}) Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mode": "ro"}]) Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mode": ""}]) - assert seen[0]["volumes"] == seen[1]["volumes"] == [{"name": "imagenet"}], seen + assert seen[0]["volumes"] == [{"name": "imagenet", "mode": "ro"}] + assert seen[1]["volumes"] == [{"name": "imagenet", "mode": ""}] def test_claim_attaches_volumes_without_mounting(node): @@ -130,13 +129,12 @@ def test_template_claim_attaches_volumes_without_mounting(node): assert sb.volumes == [{"name": "imagenet"}] -def test_claim_rejects_mount_without_mounting(node): - with pytest.raises(TypeError, match="meaningless with mount=False"): - Client(node).new("rt:24.04", volumes=[{"name": "imagenet", "mount": "/datasets"}], mount=False) - with pytest.raises(TypeError, match="meaningless with mount=False"): - Template(Client(node), node, "task:v1", "none", "small").new( - volumes=[{"name": "imagenet", "mount": "/datasets"}], mount=False - ) +def test_claim_leaves_volume_policy_to_server(node): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + entry = {"name": "imagenet", "mount": "/datasets", "attach_only": True, "mode": "future-mode"} + Client(node).new("rt:24.04", volumes=[entry], mount=False) + Template(Client(node), node, "task:v1", "none", "small").new(volumes=[entry], mount=False) + assert [request["volumes"] for request in seen] == [[entry], [entry]] def test_claim_keeps_mounting_by_default(node): diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 3226b82d..045a357e 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -46,8 +46,8 @@ def reply(body, path): @pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) def test_volume_error_preserves_typed_partial_view(spawn_node, method): partial = { - "volumes_want": [{"name": "new", "mount": "/new"}], - "volumes": [{"name": "old", "mount": "/old"}], + "volumes_want": [{"name": "new", "mode": "rw", "attach_only": True}], + "volumes": [{"name": "old", "attach_only": True}], "converging": False, "states": [ {"name": "new", "state": "pending", "error": "retry"}, @@ -164,3 +164,56 @@ def test_incomplete_volume_view_preserves_handle(spawn_node, status): def test_volume_methods_reject_a_bare_string(black_hole, call): with pytest.raises(TypeError): call(sandbox_at(black_hole)) + + +def test_mixed_volume_modes_roundtrip(spawn_node): + entries = [ + {"name": "mounted-ro", "mount": "/volumes/mounted-ro"}, + {"name": "mounted-rw", "mount": "/work", "mode": "rw"}, + {"name": "device-ro", "attach_only": True}, + {"name": "device-rw", "mode": "rw", "attach_only": True}, + ] + view = {"volumes_want": entries, "volumes": entries, "states": [], "converging": False} + seen = [] + + def reply(body, path): + seen.append(body) + return 200, view + + sb = sandbox_at(spawn_node({("GET", PATH): reply, ("PUT", PATH): reply, ("PATCH", PATH): reply})) + got = sb.get_volumes() + sb.set_volumes(got["volumes_want"]) + sb.patch_volumes(add=got["volumes"]) + assert seen == [{}, {"volumes": entries}, {"add": entries}] + assert sb.volumes == sb.volumes_want == entries + + +def test_attach_only_empty_mount_and_false_flag_remain_explicit(spawn_node): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply})) + entries = [{"name": "device", "mount": "", "attach_only": True}, {"name": "mounted", "attach_only": False}] + sb.set_volumes(entries) + assert seen == [{"volumes": entries}] + + +@pytest.mark.parametrize("status", [200, 409]) +@pytest.mark.parametrize("field", ["volumes", "volumes_want", "states"]) +def test_malformed_volume_entry_preserves_error_and_handle(spawn_node, status, field): + payload = {**VIEW, "error": "storage unavailable", field: [None]} + sb = sandbox_at(spawn_node({("GET", PATH): lambda body, path: (status, payload)})) + sb.volumes = [{"name": "old"}] + sb.volumes_want = [{"name": "wanted"}] + with pytest.raises(APIError) as caught: + sb.get_volumes() + assert sb.volumes == [{"name": "old"}] + assert sb.volumes_want == [{"name": "wanted"}] + if status == 409: + assert isinstance(caught.value, VolumeError) + assert caught.value.status == 409 + assert caught.value.message == "storage unavailable" + assert caught.value.reply == payload From 96303b2c20e2e520f65c7b1708d5fb02228ac482 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 03:12:03 +0000 Subject: [PATCH 31/39] docs: describe live volume modes and SDK compatibility Describe all four live volume modes and the coordinated daemon/SDK upgrade requirement so old default-mount behavior is not mistaken for attach-only support. --- docs/live-volumes.md | 4 +-- docs/sandboxd-api.md | 73 +++++++++++++++++++++++++++++--------------- docs/sdk-python.md | 22 +++++++++++-- docs/sdk.md | 14 +++++++-- 4 files changed, 81 insertions(+), 32 deletions(-) diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 43fa04cb..d1a1f4cc 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -13,8 +13,8 @@ A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) without editing static config. Registration persists only node-local catalog metadata and does not manage disk files. Runtime `tenants` lists follow the -[access-list rule](sandboxd-api.md#volume-access-lists). Live changes remain -limited to mounted read-only entries. +[access-list rule](sandboxd-api.md#volume-access-lists). The API also supports +[writable and attach-only entries](sandboxd-api.md#put-v1sandboxesidvolumes). Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 518cb71f..0d613381 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -87,13 +87,12 @@ Auth: `Authorization: Bearer ` (when configured). defaults to `/volumes/`; a custom value must be absolute and clean, outside the guest OS tree, unique, and non-nesting within the request. `mode` is `"ro"` (default, omitted) or `"rw"`; `"rw"` requires the catalog - entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)) -- `volumes_attach_only` (default `false`) attaches every requested volume - without mounting it, handing the whole mount contract to the workload. It - requires at least one volume, and rejects any entry carrying a `mount` — - the path is meaningless when the caller mounts the device. Everything below - describes the default, eager behaviour, which is completely unchanged; the - attach-only contract is in [its own section](#attach-only-volumes) + entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)). + `attach_only: true` attaches that entry without mounting it and requires an + empty or omitted `mount`. Mounted and attach-only entries can share a list +- `volumes_attach_only` (default `false`) is the legacy claim-wide form of + `attach_only`. The server expands it onto every entry before validation. + It requires at least one volume and rejects a nonempty `mount`. Success: @@ -135,12 +134,14 @@ an existing populated guest directory for the claim's life. ### Attach-only volumes -`"volumes_attach_only": true` stops after the attach. The device appears in -the guest, nothing is mounted, and the echoed entries carry no `mount` key: +`"attach_only": true` stops after the attach for that entry. The device appears +in the guest without a mount. Responses preserve `attach_only: true` and omit +`mount`: ```json {"id": "sb_…", "token": "…", - "volumes": [{"name": "imagenet"}, {"name": "scratch-db", "mode": "rw"}]} + "volumes": [{"name": "imagenet", "attach_only": true}, + {"name": "scratch-db", "mode": "rw", "attach_only": true}]} ``` Find each device by its virtio serial, which is the catalog name — poll this, @@ -221,7 +222,7 @@ an unknown and a forbidden volume return the same error text. Errors: 400 unknown template axis, metadata over a bound (the message names it), an unknown `on_expire`, invalid/duplicate volumes, -`volumes_attach_only` with no volumes or with an entry carrying a `mount`, +`volumes_attach_only` with no volumes, or an attach-only entry with a nonempty `mount`, `mode: "rw"` against a non-writable entry, or a volume that is unknown or forbidden (the latter two are deliberately indistinguishable), or @@ -371,22 +372,32 @@ Replaces the full desired set and converges synchronously. Auth matches GET. The owner tenant's catalog access list applies to newly admitted names, including for root. Retaining, repairing, or removing held names does not reauthorize them. A new image must exist on the owning node. The request must include a non-null `volumes` -array. An explicit empty array removes all mutable entries. +array. An explicit empty array removes all volumes. ```json {"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} ``` -Only mounted read-only catalog entries can change live (`mode` omitted or -`ro`). A PUT must retain every existing writable or attach-only entry unchanged. -The claim-time name, mount, and eight-volume limits apply to the desired set. -An omitted mount defaults to `/volumes/`, except that a held attach-only -entry is retained by sending it as the claim returned it, without `mount`. To -change an existing name's -mount or mode, first remove that name and wait for successful convergence. -An identical target can be retried. Claims with `on_expire: archive` must change -that policy before adding a volume. - +Mounted read-only, mounted writable, attach-only read-only, and attach-only +writable entries can change live, including mixed lists. The claim-time name, +mount, and eight-volume limits apply. An omitted mount defaults to +`/volumes/` unless that entry has `attach_only: true`. Responses carry +that flag directly, so a GET target can be sent back with PUT. Omitting the flag +or sending `false` does not inherit an existing attachment's shape. + +To change a held name's mount, mode, or `attach_only`, first remove the name and +wait for successful convergence. An identical target can be retried. Claims +with `on_expire: archive` must change that policy before adding a volume. + +New live writable attachments require a non-replicating pool, including a +promoted template's source pool. A replicating pool returns 409 before changing +intent, holds, or devices. Initial claim-time writable volumes remain supported. +An existing or pending replica, or this claim's unacknowledged checkpoint +retirement, returns 503 with `Retry-After`. A replica or archive intent left +with `archive_failover` disabled returns 409 because it cannot retire through +the registry. A retained local checkpoint reference alone still returns 503. +Volume mutation does not evict remote replicas. Read-only additions, exact +retries of held writers, and removals remain available under these restrictions. The sandbox must be running. Paused, archived, and failed sandboxes return 409 without waking. A failed sandbox holding volumes is release-only after its VMM exits; PUT with an empty target cannot restore its processes. Missing, invalid, @@ -407,13 +418,27 @@ unchanged. An empty patch retries the current desired set. ``` Adding an identical entry or removing an absent valid name is harmless. -Duplicate names, a name in both arrays, or changes to writable or attach-only -entries return 400. Removal refuses busy or foreign mounts and uses neither +Duplicate names, a name in both arrays, or changes to a held entry's mount, +mode, or `attach_only` return 400. Removal refuses busy or foreign mounts and uses neither lazy nor forced unmount. A foreign mount, another bind mount, or a completed ordinary unmount with nonzero exit returns 409 with the guest mount path and a safe per-volume reason. Guest transport failures remain server errors; timeouts return 504 and cancellation returns 408. +A mounted writable attachment marks the image dirty before the guest can write. +Live removal clears the marker only when this call observes and normally +unmounts the exact writable whole-device filesystem, finds no residual device +mounts, and confirms host detach. Both mount and superblock options must be +writable. A missing device or pre-unmounted filesystem does not prove clean +unmount. The marker remains on uncertainty and blocks subsequent readers. +Attach-only writers have no automatic dirty-marker protection. The caller owns +mounting, flushing, unmounting, recovery, and fencing before reuse. + +The per-entry field requires coordinated daemon and SDK releases. Older +responses without the field do not safely roundtrip attach-only intent through +new clients. The server still accepts the legacy claim-wide request flag and +normalizes old persisted empty mounts when loading claims. + ### Partial failure and recovery PUT and PATCH return the GET response shape on success. A convergence error diff --git a/docs/sdk-python.md b/docs/sdk-python.md index d726d366..c03e19fc 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -161,13 +161,29 @@ peer advertising both the template and every volume; when that intersection is empty, one volume holder may self-verify access to a shared template store before provisioning. -## Changing read-only datasets +## Changing live volumes Use `sb.get_volumes()`, `sb.set_volumes([...])`, and `sb.patch_volumes(add=[...], remove=[...])` on the same running sandbox. These methods use the client's API token on the owning node; a sandbox token cannot -manage datasets. `set_volumes([])` clears mutable entries; `None` is rejected. -PUT must retain writable and attach-only entries. PATCH retains omitted names. +manage volumes. `set_volumes([])` detaches all entries; `None` is rejected. +PATCH retains omitted names. Parameters take lists of names or `Volume` mappings. +The SDK sends volume policy to sandboxd and preserves `attach_only` directly. +Use matching daemon and SDK releases for attach-only GET-to-PUT retries. + +```python +from cocoonsandbox import Volume + +volumes: list[Volume] = [ + {"name": "dataset", "mount": "/datasets"}, + {"name": "work", "mode": "rw", "attach_only": True}, +] +view = sb.set_volumes(volumes) +``` + +The caller mounts, flushes, and unmounts attach-only devices. See the +[live volume contract](sandboxd-api.md#put-v1sandboxesidvolumes) for +writer exclusion, replica refusals, and dirty-marker limits. Each method takes `timeout=` (default: the client timeout, 120 s); a large replacement can need longer, and a call that times out cancels the server run, so retry the same target. diff --git a/docs/sdk.md b/docs/sdk.md index 2d66f2aa..c48d1eac 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -234,16 +234,17 @@ checkpoint branches support neither volumes nor a claim reference in this versio `Checkpoint.New`: the labels follow the [claim bounds](sandboxd-api.md#post-v1claim), and a branch never inherits its source's labels or expiry action. -For datasets chosen after the sandbox starts, set the desired read-only mounts -on the same claim: +For datasets chosen after the sandbox starts, set the desired volumes on +the same claim: ```go view, err := sb.SetVolumes(ctx, []sandbox.Volume{ {Name: "imagenet", Mount: "/datasets/training"}, + {Name: "work", Mode: "rw", AttachOnly: true}, }) view, err = sb.GetVolumes(ctx) view, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{"imagenet"}}) -// An explicit empty slice clears mutable mounts. A nil slice is rejected. +// An explicit empty slice detaches all volumes. A nil slice is rejected. view, err = sb.SetVolumes(ctx, []sandbox.Volume{}) ``` @@ -256,6 +257,13 @@ view = sb.patch_volumes(remove=["imagenet"]) view = sb.set_volumes([]) # None is rejected. ``` +`Volume.AttachOnly` leaves mounting and unmounting to the caller. Each entry +can independently use `Mode: "rw"` and `AttachOnly: true`. The SDK preserves +these fields directly and leaves name, mode, and mount policy to sandboxd. +Use matching daemon and SDK releases for attach-only GET-to-PUT retries. +See the [live volume contract](sandboxd-api.md#put-v1sandboxesidvolumes) for +writer exclusion, replica refusals, and dirty-marker limits. + Each response contains `volumes_want`, `volumes`, `states`, and `converging`. Go exposes them as `SandboxVolumesResponse.VolumesWant`, `.Volumes`, `.States`, and `.Converging`. `VolumePhase` has `VolumeAttached`, `VolumePending`, and From d8f2943bd0bbfb40362491ad1f3aa0625a903f82 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 03:12:32 +0000 Subject: [PATCH 32/39] volumes: clarify rollout and finish replica test lint Keep the release and dirty-marker limits beside the API and make the replica tests match repository lint rules. --- docs/sandboxd-api.md | 10 ++++++---- sandboxd/pool/volume_replica_test.go | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 0d613381..58f3e121 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -434,10 +434,12 @@ unmount. The marker remains on uncertainty and blocks subsequent readers. Attach-only writers have no automatic dirty-marker protection. The caller owns mounting, flushing, unmounting, recovery, and fencing before reuse. -The per-entry field requires coordinated daemon and SDK releases. Older -responses without the field do not safely roundtrip attach-only intent through -new clients. The server still accepts the legacy claim-wide request flag and -normalizes old persisted empty mounts when loading claims. +The per-entry field requires coordinated daemon and SDK releases. Older daemons +can ignore `attach_only` in claims, and older Python SDKs reject the field when +returned descriptors are reused in requests. Responses without the field do +not safely roundtrip attach-only intent through new clients. The server still +accepts the legacy claim-wide request flag and normalizes old persisted empty +mounts when loading claims. ### Partial failure and recovery diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 0d6c4386..0261f925 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -2,12 +2,13 @@ package pool import ( "errors" - "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/types" "slices" "sync" "testing" "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" ) func TestReplicaRecoveryDoesNotRepublishReadOnlyHold(t *testing.T) { @@ -138,7 +139,7 @@ func TestLiveWriterPolicyAndCheckpointRefusals(t *testing.T) { m.store.set(sb) m.archives = nil }, ErrReplicaPresent}, - {"unrelated checkpoint", func(m *Manager, sb *types.Sandbox) { + {"unrelated checkpoint", func(m *Manager, _ *types.Sandbox) { disableReplication(m) other := mustClaim(t, m, testKey) other.ReplicaCk = "unrelated" From 8bd6a581567074292ac7f4be3a703ae31d12cddf Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 03:21:05 +0000 Subject: [PATCH 33/39] sdk: encode accepted volume mappings as JSON objects Accepted mapping inputs need conversion to plain dictionaries before JSON serialization. Preserve string shorthand without duplicating server validation. --- sdk/go/volumes_test.go | 2 +- sdk/python/cocoonsandbox/volumes.py | 2 +- sdk/python/tests/test_volumes.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index f35563bf..57dda6ce 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -274,7 +274,7 @@ func TestVolumePolicyErrorsComeFromServer(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"mode must be ro or rw"}`)) })) diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index 586fd999..78f6dfca 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -63,4 +63,4 @@ def _volume_body(volume: str | Volume) -> Volume: return {"name": volume} if not isinstance(volume, Mapping): raise TypeError("volume must be a name string or mapping") - return volume.copy() + return cast(Volume, dict(volume)) diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 045a357e..2fae7b1c 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -1,5 +1,7 @@ """Live volume HTTP contract and partial convergence recovery.""" +from collections import UserDict + import pytest from conftest import FakeNode, sandbox_at @@ -14,6 +16,18 @@ } +def test_volume_mapping_is_json_encoded(spawn_node): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, VIEW + + sb = sandbox_at(spawn_node({("PUT", PATH): reply})) + sb.set_volumes([UserDict({"name": "data", "mount": "/data"})]) + assert seen == [{"volumes": [{"name": "data", "mount": "/data"}]}] + + @pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) def test_volume_requests_use_owner_and_api_token(spawn_node, method): seen = [] From c48f1bd75533356eff13ab4b9eb1a89a70de135b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 03:21:05 +0000 Subject: [PATCH 34/39] docs: correct live volume mode restrictions The old read-only wording no longer describes the live API. Document writable and attach-only behavior and its remaining restrictions. --- README.md | 2 +- docs/deploy.md | 11 ++++++++--- docs/live-volumes.md | 7 ++++--- docs/sdk.md | 11 +++++++---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 04468e8d..a69644d0 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ performance) — source in - `sandboxd/` — per-node control plane (Go): warm pools refilled from golden snapshot exports (online-retunable), claim/release/renew/hibernate/fork/ promote/checkpoint HTTP API, operator-catalog dataset volumes (read-only or - writable; read-only ones change on a running claim), signed preview URLs, + writable, mounted or attach-only, with live add and remove), signed preview URLs, the HTTP-upgrade byte relays to silkd and to any guest port, read-only file access to hibernated or archived claims from a warm VM of their pool (opt-in per pool `reader`), claim env with diff --git a/docs/deploy.md b/docs/deploy.md index a704a01b..70e25d4d 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -248,9 +248,13 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes -Running claims can change mounted read-only catalog entries through the +Running claims can add or remove mounted and attach-only catalog entries in +read-only or writable mode through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) -and both SDKs. The image must already exist on the owner node. Live changes +and both SDKs. Changing a held entry's descriptor requires successful removal +first. The [live volume contract](sandboxd-api.md#put-v1sandboxesidvolumes) +defines admission policy and dirty-marker handling. +The image must already exist on the owner node. Live changes require Cocoon v0.7.2 or newer for idempotent disk attach and detach. PUT and PATCH reject an older release, or a runtime whose version cannot be queried, with 503 before any intent, hold, or device change; a dev build is assumed @@ -338,7 +342,8 @@ below. With `directio=off`, readers share the host page cache; use `directio=on` when cache interference matters. A dataset mounted into an egress-lane sandbox can be uploaded wherever that tenant's egress policy permits, so treat the ACL and egress policy as one access -decision. Image replication, detach, and refcounting remain out of scope. +decision. Operators remain responsible for image distribution and cross-node +ownership. sandboxd tracks local holds and handles live detach through the volume API. #### Writable dataset volumes diff --git a/docs/live-volumes.md b/docs/live-volumes.md index d1a1f4cc..da04c979 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -65,9 +65,10 @@ with client.new("rt:24.04", ttl_seconds=900) as sb: A successful run reads `v1`, then `v2`, and preserves every identity assertion. The repeated full target verifies that a settled mount can be retained. -This example starts without other volumes. If your sandbox has writable or -attach-only volumes, retain them in a full PUT or use PATCH to replace only -the dataset names. `None`, a nil Go slice, and a missing `volumes` field do not +This example starts without other volumes. Retain any volumes you intend to +keep in a full PUT, or use PATCH to replace only the dataset names. All mounted +and attach-only entries support live removal in either mode. +`None`, a nil Go slice, and a missing `volumes` field do not mean an empty target. Go uses the same owning handle and client API token. diff --git a/docs/sdk.md b/docs/sdk.md index c48d1eac..af049af4 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -282,11 +282,14 @@ the handle retains its last known sets. Retry an interrupted operation with the same full PUT target, or set a new target to recover. `Client.Sandboxes` exposes `VolumesWant` and `Volumes` for inspection with an API token. -A PUT must retain existing writable and attach-only entries unchanged. PATCH -cannot remove or change them. Paused sandboxes are not woken, and images must -exist on the owner node. Admission holds cover the union of desired and applied +PUT and PATCH can add or remove mounted and attach-only entries in either mode. +Changing a held entry's mount, mode, or `attach_only` requires successful +removal first. See the [live volume contract](sandboxd-api.md#put-v1sandboxesidvolumes) +for admission policy and dirty-marker handling. Paused sandboxes are not woken, +and images must exist on the owner node. Admission holds cover the union of desired and applied sets. A settled retained entry receives no new attach or mount calls. `converging` -is true only during an active request. Pending or detaching entries with false +is true during an active request or while a recovery-only reference awaits an +acknowledged claim write. Pending or detaching entries with false require an explicit retry. Daemon restart replaces unfinished diagnostics with an interrupted message. Capture is refused while either set is nonempty. Release cancels convergence and hands held volumes to teardown. See the From 793e5c8325dcdc4817b7dd03a23d8b1c8ef59f5d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 07:13:37 +0000 Subject: [PATCH 35/39] docs: explain archive failover shutdown Turning off registry support first strands replica references. Document retirement checks and shared ownership risks before the operator disables failover. --- docs/deploy.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index 70e25d4d..1f7e72dd 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -145,7 +145,7 @@ gets the intersection of the two. | `reader` (pool entry) | `false` | Serve read-only file calls on this pool's hibernated or archived claims from a warm VM of the same pool instead of waking them; needs `warm` of at least 1 and a cocoon whose `vm disk attach` takes `--from-snapshot` and `--from-dir`. See [Reading sleeping claims](#reading-sleeping-claims-without-waking-them). Config-owned like `capture_trim`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one more pass, since a claim can run first in one pass and last in the next. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory; the first round of a claim after a wake or adoption uploads the whole claim, so that pass is the long pass. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one more pass, since a claim can run first in one pass and last in the next. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory; the first round of a claim after a wake or adoption uploads the whole claim, so that pass is the long pass. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 starts retiring its replicas while the registry remains enabled; see [Disabling archive failover](#disabling-archive-failover)) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | @@ -699,6 +699,32 @@ drains the warm pools; poll `GET /v1/info` until `claimed` reaches zero (or let the leases expire), then stop sandboxd. `DELETE /v1/drain` uncordons. The drain is not persisted — a restarted node serves again. +### Disabling archive failover + +Pause archive-producing work. Before setting `archive_failover` to `false`: + +1. Set every affected pool's `replicate_seconds` to `0` and restart its nodes + with failover and the registry still enabled. Keep the registry reachable + while replica retirement retries. +2. Check each node's `/claims.json` for affected claims. Wait until + `replica_ck` and `replica_pending` are absent after successful journal writes. + Use `sync_claims: true` when completion must survive power loss. +3. Wake or release existing registered archives while failover remains enabled. + Wait for `archive_pending` to clear in the journal. Verify that the affected + IDs have no rows in PostgreSQL's + `sandboxd_archives` table for your `meta_store.cell`. A consumed row is a + non-adoptable tombstone; wait for its cleanup retries to finish too. +4. Set `archive_failover` to `false` and restart the nodes. + +The public API omits replica journal fields. A fixed delay or an absence of +warnings does not prove retirement. Disabling failover first stops replica +retirement and leaves residual references that reject new live RW attachments. +A local archive can still wake and clear `archive_ck`, but that does not clear a +retained `replica_pending` or retire an old shared row. Release can remove local +state without reconciling that row. Re-enabling failover against stale rows can +fence or drop a local claim if another node has adopted it. Resolve registry +ownership before serving that claim again. + ### Shared tenant database With `meta_store` set, every node reads tenants from one PostgreSQL table From 3af81a06094297b4e78f3a7a048e799970fb0736 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 07:13:38 +0000 Subject: [PATCH 36/39] e2e: cover live volume modes and clean removal Exercise the SDK and HTTP boundary for all four modes and both dirty-marker outcomes. Keep this proof with the live feature rather than the later controller workflow. --- e2e/fakeengine_test.go | 52 ++++++++++++++--- e2e/volume_mutation_test.go | 108 ++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 39aa4951..e57368b5 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -17,6 +17,11 @@ import ( "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" ) +type fakeVolumeMount struct { + name string + rw bool +} + type fakeEngine struct { real *engine.Engine dir string @@ -27,6 +32,7 @@ type fakeEngine struct { volumeOps []string volumeVMs map[string]bool guestEnvs map[string]string + mounts map[string]map[string]fakeVolumeMount seq int readerRoot string @@ -42,6 +48,7 @@ func newFakeEngine(dir string) *fakeEngine { socks: map[string]string{}, volumeVMs: map[string]bool{}, guestEnvs: map[string]string{}, + mounts: map[string]map[string]fakeVolumeMount{}, } } @@ -66,6 +73,7 @@ func (f *fakeEngine) Remove(_ context.Context, name string) error { _ = l.Close() } delete(f.listeners, name) + delete(f.mounts, f.socks[name]) delete(f.socks, name) if f.volumeVMs[name] { delete(f.volumeVMs, name) @@ -190,10 +198,14 @@ func (f *fakeEngine) DiskAttach(_ context.Context, vmName string, spec engine.Vo return nil } -func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bool) error { +func (f *fakeEngine) MountVolume(_ context.Context, sock, name, mount string, rw bool) error { f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "mount:"+name+":"+mount+":"+volumeMode(rw)) + if f.mounts[sock] == nil { + f.mounts[sock] = map[string]fakeVolumeMount{} + } + f.mounts[sock][mount] = fakeVolumeMount{name: name, rw: rw} return nil } @@ -201,10 +213,10 @@ func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount stri return f.MountVolume(ctx, sock, name, mount, rw) } -func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { +func (f *fakeEngine) UnmountVolume(_ context.Context, sock, mount string) error { f.mu.Lock() defer f.mu.Unlock() - f.volumeOps = append(f.volumeOps, "umount:"+mount) + f.unmountVolumeLocked(sock, mount) return nil } @@ -215,16 +227,38 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (bool, error) { - if mount != "" { - if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return false, err +func (f *fakeEngine) DetachVolume(_ context.Context, _, sock, name, mount string) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + entry, mounted := f.mounts[sock][mount] + normal := mount != "" && mounted && entry.name == name && entry.rw + if mount != "" && mounted && entry.name == name { + f.unmountVolumeLocked(sock, mount) + } + for _, other := range f.mounts[sock] { + if other.name == name { + return false, &engine.VolumeMountConflict{Message: "volume still has a simulated guest mount"} } } + f.volumeOps = append(f.volumeOps, "detach:"+name) + return normal, nil +} + +func (f *fakeEngine) unmountNamedVolume(name, mount string) error { f.mu.Lock() defer f.mu.Unlock() - f.volumeOps = append(f.volumeOps, "detach:"+name) - return false, nil + for sock, mounts := range f.mounts { + if mounts[mount].name == name { + f.unmountVolumeLocked(sock, mount) + return nil + } + } + return fmt.Errorf("volume %q is not mounted at %q", name, mount) +} + +func (f *fakeEngine) unmountVolumeLocked(sock, mount string) { + f.volumeOps = append(f.volumeOps, "umount:"+mount) + delete(f.mounts[sock], mount) } func (f *fakeEngine) volumeOpsLog() []string { diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index ccd6d01d..f1310573 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -3,6 +3,7 @@ package e2e import ( "errors" "net/http" + "os" "slices" "strings" "testing" @@ -11,6 +12,59 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) +func TestLiveWritableRemovalClearsMarkerThroughSDK(t *testing.T) { + for _, tt := range []struct { + name string + preUnmount bool + }{ + {name: "daemon unmount"}, + {name: "external unmount", preUnmount: true}, + } { + t.Run(tt.name, func(t *testing.T) { + image := writeVolumeImage(t, "data.img", "original-data") + stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "data", Path: image, Writable: true}}) + writer, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer writer.Close() + reader, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer reader.Close() + if _, err = writer.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data", Mode: "rw"}}); err != nil { + t.Fatal(err) + } + if _, err = os.Stat(image + ".dirty"); err != nil { + t.Fatalf("writer has no dirty marker: %v", err) + } + if _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}); err == nil { + t.Fatal("reader admitted while writer was attached") + } + if tt.preUnmount { + if err = stack.eng.unmountNamedVolume("data", "/volumes/data"); err != nil { + t.Fatal(err) + } + } + if _, err = writer.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatal(err) + } + _, markerErr := os.Stat(image + ".dirty") + if tt.preUnmount && markerErr != nil || !tt.preUnmount && !errors.Is(markerErr, os.ErrNotExist) { + t.Fatalf("dirty marker disagrees with current-call proof: preUnmount=%t err=%v", tt.preUnmount, markerErr) + } + _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}) + if (err != nil) != tt.preUnmount { + t.Fatalf("reader admission after detach: preUnmount=%t err=%v", tt.preUnmount, err) + } + if data, readErr := os.ReadFile(image); readErr != nil || string(data) != "original-data" { + t.Fatalf("detach changed backing contents: %q, %v", data, readErr) + } + }) + } +} + func TestLiveVolumesEndToEnd(t *testing.T) { image := writeVolumeImage(t, "dataset.img", "dataset-bytes") stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "dataset", Path: image}}) @@ -77,3 +131,57 @@ func TestEmptyVolumePatchChangesNothing(t *testing.T) { t.Fatalf("empty patch: %+v err=%v", got, err) } } + +func TestMixedLiveVolumeModesThroughSDK(t *testing.T) { + volumes := []sandbox.Volume{ + {Name: "mounted-ro"}, + {Name: "mounted-rw", Mode: "rw"}, + {Name: "device-ro", AttachOnly: true}, + {Name: "device-rw", Mode: "rw", AttachOnly: true}, + } + var catalog []config.VolumeSpec + for _, volume := range volumes { + catalog = append(catalog, config.VolumeSpec{ + Name: volume.Name, Path: writeVolumeImage(t, volume.Name+".img", "seed"), Writable: volume.Mode == "rw", + }) + } + stack := startTenantStack(t, "node-token", nil, catalog) + sb, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer sb.Close() + view, err := sb.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: volumes}) + if err != nil { + t.Fatal(err) + } + want := slices.Clone(volumes) + want[0].Mount, want[1].Mount = "/volumes/mounted-ro", "/volumes/mounted-rw" + if !slices.Equal(view.Volumes, want) || !slices.Equal(sb.Volumes, want) { + t.Fatalf("mixed SDK attachment = %+v, handle=%+v, want %+v", view, sb.Volumes, want) + } + status, body := rawJSON(t, stack, http.MethodGet, "/v1/sandboxes/"+sb.ID+"/volumes", "") + if status != http.StatusOK || strings.Count(string(body), `"attach_only":true`) != 4 { + t.Fatalf("mixed HTTP view lost explicit attach_only in desired or applied entries: status=%d body=%s", status, body) + } + view, err = sb.GetVolumes(t.Context()) + if err != nil { + t.Fatal(err) + } + view, err = sb.SetVolumes(t.Context(), view.VolumesWant) + if err != nil || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("mixed GET-to-PUT = %+v, err=%v", view, err) + } + view, err = sb.SetVolumes(t.Context(), []sandbox.Volume{}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || view.Converging { + t.Fatalf("mixed removal = %+v, err=%v", view, err) + } + initial, err := stack.client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volumes...)) + if err != nil { + t.Fatalf("mixed initial claim: %v", err) + } + defer initial.Close() + if !slices.Equal(initial.Volumes, want) { + t.Fatalf("mixed initial claim = %+v, want %+v", initial.Volumes, want) + } +} From 750357875d5b6e60d5b15c9f7e4084b3d92b433f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 08:27:33 +0000 Subject: [PATCH 37/39] docs: name the archive triggers to pause Disabling failover requires stopping both idle-time archives and archive-on-expiry claims. Name the configuration and renew operation so operators can stop new archive rows before draining the registry. --- docs/deploy.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index 1f7e72dd..f9abff44 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -701,7 +701,11 @@ The drain is not persisted — a restarted node serves again. ### Disabling archive failover -Pause archive-producing work. Before setting `archive_failover` to `false`: +Stop admitting claims that use `on_expire: archive`. Set the node and affected +pools' `archive_after_seconds` to `0`, and use +[`renew`](sandboxd-api.md#post-v1sandboxesidrenew) to switch existing active claims +to `on_expire: destroy` while keeping their leases valid. These changes stop +idle-time and expiry-triggered archives. Before setting `archive_failover` to `false`: 1. Set every affected pool's `replicate_seconds` to `0` and restart its nodes with failover and the registry still enabled. Keep the registry reachable From 247e3fb8d2e8bee58ed8eae4b6312feee5e10eae Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 08:29:48 +0000 Subject: [PATCH 38/39] pool: preserve live volume staging exclusion Keep Transition across the dirty-marker stat so lifecycle changes cannot interleave between reservation and staging. Release the manager mutex during the stat and keep device calls outside Transition. Both writer-admission callers already reject archived claims under the manager mutex. Remove those unreachable checks and keep the detach proof contract in a shorter comment. The pool and end-to-end suites cover the mixed GET-to-PUT contract, so remove the duplicate server test. Make marker-removal cases declare their setup and expected outcomes directly. --- e2e/volume_mutation_test.go | 26 ++++++++------- sandboxd/engine/volume_live.go | 5 ++- sandboxd/pool/volume_mutation.go | 10 +++--- sandboxd/server/volume_mutation_test.go | 42 ------------------------- 4 files changed, 20 insertions(+), 63 deletions(-) diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index f1310573..38dd119c 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -14,11 +14,15 @@ import ( func TestLiveWritableRemovalClearsMarkerThroughSDK(t *testing.T) { for _, tt := range []struct { - name string - preUnmount bool + name string + beforeDetach func(*stack) error + markerErr error + readerBlocked bool }{ - {name: "daemon unmount"}, - {name: "external unmount", preUnmount: true}, + {name: "daemon unmount", beforeDetach: func(*stack) error { return nil }, markerErr: os.ErrNotExist}, + {name: "external unmount", beforeDetach: func(s *stack) error { + return s.eng.unmountNamedVolume("data", "/volumes/data") + }, readerBlocked: true}, } { t.Run(tt.name, func(t *testing.T) { image := writeVolumeImage(t, "data.img", "original-data") @@ -42,21 +46,19 @@ func TestLiveWritableRemovalClearsMarkerThroughSDK(t *testing.T) { if _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}); err == nil { t.Fatal("reader admitted while writer was attached") } - if tt.preUnmount { - if err = stack.eng.unmountNamedVolume("data", "/volumes/data"); err != nil { - t.Fatal(err) - } + if err = tt.beforeDetach(stack); err != nil { + t.Fatal(err) } if _, err = writer.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"data"}}); err != nil { t.Fatal(err) } _, markerErr := os.Stat(image + ".dirty") - if tt.preUnmount && markerErr != nil || !tt.preUnmount && !errors.Is(markerErr, os.ErrNotExist) { - t.Fatalf("dirty marker disagrees with current-call proof: preUnmount=%t err=%v", tt.preUnmount, markerErr) + if !errors.Is(markerErr, tt.markerErr) { + t.Fatalf("dirty marker: got %v, want %v", markerErr, tt.markerErr) } _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}) - if (err != nil) != tt.preUnmount { - t.Fatalf("reader admission after detach: preUnmount=%t err=%v", tt.preUnmount, err) + if (err != nil) != tt.readerBlocked { + t.Fatalf("reader admission after detach: blocked=%t err=%v", tt.readerBlocked, err) } if data, readErr := os.ReadFile(image); readErr != nil || string(data) != "original-data" { t.Fatalf("detach changed backing contents: %q, %v", data, readErr) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index c69ff92e..f6af7f3f 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -62,9 +62,8 @@ func (e *Engine) MountLiveVolume(ctx context.Context, vsockSocket, name, mount s return e.mountDevice(ctx, vsockSocket, device, name, mount, rw) } -// DetachVolume unmounts a guest volume before confirming host removal. -// cleanRW requires an exact whole-device mount with writable mount and superblock -// options, a normal unmount, no residual device mounts, and successful host detach. +// DetachVolume returns cleanRW only after normally unmounting the exact whole-device +// mount with writable mount and superblock options, no residual mounts, and successful host detach. func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) (cleanRW bool, err error) { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 0573087d..222fb6cc 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -265,9 +265,9 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru } func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { + sb.Transition.Lock() + defer sb.Transition.Unlock() if err := func() error { - sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -291,8 +291,6 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return claimSnapshot{}, false, err } cleanErr := m.confirmVolumesClean(resolved) - sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -318,10 +316,10 @@ func (m *Manager) liveWriterErrorLocked(sb *types.Sandbox) error { if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { return ErrReplicatingPool } - if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "" || sb.ArchiveCk != "") { + if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "") { return ErrReplicaStranded } - if sb.HasReplica() || sb.ArchivePending != "" || sb.ArchiveCk != "" || m.store.hasCheckpointReferences(sb.ID) { + if sb.HasReplica() || sb.ArchivePending != "" || m.store.hasCheckpointReferences(sb.ID) { return ErrReplicaPresent } return nil diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index baca0330..4cd91e17 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -181,48 +181,6 @@ func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { } } -func TestLiveVolumeRoutesPreserveMixedAttachOnlyRoundtrip(t *testing.T) { - want := []types.Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} - view := types.SandboxVolumesResponse{VolumesWant: want, Volumes: want} - mgr := &fakeManager{ - getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, - setVolumes: func(_, _ string, got []types.Volume) (types.SandboxVolumesResponse, error) { - if !slices.Equal(got, want) { - t.Errorf("GET-to-PUT changed mixed intent: %+v", got) - } - return view, nil - }, - patchVolumes: func(_, _ string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { - if !slices.Equal(patch.Add, want) { - t.Errorf("PATCH lost per-entry intent: %+v", patch.Add) - } - return view, nil - }, - } - ts := newTenantTestServer(t, "root", nil, mgr, nil) - get := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", "") - defer get.Body.Close() - var got types.SandboxVolumesResponse - if err := json.UnmarshalRead(get.Body, &got); err != nil || !slices.Equal(got.VolumesWant, want) { - t.Fatalf("GET lost per-entry attachment intent: %+v, %v", got, err) - } - for _, method := range []string{http.MethodPut, http.MethodPatch} { - key := "volumes" - if method == http.MethodPatch { - key = "add" - } - body, err := json.Marshal(map[string][]types.Volume{key: got.VolumesWant}) - if err != nil { - t.Fatal(err) - } - resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", string(body)) - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("%s status=%d", method, resp.StatusCode) - } - } -} - func TestClaimVolumeCompatibilityExpansion(t *testing.T) { for _, tt := range []struct { name string From e9c236d8400d374f405d1696586cd41905ef9c08 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 27 Oct 2026 08:29:14 +0000 Subject: [PATCH 39/39] sdk: focus live volume validation and wire tests The handle copies volumes and volumes_want entries before returning a view or raising VolumeError, so malformed entries must still be rejected there. States are passed through without copying, so drop their unused entry scan while retaining the response envelope checks. Replace repeated mixed-mode GET-to-PUT scenarios with concrete SDK wire checks. Keep Go zero-field omission, Python explicit false and empty mount preservation, mountless response decoding, auth, partial errors, handle copies, and cancellation coverage. Put request expectations directly in the existing case tables and shorten the claim-wide attach-only comment without dropping caller responsibilities. --- sdk/go/options.go | 3 +- sdk/go/volume_mutation_test.go | 71 +++++++++++--------------- sdk/python/cocoonsandbox/volumes.py | 2 +- sdk/python/tests/test_volumes.py | 78 ++++++++++++----------------- 4 files changed, 64 insertions(+), 90 deletions(-) diff --git a/sdk/go/options.go b/sdk/go/options.go index 036d3539..3b0b815f 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -83,8 +83,7 @@ func WithVolumes(volumes ...Volume) Option { } // WithVolumesAttachOnly is the legacy claim-wide form of Volume.AttachOnly. -// The caller owns mounting, flushing, and unmounting every attached device. -// Sandbox release does not sync these devices. +// Callers mount, flush, and unmount devices; release does not sync them. func WithVolumesAttachOnly() Option { return func(r *claimRequest) { r.VolumesAttachOnly = true } } diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index a49e2890..fab7c423 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -13,31 +13,28 @@ import ( ) func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { - for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch} { - t.Run(method, func(t *testing.T) { + for _, request := range []struct{ method, body string }{ + {http.MethodGet, ""}, + {http.MethodPut, `{"volumes":[{"name":"data"}]}`}, + {http.MethodPatch, `{"add":[{"name":"data"}],"remove":["old"]}`}, + } { + t.Run(request.method, func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != method || r.URL.Path != "/v1/sandboxes/sb_1/volumes" || r.Header.Get("Authorization") != "Bearer operator-token" { + if r.Method != request.method || r.URL.Path != "/v1/sandboxes/sb_1/volumes" || r.Header.Get("Authorization") != "Bearer operator-token" { t.Errorf("request: %s %s %s", r.Method, r.URL.Path, r.Header.Get("Authorization")) } body, _ := io.ReadAll(r.Body) - want := "" - switch method { - case http.MethodPut: - want = `{"volumes":[{"name":"data"}]}` - case http.MethodPatch: - want = `{"add":[{"name":"data"}],"remove":["old"]}` - } - if string(body) != want { - t.Errorf("body=%s want=%s", body, want) + if string(body) != request.body { + t.Errorf("body=%s want=%s", body, request.body) } - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","mount":"/data"}],"volumes":[{"name":"data","mount":"/data"}],"converging":true,"states":[{"name":"data","state":"attached"}]}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"data","attach_only":true}],"volumes":[{"name":"data","attach_only":true}],"converging":true,"states":[{"name":"data","state":"attached"}]}`) })) t.Cleanup(ts.Close) c := testClient(t, ts, WithAPIToken("operator-token")) sb := &Sandbox{ID: "sb_1", token: "sandbox-token", owner: c.addr, c: c} c.addr = "unused.invalid:7777" - got, err := callVolumes(t, sb, method) - if err != nil || !slices.Equal(got.Volumes, []Volume{{Name: "data", Mount: "/data"}}) || !slices.Equal(got.VolumesWant, got.Volumes) || got.States[0].State != VolumeAttached || !got.Converging { + got, err := callVolumes(t, sb, request.method) + if err != nil || !slices.Equal(got.Volumes, []Volume{{Name: "data", AttachOnly: true}}) || !slices.Equal(got.VolumesWant, got.Volumes) || got.States[0].State != VolumeAttached || !got.Converging { t.Fatalf("got=%+v err=%v", got, err) } got.Volumes[0].Name, got.VolumesWant[0].Name = "caller copy", "caller copy" @@ -63,7 +60,10 @@ func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { if !isVolume || !isAPI || api.Status != http.StatusInternalServerError || failure.Response == nil { t.Fatalf("%s error lost response or APIError: %v", method, err) } - if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" || sb.VolumesWant[0].Name != "new" || sb.Volumes[0].Name != "old" { + if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" { + t.Fatalf("partial view=%+v", got) + } + if !slices.Equal(sb.VolumesWant, []Volume{{Name: "new", Mode: "rw", AttachOnly: true}}) || !slices.Equal(sb.Volumes, []Volume{{Name: "old", AttachOnly: true}}) { t.Fatalf("partial view=%+v handle=%+v", got, sb) } failure.Response.Volumes[0].Name = "error caller" @@ -177,41 +177,28 @@ func TestLiveVolumeIncompleteViewPreservesHandle(t *testing.T) { } } -func TestLiveVolumeAttachOnlyRoundTrip(t *testing.T) { - const entries = `[{"name":"mounted-ro","mount":"/volumes/mounted-ro"},{"name":"mounted-rw","mount":"/work","mode":"rw"},{"name":"device-ro","attach_only":true},{"name":"device-rw","mode":"rw","attach_only":true}]` - var forwarded []map[string]any +func TestLiveVolumeMutationEncodesDeviceFields(t *testing.T) { + const want = `{"volumes":[{"name":"device","mode":"rw","attach_only":true},{"name":"mounted","mount":"/work"}]}` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPut { - var body struct { - Volumes []map[string]any `json:"volumes"` - } - if err := json.UnmarshalRead(r.Body, &body); err != nil { - t.Errorf("decode replacement: %v", err) - } - forwarded = body.Volumes + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + if r.Method != http.MethodPut || string(body) != want { + t.Errorf("request=%s %s want=PUT %s", r.Method, body, want) } - _, _ = io.WriteString(w, `{"volumes_want":`+entries+`,"volumes":`+entries+`,"states":[],"converging":false}`) + _, _ = io.WriteString(w, `{"volumes_want":[],"volumes":[],"states":[],"converging":false}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) sb := &Sandbox{ID: "sb_1", owner: c.addr, c: c} - view, err := sb.GetVolumes(t.Context()) + _, err := sb.SetVolumes(t.Context(), []Volume{ + {Name: "device", Mode: "rw", AttachOnly: true}, + {Name: "mounted", Mount: "/work", AttachOnly: false}, + }) if err != nil { t.Fatal(err) } - if _, err = sb.SetVolumes(t.Context(), view.VolumesWant); err != nil { - t.Fatal(err) - } - var want []map[string]any - if err = json.Unmarshal([]byte(entries), &want); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(forwarded, want) { - t.Fatalf("GET-to-PUT changed volume intent: got=%v want=%v", forwarded, want) - } - if !slices.Equal(sb.Volumes, view.Volumes) || !slices.Equal(sb.VolumesWant, view.VolumesWant) { - t.Fatal("roundtrip changed handle state") - } } func callVolumes(t *testing.T, sb *Sandbox, method string) (SandboxVolumesResponse, error) { diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index 78f6dfca..209aaa50 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -52,7 +52,7 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N reply is None or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")) or not isinstance(reply.get("converging"), bool) - or any(not isinstance(entry, dict) for key in ("volumes_want", "volumes", "states") for entry in reply[key]) + or any(not isinstance(entry, dict) for key in ("volumes_want", "volumes") for entry in reply[key]) ): return None return cast(SandboxVolumesResponse, reply) diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 2fae7b1c..516d1245 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -28,8 +28,19 @@ def reply(body, path): assert seen == [{"volumes": [{"name": "data", "mount": "/data"}]}] -@pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) -def test_volume_requests_use_owner_and_api_token(spawn_node, method): +@pytest.mark.parametrize( + "method,body,call", + [ + ("GET", {}, lambda sb: sb.get_volumes()), + ("PUT", {"volumes": [{"name": "data"}]}, lambda sb: sb.set_volumes(["data"])), + ( + "PATCH", + {"add": [{"name": "data"}], "remove": ["old"]}, + lambda sb: sb.patch_volumes(add=["data"], remove=["old"]), + ), + ], +) +def test_volume_requests_use_owner_and_api_token(spawn_node, method, body, call): seen = [] def reply(body, path): @@ -39,15 +50,8 @@ def reply(body, path): addr = spawn_node({(method, PATH): reply}) sb = sandbox_at(addr, api_token="operator-token") sb._client.addr = "unused.invalid:7777" - if method == "GET": - got = sb.get_volumes() - assert seen == [{}] - elif method == "PUT": - got = sb.set_volumes(["data"]) - assert seen == [{"volumes": [{"name": "data"}]}] - else: - got = sb.patch_volumes(add=["data"], remove=["old"]) - assert seen == [{"add": [{"name": "data"}], "remove": ["old"]}] + got = call(sb) + assert seen == [body] assert FakeNode.last_headers["Authorization"] == "Bearer operator-token" assert got == VIEW assert sb.volumes == VIEW["volumes"] @@ -57,8 +61,15 @@ def reply(body, path): assert sb.volumes[0]["name"] == sb.volumes_want[0]["name"] == "data" -@pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) -def test_volume_error_preserves_typed_partial_view(spawn_node, method): +@pytest.mark.parametrize( + "method,call", + [ + ("GET", lambda sb: sb.get_volumes()), + ("PUT", lambda sb: sb.set_volumes(["new"])), + ("PATCH", lambda sb: sb.patch_volumes(remove=["old"])), + ], +) +def test_volume_error_preserves_typed_partial_view(spawn_node, method, call): partial = { "volumes_want": [{"name": "new", "mode": "rw", "attach_only": True}], "volumes": [{"name": "old", "attach_only": True}], @@ -71,12 +82,7 @@ def test_volume_error_preserves_typed_partial_view(spawn_node, method): } sb = sandbox_at(spawn_node({(method, PATH): lambda body, path: (500, partial)})) with pytest.raises(VolumeError) as caught: - if method == "GET": - sb.get_volumes() - elif method == "PUT": - sb.set_volumes(["new"]) - else: - sb.patch_volumes(remove=["old"]) + call(sb) assert isinstance(caught.value, APIError) assert caught.value.status == 500 assert caught.value.response == partial @@ -180,43 +186,25 @@ def test_volume_methods_reject_a_bare_string(black_hole, call): call(sandbox_at(black_hole)) -def test_mixed_volume_modes_roundtrip(spawn_node): +def test_volume_mutations_encode_explicit_device_fields(spawn_node): entries = [ - {"name": "mounted-ro", "mount": "/volumes/mounted-ro"}, - {"name": "mounted-rw", "mount": "/work", "mode": "rw"}, - {"name": "device-ro", "attach_only": True}, - {"name": "device-rw", "mode": "rw", "attach_only": True}, + {"name": "device", "mount": "", "mode": "rw", "attach_only": True}, + {"name": "mounted", "attach_only": False}, ] - view = {"volumes_want": entries, "volumes": entries, "states": [], "converging": False} - seen = [] - - def reply(body, path): - seen.append(body) - return 200, view - - sb = sandbox_at(spawn_node({("GET", PATH): reply, ("PUT", PATH): reply, ("PATCH", PATH): reply})) - got = sb.get_volumes() - sb.set_volumes(got["volumes_want"]) - sb.patch_volumes(add=got["volumes"]) - assert seen == [{}, {"volumes": entries}, {"add": entries}] - assert sb.volumes == sb.volumes_want == entries - - -def test_attach_only_empty_mount_and_false_flag_remain_explicit(spawn_node): seen = [] def reply(body, path): seen.append(body) - return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + return 200, VIEW - sb = sandbox_at(spawn_node({("PUT", PATH): reply})) - entries = [{"name": "device", "mount": "", "attach_only": True}, {"name": "mounted", "attach_only": False}] + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) sb.set_volumes(entries) - assert seen == [{"volumes": entries}] + sb.patch_volumes(add=entries) + assert seen == [{"volumes": entries}, {"add": entries}] @pytest.mark.parametrize("status", [200, 409]) -@pytest.mark.parametrize("field", ["volumes", "volumes_want", "states"]) +@pytest.mark.parametrize("field", ["volumes", "volumes_want"]) def test_malformed_volume_entry_preserves_error_and_handle(spawn_node, status, field): payload = {**VIEW, "error": "storage unavailable", field: [None]} sb = sandbox_at(spawn_node({("GET", PATH): lambda body, path: (status, payload)}))