From 733af06c45fb6ffc6a42ce5638e56ecd5e624a29 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 17:17:14 +0800 Subject: [PATCH 1/3] pool: check one archive checkpoint pin through the reference counts archiveCkPinned copied the whole pin set under m.mu to test one id. The claim store already counts references, so the check is a map lookup: 5000 checkpoint claims, mac m2 max, interleaved, 162 us / 218 KB / 18 allocs -> 25 ns / 0 allocs. DeleteCheckpoint, retryArchiveDelete (once per marker) and dropStaleRow call it. --- sandboxd/pool/archive.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 9624529c..14a7c7ce 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -415,6 +415,8 @@ func (m *Manager) deleteMarked(ckID string) bool { } func (m *Manager) archiveCkPinned(ckID string) bool { - _, pinned := m.pinnedArchiveCks()[ckID] - return pinned + m.mu.Lock() + defer m.mu.Unlock() + _, pending := m.pendingCks[ckID] + return pending || m.store.referencesCheckpoint(ckID) } From 2a86ce3aa2c491437d5df09dc06a067ffbbb3fe0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 17:17:14 +0800 Subject: [PATCH 2/3] pool: inline the single-caller deleteCkLocked and drop a dead channel receive --- sandboxd/pool/archive_test.go | 1 - sandboxd/pool/checkpoint.go | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index f3012e57..59064b8b 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -694,7 +694,6 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { if err := <-woke; err == nil { t.Fatal("wake succeeded despite a persist failure") } - <-retried m.mu.Lock() archived := sb.ArchiveCk == ck && sb.VMName == "" m.mu.Unlock() diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index e6a32474..aed5f7c1 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -340,20 +340,15 @@ func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { if ckpt.CreatedAt.After(cutoff) { continue } - if err := m.deleteCkLocked(ctx, ckpt.ID); err != nil { + unlock := m.lockRec(ckpt.ID) + err := m.ckpts.Delete(ctx, ckpt.ID) + unlock() + if err != nil { logger.Errorf(ctx, err, "expire %s", ckpt.ID) } } } -// deleteCkLocked removes a checkpoint under its record lock, releasing it before eviction. -func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { - unlock := m.lockRec(ckID) - err := m.ckpts.Delete(ctx, ckID) - unlock() - return err -} - func (m *Manager) loadCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { if !store.CheckpointIDRe.MatchString(ckptID) { return types.Checkpoint{}, ErrUnknownCheckpoint From dc417efd032b15aa21a297307ae91c309fea0eee Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 17:17:14 +0800 Subject: [PATCH 3/3] docs: state checkpoint retention after failed journal writes and with failover off Also fix the replica release order (tombstone before replica) and the on_expire value (destroy, not release). --- docs/cluster.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 2989113a..a94f4ded 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -233,6 +233,9 @@ replication rounds write it, so the claim path is unchanged. tombstone no claim holds is purged checkpoint first. Retention is a lower bound: a purge can run up to an hour late. An archive kept forever (no `archive_delete_after_seconds`) stays until a client adopts and releases it. + A checkpoint the claims journal may still name is never deleted: after a + failed journal write its delete waits for the next write that lands, and a + replica retirement keeps its tombstone until the hourly sweep after that. - **Usage journal.** The journal stays per node: an adopted claim's `archive` is in the old owner's `usage.jsonl`, and its `unarchive`, `release` or `archive_delete` is in the adopter's or the purging node's. Fold the journals @@ -307,10 +310,10 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: about 185 s. The first round of a claim after a wake or adoption uploads the whole claim, so that pass is the long pass in this bound. While rounds fail it is the time since the last one that committed. -- **Lifecycle.** A release consumes the row, then deletes the replica and the - tombstone. A hibernate leaves the row on the last replica. An archive puts its +- **Lifecycle.** A release consumes the row, then deletes the tombstone and the + replica. A hibernate leaves the row on the last replica. An archive puts its own checkpoint over the row, with its retention deadline, and deletes the - replica. A lease expiry with `on_expire` release drops the row and the replica + replica. A lease expiry with `on_expire` destroy drops the row and the replica first, and the next reap destroys the claim; with `on_expire` archive, the archive puts its checkpoint over the row. Turning `replicate_seconds` off takes a restart, since a reload that changes it is refused; the restarted node @@ -318,8 +321,11 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: that holds a replica when the node restarts with replication off is still treated as replicated by the fence until the next restart, so during a registry outage it answers 503 like a replicated claim; that costs - availability only, never a second copy or data. A restart keeps the node's own - live row for a running claim and takes the checkpoint a crashed round left on + availability only, never a second copy or data. Turning `archive_failover` + off stops every registry read and write: a claim that holds a replica then + expires and is released like any other, and leaves its row and replica + checkpoint behind, the same residue as a node that never returns. + A restart keeps the node's own live row for a running claim and takes the checkpoint a crashed round left on it; a claim whose row another node holds, or whose row a finished adopter consumed, is dropped with its VM. A replica row has no retention deadline: the hourly sweep never purges it, because no round refreshes it while the claim is