From eba490f994131bc6e7746e20a00c9ac21e8e9322 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Tue, 17 Nov 2026 02:38:53 -0500 Subject: [PATCH 1/2] A mirroring write waits for a running arena compaction instead of being refused The engine mirror runs inside the write's control transaction, where flatsql compacts the arena only in memory, and not at all while a compaction copies its committed layout over the record stream: it answers "deferred" with nothing freed. A mirror that would pass the budget while a background pass was between its copy steps was refused although that pass was about to free the room (TestEngineArenaCompactsAtRuntimeInsteadOfRefusing, intermittently). storeOne, storeBatchChunk and importDatasetShardChunk take the store lock through lockWriteForMirror: when the mirror would pass the budget and a background pass is running, the write waits for that pass without the lock (its steps take the lock one at a time), bounded at two minutes, and checks again under the lock. Co-Authored-By: Claude Opus 5.5 --- .../storage/engine_arena_compaction.go | 57 +++++++++++++++++++ sdn-server/internal/storage/flatsql.go | 4 +- sdn-server/internal/storage/manifest.go | 2 +- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/sdn-server/internal/storage/engine_arena_compaction.go b/sdn-server/internal/storage/engine_arena_compaction.go index fd2b1f6587..ec73726fb8 100644 --- a/sdn-server/internal/storage/engine_arena_compaction.go +++ b/sdn-server/internal/storage/engine_arena_compaction.go @@ -199,6 +199,63 @@ func (s *FlatSQLStore) persistPendingArenaCompaction(reason string) { } } +// engineArenaMirrorWait bounds how long a write waits for a running +// background compaction before it takes the store lock. +const engineArenaMirrorWait = 2 * time.Minute + +// lockWriteForMirror takes the store write lock for a write that mirrors about +// n arena bytes into the engine, inside its control transaction. +// +// Inside a transaction the engine compacts only in memory, and not at all while +// a compaction copies its committed layout over the record stream (flatsql +// keeps those bytes in place until the copy is done, and answers "deferred" +// with nothing freed). A mirror that would pass the budget at that moment was +// refused although the background pass under way was about to free the room. +// So when the mirror would pass the budget and a background pass is running, +// the write waits for that pass without the lock (each of its steps takes the +// lock on its own) and takes the lock once the pass has finished. +func (s *FlatSQLStore) lockWriteForMirror(site string, n int64) func() { + if !engineArenaRuntimeCompaction { + return s.lockWrite(site) + } + c := s.arenaCompactor() + deadline := time.Now().Add(engineArenaMirrorWait) + for { + for c.running.Load() && s.engineArenaBytes.Load()+n > engineArenaBudgetBytes && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + release := s.lockWrite(site) + // Another write may have started a pass that reached its copy + // between the wait and the lock. + if !time.Now().Before(deadline) || !s.arenaCopyPinsMirrorLocked(n) { + return release + } + release() + } +} + +// datasetShardFrameBytes is the arena bytes a dataset shard chunk can mirror: +// its records' lengths plus each frame's size prefix. +func datasetShardFrameBytes(records []DatasetExportIndexRecord) int64 { + var n int64 + for _, r := range records { + n += r.Length + 4 + } + return n +} + +// arenaCopyPinsMirrorLocked: a background pass is between steps and the +// mirror would pass the budget, so a compaction inside the mirror's +// transaction may free nothing. Caller holds s.mu. +func (s *FlatSQLStore) arenaCopyPinsMirrorLocked(n int64) bool { + if s.engineArenaBytes.Load()+n <= engineArenaBudgetBytes || !s.arenaCompactor().running.Load() || + s.engineDB == nil || s.engine == nil || !s.engine.HasArenaCompaction() { + return false + } + stats, err := s.engineDB.ArenaStats() + return err == nil && stats.Compacting +} + // errArenaCompactionStopped: the store closed or the engine was replaced // between steps. Not a failure: the new engine starts clean, and a swap // already committed is finished by the next open. diff --git a/sdn-server/internal/storage/flatsql.go b/sdn-server/internal/storage/flatsql.go index 74f73527e6..86f5cb9d02 100644 --- a/sdn-server/internal/storage/flatsql.go +++ b/sdn-server/internal/storage/flatsql.go @@ -2308,7 +2308,7 @@ func (s *FlatSQLStore) storeOne(schemaName string, data []byte, peerID string, s if err := s.requireWritable("store record"); err != nil { return "", err } - defer s.lockWrite("storeOne")() + defer s.lockWriteForMirror("storeOne", engineArenaFrameBytes([][]byte{data}))() if _, err := sds.SchemaNameToTable(schemaName); err != nil { return "", fmt.Errorf("invalid schema name: %w", err) @@ -2741,7 +2741,7 @@ func (s *FlatSQLStore) storeBatch(schemaName string, records [][]byte, peerID st // storeBatchChunk stores one chunk under one store lock and one control // transaction (the pre-chunking storeBatch body). func (s *FlatSQLStore) storeBatchChunk(schemaName string, records [][]byte, peerID string, signature []byte, tags *SourceTags) (int, error) { - defer s.lockWrite("storeBatchChunk")() + defer s.lockWriteForMirror("storeBatchChunk", engineArenaFrameBytes(records))() // WS7.3d routed-only writes: pre-create the (producer, standard) table // outside the batch transaction (no DDL inside the tx). This is the ONLY diff --git a/sdn-server/internal/storage/manifest.go b/sdn-server/internal/storage/manifest.go index 81138833bd..05f1808f95 100644 --- a/sdn-server/internal/storage/manifest.go +++ b/sdn-server/internal/storage/manifest.go @@ -1063,7 +1063,7 @@ func (s *FlatSQLStore) importDatasetShardChunk(index *DatasetExportIndex, provid if s.rb != nil { return s.rb.importDatasetShardChunk(index, providerPeerID, records, readRecord) } - defer s.lockWrite("importDatasetShardChunk")() + defer s.lockWriteForMirror("importDatasetShardChunk", datasetShardFrameBytes(records))() if hook := importDatasetShardChunkHook.Load(); hook != nil { (*hook)() } From 2a405fd752f6ae5bf5ffb84cd9af0cf88b073fcd Mon Sep 17 00:00:00 2001 From: TJKoury Date: Tue, 17 Nov 2026 02:40:27 -0500 Subject: [PATCH 2/2] handoff-after-stop queues for the store lock while the daemon holds it The case took the store lock only after its own wait on the daemon process returned, by retrying a non-blocking lock every 5 ms. The helper polls the daemon's pid every 100 ms and takes the lock as soon as it sees the exit, so whichever ran first after the exit got the lock. When the helper won, it took the free lock, swapped and restarted (ledger: daemon-exited, store-locked, apply, restart), and the case failed with "the helper did not report the held store lock". The helper behaved as designed: it found the lock free. The case now checks that the running daemon holds the lock, then queues a blocking flock on store.lock before the daemon exits. The kernel grants it as the daemon's process releases the lock, before the helper can observe the exit. Co-Authored-By: Claude Opus 5.5 --- .../update_feed_recovery_e2e_test.go | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go b/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go index 3d89af608b..1f1c35c268 100644 --- a/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go +++ b/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "encoding/base64" "encoding/json" + "errors" "fmt" "net/http" "os" @@ -14,6 +15,7 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "testing" "time" @@ -373,22 +375,30 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) { case "handoff-after-stop": // The helper stops the daemon; then a process holds the store lock, so // the helper swaps nothing and restarts the build that was running. + // The request for the lock is queued while the daemon still holds + // it: the kernel hands it over as the daemon exits, before the + // helper (which polls the daemon's pid) can see the exit and ask. + // Asking only after the exit raced the helper, which sometimes took + // the lock first and swapped. + if probe, err := storage.LockStoreForMaintenance(d.store); err == nil { + _ = probe.Release() + t.Fatal("the daemon released the store lock before the request for it was queued") + } else if !errors.Is(err, storage.ErrStoreLocked) { + t.Fatalf("probe the daemon's store lock: %v", err) + } + lockAcquired, lockErr, releaseLock := queueForStoreLock(d.store) select { case <-d.exited: case <-time.After(boot + 2*time.Duration(health)*time.Second): t.Fatal("the daemon was not stopped for the hand-off") } - var lock *storage.StoreMaintenanceLock - for deadline := time.Now().Add(5 * time.Second); lock == nil; { - held, err := storage.LockStoreForMaintenance(d.store) - if err == nil { - lock = held - break - } - if time.Now().After(deadline) { - t.Fatalf("take the store lock after the daemon stopped: %v", err) + select { + case <-lockAcquired: + if err := lockErr(); err != nil { + t.Fatalf("take the store lock as the daemon stopped: %v", err) } - time.Sleep(5 * time.Millisecond) + case <-time.After(5 * time.Second): + t.Fatal("the store lock was not handed over when the daemon stopped") } var lockFailedAt time.Time for deadline := time.Now().Add(storeLockWait + 30*time.Second); lockFailedAt.IsZero(); { @@ -402,12 +412,12 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) { } } if lockFailedAt.IsZero() && time.Now().After(deadline) { - _ = lock.Release() + _ = releaseLock() t.Fatalf("the helper did not report the held store lock: %v", ledgerActions(t, d.paths.Root)) } time.Sleep(10 * time.Millisecond) } - if err := lock.Release(); err != nil { + if err := releaseLock(); err != nil { t.Fatal(err) } waitPush(t, boot, "the build that was running, restarted by the helper", func() bool { @@ -520,3 +530,33 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) { } d.stop(t) } + +// queueForStoreLock makes a blocking request for the store's single-writer +// lock (the flock the daemon holds while it runs). The kernel grants it when +// the holder releases the lock. acquired is closed once the request returns, +// and err reports how it ended; release drops the lock (call it only after +// acquired is closed). +func queueForStoreLock(store string) (acquired <-chan struct{}, err func() error, release func() error) { + done := make(chan struct{}) + var f *os.File + var lockErr error + go func() { + defer close(done) + f, lockErr = os.OpenFile(filepath.Join(store, storage.StoreLockFileName), os.O_CREATE|os.O_RDWR, 0o644) + if lockErr != nil { + return + } + for { + if lockErr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); !errors.Is(lockErr, syscall.EINTR) { + return + } + } + }() + return done, func() error { <-done; return lockErr }, func() error { + <-done + if f == nil { + return lockErr + } + return f.Close() + } +}