diff --git a/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go b/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go index 3d89af608..1f1c35c26 100644 --- a/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go +++ b/sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "encoding/base64" "encoding/json" + "errors" "fmt" "net/http" "os" @@ -14,6 +15,7 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "testing" "time" @@ -373,22 +375,30 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) { case "handoff-after-stop": // The helper stops the daemon; then a process holds the store lock, so // the helper swaps nothing and restarts the build that was running. + // The request for the lock is queued while the daemon still holds + // it: the kernel hands it over as the daemon exits, before the + // helper (which polls the daemon's pid) can see the exit and ask. + // Asking only after the exit raced the helper, which sometimes took + // the lock first and swapped. + if probe, err := storage.LockStoreForMaintenance(d.store); err == nil { + _ = probe.Release() + t.Fatal("the daemon released the store lock before the request for it was queued") + } else if !errors.Is(err, storage.ErrStoreLocked) { + t.Fatalf("probe the daemon's store lock: %v", err) + } + lockAcquired, lockErr, releaseLock := queueForStoreLock(d.store) select { case <-d.exited: case <-time.After(boot + 2*time.Duration(health)*time.Second): t.Fatal("the daemon was not stopped for the hand-off") } - var lock *storage.StoreMaintenanceLock - for deadline := time.Now().Add(5 * time.Second); lock == nil; { - held, err := storage.LockStoreForMaintenance(d.store) - if err == nil { - lock = held - break - } - if time.Now().After(deadline) { - t.Fatalf("take the store lock after the daemon stopped: %v", err) + select { + case <-lockAcquired: + if err := lockErr(); err != nil { + t.Fatalf("take the store lock as the daemon stopped: %v", err) } - time.Sleep(5 * time.Millisecond) + case <-time.After(5 * time.Second): + t.Fatal("the store lock was not handed over when the daemon stopped") } var lockFailedAt time.Time for deadline := time.Now().Add(storeLockWait + 30*time.Second); lockFailedAt.IsZero(); { @@ -402,12 +412,12 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) { } } if lockFailedAt.IsZero() && time.Now().After(deadline) { - _ = lock.Release() + _ = releaseLock() t.Fatalf("the helper did not report the held store lock: %v", ledgerActions(t, d.paths.Root)) } time.Sleep(10 * time.Millisecond) } - if err := lock.Release(); err != nil { + if err := releaseLock(); err != nil { t.Fatal(err) } waitPush(t, boot, "the build that was running, restarted by the helper", func() bool { @@ -520,3 +530,33 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) { } d.stop(t) } + +// queueForStoreLock makes a blocking request for the store's single-writer +// lock (the flock the daemon holds while it runs). The kernel grants it when +// the holder releases the lock. acquired is closed once the request returns, +// and err reports how it ended; release drops the lock (call it only after +// acquired is closed). +func queueForStoreLock(store string) (acquired <-chan struct{}, err func() error, release func() error) { + done := make(chan struct{}) + var f *os.File + var lockErr error + go func() { + defer close(done) + f, lockErr = os.OpenFile(filepath.Join(store, storage.StoreLockFileName), os.O_CREATE|os.O_RDWR, 0o644) + if lockErr != nil { + return + } + for { + if lockErr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); !errors.Is(lockErr, syscall.EINTR) { + return + } + } + }() + return done, func() error { <-done; return lockErr }, func() error { + <-done + if f == nil { + return lockErr + } + return f.Close() + } +} diff --git a/sdn-server/internal/storage/engine_arena_compaction.go b/sdn-server/internal/storage/engine_arena_compaction.go index fd2b1f658..ec73726fb 100644 --- a/sdn-server/internal/storage/engine_arena_compaction.go +++ b/sdn-server/internal/storage/engine_arena_compaction.go @@ -199,6 +199,63 @@ func (s *FlatSQLStore) persistPendingArenaCompaction(reason string) { } } +// engineArenaMirrorWait bounds how long a write waits for a running +// background compaction before it takes the store lock. +const engineArenaMirrorWait = 2 * time.Minute + +// lockWriteForMirror takes the store write lock for a write that mirrors about +// n arena bytes into the engine, inside its control transaction. +// +// Inside a transaction the engine compacts only in memory, and not at all while +// a compaction copies its committed layout over the record stream (flatsql +// keeps those bytes in place until the copy is done, and answers "deferred" +// with nothing freed). A mirror that would pass the budget at that moment was +// refused although the background pass under way was about to free the room. +// So when the mirror would pass the budget and a background pass is running, +// the write waits for that pass without the lock (each of its steps takes the +// lock on its own) and takes the lock once the pass has finished. +func (s *FlatSQLStore) lockWriteForMirror(site string, n int64) func() { + if !engineArenaRuntimeCompaction { + return s.lockWrite(site) + } + c := s.arenaCompactor() + deadline := time.Now().Add(engineArenaMirrorWait) + for { + for c.running.Load() && s.engineArenaBytes.Load()+n > engineArenaBudgetBytes && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + release := s.lockWrite(site) + // Another write may have started a pass that reached its copy + // between the wait and the lock. + if !time.Now().Before(deadline) || !s.arenaCopyPinsMirrorLocked(n) { + return release + } + release() + } +} + +// datasetShardFrameBytes is the arena bytes a dataset shard chunk can mirror: +// its records' lengths plus each frame's size prefix. +func datasetShardFrameBytes(records []DatasetExportIndexRecord) int64 { + var n int64 + for _, r := range records { + n += r.Length + 4 + } + return n +} + +// arenaCopyPinsMirrorLocked: a background pass is between steps and the +// mirror would pass the budget, so a compaction inside the mirror's +// transaction may free nothing. Caller holds s.mu. +func (s *FlatSQLStore) arenaCopyPinsMirrorLocked(n int64) bool { + if s.engineArenaBytes.Load()+n <= engineArenaBudgetBytes || !s.arenaCompactor().running.Load() || + s.engineDB == nil || s.engine == nil || !s.engine.HasArenaCompaction() { + return false + } + stats, err := s.engineDB.ArenaStats() + return err == nil && stats.Compacting +} + // errArenaCompactionStopped: the store closed or the engine was replaced // between steps. Not a failure: the new engine starts clean, and a swap // already committed is finished by the next open. diff --git a/sdn-server/internal/storage/flatsql.go b/sdn-server/internal/storage/flatsql.go index 74f73527e..86f5cb9d0 100644 --- a/sdn-server/internal/storage/flatsql.go +++ b/sdn-server/internal/storage/flatsql.go @@ -2308,7 +2308,7 @@ func (s *FlatSQLStore) storeOne(schemaName string, data []byte, peerID string, s if err := s.requireWritable("store record"); err != nil { return "", err } - defer s.lockWrite("storeOne")() + defer s.lockWriteForMirror("storeOne", engineArenaFrameBytes([][]byte{data}))() if _, err := sds.SchemaNameToTable(schemaName); err != nil { return "", fmt.Errorf("invalid schema name: %w", err) @@ -2741,7 +2741,7 @@ func (s *FlatSQLStore) storeBatch(schemaName string, records [][]byte, peerID st // storeBatchChunk stores one chunk under one store lock and one control // transaction (the pre-chunking storeBatch body). func (s *FlatSQLStore) storeBatchChunk(schemaName string, records [][]byte, peerID string, signature []byte, tags *SourceTags) (int, error) { - defer s.lockWrite("storeBatchChunk")() + defer s.lockWriteForMirror("storeBatchChunk", engineArenaFrameBytes(records))() // WS7.3d routed-only writes: pre-create the (producer, standard) table // outside the batch transaction (no DDL inside the tx). This is the ONLY diff --git a/sdn-server/internal/storage/manifest.go b/sdn-server/internal/storage/manifest.go index 81138833b..05f1808f9 100644 --- a/sdn-server/internal/storage/manifest.go +++ b/sdn-server/internal/storage/manifest.go @@ -1063,7 +1063,7 @@ func (s *FlatSQLStore) importDatasetShardChunk(index *DatasetExportIndex, provid if s.rb != nil { return s.rb.importDatasetShardChunk(index, providerPeerID, records, readRecord) } - defer s.lockWrite("importDatasetShardChunk")() + defer s.lockWriteForMirror("importDatasetShardChunk", datasetShardFrameBytes(records))() if hook := importDatasetShardChunkHook.Load(); hook != nil { (*hook)() }