Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 52 additions & 12 deletions sdn-server/cmd/spacedatanetwork/update_feed_recovery_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import (
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"

Expand Down Expand Up @@ -373,22 +375,30 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) {
case "handoff-after-stop":
// The helper stops the daemon; then a process holds the store lock, so
// the helper swaps nothing and restarts the build that was running.
// The request for the lock is queued while the daemon still holds
// it: the kernel hands it over as the daemon exits, before the
// helper (which polls the daemon's pid) can see the exit and ask.
// Asking only after the exit raced the helper, which sometimes took
// the lock first and swapped.
if probe, err := storage.LockStoreForMaintenance(d.store); err == nil {
_ = probe.Release()
t.Fatal("the daemon released the store lock before the request for it was queued")
} else if !errors.Is(err, storage.ErrStoreLocked) {
t.Fatalf("probe the daemon's store lock: %v", err)
}
lockAcquired, lockErr, releaseLock := queueForStoreLock(d.store)
select {
case <-d.exited:
case <-time.After(boot + 2*time.Duration(health)*time.Second):
t.Fatal("the daemon was not stopped for the hand-off")
}
var lock *storage.StoreMaintenanceLock
for deadline := time.Now().Add(5 * time.Second); lock == nil; {
held, err := storage.LockStoreForMaintenance(d.store)
if err == nil {
lock = held
break
}
if time.Now().After(deadline) {
t.Fatalf("take the store lock after the daemon stopped: %v", err)
select {
case <-lockAcquired:
if err := lockErr(); err != nil {
t.Fatalf("take the store lock as the daemon stopped: %v", err)
}
time.Sleep(5 * time.Millisecond)
case <-time.After(5 * time.Second):
t.Fatal("the store lock was not handed over when the daemon stopped")
}
var lockFailedAt time.Time
for deadline := time.Now().Add(storeLockWait + 30*time.Second); lockFailedAt.IsZero(); {
Expand All @@ -402,12 +412,12 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) {
}
}
if lockFailedAt.IsZero() && time.Now().After(deadline) {
_ = lock.Release()
_ = releaseLock()
t.Fatalf("the helper did not report the held store lock: %v", ledgerActions(t, d.paths.Root))
}
time.Sleep(10 * time.Millisecond)
}
if err := lock.Release(); err != nil {
if err := releaseLock(); err != nil {
t.Fatal(err)
}
waitPush(t, boot, "the build that was running, restarted by the helper", func() bool {
Expand Down Expand Up @@ -520,3 +530,33 @@ func runRealFeedRecoveryCase(t *testing.T, mode string) {
}
d.stop(t)
}

// queueForStoreLock makes a blocking request for the store's single-writer
// lock (the flock the daemon holds while it runs). The kernel grants it when
// the holder releases the lock. acquired is closed once the request returns,
// and err reports how it ended; release drops the lock (call it only after
// acquired is closed).
func queueForStoreLock(store string) (acquired <-chan struct{}, err func() error, release func() error) {
done := make(chan struct{})
var f *os.File
var lockErr error
go func() {
defer close(done)
f, lockErr = os.OpenFile(filepath.Join(store, storage.StoreLockFileName), os.O_CREATE|os.O_RDWR, 0o644)
if lockErr != nil {
return
}
for {
if lockErr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); !errors.Is(lockErr, syscall.EINTR) {
return
}
}
}()
return done, func() error { <-done; return lockErr }, func() error {
<-done
if f == nil {
return lockErr
}
return f.Close()
}
}
57 changes: 57 additions & 0 deletions sdn-server/internal/storage/engine_arena_compaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,63 @@ func (s *FlatSQLStore) persistPendingArenaCompaction(reason string) {
}
}

// engineArenaMirrorWait bounds how long a write waits for a running
// background compaction before it takes the store lock.
const engineArenaMirrorWait = 2 * time.Minute

// lockWriteForMirror takes the store write lock for a write that mirrors about
// n arena bytes into the engine, inside its control transaction.
//
// Inside a transaction the engine compacts only in memory, and not at all while
// a compaction copies its committed layout over the record stream (flatsql
// keeps those bytes in place until the copy is done, and answers "deferred"
// with nothing freed). A mirror that would pass the budget at that moment was
// refused although the background pass under way was about to free the room.
// So when the mirror would pass the budget and a background pass is running,
// the write waits for that pass without the lock (each of its steps takes the
// lock on its own) and takes the lock once the pass has finished.
func (s *FlatSQLStore) lockWriteForMirror(site string, n int64) func() {
if !engineArenaRuntimeCompaction {
return s.lockWrite(site)
}
c := s.arenaCompactor()
deadline := time.Now().Add(engineArenaMirrorWait)
for {
for c.running.Load() && s.engineArenaBytes.Load()+n > engineArenaBudgetBytes && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
release := s.lockWrite(site)
// Another write may have started a pass that reached its copy
// between the wait and the lock.
if !time.Now().Before(deadline) || !s.arenaCopyPinsMirrorLocked(n) {
return release
}
release()
}
}

// datasetShardFrameBytes is the arena bytes a dataset shard chunk can mirror:
// its records' lengths plus each frame's size prefix.
func datasetShardFrameBytes(records []DatasetExportIndexRecord) int64 {
var n int64
for _, r := range records {
n += r.Length + 4
}
return n
}

// arenaCopyPinsMirrorLocked: a background pass is between steps and the
// mirror would pass the budget, so a compaction inside the mirror's
// transaction may free nothing. Caller holds s.mu.
func (s *FlatSQLStore) arenaCopyPinsMirrorLocked(n int64) bool {
if s.engineArenaBytes.Load()+n <= engineArenaBudgetBytes || !s.arenaCompactor().running.Load() ||
s.engineDB == nil || s.engine == nil || !s.engine.HasArenaCompaction() {
return false
}
stats, err := s.engineDB.ArenaStats()
return err == nil && stats.Compacting
}

// errArenaCompactionStopped: the store closed or the engine was replaced
// between steps. Not a failure: the new engine starts clean, and a swap
// already committed is finished by the next open.
Expand Down
4 changes: 2 additions & 2 deletions sdn-server/internal/storage/flatsql.go
Original file line number Diff line number Diff line change
Expand Up @@ -2308,7 +2308,7 @@ func (s *FlatSQLStore) storeOne(schemaName string, data []byte, peerID string, s
if err := s.requireWritable("store record"); err != nil {
return "", err
}
defer s.lockWrite("storeOne")()
defer s.lockWriteForMirror("storeOne", engineArenaFrameBytes([][]byte{data}))()

if _, err := sds.SchemaNameToTable(schemaName); err != nil {
return "", fmt.Errorf("invalid schema name: %w", err)
Expand Down Expand Up @@ -2741,7 +2741,7 @@ func (s *FlatSQLStore) storeBatch(schemaName string, records [][]byte, peerID st
// storeBatchChunk stores one chunk under one store lock and one control
// transaction (the pre-chunking storeBatch body).
func (s *FlatSQLStore) storeBatchChunk(schemaName string, records [][]byte, peerID string, signature []byte, tags *SourceTags) (int, error) {
defer s.lockWrite("storeBatchChunk")()
defer s.lockWriteForMirror("storeBatchChunk", engineArenaFrameBytes(records))()

// WS7.3d routed-only writes: pre-create the (producer, standard) table
// outside the batch transaction (no DDL inside the tx). This is the ONLY
Expand Down
2 changes: 1 addition & 1 deletion sdn-server/internal/storage/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -1063,7 +1063,7 @@ func (s *FlatSQLStore) importDatasetShardChunk(index *DatasetExportIndex, provid
if s.rb != nil {
return s.rb.importDatasetShardChunk(index, providerPeerID, records, readRecord)
}
defer s.lockWrite("importDatasetShardChunk")()
defer s.lockWriteForMirror("importDatasetShardChunk", datasetShardFrameBytes(records))()
if hook := importDatasetShardChunkHook.Load(); hook != nil {
(*hook)()
}
Expand Down
Loading