From dc5b386b99fa6de4a1d32cbe0ed4e6b2907c4a9d Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Sun, 27 Sep 2026 00:20:44 -0600 Subject: [PATCH 1/2] cache: back Cache with a concurrent hash-trie The sync.Map-style read map and locked dirty map are replaced with a concurrent hash-trie modeled on Go's internal HashTrieMap. Loads are lock-free, inserts and deletes lock only the trie node they change, and Delete unlinks the entry immediately rather than waiting for a promotion. Get runs the miss function in the calling goroutine unless it has a stale value to return while the load runs. If an inline miss panics or calls runtime.Goexit, the panic reaches the caller and Gets waiting on that load run their own load. Gets that collapsed onto a load all return its result, including an error that is not cached. Expiry uses the monotonic clock, so a wall clock step no longer moves TTLs. Expire does nothing to a loading or already expired value, and never moves an expiry later. Expire replaces the loading with an expired copy rather than writing its expiry, so a concurrent Set or Swap can no longer make it lost. A stale window starts when the value expires, and TryGet, Get, and Clean agree on it. CompareAndSwap and CompareAndDelete only match unexpired, unerrored values. MaxIdleAge skips an extension that would move the expiry by less than 1s (or age/64), so hot keys do not write their expiry on every hit. The autoclean goroutine holds its cache through a weak pointer, so a cache that is never stopped is still garbage collected and its goroutine exits. StopAutoClean waits for the goroutine to exit. NewItem and NewSet now honor AutoCleanInterval, which they previously ignored. A loading holds its error and stale in a side struct that only errored or stale-carrying loadings allocate, so a Cache[int, int] value takes 48 bytes. Lookups do not make the key escape, Clean does not allocate, Get with no expiry does not read the clock, and Set.Get no longer allocates a wrapper closure per call. The tests cover every statement of the trie and cache. They include the stdlib sync.Map tests, a property test against a mutex guarded map, structural checks of the trie, hook-driven tests for specific races, and a linearizability checker. The checker runs every operation, Get with its singleflight rules, and expiry under a frozen test clock, including with forced hash collisions. CI runs them with the race detector on amd64 and arm64, and without it on 386. This requires Go 1.25: the package uses a generic type alias (1.24) and the tests use sync.WaitGroup.Go (1.25). --- .github/workflows/ci.yml | 42 + cache/autoclean_test.go | 108 +++ cache/cache.go | 1057 +++++++++++---------- cache/cache_bench_test.go | 151 +++ cache/cache_test.go | 753 +++++++++++++-- cache/concurrency_test.go | 1622 +++++++++++++++++++++++++++++++++ cache/linearizability_test.go | 433 +++++++++ cache/map_bench_test.go | 12 + cache/map_test.go | 75 ++ cache/trie.go | 363 ++++++++ cache/trie_test.go | 717 +++++++++++++++ cache/wrappers_test.go | 331 +++++++ go.mod | 2 +- 13 files changed, 5131 insertions(+), 535 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 cache/autoclean_test.go create mode 100644 cache/cache_bench_test.go create mode 100644 cache/concurrency_test.go create mode 100644 cache/linearizability_test.go create mode 100644 cache/trie.go create mode 100644 cache/trie_test.go create mode 100644 cache/wrappers_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..702fba4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: ci + +on: + push: + branches: ["main"] + pull_request: + +jobs: + test: + # amd64 and arm64: the trie/cache core is lock-free, and arm64's weaker + # memory ordering surfaces reordering bugs that x86's TSO hides. + # 1.25 is the minimum in go.mod. + name: test (${{ matrix.os }}, go ${{ matrix.go }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + go: ["1.25.x", stable] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + - run: go vet ./... + - run: go test -race ./... + + test-386: + # 32-bit: uintptr hashes are 32 bits, so the trie is 8 levels deep + # instead of 16 and the test hash constants must fit the word size. + # The race detector is unsupported on 386, so this runs without it. + name: test (linux/386) + runs-on: ubuntu-latest + env: + GOARCH: "386" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - run: go vet ./... + - run: go test ./... diff --git a/cache/autoclean_test.go b/cache/autoclean_test.go new file mode 100644 index 0000000..a7c26f6 --- /dev/null +++ b/cache/autoclean_test.go @@ -0,0 +1,108 @@ +package cache + +import ( + "runtime" + "testing" + "time" +) + +func TestAutoClean_RunsAndCleansExpired(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(2*time.Millisecond), + ) + defer c.StopAutoClean() + + c.Set("a", 1) + waitEntries(t, c, 0) +} + +// waitEntries waits until the trie holds n entries. +func waitEntries[K comparable, V any](t *testing.T, c *Cache[K, V], n int) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for trieEntries(c) != n { + if time.Now().After(deadline) { + t.Fatalf("trie has %d entries, want %d", trieEntries(c), n) + } + time.Sleep(time.Millisecond) + } +} + +func TestAutoClean_DisabledWhenStaleAgeNegative(t *testing.T) { + // AutoCleanInterval is ignored when MaxStaleAge < 0 (stales kept forever). + // We verify no goroutine is spawned by confirming StopAutoClean is a no-op + // on a cache whose quitClean channel was never created. + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(-1), + AutoCleanInterval(time.Millisecond), + ) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated when MaxStaleAge<0") + } + c.StopAutoClean() // no-op must not panic +} + +func TestAutoClean_StopIdempotent(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + c.StopAutoClean() + c.StopAutoClean() // sync.Once guards the close; second call must be safe. +} + +func TestAutoClean_ZeroIntervalNoGoroutine(t *testing.T) { + // autoCleanInterval == 0 means disabled; no goroutine, no channel. + c := New[string, int](MaxAge(time.Second)) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated with zero interval") + } + c.StopAutoClean() +} + +// Once StopAutoClean returns, no Clean runs. +func TestAutoClean_StopHaltsCleaning(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + c.Set("a", 1) + waitEntries(t, c, 0) + c.StopAutoClean() + + c.Set("b", 2) + time.Sleep(20 * time.Millisecond) + if n := trieEntries(c); n != 1 { + t.Fatalf("trie has %d entries after StopAutoClean, want 1", n) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("trie has %d entries after Clean, want 0", n) + } +} + +// A cache that is never stopped is still garbage collected, and its clean +// goroutine exits. +func TestAutoClean_ExitsWhenCacheCollected(t *testing.T) { + for _, newDone := range []func() chan struct{}{ + func() chan struct{} { return New[int, int](AutoCleanInterval(time.Millisecond)).cleanDone }, + func() chan struct{} { return NewItem[int](AutoCleanInterval(time.Millisecond)).c.cleanDone }, + func() chan struct{} { return NewSet[int](AutoCleanInterval(time.Millisecond)).c.cleanDone }, + } { + done := newDone() + deadline := time.Now().Add(10 * time.Second) + for exited := false; !exited; { + runtime.GC() + select { + case <-done: + exited = true + case <-time.After(10 * time.Millisecond): + if time.Now().After(deadline) { + t.Fatal("clean goroutine did not exit after its cache became unreachable") + } + } + } + } +} diff --git a/cache/cache.go b/cache/cache.go index 0e6d5af..4b14522 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,9 +1,9 @@ // Package cache provides a concurrency safe, mostly lock-free, singleflight // request collapsing generic cache with support for stale values. // -// The guts of this package are similar to `sync.Map`, with minor differences -// in when the internal locked write-map is promoted an atomic read-only map, -// and major differences to support singleflight request collapsing. +// The guts of this package are a concurrent hash-trie similar to the one +// backing `sync.Map`, with major differences to support singleflight request +// collapsing. // // Functions in this API that return a KeyState return the state last, rather // than the error last: the value and error are cached as a single internal @@ -16,9 +16,12 @@ package cache import ( + "errors" + "math" "sync" "sync/atomic" "time" + "weak" ) // KeyState is returned from cache operations to indicate how a key existed in @@ -50,38 +53,42 @@ type ( } loading[V any] struct { v V - err error - expires atomic.Int64 // nano at which this ent is unusable, if non-zero - stale *stale[V] + expires atomic.Int64 // see expiredClaimed for what values mean state atomic.Uint32 + wg sync.WaitGroup - wg sync.WaitGroup - mu sync.Mutex + // x is nil unless the load errored or the loading has a stale, + // which keeps the common loading small. + x atomic.Pointer[loadingExtra[V]] } - ent[V any] struct { - p atomic.Pointer[loading[V]] + + loadingWithExtra[V any] struct { + l loading[V] + x loadingExtra[V] } - read[K comparable, V any] struct { - m map[K]*ent[V] - incomplete bool + + loadingExtra[V any] struct { + err error // written before the loading is finalized + + // stale is the previous value, captured when a Get replaces an + // expired or errored loading. We return it while this load is in + // flight or if this load errors. It is set before the loading is + // published and never changes. + stale stale[V] + hasStale bool } // Cache caches comparable keys to arbitrary values. By default, the // cache grows without bounds and all keys persist forever. These // limits can be changed with options that are passed to New. Cache[K comparable, V any] struct { - r atomic.Pointer[read[K, V]] - mu sync.Mutex - dirty map[K]*ent[V] - - misses int + t trie[K, loading[V]] cfg cfg - pd *loading[V] // allocate the promotingDelete pointer once - quitOnce sync.Once quitClean chan struct{} + cleanDone chan struct{} } cfg struct { @@ -104,13 +111,54 @@ type ( const ( stateLoading uint32 = iota + stateFinalizing stateFinalized - statePromotingDelete ) -func now() int64 { return time.Now().UnixNano() } +// expiredClaimed is the loading.expires value Clean stores to claim an +// entry for deletion. The full set of values is: +// +// 0 loading, or loaded and never expires +// > 0 expires at this now() nano +// -2 claimed by Clean: expired, and its stale window is closed +// < -2 born expired (MaxAge(0), MaxErrorAge(0)): the negated now() at +// load, which is where the stale window starts +// +// Every negative value is expired, because now() is always positive. The +// epoch is padded by a second so that a negated now() is never -2. +const expiredClaimed = -2 + +// ent is what the cache stores in the trie. +type ent[K comparable, V any] = trieEntry[K, loading[V]] -func (cfg *cfg) newExpires(err error) int64 { +// We use the monotonic clock so that expiries do not move if the wall clock +// is stepped. Note that if the monotonic clock pauses while the system is +// suspended, entries only age while the system is running. +var epoch = time.Now().Add(-time.Second) + +// testNow, if non-zero, is what now returns, so that tests can freeze the +// clock. Never set in production code. +var testNow atomic.Int64 + +func now() int64 { + if n := testNow.Load(); n != 0 { + return n + } + return int64(time.Since(epoch)) +} + +// satAdd returns a+b capped at math.MaxInt64, so that a huge age does not +// wrap an expiry negative. b must be non-negative. +func satAdd(a, b int64) int64 { + if s := a + b; s >= a { + return s + } + return math.MaxInt64 +} + +// newExpires returns the expiry for a value or err loaded at n, sampling +// the clock only if it is needed and n is 0. +func (cfg *cfg) newExpires(err error, n int64) int64 { var ttl time.Duration var del0 bool if err != nil && cfg.errAgeSet { @@ -123,13 +171,16 @@ func (cfg *cfg) newExpires(err error) int64 { } del0 = cfg.ageSet && cfg.maxAge <= 0 } - if del0 { - return -1 - } - if ttl == 0 { + if !del0 && ttl == 0 { return 0 } - return time.Now().Add(ttl).UnixNano() + if n == 0 { + n = now() + } + if del0 { + return -n + } + return satAdd(n, int64(ttl)) } func (o opt) apply(c *cfg) { o.fn(c) } @@ -177,6 +228,8 @@ func MaxIdleAge(age time.Duration) Opt { // interval that is less than your MaxAge + MaxStaleAge. Values are only // candidates to be cleaned after the max age has elapsed. At worst, a value // may persist for a total of MaxAge + MaxStaleAge + AutoCleanInterval. +// +// The goroutine exits once the cache is garbage collected. func AutoCleanInterval(interval time.Duration) Opt { return opt{fn: func(c *cfg) { c.autoCleanInterval = interval }} } @@ -185,150 +238,215 @@ func AutoCleanInterval(interval time.Duration) Opt { // semantics. If you do not need to configure a cache at all, the zero value // cache is valid and usable. func New[K comparable, V any](opts ...Opt) *Cache[K, V] { - var cfg cfg + c := new(Cache[K, V]) + initCache(c, opts...) + return c +} + +// initCache initializes c in place so that NewItem and NewSet can initialize +// their embedded Cache; the autoclean goroutine references c. +func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { for _, opt := range opts { - opt.apply(&cfg) - } - c := &Cache[K, V]{ - cfg: cfg, - pd: new(loading[V]), + opt.apply(&c.cfg) } - c.pd.state.Store(statePromotingDelete) - if cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { + if c.cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { c.quitClean = make(chan struct{}) - go func() { - ticker := time.NewTicker(cfg.autoCleanInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - case <-c.quitClean: - return - } - c.Clean() - } - }() + c.cleanDone = make(chan struct{}) + go autoClean(weak.Make(c), c.cfg.autoCleanInterval, c.quitClean, c.cleanDone) } - return c } -// Get returns the cache value for k, running the miss function in a goroutine -// if the key is not yet cached. If stale values are enabled, the currently -// cached value has an error, and there is an unexpired stale value, this -// returns the stale value and no error. -func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { - r := c.read() - e := r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - return v, err, s - } - - // We missed in the read map. We lock and check again to guard against - // something concurrent. Odds are we are falling into the logic below. - c.mu.Lock() - r = c.read() - e = r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() - return v, err, s - } - - // We could have an entry in our read map that was deleted and has not - // yet gone through the promote&clear process. We only check the dirty - // map if the entry is nil. - if e == nil && r.incomplete { - e = c.dirty[k] - c.missed(r) - r = c.read() - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() - return v, err, s +// autoClean cleans the cache every interval. It holds the cache only +// weakly, so that a cache nobody stopped can still be garbage collected. +func autoClean[K comparable, V any](wc weak.Pointer[Cache[K, V]], interval time.Duration, quit, done chan struct{}) { + defer close(done) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + case <-quit: + return } + c := wc.Value() + if c == nil { + return + } + c.Clean() } +} - l := &loading[V]{ - stale: e.maybeNewStale(c.cfg.maxStaleAge), +// Get returns the cache value for k, running the miss function if the key is +// not yet cached. If stale values are enabled, the currently cached value has +// an error, and there is an unexpired stale value, this returns the stale +// value and no error. +// +// The miss function runs in the calling goroutine, unless Get returns a stale +// value: then it runs in a new goroutine and may still be running when Get +// returns. +func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { + e := c.t.loadEntry(k) + if e != nil { + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { + return v, err, s + } } - l.wg.Add(1) + return c.getSlow(k, e, miss, func() func() (V, error) { return miss }) +} - // If we have no entry, this is completely new. If we had an entry, it - // was in the read map and not yet deleted through promotion. We know - // the pointer is not promotingDelete, since that is only set while - // promoting which requires the cache lock which we have right now. - // - // In the worst case we race with a concurrent Set and we override its - // results. - if e != nil { - e.p.Store(l) - } else { - e = new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) - } - c.mu.Unlock() - - go func() { v, err := miss(); l.setve(v, err, c.cfg.newExpires(err)) }() - - // We could have set our own stale value which can be returned - // immediately rather than waiting for the get. If there is a valid - // stale to be returned now, `get` returns it. If `get` returns a Hit, - // we ended up waiting for ourself and we return a miss. There should - // be no Miss returns from `get`. - v, err, s = e.get(c.cfg.maxIdleAge) - switch s { - case Miss, Hit: - // We always return l.v and l.err. If this expires immediately, - // get may return no value. We want to return at least the - // value generated from this miss. - return l.v, l.err, Miss - } - return v, err, Stale -} - -func (c *Cache[K, V]) tryLoadEnt(k K, dirty func()) *ent[V] { - r := c.read() - e := r.m[k] - if e == nil && r.incomplete { - c.mu.Lock() - r = c.read() - e = r.m[k] - if e == nil && r.incomplete { - e = c.dirty[k] - if dirty != nil { - dirty() +// getSlow is Get after the hit check on e, which is k's entry or nil. It +// takes the miss function in two forms: load runs the load in this +// goroutine, and detach returns a function that runs the load after +// getSlow returns. We only call detach to refresh in the background, so +// load never escapes and callers only allocate for detach when we refresh. +func (c *Cache[K, V]) getSlow(k K, e *ent[K, V], load func() (V, error), detach func() func() (V, error)) (v V, err error, s KeyState) { + + // We need to load. We either create the entry with our loading, or + // replace the entry's finalized expired or errored loading with ours. + // A nil value is a deleted entry that is never written again (see + // entDel), so we unlink it and create a new entry. + var l *loading[V] +outer: + for { + if e == nil { + l = &loading[V]{} + l.wg.Add(1) + var loaded bool + if e, loaded = c.t.loadOrStoreEntry(k, l); !loaded { + break + } + } + for { + prev := e.p.Load() + if prev == nil { + c.t.deleteEntryIf(k, entDead[K, V]) + e = nil + continue outer + } + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { + return v, err, s + } + // entGet loads the value itself, so what it looked at may not + // be prev. We only return a stale if it came from prev while + // prev is still loading, and we only replace prev if prev + // itself is finalized and expired or errored. + if s == Stale && !prev.finalized() && e.p.Load() == prev { + return v, err, s + } + if !prev.finalized() || (prev.err() == nil && !prev.expired(now())) { + continue + } + if st, ok := maybeNewStale(prev, c.cfg.maxStaleAge); ok { + // One allocation for both: they have the same lifetime. + lx := &loadingWithExtra[V]{x: loadingExtra[V]{stale: st, hasStale: true}} + l = &lx.l + l.x.Store(&lx.x) + } else { + l = new(loading[V]) + } + l.wg.Add(1) + if e.p.CompareAndSwap(prev, l) { + break outer } - c.missed(r) } - c.mu.Unlock() } - return e + + if st := l.stale(); st != nil && !st.expired(now()) { + go c.load(l, detach()) + return st.v, nil, Stale + } + + // We return what our load produced, even if it is already expired: we + // do not re-derive a stale from it, and we do not extend its idle age, + // because we are a Miss. If a Swap replaced our load, the Swap may + // still be finalizing l, so we wait. + c.load(l, load) + l.wg.Wait() + return l.v, l.err(), Miss +} + +// errMissPanicked is what a load is finalized with if the miss function +// panics or exits its goroutine. It is born expired and never observable: +// every read of it is a Miss (or the previous value's stale), so waiting +// Gets run their own load. +var errMissPanicked = errors.New("cache: miss function panicked") + +func (c *Cache[K, V]) load(l *loading[V], miss func() (V, error)) { + var done bool + defer func() { + if !done { + var zero V + l.setve(zero, errMissPanicked, -now()) + } + }() + v, err := miss() + done = true + l.setve(v, err, c.cfg.newExpires(err, 0)) } // TryGet returns the value for the given key if it is cached. This returns -// either the currently stored value, or the current stale if the load errored, -// or the load error if there is no stale. If nothing is cached, or what is -// cached is expired, this returns Miss. +// either the currently stored value, or the current stale if the load errored +// or the value expired, or the load error if there is no stale. If nothing is +// cached, or what is cached is expired, this returns Miss. func (c *Cache[K, V]) TryGet(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, nil) - return e.tryGet(0, c.cfg.maxIdleAge) + e := c.t.loadEntry(k) + if e == nil { + var v V + return v, nil, Miss + } + return entTryGet(e, 0, c.cfg.maxIdleAge, c.cfg.maxStaleAge) } // Delete deletes the value for a key and returns the prior value, if stored // (i.e. the return from TryGet). func (c *Cache[K, V]) Delete(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, func() { delete(c.dirty, k) }) - defer e.del() - return e.tryGet(0, 0) + var v V + e := c.t.loadEntry(k) + if e == nil { + return v, nil, Miss + } + // We sample the clock before deleting so that we report the value as + // it was when we deleted it. + n := now() + was := entDel(e) + c.t.deleteEntryIf(k, entDead[K, V]) + if was == nil { + return v, nil, Miss + } + return loadingTryGet(was, n, 0, c.cfg.maxStaleAge) } // Expire sets a stored value to expire immediately, meaning the next Get will // be a miss. If stale values are enabled, the next Get will trigger the miss // function but still allow the now-stale value to be returned. func (c *Cache[K, V]) Expire(k K) { - e := c.tryLoadEnt(k, nil) - if l := e.load(); l != nil && l.finalized() { - l.expires.Store(now() - 1) + e := c.t.loadEntry(k) + if e == nil { + return + } + // We replace the loading with an expired copy rather than writing its + // expiry: a Set or Swap can replace the loading between our load and + // our write, and a write to the replaced loading would be lost. With + // the CAS, either we win and the Swap sees an expired value, or the + // Swap wins and we expire its value. + for { + l := e.p.Load() + if l == nil || !l.finalized() { + return + } + cur := l.expires.Load() + n := now() + if cur != 0 && cur <= n { + return // never move an expiry later + } + l2 := &loading[V]{v: l.v} + l2.x.Store(l.x.Load()) + l2.expires.Store(n - 1) + l2.state.Store(stateFinalized) + if e.p.CompareAndSwap(l, l2) { + return + } } } @@ -336,52 +454,48 @@ func (c *Cache[K, V]) Expire(k K) { func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // When ranging, repeated time.Now() calls add up, so we get the // current time when we enter range and avoid it in all tryGet calls. - now := now() - c.each(func(k K, e *ent[V]) bool { - v, err, s := e.tryGet(now, 0) + tn := now() + c.t.walk(func(e *ent[K, V]) bool { + v, err, s := entTryGet(e, tn, 0, c.cfg.maxStaleAge) if s.IsMiss() { return true } - return fn(k, v, err) + return fn(e.key, v, err) }) } -// Similar to sync.Map, we promote on range because range is O(N) (usually) and -// this amortizes out. -func (c *Cache[K, V]) each(fn func(K, *ent[V]) bool) { - r := c.read() - if r.incomplete { - c.mu.Lock() - r = c.read() - if r.incomplete { - c.promote() - r = c.read() - } - c.mu.Unlock() - } - for k, e := range r.m { - if !fn(k, e) { - return - } - } -} - // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually // expired a key. If MaxStaleAge is used and not -1, the entry must be older // than MaxAge + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. func (c *Cache[K, V]) Clean() { - // If MaxStaleAge is -1, the user opted into persisting stales forever - // and we do not clean. if c.cfg.maxStaleAge < 0 { return } - now := now() - c.each(func(k K, e *ent[V]) bool { - if l := e.load(); l != nil && l.finalized() { - expires := l.expires.Load() - if expires != 0 && now > expires+int64(c.cfg.maxStaleAge) { - c.Delete(k) + tn := now() + + // We delete exactly what TryGet would return Miss for: an expired value + // whose stale (if any) has also expired. Unlinking while we walk is + // safe: the walk holds its own references, and unlinking never moves + // an entry. + c.t.walk(func(e *ent[K, V]) bool { + l := e.p.Load() + if l == nil { + c.t.deleteEntryIf(e.key, entDead[K, V]) + return true + } + if !l.finalized() { + return true + } + expires := l.expires.Load() + expired := expires != 0 && expires <= tn + if st := l.stale(); expired && (st == nil || st.expired(tn)) && + (l.err() != nil || !staleOpen(expires, tn, c.cfg.maxStaleAge)) { + // We claim the expiry before deleting so that a racing idle + // extension, which CASes the expiry it read, fails. If the + // extension wins, the entry is in use and we leave it. + if l.expires.CompareAndSwap(expires, expiredClaimed) && e.p.CompareAndSwap(l, nil) { + c.t.deleteEntryIf(e.key, entDead[K, V]) } } return true @@ -390,11 +504,7 @@ func (c *Cache[K, V]) Clean() { // Clear deletes all keys from the cache, resetting it to an empty state. func (c *Cache[K, V]) Clear() { - c.mu.Lock() - c.dirty = nil - c.storeRead(read[K, V]{}) - c.misses = 0 - c.mu.Unlock() + c.t.clear() } // StopAutoClean stops the auto clean goroutine that is began with the @@ -403,6 +513,7 @@ func (c *Cache[K, V]) StopAutoClean() { c.quitOnce.Do(func() { if c.quitClean != nil { close(c.quitClean) + <-c.cleanDone } }) } @@ -413,85 +524,77 @@ func (c *Cache[K, V]) StopAutoClean() { // previous error if there is no stale. If nothing was cached, this returns // Miss. func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { - l := c.finalizedLoading(v) + // We use one clock sample for our value's expiry and for reporting + // what we replace. It predates whichever CAS succeeds. + n := now() + l := c.finalizedLoading(v, n) + // was is what we replaced, and wasN is the clock just before we did. var was *loading[V] + var wasN int64 defer func() { if was == nil { return } - - // The following block is similar to setve, but expanded to - // return our prior value (or stale, or err) if it exists. - now := now() if !was.finalized() { - was.mu.Lock() - if !was.finalized() { - if was.stale != nil && !was.stale.expired(now) { - old, oldState = was.stale.v, Stale + // If was is still loading, we finalize it with our value so + // that Gets waiting on it return our value. + if was.setve(v, nil, l.expires.Load()) { + if st := was.stale(); st != nil && !st.expired(wasN) { + old, oldState = st.v, Stale } - was.v = v - was.expires.Store(c.cfg.newExpires(nil)) - was.state.Store(stateFinalized) - was.wg.Done() - was.mu.Unlock() return } - was.mu.Unlock() + was.wg.Wait() // the load finalized was first } - if expired := was.expired(now); was.err != nil || expired { - if was.stale != nil && !was.stale.expired(now) { - old, oldState = was.stale.v, Stale - } - if expired { - return - } - } - old, oldErr, oldState = was.v, was.err, Hit + old, oldErr, oldState = classifyFinalized(was, was.expires.Load(), wasN, c.cfg.maxStaleAge) }() - r := c.read() - if e, ok := r.m[k]; ok { + if e := c.t.loadEntry(k); e != nil { + if rm := e.p.Load(); rm != nil { + was, wasN = rm, n + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState + } + } + } + for { + e, loaded := c.t.loadOrStoreEntry(k, l) + if !loaded { + was = nil + return old, oldErr, oldState + } for { rm := e.p.Load() - if rm != nil && rm.promotingDelete() { // deleted & currently being ignored in a promote + if rm == nil { + c.t.deleteEntryIf(k, entDead[K, V]) break } - was = rm + was, wasN = rm, n if e.p.CompareAndSwap(rm, l) { return old, oldErr, oldState } } } - - c.mu.Lock() - r = c.read() - if e := r.m[k]; e != nil { - was = e.p.Swap(l) // was not in read, but promoted by the time we entered the lock and is now in read - } else if e = c.dirty[k]; e != nil { - was = e.p.Swap(l) - } else { - e := new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) - } - c.mu.Unlock() - return old, oldErr, oldState } -func (l *loading[V]) setve(v V, err error, expires int64) { - if l.finalized() { - return +// setve finalizes l if it is still loading, returning whether we did. +func (l *loading[V]) setve(v V, err error, expires int64) bool { + if !l.state.CompareAndSwap(stateLoading, stateFinalizing) { + return false } - l.mu.Lock() - defer l.mu.Unlock() - if l.finalized() { - return + l.v = v + if err != nil { + if x := l.x.Load(); x != nil { + x.err = err + } else { + l.x.Store(&loadingExtra[V]{err: err}) + } } - l.v, l.err = v, err l.expires.Store(expires) l.state.Store(stateFinalized) l.wg.Done() + return true } // Set sets a value for a key. If the key is currently loading via Get, the @@ -501,303 +604,314 @@ func (c *Cache[K, V]) Set(k K, v V) { } // CompareAndSwap swaps the old and new values for k if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading, is not expired or errored, and is equal to old. The type V must be +// comparable. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { - r := c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if !r.incomplete { + e := c.t.loadEntry(k) + if e == nil { return false } - c.mu.Lock() - defer c.mu.Unlock() - r = c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if e, ok := c.dirty[k]; ok { - defer c.missed(r) - return c.tryCAS(e, old, new, true) - } - return false + return c.tryCAS(e, old, new, true) } -// CompareAndDelete deletes the entry for k if the value has finished loading -// and the value is equal to old. The type V must be comparable. +// CompareAndDelete deletes the entry for k if the value has finished loading, +// is not expired or errored, and is equal to old. The type V must be +// comparable. func (c *Cache[K, V]) CompareAndDelete(k K, old V) (deleted bool) { - r := c.read() - e, ok := r.m[k] - if !ok && r.incomplete { - c.mu.Lock() - r = c.read() - e, ok = r.m[k] - if !ok && r.incomplete { - e, ok = c.dirty[k] - c.missed(r) - } - c.mu.Unlock() + e := c.t.loadEntry(k) + if e == nil { + return false } - if ok { - return c.tryCAS(e, old, old, false) + if !c.tryCAS(e, old, old, false) { + return false } - return false + c.t.deleteEntryIf(k, entDead[K, V]) + return true } -func (c *Cache[K, V]) tryCAS(e *ent[V], old, new V, useNew bool) bool { +func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { l := e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { + if !casMatches(l, old) { return false } var l2 *loading[V] if useNew { - l2 = c.finalizedLoading(new) + l2 = c.finalizedLoading(new, 0) + } + if casPublishHook != nil { + casPublishHook() } for { + // We check again after allocating: if we were descheduled, the + // value may have expired. + if !casMatches(l, old) { + return false + } if e.p.CompareAndSwap(l, l2) { return true } l = e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { - return false - } } } -////////////////////// -// CACHE READ/DIRTY // -////////////////////// - -func (c *Cache[K, V]) read() read[K, V] { - p := c.r.Load() - if p == nil { - return read[K, V]{} +// casMatches returns whether l is a loaded, unerrored, unexpired value equal +// to old, i.e. whether TryGet would return old as a Hit. +func casMatches[V any](l *loading[V], old V) bool { + if l == nil || !l.finalized() || l.err() != nil || any(l.v) != any(old) { + return false } - return *p -} -func (c *Cache[K, V]) storeRead(r read[K, V]) { c.r.Store(&r) } - -func (c *Cache[K, V]) storeDirty(r read[K, V], k K, e *ent[V]) { - if !r.incomplete { - if c.dirty == nil { - c.dirty = make(map[K]*ent[V]) - } - c.storeRead(read[K, V]{m: r.m, incomplete: true}) + expires := l.expires.Load() + if expires == 0 { + return true } - c.dirty[k] = e + expires, n := l.expiresNow(expires) + return expires == 0 || expires > n } -func (c *Cache[K, V]) missed(r read[K, V]) { - c.misses++ - if len(c.dirty) == 0 || c.misses > len(r.m)>>1 { - c.promote() +/////////////////// +// ENTRY HELPERS // +/////////////////// + +func (c *Cache[K, V]) finalizedLoading(v V, n int64) *loading[V] { + l := &loading[V]{v: v} + if expires := c.cfg.newExpires(nil, n); expires != 0 { + l.expires.Store(expires) } + l.state.Store(stateFinalized) + return l } -func (c *Cache[K, V]) promote() { - r := c.read() - keep := r.m - - defer func() { - c.storeRead(read[K, V]{m: keep}) - c.misses = 0 - }() +func (l *loading[V]) finalized() bool { return l.state.Load() == stateFinalized } - if len(c.dirty) == 0 { - return +// err returns the load's error. l must be finalized. +func (l *loading[V]) err() error { + if x := l.x.Load(); x != nil { + return x.err } + return nil +} - keep = make(map[K]*ent[V], len(keep)+len(c.dirty)) - for k, e := range c.dirty { - keep[k] = e +// stale returns the loading's stale, or nil if it has none. +func (l *loading[V]) stale() *stale[V] { + if x := l.x.Load(); x != nil && x.hasStale { + return &x.stale } - c.dirty = nil + return nil +} -outer: - for k, e := range r.m { - p := e.p.Load() - for p == nil { - if e.p.CompareAndSwap(nil, c.pd) { - continue outer - } - p = e.p.Load() // concurrently deleted while promoting - } - if p != c.pd { - keep[k] = e - } - } +// entDel deletes the entry's value, returning what it was. +// +// A nil value is permanent: nothing stores a value into an entry after it +// is nil. Anything that finds a nil value unlinks the entry and creates a +// new one. This is what makes entDead a safe deleteEntryIf predicate: a +// live entry can never be unlinked. +func entDel[K comparable, V any](e *ent[K, V]) *loading[V] { + return e.p.Swap(nil) } -///////// -// ENT // -///////// +func entDead[K comparable, V any](e *ent[K, V]) bool { return e.p.Load() == nil } -func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { - l := &loading[V]{ - v: v, - } - expires := c.cfg.newExpires(nil) - if expires != 0 || c.cfg.maxStaleAge != 0 { - l.expires.Store(expires) - if c.cfg.maxStaleAge != 0 { - l.stale = newStale(v, l.expires.Load(), c.cfg.maxStaleAge) - } - } - l.state.Store(stateFinalized) - return l +func (l *loading[V]) expired(n int64) bool { + expires := l.expires.Load() + return expires != 0 && expires <= n // 0 means either miss not resolved, or no max age } -func (l *loading[V]) promotingDelete() bool { return l.state.Load() == 2 } -func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -func (l *loading[V]) onlyFinalized() bool { return l.state.Load() == 1 } +func (s *stale[V]) expired(n int64) bool { + expires := s.expires + return expires != 0 && expires <= n +} -func (e *ent[V]) del() { - if e == nil { - return +// maybeNewStale returns the stale to carry in a loading that replaces the +// finalized l: l's own stale if l errored, otherwise l's value with the +// stale window l's value has (see staleOpen). +func maybeNewStale[V any](l *loading[V], age time.Duration) (stale[V], bool) { + if age == 0 { + return stale[V]{}, false } - for { - p := e.p.Load() - if p == nil || p.promotingDelete() { - return - } - if e.p.CompareAndSwap(p, nil) { - return + if l.err() != nil { + if st := l.stale(); st != nil { + return *st, true } + return stale[V]{}, false + } + expires := l.expires.Load() + if expires == expiredClaimed { + return stale[V]{}, false } + return newStale(l.v, expires, age), true } -func (e *ent[V]) load() *loading[V] { - if e == nil { - return nil +// staleOpen returns whether an expired, unerrored value can still be +// returned as stale at n. expires must be non-zero. This must agree with +// the window newStale creates. +func staleOpen(expires, n int64, age time.Duration) bool { + if age == 0 || expires == expiredClaimed { + return false + } + if age < 0 { + return true } - p := e.p.Load() - if p == nil || p.promotingDelete() { - return nil + if expires < 0 { + expires = -expires // born expired } - return p + return n < satAdd(expires, int64(age)) } -func (l *loading[V]) expired(now int64) bool { - expires := l.expires.Load() - return expires != 0 && expires <= now // 0 means either miss not resolved, or no max age +// newStale returns a stale for v, whose loading's expires is expires. age +// must be non-zero, and expires must be an expired value's: non-zero and +// not expiredClaimed. +func newStale[V any](v V, expires int64, age time.Duration) stale[V] { + if age < 0 { + return stale[V]{v: v} + } + if expires < 0 { + expires = -expires // born expired + } + return stale[V]{v, satAdd(expires, int64(age))} } -func (s *stale[V]) expired(now int64) bool { - expires := s.expires - return expires != 0 && expires <= now -} +// expiresPairingHook, if non-nil, runs in expiresNow between the expiry load +// and the clock sample, so that tests can pause a reader there. +var expiresPairingHook func() -// If an entry is expiring, we create a new entry with this previous entry as -// a stale. +// casPublishHook, if non-nil, runs in tryCAS before the final liveness +// check, so that tests can pause a CAS there. +var casPublishHook func() + +// expiresNow returns l's expiry and a clock sample such that the expiry was +// current when the clock was sampled. expires is the caller's load of the +// expiry. // -// - if entry is nil, no stale, return nil -// - if no stale age, we are not using stales, return nil -// - if entry has an error, return prior stale -// - if age is < 0, return new unexpiring stale -// - else, return new stale with prior expiry + stale age -func (e *ent[V]) maybeNewStale(age time.Duration) *stale[V] { - if e == nil || age == 0 { - return nil +// We load the expiry before sampling the clock: Expire stores its own +// now()-1, which may be later than a clock sampled before our load. We then +// load the expiry again and retry if it changed: otherwise, an idle +// extension landing between the load and the sample would pair a dead +// expiry with a clock at which the value is live, and an Expire landing +// there would let CompareAndSwap match a value that was already expired. +func (l *loading[V]) expiresNow(expires int64) (int64, int64) { + if expiresPairingHook != nil { + expiresPairingHook() + } + for { + n := now() + again := l.expires.Load() + if again == expires { + return expires, n + } + expires = again } - l := e.load() - if l == nil || !l.finalized() { - return nil +} + +// classifyFinalized returns what a read of l at n returns. A valid stale +// takes precedence over an error; an unexpired error is returned as a Hit; +// an expired value is returned as Stale while its stale window is open. +func classifyFinalized[V any](l *loading[V], expires, n int64, staleAge time.Duration) (v V, err error, state KeyState) { + expired := expires != 0 && expires <= n + if lerr := l.err(); lerr != nil { + if st := l.stale(); st != nil && !st.expired(n) { + return st.v, nil, Stale + } + if !expired { + return l.v, lerr, Hit + } + return v, err, state } - if l.err != nil { - return l.stale + if expired { + if staleOpen(expires, n, staleAge) { + return l.v, nil, Stale + } + return v, err, state } - return newStale(l.v, l.expires.Load(), age) + return l.v, nil, Hit } -// Actually returns the stale; age must be non-zero. -func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { - if age < 0 { - return &stale[V]{v: v} +// maxIdleSlack bounds how far we let an idle extension drift from now+age +// before we write it; see MaxIdleAge. +const maxIdleSlack = time.Second + +// extend resets l's expiry to n+age if the expiry is still what we read. +// We skip the write if it would move the expiry by less than the slack, so +// that a hot key does not write its expiry on every hit. We sample the +// clock again right before the CAS so that we do not revive a value that +// expired while we were descheduled. +func (l *loading[V]) extend(expires, n int64, age time.Duration) { + target := satAdd(n, int64(age)) + slack := min(int64(age)>>6, int64(maxIdleSlack)) + if d := target - expires; d < slack && d > -slack { + return + } + if now() < expires { + l.expires.CompareAndSwap(expires, target) } - return &stale[V]{v, expires + int64(age)} } -// get always returns the value or the stale value. We do not check if our -// value is expired: we call this at the end of Get, we must always return -// something even if it is to be immediately expired. -func (e *ent[V]) get(extend time.Duration) (v V, err error, state KeyState) { - l := e.load() - var waited bool +// entGet is TryGet, but if the entry is loading with no valid stale, we +// wait for the load. If we waited and the load's result is already expired +// (MaxAge(0), MaxErrorAge(0), or a TTL shorter than the load), we return +// it anyway as a Hit: every Get that collapsed onto a load gets its result. +// The one exception is a panicked load, which has no result to share. +func entGet[K comparable, V any](e *ent[K, V], extend, staleAge time.Duration) (v V, err error, state KeyState) { + l := e.p.Load() if l == nil { - // Could be nil if deleted while in the read map, or - // promotingDelete. return v, err, state } + var waited bool if !l.finalized() { - if l.stale != nil && !l.stale.expired(now()) { - return l.stale.v, nil, Stale + if st := l.stale(); st != nil && !st.expired(now()) { + return st.v, nil, Stale } l.wg.Wait() waited = true } - - // If we did not wait and our entry is expired (value or error), or if - // our entry is not expired but has errored, we potentially return the - // stale entry. - // - // If we waited, we could immediately be expired due to time sync, or - // if the user is configured to never cache and they're just using - // request collapsing: we still want to return the now expired value. - now := now() - if (!waited && l.expired(now)) || l.err != nil { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale - } - // The stale value is expired: if our entry is not expired, - // this must be an error we waited on. - if !l.expired(now) { - return l.v, l.err, Hit - } - return v, err, state + expires := l.expires.Load() + if expires == 0 && l.x.Load() == nil { + return l.v, nil, Hit } - if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + expires, n := l.expiresNow(expires) + if waited && expires != 0 && expires <= n && l.err() != errMissPanicked { + return l.v, l.err(), Hit } - return l.v, l.err, Hit + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && err == nil { + l.extend(expires, n, extend) + } + return v, err, state } -func (e *ent[V]) tryGet(n64 int64, extend time.Duration) (v V, err error, state KeyState) { - if e == nil { - return v, err, state - } - l := e.load() - if l == nil { // deleting or promotedDelete +func entTryGet[K comparable, V any](e *ent[K, V], n int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { + l := e.p.Load() + if l == nil { return v, err, state } - // Fast path: finalized, no expiry, no error — skip time checks. - if l.onlyFinalized() && l.expires.Load() == 0 && l.err == nil { - return l.v, nil, Hit - } - if n64 == 0 { - n64 = now() - } - now := n64 + return loadingTryGet(l, n, extend, staleAge) +} - // If we are loading but there is a valid stale, return it, otherwise - // return immediately: no get. +// loadingTryGet is TryGet for a loading. If n is non-zero, it is the clock +// to use, and extend must be zero. +func loadingTryGet[V any](l *loading[V], n int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { if !l.finalized() { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale + if n == 0 { + n = now() + } + if st := l.stale(); st != nil && !st.expired(n) { + return st.v, nil, Stale } return v, err, state } - - // If we have an error or we are expired, we maybe return the stale. - if expired := l.expired(now); l.err != nil || expired { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale - } - if expired { - return v, err, state - } + // If the caller passed n, it sampled n before this load, which is the + // order expiresNow uses. + expires := l.expires.Load() + if expires == 0 && l.x.Load() == nil { + return l.v, nil, Hit } - if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + if n == 0 { + expires, n = l.expiresNow(expires) } - return l.v, l.err, Hit + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && err == nil { + l.extend(expires, n, extend) + } + return v, err, state } ////////// @@ -815,21 +929,15 @@ type Item[V any] struct { // semantics. If you do not need to configure an item at all, the zero value // item is valid and usable. func NewItem[V any](opts ...Opt) *Item[V] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Item[V]{ - c: Cache[struct{}, V]{ - cfg: c, - }, - } + i := new(Item[V]) + initCache(&i.c, opts...) + return i } -// Get returns the currently cached value, running the miss function in a -// goroutine if the item is not yet cached. If stale values are enabled, the -// currently cached value has an error, and there is an unexpired stale value, -// this returns the stale value and no error. +// Get returns the currently cached value, running the miss function if the +// item is not yet cached. If stale values are enabled, the currently cached +// value has an error, and there is an unexpired stale value, this returns the +// stale value and no error. func (i *Item[V]) Get(miss func() (V, error)) (v V, err error, state KeyState) { return i.c.Get(struct{}{}, miss) } @@ -871,13 +979,14 @@ func (i *Item[V]) Swap(v V) (old V, oldErr error, oldState KeyState) { } // CompareAndSwap swaps the old and new values if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading, is not expired or errored, and is equal to old. The type V must be +// comparable. func (i *Item[V]) CompareAndSwap(old, new V) (swapped bool) { return i.c.CompareAndSwap(struct{}{}, old, new) } -// CompareAndDelete deletes the item if the value has finished loading and the -// value is equal to old. The type V must be comparable. +// CompareAndDelete deletes the item if the value has finished loading, is not +// expired or errored, and is equal to old. The type V must be comparable. func (i *Item[V]) CompareAndDelete(old V) (deleted bool) { return i.c.CompareAndDelete(struct{}{}, old) } @@ -902,24 +1011,26 @@ type Set[K comparable] struct { // semantics. If you do not need to configure an set at all, the zero value set // is valid and usable. func NewSet[K comparable](opts ...Opt) *Set[K] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Set[K]{ - c: Cache[K, struct{}]{ - cfg: c, - }, - } + s := new(Set[K]) + initCache(&s.c, opts...) + return s } -// Get ensures the key is cached, running the miss function in a goroutine if -// the key is not yet cached. If stale keys are enabled, the currently cached -// key has an error, and there is a stale key, this returns with no error and a -// Stale key state. +// Get ensures the key is cached, running the miss function if the key is not +// yet cached. If stale keys are enabled, the currently cached key has an +// error, and there is a stale key, this returns with no error and a Stale key +// state. func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { - _, err, state = s.c.Get(k, func() (struct{}, error) { - return struct{}{}, miss() + c := &s.c + e := c.t.loadEntry(k) + if e != nil { + if _, err, state = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); state == Hit { + return err, state + } + } + load := func() (struct{}, error) { return struct{}{}, miss() } + _, err, state = c.getSlow(k, e, load, func() func() (struct{}, error) { + return func() (struct{}, error) { return struct{}{}, miss() } }) return err, state } diff --git a/cache/cache_bench_test.go b/cache/cache_bench_test.go new file mode 100644 index 0000000..df49412 --- /dev/null +++ b/cache/cache_bench_test.go @@ -0,0 +1,151 @@ +package cache + +import ( + "runtime" + "sync/atomic" + "testing" + "time" +) + +func benchGetHit(b *testing.B, opts ...Opt) { + c := New[int, int](opts...) + defer c.StopAutoClean() + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.Get(i%keys, miss) + i++ + } + }) +} + +func BenchmarkGetHit(b *testing.B) { benchGetHit(b) } +func BenchmarkGetHitMaxAge(b *testing.B) { benchGetHit(b, MaxAge(time.Hour)) } +func BenchmarkGetHitIdleAge(b *testing.B) { benchGetHit(b, MaxIdleAge(time.Hour)) } + +func BenchmarkGetHitOneKey(b *testing.B) { benchGetHitOneKey(b) } +func BenchmarkGetHitOneKeyIdleAge(b *testing.B) { benchGetHitOneKey(b, MaxIdleAge(time.Hour)) } + +func benchGetHitOneKey(b *testing.B, opts ...Opt) { + c := New[int, int](opts...) + c.Set(0, 0) + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + c.Get(0, miss) + } + }) +} + +func BenchmarkTryGetHitMaxAge(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.TryGet(i % keys) + i++ + } + }) +} + +func BenchmarkGetMissUnique(b *testing.B) { + c := New[int, int]() + var ctr atomic.Int64 + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + c.Get(int(ctr.Add(1)), miss) + } + }) +} + +func BenchmarkGetRefreshExpired(b *testing.B) { + c := New[int, int](MaxAge(0)) + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.Get(i%1024, miss) + i++ + } + }) +} + +func BenchmarkSetExisting(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.Set(i%keys, i) + i++ + } + }) +} + +func BenchmarkDeleteSet(b *testing.B) { + c := New[int, int]() + var ctr atomic.Int64 + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + k := int(ctr.Add(1)) % 4096 + c.Set(k, k) + c.Delete(k) + } + }) +} + +func BenchmarkClean(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + for i := range 4096 { + c.Set(i, i) + } + b.ResetTimer() + for range b.N { + c.Clean() + } +} + +// BenchmarkGetRefreshStale measures a refresh that returns the stale value +// while the load runs in the background, waiting for each load to finish. +func BenchmarkGetRefreshStale(b *testing.B) { + c := New[int, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set(0, 0) + miss := func() (int, error) { return 0, nil } + for range b.N { + c.Get(0, miss) + for !c.t.loadEntry(0).p.Load().finalized() { + runtime.Gosched() + } + } +} + +// BenchmarkGetHitCapturingMiss is GetHit with the miss closure a real caller +// writes: one that captures the key. +func BenchmarkGetHitCapturingMiss(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + k := i % keys + c.Get(k, func() (int, error) { return k, nil }) + i++ + } + }) +} diff --git a/cache/cache_test.go b/cache/cache_test.go index d170261..89053bc 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -2,6 +2,8 @@ package cache import ( "errors" + "math" + "runtime" "sync" "sync/atomic" "testing" @@ -58,7 +60,7 @@ func TestCollapsedGet(t *testing.T) { calls int64 ps = make(chan *int) ) - for i := 0; i < niter; i++ { + for range niter { go func() { p, _, _ := c.Get("foo", func() (*int, error) { if atomic.AddInt64(&calls, 1) != 1 { @@ -70,7 +72,7 @@ func TestCollapsedGet(t *testing.T) { ps <- p }() } - for i := 0; i < niter; i++ { + for range niter { p := <-ps if p != r { t.Error("pointer mismatch") @@ -118,19 +120,18 @@ func TestSetExpire(t *testing.T) { c.Delete(k) c.Set(k, 10) - for i := 0; i < 2; i++ { + for range 2 { var wg sync.WaitGroup - wg.Add(10) - go func() { defer wg.Done(); c.Range(func(string, int, error) bool { return false }) }() - go func() { defer wg.Done(); c.Delete(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() - go func() { defer wg.Done(); c.Expire(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() - go func() { defer wg.Done(); c.Range(func(string, int, error) bool { return false }) }() - go func() { defer wg.Done(); c.Delete(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() - go func() { defer wg.Done(); c.Expire(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() + wg.Go(func() { c.Range(func(string, int, error) bool { return false }) }) + wg.Go(func() { c.Delete(k) }) + wg.Go(func() { c.Set(k, 10) }) + wg.Go(func() { c.Expire(k) }) + wg.Go(func() { c.Set(k, 10) }) + wg.Go(func() { c.Range(func(string, int, error) bool { return false }) }) + wg.Go(func() { c.Delete(k) }) + wg.Go(func() { c.Set(k, 10) }) + wg.Go(func() { c.Expire(k) }) + wg.Go(func() { c.Set(k, 10) }) wg.Wait() v, err, s = c.Get(k, func() (int, error) { return 10, nil }) check(got[int]{v, err, s}, 10) @@ -177,10 +178,14 @@ func TestExpires(t *testing.T) { close(ch) // Here, we check that the stale is returned because it is - // expired. We sleep 10ms to allow the goroutines running the - // prior miss to finish and update our value to baz. - time.Sleep(10 * time.Millisecond) + // expired. We wait for the prior miss to finish and update our + // value to baz, and block the new miss so that it cannot finish + // before Get checks for a stale. + waitFinalized(t, &c.t, "foo") + ch = make(chan struct{}) + defer close(ch) v, err, s = c.Get("foo", func() (string, error) { + <-ch return "bar", nil }) vcheck(t, got[string]{v, err, s}, got[string]{"baz", nil, Stale}) @@ -196,7 +201,7 @@ func TestExpires(t *testing.T) { c.Expire("foo") time.Sleep(time.Millisecond) c.Get("foo", func() (string, error) { return "", errors.New("foo") }) - time.Sleep(time.Millisecond) + waitFinalized(t, &c.t, "foo") // The cache is primed: foo exists, it was expired, and we // injected an error such that the next loads should be stale. @@ -213,7 +218,7 @@ func TestExpires(t *testing.T) { v, err, s = c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Stale}) - ch <- struct{}{} + close(ch) } // Misc. bits. @@ -282,7 +287,7 @@ func TestExpires(t *testing.T) { gch := make(chan got[string], 10) ch := make(chan struct{}) - for i := 0; i < 10; i++ { + for range 10 { go func() { v, err, s := c.Get("foo", func() (string, error) { <-ch @@ -296,7 +301,7 @@ func TestExpires(t *testing.T) { // long, so not all Get's could be stacked at once. time.Sleep(100 * time.Millisecond) close(ch) - for i := 0; i < 10; i++ { + for range 10 { g := <-gch if g.s == Miss { vcheck(t, g, got[string]{"bar", nil, Miss}) @@ -361,79 +366,705 @@ func TestExpires(t *testing.T) { if len(exp) != 0 { t.Errorf("unexpected values remain: %v", exp) } + if n := trieEntries(c); n != 4 { + t.Errorf("Clean left %d entries, want 4", n) + } } } func TestMaxIdleAge(t *testing.T) { - // MaxIdleAge alone: entry stays alive while accessed within the window, - // expires after inactivity. + expiresOf := func(c *Cache[string, string]) int64 { + return c.t.loadEntry("foo").p.Load().expires.Load() + } + // tick ensures the next now() differs from the last. + tick := func() { time.Sleep(time.Millisecond) } + + // Hits from TryGet and Get extend. A 6.4s idle age has a 100ms slack + // (age/64), so we wait past it between hits. t.Run("alone", func(t *testing.T) { - c := New[string, string](MaxIdleAge(50 * time.Millisecond)) + const idle = 6400 * time.Millisecond + pastSlack := func() { time.Sleep(idle/64 + 10*time.Millisecond) } + c := New[string, string](MaxIdleAge(idle)) c.Get("foo", func() (string, error) { return "bar", nil }) - time.Sleep(10 * time.Millisecond) - - // Access within the idle window — should still be a hit and extend. - for i := 0; i < 5; i++ { - v, err, s := c.TryGet("foo") - vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) - time.Sleep(30 * time.Millisecond) // each access resets the 50ms idle + w0 := expiresOf(c) + pastSlack() + v, err, s := c.TryGet("foo") + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) + w1 := expiresOf(c) + if w1 <= w0 { + t.Fatalf("TryGet did not extend: %d <= %d", w1, w0) + } + pastSlack() + v, err, s = c.Get("foo", func() (string, error) { return "baz", nil }) + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) + if w2 := expiresOf(c); w2 <= w1 { + t.Fatalf("Get did not extend: %d <= %d", w2, w1) } + }) - // Now stop accessing. After 50ms of inactivity, it should expire. - time.Sleep(60 * time.Millisecond) + // Without accesses, the idle age is the TTL. + t.Run("lapses", func(t *testing.T) { + const idle = 20 * time.Millisecond + c := New[string, string](MaxIdleAge(idle)) + c.Get("foo", func() (string, error) { return "bar", nil }) + time.Sleep(3 * idle) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) - // MaxIdleAge + MaxAge: initial expiry uses MaxAge, subsequent accesses - // extend using MaxIdleAge. + // With MaxAge, an access resets the expiry to now + idle age, whether + // that is later or earlier than the MaxAge expiry. t.Run("with_max_age", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + c := New[string, string](MaxAge(time.Minute), MaxIdleAge(time.Hour)) c.Get("foo", func() (string, error) { return "bar", nil }) - - // Keep alive well past the original MaxAge. - for i := 0; i < 5; i++ { - time.Sleep(30 * time.Millisecond) - v, err, s := c.Get("foo", func() (string, error) { return "baz", nil }) - vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) + w0 := expiresOf(c) + c.TryGet("foo") + if w1 := expiresOf(c); w1 < w0+int64(50*time.Minute) { + t.Fatalf("access did not extend past MaxAge: %d vs %d", w1, w0) } - // Stop accessing, should expire. - time.Sleep(60 * time.Millisecond) - v, err, s := c.TryGet("foo") - vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) + c = New[string, string](MaxAge(time.Hour), MaxIdleAge(time.Minute)) + c.Get("foo", func() (string, error) { return "bar", nil }) + w0 = expiresOf(c) + c.TryGet("foo") + if w1 := expiresOf(c); w1 > w0-int64(50*time.Minute) { + t.Fatalf("access did not shorten to the idle age: %d vs %d", w1, w0) + } }) // MaxIdleAge doesn't extend errors. t.Run("no_extend_errors", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + c := New[string, string](MaxAge(time.Hour), MaxIdleAge(time.Hour)) c.Get("foo", func() (string, error) { return "", errors.New("err") }) - time.Sleep(10 * time.Millisecond) - - // Access the errored entry — it should not be extended. + w0 := expiresOf(c) + tick() v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", errors.New("err"), Hit}) + c.Get("foo", func() (string, error) { return "bar", nil }) + if w1 := expiresOf(c); w1 != w0 { + t.Fatalf("errored entry was extended: %d != %d", w1, w0) + } + }) - // Wait past expiry — should be gone. - time.Sleep(50 * time.Millisecond) + // MaxIdleAge must not resurrect entries that finalize already expired: + // with MaxAge(0), the Get that drives the load returns the value but + // does not extend it. + t.Run("no_resurrect_expired", func(t *testing.T) { + c := New[string, string](MaxAge(0), MaxIdleAge(time.Hour)) + v, err, s := c.Get("foo", func() (string, error) { return "bar", nil }) + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Miss}) v, err, s = c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) // Range doesn't extend. t.Run("range_no_extend", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + c := New[string, string](MaxAge(time.Hour), MaxIdleAge(time.Hour)) c.Get("foo", func() (string, error) { return "bar", nil }) + w0 := expiresOf(c) + tick() + var n int + c.Range(func(string, string, error) bool { n++; return true }) + if n != 1 { + t.Fatalf("Range saw %d keys, want 1", n) + } + if w1 := expiresOf(c); w1 != w0 { + t.Fatalf("Range extended: %d != %d", w1, w0) + } + }) +} - // Range over entries repeatedly — should NOT extend expiry. - for i := 0; i < 3; i++ { - time.Sleep(20 * time.Millisecond) - c.Range(func(string, string, error) bool { return true }) +// TestMaxErrorAge_DefaultsToIdleInitialTTL checks that with only MaxIdleAge +// set, a load error is cached for the idle age (not forever), and the next +// Get after it lapses retries. +func TestMaxErrorAge_DefaultsToIdleInitialTTL(t *testing.T) { + boom := errors.New("boom") + { + c := New[string, int](MaxIdleAge(time.Hour)) + c.Get("k", func() (int, error) { return 0, boom }) + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet within the idle age: err=%v s=%v, want the cached error as a Hit", err, s) + } + if w := c.t.loadEntry("k").p.Load().expires.Load(); w <= 0 { + t.Fatalf("error expiry = %d, want the idle age from now", w) } + } - // Should be expired by now (>50ms since creation, range didn't extend). - time.Sleep(20 * time.Millisecond) - v, err, s := c.TryGet("foo") - vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) + const idle = 20 * time.Millisecond + c := New[string, int](MaxIdleAge(idle)) + c.Get("k", func() (int, error) { return 0, boom }) + time.Sleep(3 * idle) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the idle age: %v, want Miss", s) + } + var retried bool + v, err, s := c.Get("k", func() (int, error) { retried = true; return 7, nil }) + if !retried || v != 7 || err != nil || s != Miss { + t.Fatalf("Get after the error lapsed: retried=%v (%d, %v, %v), want a fresh (7, nil, Miss) load", retried, v, err, s) + } +} + +// TestHugeAges verifies expiry arithmetic saturates rather than wrapping: +// absurd-but-valid Durations must mean "effectively forever", not "born +// expired" (or, for Clean, "evict everything"). +func TestHugeAges(t *testing.T) { + huge := time.Duration(math.MaxInt64) + + // MaxAge: now + ttl must not wrap negative. + { + c := New[string, int](MaxAge(huge)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet under huge MaxAge: v=%d s=%v, want 1 Hit", v, s) + } + } + + // MaxStaleAge: Clean's expires + staleAge must not wrap negative, and + // the stale's own expiry must not be born wrapped. + { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(huge)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry expired, stale effectively forever + c.Clean() // must not evict + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet after Clean under huge MaxStaleAge: v=%d s=%v, want 1 Stale", v, s) + } + } + + // MaxIdleAge: the extension n + idle must not wrap negative. + { + c := New[string, int](MaxIdleAge(huge)) + c.Set("k", 1) + c.TryGet("k") // a Hit extends; the extension must saturate + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet after huge idle extension: v=%d s=%v, want 1 Hit", v, s) + } + } +} + +// waitFinalized waits until k's value is not loading. +func waitFinalized[K comparable, V any](t *testing.T, tr *trie[K, loading[V]], k K) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if e := tr.loadEntry(k); e != nil { + if l := e.p.Load(); l != nil && l.finalized() { + return + } + } + if time.Now().After(deadline) { + t.Fatalf("%v never finished loading", k) + } + time.Sleep(time.Millisecond) + } +} + +// trieEntries counts every entry in the trie, including deleted entries and +// entries that are invisible to Range and TryGet. +func trieEntries[K comparable, V any](c *Cache[K, V]) (n int) { + c.t.walk(func(*ent[K, V]) bool { n++; return true }) + return n +} + +// TestClean_ReclaimsBornExpired verifies Clean's handling of born-expired +// entries (MaxAge(0) / MaxErrorAge(0), whose negative expiry word carries +// the negated birth): an entry is removed exactly when no read can see it +// anymore (as soon as TryGet reports Miss). That is immediately for errored +// stale-less entries, and when the birth-anchored stale window closes for +// values, regardless of process uptime. +// +// Regression test: born-expired entries used to store a bare -1 sentinel, +// and the grace window was computed as satAdd(-1, maxStaleAge), anchoring +// it at the process epoch: born-expired stale-less entries (error churn +// under MaxErrorAge(0)) were unreclaimable until process uptime exceeded +// MaxStaleAge, and forever for huge stale ages. +func TestClean_ReclaimsBornExpired(t *testing.T) { + t.Run("errored_no_stale", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead invisible born-expired entries in the trie", n) + } + }) + + t.Run("huge_stale_age", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Duration(math.MaxInt64))) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead entries (huge MaxStaleAge made them permanent)", n) + } + }) + + t.Run("keeps_valid_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // born expired; stale window anchored at the store + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean evicted a born-expired entry with a valid stale: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } + }) + + t.Run("reclaims_dead_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(5*time.Millisecond)) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries whose stale window had passed", n) + } + }) +} + +// TestStale_BornExpiredAnchorsAtBirth verifies that a born-expired value's +// (MaxAge(0)) stale window is anchored at the value's birth (carried by +// the negated-birth expiry word) for every read and refresh. Once the +// window passes, the value is gone for good: a later Get drives a fresh +// load rather than serving the dead value. +// +// Regression test: born-expired values used to store a bare -1 sentinel, +// destroying the birth; a Get-driven refresh arbitrarily later re-anchored +// the dead value's stale at the refresh (newStale's expires<=0 fallback), +// serving a value of unbounded age as a fresh Stale right after TryGet +// certified Miss. +func TestStale_BornExpiredAnchorsAtBirth(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(0), MaxStaleAge(age)) + start := time.Now() + c.Set("k", 1) + + // Within the window, the value is servable as a stale. We only check + // if we were not descheduled past the window. + if v, _, s := c.TryGet("k"); time.Since(start) < age && (s != Stale || v != 1) { + t.Fatalf("TryGet within the window: (%d, %v), want (1, Stale)", v, s) + } + + time.Sleep(2 * age) // window passed; the value is certifiably dead + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the window: %v, want Miss", s) + } + v, _, s := c.Get("k", func() (int, error) { return 2, nil }) + if v != 2 || s != Miss { + t.Fatalf("Get after the window: (%d, %v), want (2, Miss): the dead value must not be re-anchored at the refresh", v, s) + } +} + +// TestStale_GetLoadedValueServesSelfStale verifies the stale window applies +// uniformly regardless of how the value was written: a Get-loaded value, +// once expired, is served as Stale by TryGet for MaxStaleAge, exactly like +// a Set-loaded value. +// +// Regression test: only Set/Swap-created loadings used to carry a stale +// snapshot of themselves; Get-loaded values had none (their stale field is +// the previous generation's refresh companion), so TryGet reported Miss +// the instant they expired while a Get-driven refresh of the same state +// served the Stale. +func TestStale_GetLoadedValueServesSelfStale(t *testing.T) { + const ttl = 20 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 1, nil }) + time.Sleep(2 * ttl) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet on an expired Get-loaded value: (%d, %v), want (1, Stale)", v, s) + } +} + +// TestStale_ServesLatestGeneration verifies that once a refreshed value +// itself expires, every read serves that value as the stale: not the +// generation before it. +// +// Regression test: a Get-created loading's stale field is the previous +// generation's snapshot; TryGet used to serve it (v1) after the refreshed +// value (v2) expired, while a Get-driven refresh of the same state +// snapshotted and served v2. +func TestStale_ServesLatestGeneration(t *testing.T) { + const ttl = 30 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(ttl + 10*time.Millisecond) // v1 expired; within its stale window + + // Drive a refresh to v2; the driving Get returns the v1 stale while + // its (held-open) miss runs. + release := make(chan struct{}) + if v, _, s := c.Get("k", func() (int, error) { <-release; return 2, nil }); s != Stale || v != 1 { + t.Fatalf("refreshing Get: (%d, %v), want (1, Stale)", v, s) + } + close(release) + waitFinalized(t, &c.t, "k") + time.Sleep(2 * ttl) // v2 expired + + if v, _, s := c.TryGet("k"); s != Stale || v != 2 { + t.Fatalf("TryGet after v2 expired: (%d, %v), want (2, Stale): the freshest dead value", v, s) + } +} + +// TestMaxIdleAge_StaleWindowFollowsExtensions verifies that the stale +// window of an idle-extended entry anchors at the entry's actual death +// (its last extension lapsing), not at its initial expiry: derived from +// the expiry word, the window moves with every extension. +// +// Regression test: the self-stale used to be a snapshot frozen at +// creation, anchored at the initial expiry. Extensions moved only the +// expiry word, so an entry kept hot past initialExpiry+MaxStaleAge idled +// out with its stale window already closed: TryGet reported Miss the +// instant the entry expired while Get served the Stale. +func TestMaxIdleAge_StaleWindowFollowsExtensions(t *testing.T) { + const age = time.Minute + c := New[string, int](MaxAge(time.Millisecond), MaxIdleAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + if _, _, s := c.TryGet("k"); s != Hit { + // We were descheduled past the 1ms MaxAge; extend manually. + l.expires.Store(now() + int64(time.Hour)) + } + w1 := l.expires.Load() + if w1 <= w0+int64(age) { + t.Fatalf("precondition: extension %d not past the initial window end %d", w1, w0+int64(age)) + } + + // Half way into the window after the extended expiry, the initial + // window is long closed. + if v, _, s := loadingTryGet(l, w1+int64(age)/2, 0, age); s != Stale || v != 1 { + t.Fatalf("read inside the extended entry's window: (%d, %v), want (1, Stale)", v, s) + } + if _, _, s := loadingTryGet(l, w1+int64(age), 0, age); s != Miss { + t.Fatalf("read after the extended entry's window: %v, want Miss", s) + } +} + +// TestExpire_DoesNotResurrectDeadStale verifies that Expire on an entry +// whose value and stale window are both already dead is a no-op: the next +// Get drives a fresh load rather than serving the dead value. +// +// Regression test: Expire used to blindly store now()-1, moving an +// already-expired word forward; the next Get-driven refresh anchored a +// fresh stale window at the Expire and served the dead value as Stale for +// up to another MaxStaleAge. +func TestExpire_DoesNotResurrectDeadStale(t *testing.T) { + const ttl, age = 5 * time.Millisecond, 5 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(age)) + c.Set("k", 1) + time.Sleep(30 * time.Millisecond) // value expired AND stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Expire("k") // invalidating a dead key must not revive anything + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get after expiring a dead entry: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestExpire_StaleWindowAnchorsAtExpire verifies the uniform stale-window +// anchor for manually expired values: the window opens at the Expire and +// closes MaxStaleAge later, for TryGet and Get-driven refreshes alike. +// +// Regression test: TryGet used to serve a frozen creation-time snapshot +// whose window was anchored at the original expiry (an hour out here), +// outliving the documented Expire-anchored window. +func TestExpire_StaleWindowAnchorsAtExpire(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + start := time.Now() + c.Expire("k") + if v, _, s := c.TryGet("k"); time.Since(start) < age && (s != Stale || v != 1) { + t.Fatalf("TryGet just after Expire: (%d, %v), want (1, Stale)", v, s) + } + time.Sleep(2 * age) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the post-Expire window: %v, want Miss", s) + } +} + +// TestExpire_MidWindowDoesNotReanchor verifies Expire is a no-op on an +// entry that is expired but still inside its stale window: the expiry word +// (the window's anchor) must be byte-identical after the call. +// TestExpire_DoesNotResurrectDeadStale covers the window-already-closed +// case; here a forward re-anchor would extend a dead value's servable life +// mid-window. +func TestExpire_MidWindowDoesNotReanchor(t *testing.T) { + t.Run("ttl_expired", func(t *testing.T) { + c := New[string, int](MaxAge(20*time.Millisecond), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(40 * time.Millisecond) // expired; window open for ~1h + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire moved an already-expired word: %d -> %d (re-anchors the stale window)", w0, w1) + } + }) + t.Run("born_expired", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // word carries the negated birth + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + if w0 >= 0 { + t.Fatalf("precondition: want a negated-birth word, got %d", w0) + } + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire destroyed the birth anchor: %d -> %d", w0, w1) + } + }) +} + +// TestClean_KeepsExpiredErrorWithValidCompanion pins Clean's gate for +// errored entries: an expired error whose refresh companion (the previous +// generation) is still inside its window is visible to reads (TryGet +// serves the companion) so Clean must not reclaim it. +func TestClean_KeepsExpiredErrorWithValidCompanion(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour), MaxErrorAge(20*time.Millisecond)) + c.Set("k", 1) + c.Expire("k") + // Drive a refresh that errors; the new loading carries companion=1. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + time.Sleep(40 * time.Millisecond) // error now expired; companion valid ~1h + + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("precondition: TryGet=(%d,%v), want (1, Stale) via the companion", v, s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean reclaimed an entry whose companion was still servable: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } +} + +// TestStale_WindowBoundaries drives the exact-nanosecond boundaries of the +// derived stale window through loadingTryGet with hand-picked clocks (the +// n64 parameter pins the clock deterministically, exactly as Range and +// Delete do) and checks the window definitions agree bit-for-bit across +// the read path (staleOpen), the refresh companion (newStale), and +// stale.expired: the window is [expiry, expiry+age), half-open at every +// site, for positive, negated-birth, and claimed words. +func TestStale_WindowBoundaries(t *testing.T) { + const age = 100 * time.Millisecond + a := int64(age) + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + l := c.t.loadEntry("k").p.Load() + + check := func(n int64, wantS KeyState, wantV int, what string) { + t.Helper() + v, _, s := loadingTryGet(l, n, 0, age) + if s != wantS || (wantS != Miss && v != wantV) { + t.Fatalf("%s: loadingTryGet=(%d,%v), want (%d,%v)", what, v, s, wantV, wantS) + } + } + + // Positive word: expiry E, window [E, E+age). + E := now() - 1 + l.expires.Store(E) + check(E-1, Hit, 1, "one ns before expiry") + check(E, Stale, 1, "the expiry instant itself") + check(E+a-1, Stale, 1, "last open ns of the window") + check(E+a, Miss, 0, "the window-close instant") + + if st := newStale(1, E, age); st.expires != E+a { + t.Fatalf("companion window end %d != read-path window end %d", st.expires, E+a) + } else if st.expired(E+a-1) || !st.expired(E+a) { + t.Fatal("companion boundary disagrees with the read-path boundary") + } + if !staleOpen(E, E+a-1, age) || staleOpen(E, E+a, age) { + t.Fatal("staleOpen boundary inconsistent with itself") + } + + // Negated-birth word: anchor B, window [B, B+age). + B := now() + l.expires.Store(-B) + check(B+a-1, Stale, 1, "born-expired: last open ns") + check(B+a, Miss, 0, "born-expired: window-close instant") + if st := newStale(1, -B, age); st.expires != B+a { + t.Fatalf("born-expired companion end %d, want %d", st.expires, B+a) + } + + // Claimed word: no window, ever; and no companion may be derived. + l.expires.Store(expiredClaimed) + check(B, Miss, 0, "claimed word") + if staleOpen(expiredClaimed, 1, age) { + t.Fatal("staleOpen derived a window from a claimed word") + } + if st, ok := maybeNewStale(c.t.loadEntry("k").p.Load(), age); ok { + t.Fatalf("maybeNewStale snapshotted a claimed loading: %+v", st) + } +} + +// TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and +// CompareAndDelete only match live values: expired entries and errored +// entries are "not there" per TryGet, so comparing against them must fail. +func TestCompareAndSwapAgreesWithTryGet(t *testing.T) { + t.Run("expired", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatal("expired entry should TryGet Miss") + } + if c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap matched an expired value") + } + if c.CompareAndDelete("k", 1) { + t.Error("CompareAndDelete matched an expired value") + } + }) + + t.Run("errored", func(t *testing.T) { + c := New[string, int]() + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + // The errored loading's v is the zero int; CAS against 0 must not + // treat that as a cached value. + if c.CompareAndSwap("k", 0, 2) { + t.Error("CompareAndSwap matched an errored entry's placeholder value") + } + if c.CompareAndDelete("k", 0) { + t.Error("CompareAndDelete matched an errored entry's placeholder value") + } }) + + t.Run("live", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + if !c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap failed on a live matching value") + } + if !c.CompareAndDelete("k", 2) { + t.Error("CompareAndDelete failed on a live matching value") + } + }) +} + +// An idle extension that would move the expiry by less than the slack is +// skipped; a larger one is written. +func TestMaxIdleAge_Slack(t *testing.T) { + const age = time.Hour + l := &loading[int]{} + base := now() + int64(age) + l.expires.Store(base) + + l.extend(base, base-int64(age)+int64(maxIdleSlack)/2, age) + if got := l.expires.Load(); got != base { + t.Fatalf("extension within the slack was written: %d != %d", got, base) + } + n := base - int64(age) + 2*int64(maxIdleSlack) + l.extend(base, n, age) + if got := l.expires.Load(); got != n+int64(age) { + t.Fatalf("extension past the slack: expiry %d, want %d", got, n+int64(age)) + } + + // A small age uses age/64 as the slack. + const small = 32 * time.Millisecond + l.expires.Store(base) + n = base - int64(small) + int64(small)/32 + l.extend(base, n, small) + if got := l.expires.Load(); got != n+int64(small) { + t.Fatalf("small-age extension past age/64: expiry %d, want %d", got, n+int64(small)) + } +} + +// Lookups must not make the key escape: a key converted from a []byte, as +// with a builtin map lookup, must not allocate. +func TestLookupKeyDoesNotEscape(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("abc", 1) + b := []byte("abc") + for name, f := range map[string]func(){ + "TryGet": func() { c.TryGet(string(b)) }, + "Expire": func() { c.Expire(string(b) + "x") }, + "Delete": func() { c.Delete(string(b) + "x") }, + "CompareAndSwap": func() { c.CompareAndSwap(string(b), 2, 3) }, + "CompareAndDelete": func() { c.CompareAndDelete(string(b), 2) }, + } { + if n := testing.AllocsPerRun(100, f); n != 0 { + t.Errorf("%s allocated %v times, want 0", name, n) + } + } +} + +// Clean does not allocate, however much it deletes. +func TestCleanDoesNotAllocate(t *testing.T) { + c := New[int, int](MaxAge(0)) + for i := range 1000 { + c.Set(i, i) + } + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + c.Clean() + runtime.ReadMemStats(&after) + if n := after.Mallocs - before.Mallocs; n != 0 { + t.Errorf("Clean deleting 1000 entries allocated %d times, want 0", n) + } + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries, want 0", n) + } +} + +// A refresh after an error that has no stale carries no stale, even with +// stale values enabled. +func TestRefreshAfterErrorWithoutStale(t *testing.T) { + c := New[int, int](MaxStaleAge(time.Hour), MaxErrorAge(0)) + if _, err, s := c.Get(1, func() (int, error) { return 0, errors.New("boom") }); err == nil || s != Miss { + t.Fatalf("first Get: (%v, %v), want (boom, Miss)", err, s) + } + release := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + c.Get(1, func() (int, error) { <-release; return 2, nil }) + }() + waitForState(t, c, 1, func(l *loading[int]) bool { return !l.finalized() }) + if st := c.t.loadEntry(1).p.Load().stale(); st != nil { + t.Fatalf("refresh after an error with no stale carries stale %v", st.v) + } + close(release) + <-done + if v, _, s := c.TryGet(1); v != 2 || s != Hit { + t.Fatalf("TryGet: (%d, %v), want (2, Hit)", v, s) + } +} + +// waitForState waits until k's loading satisfies ok. +func waitForState(t *testing.T, c *Cache[int, int], k int, ok func(*loading[int]) bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if e := c.t.loadEntry(k); e != nil { + if l := e.p.Load(); l != nil && ok(l) { + return + } + } + if time.Now().After(deadline) { + t.Fatal("loading never reached the wanted state") + } + time.Sleep(time.Millisecond) + } } diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go new file mode 100644 index 0000000..07c3c11 --- /dev/null +++ b/cache/concurrency_test.go @@ -0,0 +1,1622 @@ +package cache + +import ( + "errors" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestSwap_CancelsInFlightGet verifies that Swap overrides a currently-loading +// entry: the goroutine waiting in Get sees the Swap value, and the underlying +// miss function's return value is discarded. +// +// Covers cache.go Swap defer block for !was.finalized() (the cancel path). +func TestSwap_CancelsInFlightGet(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + missDone := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + close(missDone) + return 99, nil + }) + getV = v + }) + + <-missStart // miss function has begun + + // Swap while the miss is still running. + _, _, oldS := c.Swap("k", 7) + if oldS != Miss { + t.Fatalf("Swap old state: got %v want Miss", oldS) + } + + // Let the miss function finish. Its value MUST be discarded. + close(missRelease) + <-missDone // ensure the miss goroutine actually ran to completion + getDone.Wait() // ensure the Get goroutine returned + + if getV != 7 { + t.Fatalf("Get value = %d, want 7 (Swap should cancel miss)", getV) + } + + // Verify the cache holds Swap's value, not the miss's. + v, _, s := c.TryGet("k") + if v != 7 || !s.IsHit() { + t.Fatalf("TryGet after Swap: v=%d s=%v, want 7 Hit", v, s) + } +} + +// TestSwap_OverExpiredFinalizedWithStale exercises the defer branch that +// reads the prior entry's stale when the prior entry has expired but a valid +// stale is still in its window. +func TestSwap_OverExpiredFinalizedWithStale(t *testing.T) { + c := New[string, int]( + MaxAge(5*time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(15 * time.Millisecond) + // Entry is expired but stale is still valid. + old, _, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 { + t.Fatalf("Swap old: got %d want 1", old) + } +} + +// TestSwap_OverErroredWithStale covers the defer branch where the prior +// entry is finalized but holds an error, and a valid stale exists. +func TestSwap_OverErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + // Drive a refresh that will error; while it is in flight, the entry is + // (loading, stale=1). The Get returns the stale immediately. + c.Expire("k") + release := make(chan struct{}) + defer close(release) + if v, _, s := c.Get("k", func() (int, error) { + <-release + return 0, errors.New("boom") + }); v != 1 || s != Stale { + t.Fatalf("refreshing Get: (%d, %v), want (1, Stale)", v, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 || oldErr != nil { + t.Fatalf("Swap old: v=%d err=%v, want v=1 err=nil", old, oldErr) + } +} + +// TestDelete_DuringInFlightLoad verifies that deleting while a load is in +// flight: (a) the in-flight Get still returns the miss fn's result, and (b) +// a subsequent Get re-triggers the miss fn. +func TestDelete_DuringInFlightLoad(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }) + + <-missStart + c.Delete("k") + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5", getV) + } + + // Next Get must re-run the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Delete") + } + if v != 6 { + t.Fatalf("post-Delete Get got %d, want 6", v) + } +} + +// TestClear_DoesNotCacheInFlightLoads pins Clear's documented interplay with +// in-flight loads (the per-key Delete caveat applied wholesale): Clear swaps +// the trie root, so a load in flight at the Clear keeps running against the +// detached tree: its result still reaches the Get waiting on it, but is +// never cached, and a later Get misses and drives a fresh load. +func TestClear_DoesNotCacheInFlightLoads(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }) + + <-missStart + c.Clear() + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5 (the load's result must still reach its waiter)", getV) + } + + // The result must not have been cached: the next Get re-runs the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Clear (the in-flight result must not be cached)") + } + if v != 6 { + t.Fatalf("post-Clear Get got %d, want 6", v) + } +} + +// TestExpire_NoOpDuringInFlightLoad verifies the documented behavior: Expire +// on an in-flight load is a no-op; the load completes with its normal TTL. +func TestExpire_NoOpDuringInFlightLoad(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getDone sync.WaitGroup + getDone.Go(func() { + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 11, nil + }) + }) + + <-missStart + c.Expire("k") // must be a no-op while loading + close(missRelease) + getDone.Wait() + + v, _, s := c.TryGet("k") + if v != 11 || !s.IsHit() { + t.Fatalf("after in-flight Expire: v=%d s=%v, want 11 Hit (Expire should not have affected the load)", v, s) + } +} + +// TestCollapsing_NoCaching runs many concurrent Gets against a cache with +// MaxAge(0) (no caching). The documented use case is collapsing simultaneous +// queries for the same key. +// +// The guarantee is weaker than it looks: with del0, the loading finalizes +// expired, so the next goroutine that re-checks the entry after the leader's +// setve sees an expired entry and triggers a fresh miss. Concurrent arrivals +// still collapse in batches, but "batch" here means "whatever queued on a +// given loading's WaitGroup before it was Done'd," which is scheduler- +// dependent. +// +// We assert: (1) miss runs at least once, (2) collapsing happens (fewer miss +// calls than workers), (3) every returned value corresponds to some miss +// invocation. +func TestCollapsing_NoCaching(t *testing.T) { + c := New[string, int](MaxAge(0)) + const workers = 100 + + var missStart sync.WaitGroup + missStart.Add(1) + missRelease := make(chan struct{}) + var missCalls int32 + + var wg sync.WaitGroup + vals := make([]int, workers) + for i := range workers { + wg.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + n := atomic.AddInt32(&missCalls, 1) + if n == 1 { + missStart.Done() + <-missRelease + } + return int(n), nil + }) + vals[i] = v + }) + } + + missStart.Wait() + time.Sleep(5 * time.Millisecond) // let as many callers as possible queue + close(missRelease) + wg.Wait() + + calls := atomic.LoadInt32(&missCalls) + if calls < 1 { + t.Fatalf("miss never called") + } + if calls >= workers { + t.Fatalf("no collapsing observed: miss called %d times across %d Gets", calls, workers) + } + for i, v := range vals { + if v < 1 || int32(v) > calls { + t.Fatalf("goroutine %d got value %d, outside valid miss-call range [1,%d]", i, v, calls) + } + } + + // Nothing is cached. A fresh Get runs the miss again. + var called bool + c.Get("k", func() (int, error) { + called = true + return 99, nil + }) + if !called { + t.Fatal("after collapsed Get, cache should not have retained a value") + } +} + +// TestGet_StaleRefreshCASRace pounds on the Get slow-path CAS-retry that +// replaces a finalized-expired-with-stale loading with a fresh loading +// whose stale is a snapshot of the old value. Many concurrent Gets on the +// same key force CAS losses and thus exercise the retry. +func TestGet_StaleRefreshCASRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + for range 200 { + c := New[string, int]( + MaxAge(time.Microsecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(time.Millisecond) // entry expired, stale valid + + const workers = 16 + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + c.Get("k", func() (int, error) { + // Slow miss so concurrent Gets all pile into the + // slow-path stale branch before any finalizes. + time.Sleep(time.Millisecond) + return 2, nil + }) + }) + } + wg.Wait() + } +} + +// TestGet_ConcurrentStaleDuringInFlight exercises Get's slow-path branch +// where a second Get observes a stale-returning loading that is already +// in flight from the first Get. The second Get must return the stale +// without attempting to install a new loading. +func TestGet_ConcurrentStaleDuringInFlight(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // entry now expired, stale still valid + + missRelease := make(chan struct{}) + missStart := make(chan struct{}) + var aDone sync.WaitGroup + aDone.Go(func() { + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 2, nil + }) + }) + <-missStart + + // Second Get observes the first Get's in-flight loading and its stale. + v, _, s := c.Get("k", func() (int, error) { + t.Error("second Get's miss function should not run while first Get is in flight") + return 3, nil + }) + if s != Stale || v != 1 { + t.Fatalf("second Get: v=%d s=%v, want v=1 Stale", v, s) + } + + close(missRelease) + aDone.Wait() +} + +// TestClean_SkipsInFlightLoads verifies Clean leaves pending loads alone. +// An entry whose loading hasn't finalized yet has no expiry to evaluate; +// Clean must skip it. +func TestClean_SkipsInFlightLoads(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond)) + missRelease := make(chan struct{}) + missStarted := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { + close(missStarted) + <-missRelease + return 42, nil + }) + }() + <-missStarted + e := c.t.loadEntry("k") + l := e.p.Load() + c.Clean() + if e2 := c.t.loadEntry("k"); e2 != e || e.p.Load() != l { + t.Fatal("Clean removed an in-flight load") + } + close(missRelease) + <-getDone +} + +// TestCleanUnderConcurrency stresses Clean racing with concurrent +// Set/Get/Delete to catch races in the clean path. Correctness here is that +// the test finishes without -race complaints or panics; we also assert that +// no value ever mysteriously appears that we never Set. +func TestCleanUnderConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Millisecond), + ) + + var stop atomic.Bool + var wg sync.WaitGroup + + // Setters. + for w := range 4 { + wg.Go(func() { + for !stop.Load() { + for i := range 64 { + c.Set(i, i*1000+w) + } + } + }) + } + + // Getters. + for range 4 { + wg.Go(func() { + for !stop.Load() { + for i := range 64 { + v, _, s := c.TryGet(i) + if s.IsHit() { + // Value is i*1000+worker; must satisfy the invariant. + if v%1000 > 3 || v/1000 != i { + t.Errorf("impossible value v=%d for key=%d", v, i) + return + } + } + } + } + }) + } + + // Deleters. + for range 2 { + wg.Go(func() { + for !stop.Load() { + for i := range 64 { + c.Delete(i) + } + } + }) + } + + // Cleaner. + wg.Go(func() { + for !stop.Load() { + c.Clean() + time.Sleep(200 * time.Microsecond) + } + }) + + time.Sleep(500 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestSwap_DeleteRangeChurn stresses Swap's fast path racing Delete's +// tombstone-then-unlink, with Range walking the trie throughout: +// Swap must observe tombstones and re-create keys (never resurrect a dying +// entry), and Range must tolerate entries being unlinked mid-walk. The +// interesting windows are ~nanoseconds wide; we stress from many angles for +// long enough that they land, leaving a generous budget for slow CI runners. +func TestSwap_DeleteRangeChurn(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + const readKeys = 32 + for i := range readKeys { + c.Set(i, i) + } + + var wg sync.WaitGroup + var stop atomic.Bool + + // Deleters churn the primed keys, repeatedly creating the tombstone + + // unlink windows that Swap's fast path must handle. + for range 8 { + wg.Go(func() { + for !stop.Load() { + for i := range readKeys { + c.Delete(i) + } + } + }) + } + // Swappers on the same primed keys exercise the unlocked fast-path CAS + // that races with the deleters. + for w := range 8 { + wg.Go(func() { + for !stop.Load() { + for i := range readKeys { + c.Swap(i, i*1000+w) + } + } + }) + } + // Churners add+delete NEW keys continuously so the trie's shape keeps + // changing (inserts expand slots, deletes prune empty parents) while + // Range walks it. + for w := range 4 { + wg.Go(func() { + base := readKeys + w*10000 + for !stop.Load() { + for i := range 64 { + c.Set(base+i, i) + c.Delete(base + i) + } + } + }) + } + // Range callers walk the trie throughout the churn. + for range 4 { + wg.Go(func() { + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }) + } + + time.Sleep(2 * time.Second) + stop.Store(true) + wg.Wait() +} + +// TestTryCAS_RetryOnRace runs concurrent CompareAndSwap increments of one +// key: every increment that reports success must land exactly once. +func TestTryCAS_RetryOnRace(t *testing.T) { + c := New[string, int]() + c.Set("k", 0) + + const workers = 32 + const iters = 1000 + + var wg sync.WaitGroup + var successes int32 + for range workers { + wg.Go(func() { + for range iters { + for { + v, _, s := c.TryGet("k") + if !s.IsHit() { + break + } + if c.CompareAndSwap("k", v, v+1) { + atomic.AddInt32(&successes, 1) + break + } + runtime.Gosched() + } + } + }) + } + wg.Wait() + + v, _, _ := c.TryGet("k") + if int32(v) != successes { + t.Fatalf("value %d != successful CAS count %d", v, successes) + } + if successes != workers*iters { + t.Fatalf("successes %d != expected %d (every worker should eventually succeed its increments)", successes, workers*iters) + } +} + +// TestCompareAndDelete_RaceOnDelete stresses the CompareAndDelete path with +// concurrent deletes/sets to exercise both the retry loop inside tryCAS and +// the value-mismatch path after a concurrent mutation. +func TestCompareAndDelete_RaceOnDelete(t *testing.T) { + c := New[int, int]() + const workers = 16 + var wg sync.WaitGroup + + var setDone atomic.Int64 + for w := range workers { + wg.Go(func() { + for i := range 1000 { + c.Set(i%8, w) + setDone.Add(1) + } + }) + } + for range workers { + wg.Go(func() { + for i := range 1000 { + v, _, s := c.TryGet(i % 8) + if s.IsHit() { + c.CompareAndDelete(i%8, v) + } + } + }) + } + wg.Wait() + // No assertion on final state; the race-detector + lack of panic is the + // signal. +} + +// TestSetve_RacesWithSwap pins the ordering where a Swap displaces a +// loading that the load is in the middle of finalizing: Swap's own +// finalize loses, so Swap must wait for the load's result and report it as +// the old value. +func TestSetve_RacesWithSwap(t *testing.T) { + c := New[string, int]() + release := make(chan struct{}) + started := make(chan struct{}) + getDone := make(chan int) + go func() { + v, _, _ := c.Get("k", func() (int, error) { + close(started) + <-release + return 1, nil + }) + getDone <- v + }() + <-started + l := c.t.loadEntry("k").p.Load() + + // Finalize as setve would, but pause between claiming and publishing. + if !l.state.CompareAndSwap(stateLoading, stateFinalizing) { + t.Fatal("precondition: loading already finalized") + } + type swapRes struct { + v int + s KeyState + } + swapDone := make(chan swapRes) + go func() { + v, _, s := c.Swap("k", 2) + swapDone <- swapRes{v, s} + }() + // Once Swap has replaced l, it is in its deferred finalize of l. + for c.t.loadEntry("k").p.Load() == l { + runtime.Gosched() + } + select { + case r := <-swapDone: + t.Fatalf("Swap returned %v before the load finished finalizing", r) + case <-time.After(10 * time.Millisecond): + } + l.v = 1 + l.state.Store(stateFinalized) + l.wg.Done() + + if r := <-swapDone; r.v != 1 || r.s != Hit { + t.Fatalf("Swap old: (%d, %v), want (1, Hit)", r.v, r.s) + } + close(release) // the load's own setve now loses + if v := <-getDone; v != 1 { + t.Fatalf("Get: %d, want 1", v) + } + if v, _, s := c.TryGet("k"); v != 2 || s != Hit { + t.Fatalf("TryGet: (%d, %v), want (2, Hit)", v, s) + } +} + +// TestClean_NoopWhenStalesArePermanent covers the MaxStaleAge<0 early return +// in Clean (the "stales kept forever" opt-in). +func TestClean_NoopWhenStalesArePermanent(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(-1)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry is now expired, stale is forever + c.Clean() // must be a no-op + // The stale is still queryable. + _, _, s := c.TryGet("k") + if s != Stale { + t.Fatalf("after Clean with MaxStaleAge=-1: state=%v want Stale", s) + } +} + +// TestGet_RacesSwapOnExpired targets Get's slow-path Hit: the fast path +// sees an expired entry (Miss), but by the time the slow path re-checks, a +// concurrent Swap has published a fresh value for the same key. If Get +// reports a Hit it must be the Swap's value, never the expired one. Hitting +// the exact timing window requires multiple iterations. +func TestGet_RacesSwapOnExpired(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int](MaxAge(time.Hour)) + c.Set(1, 1) + c.Expire(1) + + var wg sync.WaitGroup + wg.Go(func() { + c.Swap(1, iter+100) + }) + wg.Go(func() { + v, _, s := c.Get(1, func() (int, error) { return -1, nil }) + if s.IsHit() && v != iter+100 { + t.Errorf("iter %d: Hit but v=%d, want %d", iter, v, iter+100) + } + }) + wg.Wait() + } +} + +// TestCompareAndSwap_RacesSet races CompareAndSwap against the first Set of +// the same key: the CAS succeeds only if it sees the Set's value. +func TestCompareAndSwap_RacesSet(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int]() + var wg sync.WaitGroup + var swapped bool + wg.Go(func() { + c.Set(iter, iter) + }) + wg.Go(func() { + swapped = c.CompareAndSwap(iter, iter, iter+1) + }) + wg.Wait() + want := iter + if swapped { + want = iter + 1 + } + if v, _, _ := c.TryGet(iter); v != want { + t.Fatalf("iter %d: swapped=%v, value %d, want %d", iter, swapped, v, want) + } + } +} + +// TestMemoryBounded churns keys in and out of the cache and checks that +// Delete unlinks entries rather than leaving deleted entries in the trie. +func TestMemoryBounded(t *testing.T) { + c := New[int, int]() + const liveKeys = 32 + const rounds = 2000 + + for r := range rounds { + for i := range liveKeys { + c.Set(r*liveKeys+i, i) + } + // Delete the ones we just wrote except for `liveKeys` canonical keys. + for i := range liveKeys { + c.Delete(r*liveKeys + i) + } + for i := range liveKeys { + c.Set(i, i) + } + } + + if count := trieEntries(c); count != liveKeys { + t.Fatalf("after %d rounds: %d live entries, want <= %d", rounds, count, liveKeys) + } +} + +// TestSwap_NotLostDuringConcurrentDelete verifies that a Swap racing a +// Delete of the same key is never silently dropped. The dangerous +// interleaving: Delete tombstones the value slot, Swap observes the +// tombstone, and Delete's physical unlink races Swap's installation of the +// new value. A nil value slot is terminal (Swap must re-create the key as +// a fresh entry, never write into the dying one) so if Swap returns Miss +// ("I stored into nothing"), it linearized after the Delete and the swapped +// value must be visible afterwards. +// +// Regression test: an earlier version resurrected tombstones with a bare +// CAS not serialized with deleteEntryIf's unlink, losing the swapped value +// in ~0.03% of rounds. +func TestSwap_NotLostDuringConcurrentDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const rounds = 100_000 + for i := range rounds { + c := New[int, int]() + c.Set(i, 0) + + var wg sync.WaitGroup + var swapState KeyState + wg.Go(func() { + c.Delete(i) + }) + wg.Go(func() { + _, _, swapState = c.Swap(i, 1) + }) + wg.Wait() + + if swapState == Miss { + if v, _, s := c.TryGet(i); s.IsMiss() || v != 1 { + t.Fatalf("round %d: Swap returned Miss (stored after the Delete) but TryGet=(%d,%v); the swapped value was silently dropped", i, v, s) + } + } + } +} + +// TestClean_DoesNotEvictFreshEntries runs Clean continuously against Gets +// on a cache with no MaxAge and no Deletes: nothing ever expires, so Clean +// must never remove anything and every completed Get must be a subsequent +// TryGet hit. +// +// Regression test: an earlier version created Get's entry with a nil value +// slot and CAS'd the loading in afterwards; in that window the fresh entry +// was indistinguishable from a Delete tombstone and Clean would unlink it, +// losing the cached value (and with it, request collapsing for that key). +func TestClean_DoesNotEvictFreshEntries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + + stop := make(chan struct{}) + var cleanWg sync.WaitGroup + cleanWg.Go(func() { + for { + select { + case <-stop: + return + default: + c.Clean() + runtime.Gosched() + } + } + }) + + const rounds = 200_000 + for i := range rounds { + c.Get(i, func() (int, error) { return i, nil }) + if v, _, s := c.TryGet(i); s.IsMiss() || v != i { + t.Fatalf("round %d: freshly loaded entry evicted by Clean: TryGet=(%d,%v)", i, v, s) + } + } + close(stop) + cleanWg.Wait() +} + +// TestGet_DoesNotRedriveFreshlyFinalizedValue targets the slow-path window +// between Get's load of prev and entGet's own load of the slot: if the +// in-flight load it observed finalizes live in that window, Get must return +// the Hit rather than demote the fresh value to a stale and drive a +// redundant load. Every round arranges an expired-with-stale entry, races +// Gets against the first load's release, and asserts the miss function ran +// exactly once. +func TestGet_DoesNotRedriveFreshlyFinalizedValue(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 2000 { + c := New[int, int](MaxAge(time.Hour), MaxStaleAge(time.Hour)) + c.Set(iter, 1) + c.Expire(iter) // expired with a valid self-stale + + // A driving Get returns the stale and runs its miss on a + // background goroutine, so completion must be synchronized on: + // missDone is closed by the (single) miss, and a redundant + // second drive panics on the double close. + release := make(chan struct{}) + missDone := make(chan struct{}) + miss := func() (int, error) { + <-release // closed below; late drivers pass straight through + close(missDone) + return 2, nil + } + + var wg sync.WaitGroup + wg.Go(func() { + c.Get(iter, miss) // drives the refresh, returns Stale(1) + }) + wg.Go(func() { + runtime.Gosched() + close(release) + }) + // Meanwhile, hammer Gets that race the finalization. None may + // drive a second load: each must piggyback the stale or return + // the fresh Hit. + for range 4 { + c.Get(iter, miss) + } + wg.Wait() + <-missDone + } +} + +// TestExpire_NotLostToIdleExtension verifies that Expire racing an +// idle-extending read sticks: once both return, the key must be expired. +// +// Regression test: idle extension previously used a plain Store on the +// expiry, which could overwrite a concurrent Expire (read loads expiry, +// Expire stores now-1, read stores now+idle). Extension now CASes against +// the expiry it observed, so an interleaved Expire wins. +func TestExpire_NotLostToIdleExtension(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int](MaxIdleAge(time.Hour)) + const rounds = 100_000 + for i := range rounds { + c.Set(i, 1) + var wg sync.WaitGroup + wg.Go(func() { + c.TryGet(i) // a Hit extends the idle expiry + }) + wg.Go(func() { + c.Expire(i) + }) + wg.Wait() + if _, _, s := c.TryGet(i); s.IsHit() { + t.Fatalf("round %d: Expire was overwritten by a concurrent idle extension", i) + } + } +} + +// TestClean_ClaimWindowDoesNotResurrectDeadStale pins Clean's claim protocol +// against racing Gets: Clean claims an entry's expiry word before +// tombstoning, and a Get that lands between the claim and the tombstone +// snapshots that loading for a stale. The claimed word must read as "stale +// window certified closed" (never as something a stale window can be +// derived from) or the Get would serve a value past MaxAge+MaxStaleAge as +// a fresh Stale for up to another MaxStaleAge. +// +// The test performs Clean's claim by hand (same gate, same CAS) so the +// window between Clean's two CASes is held open deterministically. +// +// Regression test: Clean used to claim with -1, the born-expired sentinel, +// which newStale then re-anchored at now (born-expired words now carry +// their birth and anchor there, and a claimed word derives no window at +// all). +func TestClean_ClaimWindowDoesNotResurrectDeadStale(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + // Clean's claim, by hand: same gate, same CAS, protocol frozen between + // Clean's two CASes. + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + tn := now() + if expired := expires != 0 && expires <= tn; !expired || + (l.stale() != nil && !l.stale().expired(tn)) || + (l.err() == nil && staleOpen(expires, tn, c.cfg.maxStaleAge)) { + t.Fatalf("precondition: entry not Clean-eligible: expires=%d", expires) + } + if !l.expires.CompareAndSwap(expires, expiredClaimed) { + t.Fatal("claim CAS failed") + } + + // A Get in the claim window must not serve the dead value as a stale: + // the stale snapshot must come up empty, so the Get blocks on (and + // returns) the fresh load. + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in Clean's claim window: v=%d s=%v, want 2 Miss (a Stale here resurrects a certified-dead value)", v, s) + } + if l2 := e.p.Load(); l2 == nil || l2.stale() != nil { + t.Fatalf("the replacement loading must carry no stale snapshot of the claimed value, got %+v", l2) + } + + // Finish Clean's protocol: the tombstone CAS must fail (the Get + // replaced the slot) and the fresh value must survive. + if e.p.CompareAndSwap(l, nil) { + t.Fatal("Clean's tombstone CAS should have failed against the Get's replacement") + } + if v, _, s := c.TryGet("k"); v != 2 || !s.IsHit() { + t.Fatalf("after claim window: TryGet=(%d,%v), want (2,Hit)", v, s) + } +} + +// TestClean_ClaimWindowVsGetRace hammers the window between Clean's two +// CASes (the expiry-word claim and the tombstone) with racing Gets. Every +// round populates keys whose value AND stale are certifiably dead (TryGet +// returned Miss), then races a real Clean against a Get scan: any Get that +// returns Stale resurrected a certified-dead value. +// +// Probabilistic tripwire for the protocol pinned deterministically by +// TestClean_ClaimWindowDoesNotResurrectDeadStale; before Clean claimed with +// a dedicated sentinel, this fired within a few seconds under -race. +func TestClean_ClaimWindowVsGetRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + const ( + keys = 1024 + // Small enough that entries die within a round's sleep; large + // enough that a wrongly re-anchored stale stays observable for the + // Get that snapshotted it. + staleAge = 5 * time.Millisecond + ) + c := New[int, int](MaxAge(time.Nanosecond), MaxStaleAge(staleAge)) + miss := func() (int, error) { return 2, nil } + for start := time.Now(); time.Since(start) < 3*time.Second; { + for k := range keys { + c.Set(k, 1) + } + time.Sleep(staleAge + 2*time.Millisecond) + for k := range keys { + for { + if _, _, s := c.TryGet(k); s.IsMiss() { + break // certified: expired and stale dead + } + } + } + + var wg sync.WaitGroup + wg.Go(func() { + c.Clean() + }) + for k := range keys { + if v, _, s := c.Get(k, miss); s == Stale { + t.Fatalf("key %d: Get racing Clean returned a certified-dead value as a stale: v=%d", k, v) + } + } + wg.Wait() + } +} + +// TestExpire_DoesNotOverwriteCleanClaim pins Expire against Clean's claim +// window: once Clean has claimed an entry's expiry word (certifying the +// value expired and its stale window closed), a racing Expire must leave +// the claim in place: never replace it with a fresh timestamp that a Get +// in the window could re-derive a stale window from. Same hand-driven +// protocol as TestClean_ClaimWindowDoesNotResurrectDeadStale. +// +// Regression test: Expire used to blindly store now()-1 over any finalized +// word, including expiredClaimed; a Get between Clean's two CASes then +// served the certified-dead value as a fresh Stale. +func TestExpire_DoesNotOverwriteCleanClaim(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + if !l.expires.CompareAndSwap(expires, expiredClaimed) { // Clean's claim + t.Fatal("claim CAS failed") + } + + c.Expire("k") // lands between Clean's two CASes; must be a no-op + + if got := l.expires.Load(); got != expiredClaimed { + t.Fatalf("Expire overwrote Clean's claim: expires=%d, want expiredClaimed", got) + } + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in the claim window: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestSwap_OverFinalizedErroredWithStale is TestSwap_OverErroredWithStale +// with the race pinned the other way: the errored load has finalized by +// the time Swap displaces it. The classification must agree with TryGet: +// the valid stale, not the error, is the prior value. +// +// Regression test: the defer's finalized classification used to assign the +// stale and then fall through to overwrite it with (v, err, Hit); the +// sibling test only exercises the unfinalized branch (Swap usually beats +// the async setve), so the finalized path went unexercised. +func TestSwap_OverFinalizedErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + // Wait for the async setve to finalize the errored loading. + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + + if v, err, s := c.TryGet("k"); s != Stale || v != 1 || err != nil { + t.Fatalf("TryGet: (%d, %v, %v), want (1, nil, Stale)", v, err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestSwap_OverFinalizedErroredNoStale covers the remaining finalized +// classification arm: with no stale to return, an unexpired cached error +// is itself the prior result, and Swap reports it as a Hit carrying the +// error: agreeing with TryGet. +func TestSwap_OverFinalizedErroredNoStale(t *testing.T) { + c := New[string, int]() // errors cache forever by default + // The driving Get waits for its load, so the error is finalized here. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet: err=%v s=%v, want the cached error as a Hit", err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if old != 0 || oldErr == nil || !oldS.IsHit() { + t.Fatalf("Swap: (%d, %v, %v), want (0, boom, Hit) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestDelete_ConcurrentlyFinalizedLoadReportsHit pins Delete's documented +// classification race: Delete samples the clock before tombstoning, but +// reads the loading's finalized state afterwards, so a load finalizing +// between the two is reported as the removed value (a Hit) even though no +// read ever observed it cached. This is Delete's exact code sequence with +// the window held open; the returned triple is best effort there (see +// Delete's doc), and the cache state itself is unaffected. +func TestDelete_ConcurrentlyFinalizedLoadReportsHit(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missRelease := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { <-missRelease; return 7, nil }) + }() + var e *ent[string, int] + for e = c.t.loadEntry("k"); e == nil; e = c.t.loadEntry("k") { + runtime.Gosched() + } + + // Delete's sequence (see Cache.Delete), paused mid-window: + n := now() + was := entDel(e) + c.t.deleteEntryIf("k", entDead[string, int]) + close(missRelease) // the miss finalizes the already-detached loading + <-getDone + + if v, _, s := loadingTryGet(was, n, 0, c.cfg.maxStaleAge); !s.IsHit() || v != 7 { + t.Fatalf("classifying the concurrently finalized loading: (%d, %v), want (7, Hit) per Delete's documented race", v, s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("the key must be gone regardless: %v, want Miss", s) + } +} + +// Gets that collapsed onto a load that errors all return its error, even +// with errors uncached, and the next Get runs a new load. +func TestGet_CollapsedErrorWaitersShareError(t *testing.T) { + c := New[string, int](MaxErrorAge(0)) + var calls atomic.Int32 + release := make(chan struct{}) + started := make(chan struct{}) + boom := errors.New("boom") + miss := func() (int, error) { + if calls.Add(1) == 1 { + close(started) + <-release + } + return 0, boom + } + const waiters = 8 + errs := make(chan error, waiters) + var wg sync.WaitGroup + wg.Go(func() { _, err, _ := c.Get("k", miss); errs <- err }) + <-started + for range waiters - 1 { + wg.Go(func() { _, err, _ := c.Get("k", miss); errs <- err }) + } + waitForLoadWaiters(t, waiters-1) + close(release) + wg.Wait() + close(errs) + for err := range errs { + if err != boom { + t.Errorf("collapsed Get returned %v, want boom", err) + } + } + if n := calls.Load(); n != 1 { + t.Fatalf("miss ran %d times for %d collapsed Gets, want 1", n, waiters) + } + c.Get("k", miss) + if n := calls.Load(); n != 2 { + t.Fatalf("the Get after an uncached error did not run a new load: %d loads", n) + } +} + +// waitForLoadWaiters waits until n goroutines are blocked waiting on a +// load. +func waitForLoadWaiters(t *testing.T, n int) { + t.Helper() + buf := make([]byte, 1<<20) + deadline := time.Now().Add(10 * time.Second) + for { + stacks := string(buf[:runtime.Stack(buf, true)]) + if strings.Count(stacks, "cache.entGet[") >= n { + return + } + if time.Now().After(deadline) { + t.Fatalf("fewer than %d goroutines ever waited on a load", n) + } + time.Sleep(time.Millisecond) + } +} + +// retryTTL calls f with ttl, doubling ttl and retrying while f returns +// false. f returns false if we were descheduled past ttl before reaching +// the state under test. +func retryTTL(t *testing.T, ttl time.Duration, f func(ttl time.Duration) bool) { + t.Helper() + for range 8 { + if f(ttl) { + return + } + ttl *= 2 + } + t.Fatal("never reached the state under test within the TTL") +} + +// setExpiresPairingHook installs expiresPairingHook such that exactly the +// first read to pass through it blocks until release is closed; all later +// reads pass straight through. The hook is cleared when the test ends. +func setExpiresPairingHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + return setExpiresPairingHookNth(t, 1) +} + +// setExpiresPairingHookNth is setExpiresPairingHook parking the nth +// (1-based) paired read instead of the first, for callers whose operation +// takes multiple paired reads before the one under test. +func setExpiresPairingHookNth(t *testing.T, nth int32) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var calls atomic.Int32 + expiresPairingHook = func() { + if calls.Add(1) == nth { + close(entered) + <-release + } + } + t.Cleanup(func() { expiresPairingHook = nil }) + return entered, release +} + +// setCASPublishHook installs casPublishHook such that exactly the first +// tryCAS to pass through it blocks until release is closed; all later calls +// pass straight through. The hook is cleared when the test ends. +func setCASPublishHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var fired atomic.Bool + casPublishHook = func() { + if fired.CompareAndSwap(false, true) { + close(entered) + <-release + } + } + t.Cleanup(func() { casPublishHook = nil }) + return entered, release +} + +// TestTryCAS_RevalidatesLivenessBeforePublish pins tryCAS's pre-publish +// re-validation: a CAS caller that matched a live value and was then +// descheduled (the replacement's allocation is the widest such window in +// the call) while the value's TTL lapsed must not publish: the value it +// matched is one TryGet already certifies as Miss, and no sequential order +// explains TryGet=Miss followed by CAS=true. The residual instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// Regression test: tryCAS used to sample liveness only before the +// allocation, so the entire alloc + deschedule window sat between the +// liveness sample and the publish. +func TestTryCAS_RevalidatesLivenessBeforePublish(t *testing.T) { + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + retryTTL(t, 20*time.Millisecond, func(ttl time.Duration) bool { + c := New[string, int](MaxAge(ttl)) + c.Set("k", 1) + + entered, release := setCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + select { + case <-entered: + case <-res: + return false // the value expired before the op matched it + } + + // Let the matched value's TTL lapse while the caller is + // parked between its liveness sample and its publish. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + break + } + time.Sleep(time.Millisecond) + } + close(release) + + if <-res { + t.Fatal("published against a value that expired mid-call (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + return true + }) + }) + } +} + +// TestTryCAS_RevalidationCatchesExpire extends the pre-publish re-validation +// pin to manual expiry: a CAS caller that matched a live value and was then +// parked at the replacement's allocation while a concurrent Expire killed +// the value must not publish: Expire's word write is visible to the +// re-validation's fresh liveness check, the same way a lapsed TTL is. Only +// the instructions between the re-validation and the pointer CAS remain +// best effort for Expire, exactly as for a lapsing TTL (see +// CompareAndSwap's doc). +func TestTryCAS_RevalidationCatchesExpire(t *testing.T) { + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := setCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + <-entered + + c.Expire("k") // kills the matched value while the caller is parked + close(release) + + if <-res { + t.Fatal("published against a value a concurrent Expire had already killed (TryGet certifies Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } +} + +// TestTryCAS_RevalidationPairsWordAndClock pins HOW tryCAS's liveness +// checks read expiry: as a coherent (expiry word, clock) pair, the clock +// sampled after the word it judges (expiresNow, like every read path). A +// CAS caller descheduled inside the re-validation itself (between +// sampling the clock and loading the word, in the unpaired ordering) +// would otherwise judge the fresh word against its stale clock and +// publish against a value whose TTL had lapsed, or that a completed +// Expire had killed, while it was parked: state TryGet already certifies +// as Miss, so no sequential order explains TryGet=Miss followed by +// CAS=true followed by TryGet=Hit(new). With the paired read, any kill +// completed before the pair is taken is seen; only the instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// The pairing hook parks the caller inside the re-validation's paired +// read (the call's second paired read; the first is tryCAS's +// pre-allocation filter), and the kill lands while it is parked. +// +// Regression test: casMatches used to evaluate now() and then load the +// word (!l.expired(now())), so a deschedule between the two let both kill +// modes through the re-validation. +func TestTryCAS_RevalidationPairsWordAndClock(t *testing.T) { + const ttl = 200 * time.Millisecond + ops := []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } + kills := []struct { + name string + opts func(time.Duration) []Opt + kill func(c *Cache[string, int]) + }{ + {"ttl_lapse", func(ttl time.Duration) []Opt { return []Opt{MaxAge(ttl)} }, func(c *Cache[string, int]) { + // Let the matched value's TTL lapse while the caller is + // parked; the certifying TryGets' own paired reads pass + // straight through the one-shot hook. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + return + } + time.Sleep(time.Millisecond) + } + }}, + {"expire", func(time.Duration) []Opt { return []Opt{MaxAge(time.Hour)} }, func(c *Cache[string, int]) { + c.Expire("k") + }}, + } + for _, k := range kills { + t.Run(k.name, func(t *testing.T) { + for _, m := range ops { + t.Run(m.name, func(t *testing.T) { + retryTTL(t, ttl, func(ttl time.Duration) bool { + c := New[string, int](k.opts(ttl)...) + c.Set("k", 1) + + entered, release := setExpiresPairingHookNth(t, 2) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + select { + case <-entered: + case <-res: + return false // the value expired before the op matched it + } + + k.kill(c) + close(release) + + if <-res { + t.Fatal("published against a value that was killed mid-validation (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + return true + }) + }) + } + }) + } +} + +// TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry +// word, clock) pair a read classifies with: a reader descheduled between +// loading the word and sampling the clock, with an idle extension landing +// in that window, would otherwise classify the dead pre-extension word at +// a fresh clock and report Stale (with MaxStaleAge) or Miss (without) for +// an entry that was live at every instant of its call: and a later TryGet +// returns Hit with no intervening write, a history no sequential order +// explains. expiresNow re-confirms the word after sampling the clock, so +// the read classifies the extended (live) word instead. +// +// The hook holds the deschedule window open deterministically, like the +// hand-driven claim-window tests above. +// +// Regression test: loadingTryGet and entGet used to classify whatever word +// they loaded first at whatever clock they sampled next. +func TestTryGet_PairedReadVsIdleExtension(t *testing.T) { + for _, tt := range []struct { + name string + opts []Opt + }{ + {"no_stale_age", []Opt{MaxIdleAge(time.Hour)}}, + {"with_stale_age", []Opt{MaxIdleAge(time.Hour), MaxStaleAge(time.Hour)}}, + } { + t.Run(tt.name, func(t *testing.T) { + retryTTL(t, 100*time.Millisecond, func(ttl time.Duration) bool { + c := New[string, int](append(tt.opts, MaxAge(ttl))...) + c.Set("k", 1) + + entered, release := setExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") // loads the initial expiry, parks in the hook + r1c <- res{v, s} + }() + <-entered + + // A second TryGet (the hook is one-shot) hits and extends the + // entry to ~now+1h; the entry is now live well past the + // original expiry. + if _, _, s := c.TryGet("k"); s != Hit { + close(release) // the entry expired before we extended it + <-r1c + return false + } + // Let real time pass the original expiry while the first + // reader is still parked between its two loads. + time.Sleep(ttl + 50*time.Millisecond) + close(release) + r1 := <-r1c + + if v, _, s := c.TryGet("k"); s != Hit || v != 1 { + t.Fatalf("entry must still be live after the extension: (%d, %v)", v, s) + } + if r1.s != Hit || r1.v != 1 { + t.Fatalf("paired read returned (%d, %v) for an entry that was live throughout its call, want (1, Hit)", r1.v, r1.s) + } + return true + }) + }) + } +} + +// TestTryGet_PairedReadVsExpire pins the safe direction of the same window: +// otherwise the word only ever moves backward (Expire, Clean's claim), and +// a parked reader lands on one side of the race or the other: Hit per the +// pre-Expire live word (its call overlaps the Expire, so ordering the read +// first explains the history) or Miss per the expired word. Whichever side +// it lands on, no stale is conjured for an entry with no stale configured. +func TestTryGet_PairedReadVsExpire(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := setExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") + r1c <- res{v, s} + }() + <-entered + + c.Expire("k") // moves the word backward while the reader is parked + close(release) + r1 := <-r1c + + if r1.s == Stale { + t.Fatalf("no stale is configured; got (%d, %v)", r1.v, r1.s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("after Expire: %v, want Miss", s) + } +} + +// TestSwap_OverBornExpired pins the born-expired classification cell of +// Swap's displaced-value report against TryGet: a displaced born-expired +// value (negated-birth expiry word) is the prior value (Stale) while its +// birth-anchored window is open, and Miss after: exactly as TryGet +// reports the same state. +func TestSwap_OverBornExpired(t *testing.T) { + t.Run("window_open", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet: (%d, %v), want (1, Stale)", v, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } + }) + t.Run("window_closed", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(30*time.Millisecond)) + c.Set("k", 1) + time.Sleep(60 * time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet: %v, want Miss", s) + } + if _, _, oldS := c.Swap("k", 2); oldS != Miss { + t.Fatalf("Swap: %v, want Miss to agree with TryGet", oldS) + } + }) +} + +// If an inline miss panics, the panic reaches the caller, and Gets waiting +// on that load run their own load rather than seeing an error. +func TestGet_MissPanic(t *testing.T) { + c := New[string, int]() + started := make(chan struct{}) + release := make(chan struct{}) + leaderDone := make(chan any) + go func() { + defer func() { leaderDone <- recover() }() + c.Get("k", func() (int, error) { + close(started) + <-release + panic("boom") + }) + }() + <-started + + type res struct { + v int + err error + s KeyState + } + waiterDone := make(chan res) + go func() { + v, err, s := c.Get("k", func() (int, error) { return 2, nil }) + waiterDone <- res{v, err, s} + }() + time.Sleep(10 * time.Millisecond) // let the waiter queue on the leader + close(release) + + if r := <-leaderDone; r != "boom" { + t.Fatalf("leader recovered %v, want boom", r) + } + if r := <-waiterDone; r.v != 2 || r.err != nil || r.s != Miss { + t.Fatalf("waiter: (%d, %v, %v), want (2, nil, Miss)", r.v, r.err, r.s) + } + if v, err, s := c.TryGet("k"); v != 2 || err != nil || s != Hit { + t.Fatalf("TryGet: (%d, %v, %v), want (2, nil, Hit)", v, err, s) + } +} + +// A panicked load is never returned or cached, even with errors cached +// forever. +func TestGet_MissPanicNotCached(t *testing.T) { + c := New[string, int]() + func() { + defer func() { recover() }() + c.Get("k", func() (int, error) { panic("boom") }) + }() + if _, err, s := c.TryGet("k"); s != Miss || err != nil { + t.Fatalf("TryGet after panic: (%v, %v), want (nil, Miss)", err, s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries, want 0", n) + } + if v, err, s := c.Get("k", func() (int, error) { return 1, nil }); v != 1 || err != nil || s != Miss { + t.Fatalf("Get after panic: (%d, %v, %v), want (1, nil, Miss)", v, err, s) + } +} + +// A miss that exits its goroutine does not leave the load unfinished. +func TestGet_MissGoexit(t *testing.T) { + c := New[string, int]() + done := make(chan struct{}) + go func() { + defer close(done) + c.Get("k", func() (int, error) { runtime.Goexit(); return 0, nil }) + }() + <-done + if v, _, s := c.Get("k", func() (int, error) { return 1, nil }); v != 1 || s != Miss { + t.Fatalf("Get after Goexit: (%d, %v), want (1, Miss)", v, s) + } +} diff --git a/cache/linearizability_test.go b/cache/linearizability_test.go new file mode 100644 index 0000000..1e7d6a8 --- /dev/null +++ b/cache/linearizability_test.go @@ -0,0 +1,433 @@ +package cache + +import ( + "fmt" + "math/rand/v2" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// This file implements a small linearizability checker: each round runs a +// handful of concurrent operations against one or more keys, records every +// operation's invocation/response order and return values, and then +// verifies that SOME linearization exists for each key: a total order of +// the operations consistent with real time (op A precedes op B if A +// responded before B was invoked) under which the sequential model below +// produces exactly the observed returns. If no such order exists, an +// atomicity bug has been observed and the full history is dumped. Keys are +// independent, so checking each key's history alone is enough. +// +// The clock is frozen for each round (see testNow), so expiry is part of +// the sequential model. +// +// Get is modeled with its singleflight rules rather than as a plain map +// operation: +// +// - A Get that runs its own load takes effect when the load finishes: the +// key must not be visible then, and afterward holds the load's value. +// Until then, the in-flight load looks absent to every other operation. +// - If a Set or Swap replaces the in-flight load, the Get returns that +// value, so the Get reads it at some point after the replacement. +// - If a Delete or Clear removes the in-flight load, the Get still +// returns its load's value, the key stays absent, and Gets that waited +// on the load may also return that value (the model's ghost value). + +type linKind uint8 + +const ( + linSet linKind = iota + linSwap + linDelete + linTryGet + linCAS + linCAD + linExpire + linClean + linGet + linClear +) + +func (k linKind) String() string { + return [...]string{"Set", "Swap", "Delete", "TryGet", "CAS", "CAD", "Expire", "Clean", "Get", "Clear"}[k] +} + +type linOp struct { + kind linKind + arg int // value for Set/Swap, old for CAS/CAD, the load's value for Get + arg2 int // new for CAS + + retV int + retS KeyState + retB bool + + // delOK, for a Get, is whether a Delete that returned Miss or a Clear + // overlapped it, so its in-flight load may have been removed. + delOK bool + + start, end int64 // global ticket counter values around the call +} + +func (o *linOp) String() string { + span := fmt.Sprintf("[%d,%d]", o.start, o.end) + switch o.kind { + case linSet: + return fmt.Sprintf("Set(%d) %s", o.arg, span) + case linSwap: + return fmt.Sprintf("Swap(%d)=(%d,%v) %s", o.arg, o.retV, o.retS, span) + case linDelete: + return fmt.Sprintf("Delete=(%d,%v) %s", o.retV, o.retS, span) + case linTryGet: + return fmt.Sprintf("TryGet=(%d,%v) %s", o.retV, o.retS, span) + case linCAS: + return fmt.Sprintf("CAS(%d,%d)=%v %s", o.arg, o.arg2, o.retB, span) + case linCAD: + return fmt.Sprintf("CAD(%d)=%v %s", o.arg, o.retB, span) + case linGet: + return fmt.Sprintf("Get(load %d)=(%d,%v) delOK=%v %s", o.arg, o.retV, o.retS, o.delOK, span) + default: + return fmt.Sprintf("%v %s", o.kind, span) + } +} + +// linModel is the cache configuration and the frozen clock for a round. +type linModel struct { + ttl, staleAge int64 // MaxAge and MaxStaleAge; 0 means unset + now int64 +} + +type linState struct { + present bool + val int + exp int64 // 0 never expires + ghost int // a removed in-flight Get's value that its waiters may return +} + +func (m linModel) newExp() int64 { + if m.ttl > 0 { + return m.now + m.ttl + } + return 0 +} + +// class returns what TryGet reports for st. +func (m linModel) class(st linState) KeyState { + switch { + case !st.present: + return Miss + case st.exp == 0 || m.now < st.exp: + return Hit + case m.staleAge < 0 || m.staleAge > 0 && m.now < st.exp+m.staleAge: + return Stale + } + return Miss +} + +// linApply returns every state op can leave st in, given the op's recorded +// returns; none if the returns are impossible at this point in the order. +func (m linModel) linApply(st linState, o *linOp) []linState { + readOK := func(s KeyState, v int) bool { + c := m.class(st) + return c == s && (c == Miss || v == st.val) + } + one := func(ok bool, next linState) []linState { + if !ok { + return nil + } + next.ghost = st.ghost + return []linState{next} + } + live := m.class(st) == Hit + switch o.kind { + case linSet: + return one(true, linState{present: true, val: o.arg, exp: m.newExp()}) + case linSwap: + return one(readOK(o.retS, o.retV), linState{present: true, val: o.arg, exp: m.newExp()}) + case linDelete: + return one(readOK(o.retS, o.retV), linState{}) + case linTryGet: + return one(readOK(o.retS, o.retV), st) + case linCAS: + if live && st.val == o.arg { + return one(o.retB, linState{present: true, val: o.arg2, exp: m.newExp()}) + } + return one(!o.retB, st) + case linCAD: + if live && st.val == o.arg { + return one(o.retB, linState{}) + } + return one(!o.retB, st) + case linExpire: + if live { + next := st + next.exp = m.now - 1 + return one(true, next) + } + return one(true, st) + case linClean: + return one(true, st) // removes only what no read can see + case linClear: + return one(true, linState{}) + case linGet: + switch { + case o.retS == Hit: + return one(live && st.val == o.retV || st.ghost != 0 && st.ghost == o.retV, st) + case o.retS != Miss: + return nil + case o.retV != o.arg: + // A Set or Swap replaced our load; we read its value. + return one(live && st.val == o.retV, st) + case live: + return nil // we only load a key that is not visible + } + out := []linState{{present: true, val: o.arg, exp: m.newExp(), ghost: st.ghost}} + if o.delOK { + out = append(out, linState{ghost: o.arg}) + } + return out + } + panic("unreachable") +} + +// linearizable reports whether some real-time-consistent total order of ops +// explains every op's returns starting from initial. +func (m linModel) linearizable(initial linState, ops []*linOp) bool { + used := make([]bool, len(ops)) + var rec func(st linState, depth int) bool + rec = func(st linState, depth int) bool { + if depth == len(ops) { + return true + } + next: + for i, o := range ops { + if used[i] { + continue + } + // o may go next only if no other unscheduled op wholly + // preceded it in real time. + for j, p := range ops { + if !used[j] && j != i && p.end < o.start { + continue next + } + } + for _, st2 := range m.linApply(st, o) { + used[i] = true + if rec(st2, depth+1) { + return true + } + used[i] = false + } + } + return false + } + return rec(initial, 0) +} + +// freezeClock sets now() to a fixed value for the rest of the test. +func freezeClock(t *testing.T) *atomic.Int64 { + testNow.Store(int64(time.Hour)) + t.Cleanup(func() { testNow.Store(0) }) + return &testNow +} + +type linConfig struct { + opts []Opt + model linModel + kinds []linKind + keys int // keys per round + hashFn func(uintptr) uintptr // forces collisions if non-nil + prime func(c *Cache[int, int], k int, st *linState, m *linModel, clock *atomic.Int64) +} + +func runLinearizability(t *testing.T, cfg linConfig, rounds int) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const ( + opsPerKey = 4 + valSpace = 3 // small domain so CAS/CAD old values plausibly match + ) + clock := freezeClock(t) + var c *Cache[int, int] + newCache := func() { + c = New[int, int](cfg.opts...) + if cfg.hashFn != nil { + c.t.hashFn = cfg.hashFn + } + } + newCache() + + var ticket atomic.Int64 + for round := range rounds { + if cfg.hashFn != nil && round%64 == 0 { + newCache() // keep collision chains short + } + m := cfg.model + type keyRound struct { + k int + initial linState + ops []*linOp + } + krs := make([]*keyRound, cfg.keys) + for i := range krs { + kr := &keyRound{k: round*cfg.keys + i} + if rand.IntN(2) == 1 { + kr.initial = linState{present: true, val: 1 + rand.IntN(valSpace)} + m.now = clock.Load() + kr.initial.exp = m.newExp() + c.Set(kr.k, kr.initial.val) // sequentially primed before the round + if cfg.prime != nil { + cfg.prime(c, kr.k, &kr.initial, &m, clock) + } + } + for j := range opsPerKey { + kr.ops = append(kr.ops, &linOp{ + kind: cfg.kinds[rand.IntN(len(cfg.kinds))], + arg: 1 + rand.IntN(valSpace), + arg2: 1 + rand.IntN(valSpace), + }) + if o := kr.ops[j]; o.kind == linGet { + o.arg = 100 + j // each Get's load returns a unique value + } + } + krs[i] = kr + } + m.now = clock.Load() + + var wg sync.WaitGroup + for _, kr := range krs { + for _, o := range kr.ops { + k := kr.k + wg.Go(func() { + o.start = ticket.Add(1) + switch o.kind { + case linSet: + c.Set(k, o.arg) + case linSwap: + o.retV, _, o.retS = c.Swap(k, o.arg) + case linDelete: + o.retV, _, o.retS = c.Delete(k) + case linTryGet: + o.retV, _, o.retS = c.TryGet(k) + case linCAS: + o.retB = c.CompareAndSwap(k, o.arg, o.arg2) + case linCAD: + o.retB = c.CompareAndDelete(k, o.arg) + case linExpire: + c.Expire(k) + case linClean: + c.Clean() + case linClear: + c.Clear() + case linGet: + o.retV, _, o.retS = c.Get(k, func() (int, error) { + for range rand.IntN(3) { + runtime.Gosched() + } + return o.arg, nil + }) + } + o.end = ticket.Add(1) + }) + } + } + wg.Wait() + + // A Clear on any key clears every key in this round. + var clears []*linOp + for _, kr := range krs { + for _, o := range kr.ops { + if o.kind == linClear { + clears = append(clears, o) + } + } + } + for _, kr := range krs { + ops := kr.ops + for _, cl := range clears { + if !contains(ops, cl) { + ops = append(ops, cl) + } + } + for _, o := range ops { + if o.kind != linGet { + continue + } + for _, p := range ops { + overlap := p.start < o.end && o.start < p.end + if overlap && (p.kind == linClear || p.kind == linDelete && p.retS == Miss) { + o.delOK = true + } + } + } + if !m.linearizable(kr.initial, ops) { + var dump strings.Builder + for _, o := range ops { + dump.WriteString("\n\t") + dump.WriteString(o.String()) + } + t.Fatalf("round %d key %d: no linearization explains this history (initial %+v, now %d):%s", + round, kr.k, kr.initial, m.now, dump.String()) + } + } + } +} + +func contains(ops []*linOp, o *linOp) bool { + for _, p := range ops { + if p == o { + return true + } + } + return false +} + +var linMapKinds = []linKind{linSet, linSwap, linDelete, linTryGet, linCAS, linCAD, linExpire, linClean, linGet, linClear} + +// TestLinearizability checks every map operation, Get, Expire, Clean, and +// Clear against the model with no expiry configured. +func TestLinearizability(t *testing.T) { + runLinearizability(t, linConfig{kinds: linMapKinds, keys: 1}, 30_000) +} + +// TestLinearizabilityCollisions is TestLinearizability with several keys +// per round whose hashes collide, so overflow chains and trie expansion and +// pruning race with each other. +func TestLinearizabilityCollisions(t *testing.T) { + runLinearizability(t, linConfig{ + kinds: linMapKinds, + keys: 3, + hashFn: func(h uintptr) uintptr { return h & 3 }, // 4 full hashes, 15 shared nibbles + }, 20_000) +} + +// TestLinearizabilityExpiry checks the map operations, Expire, and Clean +// with MaxAge and MaxStaleAge. Before each round, a primed key is aged into +// its live, stale, or dead phase by moving the frozen clock. +func TestLinearizabilityExpiry(t *testing.T) { + const ttl, staleAge = 100, 50 + for _, sa := range []int64{0, staleAge, -1} { + t.Run(fmt.Sprint("stale_age_", sa), func(t *testing.T) { + runLinearizability(t, linConfig{ + opts: []Opt{MaxAge(ttl), MaxStaleAge(time.Duration(sa))}, + model: linModel{ttl: ttl, staleAge: sa}, + kinds: []linKind{linSet, linSwap, linDelete, linTryGet, linCAS, linCAD, linExpire, linClean}, + keys: 1, + prime: func(c *Cache[int, int], k int, st *linState, m *linModel, clock *atomic.Int64) { + if rand.IntN(4) == 0 { + m.now = clock.Load() + c.Expire(k) + st.exp = m.now - 1 + } + // Age into live, the expiry instant, stale, the window + // close, or dead. + ages := []int64{0, ttl - 1, ttl, ttl + staleAge/2, ttl + staleAge, ttl + 2*staleAge} + clock.Add(ages[rand.IntN(len(ages))]) + }, + }, 20_000) + }) + } +} diff --git a/cache/map_bench_test.go b/cache/map_bench_test.go index aeec419..4c8625f 100644 --- a/cache/map_bench_test.go +++ b/cache/map_bench_test.go @@ -513,3 +513,15 @@ func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { }, }) } + +func BenchmarkClear(b *testing.B) { + benchMap(b, bench{ + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface[int]) { + for ; pb.Next(); i++ { + k, v := i%256, i%256 + m.Clear() + m.Store(k, v) + } + }, + }) +} diff --git a/cache/map_test.go b/cache/map_test.go index 7fb15a8..77114a8 100644 --- a/cache/map_test.go +++ b/cache/map_test.go @@ -276,3 +276,78 @@ func TestCompareAndSwap_NonExistingKey(t *testing.T) { t.Fatalf("CompareAndSwap on an non-existing key succeeded") } } + +// TestConcurrentClear is adapted from sync/map_test.go. Spawns 10 writers, +// 10 readers, and 10 Clear goroutines; correctness is that nothing panics +// or trips the race detector. +func TestConcurrentClear(t *testing.T) { + var m CacheMap[int] + + var wg sync.WaitGroup + wg.Add(30) + + for i := range 10 { + go func(k, v int) { + defer wg.Done() + m.Store(k, v) + }(i, i*10) + } + for i := range 10 { + go func(k int) { + defer wg.Done() + if _, ok := m.Load(k); ok { + _ = ok + } + }(i) + } + for range 10 { + go func() { + defer wg.Done() + m.Clear() + }() + } + + wg.Wait() + + // After all Clears and Stores have run, the map may be empty or contain a + // subset of the Stores depending on interleaving. Verify no phantom keys + // (keys we never Stored) materialized. + m.Range(func(k int, _ any) bool { + if k < 0 || k >= 10 { + t.Errorf("Range after concurrent Clear/Store saw key %d, never Stored", k) + } + return true + }) +} + +// TestMapClearOneAllocation is adapted from sync/map_test.go. Cache.Clear +// replaces the trie root with one fresh indirect node, which should be the +// only allocation. +func TestMapClearOneAllocation(t *testing.T) { + var m CacheMap[int] + // Clear of a never-used cache does nothing, so prime it. + m.Store(0, 0) + allocs := testing.AllocsPerRun(10, func() { + m.Clear() + }) + if allocs > 1 { + t.Errorf("AllocsPerRun of Clear = %v; want 1", allocs) + } +} + +// TestMapRangeNoAllocations is adapted from sync/map_test.go. Range must not +// allocate. +func TestMapRangeNoAllocations(t *testing.T) { + var m CacheMap[int] + for i := range 100 { + m.Store(i, i) + } + allocs := testing.AllocsPerRun(10, func() { + m.Range(func(k int, _ any) bool { + return true + }) + }) + if allocs > 0 { + t.Errorf("AllocsPerRun of Range = %v; want 0", allocs) + } +} diff --git a/cache/trie.go b/cache/trie.go new file mode 100644 index 0000000..dda6370 --- /dev/null +++ b/cache/trie.go @@ -0,0 +1,363 @@ +package cache + +import ( + "hash/maphash" + "sync" + "sync/atomic" + "unsafe" +) + +// trie is a concurrent hash-trie map, modeled on Go's internal/sync +// HashTrieMap. Loads are lock-free. Inserts and deletes lock only the +// indirect node that holds the slot being changed. Deletes prune empty +// indirect nodes bottom-up, locking hand over hand from child to parent. +// +// Unlike internal/sync, we only have userspace tools: hash/maphash gives us +// the runtime's hasher for any comparable type. +type trie[K comparable, V any] struct { + root atomic.Pointer[trieIndirect[K, V]] // nil until first insert + initMu sync.Mutex + seed maphash.Seed + + // hashFn, if non-nil, remaps every hash so that tests can force + // collisions. It takes the hash rather than the key: calling a func + // value with the key would make every key escape to the heap. + hashFn func(uintptr) uintptr +} + +const ( + trieBranchingLog2 = 4 + trieBranching = 1 << trieBranchingLog2 + trieBranchingMask = trieBranching - 1 +) + +// ptrBits is the number of hash bits we use: the trie is at most +// ptrBits/trieBranchingLog2 levels deep. +const ptrBits = uint(unsafe.Sizeof(uintptr(0))) * 8 + +// trieNode is the header shared by trieIndirect and trieEntry; isEntry +// says which one a *trieNode actually is. +type trieNode[K comparable, V any] struct { + isEntry bool +} + +type trieIndirect[K comparable, V any] struct { + trieNode[K, V] + dead atomic.Bool // set once pruned; the node must not be written again + mu sync.Mutex + parent *trieIndirect[K, V] + children [trieBranching]atomic.Pointer[trieNode[K, V]] +} + +// trieEntry is a leaf. Keys whose full hashes are equal are chained through +// overflow. The trie never reads p; the cache owns what it means (see +// entDel). +type trieEntry[K comparable, V any] struct { + trieNode[K, V] + overflow atomic.Pointer[trieEntry[K, V]] + key K + p atomic.Pointer[V] +} + +func newTrieIndirect[K comparable, V any](parent *trieIndirect[K, V]) *trieIndirect[K, V] { + return &trieIndirect[K, V]{parent: parent} +} + +func newTrieEntry[K comparable, V any](key K, value *V) *trieEntry[K, V] { + e := &trieEntry[K, V]{key: key} + e.isEntry = true + if value != nil { + e.p.Store(value) + } + return e +} + +func (n *trieNode[K, V]) entry() *trieEntry[K, V] { + return (*trieEntry[K, V])(unsafe.Pointer(n)) +} + +func (n *trieNode[K, V]) indirect() *trieIndirect[K, V] { + return (*trieIndirect[K, V])(unsafe.Pointer(n)) +} + +func (i *trieIndirect[K, V]) empty() bool { + for j := range i.children { + if i.children[j].Load() != nil { + return false + } + } + return true +} + +// initRoot returns the root, creating it and the hash seed on first use. +// The seed is written before the root is published, so anything that loads +// a non-nil root can hash. +func (t *trie[K, V]) initRoot() *trieIndirect[K, V] { + if i := t.root.Load(); i != nil { + return i + } + t.initMu.Lock() + defer t.initMu.Unlock() + if i := t.root.Load(); i != nil { + return i + } + if t.hashFn == nil { + t.seed = maphash.MakeSeed() + } + i := newTrieIndirect[K, V](nil) + t.root.Store(i) + return i +} + +func (t *trie[K, V]) hash(k K) uintptr { + h := uintptr(maphash.Comparable(t.seed, k)) + if t.hashFn != nil { + h = t.hashFn(h) + } + return h +} + +// loadEntry returns the entry for k, or nil if there is none. +func (t *trie[K, V]) loadEntry(k K) *trieEntry[K, V] { + i := t.root.Load() + if i == nil { + return nil + } + h := t.hash(k) + for shift := ptrBits; shift != 0; { + shift -= trieBranchingLog2 + n := i.children[(h>>shift)&trieBranchingMask].Load() + if n == nil { + return nil + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e + } + } + return nil + } + i = n.indirect() + } + panic("cache: trie ran out of hash bits") +} + +// loadOrStoreEntry returns the existing entry for k and true, or inserts a +// new entry holding value (which may be nil) and returns it and false. +func (t *trie[K, V]) loadOrStoreEntry(k K, value *V) (*trieEntry[K, V], bool) { + t.initRoot() + h := t.hash(k) + var ( + i *trieIndirect[K, V] + shift uint + slot *atomic.Pointer[trieNode[K, V]] + n *trieNode[K, V] + ) + for { + // Find the slot we would insert into without locking, then lock + // its node and retry if the slot has since become an indirect or + // the node was pruned. + i = t.root.Load() + shift = ptrBits + for { + if shift == 0 { + panic("cache: trie ran out of hash bits") + } + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + break + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + break + } + i = n.indirect() + } + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + break + } + i.mu.Unlock() + } + defer i.mu.Unlock() + + var old *trieEntry[K, V] + if n != nil { + old = n.entry() + for e := old; e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + } + e := newTrieEntry(k, value) + if old == nil { + slot.Store(&e.trieNode) + } else { + slot.Store(t.expand(old, e, h, shift, i)) + } + return e, false +} + +// expand returns the node to replace old's slot with so that both old and +// new are reachable: new chained in front of old if their full hashes are +// equal, otherwise indirect nodes down to the first hash bits that differ. +func (t *trie[K, V]) expand(old, new *trieEntry[K, V], newHash uintptr, shift uint, parent *trieIndirect[K, V]) *trieNode[K, V] { + oldHash := t.hash(old.key) + if oldHash == newHash { + new.overflow.Store(old) + return &new.trieNode + } + top := newTrieIndirect(parent) + cur := top + for { + if shift == 0 { + panic("cache: trie ran out of hash bits") + } + shift -= trieBranchingLog2 + oi := (oldHash >> shift) & trieBranchingMask + ni := (newHash >> shift) & trieBranchingMask + if oi != ni { + cur.children[oi].Store(&old.trieNode) + cur.children[ni].Store(&new.trieNode) + return &top.trieNode + } + next := newTrieIndirect(cur) + cur.children[oi].Store(&next.trieNode) + cur = next + } +} + +// deleteEntryIf unlinks and returns the entry for k if pred, evaluated +// while the entry's node is locked, returns true. A nil pred always +// deletes. +func (t *trie[K, V]) deleteEntryIf(k K, pred func(*trieEntry[K, V]) bool) *trieEntry[K, V] { + i := t.root.Load() + if i == nil { + return nil + } + h := t.hash(k) + var ( + shift uint + slot *atomic.Pointer[trieNode[K, V]] + n *trieNode[K, V] + ) + for { + i = t.root.Load() + shift = ptrBits + for { + if shift == 0 { + panic("cache: trie ran out of hash bits") + } + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + return nil + } + if n.isEntry { + break + } + i = n.indirect() + } + i.mu.Lock() + if i.dead.Load() { + i.mu.Unlock() + continue + } + n = slot.Load() + if n == nil { + i.mu.Unlock() + return nil + } + if n.isEntry { + break + } + // A concurrent insert expanded the slot; k may now be deeper. + i.mu.Unlock() + } + + head := n.entry() + var prev, target *trieEntry[K, V] + for e := head; e != nil; prev, e = e, e.overflow.Load() { + if e.key == k { + target = e + break + } + } + if target == nil || pred != nil && !pred(target) { + i.mu.Unlock() + return nil + } + if prev != nil { + prev.overflow.Store(target.overflow.Load()) + i.mu.Unlock() + return target + } + if next := target.overflow.Load(); next != nil { + slot.Store(&next.trieNode) + i.mu.Unlock() + return target + } + slot.Store(nil) + + // Prune now-empty indirect nodes up toward the root. We hold i's lock + // until i is marked dead, so an insert that locks i afterward sees + // dead and retries from the root. + for i.parent != nil && i.empty() { + shift += trieBranchingLog2 + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(h>>shift)&trieBranchingMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return target +} + +// walk calls f on every entry until f returns false. We do not snapshot: +// concurrent changes may or may not be seen, but no key is visited twice. +func (t *trie[K, V]) walk(f func(*trieEntry[K, V]) bool) { + if i := t.root.Load(); i != nil { + walkIndirect(i, f) + } +} + +func walkIndirect[K comparable, V any](i *trieIndirect[K, V], f func(*trieEntry[K, V]) bool) bool { + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + if !n.isEntry { + if !walkIndirect(n.indirect(), f) { + return false + } + continue + } + for e := n.entry(); e != nil; e = e.overflow.Load() { + if !f(e) { + return false + } + } + } + return true +} + +// clear swaps in an empty root. Anything still working on the old tree +// finishes on the old tree. +func (t *trie[K, V]) clear() { + if t.root.Load() != nil { + t.root.Store(newTrieIndirect[K, V](nil)) + } +} diff --git a/cache/trie_test.go b/cache/trie_test.go new file mode 100644 index 0000000..fd26837 --- /dev/null +++ b/cache/trie_test.go @@ -0,0 +1,717 @@ +package cache + +import ( + "fmt" + "hash/maphash" + "math/rand/v2" + "sync" + "sync/atomic" + "testing" +) + +// Helper: store or replace the value slot with a fresh pointer carrying v. +// The trie owns the entry; the caller owns the value pointer identity. +func triePut[K comparable, V any](t *trie[K, V], k K, v V) *V { + p := &v + e, _ := t.loadOrStoreEntry(k, p) + e.p.Store(p) + return p +} + +func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { + return t.deleteEntryIf(k, nil) +} + +// setHashes makes each key in hashes hash to its value in tr. +func setHashes[K comparable, V any](tr *trie[K, V], hashes map[K]uintptr) { + tr.initRoot() // creates the seed + raw := make(map[uintptr]uintptr, len(hashes)) + for k, h := range hashes { + raw[uintptr(maphash.Comparable(tr.seed, k))] = h + } + tr.hashFn = func(h uintptr) uintptr { return raw[h] } +} + +func trieGet[K comparable, V any](t *trie[K, V], k K) (V, bool) { + e := t.loadEntry(k) + if e == nil { + return *new(V), false + } + p := e.p.Load() + if p == nil { + return *new(V), false + } + return *p, true +} + +// verifyTrie checks structural invariants once concurrent operations have +// quiesced: every entry hangs at the position prescribed by its hash, +// overflow chains hold only full-hash collisions, parent pointers are +// consistent, no linked node is marked dead, and no non-root indirect is +// empty (pruning must have removed it). It must not run concurrently with +// mutations. +func verifyTrie[K comparable, V any](t *testing.T, tr *trie[K, V]) { + t.Helper() + if tr.root.Load() == nil { + return + } + root := tr.root.Load() + var walk func(i *trieIndirect[K, V], prefix uintptr, nbits uint) + walk = func(i *trieIndirect[K, V], prefix uintptr, nbits uint) { + if i.dead.Load() { + t.Fatalf("linked indirect at depth %d is marked dead", nbits/trieBranchingLog2) + } + if i != root && i.empty() { + t.Fatalf("empty non-root indirect linked at depth %d (pruning missed it)", nbits/trieBranchingLog2) + } + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + cp := prefix< ptrBits { + t.Fatal("trie deeper than the hash bit budget") + } + if n.isEntry { + head := n.entry() + headHash := tr.hash(head.key) + for e := head; e != nil; e = e.overflow.Load() { + h := tr.hash(e.key) + if h>>(ptrBits-cb) != cp { + t.Fatalf("entry for key %v misplaced: hash %#x does not match its path %#x at depth %d", + e.key, h, cp, cb/trieBranchingLog2) + } + if h != headHash { + t.Fatalf("overflow chain mixes hashes: key %v hash %#x vs head %v hash %#x", + e.key, h, head.key, headHash) + } + } + } else { + ci := n.indirect() + if ci.parent != i { + t.Fatalf("indirect at depth %d has a wrong parent pointer", cb/trieBranchingLog2) + } + walk(ci, cp, cb) + } + } + } + walk(root, 0, 0) +} + +func TestTrie_ZeroValueLoad(t *testing.T) { + var tr trie[string, int] + if e := tr.loadEntry("missing"); e != nil { + t.Fatalf("loadEntry on zero-value trie returned %v, want nil", e) + } + var visited int + tr.walk(func(*trieEntry[string, int]) bool { visited++; return true }) + if visited != 0 { + t.Fatalf("walk on zero-value trie visited %d, want 0", visited) + } + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry on zero-value trie returned %v, want nil", e) + } +} + +func TestTrie_StoreLoad(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + if v, ok := trieGet(&tr, "a"); !ok || v != 1 { + t.Fatalf("get a: v=%d ok=%v, want 1 true", v, ok) + } + if v, ok := trieGet(&tr, "b"); !ok || v != 2 { + t.Fatalf("get b: v=%d ok=%v, want 2 true", v, ok) + } + if _, ok := trieGet(&tr, "c"); ok { + t.Fatal("get c should be missing") + } +} + +func TestTrie_LoadOrStoreExisting(t *testing.T) { + var tr trie[string, int] + v1 := 1 + e1, loaded := tr.loadOrStoreEntry("k", &v1) + if loaded { + t.Fatal("first loadOrStoreEntry should not report loaded") + } + v2 := 2 + e2, loaded := tr.loadOrStoreEntry("k", &v2) + if !loaded { + t.Fatal("second loadOrStoreEntry should report loaded") + } + if e1 != e2 { + t.Fatal("loadOrStoreEntry should return the same entry object on subsequent calls") + } + // The stored value is v1's pointer; v2 was ignored. + if got := *e2.p.Load(); got != 1 { + t.Fatalf("stored value = %d, want 1 (new value must not overwrite)", got) + } +} + +func TestTrie_LoadOrStoreWithNilValue(t *testing.T) { + // Cache always creates entries with a value in place (a nil slot means + // tombstoned), but the trie itself is value-agnostic: verify + // loadOrStoreEntry(k, nil) is legal and the entry.p is nil. + var tr trie[string, int] + e, loaded := tr.loadOrStoreEntry("k", nil) + if loaded { + t.Fatal("expected not-loaded for first insert") + } + if e.p.Load() != nil { + t.Fatal("nil-initialized entry should have nil value slot") + } + v := 42 + e.p.Store(&v) + if got, ok := trieGet(&tr, "k"); !ok || got != 42 { + t.Fatalf("after Store: got=%d ok=%v, want 42 true", got, ok) + } +} + +func TestTrie_Delete(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + + e := tr.deleteEntry("a") + if e == nil || e.key != "a" { + t.Fatalf("deleteEntry(a) = %v, want entry for a", e) + } + if _, ok := trieGet(&tr, "a"); ok { + t.Fatal("after delete, a should be missing") + } + if _, ok := trieGet(&tr, "b"); !ok { + t.Fatal("after delete of a, b should still be present") + } + if e := tr.deleteEntry("a"); e != nil { + t.Fatalf("second deleteEntry(a) = %v, want nil", e) + } +} + +func TestTrie_DeletePrunesEmptyParents(t *testing.T) { + // Insert many keys so the tree grows, then delete them all. After, the + // root should be empty (all children nil). + var tr trie[int, int] + const n = 1000 + for i := range n { + triePut(&tr, i, i) + } + for i := range n { + if e := tr.deleteEntry(i); e == nil { + t.Fatalf("deleteEntry(%d) unexpectedly returned nil", i) + } + } + root := tr.root.Load() + for i := range root.children { + if n := root.children[i].Load(); n != nil { + t.Fatalf("root.children[%d] = %v, want nil (pruning should have cleared)", i, n) + } + } +} + +func TestTrie_Walk(t *testing.T) { + var tr trie[int, int] + const n = 256 + for i := range n { + triePut(&tr, i, i*10) + } + seen := make(map[int]int) + tr.walk(func(e *trieEntry[int, int]) bool { + if p := e.p.Load(); p != nil { + seen[e.key] = *p + } + return true + }) + if len(seen) != n { + t.Fatalf("walk saw %d entries, want %d", len(seen), n) + } + for k, v := range seen { + if v != k*10 { + t.Fatalf("seen[%d] = %d, want %d", k, v, k*10) + } + } +} + +func TestTrie_WalkEarlyStop(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { + count++ + return count < 5 + }) + if count != 5 { + t.Fatalf("walk visited %d, want 5 (early stop)", count) + } +} + +func TestTrie_Clear(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + tr.clear() + for i := range 100 { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("after clear, key %d should be missing", i) + } + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { count++; return true }) + if count != 0 { + t.Fatalf("walk after clear visited %d, want 0", count) + } + // Trie is still usable after clear. + triePut(&tr, 1, 1) + if v, ok := trieGet(&tr, 1); !ok || v != 1 { + t.Fatalf("reuse after clear: v=%d ok=%v", v, ok) + } +} + +// TestTrie_ManyKeys exercises the expand path repeatedly: random keys will +// routinely collide on short hash prefixes, forcing the trie to extend its +// depth. 20k keys is enough that most hash prefixes of length 4 will +// collide multiple times. +func TestTrie_ManyKeys(t *testing.T) { + var tr trie[int, int] + const n = 20_000 + keys := rand.Perm(n) + for _, k := range keys { + triePut(&tr, k, k*2) + } + for _, k := range keys { + v, ok := trieGet(&tr, k) + if !ok || v != k*2 { + t.Fatalf("lookup(%d): v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + // Delete half and re-verify. + for i, k := range keys { + if i%2 == 0 { + tr.deleteEntry(k) + } + } + for i, k := range keys { + v, ok := trieGet(&tr, k) + switch i % 2 { + case 0: + if ok { + t.Fatalf("deleted key %d still present: v=%d", k, v) + } + case 1: + if !ok || v != k*2 { + t.Fatalf("surviving key %d: v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + } + verifyTrie(t, &tr) +} + +// TestTrie_FullHashCollision verifies the overflow-chain behavior by forcing +// two distinct keys to hash identically via the test-only hashFn hook. +func TestTrie_FullHashCollision(t *testing.T) { + var tr trie[string, int] + // Collide everything in a single hash bucket. + tr.hashFn = func(uintptr) uintptr { return 0xdeadbeef } + + const n = 10 + for i := range n { + k := fmt.Sprintf("k%d", i) + triePut(&tr, k, i) + } + // All n entries share one overflow chain under the same leaf slot. + for i := range n { + k := fmt.Sprintf("k%d", i) + v, ok := trieGet(&tr, k) + if !ok || v != i { + t.Fatalf("get %q: v=%d ok=%v, want %d true", k, v, ok, i) + } + } + // Delete one from the middle; the rest remain. + tr.deleteEntry("k5") + if _, ok := trieGet(&tr, "k5"); ok { + t.Fatal("k5 should be deleted") + } + for i := range n { + if i == 5 { + continue + } + k := fmt.Sprintf("k%d", i) + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%q should survive delete of k5", k) + } + } +} + +// TestTrie_FullHashCollisionDeleteHead deletes the head of the overflow +// chain. The chain head is what the slot points at, so slot must be +// updated to the next entry. +func TestTrie_FullHashCollisionDeleteHead(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(uintptr) uintptr { return 0x42 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + + // Delete the most-recently-inserted (head of chain, per expand logic). + if e := tr.deleteEntry("c"); e == nil || e.key != "c" { + t.Fatalf("delete c: %v", e) + } + for _, k := range []string{"a", "b"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s missing after delete of chain head", k) + } + } +} + +func TestTrie_LoadOrStoreDuringCollision(t *testing.T) { + // After a collision chain exists, loadOrStoreEntry for an already-present + // key must find it via the overflow walk, not append a duplicate. + var tr trie[string, int] + tr.hashFn = func(uintptr) uintptr { return 0xabcd } + triePut(&tr, "x", 1) + triePut(&tr, "y", 2) + v := 999 + e, loaded := tr.loadOrStoreEntry("x", &v) + if !loaded { + t.Fatal("second loadOrStoreEntry(x) should report loaded") + } + if got := *e.p.Load(); got != 1 { + t.Fatalf("existing entry value=%d, want 1 (new value must be ignored)", got) + } +} + +// TestTrie_ConcurrentStoreLoad runs many goroutines performing +// loadOrStoreEntry and loadEntry over disjoint key ranges, verifying no +// race and eventual consistency. +func TestTrie_ConcurrentStoreLoad(t *testing.T) { + var tr trie[int, int] + const workers = 8 + const perWorker = 2000 + + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + base := w * perWorker + for i := range perWorker { + triePut(&tr, base+i, base+i) + } + for i := range perWorker { + v, ok := trieGet(&tr, base+i) + if !ok || v != base+i { + t.Errorf("worker %d, key %d: v=%d ok=%v", w, base+i, v, ok) + return + } + } + }) + } + wg.Wait() + verifyTrie(t, &tr) +} + +// TestTrie_ConcurrentSameKey hammers the same key with many goroutines +// doing loadOrStoreEntry. Exactly one must report loaded=false; all others +// must observe the same entry object. +func TestTrie_ConcurrentSameKey(t *testing.T) { + const workers = 64 + for iter := range 100 { + var tr trie[int, int] + var wg sync.WaitGroup + entries := make([]*trieEntry[int, int], workers) + loadeds := make([]bool, workers) + for w := range workers { + wg.Go(func() { + v := w + e, loaded := tr.loadOrStoreEntry(iter, &v) + entries[w] = e + loadeds[w] = loaded + }) + } + wg.Wait() + + var inserted int + for _, l := range loadeds { + if !l { + inserted++ + } + } + if inserted != 1 { + t.Fatalf("iter %d: %d inserters, want exactly 1", iter, inserted) + } + first := entries[0] + for i, e := range entries { + if e != first { + t.Fatalf("iter %d: worker %d saw entry %p, want %p", iter, i, e, first) + } + } + verifyTrie(t, &tr) + } +} + +// TestTrie_ConcurrentDeletePrune exercises concurrent deletes across many +// keys so parent pruning fires from many angles simultaneously. +func TestTrie_ConcurrentDeletePrune(t *testing.T) { + var tr trie[int, int] + const n = 10_000 + for i := range n { + triePut(&tr, i, i) + } + const workers = 8 + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + for i := w; i < n; i += workers { + tr.deleteEntry(i) + } + }) + } + wg.Wait() + for i := range n { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("key %d should be deleted", i) + } + } + // After all deletions, root should be empty. + if root := tr.root.Load(); !root.empty() { + t.Fatal("root not pruned: has surviving children") + } + verifyTrie(t, &tr) +} + +// TestTrie_RangeDuringMutation runs walk concurrently with store and +// delete. No snapshot is promised; we assert only no-crash + that every +// observed key-value pair is plausible (key was stored, value is key*2 or +// nil). +func TestTrie_RangeDuringMutation(t *testing.T) { + var tr trie[int, int] + const n = 2000 + for i := range n { + triePut(&tr, i, i*2) + } + + var stop atomic.Bool + var wg sync.WaitGroup + + wg.Go(func() { + for !stop.Load() { + i := rand.IntN(n) + triePut(&tr, i, i*2) + tr.deleteEntry(i) + } + }) + + wg.Go(func() { + for !stop.Load() { + tr.walk(func(e *trieEntry[int, int]) bool { + p := e.p.Load() + if p != nil && *p != e.key*2 { + t.Errorf("invariant break: key=%d value=%d", e.key, *p) + return false + } + return true + }) + } + }) + + // Let the race run briefly. + for range 50_000 { + i := rand.IntN(n) + if _, ok := trieGet(&tr, i); ok { + // just a read + } + } + stop.Store(true) + wg.Wait() + verifyTrie(t, &tr) +} + +// TestTrie_DeleteMissingInCollisionChain walks an overflow chain in search +// of a key that is never present, exercising deleteEntryIf's +// "walk the chain to the end without finding k" branch. +func TestTrie_DeleteMissingInCollisionChain(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(uintptr) uintptr { return 0x1234 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + // "missing" collides with a/b/c but was never stored. + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry of never-stored colliding key returned %v, want nil", e) + } + // Original entries survive. + for _, k := range []string{"a", "b", "c"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s should survive delete of missing colliding key", k) + } + } +} + +// TestTrie_DeleteRaceRetries stresses deleteEntry's lock-then-retry paths: +// the "parent is dead" retry and the "slot became nil after lock" +// early-return. Both fire when a concurrent delete prunes or empties the +// slot between our unlocked walk and our mu.Lock. +// +// The stress needs sustained concurrent deletes on many hash-adjacent +// keys so that pruning and slot clearing happen continuously. +func TestTrie_DeleteRaceRetries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const workers = 16 + const keySpace = 256 + for range 200 { + var tr trie[int, int] + // Prime: every key is present. + for i := range keySpace { + triePut(&tr, i, i) + } + var wg sync.WaitGroup + // Deleter swarms. + for w := range workers { + wg.Go(func() { + for i := w; i < keySpace; i += workers { + tr.deleteEntry(i) + } + }) + } + // Concurrent deleter attempting same keys (likely to race on lock). + for w := range workers { + wg.Go(func() { + for i := keySpace - 1 - w; i >= 0; i -= workers { + tr.deleteEntry(i) + } + }) + } + wg.Wait() + verifyTrie(t, &tr) + } +} + +// TestTrie_DeleteRaceAgainstExpand targets deleteEntryIf's "slot changed to +// non-entry after lock" branch. An indirect-node split (expand) during the +// window between delete's unlocked walk and its parent.Lock forces the +// reload to see a non-entry node, which must trigger a retry from the root +// (matching internal/sync.HashTrieMap's find), not a give-up: "a" is present +// for the whole round until the delete removes it, so deleteEntry("a") must +// always find and return it. +// +// We use a controlled hashFn that collides a few keys in the top bits but +// not all bits, so inserting them triggers expand. Concurrent insert+delete +// races through the expand window. +func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + // Keys whose hashes match in the top 4 bits but diverge later: they + // share a slot at depth 1 and a leaf at depth 2 (via overflow then + // expand). The constants are built by shifting so they fit a uintptr + // on both 64-bit and 32-bit platforms. + hashes := map[string]uintptr{ + "a": 0xA << (ptrBits - 4), + "b": 0xA<<(ptrBits-4) | 1, + "c": 0xA<<(ptrBits-4) | 2, + "d": 0xB << (ptrBits - 4), + } + for i := range 20_000 { + var tr trie[string, int] + setHashes(&tr, hashes) + triePut(&tr, "a", 1) + triePut(&tr, "d", 4) + + var wg sync.WaitGroup + var deleted *trieEntry[string, int] + wg.Go(func() { + triePut(&tr, "b", 2) + }) + wg.Go(func() { + triePut(&tr, "c", 3) + }) + wg.Go(func() { + deleted = tr.deleteEntry("a") + }) + wg.Wait() + + if deleted == nil || deleted.key != "a" { + t.Fatalf("round %d: deleteEntry(a) = %v, want the entry for a (delete must retry across a concurrent expand)", i, deleted) + } + if e := tr.loadEntry("a"); e != nil { + t.Fatalf("round %d: a still present after deleteEntry returned it", i) + } + for _, k := range []string{"b", "c", "d"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("round %d: %s missing after delete of a", i, k) + } + } + verifyTrie(t, &tr) + } +} + +// TestTrie_DeleteDuringConcurrentStore: a goroutine repeatedly deletes key +// K while another repeatedly stores it. Both succeed; the final state is +// "either present or absent" and neither crashes. +func TestTrie_DeleteDuringConcurrentStore(t *testing.T) { + var tr trie[int, int] + var stop atomic.Bool + var wg sync.WaitGroup + wg.Go(func() { + for !stop.Load() { + triePut(&tr, 7, 42) + } + }) + wg.Go(func() { + for !stop.Load() { + tr.deleteEntry(7) + } + }) + for range 100_000 { + trieGet(&tr, 7) + } + stop.Store(true) + wg.Wait() + verifyTrie(t, &tr) +} + +// deepTrie returns a trie with an indirect node at every level of the path +// for hash 0, which no correct trie has, and makes every key hash to 0. +func deepTrie() *trie[int, int] { + tr := new(trie[int, int]) + i := tr.initRoot() + tr.hashFn = func(uintptr) uintptr { return 0 } + for range ptrBits / trieBranchingLog2 { + n := newTrieIndirect(i) + i.children[0].Store(&n.trieNode) + i = n + } + return tr +} + +// A trie whose depth exceeds the hash bits is corrupt: every walk must +// panic rather than read past the hash. +func TestTrie_RanOutOfHashBitsPanics(t *testing.T) { + for name, f := range map[string]func(){ + "loadEntry": func() { deepTrie().loadEntry(1) }, + "loadOrStoreEntry": func() { deepTrie().loadOrStoreEntry(1, nil) }, + "deleteEntryIf": func() { deepTrie().deleteEntryIf(1, nil) }, + "expand": func() { + tr := new(trie[int, int]) + parent := tr.initRoot() + tr.hashFn = func(h uintptr) uintptr { return 0 } + old := newTrieEntry[int, int](1, nil) + // Different hashes with no bits left to tell them apart. + tr.expand(old, newTrieEntry[int, int](2, nil), 1, 0, parent) + }, + } { + func() { + defer func() { + if r := recover(); r != "cache: trie ran out of hash bits" { + t.Errorf("%s: recovered %v, want the ran out of hash bits panic", name, r) + } + }() + f() + }() + } +} diff --git a/cache/wrappers_test.go b/cache/wrappers_test.go new file mode 100644 index 0000000..f994ed9 --- /dev/null +++ b/cache/wrappers_test.go @@ -0,0 +1,331 @@ +package cache + +import ( + "errors" + "sort" + "sync/atomic" + "testing" + "time" +) + +// Item delegates to Cache[struct{}, V]. These tests exercise every method so +// the thin wrappers contribute to coverage and so a refactor of the underlying +// Cache that forgets to propagate a method would be caught. + +func TestItem_BasicGetAndCaches(t *testing.T) { + i := NewItem[int]() + var calls int32 + v, err, s := i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return 42, nil + }) + if v != 42 || err != nil || !s.IsMiss() { + t.Fatalf("first Get: got v=%d err=%v s=%v, want v=42 err=nil Miss", v, err, s) + } + v, err, s = i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return -1, nil + }) + if v != 42 || err != nil || !s.IsHit() { + t.Fatalf("second Get: got v=%d err=%v s=%v, want v=42 err=nil Hit", v, err, s) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestItem_TryGet(t *testing.T) { + i := NewItem[string]() + _, _, s := i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet on empty item: state=%v want Miss", s) + } + i.Set("hello") + v, err, s := i.TryGet() + if v != "hello" || err != nil || !s.IsHit() { + t.Fatalf("TryGet after Set: v=%q err=%v s=%v", v, err, s) + } +} + +func TestItem_Delete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + v, err, s := i.Delete() + if v != 7 || err != nil || !s.IsHit() { + t.Fatalf("Delete of set value: v=%d err=%v s=%v", v, err, s) + } + _, _, s = i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet after Delete: state=%v want Miss", s) + } + // Deleting an already-empty item is a no-op. + _, _, s = i.Delete() + if !s.IsMiss() { + t.Fatalf("Delete of empty item: state=%v want Miss", s) + } +} + +func TestItem_Expire(t *testing.T) { + i := NewItem[int](MaxAge(time.Hour)) + i.Set(99) + if _, _, s := i.TryGet(); !s.IsHit() { + t.Fatalf("before Expire: want Hit") + } + i.Expire() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Expire: want Miss") + } +} + +func TestItem_Swap(t *testing.T) { + i := NewItem[int]() + old, err, s := i.Swap(5) + if err != nil || !s.IsMiss() { + t.Fatalf("first Swap: old=%d err=%v s=%v, want Miss", old, err, s) + } + old, err, s = i.Swap(6) + if old != 5 || err != nil || !s.IsHit() { + t.Fatalf("second Swap: old=%d err=%v s=%v, want 5 Hit", old, err, s) + } + if v, _, _ := i.TryGet(); v != 6 { + t.Fatalf("after second Swap: TryGet=%d, want 6", v) + } +} + +func TestItem_CompareAndSwap(t *testing.T) { + i := NewItem[int]() + i.Set(10) + if !i.CompareAndSwap(10, 11) { + t.Fatal("CAS 10->11 should succeed") + } + if i.CompareAndSwap(10, 12) { + t.Fatal("CAS 10->12 should fail after value is 11") + } + if v, _, _ := i.TryGet(); v != 11 { + t.Fatalf("after CAS: TryGet=%d, want 11", v) + } +} + +func TestItem_CompareAndDelete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + if i.CompareAndDelete(8) { + t.Fatal("CAD of wrong value should fail") + } + if !i.CompareAndDelete(7) { + t.Fatal("CAD of correct value should succeed") + } + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after CAD: state=%v want Miss", s) + } +} + +func TestItem_Clear(t *testing.T) { + i := NewItem[int]() + i.Set(1) + i.Clear() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", s) + } +} + +func TestItem_ZeroValue(t *testing.T) { + // Docstring says zero-value Item is valid. + var i Item[int] + v, err, s := i.Get(func() (int, error) { return 3, nil }) + if v != 3 || err != nil || !s.IsMiss() { + t.Fatalf("zero Item Get: v=%d err=%v s=%v", v, err, s) + } + if v, _, _ := i.TryGet(); v != 3 { + t.Fatalf("zero Item TryGet: v=%d want 3", v) + } +} + +// Set is a key-only cache over Cache[K, struct{}]. + +func TestSet_BasicGet(t *testing.T) { + s := NewSet[string]() + var calls int32 + err, ks := s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsMiss() { + t.Fatalf("first Get: err=%v state=%v want nil Miss", err, ks) + } + err, ks = s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsHit() { + t.Fatalf("second Get: err=%v state=%v want nil Hit", err, ks) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestSet_GetError(t *testing.T) { + s := NewSet[string]() + wantErr := errors.New("boom") + err, ks := s.Get("a", func() error { return wantErr }) + if !errors.Is(err, wantErr) || !ks.IsMiss() { + t.Fatalf("Get with error: err=%v state=%v", err, ks) + } +} + +func TestSet_TryGet(t *testing.T) { + s := NewSet[string]() + _, ks := s.TryGet("missing") + if !ks.IsMiss() { + t.Fatalf("TryGet missing: state=%v", ks) + } + s.Set("k") + err, ks := s.TryGet("k") + if err != nil || !ks.IsHit() { + t.Fatalf("TryGet after Set: err=%v state=%v", err, ks) + } +} + +func TestSet_Delete(t *testing.T) { + s := NewSet[string]() + s.Set("k") + _, ks := s.Delete("k") + if !ks.IsHit() { + t.Fatalf("Delete existing: state=%v want Hit", ks) + } + _, ks = s.Delete("k") + if !ks.IsMiss() { + t.Fatalf("Delete missing: state=%v want Miss", ks) + } +} + +func TestSet_Expire(t *testing.T) { + s := NewSet[string](MaxAge(time.Hour)) + s.Set("k") + s.Expire("k") + if _, ks := s.TryGet("k"); !ks.IsMiss() { + t.Fatalf("TryGet after Expire: state=%v want Miss", ks) + } +} + +func TestSet_Range(t *testing.T) { + s := NewSet[string]() + for _, k := range []string{"a", "b", "c"} { + s.Set(k) + } + var got []string + s.Range(func(k string, err error) bool { + if err != nil { + t.Errorf("unexpected err for key %q: %v", k, err) + } + got = append(got, k) + return true + }) + sort.Strings(got) + want := []string{"a", "b", "c"} + if len(got) != len(want) { + t.Fatalf("Range yielded %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Range yielded %v, want %v", got, want) + } + } + + // Early stop. + count := 0 + s.Range(func(string, error) bool { + count++ + return false + }) + if count != 1 { + t.Fatalf("Range early-stop visited %d, want 1", count) + } +} + +func TestSet_Clean(t *testing.T) { + s := NewSet[string](MaxAge(time.Nanosecond)) + s.Set("k") + time.Sleep(time.Millisecond) + s.Clean() + if n := trieEntries(&s.c); n != 0 { + t.Fatalf("Clean left %d entries, want 0", n) + } +} + +func TestSet_Clear(t *testing.T) { + s := NewSet[string]() + s.Set("a") + s.Set("b") + s.Clear() + if _, ks := s.TryGet("a"); !ks.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", ks) + } + // Range yields nothing. + var got []string + s.Range(func(k string, _ error) bool { + got = append(got, k) + return true + }) + if len(got) != 0 { + t.Fatalf("Range after Clear: %v, want empty", got) + } +} + +func TestSet_AutoClean(t *testing.T) { + // NewSet must actually start the autoclean goroutine (a regression had + // NewSet building the Cache directly, silently ignoring the option); + // the cleaning behavior itself is covered by the Cache autoclean tests. + s := NewSet[string](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if s.c.quitClean == nil { + t.Fatal("NewSet with AutoCleanInterval should start the autoclean goroutine") + } + s.StopAutoClean() + // Calling twice is safe (sync.Once on the underlying Cache). + s.StopAutoClean() +} + +func TestItem_AutoClean(t *testing.T) { + i := NewItem[int](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if i.c.quitClean == nil { + t.Fatal("NewItem with AutoCleanInterval should start the autoclean goroutine") + } + defer i.c.StopAutoClean() + i.Set(1) + waitEntries(t, &i.c, 0) +} + +func TestSet_ZeroValue(t *testing.T) { + var s Set[string] + err, ks := s.Get("a", func() error { return nil }) + if err != nil || !ks.IsMiss() { + t.Fatalf("zero Set Get: err=%v state=%v", err, ks) + } +} + +func TestSet_GetHit(t *testing.T) { + s := NewSet[string]() + s.Set("k") + if err, st := s.Get("k", func() error { + t.Error("miss ran for a cached key") + return nil + }); err != nil || st != Hit { + t.Fatalf("Get: (%v, %v), want (nil, Hit)", err, st) + } +} + +// A stale Set.Get refreshes in the background. +func TestSet_GetStaleRefresh(t *testing.T) { + s := NewSet[int](MaxAge(0), MaxStaleAge(time.Hour)) + s.Set(1) + ran := make(chan struct{}) + if err, st := s.Get(1, func() error { close(ran); return nil }); err != nil || st != Stale { + t.Fatalf("Get: (%v, %v), want (nil, Stale)", err, st) + } + select { + case <-ran: + case <-time.After(10 * time.Second): + t.Fatal("the background refresh never ran") + } +} diff --git a/go.mod b/go.mod index da78d2d..3e25632 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/twmb/go-cache -go 1.18 +go 1.25 retract v1.1.0 // The newly added API cannot work. From 10ea4ee80cd93840efe54a7904eb09d3204ecc92 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Sun, 27 Sep 2026 00:20:44 -0600 Subject: [PATCH 2/2] README: describe the trie and benchmark against sync.Map --- README.md | 130 ++++++++++++++++++------------------------------------ 1 file changed, 43 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 64bb839..d416bc8 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ cache Package cache provides a concurrency safe, mostly lock-free, singleflight request collapsing generic cache with support for stale values. -The guts of this package are similar to `sync.Map`, with minor differences in -when the internal locked write-map is promoted an atomic read-only map, and -major differences to support singleflight request collapsing. +The storage is a concurrent hash-trie similar to Go's internal +`HashTrieMap`, which backs `sync.Map`: loads are lock-free, and inserts and +deletes lock only the trie node being changed. Functions in this API that return a `KeyState` return the state last, rather than the error last: the value and error are cached as a single internal unit @@ -34,9 +34,12 @@ value can be kept and returned from `Get` during a refresh / if a refresh fails. Keys can be manually expired with `Expire`. Internally expired values or errors can be occasionally cleaned with `Clean`. -Out of an abundance of paranoia that this code is correct, there are unit tests -to hit 97% coverage of the `cache.Cache` type. As well, all tests against -`sync.Map` are copied into this library and used against `cache.Cache`. +Out of an abundance of paranoia that this code is correct, the tests include +all stdlib `sync.Map` tests run against `cache.Cache`, a property test +against a mutex guarded map, a linearizability checker for concurrent +operations on a single key, structural checks of the trie after concurrent +stress, and regression tests for specific races. CI runs the tests with the +race detector on amd64 and arm64, and without it on 386. This package also provides a cached `Item` and a `Set`. `Item` can be used to populate an expensive value once and expire or replace it when needed, similar @@ -54,87 +57,40 @@ Documentation Benchmarking ------------ -Some of the APIs are actually faster or more memory efficient than `sync.Map` -because of generics. Some are slower and require more allocs because of the -request collapsing aspect. Some benchmarks below show more CPU, but this is due -to a few slight changes in how the write (dirty) map is internally promoted to -the read map. - -The benchmarks are from the stdlib using a key and value of type int: +These are the stdlib `sync.Map` benchmarks, run against `sync.Map` and against +a `cache.Cache[int, any]` adapted to the same interface. Both are hash-tries +underneath. The numbers are medians of 8 runs on an Apple M2 with Go 1.27; +the last column is allocs per op, `sync.Map` / cache. ``` -name old time/op new time/op delta -LoadMostlyHits 12.8ns ±44% 14.0ns ±50% ~ (p=0.393 n=10+10) -LoadMostlyMisses 10.7ns ±53% 4.6ns ±47% -57.31% (p=0.000 n=10+10) -LoadOrStoreBalanced 351ns ± 3% 705ns ±27% +100.68% (p=0.000 n=8+10) -LoadOrStoreUnique 663ns ±13% 1296ns ±31% +95.53% (p=0.000 n=10+10) -LoadOrStoreCollision 9.71ns ±60% 29.26ns ±46% +201.44% (p=0.000 n=10+10) -LoadAndDeleteBalanced 11.2ns ±42% 7.1ns ±48% -36.06% (p=0.035 n=10+10) -LoadAndDeleteUnique 9.19ns ±62% 4.17ns ±47% -54.59% (p=0.001 n=10+10) -LoadAndDeleteCollision 10.9ns ±41% 14.3ns ±35% +31.18% (p=0.023 n=10+10) -Range 3.73µs ±48% 5.40µs ±45% ~ (p=0.063 n=10+10) -AdversarialAlloc 298ns ±22% 259ns ±21% ~ (p=0.063 n=10+10) -AdversarialDelete 83.7ns ±14% 105.6ns ±20% ~ (p=0.052 n=10+10) -DeleteCollision 6.87ns ±71% 4.86ns ±100% ~ (p=0.424 n=10+10) -SwapCollision 201ns ±26% 266ns ±31% ~ (p=0.143 n=10+10) -SwapMostlyHits 40.3ns ±42% 59.7ns ±51% ~ (p=0.143 n=10+10) -SwapMostlyMisses 662ns ±18% 607ns ±24% -8.19% (p=0.023 n=10+10) -CompareAndSwapCollision 33.4ns ±41% 24.9ns ±65% -25.44% (p=0.023 n=10+10) -CompareAndSwapNoExistingKey 10.3ns ±46% 2.0ns ±53% -80.15% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 9.64ns ±51% 6.02ns ±95% -37.49% (p=0.019 n=10+10) -CompareAndSwapMostlyHits 46.9ns ±39% 56.8ns ±47% ~ (p=0.143 n=10+10) -CompareAndSwapMostlyMisses 22.9ns ±39% 13.4ns ±48% -41.48% (p=0.023 n=10+10) -CompareAndDeleteCollision 28.1ns ±47% 18.3ns ±37% -34.88% (p=0.019 n=10+10) -CompareAndDeleteMostlyHits 64.0ns ±38% 63.9ns ±52% ~ (p=0.165 n=10+10) -CompareAndDeleteMostlyMisses 18.3ns ±37% 8.7ns ±53% -52.65% (p=0.023 n=10+10) - -name old alloc/op new alloc/op delta -LoadMostlyHits 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadMostlyMisses 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadOrStoreBalanced 74.7B ±16% 195.3B ± 1% +161.45% (p=0.000 n=10+10) -LoadOrStoreUnique 159B ± 7% 330B ± 7% +107.15% (p=0.000 n=9+10) -LoadOrStoreCollision 0.00B 32.00B ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 4.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteUnique 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteCollision 1.00B ± 0% 1.00B ± 0% ~ (all equal) -Range 16.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) -AdversarialAlloc 53.4B ± 3% 64.8B ± 3% +21.35% (p=0.000 n=10+10) -AdversarialDelete 22.6B ±12% 39.0B ± 0% +72.57% (p=0.000 n=10+10) -DeleteCollision 0.00B 0.00B ~ (all equal) -SwapCollision 16.0B ± 0% 80.0B ± 0% +400.00% (p=0.000 n=10+10) -SwapMostlyHits 28.0B ± 0% 86.0B ± 0% +207.14% (p=0.000 n=10+10) -SwapMostlyMisses 124B ± 0% 136B ± 1% +9.71% (p=0.000 n=10+10) -CompareAndSwapCollision 7.60B ± 8% 21.70B ±17% +185.53% (p=0.000 n=10+10) -CompareAndSwapNoExistingKey 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 0.00B 0.00B ~ (all equal) -CompareAndSwapMostlyHits 33.0B ± 0% 91.0B ± 0% +175.76% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 23.0B ± 0% 16.0B ± 0% -30.43% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00B 2.00B ± 0% +Inf% (p=0.000 n=10+8) -CompareAndDeleteMostlyHits 39.0B ± 0% 90.6B ± 1% +132.31% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 16.0B ± 0% 8.0B ± 0% -50.00% (p=0.002 n=8+10) - -name old allocs/op new allocs/op delta -LoadMostlyHits 0.00 0.00 ~ (all equal) -LoadMostlyMisses 0.00 0.00 ~ (all equal) -LoadOrStoreBalanced 2.00 ± 0% 3.00 ± 0% +50.00% (p=0.000 n=10+10) -LoadOrStoreUnique 4.00 ± 0% 5.00 ± 0% +25.00% (p=0.000 n=10+10) -LoadOrStoreCollision 0.00 1.00 ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 0.00 0.00 ~ (all equal) -LoadAndDeleteUnique 0.60 ±100% 0.00 -100.00% (p=0.011 n=10+10) -LoadAndDeleteCollision 0.00 0.00 ~ (all equal) -Range 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialAlloc 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialDelete 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -DeleteCollision 0.00 0.00 ~ (all equal) -SwapCollision 1.00 ± 0% 1.00 ± 0% ~ (all equal) -SwapMostlyHits 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -SwapMostlyMisses 6.00 ± 0% 4.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapCollision 0.00 0.00 ~ (all equal) -CompareAndSwapNoExistingKey 0.50 ±100% 0.00 -100.00% (p=0.033 n=10+10) -CompareAndSwapValueNotEqual 0.00 0.00 ~ (all equal) -CompareAndSwapMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00 0.00 ~ (all equal) -CompareAndDeleteMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) +name sync.Map cache allocs +LoadMostlyHits 3.10ns 3.28ns 0 / 0 +LoadMostlyMisses 1.66ns 1.81ns 0 / 0 +LoadOrStoreBalanced 47.2ns 80.0ns 2 / 3 +LoadOrStoreUnique 95.1ns 121ns 3 / 4 +LoadOrStoreCollision 2.53ns 10.3ns 0 / 1 +LoadAndDeleteBalanced 2.05ns 1.86ns 0 / 0 +LoadAndDeleteUnique 2.13ns 0.65ns 0 / 0 +LoadAndDeleteCollision 3.43ns 2.15ns 0 / 0 +Range 1.45µs 2.25µs 0 / 0 +AdversarialAlloc 5.61ns 4.59ns 0 / 0 +AdversarialDelete 2.98ns 2.91ns 0 / 0 +DeleteCollision 1.90ns 1.66ns 0 / 0 +SwapCollision 122ns 258ns 1 / 1 +SwapMostlyHits 35.9ns 30.2ns 2 / 1 +SwapMostlyMisses 208ns 245ns 3 / 3 +CompareAndSwapCollision 6.02ns 9.11ns 0 / 0 +CompareAndSwapNoExistingKey 4.81ns 0.80ns 0 / 0 +CompareAndSwapValueNotEqual 3.69ns 3.07ns 0 / 0 +CompareAndSwapMostlyHits 38.3ns 27.7ns 3 / 2 +CompareAndSwapMostlyMisses 11.6ns 7.93ns 2 / 1 +CompareAndDeleteCollision 3.08ns 4.36ns 0 / 0 +CompareAndDeleteMostlyHits 43.5ns 62.9ns 3 / 3 +CompareAndDeleteMostlyMisses 5.25ns 5.31ns 0 / 1 +Clear 84.8ns 128ns 2 / 3 ``` + +Loads are within 10% of `sync.Map`, and deletes are faster. Stores cost more: +each stored value is a `loading` that also carries singleflight and expiry +state, and the adapter's `LoadOrStore` allocates a closure for its miss +function on every call.