diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..702fba4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: ci + +on: + push: + branches: ["main"] + pull_request: + +jobs: + test: + # amd64 and arm64: the trie/cache core is lock-free, and arm64's weaker + # memory ordering surfaces reordering bugs that x86's TSO hides. + # 1.25 is the minimum in go.mod. + name: test (${{ matrix.os }}, go ${{ matrix.go }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + go: ["1.25.x", stable] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + - run: go vet ./... + - run: go test -race ./... + + test-386: + # 32-bit: uintptr hashes are 32 bits, so the trie is 8 levels deep + # instead of 16 and the test hash constants must fit the word size. + # The race detector is unsupported on 386, so this runs without it. + name: test (linux/386) + runs-on: ubuntu-latest + env: + GOARCH: "386" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - run: go vet ./... + - run: go test ./... diff --git a/README.md b/README.md index 64bb839..d416bc8 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ cache Package cache provides a concurrency safe, mostly lock-free, singleflight request collapsing generic cache with support for stale values. -The guts of this package are similar to `sync.Map`, with minor differences in -when the internal locked write-map is promoted an atomic read-only map, and -major differences to support singleflight request collapsing. +The storage is a concurrent hash-trie similar to Go's internal +`HashTrieMap`, which backs `sync.Map`: loads are lock-free, and inserts and +deletes lock only the trie node being changed. Functions in this API that return a `KeyState` return the state last, rather than the error last: the value and error are cached as a single internal unit @@ -34,9 +34,12 @@ value can be kept and returned from `Get` during a refresh / if a refresh fails. Keys can be manually expired with `Expire`. Internally expired values or errors can be occasionally cleaned with `Clean`. -Out of an abundance of paranoia that this code is correct, there are unit tests -to hit 97% coverage of the `cache.Cache` type. As well, all tests against -`sync.Map` are copied into this library and used against `cache.Cache`. +Out of an abundance of paranoia that this code is correct, the tests include +all stdlib `sync.Map` tests run against `cache.Cache`, a property test +against a mutex guarded map, a linearizability checker for concurrent +operations on a single key, structural checks of the trie after concurrent +stress, and regression tests for specific races. CI runs the tests with the +race detector on amd64 and arm64, and without it on 386. This package also provides a cached `Item` and a `Set`. `Item` can be used to populate an expensive value once and expire or replace it when needed, similar @@ -54,87 +57,40 @@ Documentation Benchmarking ------------ -Some of the APIs are actually faster or more memory efficient than `sync.Map` -because of generics. Some are slower and require more allocs because of the -request collapsing aspect. Some benchmarks below show more CPU, but this is due -to a few slight changes in how the write (dirty) map is internally promoted to -the read map. - -The benchmarks are from the stdlib using a key and value of type int: +These are the stdlib `sync.Map` benchmarks, run against `sync.Map` and against +a `cache.Cache[int, any]` adapted to the same interface. Both are hash-tries +underneath. The numbers are medians of 8 runs on an Apple M2 with Go 1.27; +the last column is allocs per op, `sync.Map` / cache. ``` -name old time/op new time/op delta -LoadMostlyHits 12.8ns ±44% 14.0ns ±50% ~ (p=0.393 n=10+10) -LoadMostlyMisses 10.7ns ±53% 4.6ns ±47% -57.31% (p=0.000 n=10+10) -LoadOrStoreBalanced 351ns ± 3% 705ns ±27% +100.68% (p=0.000 n=8+10) -LoadOrStoreUnique 663ns ±13% 1296ns ±31% +95.53% (p=0.000 n=10+10) -LoadOrStoreCollision 9.71ns ±60% 29.26ns ±46% +201.44% (p=0.000 n=10+10) -LoadAndDeleteBalanced 11.2ns ±42% 7.1ns ±48% -36.06% (p=0.035 n=10+10) -LoadAndDeleteUnique 9.19ns ±62% 4.17ns ±47% -54.59% (p=0.001 n=10+10) -LoadAndDeleteCollision 10.9ns ±41% 14.3ns ±35% +31.18% (p=0.023 n=10+10) -Range 3.73µs ±48% 5.40µs ±45% ~ (p=0.063 n=10+10) -AdversarialAlloc 298ns ±22% 259ns ±21% ~ (p=0.063 n=10+10) -AdversarialDelete 83.7ns ±14% 105.6ns ±20% ~ (p=0.052 n=10+10) -DeleteCollision 6.87ns ±71% 4.86ns ±100% ~ (p=0.424 n=10+10) -SwapCollision 201ns ±26% 266ns ±31% ~ (p=0.143 n=10+10) -SwapMostlyHits 40.3ns ±42% 59.7ns ±51% ~ (p=0.143 n=10+10) -SwapMostlyMisses 662ns ±18% 607ns ±24% -8.19% (p=0.023 n=10+10) -CompareAndSwapCollision 33.4ns ±41% 24.9ns ±65% -25.44% (p=0.023 n=10+10) -CompareAndSwapNoExistingKey 10.3ns ±46% 2.0ns ±53% -80.15% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 9.64ns ±51% 6.02ns ±95% -37.49% (p=0.019 n=10+10) -CompareAndSwapMostlyHits 46.9ns ±39% 56.8ns ±47% ~ (p=0.143 n=10+10) -CompareAndSwapMostlyMisses 22.9ns ±39% 13.4ns ±48% -41.48% (p=0.023 n=10+10) -CompareAndDeleteCollision 28.1ns ±47% 18.3ns ±37% -34.88% (p=0.019 n=10+10) -CompareAndDeleteMostlyHits 64.0ns ±38% 63.9ns ±52% ~ (p=0.165 n=10+10) -CompareAndDeleteMostlyMisses 18.3ns ±37% 8.7ns ±53% -52.65% (p=0.023 n=10+10) - -name old alloc/op new alloc/op delta -LoadMostlyHits 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadMostlyMisses 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadOrStoreBalanced 74.7B ±16% 195.3B ± 1% +161.45% (p=0.000 n=10+10) -LoadOrStoreUnique 159B ± 7% 330B ± 7% +107.15% (p=0.000 n=9+10) -LoadOrStoreCollision 0.00B 32.00B ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 4.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteUnique 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteCollision 1.00B ± 0% 1.00B ± 0% ~ (all equal) -Range 16.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) -AdversarialAlloc 53.4B ± 3% 64.8B ± 3% +21.35% (p=0.000 n=10+10) -AdversarialDelete 22.6B ±12% 39.0B ± 0% +72.57% (p=0.000 n=10+10) -DeleteCollision 0.00B 0.00B ~ (all equal) -SwapCollision 16.0B ± 0% 80.0B ± 0% +400.00% (p=0.000 n=10+10) -SwapMostlyHits 28.0B ± 0% 86.0B ± 0% +207.14% (p=0.000 n=10+10) -SwapMostlyMisses 124B ± 0% 136B ± 1% +9.71% (p=0.000 n=10+10) -CompareAndSwapCollision 7.60B ± 8% 21.70B ±17% +185.53% (p=0.000 n=10+10) -CompareAndSwapNoExistingKey 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 0.00B 0.00B ~ (all equal) -CompareAndSwapMostlyHits 33.0B ± 0% 91.0B ± 0% +175.76% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 23.0B ± 0% 16.0B ± 0% -30.43% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00B 2.00B ± 0% +Inf% (p=0.000 n=10+8) -CompareAndDeleteMostlyHits 39.0B ± 0% 90.6B ± 1% +132.31% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 16.0B ± 0% 8.0B ± 0% -50.00% (p=0.002 n=8+10) - -name old allocs/op new allocs/op delta -LoadMostlyHits 0.00 0.00 ~ (all equal) -LoadMostlyMisses 0.00 0.00 ~ (all equal) -LoadOrStoreBalanced 2.00 ± 0% 3.00 ± 0% +50.00% (p=0.000 n=10+10) -LoadOrStoreUnique 4.00 ± 0% 5.00 ± 0% +25.00% (p=0.000 n=10+10) -LoadOrStoreCollision 0.00 1.00 ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 0.00 0.00 ~ (all equal) -LoadAndDeleteUnique 0.60 ±100% 0.00 -100.00% (p=0.011 n=10+10) -LoadAndDeleteCollision 0.00 0.00 ~ (all equal) -Range 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialAlloc 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialDelete 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -DeleteCollision 0.00 0.00 ~ (all equal) -SwapCollision 1.00 ± 0% 1.00 ± 0% ~ (all equal) -SwapMostlyHits 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -SwapMostlyMisses 6.00 ± 0% 4.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapCollision 0.00 0.00 ~ (all equal) -CompareAndSwapNoExistingKey 0.50 ±100% 0.00 -100.00% (p=0.033 n=10+10) -CompareAndSwapValueNotEqual 0.00 0.00 ~ (all equal) -CompareAndSwapMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00 0.00 ~ (all equal) -CompareAndDeleteMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) +name sync.Map cache allocs +LoadMostlyHits 3.10ns 3.28ns 0 / 0 +LoadMostlyMisses 1.66ns 1.81ns 0 / 0 +LoadOrStoreBalanced 47.2ns 80.0ns 2 / 3 +LoadOrStoreUnique 95.1ns 121ns 3 / 4 +LoadOrStoreCollision 2.53ns 10.3ns 0 / 1 +LoadAndDeleteBalanced 2.05ns 1.86ns 0 / 0 +LoadAndDeleteUnique 2.13ns 0.65ns 0 / 0 +LoadAndDeleteCollision 3.43ns 2.15ns 0 / 0 +Range 1.45µs 2.25µs 0 / 0 +AdversarialAlloc 5.61ns 4.59ns 0 / 0 +AdversarialDelete 2.98ns 2.91ns 0 / 0 +DeleteCollision 1.90ns 1.66ns 0 / 0 +SwapCollision 122ns 258ns 1 / 1 +SwapMostlyHits 35.9ns 30.2ns 2 / 1 +SwapMostlyMisses 208ns 245ns 3 / 3 +CompareAndSwapCollision 6.02ns 9.11ns 0 / 0 +CompareAndSwapNoExistingKey 4.81ns 0.80ns 0 / 0 +CompareAndSwapValueNotEqual 3.69ns 3.07ns 0 / 0 +CompareAndSwapMostlyHits 38.3ns 27.7ns 3 / 2 +CompareAndSwapMostlyMisses 11.6ns 7.93ns 2 / 1 +CompareAndDeleteCollision 3.08ns 4.36ns 0 / 0 +CompareAndDeleteMostlyHits 43.5ns 62.9ns 3 / 3 +CompareAndDeleteMostlyMisses 5.25ns 5.31ns 0 / 1 +Clear 84.8ns 128ns 2 / 3 ``` + +Loads are within 10% of `sync.Map`, and deletes are faster. Stores cost more: +each stored value is a `loading` that also carries singleflight and expiry +state, and the adapter's `LoadOrStore` allocates a closure for its miss +function on every call. diff --git a/cache/autoclean_test.go b/cache/autoclean_test.go new file mode 100644 index 0000000..a7c26f6 --- /dev/null +++ b/cache/autoclean_test.go @@ -0,0 +1,108 @@ +package cache + +import ( + "runtime" + "testing" + "time" +) + +func TestAutoClean_RunsAndCleansExpired(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(2*time.Millisecond), + ) + defer c.StopAutoClean() + + c.Set("a", 1) + waitEntries(t, c, 0) +} + +// waitEntries waits until the trie holds n entries. +func waitEntries[K comparable, V any](t *testing.T, c *Cache[K, V], n int) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for trieEntries(c) != n { + if time.Now().After(deadline) { + t.Fatalf("trie has %d entries, want %d", trieEntries(c), n) + } + time.Sleep(time.Millisecond) + } +} + +func TestAutoClean_DisabledWhenStaleAgeNegative(t *testing.T) { + // AutoCleanInterval is ignored when MaxStaleAge < 0 (stales kept forever). + // We verify no goroutine is spawned by confirming StopAutoClean is a no-op + // on a cache whose quitClean channel was never created. + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(-1), + AutoCleanInterval(time.Millisecond), + ) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated when MaxStaleAge<0") + } + c.StopAutoClean() // no-op must not panic +} + +func TestAutoClean_StopIdempotent(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + c.StopAutoClean() + c.StopAutoClean() // sync.Once guards the close; second call must be safe. +} + +func TestAutoClean_ZeroIntervalNoGoroutine(t *testing.T) { + // autoCleanInterval == 0 means disabled; no goroutine, no channel. + c := New[string, int](MaxAge(time.Second)) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated with zero interval") + } + c.StopAutoClean() +} + +// Once StopAutoClean returns, no Clean runs. +func TestAutoClean_StopHaltsCleaning(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + c.Set("a", 1) + waitEntries(t, c, 0) + c.StopAutoClean() + + c.Set("b", 2) + time.Sleep(20 * time.Millisecond) + if n := trieEntries(c); n != 1 { + t.Fatalf("trie has %d entries after StopAutoClean, want 1", n) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("trie has %d entries after Clean, want 0", n) + } +} + +// A cache that is never stopped is still garbage collected, and its clean +// goroutine exits. +func TestAutoClean_ExitsWhenCacheCollected(t *testing.T) { + for _, newDone := range []func() chan struct{}{ + func() chan struct{} { return New[int, int](AutoCleanInterval(time.Millisecond)).cleanDone }, + func() chan struct{} { return NewItem[int](AutoCleanInterval(time.Millisecond)).c.cleanDone }, + func() chan struct{} { return NewSet[int](AutoCleanInterval(time.Millisecond)).c.cleanDone }, + } { + done := newDone() + deadline := time.Now().Add(10 * time.Second) + for exited := false; !exited; { + runtime.GC() + select { + case <-done: + exited = true + case <-time.After(10 * time.Millisecond): + if time.Now().After(deadline) { + t.Fatal("clean goroutine did not exit after its cache became unreachable") + } + } + } + } +} diff --git a/cache/cache.go b/cache/cache.go index 0e6d5af..4b14522 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,9 +1,9 @@ // Package cache provides a concurrency safe, mostly lock-free, singleflight // request collapsing generic cache with support for stale values. // -// The guts of this package are similar to `sync.Map`, with minor differences -// in when the internal locked write-map is promoted an atomic read-only map, -// and major differences to support singleflight request collapsing. +// The guts of this package are a concurrent hash-trie similar to the one +// backing `sync.Map`, with major differences to support singleflight request +// collapsing. // // Functions in this API that return a KeyState return the state last, rather // than the error last: the value and error are cached as a single internal @@ -16,9 +16,12 @@ package cache import ( + "errors" + "math" "sync" "sync/atomic" "time" + "weak" ) // KeyState is returned from cache operations to indicate how a key existed in @@ -50,38 +53,42 @@ type ( } loading[V any] struct { v V - err error - expires atomic.Int64 // nano at which this ent is unusable, if non-zero - stale *stale[V] + expires atomic.Int64 // see expiredClaimed for what values mean state atomic.Uint32 + wg sync.WaitGroup - wg sync.WaitGroup - mu sync.Mutex + // x is nil unless the load errored or the loading has a stale, + // which keeps the common loading small. + x atomic.Pointer[loadingExtra[V]] } - ent[V any] struct { - p atomic.Pointer[loading[V]] + + loadingWithExtra[V any] struct { + l loading[V] + x loadingExtra[V] } - read[K comparable, V any] struct { - m map[K]*ent[V] - incomplete bool + + loadingExtra[V any] struct { + err error // written before the loading is finalized + + // stale is the previous value, captured when a Get replaces an + // expired or errored loading. We return it while this load is in + // flight or if this load errors. It is set before the loading is + // published and never changes. + stale stale[V] + hasStale bool } // Cache caches comparable keys to arbitrary values. By default, the // cache grows without bounds and all keys persist forever. These // limits can be changed with options that are passed to New. Cache[K comparable, V any] struct { - r atomic.Pointer[read[K, V]] - mu sync.Mutex - dirty map[K]*ent[V] - - misses int + t trie[K, loading[V]] cfg cfg - pd *loading[V] // allocate the promotingDelete pointer once - quitOnce sync.Once quitClean chan struct{} + cleanDone chan struct{} } cfg struct { @@ -104,13 +111,54 @@ type ( const ( stateLoading uint32 = iota + stateFinalizing stateFinalized - statePromotingDelete ) -func now() int64 { return time.Now().UnixNano() } +// expiredClaimed is the loading.expires value Clean stores to claim an +// entry for deletion. The full set of values is: +// +// 0 loading, or loaded and never expires +// > 0 expires at this now() nano +// -2 claimed by Clean: expired, and its stale window is closed +// < -2 born expired (MaxAge(0), MaxErrorAge(0)): the negated now() at +// load, which is where the stale window starts +// +// Every negative value is expired, because now() is always positive. The +// epoch is padded by a second so that a negated now() is never -2. +const expiredClaimed = -2 + +// ent is what the cache stores in the trie. +type ent[K comparable, V any] = trieEntry[K, loading[V]] -func (cfg *cfg) newExpires(err error) int64 { +// We use the monotonic clock so that expiries do not move if the wall clock +// is stepped. Note that if the monotonic clock pauses while the system is +// suspended, entries only age while the system is running. +var epoch = time.Now().Add(-time.Second) + +// testNow, if non-zero, is what now returns, so that tests can freeze the +// clock. Never set in production code. +var testNow atomic.Int64 + +func now() int64 { + if n := testNow.Load(); n != 0 { + return n + } + return int64(time.Since(epoch)) +} + +// satAdd returns a+b capped at math.MaxInt64, so that a huge age does not +// wrap an expiry negative. b must be non-negative. +func satAdd(a, b int64) int64 { + if s := a + b; s >= a { + return s + } + return math.MaxInt64 +} + +// newExpires returns the expiry for a value or err loaded at n, sampling +// the clock only if it is needed and n is 0. +func (cfg *cfg) newExpires(err error, n int64) int64 { var ttl time.Duration var del0 bool if err != nil && cfg.errAgeSet { @@ -123,13 +171,16 @@ func (cfg *cfg) newExpires(err error) int64 { } del0 = cfg.ageSet && cfg.maxAge <= 0 } - if del0 { - return -1 - } - if ttl == 0 { + if !del0 && ttl == 0 { return 0 } - return time.Now().Add(ttl).UnixNano() + if n == 0 { + n = now() + } + if del0 { + return -n + } + return satAdd(n, int64(ttl)) } func (o opt) apply(c *cfg) { o.fn(c) } @@ -177,6 +228,8 @@ func MaxIdleAge(age time.Duration) Opt { // interval that is less than your MaxAge + MaxStaleAge. Values are only // candidates to be cleaned after the max age has elapsed. At worst, a value // may persist for a total of MaxAge + MaxStaleAge + AutoCleanInterval. +// +// The goroutine exits once the cache is garbage collected. func AutoCleanInterval(interval time.Duration) Opt { return opt{fn: func(c *cfg) { c.autoCleanInterval = interval }} } @@ -185,150 +238,215 @@ func AutoCleanInterval(interval time.Duration) Opt { // semantics. If you do not need to configure a cache at all, the zero value // cache is valid and usable. func New[K comparable, V any](opts ...Opt) *Cache[K, V] { - var cfg cfg + c := new(Cache[K, V]) + initCache(c, opts...) + return c +} + +// initCache initializes c in place so that NewItem and NewSet can initialize +// their embedded Cache; the autoclean goroutine references c. +func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { for _, opt := range opts { - opt.apply(&cfg) - } - c := &Cache[K, V]{ - cfg: cfg, - pd: new(loading[V]), + opt.apply(&c.cfg) } - c.pd.state.Store(statePromotingDelete) - if cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { + if c.cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { c.quitClean = make(chan struct{}) - go func() { - ticker := time.NewTicker(cfg.autoCleanInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - case <-c.quitClean: - return - } - c.Clean() - } - }() + c.cleanDone = make(chan struct{}) + go autoClean(weak.Make(c), c.cfg.autoCleanInterval, c.quitClean, c.cleanDone) } - return c } -// Get returns the cache value for k, running the miss function in a goroutine -// if the key is not yet cached. If stale values are enabled, the currently -// cached value has an error, and there is an unexpired stale value, this -// returns the stale value and no error. -func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { - r := c.read() - e := r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - return v, err, s - } - - // We missed in the read map. We lock and check again to guard against - // something concurrent. Odds are we are falling into the logic below. - c.mu.Lock() - r = c.read() - e = r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() - return v, err, s - } - - // We could have an entry in our read map that was deleted and has not - // yet gone through the promote&clear process. We only check the dirty - // map if the entry is nil. - if e == nil && r.incomplete { - e = c.dirty[k] - c.missed(r) - r = c.read() - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() - return v, err, s +// autoClean cleans the cache every interval. It holds the cache only +// weakly, so that a cache nobody stopped can still be garbage collected. +func autoClean[K comparable, V any](wc weak.Pointer[Cache[K, V]], interval time.Duration, quit, done chan struct{}) { + defer close(done) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + case <-quit: + return } + c := wc.Value() + if c == nil { + return + } + c.Clean() } +} - l := &loading[V]{ - stale: e.maybeNewStale(c.cfg.maxStaleAge), +// Get returns the cache value for k, running the miss function if the key is +// not yet cached. If stale values are enabled, the currently cached value has +// an error, and there is an unexpired stale value, this returns the stale +// value and no error. +// +// The miss function runs in the calling goroutine, unless Get returns a stale +// value: then it runs in a new goroutine and may still be running when Get +// returns. +func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { + e := c.t.loadEntry(k) + if e != nil { + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { + return v, err, s + } } - l.wg.Add(1) + return c.getSlow(k, e, miss, func() func() (V, error) { return miss }) +} - // If we have no entry, this is completely new. If we had an entry, it - // was in the read map and not yet deleted through promotion. We know - // the pointer is not promotingDelete, since that is only set while - // promoting which requires the cache lock which we have right now. - // - // In the worst case we race with a concurrent Set and we override its - // results. - if e != nil { - e.p.Store(l) - } else { - e = new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) - } - c.mu.Unlock() - - go func() { v, err := miss(); l.setve(v, err, c.cfg.newExpires(err)) }() - - // We could have set our own stale value which can be returned - // immediately rather than waiting for the get. If there is a valid - // stale to be returned now, `get` returns it. If `get` returns a Hit, - // we ended up waiting for ourself and we return a miss. There should - // be no Miss returns from `get`. - v, err, s = e.get(c.cfg.maxIdleAge) - switch s { - case Miss, Hit: - // We always return l.v and l.err. If this expires immediately, - // get may return no value. We want to return at least the - // value generated from this miss. - return l.v, l.err, Miss - } - return v, err, Stale -} - -func (c *Cache[K, V]) tryLoadEnt(k K, dirty func()) *ent[V] { - r := c.read() - e := r.m[k] - if e == nil && r.incomplete { - c.mu.Lock() - r = c.read() - e = r.m[k] - if e == nil && r.incomplete { - e = c.dirty[k] - if dirty != nil { - dirty() +// getSlow is Get after the hit check on e, which is k's entry or nil. It +// takes the miss function in two forms: load runs the load in this +// goroutine, and detach returns a function that runs the load after +// getSlow returns. We only call detach to refresh in the background, so +// load never escapes and callers only allocate for detach when we refresh. +func (c *Cache[K, V]) getSlow(k K, e *ent[K, V], load func() (V, error), detach func() func() (V, error)) (v V, err error, s KeyState) { + + // We need to load. We either create the entry with our loading, or + // replace the entry's finalized expired or errored loading with ours. + // A nil value is a deleted entry that is never written again (see + // entDel), so we unlink it and create a new entry. + var l *loading[V] +outer: + for { + if e == nil { + l = &loading[V]{} + l.wg.Add(1) + var loaded bool + if e, loaded = c.t.loadOrStoreEntry(k, l); !loaded { + break + } + } + for { + prev := e.p.Load() + if prev == nil { + c.t.deleteEntryIf(k, entDead[K, V]) + e = nil + continue outer + } + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { + return v, err, s + } + // entGet loads the value itself, so what it looked at may not + // be prev. We only return a stale if it came from prev while + // prev is still loading, and we only replace prev if prev + // itself is finalized and expired or errored. + if s == Stale && !prev.finalized() && e.p.Load() == prev { + return v, err, s + } + if !prev.finalized() || (prev.err() == nil && !prev.expired(now())) { + continue + } + if st, ok := maybeNewStale(prev, c.cfg.maxStaleAge); ok { + // One allocation for both: they have the same lifetime. + lx := &loadingWithExtra[V]{x: loadingExtra[V]{stale: st, hasStale: true}} + l = &lx.l + l.x.Store(&lx.x) + } else { + l = new(loading[V]) + } + l.wg.Add(1) + if e.p.CompareAndSwap(prev, l) { + break outer } - c.missed(r) } - c.mu.Unlock() } - return e + + if st := l.stale(); st != nil && !st.expired(now()) { + go c.load(l, detach()) + return st.v, nil, Stale + } + + // We return what our load produced, even if it is already expired: we + // do not re-derive a stale from it, and we do not extend its idle age, + // because we are a Miss. If a Swap replaced our load, the Swap may + // still be finalizing l, so we wait. + c.load(l, load) + l.wg.Wait() + return l.v, l.err(), Miss +} + +// errMissPanicked is what a load is finalized with if the miss function +// panics or exits its goroutine. It is born expired and never observable: +// every read of it is a Miss (or the previous value's stale), so waiting +// Gets run their own load. +var errMissPanicked = errors.New("cache: miss function panicked") + +func (c *Cache[K, V]) load(l *loading[V], miss func() (V, error)) { + var done bool + defer func() { + if !done { + var zero V + l.setve(zero, errMissPanicked, -now()) + } + }() + v, err := miss() + done = true + l.setve(v, err, c.cfg.newExpires(err, 0)) } // TryGet returns the value for the given key if it is cached. This returns -// either the currently stored value, or the current stale if the load errored, -// or the load error if there is no stale. If nothing is cached, or what is -// cached is expired, this returns Miss. +// either the currently stored value, or the current stale if the load errored +// or the value expired, or the load error if there is no stale. If nothing is +// cached, or what is cached is expired, this returns Miss. func (c *Cache[K, V]) TryGet(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, nil) - return e.tryGet(0, c.cfg.maxIdleAge) + e := c.t.loadEntry(k) + if e == nil { + var v V + return v, nil, Miss + } + return entTryGet(e, 0, c.cfg.maxIdleAge, c.cfg.maxStaleAge) } // Delete deletes the value for a key and returns the prior value, if stored // (i.e. the return from TryGet). func (c *Cache[K, V]) Delete(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, func() { delete(c.dirty, k) }) - defer e.del() - return e.tryGet(0, 0) + var v V + e := c.t.loadEntry(k) + if e == nil { + return v, nil, Miss + } + // We sample the clock before deleting so that we report the value as + // it was when we deleted it. + n := now() + was := entDel(e) + c.t.deleteEntryIf(k, entDead[K, V]) + if was == nil { + return v, nil, Miss + } + return loadingTryGet(was, n, 0, c.cfg.maxStaleAge) } // Expire sets a stored value to expire immediately, meaning the next Get will // be a miss. If stale values are enabled, the next Get will trigger the miss // function but still allow the now-stale value to be returned. func (c *Cache[K, V]) Expire(k K) { - e := c.tryLoadEnt(k, nil) - if l := e.load(); l != nil && l.finalized() { - l.expires.Store(now() - 1) + e := c.t.loadEntry(k) + if e == nil { + return + } + // We replace the loading with an expired copy rather than writing its + // expiry: a Set or Swap can replace the loading between our load and + // our write, and a write to the replaced loading would be lost. With + // the CAS, either we win and the Swap sees an expired value, or the + // Swap wins and we expire its value. + for { + l := e.p.Load() + if l == nil || !l.finalized() { + return + } + cur := l.expires.Load() + n := now() + if cur != 0 && cur <= n { + return // never move an expiry later + } + l2 := &loading[V]{v: l.v} + l2.x.Store(l.x.Load()) + l2.expires.Store(n - 1) + l2.state.Store(stateFinalized) + if e.p.CompareAndSwap(l, l2) { + return + } } } @@ -336,52 +454,48 @@ func (c *Cache[K, V]) Expire(k K) { func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // When ranging, repeated time.Now() calls add up, so we get the // current time when we enter range and avoid it in all tryGet calls. - now := now() - c.each(func(k K, e *ent[V]) bool { - v, err, s := e.tryGet(now, 0) + tn := now() + c.t.walk(func(e *ent[K, V]) bool { + v, err, s := entTryGet(e, tn, 0, c.cfg.maxStaleAge) if s.IsMiss() { return true } - return fn(k, v, err) + return fn(e.key, v, err) }) } -// Similar to sync.Map, we promote on range because range is O(N) (usually) and -// this amortizes out. -func (c *Cache[K, V]) each(fn func(K, *ent[V]) bool) { - r := c.read() - if r.incomplete { - c.mu.Lock() - r = c.read() - if r.incomplete { - c.promote() - r = c.read() - } - c.mu.Unlock() - } - for k, e := range r.m { - if !fn(k, e) { - return - } - } -} - // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually // expired a key. If MaxStaleAge is used and not -1, the entry must be older // than MaxAge + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. func (c *Cache[K, V]) Clean() { - // If MaxStaleAge is -1, the user opted into persisting stales forever - // and we do not clean. if c.cfg.maxStaleAge < 0 { return } - now := now() - c.each(func(k K, e *ent[V]) bool { - if l := e.load(); l != nil && l.finalized() { - expires := l.expires.Load() - if expires != 0 && now > expires+int64(c.cfg.maxStaleAge) { - c.Delete(k) + tn := now() + + // We delete exactly what TryGet would return Miss for: an expired value + // whose stale (if any) has also expired. Unlinking while we walk is + // safe: the walk holds its own references, and unlinking never moves + // an entry. + c.t.walk(func(e *ent[K, V]) bool { + l := e.p.Load() + if l == nil { + c.t.deleteEntryIf(e.key, entDead[K, V]) + return true + } + if !l.finalized() { + return true + } + expires := l.expires.Load() + expired := expires != 0 && expires <= tn + if st := l.stale(); expired && (st == nil || st.expired(tn)) && + (l.err() != nil || !staleOpen(expires, tn, c.cfg.maxStaleAge)) { + // We claim the expiry before deleting so that a racing idle + // extension, which CASes the expiry it read, fails. If the + // extension wins, the entry is in use and we leave it. + if l.expires.CompareAndSwap(expires, expiredClaimed) && e.p.CompareAndSwap(l, nil) { + c.t.deleteEntryIf(e.key, entDead[K, V]) } } return true @@ -390,11 +504,7 @@ func (c *Cache[K, V]) Clean() { // Clear deletes all keys from the cache, resetting it to an empty state. func (c *Cache[K, V]) Clear() { - c.mu.Lock() - c.dirty = nil - c.storeRead(read[K, V]{}) - c.misses = 0 - c.mu.Unlock() + c.t.clear() } // StopAutoClean stops the auto clean goroutine that is began with the @@ -403,6 +513,7 @@ func (c *Cache[K, V]) StopAutoClean() { c.quitOnce.Do(func() { if c.quitClean != nil { close(c.quitClean) + <-c.cleanDone } }) } @@ -413,85 +524,77 @@ func (c *Cache[K, V]) StopAutoClean() { // previous error if there is no stale. If nothing was cached, this returns // Miss. func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { - l := c.finalizedLoading(v) + // We use one clock sample for our value's expiry and for reporting + // what we replace. It predates whichever CAS succeeds. + n := now() + l := c.finalizedLoading(v, n) + // was is what we replaced, and wasN is the clock just before we did. var was *loading[V] + var wasN int64 defer func() { if was == nil { return } - - // The following block is similar to setve, but expanded to - // return our prior value (or stale, or err) if it exists. - now := now() if !was.finalized() { - was.mu.Lock() - if !was.finalized() { - if was.stale != nil && !was.stale.expired(now) { - old, oldState = was.stale.v, Stale + // If was is still loading, we finalize it with our value so + // that Gets waiting on it return our value. + if was.setve(v, nil, l.expires.Load()) { + if st := was.stale(); st != nil && !st.expired(wasN) { + old, oldState = st.v, Stale } - was.v = v - was.expires.Store(c.cfg.newExpires(nil)) - was.state.Store(stateFinalized) - was.wg.Done() - was.mu.Unlock() return } - was.mu.Unlock() + was.wg.Wait() // the load finalized was first } - if expired := was.expired(now); was.err != nil || expired { - if was.stale != nil && !was.stale.expired(now) { - old, oldState = was.stale.v, Stale - } - if expired { - return - } - } - old, oldErr, oldState = was.v, was.err, Hit + old, oldErr, oldState = classifyFinalized(was, was.expires.Load(), wasN, c.cfg.maxStaleAge) }() - r := c.read() - if e, ok := r.m[k]; ok { + if e := c.t.loadEntry(k); e != nil { + if rm := e.p.Load(); rm != nil { + was, wasN = rm, n + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState + } + } + } + for { + e, loaded := c.t.loadOrStoreEntry(k, l) + if !loaded { + was = nil + return old, oldErr, oldState + } for { rm := e.p.Load() - if rm != nil && rm.promotingDelete() { // deleted & currently being ignored in a promote + if rm == nil { + c.t.deleteEntryIf(k, entDead[K, V]) break } - was = rm + was, wasN = rm, n if e.p.CompareAndSwap(rm, l) { return old, oldErr, oldState } } } - - c.mu.Lock() - r = c.read() - if e := r.m[k]; e != nil { - was = e.p.Swap(l) // was not in read, but promoted by the time we entered the lock and is now in read - } else if e = c.dirty[k]; e != nil { - was = e.p.Swap(l) - } else { - e := new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) - } - c.mu.Unlock() - return old, oldErr, oldState } -func (l *loading[V]) setve(v V, err error, expires int64) { - if l.finalized() { - return +// setve finalizes l if it is still loading, returning whether we did. +func (l *loading[V]) setve(v V, err error, expires int64) bool { + if !l.state.CompareAndSwap(stateLoading, stateFinalizing) { + return false } - l.mu.Lock() - defer l.mu.Unlock() - if l.finalized() { - return + l.v = v + if err != nil { + if x := l.x.Load(); x != nil { + x.err = err + } else { + l.x.Store(&loadingExtra[V]{err: err}) + } } - l.v, l.err = v, err l.expires.Store(expires) l.state.Store(stateFinalized) l.wg.Done() + return true } // Set sets a value for a key. If the key is currently loading via Get, the @@ -501,303 +604,314 @@ func (c *Cache[K, V]) Set(k K, v V) { } // CompareAndSwap swaps the old and new values for k if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading, is not expired or errored, and is equal to old. The type V must be +// comparable. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { - r := c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if !r.incomplete { + e := c.t.loadEntry(k) + if e == nil { return false } - c.mu.Lock() - defer c.mu.Unlock() - r = c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if e, ok := c.dirty[k]; ok { - defer c.missed(r) - return c.tryCAS(e, old, new, true) - } - return false + return c.tryCAS(e, old, new, true) } -// CompareAndDelete deletes the entry for k if the value has finished loading -// and the value is equal to old. The type V must be comparable. +// CompareAndDelete deletes the entry for k if the value has finished loading, +// is not expired or errored, and is equal to old. The type V must be +// comparable. func (c *Cache[K, V]) CompareAndDelete(k K, old V) (deleted bool) { - r := c.read() - e, ok := r.m[k] - if !ok && r.incomplete { - c.mu.Lock() - r = c.read() - e, ok = r.m[k] - if !ok && r.incomplete { - e, ok = c.dirty[k] - c.missed(r) - } - c.mu.Unlock() + e := c.t.loadEntry(k) + if e == nil { + return false } - if ok { - return c.tryCAS(e, old, old, false) + if !c.tryCAS(e, old, old, false) { + return false } - return false + c.t.deleteEntryIf(k, entDead[K, V]) + return true } -func (c *Cache[K, V]) tryCAS(e *ent[V], old, new V, useNew bool) bool { +func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { l := e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { + if !casMatches(l, old) { return false } var l2 *loading[V] if useNew { - l2 = c.finalizedLoading(new) + l2 = c.finalizedLoading(new, 0) + } + if casPublishHook != nil { + casPublishHook() } for { + // We check again after allocating: if we were descheduled, the + // value may have expired. + if !casMatches(l, old) { + return false + } if e.p.CompareAndSwap(l, l2) { return true } l = e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { - return false - } } } -////////////////////// -// CACHE READ/DIRTY // -////////////////////// - -func (c *Cache[K, V]) read() read[K, V] { - p := c.r.Load() - if p == nil { - return read[K, V]{} +// casMatches returns whether l is a loaded, unerrored, unexpired value equal +// to old, i.e. whether TryGet would return old as a Hit. +func casMatches[V any](l *loading[V], old V) bool { + if l == nil || !l.finalized() || l.err() != nil || any(l.v) != any(old) { + return false } - return *p -} -func (c *Cache[K, V]) storeRead(r read[K, V]) { c.r.Store(&r) } - -func (c *Cache[K, V]) storeDirty(r read[K, V], k K, e *ent[V]) { - if !r.incomplete { - if c.dirty == nil { - c.dirty = make(map[K]*ent[V]) - } - c.storeRead(read[K, V]{m: r.m, incomplete: true}) + expires := l.expires.Load() + if expires == 0 { + return true } - c.dirty[k] = e + expires, n := l.expiresNow(expires) + return expires == 0 || expires > n } -func (c *Cache[K, V]) missed(r read[K, V]) { - c.misses++ - if len(c.dirty) == 0 || c.misses > len(r.m)>>1 { - c.promote() +/////////////////// +// ENTRY HELPERS // +/////////////////// + +func (c *Cache[K, V]) finalizedLoading(v V, n int64) *loading[V] { + l := &loading[V]{v: v} + if expires := c.cfg.newExpires(nil, n); expires != 0 { + l.expires.Store(expires) } + l.state.Store(stateFinalized) + return l } -func (c *Cache[K, V]) promote() { - r := c.read() - keep := r.m - - defer func() { - c.storeRead(read[K, V]{m: keep}) - c.misses = 0 - }() +func (l *loading[V]) finalized() bool { return l.state.Load() == stateFinalized } - if len(c.dirty) == 0 { - return +// err returns the load's error. l must be finalized. +func (l *loading[V]) err() error { + if x := l.x.Load(); x != nil { + return x.err } + return nil +} - keep = make(map[K]*ent[V], len(keep)+len(c.dirty)) - for k, e := range c.dirty { - keep[k] = e +// stale returns the loading's stale, or nil if it has none. +func (l *loading[V]) stale() *stale[V] { + if x := l.x.Load(); x != nil && x.hasStale { + return &x.stale } - c.dirty = nil + return nil +} -outer: - for k, e := range r.m { - p := e.p.Load() - for p == nil { - if e.p.CompareAndSwap(nil, c.pd) { - continue outer - } - p = e.p.Load() // concurrently deleted while promoting - } - if p != c.pd { - keep[k] = e - } - } +// entDel deletes the entry's value, returning what it was. +// +// A nil value is permanent: nothing stores a value into an entry after it +// is nil. Anything that finds a nil value unlinks the entry and creates a +// new one. This is what makes entDead a safe deleteEntryIf predicate: a +// live entry can never be unlinked. +func entDel[K comparable, V any](e *ent[K, V]) *loading[V] { + return e.p.Swap(nil) } -///////// -// ENT // -///////// +func entDead[K comparable, V any](e *ent[K, V]) bool { return e.p.Load() == nil } -func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { - l := &loading[V]{ - v: v, - } - expires := c.cfg.newExpires(nil) - if expires != 0 || c.cfg.maxStaleAge != 0 { - l.expires.Store(expires) - if c.cfg.maxStaleAge != 0 { - l.stale = newStale(v, l.expires.Load(), c.cfg.maxStaleAge) - } - } - l.state.Store(stateFinalized) - return l +func (l *loading[V]) expired(n int64) bool { + expires := l.expires.Load() + return expires != 0 && expires <= n // 0 means either miss not resolved, or no max age } -func (l *loading[V]) promotingDelete() bool { return l.state.Load() == 2 } -func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -func (l *loading[V]) onlyFinalized() bool { return l.state.Load() == 1 } +func (s *stale[V]) expired(n int64) bool { + expires := s.expires + return expires != 0 && expires <= n +} -func (e *ent[V]) del() { - if e == nil { - return +// maybeNewStale returns the stale to carry in a loading that replaces the +// finalized l: l's own stale if l errored, otherwise l's value with the +// stale window l's value has (see staleOpen). +func maybeNewStale[V any](l *loading[V], age time.Duration) (stale[V], bool) { + if age == 0 { + return stale[V]{}, false } - for { - p := e.p.Load() - if p == nil || p.promotingDelete() { - return - } - if e.p.CompareAndSwap(p, nil) { - return + if l.err() != nil { + if st := l.stale(); st != nil { + return *st, true } + return stale[V]{}, false + } + expires := l.expires.Load() + if expires == expiredClaimed { + return stale[V]{}, false } + return newStale(l.v, expires, age), true } -func (e *ent[V]) load() *loading[V] { - if e == nil { - return nil +// staleOpen returns whether an expired, unerrored value can still be +// returned as stale at n. expires must be non-zero. This must agree with +// the window newStale creates. +func staleOpen(expires, n int64, age time.Duration) bool { + if age == 0 || expires == expiredClaimed { + return false + } + if age < 0 { + return true } - p := e.p.Load() - if p == nil || p.promotingDelete() { - return nil + if expires < 0 { + expires = -expires // born expired } - return p + return n < satAdd(expires, int64(age)) } -func (l *loading[V]) expired(now int64) bool { - expires := l.expires.Load() - return expires != 0 && expires <= now // 0 means either miss not resolved, or no max age +// newStale returns a stale for v, whose loading's expires is expires. age +// must be non-zero, and expires must be an expired value's: non-zero and +// not expiredClaimed. +func newStale[V any](v V, expires int64, age time.Duration) stale[V] { + if age < 0 { + return stale[V]{v: v} + } + if expires < 0 { + expires = -expires // born expired + } + return stale[V]{v, satAdd(expires, int64(age))} } -func (s *stale[V]) expired(now int64) bool { - expires := s.expires - return expires != 0 && expires <= now -} +// expiresPairingHook, if non-nil, runs in expiresNow between the expiry load +// and the clock sample, so that tests can pause a reader there. +var expiresPairingHook func() -// If an entry is expiring, we create a new entry with this previous entry as -// a stale. +// casPublishHook, if non-nil, runs in tryCAS before the final liveness +// check, so that tests can pause a CAS there. +var casPublishHook func() + +// expiresNow returns l's expiry and a clock sample such that the expiry was +// current when the clock was sampled. expires is the caller's load of the +// expiry. // -// - if entry is nil, no stale, return nil -// - if no stale age, we are not using stales, return nil -// - if entry has an error, return prior stale -// - if age is < 0, return new unexpiring stale -// - else, return new stale with prior expiry + stale age -func (e *ent[V]) maybeNewStale(age time.Duration) *stale[V] { - if e == nil || age == 0 { - return nil +// We load the expiry before sampling the clock: Expire stores its own +// now()-1, which may be later than a clock sampled before our load. We then +// load the expiry again and retry if it changed: otherwise, an idle +// extension landing between the load and the sample would pair a dead +// expiry with a clock at which the value is live, and an Expire landing +// there would let CompareAndSwap match a value that was already expired. +func (l *loading[V]) expiresNow(expires int64) (int64, int64) { + if expiresPairingHook != nil { + expiresPairingHook() + } + for { + n := now() + again := l.expires.Load() + if again == expires { + return expires, n + } + expires = again } - l := e.load() - if l == nil || !l.finalized() { - return nil +} + +// classifyFinalized returns what a read of l at n returns. A valid stale +// takes precedence over an error; an unexpired error is returned as a Hit; +// an expired value is returned as Stale while its stale window is open. +func classifyFinalized[V any](l *loading[V], expires, n int64, staleAge time.Duration) (v V, err error, state KeyState) { + expired := expires != 0 && expires <= n + if lerr := l.err(); lerr != nil { + if st := l.stale(); st != nil && !st.expired(n) { + return st.v, nil, Stale + } + if !expired { + return l.v, lerr, Hit + } + return v, err, state } - if l.err != nil { - return l.stale + if expired { + if staleOpen(expires, n, staleAge) { + return l.v, nil, Stale + } + return v, err, state } - return newStale(l.v, l.expires.Load(), age) + return l.v, nil, Hit } -// Actually returns the stale; age must be non-zero. -func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { - if age < 0 { - return &stale[V]{v: v} +// maxIdleSlack bounds how far we let an idle extension drift from now+age +// before we write it; see MaxIdleAge. +const maxIdleSlack = time.Second + +// extend resets l's expiry to n+age if the expiry is still what we read. +// We skip the write if it would move the expiry by less than the slack, so +// that a hot key does not write its expiry on every hit. We sample the +// clock again right before the CAS so that we do not revive a value that +// expired while we were descheduled. +func (l *loading[V]) extend(expires, n int64, age time.Duration) { + target := satAdd(n, int64(age)) + slack := min(int64(age)>>6, int64(maxIdleSlack)) + if d := target - expires; d < slack && d > -slack { + return + } + if now() < expires { + l.expires.CompareAndSwap(expires, target) } - return &stale[V]{v, expires + int64(age)} } -// get always returns the value or the stale value. We do not check if our -// value is expired: we call this at the end of Get, we must always return -// something even if it is to be immediately expired. -func (e *ent[V]) get(extend time.Duration) (v V, err error, state KeyState) { - l := e.load() - var waited bool +// entGet is TryGet, but if the entry is loading with no valid stale, we +// wait for the load. If we waited and the load's result is already expired +// (MaxAge(0), MaxErrorAge(0), or a TTL shorter than the load), we return +// it anyway as a Hit: every Get that collapsed onto a load gets its result. +// The one exception is a panicked load, which has no result to share. +func entGet[K comparable, V any](e *ent[K, V], extend, staleAge time.Duration) (v V, err error, state KeyState) { + l := e.p.Load() if l == nil { - // Could be nil if deleted while in the read map, or - // promotingDelete. return v, err, state } + var waited bool if !l.finalized() { - if l.stale != nil && !l.stale.expired(now()) { - return l.stale.v, nil, Stale + if st := l.stale(); st != nil && !st.expired(now()) { + return st.v, nil, Stale } l.wg.Wait() waited = true } - - // If we did not wait and our entry is expired (value or error), or if - // our entry is not expired but has errored, we potentially return the - // stale entry. - // - // If we waited, we could immediately be expired due to time sync, or - // if the user is configured to never cache and they're just using - // request collapsing: we still want to return the now expired value. - now := now() - if (!waited && l.expired(now)) || l.err != nil { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale - } - // The stale value is expired: if our entry is not expired, - // this must be an error we waited on. - if !l.expired(now) { - return l.v, l.err, Hit - } - return v, err, state + expires := l.expires.Load() + if expires == 0 && l.x.Load() == nil { + return l.v, nil, Hit } - if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + expires, n := l.expiresNow(expires) + if waited && expires != 0 && expires <= n && l.err() != errMissPanicked { + return l.v, l.err(), Hit } - return l.v, l.err, Hit + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && err == nil { + l.extend(expires, n, extend) + } + return v, err, state } -func (e *ent[V]) tryGet(n64 int64, extend time.Duration) (v V, err error, state KeyState) { - if e == nil { - return v, err, state - } - l := e.load() - if l == nil { // deleting or promotedDelete +func entTryGet[K comparable, V any](e *ent[K, V], n int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { + l := e.p.Load() + if l == nil { return v, err, state } - // Fast path: finalized, no expiry, no error — skip time checks. - if l.onlyFinalized() && l.expires.Load() == 0 && l.err == nil { - return l.v, nil, Hit - } - if n64 == 0 { - n64 = now() - } - now := n64 + return loadingTryGet(l, n, extend, staleAge) +} - // If we are loading but there is a valid stale, return it, otherwise - // return immediately: no get. +// loadingTryGet is TryGet for a loading. If n is non-zero, it is the clock +// to use, and extend must be zero. +func loadingTryGet[V any](l *loading[V], n int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { if !l.finalized() { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale + if n == 0 { + n = now() + } + if st := l.stale(); st != nil && !st.expired(n) { + return st.v, nil, Stale } return v, err, state } - - // If we have an error or we are expired, we maybe return the stale. - if expired := l.expired(now); l.err != nil || expired { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale - } - if expired { - return v, err, state - } + // If the caller passed n, it sampled n before this load, which is the + // order expiresNow uses. + expires := l.expires.Load() + if expires == 0 && l.x.Load() == nil { + return l.v, nil, Hit } - if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + if n == 0 { + expires, n = l.expiresNow(expires) } - return l.v, l.err, Hit + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && err == nil { + l.extend(expires, n, extend) + } + return v, err, state } ////////// @@ -815,21 +929,15 @@ type Item[V any] struct { // semantics. If you do not need to configure an item at all, the zero value // item is valid and usable. func NewItem[V any](opts ...Opt) *Item[V] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Item[V]{ - c: Cache[struct{}, V]{ - cfg: c, - }, - } + i := new(Item[V]) + initCache(&i.c, opts...) + return i } -// Get returns the currently cached value, running the miss function in a -// goroutine if the item is not yet cached. If stale values are enabled, the -// currently cached value has an error, and there is an unexpired stale value, -// this returns the stale value and no error. +// Get returns the currently cached value, running the miss function if the +// item is not yet cached. If stale values are enabled, the currently cached +// value has an error, and there is an unexpired stale value, this returns the +// stale value and no error. func (i *Item[V]) Get(miss func() (V, error)) (v V, err error, state KeyState) { return i.c.Get(struct{}{}, miss) } @@ -871,13 +979,14 @@ func (i *Item[V]) Swap(v V) (old V, oldErr error, oldState KeyState) { } // CompareAndSwap swaps the old and new values if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading, is not expired or errored, and is equal to old. The type V must be +// comparable. func (i *Item[V]) CompareAndSwap(old, new V) (swapped bool) { return i.c.CompareAndSwap(struct{}{}, old, new) } -// CompareAndDelete deletes the item if the value has finished loading and the -// value is equal to old. The type V must be comparable. +// CompareAndDelete deletes the item if the value has finished loading, is not +// expired or errored, and is equal to old. The type V must be comparable. func (i *Item[V]) CompareAndDelete(old V) (deleted bool) { return i.c.CompareAndDelete(struct{}{}, old) } @@ -902,24 +1011,26 @@ type Set[K comparable] struct { // semantics. If you do not need to configure an set at all, the zero value set // is valid and usable. func NewSet[K comparable](opts ...Opt) *Set[K] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Set[K]{ - c: Cache[K, struct{}]{ - cfg: c, - }, - } + s := new(Set[K]) + initCache(&s.c, opts...) + return s } -// Get ensures the key is cached, running the miss function in a goroutine if -// the key is not yet cached. If stale keys are enabled, the currently cached -// key has an error, and there is a stale key, this returns with no error and a -// Stale key state. +// Get ensures the key is cached, running the miss function if the key is not +// yet cached. If stale keys are enabled, the currently cached key has an +// error, and there is a stale key, this returns with no error and a Stale key +// state. func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { - _, err, state = s.c.Get(k, func() (struct{}, error) { - return struct{}{}, miss() + c := &s.c + e := c.t.loadEntry(k) + if e != nil { + if _, err, state = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); state == Hit { + return err, state + } + } + load := func() (struct{}, error) { return struct{}{}, miss() } + _, err, state = c.getSlow(k, e, load, func() func() (struct{}, error) { + return func() (struct{}, error) { return struct{}{}, miss() } }) return err, state } diff --git a/cache/cache_bench_test.go b/cache/cache_bench_test.go new file mode 100644 index 0000000..df49412 --- /dev/null +++ b/cache/cache_bench_test.go @@ -0,0 +1,151 @@ +package cache + +import ( + "runtime" + "sync/atomic" + "testing" + "time" +) + +func benchGetHit(b *testing.B, opts ...Opt) { + c := New[int, int](opts...) + defer c.StopAutoClean() + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.Get(i%keys, miss) + i++ + } + }) +} + +func BenchmarkGetHit(b *testing.B) { benchGetHit(b) } +func BenchmarkGetHitMaxAge(b *testing.B) { benchGetHit(b, MaxAge(time.Hour)) } +func BenchmarkGetHitIdleAge(b *testing.B) { benchGetHit(b, MaxIdleAge(time.Hour)) } + +func BenchmarkGetHitOneKey(b *testing.B) { benchGetHitOneKey(b) } +func BenchmarkGetHitOneKeyIdleAge(b *testing.B) { benchGetHitOneKey(b, MaxIdleAge(time.Hour)) } + +func benchGetHitOneKey(b *testing.B, opts ...Opt) { + c := New[int, int](opts...) + c.Set(0, 0) + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + c.Get(0, miss) + } + }) +} + +func BenchmarkTryGetHitMaxAge(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.TryGet(i % keys) + i++ + } + }) +} + +func BenchmarkGetMissUnique(b *testing.B) { + c := New[int, int]() + var ctr atomic.Int64 + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + c.Get(int(ctr.Add(1)), miss) + } + }) +} + +func BenchmarkGetRefreshExpired(b *testing.B) { + c := New[int, int](MaxAge(0)) + miss := func() (int, error) { return 0, nil } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.Get(i%1024, miss) + i++ + } + }) +} + +func BenchmarkSetExisting(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + c.Set(i%keys, i) + i++ + } + }) +} + +func BenchmarkDeleteSet(b *testing.B) { + c := New[int, int]() + var ctr atomic.Int64 + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + k := int(ctr.Add(1)) % 4096 + c.Set(k, k) + c.Delete(k) + } + }) +} + +func BenchmarkClean(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + for i := range 4096 { + c.Set(i, i) + } + b.ResetTimer() + for range b.N { + c.Clean() + } +} + +// BenchmarkGetRefreshStale measures a refresh that returns the stale value +// while the load runs in the background, waiting for each load to finish. +func BenchmarkGetRefreshStale(b *testing.B) { + c := New[int, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set(0, 0) + miss := func() (int, error) { return 0, nil } + for range b.N { + c.Get(0, miss) + for !c.t.loadEntry(0).p.Load().finalized() { + runtime.Gosched() + } + } +} + +// BenchmarkGetHitCapturingMiss is GetHit with the miss closure a real caller +// writes: one that captures the key. +func BenchmarkGetHitCapturingMiss(b *testing.B) { + c := New[int, int](MaxAge(time.Hour)) + const keys = 1024 + for i := range keys { + c.Set(i, i) + } + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + k := i % keys + c.Get(k, func() (int, error) { return k, nil }) + i++ + } + }) +} diff --git a/cache/cache_test.go b/cache/cache_test.go index d170261..89053bc 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -2,6 +2,8 @@ package cache import ( "errors" + "math" + "runtime" "sync" "sync/atomic" "testing" @@ -58,7 +60,7 @@ func TestCollapsedGet(t *testing.T) { calls int64 ps = make(chan *int) ) - for i := 0; i < niter; i++ { + for range niter { go func() { p, _, _ := c.Get("foo", func() (*int, error) { if atomic.AddInt64(&calls, 1) != 1 { @@ -70,7 +72,7 @@ func TestCollapsedGet(t *testing.T) { ps <- p }() } - for i := 0; i < niter; i++ { + for range niter { p := <-ps if p != r { t.Error("pointer mismatch") @@ -118,19 +120,18 @@ func TestSetExpire(t *testing.T) { c.Delete(k) c.Set(k, 10) - for i := 0; i < 2; i++ { + for range 2 { var wg sync.WaitGroup - wg.Add(10) - go func() { defer wg.Done(); c.Range(func(string, int, error) bool { return false }) }() - go func() { defer wg.Done(); c.Delete(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() - go func() { defer wg.Done(); c.Expire(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() - go func() { defer wg.Done(); c.Range(func(string, int, error) bool { return false }) }() - go func() { defer wg.Done(); c.Delete(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() - go func() { defer wg.Done(); c.Expire(k) }() - go func() { defer wg.Done(); c.Set(k, 10) }() + wg.Go(func() { c.Range(func(string, int, error) bool { return false }) }) + wg.Go(func() { c.Delete(k) }) + wg.Go(func() { c.Set(k, 10) }) + wg.Go(func() { c.Expire(k) }) + wg.Go(func() { c.Set(k, 10) }) + wg.Go(func() { c.Range(func(string, int, error) bool { return false }) }) + wg.Go(func() { c.Delete(k) }) + wg.Go(func() { c.Set(k, 10) }) + wg.Go(func() { c.Expire(k) }) + wg.Go(func() { c.Set(k, 10) }) wg.Wait() v, err, s = c.Get(k, func() (int, error) { return 10, nil }) check(got[int]{v, err, s}, 10) @@ -177,10 +178,14 @@ func TestExpires(t *testing.T) { close(ch) // Here, we check that the stale is returned because it is - // expired. We sleep 10ms to allow the goroutines running the - // prior miss to finish and update our value to baz. - time.Sleep(10 * time.Millisecond) + // expired. We wait for the prior miss to finish and update our + // value to baz, and block the new miss so that it cannot finish + // before Get checks for a stale. + waitFinalized(t, &c.t, "foo") + ch = make(chan struct{}) + defer close(ch) v, err, s = c.Get("foo", func() (string, error) { + <-ch return "bar", nil }) vcheck(t, got[string]{v, err, s}, got[string]{"baz", nil, Stale}) @@ -196,7 +201,7 @@ func TestExpires(t *testing.T) { c.Expire("foo") time.Sleep(time.Millisecond) c.Get("foo", func() (string, error) { return "", errors.New("foo") }) - time.Sleep(time.Millisecond) + waitFinalized(t, &c.t, "foo") // The cache is primed: foo exists, it was expired, and we // injected an error such that the next loads should be stale. @@ -213,7 +218,7 @@ func TestExpires(t *testing.T) { v, err, s = c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Stale}) - ch <- struct{}{} + close(ch) } // Misc. bits. @@ -282,7 +287,7 @@ func TestExpires(t *testing.T) { gch := make(chan got[string], 10) ch := make(chan struct{}) - for i := 0; i < 10; i++ { + for range 10 { go func() { v, err, s := c.Get("foo", func() (string, error) { <-ch @@ -296,7 +301,7 @@ func TestExpires(t *testing.T) { // long, so not all Get's could be stacked at once. time.Sleep(100 * time.Millisecond) close(ch) - for i := 0; i < 10; i++ { + for range 10 { g := <-gch if g.s == Miss { vcheck(t, g, got[string]{"bar", nil, Miss}) @@ -361,79 +366,705 @@ func TestExpires(t *testing.T) { if len(exp) != 0 { t.Errorf("unexpected values remain: %v", exp) } + if n := trieEntries(c); n != 4 { + t.Errorf("Clean left %d entries, want 4", n) + } } } func TestMaxIdleAge(t *testing.T) { - // MaxIdleAge alone: entry stays alive while accessed within the window, - // expires after inactivity. + expiresOf := func(c *Cache[string, string]) int64 { + return c.t.loadEntry("foo").p.Load().expires.Load() + } + // tick ensures the next now() differs from the last. + tick := func() { time.Sleep(time.Millisecond) } + + // Hits from TryGet and Get extend. A 6.4s idle age has a 100ms slack + // (age/64), so we wait past it between hits. t.Run("alone", func(t *testing.T) { - c := New[string, string](MaxIdleAge(50 * time.Millisecond)) + const idle = 6400 * time.Millisecond + pastSlack := func() { time.Sleep(idle/64 + 10*time.Millisecond) } + c := New[string, string](MaxIdleAge(idle)) c.Get("foo", func() (string, error) { return "bar", nil }) - time.Sleep(10 * time.Millisecond) - - // Access within the idle window — should still be a hit and extend. - for i := 0; i < 5; i++ { - v, err, s := c.TryGet("foo") - vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) - time.Sleep(30 * time.Millisecond) // each access resets the 50ms idle + w0 := expiresOf(c) + pastSlack() + v, err, s := c.TryGet("foo") + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) + w1 := expiresOf(c) + if w1 <= w0 { + t.Fatalf("TryGet did not extend: %d <= %d", w1, w0) + } + pastSlack() + v, err, s = c.Get("foo", func() (string, error) { return "baz", nil }) + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) + if w2 := expiresOf(c); w2 <= w1 { + t.Fatalf("Get did not extend: %d <= %d", w2, w1) } + }) - // Now stop accessing. After 50ms of inactivity, it should expire. - time.Sleep(60 * time.Millisecond) + // Without accesses, the idle age is the TTL. + t.Run("lapses", func(t *testing.T) { + const idle = 20 * time.Millisecond + c := New[string, string](MaxIdleAge(idle)) + c.Get("foo", func() (string, error) { return "bar", nil }) + time.Sleep(3 * idle) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) - // MaxIdleAge + MaxAge: initial expiry uses MaxAge, subsequent accesses - // extend using MaxIdleAge. + // With MaxAge, an access resets the expiry to now + idle age, whether + // that is later or earlier than the MaxAge expiry. t.Run("with_max_age", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + c := New[string, string](MaxAge(time.Minute), MaxIdleAge(time.Hour)) c.Get("foo", func() (string, error) { return "bar", nil }) - - // Keep alive well past the original MaxAge. - for i := 0; i < 5; i++ { - time.Sleep(30 * time.Millisecond) - v, err, s := c.Get("foo", func() (string, error) { return "baz", nil }) - vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) + w0 := expiresOf(c) + c.TryGet("foo") + if w1 := expiresOf(c); w1 < w0+int64(50*time.Minute) { + t.Fatalf("access did not extend past MaxAge: %d vs %d", w1, w0) } - // Stop accessing, should expire. - time.Sleep(60 * time.Millisecond) - v, err, s := c.TryGet("foo") - vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) + c = New[string, string](MaxAge(time.Hour), MaxIdleAge(time.Minute)) + c.Get("foo", func() (string, error) { return "bar", nil }) + w0 = expiresOf(c) + c.TryGet("foo") + if w1 := expiresOf(c); w1 > w0-int64(50*time.Minute) { + t.Fatalf("access did not shorten to the idle age: %d vs %d", w1, w0) + } }) // MaxIdleAge doesn't extend errors. t.Run("no_extend_errors", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + c := New[string, string](MaxAge(time.Hour), MaxIdleAge(time.Hour)) c.Get("foo", func() (string, error) { return "", errors.New("err") }) - time.Sleep(10 * time.Millisecond) - - // Access the errored entry — it should not be extended. + w0 := expiresOf(c) + tick() v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", errors.New("err"), Hit}) + c.Get("foo", func() (string, error) { return "bar", nil }) + if w1 := expiresOf(c); w1 != w0 { + t.Fatalf("errored entry was extended: %d != %d", w1, w0) + } + }) - // Wait past expiry — should be gone. - time.Sleep(50 * time.Millisecond) + // MaxIdleAge must not resurrect entries that finalize already expired: + // with MaxAge(0), the Get that drives the load returns the value but + // does not extend it. + t.Run("no_resurrect_expired", func(t *testing.T) { + c := New[string, string](MaxAge(0), MaxIdleAge(time.Hour)) + v, err, s := c.Get("foo", func() (string, error) { return "bar", nil }) + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Miss}) v, err, s = c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) // Range doesn't extend. t.Run("range_no_extend", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + c := New[string, string](MaxAge(time.Hour), MaxIdleAge(time.Hour)) c.Get("foo", func() (string, error) { return "bar", nil }) + w0 := expiresOf(c) + tick() + var n int + c.Range(func(string, string, error) bool { n++; return true }) + if n != 1 { + t.Fatalf("Range saw %d keys, want 1", n) + } + if w1 := expiresOf(c); w1 != w0 { + t.Fatalf("Range extended: %d != %d", w1, w0) + } + }) +} - // Range over entries repeatedly — should NOT extend expiry. - for i := 0; i < 3; i++ { - time.Sleep(20 * time.Millisecond) - c.Range(func(string, string, error) bool { return true }) +// TestMaxErrorAge_DefaultsToIdleInitialTTL checks that with only MaxIdleAge +// set, a load error is cached for the idle age (not forever), and the next +// Get after it lapses retries. +func TestMaxErrorAge_DefaultsToIdleInitialTTL(t *testing.T) { + boom := errors.New("boom") + { + c := New[string, int](MaxIdleAge(time.Hour)) + c.Get("k", func() (int, error) { return 0, boom }) + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet within the idle age: err=%v s=%v, want the cached error as a Hit", err, s) + } + if w := c.t.loadEntry("k").p.Load().expires.Load(); w <= 0 { + t.Fatalf("error expiry = %d, want the idle age from now", w) } + } - // Should be expired by now (>50ms since creation, range didn't extend). - time.Sleep(20 * time.Millisecond) - v, err, s := c.TryGet("foo") - vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) + const idle = 20 * time.Millisecond + c := New[string, int](MaxIdleAge(idle)) + c.Get("k", func() (int, error) { return 0, boom }) + time.Sleep(3 * idle) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the idle age: %v, want Miss", s) + } + var retried bool + v, err, s := c.Get("k", func() (int, error) { retried = true; return 7, nil }) + if !retried || v != 7 || err != nil || s != Miss { + t.Fatalf("Get after the error lapsed: retried=%v (%d, %v, %v), want a fresh (7, nil, Miss) load", retried, v, err, s) + } +} + +// TestHugeAges verifies expiry arithmetic saturates rather than wrapping: +// absurd-but-valid Durations must mean "effectively forever", not "born +// expired" (or, for Clean, "evict everything"). +func TestHugeAges(t *testing.T) { + huge := time.Duration(math.MaxInt64) + + // MaxAge: now + ttl must not wrap negative. + { + c := New[string, int](MaxAge(huge)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet under huge MaxAge: v=%d s=%v, want 1 Hit", v, s) + } + } + + // MaxStaleAge: Clean's expires + staleAge must not wrap negative, and + // the stale's own expiry must not be born wrapped. + { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(huge)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry expired, stale effectively forever + c.Clean() // must not evict + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet after Clean under huge MaxStaleAge: v=%d s=%v, want 1 Stale", v, s) + } + } + + // MaxIdleAge: the extension n + idle must not wrap negative. + { + c := New[string, int](MaxIdleAge(huge)) + c.Set("k", 1) + c.TryGet("k") // a Hit extends; the extension must saturate + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet after huge idle extension: v=%d s=%v, want 1 Hit", v, s) + } + } +} + +// waitFinalized waits until k's value is not loading. +func waitFinalized[K comparable, V any](t *testing.T, tr *trie[K, loading[V]], k K) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if e := tr.loadEntry(k); e != nil { + if l := e.p.Load(); l != nil && l.finalized() { + return + } + } + if time.Now().After(deadline) { + t.Fatalf("%v never finished loading", k) + } + time.Sleep(time.Millisecond) + } +} + +// trieEntries counts every entry in the trie, including deleted entries and +// entries that are invisible to Range and TryGet. +func trieEntries[K comparable, V any](c *Cache[K, V]) (n int) { + c.t.walk(func(*ent[K, V]) bool { n++; return true }) + return n +} + +// TestClean_ReclaimsBornExpired verifies Clean's handling of born-expired +// entries (MaxAge(0) / MaxErrorAge(0), whose negative expiry word carries +// the negated birth): an entry is removed exactly when no read can see it +// anymore (as soon as TryGet reports Miss). That is immediately for errored +// stale-less entries, and when the birth-anchored stale window closes for +// values, regardless of process uptime. +// +// Regression test: born-expired entries used to store a bare -1 sentinel, +// and the grace window was computed as satAdd(-1, maxStaleAge), anchoring +// it at the process epoch: born-expired stale-less entries (error churn +// under MaxErrorAge(0)) were unreclaimable until process uptime exceeded +// MaxStaleAge, and forever for huge stale ages. +func TestClean_ReclaimsBornExpired(t *testing.T) { + t.Run("errored_no_stale", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead invisible born-expired entries in the trie", n) + } + }) + + t.Run("huge_stale_age", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Duration(math.MaxInt64))) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead entries (huge MaxStaleAge made them permanent)", n) + } + }) + + t.Run("keeps_valid_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // born expired; stale window anchored at the store + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean evicted a born-expired entry with a valid stale: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } + }) + + t.Run("reclaims_dead_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(5*time.Millisecond)) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries whose stale window had passed", n) + } + }) +} + +// TestStale_BornExpiredAnchorsAtBirth verifies that a born-expired value's +// (MaxAge(0)) stale window is anchored at the value's birth (carried by +// the negated-birth expiry word) for every read and refresh. Once the +// window passes, the value is gone for good: a later Get drives a fresh +// load rather than serving the dead value. +// +// Regression test: born-expired values used to store a bare -1 sentinel, +// destroying the birth; a Get-driven refresh arbitrarily later re-anchored +// the dead value's stale at the refresh (newStale's expires<=0 fallback), +// serving a value of unbounded age as a fresh Stale right after TryGet +// certified Miss. +func TestStale_BornExpiredAnchorsAtBirth(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(0), MaxStaleAge(age)) + start := time.Now() + c.Set("k", 1) + + // Within the window, the value is servable as a stale. We only check + // if we were not descheduled past the window. + if v, _, s := c.TryGet("k"); time.Since(start) < age && (s != Stale || v != 1) { + t.Fatalf("TryGet within the window: (%d, %v), want (1, Stale)", v, s) + } + + time.Sleep(2 * age) // window passed; the value is certifiably dead + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the window: %v, want Miss", s) + } + v, _, s := c.Get("k", func() (int, error) { return 2, nil }) + if v != 2 || s != Miss { + t.Fatalf("Get after the window: (%d, %v), want (2, Miss): the dead value must not be re-anchored at the refresh", v, s) + } +} + +// TestStale_GetLoadedValueServesSelfStale verifies the stale window applies +// uniformly regardless of how the value was written: a Get-loaded value, +// once expired, is served as Stale by TryGet for MaxStaleAge, exactly like +// a Set-loaded value. +// +// Regression test: only Set/Swap-created loadings used to carry a stale +// snapshot of themselves; Get-loaded values had none (their stale field is +// the previous generation's refresh companion), so TryGet reported Miss +// the instant they expired while a Get-driven refresh of the same state +// served the Stale. +func TestStale_GetLoadedValueServesSelfStale(t *testing.T) { + const ttl = 20 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 1, nil }) + time.Sleep(2 * ttl) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet on an expired Get-loaded value: (%d, %v), want (1, Stale)", v, s) + } +} + +// TestStale_ServesLatestGeneration verifies that once a refreshed value +// itself expires, every read serves that value as the stale: not the +// generation before it. +// +// Regression test: a Get-created loading's stale field is the previous +// generation's snapshot; TryGet used to serve it (v1) after the refreshed +// value (v2) expired, while a Get-driven refresh of the same state +// snapshotted and served v2. +func TestStale_ServesLatestGeneration(t *testing.T) { + const ttl = 30 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(ttl + 10*time.Millisecond) // v1 expired; within its stale window + + // Drive a refresh to v2; the driving Get returns the v1 stale while + // its (held-open) miss runs. + release := make(chan struct{}) + if v, _, s := c.Get("k", func() (int, error) { <-release; return 2, nil }); s != Stale || v != 1 { + t.Fatalf("refreshing Get: (%d, %v), want (1, Stale)", v, s) + } + close(release) + waitFinalized(t, &c.t, "k") + time.Sleep(2 * ttl) // v2 expired + + if v, _, s := c.TryGet("k"); s != Stale || v != 2 { + t.Fatalf("TryGet after v2 expired: (%d, %v), want (2, Stale): the freshest dead value", v, s) + } +} + +// TestMaxIdleAge_StaleWindowFollowsExtensions verifies that the stale +// window of an idle-extended entry anchors at the entry's actual death +// (its last extension lapsing), not at its initial expiry: derived from +// the expiry word, the window moves with every extension. +// +// Regression test: the self-stale used to be a snapshot frozen at +// creation, anchored at the initial expiry. Extensions moved only the +// expiry word, so an entry kept hot past initialExpiry+MaxStaleAge idled +// out with its stale window already closed: TryGet reported Miss the +// instant the entry expired while Get served the Stale. +func TestMaxIdleAge_StaleWindowFollowsExtensions(t *testing.T) { + const age = time.Minute + c := New[string, int](MaxAge(time.Millisecond), MaxIdleAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + if _, _, s := c.TryGet("k"); s != Hit { + // We were descheduled past the 1ms MaxAge; extend manually. + l.expires.Store(now() + int64(time.Hour)) + } + w1 := l.expires.Load() + if w1 <= w0+int64(age) { + t.Fatalf("precondition: extension %d not past the initial window end %d", w1, w0+int64(age)) + } + + // Half way into the window after the extended expiry, the initial + // window is long closed. + if v, _, s := loadingTryGet(l, w1+int64(age)/2, 0, age); s != Stale || v != 1 { + t.Fatalf("read inside the extended entry's window: (%d, %v), want (1, Stale)", v, s) + } + if _, _, s := loadingTryGet(l, w1+int64(age), 0, age); s != Miss { + t.Fatalf("read after the extended entry's window: %v, want Miss", s) + } +} + +// TestExpire_DoesNotResurrectDeadStale verifies that Expire on an entry +// whose value and stale window are both already dead is a no-op: the next +// Get drives a fresh load rather than serving the dead value. +// +// Regression test: Expire used to blindly store now()-1, moving an +// already-expired word forward; the next Get-driven refresh anchored a +// fresh stale window at the Expire and served the dead value as Stale for +// up to another MaxStaleAge. +func TestExpire_DoesNotResurrectDeadStale(t *testing.T) { + const ttl, age = 5 * time.Millisecond, 5 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(age)) + c.Set("k", 1) + time.Sleep(30 * time.Millisecond) // value expired AND stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Expire("k") // invalidating a dead key must not revive anything + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get after expiring a dead entry: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestExpire_StaleWindowAnchorsAtExpire verifies the uniform stale-window +// anchor for manually expired values: the window opens at the Expire and +// closes MaxStaleAge later, for TryGet and Get-driven refreshes alike. +// +// Regression test: TryGet used to serve a frozen creation-time snapshot +// whose window was anchored at the original expiry (an hour out here), +// outliving the documented Expire-anchored window. +func TestExpire_StaleWindowAnchorsAtExpire(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + start := time.Now() + c.Expire("k") + if v, _, s := c.TryGet("k"); time.Since(start) < age && (s != Stale || v != 1) { + t.Fatalf("TryGet just after Expire: (%d, %v), want (1, Stale)", v, s) + } + time.Sleep(2 * age) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the post-Expire window: %v, want Miss", s) + } +} + +// TestExpire_MidWindowDoesNotReanchor verifies Expire is a no-op on an +// entry that is expired but still inside its stale window: the expiry word +// (the window's anchor) must be byte-identical after the call. +// TestExpire_DoesNotResurrectDeadStale covers the window-already-closed +// case; here a forward re-anchor would extend a dead value's servable life +// mid-window. +func TestExpire_MidWindowDoesNotReanchor(t *testing.T) { + t.Run("ttl_expired", func(t *testing.T) { + c := New[string, int](MaxAge(20*time.Millisecond), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(40 * time.Millisecond) // expired; window open for ~1h + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire moved an already-expired word: %d -> %d (re-anchors the stale window)", w0, w1) + } + }) + t.Run("born_expired", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // word carries the negated birth + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + if w0 >= 0 { + t.Fatalf("precondition: want a negated-birth word, got %d", w0) + } + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire destroyed the birth anchor: %d -> %d", w0, w1) + } + }) +} + +// TestClean_KeepsExpiredErrorWithValidCompanion pins Clean's gate for +// errored entries: an expired error whose refresh companion (the previous +// generation) is still inside its window is visible to reads (TryGet +// serves the companion) so Clean must not reclaim it. +func TestClean_KeepsExpiredErrorWithValidCompanion(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour), MaxErrorAge(20*time.Millisecond)) + c.Set("k", 1) + c.Expire("k") + // Drive a refresh that errors; the new loading carries companion=1. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + time.Sleep(40 * time.Millisecond) // error now expired; companion valid ~1h + + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("precondition: TryGet=(%d,%v), want (1, Stale) via the companion", v, s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean reclaimed an entry whose companion was still servable: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } +} + +// TestStale_WindowBoundaries drives the exact-nanosecond boundaries of the +// derived stale window through loadingTryGet with hand-picked clocks (the +// n64 parameter pins the clock deterministically, exactly as Range and +// Delete do) and checks the window definitions agree bit-for-bit across +// the read path (staleOpen), the refresh companion (newStale), and +// stale.expired: the window is [expiry, expiry+age), half-open at every +// site, for positive, negated-birth, and claimed words. +func TestStale_WindowBoundaries(t *testing.T) { + const age = 100 * time.Millisecond + a := int64(age) + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + l := c.t.loadEntry("k").p.Load() + + check := func(n int64, wantS KeyState, wantV int, what string) { + t.Helper() + v, _, s := loadingTryGet(l, n, 0, age) + if s != wantS || (wantS != Miss && v != wantV) { + t.Fatalf("%s: loadingTryGet=(%d,%v), want (%d,%v)", what, v, s, wantV, wantS) + } + } + + // Positive word: expiry E, window [E, E+age). + E := now() - 1 + l.expires.Store(E) + check(E-1, Hit, 1, "one ns before expiry") + check(E, Stale, 1, "the expiry instant itself") + check(E+a-1, Stale, 1, "last open ns of the window") + check(E+a, Miss, 0, "the window-close instant") + + if st := newStale(1, E, age); st.expires != E+a { + t.Fatalf("companion window end %d != read-path window end %d", st.expires, E+a) + } else if st.expired(E+a-1) || !st.expired(E+a) { + t.Fatal("companion boundary disagrees with the read-path boundary") + } + if !staleOpen(E, E+a-1, age) || staleOpen(E, E+a, age) { + t.Fatal("staleOpen boundary inconsistent with itself") + } + + // Negated-birth word: anchor B, window [B, B+age). + B := now() + l.expires.Store(-B) + check(B+a-1, Stale, 1, "born-expired: last open ns") + check(B+a, Miss, 0, "born-expired: window-close instant") + if st := newStale(1, -B, age); st.expires != B+a { + t.Fatalf("born-expired companion end %d, want %d", st.expires, B+a) + } + + // Claimed word: no window, ever; and no companion may be derived. + l.expires.Store(expiredClaimed) + check(B, Miss, 0, "claimed word") + if staleOpen(expiredClaimed, 1, age) { + t.Fatal("staleOpen derived a window from a claimed word") + } + if st, ok := maybeNewStale(c.t.loadEntry("k").p.Load(), age); ok { + t.Fatalf("maybeNewStale snapshotted a claimed loading: %+v", st) + } +} + +// TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and +// CompareAndDelete only match live values: expired entries and errored +// entries are "not there" per TryGet, so comparing against them must fail. +func TestCompareAndSwapAgreesWithTryGet(t *testing.T) { + t.Run("expired", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatal("expired entry should TryGet Miss") + } + if c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap matched an expired value") + } + if c.CompareAndDelete("k", 1) { + t.Error("CompareAndDelete matched an expired value") + } + }) + + t.Run("errored", func(t *testing.T) { + c := New[string, int]() + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + // The errored loading's v is the zero int; CAS against 0 must not + // treat that as a cached value. + if c.CompareAndSwap("k", 0, 2) { + t.Error("CompareAndSwap matched an errored entry's placeholder value") + } + if c.CompareAndDelete("k", 0) { + t.Error("CompareAndDelete matched an errored entry's placeholder value") + } }) + + t.Run("live", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + if !c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap failed on a live matching value") + } + if !c.CompareAndDelete("k", 2) { + t.Error("CompareAndDelete failed on a live matching value") + } + }) +} + +// An idle extension that would move the expiry by less than the slack is +// skipped; a larger one is written. +func TestMaxIdleAge_Slack(t *testing.T) { + const age = time.Hour + l := &loading[int]{} + base := now() + int64(age) + l.expires.Store(base) + + l.extend(base, base-int64(age)+int64(maxIdleSlack)/2, age) + if got := l.expires.Load(); got != base { + t.Fatalf("extension within the slack was written: %d != %d", got, base) + } + n := base - int64(age) + 2*int64(maxIdleSlack) + l.extend(base, n, age) + if got := l.expires.Load(); got != n+int64(age) { + t.Fatalf("extension past the slack: expiry %d, want %d", got, n+int64(age)) + } + + // A small age uses age/64 as the slack. + const small = 32 * time.Millisecond + l.expires.Store(base) + n = base - int64(small) + int64(small)/32 + l.extend(base, n, small) + if got := l.expires.Load(); got != n+int64(small) { + t.Fatalf("small-age extension past age/64: expiry %d, want %d", got, n+int64(small)) + } +} + +// Lookups must not make the key escape: a key converted from a []byte, as +// with a builtin map lookup, must not allocate. +func TestLookupKeyDoesNotEscape(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("abc", 1) + b := []byte("abc") + for name, f := range map[string]func(){ + "TryGet": func() { c.TryGet(string(b)) }, + "Expire": func() { c.Expire(string(b) + "x") }, + "Delete": func() { c.Delete(string(b) + "x") }, + "CompareAndSwap": func() { c.CompareAndSwap(string(b), 2, 3) }, + "CompareAndDelete": func() { c.CompareAndDelete(string(b), 2) }, + } { + if n := testing.AllocsPerRun(100, f); n != 0 { + t.Errorf("%s allocated %v times, want 0", name, n) + } + } +} + +// Clean does not allocate, however much it deletes. +func TestCleanDoesNotAllocate(t *testing.T) { + c := New[int, int](MaxAge(0)) + for i := range 1000 { + c.Set(i, i) + } + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + c.Clean() + runtime.ReadMemStats(&after) + if n := after.Mallocs - before.Mallocs; n != 0 { + t.Errorf("Clean deleting 1000 entries allocated %d times, want 0", n) + } + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries, want 0", n) + } +} + +// A refresh after an error that has no stale carries no stale, even with +// stale values enabled. +func TestRefreshAfterErrorWithoutStale(t *testing.T) { + c := New[int, int](MaxStaleAge(time.Hour), MaxErrorAge(0)) + if _, err, s := c.Get(1, func() (int, error) { return 0, errors.New("boom") }); err == nil || s != Miss { + t.Fatalf("first Get: (%v, %v), want (boom, Miss)", err, s) + } + release := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + c.Get(1, func() (int, error) { <-release; return 2, nil }) + }() + waitForState(t, c, 1, func(l *loading[int]) bool { return !l.finalized() }) + if st := c.t.loadEntry(1).p.Load().stale(); st != nil { + t.Fatalf("refresh after an error with no stale carries stale %v", st.v) + } + close(release) + <-done + if v, _, s := c.TryGet(1); v != 2 || s != Hit { + t.Fatalf("TryGet: (%d, %v), want (2, Hit)", v, s) + } +} + +// waitForState waits until k's loading satisfies ok. +func waitForState(t *testing.T, c *Cache[int, int], k int, ok func(*loading[int]) bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if e := c.t.loadEntry(k); e != nil { + if l := e.p.Load(); l != nil && ok(l) { + return + } + } + if time.Now().After(deadline) { + t.Fatal("loading never reached the wanted state") + } + time.Sleep(time.Millisecond) + } } diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go new file mode 100644 index 0000000..07c3c11 --- /dev/null +++ b/cache/concurrency_test.go @@ -0,0 +1,1622 @@ +package cache + +import ( + "errors" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestSwap_CancelsInFlightGet verifies that Swap overrides a currently-loading +// entry: the goroutine waiting in Get sees the Swap value, and the underlying +// miss function's return value is discarded. +// +// Covers cache.go Swap defer block for !was.finalized() (the cancel path). +func TestSwap_CancelsInFlightGet(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + missDone := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + close(missDone) + return 99, nil + }) + getV = v + }) + + <-missStart // miss function has begun + + // Swap while the miss is still running. + _, _, oldS := c.Swap("k", 7) + if oldS != Miss { + t.Fatalf("Swap old state: got %v want Miss", oldS) + } + + // Let the miss function finish. Its value MUST be discarded. + close(missRelease) + <-missDone // ensure the miss goroutine actually ran to completion + getDone.Wait() // ensure the Get goroutine returned + + if getV != 7 { + t.Fatalf("Get value = %d, want 7 (Swap should cancel miss)", getV) + } + + // Verify the cache holds Swap's value, not the miss's. + v, _, s := c.TryGet("k") + if v != 7 || !s.IsHit() { + t.Fatalf("TryGet after Swap: v=%d s=%v, want 7 Hit", v, s) + } +} + +// TestSwap_OverExpiredFinalizedWithStale exercises the defer branch that +// reads the prior entry's stale when the prior entry has expired but a valid +// stale is still in its window. +func TestSwap_OverExpiredFinalizedWithStale(t *testing.T) { + c := New[string, int]( + MaxAge(5*time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(15 * time.Millisecond) + // Entry is expired but stale is still valid. + old, _, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 { + t.Fatalf("Swap old: got %d want 1", old) + } +} + +// TestSwap_OverErroredWithStale covers the defer branch where the prior +// entry is finalized but holds an error, and a valid stale exists. +func TestSwap_OverErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + // Drive a refresh that will error; while it is in flight, the entry is + // (loading, stale=1). The Get returns the stale immediately. + c.Expire("k") + release := make(chan struct{}) + defer close(release) + if v, _, s := c.Get("k", func() (int, error) { + <-release + return 0, errors.New("boom") + }); v != 1 || s != Stale { + t.Fatalf("refreshing Get: (%d, %v), want (1, Stale)", v, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 || oldErr != nil { + t.Fatalf("Swap old: v=%d err=%v, want v=1 err=nil", old, oldErr) + } +} + +// TestDelete_DuringInFlightLoad verifies that deleting while a load is in +// flight: (a) the in-flight Get still returns the miss fn's result, and (b) +// a subsequent Get re-triggers the miss fn. +func TestDelete_DuringInFlightLoad(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }) + + <-missStart + c.Delete("k") + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5", getV) + } + + // Next Get must re-run the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Delete") + } + if v != 6 { + t.Fatalf("post-Delete Get got %d, want 6", v) + } +} + +// TestClear_DoesNotCacheInFlightLoads pins Clear's documented interplay with +// in-flight loads (the per-key Delete caveat applied wholesale): Clear swaps +// the trie root, so a load in flight at the Clear keeps running against the +// detached tree: its result still reaches the Get waiting on it, but is +// never cached, and a later Get misses and drives a fresh load. +func TestClear_DoesNotCacheInFlightLoads(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }) + + <-missStart + c.Clear() + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5 (the load's result must still reach its waiter)", getV) + } + + // The result must not have been cached: the next Get re-runs the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Clear (the in-flight result must not be cached)") + } + if v != 6 { + t.Fatalf("post-Clear Get got %d, want 6", v) + } +} + +// TestExpire_NoOpDuringInFlightLoad verifies the documented behavior: Expire +// on an in-flight load is a no-op; the load completes with its normal TTL. +func TestExpire_NoOpDuringInFlightLoad(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getDone sync.WaitGroup + getDone.Go(func() { + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 11, nil + }) + }) + + <-missStart + c.Expire("k") // must be a no-op while loading + close(missRelease) + getDone.Wait() + + v, _, s := c.TryGet("k") + if v != 11 || !s.IsHit() { + t.Fatalf("after in-flight Expire: v=%d s=%v, want 11 Hit (Expire should not have affected the load)", v, s) + } +} + +// TestCollapsing_NoCaching runs many concurrent Gets against a cache with +// MaxAge(0) (no caching). The documented use case is collapsing simultaneous +// queries for the same key. +// +// The guarantee is weaker than it looks: with del0, the loading finalizes +// expired, so the next goroutine that re-checks the entry after the leader's +// setve sees an expired entry and triggers a fresh miss. Concurrent arrivals +// still collapse in batches, but "batch" here means "whatever queued on a +// given loading's WaitGroup before it was Done'd," which is scheduler- +// dependent. +// +// We assert: (1) miss runs at least once, (2) collapsing happens (fewer miss +// calls than workers), (3) every returned value corresponds to some miss +// invocation. +func TestCollapsing_NoCaching(t *testing.T) { + c := New[string, int](MaxAge(0)) + const workers = 100 + + var missStart sync.WaitGroup + missStart.Add(1) + missRelease := make(chan struct{}) + var missCalls int32 + + var wg sync.WaitGroup + vals := make([]int, workers) + for i := range workers { + wg.Go(func() { + v, _, _ := c.Get("k", func() (int, error) { + n := atomic.AddInt32(&missCalls, 1) + if n == 1 { + missStart.Done() + <-missRelease + } + return int(n), nil + }) + vals[i] = v + }) + } + + missStart.Wait() + time.Sleep(5 * time.Millisecond) // let as many callers as possible queue + close(missRelease) + wg.Wait() + + calls := atomic.LoadInt32(&missCalls) + if calls < 1 { + t.Fatalf("miss never called") + } + if calls >= workers { + t.Fatalf("no collapsing observed: miss called %d times across %d Gets", calls, workers) + } + for i, v := range vals { + if v < 1 || int32(v) > calls { + t.Fatalf("goroutine %d got value %d, outside valid miss-call range [1,%d]", i, v, calls) + } + } + + // Nothing is cached. A fresh Get runs the miss again. + var called bool + c.Get("k", func() (int, error) { + called = true + return 99, nil + }) + if !called { + t.Fatal("after collapsed Get, cache should not have retained a value") + } +} + +// TestGet_StaleRefreshCASRace pounds on the Get slow-path CAS-retry that +// replaces a finalized-expired-with-stale loading with a fresh loading +// whose stale is a snapshot of the old value. Many concurrent Gets on the +// same key force CAS losses and thus exercise the retry. +func TestGet_StaleRefreshCASRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + for range 200 { + c := New[string, int]( + MaxAge(time.Microsecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(time.Millisecond) // entry expired, stale valid + + const workers = 16 + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + c.Get("k", func() (int, error) { + // Slow miss so concurrent Gets all pile into the + // slow-path stale branch before any finalizes. + time.Sleep(time.Millisecond) + return 2, nil + }) + }) + } + wg.Wait() + } +} + +// TestGet_ConcurrentStaleDuringInFlight exercises Get's slow-path branch +// where a second Get observes a stale-returning loading that is already +// in flight from the first Get. The second Get must return the stale +// without attempting to install a new loading. +func TestGet_ConcurrentStaleDuringInFlight(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // entry now expired, stale still valid + + missRelease := make(chan struct{}) + missStart := make(chan struct{}) + var aDone sync.WaitGroup + aDone.Go(func() { + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 2, nil + }) + }) + <-missStart + + // Second Get observes the first Get's in-flight loading and its stale. + v, _, s := c.Get("k", func() (int, error) { + t.Error("second Get's miss function should not run while first Get is in flight") + return 3, nil + }) + if s != Stale || v != 1 { + t.Fatalf("second Get: v=%d s=%v, want v=1 Stale", v, s) + } + + close(missRelease) + aDone.Wait() +} + +// TestClean_SkipsInFlightLoads verifies Clean leaves pending loads alone. +// An entry whose loading hasn't finalized yet has no expiry to evaluate; +// Clean must skip it. +func TestClean_SkipsInFlightLoads(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond)) + missRelease := make(chan struct{}) + missStarted := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { + close(missStarted) + <-missRelease + return 42, nil + }) + }() + <-missStarted + e := c.t.loadEntry("k") + l := e.p.Load() + c.Clean() + if e2 := c.t.loadEntry("k"); e2 != e || e.p.Load() != l { + t.Fatal("Clean removed an in-flight load") + } + close(missRelease) + <-getDone +} + +// TestCleanUnderConcurrency stresses Clean racing with concurrent +// Set/Get/Delete to catch races in the clean path. Correctness here is that +// the test finishes without -race complaints or panics; we also assert that +// no value ever mysteriously appears that we never Set. +func TestCleanUnderConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Millisecond), + ) + + var stop atomic.Bool + var wg sync.WaitGroup + + // Setters. + for w := range 4 { + wg.Go(func() { + for !stop.Load() { + for i := range 64 { + c.Set(i, i*1000+w) + } + } + }) + } + + // Getters. + for range 4 { + wg.Go(func() { + for !stop.Load() { + for i := range 64 { + v, _, s := c.TryGet(i) + if s.IsHit() { + // Value is i*1000+worker; must satisfy the invariant. + if v%1000 > 3 || v/1000 != i { + t.Errorf("impossible value v=%d for key=%d", v, i) + return + } + } + } + } + }) + } + + // Deleters. + for range 2 { + wg.Go(func() { + for !stop.Load() { + for i := range 64 { + c.Delete(i) + } + } + }) + } + + // Cleaner. + wg.Go(func() { + for !stop.Load() { + c.Clean() + time.Sleep(200 * time.Microsecond) + } + }) + + time.Sleep(500 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestSwap_DeleteRangeChurn stresses Swap's fast path racing Delete's +// tombstone-then-unlink, with Range walking the trie throughout: +// Swap must observe tombstones and re-create keys (never resurrect a dying +// entry), and Range must tolerate entries being unlinked mid-walk. The +// interesting windows are ~nanoseconds wide; we stress from many angles for +// long enough that they land, leaving a generous budget for slow CI runners. +func TestSwap_DeleteRangeChurn(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + const readKeys = 32 + for i := range readKeys { + c.Set(i, i) + } + + var wg sync.WaitGroup + var stop atomic.Bool + + // Deleters churn the primed keys, repeatedly creating the tombstone + + // unlink windows that Swap's fast path must handle. + for range 8 { + wg.Go(func() { + for !stop.Load() { + for i := range readKeys { + c.Delete(i) + } + } + }) + } + // Swappers on the same primed keys exercise the unlocked fast-path CAS + // that races with the deleters. + for w := range 8 { + wg.Go(func() { + for !stop.Load() { + for i := range readKeys { + c.Swap(i, i*1000+w) + } + } + }) + } + // Churners add+delete NEW keys continuously so the trie's shape keeps + // changing (inserts expand slots, deletes prune empty parents) while + // Range walks it. + for w := range 4 { + wg.Go(func() { + base := readKeys + w*10000 + for !stop.Load() { + for i := range 64 { + c.Set(base+i, i) + c.Delete(base + i) + } + } + }) + } + // Range callers walk the trie throughout the churn. + for range 4 { + wg.Go(func() { + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }) + } + + time.Sleep(2 * time.Second) + stop.Store(true) + wg.Wait() +} + +// TestTryCAS_RetryOnRace runs concurrent CompareAndSwap increments of one +// key: every increment that reports success must land exactly once. +func TestTryCAS_RetryOnRace(t *testing.T) { + c := New[string, int]() + c.Set("k", 0) + + const workers = 32 + const iters = 1000 + + var wg sync.WaitGroup + var successes int32 + for range workers { + wg.Go(func() { + for range iters { + for { + v, _, s := c.TryGet("k") + if !s.IsHit() { + break + } + if c.CompareAndSwap("k", v, v+1) { + atomic.AddInt32(&successes, 1) + break + } + runtime.Gosched() + } + } + }) + } + wg.Wait() + + v, _, _ := c.TryGet("k") + if int32(v) != successes { + t.Fatalf("value %d != successful CAS count %d", v, successes) + } + if successes != workers*iters { + t.Fatalf("successes %d != expected %d (every worker should eventually succeed its increments)", successes, workers*iters) + } +} + +// TestCompareAndDelete_RaceOnDelete stresses the CompareAndDelete path with +// concurrent deletes/sets to exercise both the retry loop inside tryCAS and +// the value-mismatch path after a concurrent mutation. +func TestCompareAndDelete_RaceOnDelete(t *testing.T) { + c := New[int, int]() + const workers = 16 + var wg sync.WaitGroup + + var setDone atomic.Int64 + for w := range workers { + wg.Go(func() { + for i := range 1000 { + c.Set(i%8, w) + setDone.Add(1) + } + }) + } + for range workers { + wg.Go(func() { + for i := range 1000 { + v, _, s := c.TryGet(i % 8) + if s.IsHit() { + c.CompareAndDelete(i%8, v) + } + } + }) + } + wg.Wait() + // No assertion on final state; the race-detector + lack of panic is the + // signal. +} + +// TestSetve_RacesWithSwap pins the ordering where a Swap displaces a +// loading that the load is in the middle of finalizing: Swap's own +// finalize loses, so Swap must wait for the load's result and report it as +// the old value. +func TestSetve_RacesWithSwap(t *testing.T) { + c := New[string, int]() + release := make(chan struct{}) + started := make(chan struct{}) + getDone := make(chan int) + go func() { + v, _, _ := c.Get("k", func() (int, error) { + close(started) + <-release + return 1, nil + }) + getDone <- v + }() + <-started + l := c.t.loadEntry("k").p.Load() + + // Finalize as setve would, but pause between claiming and publishing. + if !l.state.CompareAndSwap(stateLoading, stateFinalizing) { + t.Fatal("precondition: loading already finalized") + } + type swapRes struct { + v int + s KeyState + } + swapDone := make(chan swapRes) + go func() { + v, _, s := c.Swap("k", 2) + swapDone <- swapRes{v, s} + }() + // Once Swap has replaced l, it is in its deferred finalize of l. + for c.t.loadEntry("k").p.Load() == l { + runtime.Gosched() + } + select { + case r := <-swapDone: + t.Fatalf("Swap returned %v before the load finished finalizing", r) + case <-time.After(10 * time.Millisecond): + } + l.v = 1 + l.state.Store(stateFinalized) + l.wg.Done() + + if r := <-swapDone; r.v != 1 || r.s != Hit { + t.Fatalf("Swap old: (%d, %v), want (1, Hit)", r.v, r.s) + } + close(release) // the load's own setve now loses + if v := <-getDone; v != 1 { + t.Fatalf("Get: %d, want 1", v) + } + if v, _, s := c.TryGet("k"); v != 2 || s != Hit { + t.Fatalf("TryGet: (%d, %v), want (2, Hit)", v, s) + } +} + +// TestClean_NoopWhenStalesArePermanent covers the MaxStaleAge<0 early return +// in Clean (the "stales kept forever" opt-in). +func TestClean_NoopWhenStalesArePermanent(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(-1)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry is now expired, stale is forever + c.Clean() // must be a no-op + // The stale is still queryable. + _, _, s := c.TryGet("k") + if s != Stale { + t.Fatalf("after Clean with MaxStaleAge=-1: state=%v want Stale", s) + } +} + +// TestGet_RacesSwapOnExpired targets Get's slow-path Hit: the fast path +// sees an expired entry (Miss), but by the time the slow path re-checks, a +// concurrent Swap has published a fresh value for the same key. If Get +// reports a Hit it must be the Swap's value, never the expired one. Hitting +// the exact timing window requires multiple iterations. +func TestGet_RacesSwapOnExpired(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int](MaxAge(time.Hour)) + c.Set(1, 1) + c.Expire(1) + + var wg sync.WaitGroup + wg.Go(func() { + c.Swap(1, iter+100) + }) + wg.Go(func() { + v, _, s := c.Get(1, func() (int, error) { return -1, nil }) + if s.IsHit() && v != iter+100 { + t.Errorf("iter %d: Hit but v=%d, want %d", iter, v, iter+100) + } + }) + wg.Wait() + } +} + +// TestCompareAndSwap_RacesSet races CompareAndSwap against the first Set of +// the same key: the CAS succeeds only if it sees the Set's value. +func TestCompareAndSwap_RacesSet(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int]() + var wg sync.WaitGroup + var swapped bool + wg.Go(func() { + c.Set(iter, iter) + }) + wg.Go(func() { + swapped = c.CompareAndSwap(iter, iter, iter+1) + }) + wg.Wait() + want := iter + if swapped { + want = iter + 1 + } + if v, _, _ := c.TryGet(iter); v != want { + t.Fatalf("iter %d: swapped=%v, value %d, want %d", iter, swapped, v, want) + } + } +} + +// TestMemoryBounded churns keys in and out of the cache and checks that +// Delete unlinks entries rather than leaving deleted entries in the trie. +func TestMemoryBounded(t *testing.T) { + c := New[int, int]() + const liveKeys = 32 + const rounds = 2000 + + for r := range rounds { + for i := range liveKeys { + c.Set(r*liveKeys+i, i) + } + // Delete the ones we just wrote except for `liveKeys` canonical keys. + for i := range liveKeys { + c.Delete(r*liveKeys + i) + } + for i := range liveKeys { + c.Set(i, i) + } + } + + if count := trieEntries(c); count != liveKeys { + t.Fatalf("after %d rounds: %d live entries, want <= %d", rounds, count, liveKeys) + } +} + +// TestSwap_NotLostDuringConcurrentDelete verifies that a Swap racing a +// Delete of the same key is never silently dropped. The dangerous +// interleaving: Delete tombstones the value slot, Swap observes the +// tombstone, and Delete's physical unlink races Swap's installation of the +// new value. A nil value slot is terminal (Swap must re-create the key as +// a fresh entry, never write into the dying one) so if Swap returns Miss +// ("I stored into nothing"), it linearized after the Delete and the swapped +// value must be visible afterwards. +// +// Regression test: an earlier version resurrected tombstones with a bare +// CAS not serialized with deleteEntryIf's unlink, losing the swapped value +// in ~0.03% of rounds. +func TestSwap_NotLostDuringConcurrentDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const rounds = 100_000 + for i := range rounds { + c := New[int, int]() + c.Set(i, 0) + + var wg sync.WaitGroup + var swapState KeyState + wg.Go(func() { + c.Delete(i) + }) + wg.Go(func() { + _, _, swapState = c.Swap(i, 1) + }) + wg.Wait() + + if swapState == Miss { + if v, _, s := c.TryGet(i); s.IsMiss() || v != 1 { + t.Fatalf("round %d: Swap returned Miss (stored after the Delete) but TryGet=(%d,%v); the swapped value was silently dropped", i, v, s) + } + } + } +} + +// TestClean_DoesNotEvictFreshEntries runs Clean continuously against Gets +// on a cache with no MaxAge and no Deletes: nothing ever expires, so Clean +// must never remove anything and every completed Get must be a subsequent +// TryGet hit. +// +// Regression test: an earlier version created Get's entry with a nil value +// slot and CAS'd the loading in afterwards; in that window the fresh entry +// was indistinguishable from a Delete tombstone and Clean would unlink it, +// losing the cached value (and with it, request collapsing for that key). +func TestClean_DoesNotEvictFreshEntries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + + stop := make(chan struct{}) + var cleanWg sync.WaitGroup + cleanWg.Go(func() { + for { + select { + case <-stop: + return + default: + c.Clean() + runtime.Gosched() + } + } + }) + + const rounds = 200_000 + for i := range rounds { + c.Get(i, func() (int, error) { return i, nil }) + if v, _, s := c.TryGet(i); s.IsMiss() || v != i { + t.Fatalf("round %d: freshly loaded entry evicted by Clean: TryGet=(%d,%v)", i, v, s) + } + } + close(stop) + cleanWg.Wait() +} + +// TestGet_DoesNotRedriveFreshlyFinalizedValue targets the slow-path window +// between Get's load of prev and entGet's own load of the slot: if the +// in-flight load it observed finalizes live in that window, Get must return +// the Hit rather than demote the fresh value to a stale and drive a +// redundant load. Every round arranges an expired-with-stale entry, races +// Gets against the first load's release, and asserts the miss function ran +// exactly once. +func TestGet_DoesNotRedriveFreshlyFinalizedValue(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 2000 { + c := New[int, int](MaxAge(time.Hour), MaxStaleAge(time.Hour)) + c.Set(iter, 1) + c.Expire(iter) // expired with a valid self-stale + + // A driving Get returns the stale and runs its miss on a + // background goroutine, so completion must be synchronized on: + // missDone is closed by the (single) miss, and a redundant + // second drive panics on the double close. + release := make(chan struct{}) + missDone := make(chan struct{}) + miss := func() (int, error) { + <-release // closed below; late drivers pass straight through + close(missDone) + return 2, nil + } + + var wg sync.WaitGroup + wg.Go(func() { + c.Get(iter, miss) // drives the refresh, returns Stale(1) + }) + wg.Go(func() { + runtime.Gosched() + close(release) + }) + // Meanwhile, hammer Gets that race the finalization. None may + // drive a second load: each must piggyback the stale or return + // the fresh Hit. + for range 4 { + c.Get(iter, miss) + } + wg.Wait() + <-missDone + } +} + +// TestExpire_NotLostToIdleExtension verifies that Expire racing an +// idle-extending read sticks: once both return, the key must be expired. +// +// Regression test: idle extension previously used a plain Store on the +// expiry, which could overwrite a concurrent Expire (read loads expiry, +// Expire stores now-1, read stores now+idle). Extension now CASes against +// the expiry it observed, so an interleaved Expire wins. +func TestExpire_NotLostToIdleExtension(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int](MaxIdleAge(time.Hour)) + const rounds = 100_000 + for i := range rounds { + c.Set(i, 1) + var wg sync.WaitGroup + wg.Go(func() { + c.TryGet(i) // a Hit extends the idle expiry + }) + wg.Go(func() { + c.Expire(i) + }) + wg.Wait() + if _, _, s := c.TryGet(i); s.IsHit() { + t.Fatalf("round %d: Expire was overwritten by a concurrent idle extension", i) + } + } +} + +// TestClean_ClaimWindowDoesNotResurrectDeadStale pins Clean's claim protocol +// against racing Gets: Clean claims an entry's expiry word before +// tombstoning, and a Get that lands between the claim and the tombstone +// snapshots that loading for a stale. The claimed word must read as "stale +// window certified closed" (never as something a stale window can be +// derived from) or the Get would serve a value past MaxAge+MaxStaleAge as +// a fresh Stale for up to another MaxStaleAge. +// +// The test performs Clean's claim by hand (same gate, same CAS) so the +// window between Clean's two CASes is held open deterministically. +// +// Regression test: Clean used to claim with -1, the born-expired sentinel, +// which newStale then re-anchored at now (born-expired words now carry +// their birth and anchor there, and a claimed word derives no window at +// all). +func TestClean_ClaimWindowDoesNotResurrectDeadStale(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + // Clean's claim, by hand: same gate, same CAS, protocol frozen between + // Clean's two CASes. + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + tn := now() + if expired := expires != 0 && expires <= tn; !expired || + (l.stale() != nil && !l.stale().expired(tn)) || + (l.err() == nil && staleOpen(expires, tn, c.cfg.maxStaleAge)) { + t.Fatalf("precondition: entry not Clean-eligible: expires=%d", expires) + } + if !l.expires.CompareAndSwap(expires, expiredClaimed) { + t.Fatal("claim CAS failed") + } + + // A Get in the claim window must not serve the dead value as a stale: + // the stale snapshot must come up empty, so the Get blocks on (and + // returns) the fresh load. + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in Clean's claim window: v=%d s=%v, want 2 Miss (a Stale here resurrects a certified-dead value)", v, s) + } + if l2 := e.p.Load(); l2 == nil || l2.stale() != nil { + t.Fatalf("the replacement loading must carry no stale snapshot of the claimed value, got %+v", l2) + } + + // Finish Clean's protocol: the tombstone CAS must fail (the Get + // replaced the slot) and the fresh value must survive. + if e.p.CompareAndSwap(l, nil) { + t.Fatal("Clean's tombstone CAS should have failed against the Get's replacement") + } + if v, _, s := c.TryGet("k"); v != 2 || !s.IsHit() { + t.Fatalf("after claim window: TryGet=(%d,%v), want (2,Hit)", v, s) + } +} + +// TestClean_ClaimWindowVsGetRace hammers the window between Clean's two +// CASes (the expiry-word claim and the tombstone) with racing Gets. Every +// round populates keys whose value AND stale are certifiably dead (TryGet +// returned Miss), then races a real Clean against a Get scan: any Get that +// returns Stale resurrected a certified-dead value. +// +// Probabilistic tripwire for the protocol pinned deterministically by +// TestClean_ClaimWindowDoesNotResurrectDeadStale; before Clean claimed with +// a dedicated sentinel, this fired within a few seconds under -race. +func TestClean_ClaimWindowVsGetRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + const ( + keys = 1024 + // Small enough that entries die within a round's sleep; large + // enough that a wrongly re-anchored stale stays observable for the + // Get that snapshotted it. + staleAge = 5 * time.Millisecond + ) + c := New[int, int](MaxAge(time.Nanosecond), MaxStaleAge(staleAge)) + miss := func() (int, error) { return 2, nil } + for start := time.Now(); time.Since(start) < 3*time.Second; { + for k := range keys { + c.Set(k, 1) + } + time.Sleep(staleAge + 2*time.Millisecond) + for k := range keys { + for { + if _, _, s := c.TryGet(k); s.IsMiss() { + break // certified: expired and stale dead + } + } + } + + var wg sync.WaitGroup + wg.Go(func() { + c.Clean() + }) + for k := range keys { + if v, _, s := c.Get(k, miss); s == Stale { + t.Fatalf("key %d: Get racing Clean returned a certified-dead value as a stale: v=%d", k, v) + } + } + wg.Wait() + } +} + +// TestExpire_DoesNotOverwriteCleanClaim pins Expire against Clean's claim +// window: once Clean has claimed an entry's expiry word (certifying the +// value expired and its stale window closed), a racing Expire must leave +// the claim in place: never replace it with a fresh timestamp that a Get +// in the window could re-derive a stale window from. Same hand-driven +// protocol as TestClean_ClaimWindowDoesNotResurrectDeadStale. +// +// Regression test: Expire used to blindly store now()-1 over any finalized +// word, including expiredClaimed; a Get between Clean's two CASes then +// served the certified-dead value as a fresh Stale. +func TestExpire_DoesNotOverwriteCleanClaim(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + if !l.expires.CompareAndSwap(expires, expiredClaimed) { // Clean's claim + t.Fatal("claim CAS failed") + } + + c.Expire("k") // lands between Clean's two CASes; must be a no-op + + if got := l.expires.Load(); got != expiredClaimed { + t.Fatalf("Expire overwrote Clean's claim: expires=%d, want expiredClaimed", got) + } + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in the claim window: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestSwap_OverFinalizedErroredWithStale is TestSwap_OverErroredWithStale +// with the race pinned the other way: the errored load has finalized by +// the time Swap displaces it. The classification must agree with TryGet: +// the valid stale, not the error, is the prior value. +// +// Regression test: the defer's finalized classification used to assign the +// stale and then fall through to overwrite it with (v, err, Hit); the +// sibling test only exercises the unfinalized branch (Swap usually beats +// the async setve), so the finalized path went unexercised. +func TestSwap_OverFinalizedErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + // Wait for the async setve to finalize the errored loading. + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + + if v, err, s := c.TryGet("k"); s != Stale || v != 1 || err != nil { + t.Fatalf("TryGet: (%d, %v, %v), want (1, nil, Stale)", v, err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestSwap_OverFinalizedErroredNoStale covers the remaining finalized +// classification arm: with no stale to return, an unexpired cached error +// is itself the prior result, and Swap reports it as a Hit carrying the +// error: agreeing with TryGet. +func TestSwap_OverFinalizedErroredNoStale(t *testing.T) { + c := New[string, int]() // errors cache forever by default + // The driving Get waits for its load, so the error is finalized here. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet: err=%v s=%v, want the cached error as a Hit", err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if old != 0 || oldErr == nil || !oldS.IsHit() { + t.Fatalf("Swap: (%d, %v, %v), want (0, boom, Hit) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestDelete_ConcurrentlyFinalizedLoadReportsHit pins Delete's documented +// classification race: Delete samples the clock before tombstoning, but +// reads the loading's finalized state afterwards, so a load finalizing +// between the two is reported as the removed value (a Hit) even though no +// read ever observed it cached. This is Delete's exact code sequence with +// the window held open; the returned triple is best effort there (see +// Delete's doc), and the cache state itself is unaffected. +func TestDelete_ConcurrentlyFinalizedLoadReportsHit(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missRelease := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { <-missRelease; return 7, nil }) + }() + var e *ent[string, int] + for e = c.t.loadEntry("k"); e == nil; e = c.t.loadEntry("k") { + runtime.Gosched() + } + + // Delete's sequence (see Cache.Delete), paused mid-window: + n := now() + was := entDel(e) + c.t.deleteEntryIf("k", entDead[string, int]) + close(missRelease) // the miss finalizes the already-detached loading + <-getDone + + if v, _, s := loadingTryGet(was, n, 0, c.cfg.maxStaleAge); !s.IsHit() || v != 7 { + t.Fatalf("classifying the concurrently finalized loading: (%d, %v), want (7, Hit) per Delete's documented race", v, s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("the key must be gone regardless: %v, want Miss", s) + } +} + +// Gets that collapsed onto a load that errors all return its error, even +// with errors uncached, and the next Get runs a new load. +func TestGet_CollapsedErrorWaitersShareError(t *testing.T) { + c := New[string, int](MaxErrorAge(0)) + var calls atomic.Int32 + release := make(chan struct{}) + started := make(chan struct{}) + boom := errors.New("boom") + miss := func() (int, error) { + if calls.Add(1) == 1 { + close(started) + <-release + } + return 0, boom + } + const waiters = 8 + errs := make(chan error, waiters) + var wg sync.WaitGroup + wg.Go(func() { _, err, _ := c.Get("k", miss); errs <- err }) + <-started + for range waiters - 1 { + wg.Go(func() { _, err, _ := c.Get("k", miss); errs <- err }) + } + waitForLoadWaiters(t, waiters-1) + close(release) + wg.Wait() + close(errs) + for err := range errs { + if err != boom { + t.Errorf("collapsed Get returned %v, want boom", err) + } + } + if n := calls.Load(); n != 1 { + t.Fatalf("miss ran %d times for %d collapsed Gets, want 1", n, waiters) + } + c.Get("k", miss) + if n := calls.Load(); n != 2 { + t.Fatalf("the Get after an uncached error did not run a new load: %d loads", n) + } +} + +// waitForLoadWaiters waits until n goroutines are blocked waiting on a +// load. +func waitForLoadWaiters(t *testing.T, n int) { + t.Helper() + buf := make([]byte, 1<<20) + deadline := time.Now().Add(10 * time.Second) + for { + stacks := string(buf[:runtime.Stack(buf, true)]) + if strings.Count(stacks, "cache.entGet[") >= n { + return + } + if time.Now().After(deadline) { + t.Fatalf("fewer than %d goroutines ever waited on a load", n) + } + time.Sleep(time.Millisecond) + } +} + +// retryTTL calls f with ttl, doubling ttl and retrying while f returns +// false. f returns false if we were descheduled past ttl before reaching +// the state under test. +func retryTTL(t *testing.T, ttl time.Duration, f func(ttl time.Duration) bool) { + t.Helper() + for range 8 { + if f(ttl) { + return + } + ttl *= 2 + } + t.Fatal("never reached the state under test within the TTL") +} + +// setExpiresPairingHook installs expiresPairingHook such that exactly the +// first read to pass through it blocks until release is closed; all later +// reads pass straight through. The hook is cleared when the test ends. +func setExpiresPairingHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + return setExpiresPairingHookNth(t, 1) +} + +// setExpiresPairingHookNth is setExpiresPairingHook parking the nth +// (1-based) paired read instead of the first, for callers whose operation +// takes multiple paired reads before the one under test. +func setExpiresPairingHookNth(t *testing.T, nth int32) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var calls atomic.Int32 + expiresPairingHook = func() { + if calls.Add(1) == nth { + close(entered) + <-release + } + } + t.Cleanup(func() { expiresPairingHook = nil }) + return entered, release +} + +// setCASPublishHook installs casPublishHook such that exactly the first +// tryCAS to pass through it blocks until release is closed; all later calls +// pass straight through. The hook is cleared when the test ends. +func setCASPublishHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var fired atomic.Bool + casPublishHook = func() { + if fired.CompareAndSwap(false, true) { + close(entered) + <-release + } + } + t.Cleanup(func() { casPublishHook = nil }) + return entered, release +} + +// TestTryCAS_RevalidatesLivenessBeforePublish pins tryCAS's pre-publish +// re-validation: a CAS caller that matched a live value and was then +// descheduled (the replacement's allocation is the widest such window in +// the call) while the value's TTL lapsed must not publish: the value it +// matched is one TryGet already certifies as Miss, and no sequential order +// explains TryGet=Miss followed by CAS=true. The residual instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// Regression test: tryCAS used to sample liveness only before the +// allocation, so the entire alloc + deschedule window sat between the +// liveness sample and the publish. +func TestTryCAS_RevalidatesLivenessBeforePublish(t *testing.T) { + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + retryTTL(t, 20*time.Millisecond, func(ttl time.Duration) bool { + c := New[string, int](MaxAge(ttl)) + c.Set("k", 1) + + entered, release := setCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + select { + case <-entered: + case <-res: + return false // the value expired before the op matched it + } + + // Let the matched value's TTL lapse while the caller is + // parked between its liveness sample and its publish. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + break + } + time.Sleep(time.Millisecond) + } + close(release) + + if <-res { + t.Fatal("published against a value that expired mid-call (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + return true + }) + }) + } +} + +// TestTryCAS_RevalidationCatchesExpire extends the pre-publish re-validation +// pin to manual expiry: a CAS caller that matched a live value and was then +// parked at the replacement's allocation while a concurrent Expire killed +// the value must not publish: Expire's word write is visible to the +// re-validation's fresh liveness check, the same way a lapsed TTL is. Only +// the instructions between the re-validation and the pointer CAS remain +// best effort for Expire, exactly as for a lapsing TTL (see +// CompareAndSwap's doc). +func TestTryCAS_RevalidationCatchesExpire(t *testing.T) { + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := setCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + <-entered + + c.Expire("k") // kills the matched value while the caller is parked + close(release) + + if <-res { + t.Fatal("published against a value a concurrent Expire had already killed (TryGet certifies Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } +} + +// TestTryCAS_RevalidationPairsWordAndClock pins HOW tryCAS's liveness +// checks read expiry: as a coherent (expiry word, clock) pair, the clock +// sampled after the word it judges (expiresNow, like every read path). A +// CAS caller descheduled inside the re-validation itself (between +// sampling the clock and loading the word, in the unpaired ordering) +// would otherwise judge the fresh word against its stale clock and +// publish against a value whose TTL had lapsed, or that a completed +// Expire had killed, while it was parked: state TryGet already certifies +// as Miss, so no sequential order explains TryGet=Miss followed by +// CAS=true followed by TryGet=Hit(new). With the paired read, any kill +// completed before the pair is taken is seen; only the instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// The pairing hook parks the caller inside the re-validation's paired +// read (the call's second paired read; the first is tryCAS's +// pre-allocation filter), and the kill lands while it is parked. +// +// Regression test: casMatches used to evaluate now() and then load the +// word (!l.expired(now())), so a deschedule between the two let both kill +// modes through the re-validation. +func TestTryCAS_RevalidationPairsWordAndClock(t *testing.T) { + const ttl = 200 * time.Millisecond + ops := []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } + kills := []struct { + name string + opts func(time.Duration) []Opt + kill func(c *Cache[string, int]) + }{ + {"ttl_lapse", func(ttl time.Duration) []Opt { return []Opt{MaxAge(ttl)} }, func(c *Cache[string, int]) { + // Let the matched value's TTL lapse while the caller is + // parked; the certifying TryGets' own paired reads pass + // straight through the one-shot hook. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + return + } + time.Sleep(time.Millisecond) + } + }}, + {"expire", func(time.Duration) []Opt { return []Opt{MaxAge(time.Hour)} }, func(c *Cache[string, int]) { + c.Expire("k") + }}, + } + for _, k := range kills { + t.Run(k.name, func(t *testing.T) { + for _, m := range ops { + t.Run(m.name, func(t *testing.T) { + retryTTL(t, ttl, func(ttl time.Duration) bool { + c := New[string, int](k.opts(ttl)...) + c.Set("k", 1) + + entered, release := setExpiresPairingHookNth(t, 2) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + select { + case <-entered: + case <-res: + return false // the value expired before the op matched it + } + + k.kill(c) + close(release) + + if <-res { + t.Fatal("published against a value that was killed mid-validation (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + return true + }) + }) + } + }) + } +} + +// TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry +// word, clock) pair a read classifies with: a reader descheduled between +// loading the word and sampling the clock, with an idle extension landing +// in that window, would otherwise classify the dead pre-extension word at +// a fresh clock and report Stale (with MaxStaleAge) or Miss (without) for +// an entry that was live at every instant of its call: and a later TryGet +// returns Hit with no intervening write, a history no sequential order +// explains. expiresNow re-confirms the word after sampling the clock, so +// the read classifies the extended (live) word instead. +// +// The hook holds the deschedule window open deterministically, like the +// hand-driven claim-window tests above. +// +// Regression test: loadingTryGet and entGet used to classify whatever word +// they loaded first at whatever clock they sampled next. +func TestTryGet_PairedReadVsIdleExtension(t *testing.T) { + for _, tt := range []struct { + name string + opts []Opt + }{ + {"no_stale_age", []Opt{MaxIdleAge(time.Hour)}}, + {"with_stale_age", []Opt{MaxIdleAge(time.Hour), MaxStaleAge(time.Hour)}}, + } { + t.Run(tt.name, func(t *testing.T) { + retryTTL(t, 100*time.Millisecond, func(ttl time.Duration) bool { + c := New[string, int](append(tt.opts, MaxAge(ttl))...) + c.Set("k", 1) + + entered, release := setExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") // loads the initial expiry, parks in the hook + r1c <- res{v, s} + }() + <-entered + + // A second TryGet (the hook is one-shot) hits and extends the + // entry to ~now+1h; the entry is now live well past the + // original expiry. + if _, _, s := c.TryGet("k"); s != Hit { + close(release) // the entry expired before we extended it + <-r1c + return false + } + // Let real time pass the original expiry while the first + // reader is still parked between its two loads. + time.Sleep(ttl + 50*time.Millisecond) + close(release) + r1 := <-r1c + + if v, _, s := c.TryGet("k"); s != Hit || v != 1 { + t.Fatalf("entry must still be live after the extension: (%d, %v)", v, s) + } + if r1.s != Hit || r1.v != 1 { + t.Fatalf("paired read returned (%d, %v) for an entry that was live throughout its call, want (1, Hit)", r1.v, r1.s) + } + return true + }) + }) + } +} + +// TestTryGet_PairedReadVsExpire pins the safe direction of the same window: +// otherwise the word only ever moves backward (Expire, Clean's claim), and +// a parked reader lands on one side of the race or the other: Hit per the +// pre-Expire live word (its call overlaps the Expire, so ordering the read +// first explains the history) or Miss per the expired word. Whichever side +// it lands on, no stale is conjured for an entry with no stale configured. +func TestTryGet_PairedReadVsExpire(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := setExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") + r1c <- res{v, s} + }() + <-entered + + c.Expire("k") // moves the word backward while the reader is parked + close(release) + r1 := <-r1c + + if r1.s == Stale { + t.Fatalf("no stale is configured; got (%d, %v)", r1.v, r1.s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("after Expire: %v, want Miss", s) + } +} + +// TestSwap_OverBornExpired pins the born-expired classification cell of +// Swap's displaced-value report against TryGet: a displaced born-expired +// value (negated-birth expiry word) is the prior value (Stale) while its +// birth-anchored window is open, and Miss after: exactly as TryGet +// reports the same state. +func TestSwap_OverBornExpired(t *testing.T) { + t.Run("window_open", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet: (%d, %v), want (1, Stale)", v, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } + }) + t.Run("window_closed", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(30*time.Millisecond)) + c.Set("k", 1) + time.Sleep(60 * time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet: %v, want Miss", s) + } + if _, _, oldS := c.Swap("k", 2); oldS != Miss { + t.Fatalf("Swap: %v, want Miss to agree with TryGet", oldS) + } + }) +} + +// If an inline miss panics, the panic reaches the caller, and Gets waiting +// on that load run their own load rather than seeing an error. +func TestGet_MissPanic(t *testing.T) { + c := New[string, int]() + started := make(chan struct{}) + release := make(chan struct{}) + leaderDone := make(chan any) + go func() { + defer func() { leaderDone <- recover() }() + c.Get("k", func() (int, error) { + close(started) + <-release + panic("boom") + }) + }() + <-started + + type res struct { + v int + err error + s KeyState + } + waiterDone := make(chan res) + go func() { + v, err, s := c.Get("k", func() (int, error) { return 2, nil }) + waiterDone <- res{v, err, s} + }() + time.Sleep(10 * time.Millisecond) // let the waiter queue on the leader + close(release) + + if r := <-leaderDone; r != "boom" { + t.Fatalf("leader recovered %v, want boom", r) + } + if r := <-waiterDone; r.v != 2 || r.err != nil || r.s != Miss { + t.Fatalf("waiter: (%d, %v, %v), want (2, nil, Miss)", r.v, r.err, r.s) + } + if v, err, s := c.TryGet("k"); v != 2 || err != nil || s != Hit { + t.Fatalf("TryGet: (%d, %v, %v), want (2, nil, Hit)", v, err, s) + } +} + +// A panicked load is never returned or cached, even with errors cached +// forever. +func TestGet_MissPanicNotCached(t *testing.T) { + c := New[string, int]() + func() { + defer func() { recover() }() + c.Get("k", func() (int, error) { panic("boom") }) + }() + if _, err, s := c.TryGet("k"); s != Miss || err != nil { + t.Fatalf("TryGet after panic: (%v, %v), want (nil, Miss)", err, s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries, want 0", n) + } + if v, err, s := c.Get("k", func() (int, error) { return 1, nil }); v != 1 || err != nil || s != Miss { + t.Fatalf("Get after panic: (%d, %v, %v), want (1, nil, Miss)", v, err, s) + } +} + +// A miss that exits its goroutine does not leave the load unfinished. +func TestGet_MissGoexit(t *testing.T) { + c := New[string, int]() + done := make(chan struct{}) + go func() { + defer close(done) + c.Get("k", func() (int, error) { runtime.Goexit(); return 0, nil }) + }() + <-done + if v, _, s := c.Get("k", func() (int, error) { return 1, nil }); v != 1 || s != Miss { + t.Fatalf("Get after Goexit: (%d, %v), want (1, Miss)", v, s) + } +} diff --git a/cache/linearizability_test.go b/cache/linearizability_test.go new file mode 100644 index 0000000..1e7d6a8 --- /dev/null +++ b/cache/linearizability_test.go @@ -0,0 +1,433 @@ +package cache + +import ( + "fmt" + "math/rand/v2" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// This file implements a small linearizability checker: each round runs a +// handful of concurrent operations against one or more keys, records every +// operation's invocation/response order and return values, and then +// verifies that SOME linearization exists for each key: a total order of +// the operations consistent with real time (op A precedes op B if A +// responded before B was invoked) under which the sequential model below +// produces exactly the observed returns. If no such order exists, an +// atomicity bug has been observed and the full history is dumped. Keys are +// independent, so checking each key's history alone is enough. +// +// The clock is frozen for each round (see testNow), so expiry is part of +// the sequential model. +// +// Get is modeled with its singleflight rules rather than as a plain map +// operation: +// +// - A Get that runs its own load takes effect when the load finishes: the +// key must not be visible then, and afterward holds the load's value. +// Until then, the in-flight load looks absent to every other operation. +// - If a Set or Swap replaces the in-flight load, the Get returns that +// value, so the Get reads it at some point after the replacement. +// - If a Delete or Clear removes the in-flight load, the Get still +// returns its load's value, the key stays absent, and Gets that waited +// on the load may also return that value (the model's ghost value). + +type linKind uint8 + +const ( + linSet linKind = iota + linSwap + linDelete + linTryGet + linCAS + linCAD + linExpire + linClean + linGet + linClear +) + +func (k linKind) String() string { + return [...]string{"Set", "Swap", "Delete", "TryGet", "CAS", "CAD", "Expire", "Clean", "Get", "Clear"}[k] +} + +type linOp struct { + kind linKind + arg int // value for Set/Swap, old for CAS/CAD, the load's value for Get + arg2 int // new for CAS + + retV int + retS KeyState + retB bool + + // delOK, for a Get, is whether a Delete that returned Miss or a Clear + // overlapped it, so its in-flight load may have been removed. + delOK bool + + start, end int64 // global ticket counter values around the call +} + +func (o *linOp) String() string { + span := fmt.Sprintf("[%d,%d]", o.start, o.end) + switch o.kind { + case linSet: + return fmt.Sprintf("Set(%d) %s", o.arg, span) + case linSwap: + return fmt.Sprintf("Swap(%d)=(%d,%v) %s", o.arg, o.retV, o.retS, span) + case linDelete: + return fmt.Sprintf("Delete=(%d,%v) %s", o.retV, o.retS, span) + case linTryGet: + return fmt.Sprintf("TryGet=(%d,%v) %s", o.retV, o.retS, span) + case linCAS: + return fmt.Sprintf("CAS(%d,%d)=%v %s", o.arg, o.arg2, o.retB, span) + case linCAD: + return fmt.Sprintf("CAD(%d)=%v %s", o.arg, o.retB, span) + case linGet: + return fmt.Sprintf("Get(load %d)=(%d,%v) delOK=%v %s", o.arg, o.retV, o.retS, o.delOK, span) + default: + return fmt.Sprintf("%v %s", o.kind, span) + } +} + +// linModel is the cache configuration and the frozen clock for a round. +type linModel struct { + ttl, staleAge int64 // MaxAge and MaxStaleAge; 0 means unset + now int64 +} + +type linState struct { + present bool + val int + exp int64 // 0 never expires + ghost int // a removed in-flight Get's value that its waiters may return +} + +func (m linModel) newExp() int64 { + if m.ttl > 0 { + return m.now + m.ttl + } + return 0 +} + +// class returns what TryGet reports for st. +func (m linModel) class(st linState) KeyState { + switch { + case !st.present: + return Miss + case st.exp == 0 || m.now < st.exp: + return Hit + case m.staleAge < 0 || m.staleAge > 0 && m.now < st.exp+m.staleAge: + return Stale + } + return Miss +} + +// linApply returns every state op can leave st in, given the op's recorded +// returns; none if the returns are impossible at this point in the order. +func (m linModel) linApply(st linState, o *linOp) []linState { + readOK := func(s KeyState, v int) bool { + c := m.class(st) + return c == s && (c == Miss || v == st.val) + } + one := func(ok bool, next linState) []linState { + if !ok { + return nil + } + next.ghost = st.ghost + return []linState{next} + } + live := m.class(st) == Hit + switch o.kind { + case linSet: + return one(true, linState{present: true, val: o.arg, exp: m.newExp()}) + case linSwap: + return one(readOK(o.retS, o.retV), linState{present: true, val: o.arg, exp: m.newExp()}) + case linDelete: + return one(readOK(o.retS, o.retV), linState{}) + case linTryGet: + return one(readOK(o.retS, o.retV), st) + case linCAS: + if live && st.val == o.arg { + return one(o.retB, linState{present: true, val: o.arg2, exp: m.newExp()}) + } + return one(!o.retB, st) + case linCAD: + if live && st.val == o.arg { + return one(o.retB, linState{}) + } + return one(!o.retB, st) + case linExpire: + if live { + next := st + next.exp = m.now - 1 + return one(true, next) + } + return one(true, st) + case linClean: + return one(true, st) // removes only what no read can see + case linClear: + return one(true, linState{}) + case linGet: + switch { + case o.retS == Hit: + return one(live && st.val == o.retV || st.ghost != 0 && st.ghost == o.retV, st) + case o.retS != Miss: + return nil + case o.retV != o.arg: + // A Set or Swap replaced our load; we read its value. + return one(live && st.val == o.retV, st) + case live: + return nil // we only load a key that is not visible + } + out := []linState{{present: true, val: o.arg, exp: m.newExp(), ghost: st.ghost}} + if o.delOK { + out = append(out, linState{ghost: o.arg}) + } + return out + } + panic("unreachable") +} + +// linearizable reports whether some real-time-consistent total order of ops +// explains every op's returns starting from initial. +func (m linModel) linearizable(initial linState, ops []*linOp) bool { + used := make([]bool, len(ops)) + var rec func(st linState, depth int) bool + rec = func(st linState, depth int) bool { + if depth == len(ops) { + return true + } + next: + for i, o := range ops { + if used[i] { + continue + } + // o may go next only if no other unscheduled op wholly + // preceded it in real time. + for j, p := range ops { + if !used[j] && j != i && p.end < o.start { + continue next + } + } + for _, st2 := range m.linApply(st, o) { + used[i] = true + if rec(st2, depth+1) { + return true + } + used[i] = false + } + } + return false + } + return rec(initial, 0) +} + +// freezeClock sets now() to a fixed value for the rest of the test. +func freezeClock(t *testing.T) *atomic.Int64 { + testNow.Store(int64(time.Hour)) + t.Cleanup(func() { testNow.Store(0) }) + return &testNow +} + +type linConfig struct { + opts []Opt + model linModel + kinds []linKind + keys int // keys per round + hashFn func(uintptr) uintptr // forces collisions if non-nil + prime func(c *Cache[int, int], k int, st *linState, m *linModel, clock *atomic.Int64) +} + +func runLinearizability(t *testing.T, cfg linConfig, rounds int) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const ( + opsPerKey = 4 + valSpace = 3 // small domain so CAS/CAD old values plausibly match + ) + clock := freezeClock(t) + var c *Cache[int, int] + newCache := func() { + c = New[int, int](cfg.opts...) + if cfg.hashFn != nil { + c.t.hashFn = cfg.hashFn + } + } + newCache() + + var ticket atomic.Int64 + for round := range rounds { + if cfg.hashFn != nil && round%64 == 0 { + newCache() // keep collision chains short + } + m := cfg.model + type keyRound struct { + k int + initial linState + ops []*linOp + } + krs := make([]*keyRound, cfg.keys) + for i := range krs { + kr := &keyRound{k: round*cfg.keys + i} + if rand.IntN(2) == 1 { + kr.initial = linState{present: true, val: 1 + rand.IntN(valSpace)} + m.now = clock.Load() + kr.initial.exp = m.newExp() + c.Set(kr.k, kr.initial.val) // sequentially primed before the round + if cfg.prime != nil { + cfg.prime(c, kr.k, &kr.initial, &m, clock) + } + } + for j := range opsPerKey { + kr.ops = append(kr.ops, &linOp{ + kind: cfg.kinds[rand.IntN(len(cfg.kinds))], + arg: 1 + rand.IntN(valSpace), + arg2: 1 + rand.IntN(valSpace), + }) + if o := kr.ops[j]; o.kind == linGet { + o.arg = 100 + j // each Get's load returns a unique value + } + } + krs[i] = kr + } + m.now = clock.Load() + + var wg sync.WaitGroup + for _, kr := range krs { + for _, o := range kr.ops { + k := kr.k + wg.Go(func() { + o.start = ticket.Add(1) + switch o.kind { + case linSet: + c.Set(k, o.arg) + case linSwap: + o.retV, _, o.retS = c.Swap(k, o.arg) + case linDelete: + o.retV, _, o.retS = c.Delete(k) + case linTryGet: + o.retV, _, o.retS = c.TryGet(k) + case linCAS: + o.retB = c.CompareAndSwap(k, o.arg, o.arg2) + case linCAD: + o.retB = c.CompareAndDelete(k, o.arg) + case linExpire: + c.Expire(k) + case linClean: + c.Clean() + case linClear: + c.Clear() + case linGet: + o.retV, _, o.retS = c.Get(k, func() (int, error) { + for range rand.IntN(3) { + runtime.Gosched() + } + return o.arg, nil + }) + } + o.end = ticket.Add(1) + }) + } + } + wg.Wait() + + // A Clear on any key clears every key in this round. + var clears []*linOp + for _, kr := range krs { + for _, o := range kr.ops { + if o.kind == linClear { + clears = append(clears, o) + } + } + } + for _, kr := range krs { + ops := kr.ops + for _, cl := range clears { + if !contains(ops, cl) { + ops = append(ops, cl) + } + } + for _, o := range ops { + if o.kind != linGet { + continue + } + for _, p := range ops { + overlap := p.start < o.end && o.start < p.end + if overlap && (p.kind == linClear || p.kind == linDelete && p.retS == Miss) { + o.delOK = true + } + } + } + if !m.linearizable(kr.initial, ops) { + var dump strings.Builder + for _, o := range ops { + dump.WriteString("\n\t") + dump.WriteString(o.String()) + } + t.Fatalf("round %d key %d: no linearization explains this history (initial %+v, now %d):%s", + round, kr.k, kr.initial, m.now, dump.String()) + } + } + } +} + +func contains(ops []*linOp, o *linOp) bool { + for _, p := range ops { + if p == o { + return true + } + } + return false +} + +var linMapKinds = []linKind{linSet, linSwap, linDelete, linTryGet, linCAS, linCAD, linExpire, linClean, linGet, linClear} + +// TestLinearizability checks every map operation, Get, Expire, Clean, and +// Clear against the model with no expiry configured. +func TestLinearizability(t *testing.T) { + runLinearizability(t, linConfig{kinds: linMapKinds, keys: 1}, 30_000) +} + +// TestLinearizabilityCollisions is TestLinearizability with several keys +// per round whose hashes collide, so overflow chains and trie expansion and +// pruning race with each other. +func TestLinearizabilityCollisions(t *testing.T) { + runLinearizability(t, linConfig{ + kinds: linMapKinds, + keys: 3, + hashFn: func(h uintptr) uintptr { return h & 3 }, // 4 full hashes, 15 shared nibbles + }, 20_000) +} + +// TestLinearizabilityExpiry checks the map operations, Expire, and Clean +// with MaxAge and MaxStaleAge. Before each round, a primed key is aged into +// its live, stale, or dead phase by moving the frozen clock. +func TestLinearizabilityExpiry(t *testing.T) { + const ttl, staleAge = 100, 50 + for _, sa := range []int64{0, staleAge, -1} { + t.Run(fmt.Sprint("stale_age_", sa), func(t *testing.T) { + runLinearizability(t, linConfig{ + opts: []Opt{MaxAge(ttl), MaxStaleAge(time.Duration(sa))}, + model: linModel{ttl: ttl, staleAge: sa}, + kinds: []linKind{linSet, linSwap, linDelete, linTryGet, linCAS, linCAD, linExpire, linClean}, + keys: 1, + prime: func(c *Cache[int, int], k int, st *linState, m *linModel, clock *atomic.Int64) { + if rand.IntN(4) == 0 { + m.now = clock.Load() + c.Expire(k) + st.exp = m.now - 1 + } + // Age into live, the expiry instant, stale, the window + // close, or dead. + ages := []int64{0, ttl - 1, ttl, ttl + staleAge/2, ttl + staleAge, ttl + 2*staleAge} + clock.Add(ages[rand.IntN(len(ages))]) + }, + }, 20_000) + }) + } +} diff --git a/cache/map_bench_test.go b/cache/map_bench_test.go index aeec419..4c8625f 100644 --- a/cache/map_bench_test.go +++ b/cache/map_bench_test.go @@ -513,3 +513,15 @@ func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { }, }) } + +func BenchmarkClear(b *testing.B) { + benchMap(b, bench{ + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface[int]) { + for ; pb.Next(); i++ { + k, v := i%256, i%256 + m.Clear() + m.Store(k, v) + } + }, + }) +} diff --git a/cache/map_test.go b/cache/map_test.go index 7fb15a8..77114a8 100644 --- a/cache/map_test.go +++ b/cache/map_test.go @@ -276,3 +276,78 @@ func TestCompareAndSwap_NonExistingKey(t *testing.T) { t.Fatalf("CompareAndSwap on an non-existing key succeeded") } } + +// TestConcurrentClear is adapted from sync/map_test.go. Spawns 10 writers, +// 10 readers, and 10 Clear goroutines; correctness is that nothing panics +// or trips the race detector. +func TestConcurrentClear(t *testing.T) { + var m CacheMap[int] + + var wg sync.WaitGroup + wg.Add(30) + + for i := range 10 { + go func(k, v int) { + defer wg.Done() + m.Store(k, v) + }(i, i*10) + } + for i := range 10 { + go func(k int) { + defer wg.Done() + if _, ok := m.Load(k); ok { + _ = ok + } + }(i) + } + for range 10 { + go func() { + defer wg.Done() + m.Clear() + }() + } + + wg.Wait() + + // After all Clears and Stores have run, the map may be empty or contain a + // subset of the Stores depending on interleaving. Verify no phantom keys + // (keys we never Stored) materialized. + m.Range(func(k int, _ any) bool { + if k < 0 || k >= 10 { + t.Errorf("Range after concurrent Clear/Store saw key %d, never Stored", k) + } + return true + }) +} + +// TestMapClearOneAllocation is adapted from sync/map_test.go. Cache.Clear +// replaces the trie root with one fresh indirect node, which should be the +// only allocation. +func TestMapClearOneAllocation(t *testing.T) { + var m CacheMap[int] + // Clear of a never-used cache does nothing, so prime it. + m.Store(0, 0) + allocs := testing.AllocsPerRun(10, func() { + m.Clear() + }) + if allocs > 1 { + t.Errorf("AllocsPerRun of Clear = %v; want 1", allocs) + } +} + +// TestMapRangeNoAllocations is adapted from sync/map_test.go. Range must not +// allocate. +func TestMapRangeNoAllocations(t *testing.T) { + var m CacheMap[int] + for i := range 100 { + m.Store(i, i) + } + allocs := testing.AllocsPerRun(10, func() { + m.Range(func(k int, _ any) bool { + return true + }) + }) + if allocs > 0 { + t.Errorf("AllocsPerRun of Range = %v; want 0", allocs) + } +} diff --git a/cache/trie.go b/cache/trie.go new file mode 100644 index 0000000..dda6370 --- /dev/null +++ b/cache/trie.go @@ -0,0 +1,363 @@ +package cache + +import ( + "hash/maphash" + "sync" + "sync/atomic" + "unsafe" +) + +// trie is a concurrent hash-trie map, modeled on Go's internal/sync +// HashTrieMap. Loads are lock-free. Inserts and deletes lock only the +// indirect node that holds the slot being changed. Deletes prune empty +// indirect nodes bottom-up, locking hand over hand from child to parent. +// +// Unlike internal/sync, we only have userspace tools: hash/maphash gives us +// the runtime's hasher for any comparable type. +type trie[K comparable, V any] struct { + root atomic.Pointer[trieIndirect[K, V]] // nil until first insert + initMu sync.Mutex + seed maphash.Seed + + // hashFn, if non-nil, remaps every hash so that tests can force + // collisions. It takes the hash rather than the key: calling a func + // value with the key would make every key escape to the heap. + hashFn func(uintptr) uintptr +} + +const ( + trieBranchingLog2 = 4 + trieBranching = 1 << trieBranchingLog2 + trieBranchingMask = trieBranching - 1 +) + +// ptrBits is the number of hash bits we use: the trie is at most +// ptrBits/trieBranchingLog2 levels deep. +const ptrBits = uint(unsafe.Sizeof(uintptr(0))) * 8 + +// trieNode is the header shared by trieIndirect and trieEntry; isEntry +// says which one a *trieNode actually is. +type trieNode[K comparable, V any] struct { + isEntry bool +} + +type trieIndirect[K comparable, V any] struct { + trieNode[K, V] + dead atomic.Bool // set once pruned; the node must not be written again + mu sync.Mutex + parent *trieIndirect[K, V] + children [trieBranching]atomic.Pointer[trieNode[K, V]] +} + +// trieEntry is a leaf. Keys whose full hashes are equal are chained through +// overflow. The trie never reads p; the cache owns what it means (see +// entDel). +type trieEntry[K comparable, V any] struct { + trieNode[K, V] + overflow atomic.Pointer[trieEntry[K, V]] + key K + p atomic.Pointer[V] +} + +func newTrieIndirect[K comparable, V any](parent *trieIndirect[K, V]) *trieIndirect[K, V] { + return &trieIndirect[K, V]{parent: parent} +} + +func newTrieEntry[K comparable, V any](key K, value *V) *trieEntry[K, V] { + e := &trieEntry[K, V]{key: key} + e.isEntry = true + if value != nil { + e.p.Store(value) + } + return e +} + +func (n *trieNode[K, V]) entry() *trieEntry[K, V] { + return (*trieEntry[K, V])(unsafe.Pointer(n)) +} + +func (n *trieNode[K, V]) indirect() *trieIndirect[K, V] { + return (*trieIndirect[K, V])(unsafe.Pointer(n)) +} + +func (i *trieIndirect[K, V]) empty() bool { + for j := range i.children { + if i.children[j].Load() != nil { + return false + } + } + return true +} + +// initRoot returns the root, creating it and the hash seed on first use. +// The seed is written before the root is published, so anything that loads +// a non-nil root can hash. +func (t *trie[K, V]) initRoot() *trieIndirect[K, V] { + if i := t.root.Load(); i != nil { + return i + } + t.initMu.Lock() + defer t.initMu.Unlock() + if i := t.root.Load(); i != nil { + return i + } + if t.hashFn == nil { + t.seed = maphash.MakeSeed() + } + i := newTrieIndirect[K, V](nil) + t.root.Store(i) + return i +} + +func (t *trie[K, V]) hash(k K) uintptr { + h := uintptr(maphash.Comparable(t.seed, k)) + if t.hashFn != nil { + h = t.hashFn(h) + } + return h +} + +// loadEntry returns the entry for k, or nil if there is none. +func (t *trie[K, V]) loadEntry(k K) *trieEntry[K, V] { + i := t.root.Load() + if i == nil { + return nil + } + h := t.hash(k) + for shift := ptrBits; shift != 0; { + shift -= trieBranchingLog2 + n := i.children[(h>>shift)&trieBranchingMask].Load() + if n == nil { + return nil + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e + } + } + return nil + } + i = n.indirect() + } + panic("cache: trie ran out of hash bits") +} + +// loadOrStoreEntry returns the existing entry for k and true, or inserts a +// new entry holding value (which may be nil) and returns it and false. +func (t *trie[K, V]) loadOrStoreEntry(k K, value *V) (*trieEntry[K, V], bool) { + t.initRoot() + h := t.hash(k) + var ( + i *trieIndirect[K, V] + shift uint + slot *atomic.Pointer[trieNode[K, V]] + n *trieNode[K, V] + ) + for { + // Find the slot we would insert into without locking, then lock + // its node and retry if the slot has since become an indirect or + // the node was pruned. + i = t.root.Load() + shift = ptrBits + for { + if shift == 0 { + panic("cache: trie ran out of hash bits") + } + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + break + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + break + } + i = n.indirect() + } + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + break + } + i.mu.Unlock() + } + defer i.mu.Unlock() + + var old *trieEntry[K, V] + if n != nil { + old = n.entry() + for e := old; e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + } + e := newTrieEntry(k, value) + if old == nil { + slot.Store(&e.trieNode) + } else { + slot.Store(t.expand(old, e, h, shift, i)) + } + return e, false +} + +// expand returns the node to replace old's slot with so that both old and +// new are reachable: new chained in front of old if their full hashes are +// equal, otherwise indirect nodes down to the first hash bits that differ. +func (t *trie[K, V]) expand(old, new *trieEntry[K, V], newHash uintptr, shift uint, parent *trieIndirect[K, V]) *trieNode[K, V] { + oldHash := t.hash(old.key) + if oldHash == newHash { + new.overflow.Store(old) + return &new.trieNode + } + top := newTrieIndirect(parent) + cur := top + for { + if shift == 0 { + panic("cache: trie ran out of hash bits") + } + shift -= trieBranchingLog2 + oi := (oldHash >> shift) & trieBranchingMask + ni := (newHash >> shift) & trieBranchingMask + if oi != ni { + cur.children[oi].Store(&old.trieNode) + cur.children[ni].Store(&new.trieNode) + return &top.trieNode + } + next := newTrieIndirect(cur) + cur.children[oi].Store(&next.trieNode) + cur = next + } +} + +// deleteEntryIf unlinks and returns the entry for k if pred, evaluated +// while the entry's node is locked, returns true. A nil pred always +// deletes. +func (t *trie[K, V]) deleteEntryIf(k K, pred func(*trieEntry[K, V]) bool) *trieEntry[K, V] { + i := t.root.Load() + if i == nil { + return nil + } + h := t.hash(k) + var ( + shift uint + slot *atomic.Pointer[trieNode[K, V]] + n *trieNode[K, V] + ) + for { + i = t.root.Load() + shift = ptrBits + for { + if shift == 0 { + panic("cache: trie ran out of hash bits") + } + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + return nil + } + if n.isEntry { + break + } + i = n.indirect() + } + i.mu.Lock() + if i.dead.Load() { + i.mu.Unlock() + continue + } + n = slot.Load() + if n == nil { + i.mu.Unlock() + return nil + } + if n.isEntry { + break + } + // A concurrent insert expanded the slot; k may now be deeper. + i.mu.Unlock() + } + + head := n.entry() + var prev, target *trieEntry[K, V] + for e := head; e != nil; prev, e = e, e.overflow.Load() { + if e.key == k { + target = e + break + } + } + if target == nil || pred != nil && !pred(target) { + i.mu.Unlock() + return nil + } + if prev != nil { + prev.overflow.Store(target.overflow.Load()) + i.mu.Unlock() + return target + } + if next := target.overflow.Load(); next != nil { + slot.Store(&next.trieNode) + i.mu.Unlock() + return target + } + slot.Store(nil) + + // Prune now-empty indirect nodes up toward the root. We hold i's lock + // until i is marked dead, so an insert that locks i afterward sees + // dead and retries from the root. + for i.parent != nil && i.empty() { + shift += trieBranchingLog2 + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(h>>shift)&trieBranchingMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return target +} + +// walk calls f on every entry until f returns false. We do not snapshot: +// concurrent changes may or may not be seen, but no key is visited twice. +func (t *trie[K, V]) walk(f func(*trieEntry[K, V]) bool) { + if i := t.root.Load(); i != nil { + walkIndirect(i, f) + } +} + +func walkIndirect[K comparable, V any](i *trieIndirect[K, V], f func(*trieEntry[K, V]) bool) bool { + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + if !n.isEntry { + if !walkIndirect(n.indirect(), f) { + return false + } + continue + } + for e := n.entry(); e != nil; e = e.overflow.Load() { + if !f(e) { + return false + } + } + } + return true +} + +// clear swaps in an empty root. Anything still working on the old tree +// finishes on the old tree. +func (t *trie[K, V]) clear() { + if t.root.Load() != nil { + t.root.Store(newTrieIndirect[K, V](nil)) + } +} diff --git a/cache/trie_test.go b/cache/trie_test.go new file mode 100644 index 0000000..fd26837 --- /dev/null +++ b/cache/trie_test.go @@ -0,0 +1,717 @@ +package cache + +import ( + "fmt" + "hash/maphash" + "math/rand/v2" + "sync" + "sync/atomic" + "testing" +) + +// Helper: store or replace the value slot with a fresh pointer carrying v. +// The trie owns the entry; the caller owns the value pointer identity. +func triePut[K comparable, V any](t *trie[K, V], k K, v V) *V { + p := &v + e, _ := t.loadOrStoreEntry(k, p) + e.p.Store(p) + return p +} + +func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { + return t.deleteEntryIf(k, nil) +} + +// setHashes makes each key in hashes hash to its value in tr. +func setHashes[K comparable, V any](tr *trie[K, V], hashes map[K]uintptr) { + tr.initRoot() // creates the seed + raw := make(map[uintptr]uintptr, len(hashes)) + for k, h := range hashes { + raw[uintptr(maphash.Comparable(tr.seed, k))] = h + } + tr.hashFn = func(h uintptr) uintptr { return raw[h] } +} + +func trieGet[K comparable, V any](t *trie[K, V], k K) (V, bool) { + e := t.loadEntry(k) + if e == nil { + return *new(V), false + } + p := e.p.Load() + if p == nil { + return *new(V), false + } + return *p, true +} + +// verifyTrie checks structural invariants once concurrent operations have +// quiesced: every entry hangs at the position prescribed by its hash, +// overflow chains hold only full-hash collisions, parent pointers are +// consistent, no linked node is marked dead, and no non-root indirect is +// empty (pruning must have removed it). It must not run concurrently with +// mutations. +func verifyTrie[K comparable, V any](t *testing.T, tr *trie[K, V]) { + t.Helper() + if tr.root.Load() == nil { + return + } + root := tr.root.Load() + var walk func(i *trieIndirect[K, V], prefix uintptr, nbits uint) + walk = func(i *trieIndirect[K, V], prefix uintptr, nbits uint) { + if i.dead.Load() { + t.Fatalf("linked indirect at depth %d is marked dead", nbits/trieBranchingLog2) + } + if i != root && i.empty() { + t.Fatalf("empty non-root indirect linked at depth %d (pruning missed it)", nbits/trieBranchingLog2) + } + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + cp := prefix< ptrBits { + t.Fatal("trie deeper than the hash bit budget") + } + if n.isEntry { + head := n.entry() + headHash := tr.hash(head.key) + for e := head; e != nil; e = e.overflow.Load() { + h := tr.hash(e.key) + if h>>(ptrBits-cb) != cp { + t.Fatalf("entry for key %v misplaced: hash %#x does not match its path %#x at depth %d", + e.key, h, cp, cb/trieBranchingLog2) + } + if h != headHash { + t.Fatalf("overflow chain mixes hashes: key %v hash %#x vs head %v hash %#x", + e.key, h, head.key, headHash) + } + } + } else { + ci := n.indirect() + if ci.parent != i { + t.Fatalf("indirect at depth %d has a wrong parent pointer", cb/trieBranchingLog2) + } + walk(ci, cp, cb) + } + } + } + walk(root, 0, 0) +} + +func TestTrie_ZeroValueLoad(t *testing.T) { + var tr trie[string, int] + if e := tr.loadEntry("missing"); e != nil { + t.Fatalf("loadEntry on zero-value trie returned %v, want nil", e) + } + var visited int + tr.walk(func(*trieEntry[string, int]) bool { visited++; return true }) + if visited != 0 { + t.Fatalf("walk on zero-value trie visited %d, want 0", visited) + } + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry on zero-value trie returned %v, want nil", e) + } +} + +func TestTrie_StoreLoad(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + if v, ok := trieGet(&tr, "a"); !ok || v != 1 { + t.Fatalf("get a: v=%d ok=%v, want 1 true", v, ok) + } + if v, ok := trieGet(&tr, "b"); !ok || v != 2 { + t.Fatalf("get b: v=%d ok=%v, want 2 true", v, ok) + } + if _, ok := trieGet(&tr, "c"); ok { + t.Fatal("get c should be missing") + } +} + +func TestTrie_LoadOrStoreExisting(t *testing.T) { + var tr trie[string, int] + v1 := 1 + e1, loaded := tr.loadOrStoreEntry("k", &v1) + if loaded { + t.Fatal("first loadOrStoreEntry should not report loaded") + } + v2 := 2 + e2, loaded := tr.loadOrStoreEntry("k", &v2) + if !loaded { + t.Fatal("second loadOrStoreEntry should report loaded") + } + if e1 != e2 { + t.Fatal("loadOrStoreEntry should return the same entry object on subsequent calls") + } + // The stored value is v1's pointer; v2 was ignored. + if got := *e2.p.Load(); got != 1 { + t.Fatalf("stored value = %d, want 1 (new value must not overwrite)", got) + } +} + +func TestTrie_LoadOrStoreWithNilValue(t *testing.T) { + // Cache always creates entries with a value in place (a nil slot means + // tombstoned), but the trie itself is value-agnostic: verify + // loadOrStoreEntry(k, nil) is legal and the entry.p is nil. + var tr trie[string, int] + e, loaded := tr.loadOrStoreEntry("k", nil) + if loaded { + t.Fatal("expected not-loaded for first insert") + } + if e.p.Load() != nil { + t.Fatal("nil-initialized entry should have nil value slot") + } + v := 42 + e.p.Store(&v) + if got, ok := trieGet(&tr, "k"); !ok || got != 42 { + t.Fatalf("after Store: got=%d ok=%v, want 42 true", got, ok) + } +} + +func TestTrie_Delete(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + + e := tr.deleteEntry("a") + if e == nil || e.key != "a" { + t.Fatalf("deleteEntry(a) = %v, want entry for a", e) + } + if _, ok := trieGet(&tr, "a"); ok { + t.Fatal("after delete, a should be missing") + } + if _, ok := trieGet(&tr, "b"); !ok { + t.Fatal("after delete of a, b should still be present") + } + if e := tr.deleteEntry("a"); e != nil { + t.Fatalf("second deleteEntry(a) = %v, want nil", e) + } +} + +func TestTrie_DeletePrunesEmptyParents(t *testing.T) { + // Insert many keys so the tree grows, then delete them all. After, the + // root should be empty (all children nil). + var tr trie[int, int] + const n = 1000 + for i := range n { + triePut(&tr, i, i) + } + for i := range n { + if e := tr.deleteEntry(i); e == nil { + t.Fatalf("deleteEntry(%d) unexpectedly returned nil", i) + } + } + root := tr.root.Load() + for i := range root.children { + if n := root.children[i].Load(); n != nil { + t.Fatalf("root.children[%d] = %v, want nil (pruning should have cleared)", i, n) + } + } +} + +func TestTrie_Walk(t *testing.T) { + var tr trie[int, int] + const n = 256 + for i := range n { + triePut(&tr, i, i*10) + } + seen := make(map[int]int) + tr.walk(func(e *trieEntry[int, int]) bool { + if p := e.p.Load(); p != nil { + seen[e.key] = *p + } + return true + }) + if len(seen) != n { + t.Fatalf("walk saw %d entries, want %d", len(seen), n) + } + for k, v := range seen { + if v != k*10 { + t.Fatalf("seen[%d] = %d, want %d", k, v, k*10) + } + } +} + +func TestTrie_WalkEarlyStop(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { + count++ + return count < 5 + }) + if count != 5 { + t.Fatalf("walk visited %d, want 5 (early stop)", count) + } +} + +func TestTrie_Clear(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + tr.clear() + for i := range 100 { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("after clear, key %d should be missing", i) + } + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { count++; return true }) + if count != 0 { + t.Fatalf("walk after clear visited %d, want 0", count) + } + // Trie is still usable after clear. + triePut(&tr, 1, 1) + if v, ok := trieGet(&tr, 1); !ok || v != 1 { + t.Fatalf("reuse after clear: v=%d ok=%v", v, ok) + } +} + +// TestTrie_ManyKeys exercises the expand path repeatedly: random keys will +// routinely collide on short hash prefixes, forcing the trie to extend its +// depth. 20k keys is enough that most hash prefixes of length 4 will +// collide multiple times. +func TestTrie_ManyKeys(t *testing.T) { + var tr trie[int, int] + const n = 20_000 + keys := rand.Perm(n) + for _, k := range keys { + triePut(&tr, k, k*2) + } + for _, k := range keys { + v, ok := trieGet(&tr, k) + if !ok || v != k*2 { + t.Fatalf("lookup(%d): v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + // Delete half and re-verify. + for i, k := range keys { + if i%2 == 0 { + tr.deleteEntry(k) + } + } + for i, k := range keys { + v, ok := trieGet(&tr, k) + switch i % 2 { + case 0: + if ok { + t.Fatalf("deleted key %d still present: v=%d", k, v) + } + case 1: + if !ok || v != k*2 { + t.Fatalf("surviving key %d: v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + } + verifyTrie(t, &tr) +} + +// TestTrie_FullHashCollision verifies the overflow-chain behavior by forcing +// two distinct keys to hash identically via the test-only hashFn hook. +func TestTrie_FullHashCollision(t *testing.T) { + var tr trie[string, int] + // Collide everything in a single hash bucket. + tr.hashFn = func(uintptr) uintptr { return 0xdeadbeef } + + const n = 10 + for i := range n { + k := fmt.Sprintf("k%d", i) + triePut(&tr, k, i) + } + // All n entries share one overflow chain under the same leaf slot. + for i := range n { + k := fmt.Sprintf("k%d", i) + v, ok := trieGet(&tr, k) + if !ok || v != i { + t.Fatalf("get %q: v=%d ok=%v, want %d true", k, v, ok, i) + } + } + // Delete one from the middle; the rest remain. + tr.deleteEntry("k5") + if _, ok := trieGet(&tr, "k5"); ok { + t.Fatal("k5 should be deleted") + } + for i := range n { + if i == 5 { + continue + } + k := fmt.Sprintf("k%d", i) + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%q should survive delete of k5", k) + } + } +} + +// TestTrie_FullHashCollisionDeleteHead deletes the head of the overflow +// chain. The chain head is what the slot points at, so slot must be +// updated to the next entry. +func TestTrie_FullHashCollisionDeleteHead(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(uintptr) uintptr { return 0x42 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + + // Delete the most-recently-inserted (head of chain, per expand logic). + if e := tr.deleteEntry("c"); e == nil || e.key != "c" { + t.Fatalf("delete c: %v", e) + } + for _, k := range []string{"a", "b"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s missing after delete of chain head", k) + } + } +} + +func TestTrie_LoadOrStoreDuringCollision(t *testing.T) { + // After a collision chain exists, loadOrStoreEntry for an already-present + // key must find it via the overflow walk, not append a duplicate. + var tr trie[string, int] + tr.hashFn = func(uintptr) uintptr { return 0xabcd } + triePut(&tr, "x", 1) + triePut(&tr, "y", 2) + v := 999 + e, loaded := tr.loadOrStoreEntry("x", &v) + if !loaded { + t.Fatal("second loadOrStoreEntry(x) should report loaded") + } + if got := *e.p.Load(); got != 1 { + t.Fatalf("existing entry value=%d, want 1 (new value must be ignored)", got) + } +} + +// TestTrie_ConcurrentStoreLoad runs many goroutines performing +// loadOrStoreEntry and loadEntry over disjoint key ranges, verifying no +// race and eventual consistency. +func TestTrie_ConcurrentStoreLoad(t *testing.T) { + var tr trie[int, int] + const workers = 8 + const perWorker = 2000 + + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + base := w * perWorker + for i := range perWorker { + triePut(&tr, base+i, base+i) + } + for i := range perWorker { + v, ok := trieGet(&tr, base+i) + if !ok || v != base+i { + t.Errorf("worker %d, key %d: v=%d ok=%v", w, base+i, v, ok) + return + } + } + }) + } + wg.Wait() + verifyTrie(t, &tr) +} + +// TestTrie_ConcurrentSameKey hammers the same key with many goroutines +// doing loadOrStoreEntry. Exactly one must report loaded=false; all others +// must observe the same entry object. +func TestTrie_ConcurrentSameKey(t *testing.T) { + const workers = 64 + for iter := range 100 { + var tr trie[int, int] + var wg sync.WaitGroup + entries := make([]*trieEntry[int, int], workers) + loadeds := make([]bool, workers) + for w := range workers { + wg.Go(func() { + v := w + e, loaded := tr.loadOrStoreEntry(iter, &v) + entries[w] = e + loadeds[w] = loaded + }) + } + wg.Wait() + + var inserted int + for _, l := range loadeds { + if !l { + inserted++ + } + } + if inserted != 1 { + t.Fatalf("iter %d: %d inserters, want exactly 1", iter, inserted) + } + first := entries[0] + for i, e := range entries { + if e != first { + t.Fatalf("iter %d: worker %d saw entry %p, want %p", iter, i, e, first) + } + } + verifyTrie(t, &tr) + } +} + +// TestTrie_ConcurrentDeletePrune exercises concurrent deletes across many +// keys so parent pruning fires from many angles simultaneously. +func TestTrie_ConcurrentDeletePrune(t *testing.T) { + var tr trie[int, int] + const n = 10_000 + for i := range n { + triePut(&tr, i, i) + } + const workers = 8 + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + for i := w; i < n; i += workers { + tr.deleteEntry(i) + } + }) + } + wg.Wait() + for i := range n { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("key %d should be deleted", i) + } + } + // After all deletions, root should be empty. + if root := tr.root.Load(); !root.empty() { + t.Fatal("root not pruned: has surviving children") + } + verifyTrie(t, &tr) +} + +// TestTrie_RangeDuringMutation runs walk concurrently with store and +// delete. No snapshot is promised; we assert only no-crash + that every +// observed key-value pair is plausible (key was stored, value is key*2 or +// nil). +func TestTrie_RangeDuringMutation(t *testing.T) { + var tr trie[int, int] + const n = 2000 + for i := range n { + triePut(&tr, i, i*2) + } + + var stop atomic.Bool + var wg sync.WaitGroup + + wg.Go(func() { + for !stop.Load() { + i := rand.IntN(n) + triePut(&tr, i, i*2) + tr.deleteEntry(i) + } + }) + + wg.Go(func() { + for !stop.Load() { + tr.walk(func(e *trieEntry[int, int]) bool { + p := e.p.Load() + if p != nil && *p != e.key*2 { + t.Errorf("invariant break: key=%d value=%d", e.key, *p) + return false + } + return true + }) + } + }) + + // Let the race run briefly. + for range 50_000 { + i := rand.IntN(n) + if _, ok := trieGet(&tr, i); ok { + // just a read + } + } + stop.Store(true) + wg.Wait() + verifyTrie(t, &tr) +} + +// TestTrie_DeleteMissingInCollisionChain walks an overflow chain in search +// of a key that is never present, exercising deleteEntryIf's +// "walk the chain to the end without finding k" branch. +func TestTrie_DeleteMissingInCollisionChain(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(uintptr) uintptr { return 0x1234 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + // "missing" collides with a/b/c but was never stored. + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry of never-stored colliding key returned %v, want nil", e) + } + // Original entries survive. + for _, k := range []string{"a", "b", "c"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s should survive delete of missing colliding key", k) + } + } +} + +// TestTrie_DeleteRaceRetries stresses deleteEntry's lock-then-retry paths: +// the "parent is dead" retry and the "slot became nil after lock" +// early-return. Both fire when a concurrent delete prunes or empties the +// slot between our unlocked walk and our mu.Lock. +// +// The stress needs sustained concurrent deletes on many hash-adjacent +// keys so that pruning and slot clearing happen continuously. +func TestTrie_DeleteRaceRetries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const workers = 16 + const keySpace = 256 + for range 200 { + var tr trie[int, int] + // Prime: every key is present. + for i := range keySpace { + triePut(&tr, i, i) + } + var wg sync.WaitGroup + // Deleter swarms. + for w := range workers { + wg.Go(func() { + for i := w; i < keySpace; i += workers { + tr.deleteEntry(i) + } + }) + } + // Concurrent deleter attempting same keys (likely to race on lock). + for w := range workers { + wg.Go(func() { + for i := keySpace - 1 - w; i >= 0; i -= workers { + tr.deleteEntry(i) + } + }) + } + wg.Wait() + verifyTrie(t, &tr) + } +} + +// TestTrie_DeleteRaceAgainstExpand targets deleteEntryIf's "slot changed to +// non-entry after lock" branch. An indirect-node split (expand) during the +// window between delete's unlocked walk and its parent.Lock forces the +// reload to see a non-entry node, which must trigger a retry from the root +// (matching internal/sync.HashTrieMap's find), not a give-up: "a" is present +// for the whole round until the delete removes it, so deleteEntry("a") must +// always find and return it. +// +// We use a controlled hashFn that collides a few keys in the top bits but +// not all bits, so inserting them triggers expand. Concurrent insert+delete +// races through the expand window. +func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + // Keys whose hashes match in the top 4 bits but diverge later: they + // share a slot at depth 1 and a leaf at depth 2 (via overflow then + // expand). The constants are built by shifting so they fit a uintptr + // on both 64-bit and 32-bit platforms. + hashes := map[string]uintptr{ + "a": 0xA << (ptrBits - 4), + "b": 0xA<<(ptrBits-4) | 1, + "c": 0xA<<(ptrBits-4) | 2, + "d": 0xB << (ptrBits - 4), + } + for i := range 20_000 { + var tr trie[string, int] + setHashes(&tr, hashes) + triePut(&tr, "a", 1) + triePut(&tr, "d", 4) + + var wg sync.WaitGroup + var deleted *trieEntry[string, int] + wg.Go(func() { + triePut(&tr, "b", 2) + }) + wg.Go(func() { + triePut(&tr, "c", 3) + }) + wg.Go(func() { + deleted = tr.deleteEntry("a") + }) + wg.Wait() + + if deleted == nil || deleted.key != "a" { + t.Fatalf("round %d: deleteEntry(a) = %v, want the entry for a (delete must retry across a concurrent expand)", i, deleted) + } + if e := tr.loadEntry("a"); e != nil { + t.Fatalf("round %d: a still present after deleteEntry returned it", i) + } + for _, k := range []string{"b", "c", "d"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("round %d: %s missing after delete of a", i, k) + } + } + verifyTrie(t, &tr) + } +} + +// TestTrie_DeleteDuringConcurrentStore: a goroutine repeatedly deletes key +// K while another repeatedly stores it. Both succeed; the final state is +// "either present or absent" and neither crashes. +func TestTrie_DeleteDuringConcurrentStore(t *testing.T) { + var tr trie[int, int] + var stop atomic.Bool + var wg sync.WaitGroup + wg.Go(func() { + for !stop.Load() { + triePut(&tr, 7, 42) + } + }) + wg.Go(func() { + for !stop.Load() { + tr.deleteEntry(7) + } + }) + for range 100_000 { + trieGet(&tr, 7) + } + stop.Store(true) + wg.Wait() + verifyTrie(t, &tr) +} + +// deepTrie returns a trie with an indirect node at every level of the path +// for hash 0, which no correct trie has, and makes every key hash to 0. +func deepTrie() *trie[int, int] { + tr := new(trie[int, int]) + i := tr.initRoot() + tr.hashFn = func(uintptr) uintptr { return 0 } + for range ptrBits / trieBranchingLog2 { + n := newTrieIndirect(i) + i.children[0].Store(&n.trieNode) + i = n + } + return tr +} + +// A trie whose depth exceeds the hash bits is corrupt: every walk must +// panic rather than read past the hash. +func TestTrie_RanOutOfHashBitsPanics(t *testing.T) { + for name, f := range map[string]func(){ + "loadEntry": func() { deepTrie().loadEntry(1) }, + "loadOrStoreEntry": func() { deepTrie().loadOrStoreEntry(1, nil) }, + "deleteEntryIf": func() { deepTrie().deleteEntryIf(1, nil) }, + "expand": func() { + tr := new(trie[int, int]) + parent := tr.initRoot() + tr.hashFn = func(h uintptr) uintptr { return 0 } + old := newTrieEntry[int, int](1, nil) + // Different hashes with no bits left to tell them apart. + tr.expand(old, newTrieEntry[int, int](2, nil), 1, 0, parent) + }, + } { + func() { + defer func() { + if r := recover(); r != "cache: trie ran out of hash bits" { + t.Errorf("%s: recovered %v, want the ran out of hash bits panic", name, r) + } + }() + f() + }() + } +} diff --git a/cache/wrappers_test.go b/cache/wrappers_test.go new file mode 100644 index 0000000..f994ed9 --- /dev/null +++ b/cache/wrappers_test.go @@ -0,0 +1,331 @@ +package cache + +import ( + "errors" + "sort" + "sync/atomic" + "testing" + "time" +) + +// Item delegates to Cache[struct{}, V]. These tests exercise every method so +// the thin wrappers contribute to coverage and so a refactor of the underlying +// Cache that forgets to propagate a method would be caught. + +func TestItem_BasicGetAndCaches(t *testing.T) { + i := NewItem[int]() + var calls int32 + v, err, s := i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return 42, nil + }) + if v != 42 || err != nil || !s.IsMiss() { + t.Fatalf("first Get: got v=%d err=%v s=%v, want v=42 err=nil Miss", v, err, s) + } + v, err, s = i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return -1, nil + }) + if v != 42 || err != nil || !s.IsHit() { + t.Fatalf("second Get: got v=%d err=%v s=%v, want v=42 err=nil Hit", v, err, s) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestItem_TryGet(t *testing.T) { + i := NewItem[string]() + _, _, s := i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet on empty item: state=%v want Miss", s) + } + i.Set("hello") + v, err, s := i.TryGet() + if v != "hello" || err != nil || !s.IsHit() { + t.Fatalf("TryGet after Set: v=%q err=%v s=%v", v, err, s) + } +} + +func TestItem_Delete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + v, err, s := i.Delete() + if v != 7 || err != nil || !s.IsHit() { + t.Fatalf("Delete of set value: v=%d err=%v s=%v", v, err, s) + } + _, _, s = i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet after Delete: state=%v want Miss", s) + } + // Deleting an already-empty item is a no-op. + _, _, s = i.Delete() + if !s.IsMiss() { + t.Fatalf("Delete of empty item: state=%v want Miss", s) + } +} + +func TestItem_Expire(t *testing.T) { + i := NewItem[int](MaxAge(time.Hour)) + i.Set(99) + if _, _, s := i.TryGet(); !s.IsHit() { + t.Fatalf("before Expire: want Hit") + } + i.Expire() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Expire: want Miss") + } +} + +func TestItem_Swap(t *testing.T) { + i := NewItem[int]() + old, err, s := i.Swap(5) + if err != nil || !s.IsMiss() { + t.Fatalf("first Swap: old=%d err=%v s=%v, want Miss", old, err, s) + } + old, err, s = i.Swap(6) + if old != 5 || err != nil || !s.IsHit() { + t.Fatalf("second Swap: old=%d err=%v s=%v, want 5 Hit", old, err, s) + } + if v, _, _ := i.TryGet(); v != 6 { + t.Fatalf("after second Swap: TryGet=%d, want 6", v) + } +} + +func TestItem_CompareAndSwap(t *testing.T) { + i := NewItem[int]() + i.Set(10) + if !i.CompareAndSwap(10, 11) { + t.Fatal("CAS 10->11 should succeed") + } + if i.CompareAndSwap(10, 12) { + t.Fatal("CAS 10->12 should fail after value is 11") + } + if v, _, _ := i.TryGet(); v != 11 { + t.Fatalf("after CAS: TryGet=%d, want 11", v) + } +} + +func TestItem_CompareAndDelete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + if i.CompareAndDelete(8) { + t.Fatal("CAD of wrong value should fail") + } + if !i.CompareAndDelete(7) { + t.Fatal("CAD of correct value should succeed") + } + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after CAD: state=%v want Miss", s) + } +} + +func TestItem_Clear(t *testing.T) { + i := NewItem[int]() + i.Set(1) + i.Clear() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", s) + } +} + +func TestItem_ZeroValue(t *testing.T) { + // Docstring says zero-value Item is valid. + var i Item[int] + v, err, s := i.Get(func() (int, error) { return 3, nil }) + if v != 3 || err != nil || !s.IsMiss() { + t.Fatalf("zero Item Get: v=%d err=%v s=%v", v, err, s) + } + if v, _, _ := i.TryGet(); v != 3 { + t.Fatalf("zero Item TryGet: v=%d want 3", v) + } +} + +// Set is a key-only cache over Cache[K, struct{}]. + +func TestSet_BasicGet(t *testing.T) { + s := NewSet[string]() + var calls int32 + err, ks := s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsMiss() { + t.Fatalf("first Get: err=%v state=%v want nil Miss", err, ks) + } + err, ks = s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsHit() { + t.Fatalf("second Get: err=%v state=%v want nil Hit", err, ks) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestSet_GetError(t *testing.T) { + s := NewSet[string]() + wantErr := errors.New("boom") + err, ks := s.Get("a", func() error { return wantErr }) + if !errors.Is(err, wantErr) || !ks.IsMiss() { + t.Fatalf("Get with error: err=%v state=%v", err, ks) + } +} + +func TestSet_TryGet(t *testing.T) { + s := NewSet[string]() + _, ks := s.TryGet("missing") + if !ks.IsMiss() { + t.Fatalf("TryGet missing: state=%v", ks) + } + s.Set("k") + err, ks := s.TryGet("k") + if err != nil || !ks.IsHit() { + t.Fatalf("TryGet after Set: err=%v state=%v", err, ks) + } +} + +func TestSet_Delete(t *testing.T) { + s := NewSet[string]() + s.Set("k") + _, ks := s.Delete("k") + if !ks.IsHit() { + t.Fatalf("Delete existing: state=%v want Hit", ks) + } + _, ks = s.Delete("k") + if !ks.IsMiss() { + t.Fatalf("Delete missing: state=%v want Miss", ks) + } +} + +func TestSet_Expire(t *testing.T) { + s := NewSet[string](MaxAge(time.Hour)) + s.Set("k") + s.Expire("k") + if _, ks := s.TryGet("k"); !ks.IsMiss() { + t.Fatalf("TryGet after Expire: state=%v want Miss", ks) + } +} + +func TestSet_Range(t *testing.T) { + s := NewSet[string]() + for _, k := range []string{"a", "b", "c"} { + s.Set(k) + } + var got []string + s.Range(func(k string, err error) bool { + if err != nil { + t.Errorf("unexpected err for key %q: %v", k, err) + } + got = append(got, k) + return true + }) + sort.Strings(got) + want := []string{"a", "b", "c"} + if len(got) != len(want) { + t.Fatalf("Range yielded %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Range yielded %v, want %v", got, want) + } + } + + // Early stop. + count := 0 + s.Range(func(string, error) bool { + count++ + return false + }) + if count != 1 { + t.Fatalf("Range early-stop visited %d, want 1", count) + } +} + +func TestSet_Clean(t *testing.T) { + s := NewSet[string](MaxAge(time.Nanosecond)) + s.Set("k") + time.Sleep(time.Millisecond) + s.Clean() + if n := trieEntries(&s.c); n != 0 { + t.Fatalf("Clean left %d entries, want 0", n) + } +} + +func TestSet_Clear(t *testing.T) { + s := NewSet[string]() + s.Set("a") + s.Set("b") + s.Clear() + if _, ks := s.TryGet("a"); !ks.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", ks) + } + // Range yields nothing. + var got []string + s.Range(func(k string, _ error) bool { + got = append(got, k) + return true + }) + if len(got) != 0 { + t.Fatalf("Range after Clear: %v, want empty", got) + } +} + +func TestSet_AutoClean(t *testing.T) { + // NewSet must actually start the autoclean goroutine (a regression had + // NewSet building the Cache directly, silently ignoring the option); + // the cleaning behavior itself is covered by the Cache autoclean tests. + s := NewSet[string](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if s.c.quitClean == nil { + t.Fatal("NewSet with AutoCleanInterval should start the autoclean goroutine") + } + s.StopAutoClean() + // Calling twice is safe (sync.Once on the underlying Cache). + s.StopAutoClean() +} + +func TestItem_AutoClean(t *testing.T) { + i := NewItem[int](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if i.c.quitClean == nil { + t.Fatal("NewItem with AutoCleanInterval should start the autoclean goroutine") + } + defer i.c.StopAutoClean() + i.Set(1) + waitEntries(t, &i.c, 0) +} + +func TestSet_ZeroValue(t *testing.T) { + var s Set[string] + err, ks := s.Get("a", func() error { return nil }) + if err != nil || !ks.IsMiss() { + t.Fatalf("zero Set Get: err=%v state=%v", err, ks) + } +} + +func TestSet_GetHit(t *testing.T) { + s := NewSet[string]() + s.Set("k") + if err, st := s.Get("k", func() error { + t.Error("miss ran for a cached key") + return nil + }); err != nil || st != Hit { + t.Fatalf("Get: (%v, %v), want (nil, Hit)", err, st) + } +} + +// A stale Set.Get refreshes in the background. +func TestSet_GetStaleRefresh(t *testing.T) { + s := NewSet[int](MaxAge(0), MaxStaleAge(time.Hour)) + s.Set(1) + ran := make(chan struct{}) + if err, st := s.Get(1, func() error { close(ran); return nil }); err != nil || st != Stale { + t.Fatalf("Get: (%v, %v), want (nil, Stale)", err, st) + } + select { + case <-ran: + case <-time.After(10 * time.Second): + t.Fatal("the background refresh never ran") + } +} diff --git a/go.mod b/go.mod index da78d2d..3e25632 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/twmb/go-cache -go 1.18 +go 1.25 retract v1.1.0 // The newly added API cannot work.