diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7461128b..9d040d1b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -262,3 +262,10 @@ jobs: - name: Run Redis e2e tests working-directory: playgrounds/redis run: pnpm run test:e2e + + # Provider behaviour the playground's single configuration never reaches, + # such as purging with useRedisSets off. + - name: Run provider checks against Redis + run: pnpm run test:redis:live + env: + REDIS_URL: redis://127.0.0.1:6379/15 diff --git a/package.json b/package.json index 9d615119..fe3090f2 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "docs:api": "typedoc", "docs:build:only": "vitepress build docs", "build:providers": "pnpm --filter \"./packages/provider-*\" run build", - "test:cluster:live": "node scripts/redis-cluster-live.cjs" + "test:cluster:live": "node scripts/redis-cluster-live.cjs", + "test:redis:live": "node scripts/redis-standalone-live.cjs" }, "license": "MIT", "devDependencies": { diff --git a/packages/provider-rest-cache-redis/src/RedisCacheProvider.ts b/packages/provider-rest-cache-redis/src/RedisCacheProvider.ts index 5ac26f55..e32772e9 100644 --- a/packages/provider-rest-cache-redis/src/RedisCacheProvider.ts +++ b/packages/provider-rest-cache-redis/src/RedisCacheProvider.ts @@ -44,6 +44,9 @@ export interface RedisClient { unlink(...keys: string[]): Promise; srem(key: string, ...members: string[]): Promise; multi(): RedisMulti; + scan?(cursor: string, ...args: Array): Promise<[string, string[]]>; + /** On an ioredis Cluster: one client per node. */ + nodes?(role: 'master'): RedisClient[]; } /** @@ -248,6 +251,31 @@ export class RedisCacheProvider extends CacheProvider { const store = redisStoreOf(this.cache); const namespace = store._getNamespace?.(); + // With useRedisSets off, @keyv/redis keeps no tracking set and names every + // entry `sets:namespace:::`. Reading the set below returned + // nothing, so every pattern purge matched nothing and entries survived until + // maxAge. Find them by that name instead, on every master when clustered: + // SCAN only sees the node it is sent to. + if (store.opts?.useRedisSets === false && store.redis && namespace && store.namespace) { + const named = `sets:${namespace}:`; + const prefix = `${store.namespace}:`; + const nodes = store.redis.isCluster ? store.redis.nodes?.('master') ?? [] : [store.redis]; + const found: string[] = []; + + for (const node of nodes) { + let cursor = '0'; + do { + const [next, batch] = await node.scan!(cursor, 'MATCH', `${named}*`, 'COUNT', 1000); + cursor = next; + found.push(...batch); + } while (cursor !== '0'); + } + + return found + .map((key) => key.slice(named.length)) + .map((key) => (key.startsWith(prefix) ? key.slice(prefix.length) : key)); + } + if (typeof store.redis?.smembers === 'function' && namespace && store.namespace) { const members = await store.redis.smembers(namespace); diff --git a/scripts/redis-cluster-live.cjs b/scripts/redis-cluster-live.cjs index bd184191..a742f481 100644 --- a/scripts/redis-cluster-live.cjs +++ b/scripts/redis-cluster-live.cjs @@ -117,6 +117,29 @@ async function allKeys(cluster) { assert.deepStrictEqual(await allKeys(cluster), []); }); + // Without redis sets there is no tracking set to read, so keys() has to find + // entries itself - on every master, since SCAN only sees the node it is sent + // to. Reading one node, or the empty set, made pattern purges match nothing. + const unset = new RedisCacheProvider(cluster, { useRedisSets: false }); + unset.cache.stores[0].on('error', (error) => storeErrors.push(error.message)); + + await check('without redis sets: keys() finds entries on every master', async () => { + await flushAll(cluster); + for (const key of KEYS) await unset.set(key, { key }, MAX_AGE); + noStoreErrors(); + const populated = ( + await Promise.all(cluster.nodes('master').map((node) => node.dbsize())) + ).filter((size) => size > 0).length; + assert.ok(populated > 1, 'entries did not spread across masters'); + assert.deepStrictEqual((await unset.keys()).sort(), [...KEYS].sort()); + }); + + await check('without redis sets: purging what keys() lists removes it', async () => { + await unset.delMany(await unset.keys()); + noStoreErrors(); + assert.deepStrictEqual(await allKeys(cluster), []); + }); + await flushAll(cluster); cluster.disconnect(); diff --git a/scripts/redis-standalone-live.cjs b/scripts/redis-standalone-live.cjs new file mode 100644 index 00000000..e6711372 --- /dev/null +++ b/scripts/redis-standalone-live.cjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node +'use strict'; + +/** + * Drives the redis provider against a real standalone Redis, for behaviour the + * e2e suite cannot reach because the playground uses one fixed configuration. + * + * Needs a Redis it may flush: REDIS_URL (default redis://127.0.0.1:6379/15). + */ + +const assert = require('assert'); +const path = require('path'); +const { createRequire } = require('module'); + +const ROOT = path.join(__dirname, '..'); +const PROVIDER = path.join(ROOT, 'packages/provider-rest-cache-redis'); +const requireFromProvider = createRequire(path.join(PROVIDER, 'package.json')); +// The ioredis the provider actually runs on is @keyv/redis's. +const Redis = createRequire(requireFromProvider.resolve('@keyv/redis'))('ioredis'); +const { RedisCacheProvider } = require(path.join(PROVIDER, 'dist/RedisCacheProvider.js')); + +const REDIS_URL = process.env.REDIS_URL || 'redis://127.0.0.1:6379/15'; +const KEYS = ['/api/articles?', '/api/articles/1?populate=*', '/api/categories?locale=fr']; +const MAX_AGE = 600000; + +const results = []; +async function check(name, fn) { + try { + await fn(); + results.push([true, name, '']); + } catch (error) { + results.push([false, name, error.message.split('\n')[0]]); + } +} + +(async () => { + const client = new Redis(REDIS_URL); + + // With useRedisSets off, @keyv/redis keeps no tracking set and names every + // entry `sets:namespace:keyv:keyv:`. keys() read the (empty) tracking + // set anyway and returned nothing, so every pattern purge matched nothing + // and the entries survived until maxAge. + for (const options of [{}, { useRedisSets: false }]) { + const mode = options.useRedisSets === false ? 'without redis sets' : 'with redis sets'; + const provider = new RedisCacheProvider(client, options); + + await check(`${mode}: keys() lists every entry, unqualified`, async () => { + await client.flushdb(); + for (const key of KEYS) await provider.set(key, { key }, MAX_AGE); + assert.deepStrictEqual((await provider.keys()).sort(), [...KEYS].sort()); + }); + + await check(`${mode}: purging what keys() lists removes it`, async () => { + const matched = (await provider.keys()).filter((key) => key.startsWith('/api/articles')); + assert.strictEqual(matched.length, 2, 'pattern matched nothing'); + await provider.delMany(matched); + assert.deepStrictEqual((await provider.keys()).sort(), ['/api/categories?locale=fr']); + for (const key of matched) assert.ok((await provider.get(key)) == null, `${key} survived`); + }); + } + + await client.flushdb(); + await client.quit(); + + console.log('redis standalone (live)\n'); + for (const [ok, name, detail] of results) { + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${detail ? `\n ${detail}` : ''}`); + } + const failed = results.filter(([ok]) => !ok).length; + console.log(`\n${results.length - failed}/${results.length} passed`); + process.exit(failed > 0 ? 1 : 0); +})().catch((error) => { + console.error(error); + process.exit(1); +});