From 268562a59f212da3b59f02ae2764371b9ddc5302 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:38:01 -0600 Subject: [PATCH 01/23] server: --role=master|tx-worker, and with no role nothing changes library/role.js reads the role from --role (or FHIRSMITH_ROLE), the instance code from --instance-code (or FHIRSMITH_INSTANCE_CODE, or modules.tx.instanceCode for the master) and the port from --port. applyRole changes the parsed config.json in place: - master: every module and background job as configured; tx gets the instance code, clusterRole 'master', and the VSAC provider is told it is the leader - tx-worker: every module but tx disabled; the statistics database, npm audit and daily report off (the master has them); its own instance code (required: the config's is the master's); VSAC follower Both turn on the group memory measure (server.memory.group). server.js calls applyRole only when parseRole returns a role, and parseRole returns null when neither --role nor FHIRSMITH_ROLE is set. --port or --instance-code without a role is refused, so a mistyped command doesn't silently start an ordinary server. The other two server.js changes are `if (role)` and `(role && role.port) ||`, so `node server.js` runs exactly as before. Co-Authored-By: Claude Opus 5.5 --- library/role.js | 107 +++++++++++++++++++++++++++++++++++++ server.js | 19 ++++++- tests/library/role.test.js | 89 ++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 library/role.js create mode 100644 tests/library/role.test.js diff --git a/library/role.js b/library/role.js new file mode 100644 index 00000000..b56b59f3 --- /dev/null +++ b/library/role.js @@ -0,0 +1,107 @@ +/** + * Process roles, for running one master and several terminology workers on one data + * directory (see cluster.md). + * + * node server.js no role: everything exactly as before + * node server.js --role=master --instance-code=m0 --port=3001 + * node server.js --role=tx-worker --instance-code=tx1 --port=3002 + * + * The role also comes from FHIRSMITH_ROLE and the instance code from FHIRSMITH_INSTANCE_CODE + * (flags win). With no role, nothing here changes the configuration: server.js only calls + * applyRole when parseRole returns a role. + * + * - master: every configured module, every background job, $closure and the syndication + * feed. The VSAC provider syncs and, after each sync, bumps a generation counter that the + * workers watch. + * - tx-worker: the tx module only. No background jobs, no statistics database, no $closure + * or syndication store (those requests go to the master), and the VSAC provider only + * reads what the master's sync wrote. + * + * @module library/role + */ + +const ROLES = ['master', 'tx-worker']; +const FLAGS = ['role', 'instance-code', 'port']; + +/** + * The role this process was started with, or null for none. + * @param {string[]} argv - process.argv.slice(2) + * @param {Object} env - process.env + * @returns {{role: string, instanceCode: string|null, port: number|null}|null} + */ +function parseRole(argv = [], env = {}) { + const flags = {}; + for (let i = 0; i < argv.length; i++) { + const m = /^--([a-z-]+)(?:=(.*))?$/.exec(argv[i]); + if (m && FLAGS.includes(m[1])) { + flags[m[1]] = m[2] !== undefined ? m[2] : argv[++i]; + } + } + const role = flags.role || env.FHIRSMITH_ROLE || null; + if (!role) { + if (flags['instance-code'] !== undefined || flags.port !== undefined) { + throw new Error('--instance-code and --port are only used with --role (master or tx-worker)'); + } + return null; + } + if (!ROLES.includes(role)) { + throw new Error(`Unknown role '${role}': use master or tx-worker`); + } + let port = null; + if (flags.port !== undefined) { + port = Number(flags.port); + if (!Number.isInteger(port) || port <= 0 || port > 65535) { + throw new Error(`--port '${flags.port}' is not a port number`); + } + } + return { role, instanceCode: flags['instance-code'] || env.FHIRSMITH_INSTANCE_CODE || null, port }; +} + +/** + * Change the configuration (in place) for the role. + * @param {Object} config - the parsed config.json + * @param {{role: string, instanceCode: string|null}} role - from parseRole + * @returns {Object} config + */ +function applyRole(config, role) { + const tx = config.modules && config.modules.tx; + if (!tx || !tx.enabled) { + throw new Error(`--role=${role.role} needs the tx module enabled in config.json`); + } + if (role.role === 'tx-worker' && !role.instanceCode) { + // the config's code is the master's: every process needs its own + throw new Error('A tx-worker needs its own instance code: --instance-code or FHIRSMITH_INSTANCE_CODE'); + } + tx.instanceCode = role.instanceCode || tx.instanceCode; + if (!tx.instanceCode) { + throw new Error('The master needs an instance code: --instance-code, FHIRSMITH_INSTANCE_CODE or modules.tx.instanceCode'); + } + tx.clusterRole = role.role; + if (tx.vsacCfg) { + tx.vsacCfg = { ...tx.vsacCfg, cluster: role.role === 'master' ? 'leader' : 'follower' }; + } + config.server = config.server || {}; + // the memory limit is the whole group's (master and workers in one cgroup) + config.server.memory = { ...(config.server.memory || {}), group: true }; + if (role.role === 'tx-worker') { + for (const [name, mod] of Object.entries(config.modules)) { + if (name !== 'tx' && mod) { + mod.enabled = false; + } + } + // the master keeps the statistics database, runs the npm audit and sends the report + config.stats = { ...(config.stats || {}), enabled: false }; + config.npmAudit = { ...(config.npmAudit || {}), enabled: false }; + config.dailyReport = { ...(config.dailyReport || {}), enabled: false }; + } + return config; +} + +function describeRole(role, config) { + const code = config.modules.tx.instanceCode; + return role.role === 'master' + ? `Role: master (instance code ${code}): all modules, background jobs, $closure` + : `Role: tx-worker (instance code ${code}): the tx module only, no background jobs`; +} + +module.exports = { parseRole, applyRole, describeRole, ROLES }; diff --git a/server.js b/server.js index cc724cd9..8caec884 100644 --- a/server.js +++ b/server.js @@ -33,6 +33,20 @@ try { process.exit(1); } +// --role=master|tx-worker (cluster.md). With no role this is null and the configuration is +// untouched: every `if (role)` below is skipped +const { parseRole, applyRole, describeRole } = require('./library/role'); +let role; +try { + role = parseRole(process.argv.slice(2), process.env); + if (role) { + applyRole(config, role); + } +} catch (error) { + console.error(error.message); + process.exit(1); +} + const Logger = require('./library/logger'); const serverLog = Logger.getInstance(config.logging || {}).child({ module: 'server' }); const packageJson = require('./package.json'); @@ -46,6 +60,9 @@ serverLog.info(`Node.js ${process.version} on ${os.type()} ${os.release()} (${os serverLog.info(`Memory: ${freeMemGB} GB free / ${totalMemGB} GB total`); memoryGuard.configure(config.server.memory || {}); serverLog.info(memoryGuard.describe()); +if (role) { + serverLog.info(describeRole(role, config)); +} serverLog.info(`Data directory: ${folders.dataDir()}`); serverLog.info(`========================================`); @@ -87,7 +104,7 @@ const app = express(); const DEFAULT_TRUST_PROXY = 'loopback, linklocal, uniquelocal'; app.set('trust proxy', config.server.trustProxy ?? DEFAULT_TRUST_PROXY); -const PORT = process.env.PORT || config.server.port || 3000; +const PORT = (role && role.port) || process.env.PORT || config.server.port || 3000; // Middleware // Refuse new requests (503) while memory is near the limit - first, so a refused request diff --git a/tests/library/role.test.js b/tests/library/role.test.js new file mode 100644 index 00000000..755836b1 --- /dev/null +++ b/tests/library/role.test.js @@ -0,0 +1,89 @@ +const { parseRole, applyRole } = require('../../library/role'); + +const baseConfig = () => ({ + server: { port: 3000, memory: { rejectFraction: 0.8 } }, + stats: { intervalMinutes: 5 }, + modules: { + tx: { enabled: true, instanceCode: 'm0', vsacCfg: { apiKey: 'k' }, closure: { enabled: true } }, + packages: { enabled: true }, + xig: { enabled: true }, + registry: { enabled: false } + } +}); + +describe('parseRole', () => { + test('no flags and no environment: no role', () => { + expect(parseRole([], {})).toBeNull(); + // other arguments and an instance code in the environment don't make a role + expect(parseRole(['--inspect', 'foo'], { FHIRSMITH_INSTANCE_CODE: 'tx1', PORT: '4000' })).toBeNull(); + }); + + test('flags, in either form', () => { + expect(parseRole(['--role=tx-worker', '--instance-code=tx1', '--port=20001'], {})) + .toEqual({ role: 'tx-worker', instanceCode: 'tx1', port: 20001 }); + expect(parseRole(['--role', 'master', '--port', '20000'], {})) + .toEqual({ role: 'master', instanceCode: null, port: 20000 }); + }); + + test('the environment, with the flags winning', () => { + expect(parseRole([], { FHIRSMITH_ROLE: 'tx-worker', FHIRSMITH_INSTANCE_CODE: 'tx2' })) + .toEqual({ role: 'tx-worker', instanceCode: 'tx2', port: null }); + expect(parseRole(['--role=master', '--instance-code=a'], { FHIRSMITH_ROLE: 'tx-worker', FHIRSMITH_INSTANCE_CODE: 'b' })) + .toEqual({ role: 'master', instanceCode: 'a', port: null }); + }); + + test('mistakes are refused', () => { + expect(() => parseRole(['--role=worker'], {})).toThrow(/Unknown role/); + expect(() => parseRole(['--role=master', '--port=x'], {})).toThrow(/not a port/); + expect(() => parseRole(['--port=3001'], {})).toThrow(/only used with --role/); + expect(() => parseRole(['--instance-code=tx1'], {})).toThrow(/only used with --role/); + }); +}); + +describe('applyRole', () => { + test('without a role, server.js never calls it: the configuration is untouched', () => { + // what server.js does: applyRole only when parseRole returns a role + const config = baseConfig(); + const before = JSON.stringify(config); + const role = parseRole([], {}); + if (role) { + applyRole(config, role); + } + expect(JSON.stringify(config)).toBe(before); + }); + + test('master: every module, the VSAC sync, the group memory measure', () => { + const config = applyRole(baseConfig(), { role: 'master', instanceCode: null }); + expect(config.modules.tx.instanceCode).toBe('m0'); + expect(config.modules.tx.clusterRole).toBe('master'); + expect(config.modules.tx.vsacCfg).toEqual({ apiKey: 'k', cluster: 'leader' }); + expect(config.modules.packages.enabled).toBe(true); + expect(config.modules.xig.enabled).toBe(true); + expect(config.stats.enabled).toBeUndefined(); + expect(config.server.memory).toEqual({ rejectFraction: 0.8, group: true }); + }); + + test('tx-worker: the tx module only, no background jobs, its own instance code', () => { + const config = applyRole(baseConfig(), { role: 'tx-worker', instanceCode: 'tx1' }); + expect(config.modules.tx.enabled).toBe(true); + expect(config.modules.tx.instanceCode).toBe('tx1'); + expect(config.modules.tx.clusterRole).toBe('tx-worker'); + expect(config.modules.tx.vsacCfg.cluster).toBe('follower'); + expect(config.modules.packages.enabled).toBe(false); + expect(config.modules.xig.enabled).toBe(false); + expect(config.modules.registry.enabled).toBe(false); + expect(config.stats).toEqual({ intervalMinutes: 5, enabled: false }); + expect(config.npmAudit.enabled).toBe(false); + expect(config.dailyReport.enabled).toBe(false); + }); + + test('a worker needs its own instance code, and tx must be enabled', () => { + expect(() => applyRole(baseConfig(), { role: 'tx-worker', instanceCode: null })).toThrow(/its own instance code/); + const noCode = baseConfig(); + delete noCode.modules.tx.instanceCode; + expect(() => applyRole(noCode, { role: 'master', instanceCode: null })).toThrow(/needs an instance code/); + const noTx = baseConfig(); + noTx.modules.tx.enabled = false; + expect(() => applyRole(noTx, { role: 'master', instanceCode: 'm0' })).toThrow(/tx module enabled/); + }); +}); From 38517864d2505235aa8580d6e07ebae826c19590 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:38:01 -0600 Subject: [PATCH 02/23] tx-worker: the master keeps $closure and the syndication store; own local source caches A tx-worker (modules.tx.clusterRole, set by --role) doesn't open the closure database or the syndication store. Both are the master's: the launcher and the nginx example route $closure and /syndication to it. The worker's CapabilityStatement still declares $closure when closure is enabled, since the service offers it; one that reaches a worker anyway gets the existing 501 not-supported. A folder: or file: source wipes and rebuilds its cache folder (terminology-cache/folder-source-) on every load, and a load prunes other such folders untouched for a day. Several processes on one data directory would wipe each other's. With a role, each process's folders carry its instance code (folder-source--) and it doesn't prune. Without a role, clusterRole and localSourceTag are unset and both paths are as before. Co-Authored-By: Claude Opus 5.5 --- tx/library.js | 12 ++++++++++-- tx/tx.js | 12 +++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tx/library.js b/tx/library.js index 5098a813..df3700bd 100644 --- a/tx/library.js +++ b/tx/library.js @@ -103,6 +103,12 @@ class Library { lastTime = null; totalDownloaded = 0; vsacCfg = undefined; + /** + * {string|null} with several processes on one data directory (cluster.md), this one's + * instance code: its folder-source-* caches are its own, since each load wipes and + * rebuilds them, and it leaves the other processes' alone (no pruning) + */ + localSourceTag = null; /** * {SyndicationStore|null} where the content this library loads is recorded for the @@ -249,7 +255,9 @@ class Library { external: endMemory.external - this.startMemory.external }; - await this.#pruneLocalSourceCaches(); + if (!this.localSourceTag) { + await this.#pruneLocalSourceCaches(); + } this.log.info(`Loading Time: ${(totalTime / 1000).toLocaleString()}s`); this.log.info(`Memory Used: ${(memoryIncrease.rss / 1024 / 1024).toFixed(2)} MB`); @@ -923,7 +931,7 @@ class Library { */ async #prepareLocalSourceCache(sourcePath) { const hash = crypto.createHash('sha1').update(sourcePath).digest('hex').substring(0, 16); - const name = LOCAL_SOURCE_CACHE_PREFIX + hash; + const name = LOCAL_SOURCE_CACHE_PREFIX + (this.localSourceTag ? this.localSourceTag + '-' : '') + hash; const cacheSubdir = path.join(this.cacheFolder, name); this.#localSourceCaches.add(name); await fs.rm(cacheSubdir, { recursive: true, force: true }); diff --git a/tx/tx.js b/tx/tx.js index 09d5ec42..4ae9a0a8 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -202,7 +202,10 @@ class TXModule { // the database can't be opened, that stops startup - silently running without it // would be worse. this.closureStore = null; - if (config.closure && config.closure.enabled) { + // a tx-worker (cluster.md) opens neither the closure tables nor the syndication store: + // the master owns them, and $closure and the feed are routed to it + const worker = config.clusterRole === 'tx-worker'; + if (config.closure && config.closure.enabled && !worker) { this.closureStore = new ClosureStore(config.closure, this.log); this.closureStore.open(); this.log.info(`$closure enabled: tables in ${this.closureStore.path}`); @@ -225,7 +228,7 @@ class TXModule { // Initialize metadata handler with config this.metadataHandler = new MetadataHandler({ - closure: this.closureStore != null, + closure: this.closureStore != null || (worker && !!(config.closure && config.closure.enabled)), baseUrl: config.baseUrl, serverVersion: packageJson.version, txVersion: packageJson.txVersion, @@ -244,13 +247,16 @@ class TXModule { // Load the library from YAML this.log.info(`Loading library from: ${config.librarySource}`); this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats); + if (config.clusterRole) { + this.library.localSourceTag = config.instanceCode; + } // The syndication feed: content is recorded in the store as the library loads it, from // the time the administrator turns it on (see tx/syndication). One store serves every // endpoint, since the library is shared. If it is turned on and the store can't be // opened, that stops startup. this.syndicationStore = null; - if (config.syndication && config.syndication.enabled) { + if (config.syndication && config.syndication.enabled && !worker) { this.syndicationStore = new SyndicationStore(path.join(this.library.cacheFolder, 'syndication'), this.log).open(); this.library.syndication = this.syndicationStore; this.log.info(`Syndication feed enabled: store in ${this.syndicationStore.folder}`); From 92708e01a845ca3bc02e440cdeb42c5ce43e91da Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:38:01 -0600 Subject: [PATCH 03/23] VSAC: the workers catch up with the master's sync by a generation counter Ported from #336 (cf38dde5), for the master/worker split only. With vsacCfg.cluster (set by --role): - leader (the master) syncs as before, and bumps vsac_settings 'generation' once the sync has written its value sets (and after a failed run, and after an operator resync of one value set) - follower (a tx-worker) never syncs and runs no refresh timer. It loads the list at start if the database exists, recording the generation (read before the list, so a sync finishing meanwhile makes it load again, never the reverse). When a request uses the value sets (fetch by url or id, search) it reads the generation at most once a second (one read by key) and reloads in the background when it has moved. It doesn't open the database before the master has created it, since opening it would create an empty file. Its resync form is off: the info pages go to the master. Both open the database in WAL mode with a 5 s busy timeout, so a worker's reload doesn't make the master's writes fail. Without a role none of this runs: every new line is behind this.cluster or this.shared. Test: two real processes on one VSAC database, syncing against a fake VSAC. Co-Authored-By: Claude Opus 5.5 --- tests/tx/vsac-generation.test.js | 130 +++++++++++++++++++++++++++++++ tests/utils/vsac-child.js | 73 +++++++++++++++++ tx/vs/vs-database.js | 35 ++++++++- tx/vs/vs-vsac.js | 82 ++++++++++++++++++- 4 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 tests/tx/vsac-generation.test.js create mode 100644 tests/utils/vsac-child.js diff --git a/tests/tx/vsac-generation.test.js b/tests/tx/vsac-generation.test.js new file mode 100644 index 00000000..5d18843c --- /dev/null +++ b/tests/tx/vsac-generation.test.js @@ -0,0 +1,130 @@ +/** + * VSAC with --role (cluster.md): the master's provider (leader) syncs and bumps the + * database's generation; a worker's (follower) never syncs, and when a request uses the + * value sets it compares the generation with the one it loaded (at most once a second), + * reloading only when it differs. Two real processes on one VSAC database. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { fork } = require('child_process'); + +const CHILD = path.join(__dirname, '..', 'utils', 'vsac-child.js'); +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +const vs = (id, version) => ({ + resourceType: 'ValueSet', id, url: `http://cts.nlm.nih.gov/fhir/ValueSet/${id}`, version, status: 'active', + compose: { include: [{ system: 'http://loinc.org', concept: [{ code: '1-8' }] }] } +}); + +describe('VSAC value sets, master and worker', () => { + let dir; + const children = []; + + const spawn = (role) => { + const child = fork(CHILD, [role], { env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', 'inherit', 'inherit', 'ipc'] }); + children.push(child); + let seq = 0; + const waiting = new Map(); + let readyResolve; + const ready = new Promise(resolve => { readyResolve = resolve; }); + child.on('message', (m) => { + if (m.ready) { + readyResolve(); + return; + } + const w = waiting.get(m.seq); + if (m.error) { + w.reject(new Error(m.error)); + } else { + w.resolve(m.result); + } + }); + const call = (op, args = {}) => new Promise((resolve, reject) => { + const s = ++seq; + waiting.set(s, { resolve, reject }); + child.send({ seq: s, op, ...args }); + }); + return ready.then(() => ({ call })); + }; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vsac-gen-')); + }); + afterEach(async () => { + for (const c of children.splice(0)) { + if (c.exitCode === null && c.signalCode === null) { + await new Promise(resolve => { + c.once('exit', resolve); + c.kill(); + }); + } + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('the worker picks up a sync on use, once, and only after it', async () => { + // the master's first sync (of an empty VSAC) takes the generation to 1 + const a = await spawn('leader'); + expect(await a.call('init')).toBe('leader'); + const b = await spawn('follower'); + expect(await b.call('init')).toBe('follower'); + expect(await b.call('catchUpNow')).toBe(1); + + const url = 'http://cts.nlm.nih.gov/fhir/ValueSet/one'; + expect(await b.call('fetch', { url, version: '20250101' })).toBeNull(); + + expect(await a.call('sync', { valueSets: [vs('one', '20250101')] })).toBe(2); + expect(await a.call('fetch', { url, version: '20250101' })).toBe('20250101'); + + // the worker notices on use, within a second or so (the check is throttled, and the + // reload runs in the background, so the request that notices still has the old list) + const before = await b.call('counts'); + let seen = null; + const t0 = Date.now(); + while (Date.now() - t0 < 5000 && !(seen = await b.call('fetch', { url, version: '20250101' }))) { + await sleep(50); + } + expect(seen).toBe('20250101'); + expect(Date.now() - t0).toBeLessThan(2500); + const after = await b.call('counts'); + expect(after.loaded).toBe(2); + expect(after.reloads - before.reloads).toBe(1); + + // nothing changes: many uses over 1.5 s cost at most two generation reads, and no reload + const quiet = await b.call('counts'); + const t1 = Date.now(); + let uses = 0; + while (Date.now() - t1 < 1500) { + await b.call('fetch', { url, version: '20250101' }); + uses++; + } + const end = await b.call('counts'); + expect(uses).toBeGreaterThan(20); + expect(end.generationReads - quiet.generationReads).toBeLessThanOrEqual(2); + expect(end.reloads).toBe(quiet.reloads); + }); + + test('a resync of one value set on the master reaches the worker', async () => { + const a = await spawn('leader'); + await a.call('init'); + const b = await spawn('follower'); + await b.call('init'); + await a.call('sync', { valueSets: [vs('one', '1')] }); + expect(await b.call('catchUpNow')).toBe(2); + const url = 'http://cts.nlm.nih.gov/fhir/ValueSet/one'; + expect(await a.call('resync', { url, valueSets: [vs('one', '1'), vs('one', '2')] })).toBe(3); + expect(await b.call('catchUpNow')).toBe(3); + expect(await b.call('fetch', { url, version: '2' })).toBe('2'); + }); + + test('a worker started before the master has made the database leaves it alone', async () => { + const b = await spawn('follower'); + expect(await b.call('init')).toBe('follower'); + expect(await b.call('catchUpNow')).toBeNull(); + expect(fs.existsSync(path.join(dir, 'terminology-cache', 'vsac', 'vsac-valuesets.db'))).toBe(false); + const a = await spawn('leader'); + await a.call('init'); + expect(await b.call('catchUpNow')).toBe(1); + }); +}); diff --git a/tests/utils/vsac-child.js b/tests/utils/vsac-child.js new file mode 100644 index 00000000..b6ebd86c --- /dev/null +++ b/tests/utils/vsac-child.js @@ -0,0 +1,73 @@ +// A process with the VSAC value set provider, for tests/tx/vsac-generation.test.js: +// FHIRSMITH_DATA_DIR= node vsac-child.js leader|follower +// Its "VSAC" is a list of value sets the test sends: the provider's real refresh runs +// against it (_fetchBundle answers from the list), so a sync writes the database and +// bumps its generation exactly as a real one does. +const { VSACValueSetProvider } = require('../../tx/vs/vs-vsac'); + +const stats = { addTask() {}, task() {}, taskDone() {}, taskError() {} }; +const provider = new VSACValueSetProvider({ apiKey: 'test', cluster: process.argv[2] }, stats); +provider.log = { error() {} }; +let upstream = []; +provider._scanLastUpdated = async () => 0; +provider._fetchBundle = async (url) => { + const m = /[?&]url=([^&]+)/.exec(url); + const list = m ? upstream.filter(vs => vs.url === decodeURIComponent(m[1])) : upstream; + return { resourceType: 'Bundle', total: list.length, entry: list.map(resource => ({ resource })), link: [] }; +}; +// count the generation reads and the reloads, to show the check is throttled +let generationReads = 0; +const getGeneration = provider.database.getGeneration.bind(provider.database); +provider.database.getGeneration = async () => { + generationReads++; + return getGeneration(); +}; +let reloads = 0; +const reload = provider._reloadMap.bind(provider); +provider._reloadMap = async () => { + reloads++; + return reload(); +}; +const origLog = console.log; +console.log = () => {}; + +process.on('message', async (msg) => { + try { + let result; + switch (msg.op) { + case 'init': + await provider.initialize(); + provider.stopRefreshTimer(); + result = provider.cluster; + break; + case 'sync': + upstream = msg.valueSets; + await provider.refreshValueSets(); + result = await provider.database.getGeneration(); + break; + case 'resync': + upstream = msg.valueSets; + await provider.resyncValueSet(msg.url); + result = await provider.database.getGeneration(); + break; + case 'fetch': { + const vs = await provider.fetchValueSet(msg.url, msg.version || ''); + result = vs ? vs.version : null; + break; + } + case 'catchUpNow': + provider.generationCheckedAt = 0; + await provider.catchUp(); + result = provider.loadedGeneration; + break; + case 'counts': + result = { generationReads, reloads, loaded: provider.loadedGeneration }; + break; + } + process.send({ seq: msg.seq, result }); + } catch (e) { + origLog(e); + process.send({ seq: msg.seq, error: e.message }); + } +}); +process.send({ ready: true }); diff --git a/tx/vs/vs-database.js b/tx/vs/vs-database.js index 5ebeae88..e8f0fc5c 100644 --- a/tx/vs/vs-database.js +++ b/tx/vs/vs-database.js @@ -6,6 +6,9 @@ const ValueSet = require("../library/valueset"); // Settings key written once a package cache has been fully populated (see isComplete) const CACHE_COMPLETE_KEY = 'cache-complete'; +// A counter in vsac_settings that the VSAC sync bumps when it has written (cluster.md): +// a process that loaded the value set list at generation n is up to date while it says n +const GENERATION_KEY = 'generation'; // Columns that can be returned directly without parsing JSON const INDEXED_COLUMNS = ['id', 'url', 'version', 'date', 'description', 'name', 'publisher', 'status', 'title']; @@ -43,9 +46,13 @@ class ValueSetDatabase { /** * @param {string} dbPath - Path to the SQLite database file + * @param {Object} [options] - {shared: true} when several processes use the file (one + * writing, the others reading - cluster.md): WAL, so the readers don't block the writer, + * and a wait of up to 5 s for another process's lock instead of failing at once */ - constructor(dbPath) { + constructor(dbPath, options = {}) { this.dbPath = dbPath; + this.shared = !!options.shared; // Single read-write connection used for everything. Using a separate // OPEN_READONLY connection for reads can miss WAL-based schema changes // made through the write connection (because read-only opens can't fully @@ -76,6 +83,10 @@ class ValueSetDatabase { }); return (async () => { + if (this.shared) { + db.configure('busyTimeout', 5000); + await all('PRAGMA journal_mode = WAL'); + } const cols = await all("PRAGMA table_info(valuesets)"); const hasDateFirstSeen = cols.some(c => c.name === 'date_first_seen'); const hasContentHash = cols.some(c => c.name === 'content_hash'); @@ -470,6 +481,28 @@ class ValueSetDatabase { }); } + /** + * The generation (see GENERATION_KEY): 0 until a sync has bumped it. One read by key. + * @returns {Promise} + */ + async getGeneration() { + const value = await this.getSetting(GENERATION_KEY); + return value === null ? 0 : Number(value); + } + + /** + * Add one to the generation, in one statement. + * @returns {Promise} + */ + async bumpGeneration() { + const db = await this._getWriteConnection(); + return new Promise((resolve, reject) => { + db.run(`INSERT INTO vsac_settings (key, value) VALUES (?, '1') + ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1`, + [GENERATION_KEY], err => err ? reject(err) : resolve()); + }); + } + /** * Set a setting value in the vsac_settings table * @param {string} key - The setting key diff --git a/tx/vs/vs-vsac.js b/tx/vs/vs-vsac.js index 337a3573..dd2616c4 100644 --- a/tx/vs/vs-vsac.js +++ b/tx/vs/vs-vsac.js @@ -11,6 +11,10 @@ const { applyIdPrefix, stripIdPrefix } = require("../library/resource-ids"); // Persisted watermark for the phase-1b _lastUpdated scan. const VSAC_LAST_UPDATED_KEY = 'vsac_last_updated_date'; +// A follower (cluster.md) compares its generation with the database's at most this often, +// when a request uses the value sets +const GENERATION_CHECK_MS = 1000; + // Canonical URL prefix for VSAC value sets, so operators can enter a bare OID. const VSAC_VALUESET_URL_PREFIX = 'http://cts.nlm.nih.gov/fhir/ValueSet/'; @@ -30,6 +34,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { * @param {number} [config.timeoutMs=120000] - HTTP request timeout in milliseconds * @param {string} [config.resyncPassword] - If set, enables the operator "resync a ValueSet" * form on the /info page, gated by this password. If unset, the form is not offered. + * @param {string} [config.cluster] - set by --role (library/role.js), else absent: 'leader' + * (the master) syncs, and bumps the database's generation after each sync; 'follower' (a + * tx-worker) never syncs, and reloads its list when the generation moves (catchUp) */ constructor(config, stats) { super(); @@ -52,8 +59,13 @@ class VSACValueSetProvider extends AbstractValueSetProvider { this.refreshIntervalHours = config.refreshIntervalHours || 24; this.dbPath = path.join(this.cacheFolder, 'vsac-valuesets.db'); - this.database = new ValueSetDatabase(this.dbPath); + this.cluster = config.cluster || null; + this.database = new ValueSetDatabase(this.dbPath, { shared: !!this.cluster }); this.valueSetMap = new Map(); + // a follower: the generation its map was loaded at, and when it last compared + this.loadedGeneration = null; + this.generationCheckedAt = 0; + this.catchingUp = null; this.initialized = false; this.refreshTimer = null; this.isRefreshing = false; @@ -88,6 +100,14 @@ class VSACValueSetProvider extends AbstractValueSetProvider { if (this.initialized) { return; } + if (this.cluster === 'follower') { + // the master creates the database and syncs; this process only reads + if (await this.database.exists()) { + await this._reloadMap(); + } + this.initialized = true; + return; + } this.stats.addTask('VSAC Sync', `${this.refreshIntervalHours} hours`); // Create database if it doesn't exist @@ -235,6 +255,10 @@ class VSACValueSetProvider extends AbstractValueSetProvider { tracking.count++; } + // the followers reload when they see the generation move + if (this.cluster) { + await this.database.bumpGeneration(); + } // Reload map with fresh data await this._reloadMap(); @@ -260,6 +284,10 @@ class VSACValueSetProvider extends AbstractValueSetProvider { debugLog(error, 'Error during VSAC refresh:'); this.stats.taskError('VSAC Sync', `Error (${error.message})`); await this.database.failRun(runId, error.message); + if (this.cluster) { + // what the run wrote before it failed is in the database + await this.database.bumpGeneration(); + } throw error; } finally { this.isRefreshing = false; @@ -428,6 +456,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { * @private */ async _reloadMap() { + // the generation before the list: a sync finishing meanwhile makes the next check load + // again, never the reverse + const generation = this.cluster ? await this.database.getGeneration() : null; const newMap = await this.database.loadAllValueSets(this.sourcePackage()); for (const vs of newMap.values()) { if (vs.jsonObj.compose) { @@ -448,6 +479,40 @@ class VSACValueSetProvider extends AbstractValueSetProvider { } // Atomic replacement of the map this.valueSetMap = newMap; + if (this.cluster) { + this.loadedGeneration = generation; + } + } + + /** + * A follower: pick up what the master's sync wrote. At most every GENERATION_CHECK_MS, + * read the database's generation; if it isn't the one the map was loaded at, reload the + * map. Runs in the background (requests meanwhile use the map they have) and never + * rejects. Returns the promise of the check, for tests. + * @returns {Promise} + */ + catchUp() { + const now = Date.now(); + if (this.catchingUp || now - this.generationCheckedAt < GENERATION_CHECK_MS) { + return this.catchingUp || Promise.resolve(); + } + this.generationCheckedAt = now; + this.catchingUp = (async () => { + // not before the master has made it: opening it would create an empty file + if (this.loadedGeneration === null && !(await this.database.exists())) { + return; + } + const generation = await this.database.getGeneration(); + if (generation !== this.loadedGeneration) { + console.log(`VSAC: the database is at generation ${generation}, this process at ${this.loadedGeneration}; reloading`); + await this._reloadMap(); + } + })().catch((error) => { + debugLog(error, 'VSAC: catching up with the database failed'); + }).finally(() => { + this.catchingUp = null; + }); + return this.catchingUp; } /** @@ -458,6 +523,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { */ async fetchValueSet(url, version) { await this.initialize(); + if (this.cluster === 'follower') { + void this.catchUp(); + } this._validateFetchParams(url, version); // Try exact match first: url|version @@ -490,6 +558,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { } async fetchValueSetById(id) { + if (this.cluster === 'follower') { + void this.catchUp(); + } // the map is keyed by the value set's own id (and its url); the value set itself // carries the prefixed id, which is how a url key that happens to match is excluded const vs = this.valueSetMap.get(stripIdPrefix(id, this.spaceId)); @@ -502,6 +573,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { */ async searchValueSets(searchParams, elements) { await this.initialize(); + if (this.cluster === 'follower') { + void this.catchUp(); + } this._validateSearchParams(searchParams); if (searchParams.length === 0) { @@ -703,7 +777,8 @@ class VSACValueSetProvider extends AbstractValueSetProvider { * Whether the on-demand resync form is enabled (a resyncPassword is configured). */ _resyncEnabled() { - return !!this.resyncPassword; + // a follower doesn't write: the info pages are routed to the master + return !!this.resyncPassword && this.cluster !== 'follower'; } /** @@ -731,6 +806,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { async resyncValueSet(url) { const tracking = { totalFetched: 0, totalNew: 0, totalUpdated: 0, count: 0, newCount: 0 }; await this.processContentAndHistory(url, tracking, 1); + if (this.cluster) { + await this.database.bumpGeneration(); + } await this._reloadMap(); return tracking.totalFetched; } From 96f8458141860678fd1c56866807f883fbed8f17 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:38:01 -0600 Subject: [PATCH 04/23] memory guard: with a role, measure the cgroup, and abort only in its largest process The master and its workers run in one cgroup, and the kernel's limit is on all of them together; each measuring its own RSS against that limit lets the group go well over it. With server.memory.group (set by --role) and a cgroup that has the limit on it (MemoryMax on the unit or scope, a container), each process reads the cgroup's private memory (memory.stat anon; v1 total_rss) itself: - every process refuses new requests above rejectFraction - above abortFraction, only the process with the largest RSS in the cgroup (from cgroup.procs and /proc//status) aborts its operations; the others carry on - the in-operation check reuses a measure at most 100 ms old, since it reads a file - the forced GC while refusing needs only this process's share of the excess From #336's guard (9466ddb6 and follow-ups) this keeps the cgroup measure and the largest-aborts rule, and drops the IPC (RSS reports to a primary), the "processes" measure for platforms without cgroups, and draining and recycling the largest worker. The launcher restarts a process the kernel kills. Without a role state.group is null and the guard is unchanged. Co-Authored-By: Claude Opus 5.5 --- library/cgroup-memory.js | 93 +++++++++++++++++++++++- library/memory-guard.js | 72 ++++++++++++++++-- tests/library/memory-guard-group.test.js | 83 +++++++++++++++++++++ 3 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 tests/library/memory-guard-group.test.js diff --git a/library/cgroup-memory.js b/library/cgroup-memory.js index dd8b0ee2..f46a8d09 100644 --- a/library/cgroup-memory.js +++ b/library/cgroup-memory.js @@ -110,4 +110,95 @@ function readCgroupMemoryLimit(opts = {}) { return { limit: 0, source: null }; } -module.exports = { readCgroupMemoryLimit }; +/** + * The directory of this process's own cgroup (memory controller), and whether it is v2 or + * v1. null when there is none to be found (not Linux, or no /proc). + * @param {Object} [opts] - procFile, cgroupRoot (tests) + * @returns {{dir: string, version: 2|1}|null} + */ +function ownCgroupDir(opts = {}) { + const procFile = opts.procFile || '/proc/self/cgroup'; + const cgroupRoot = opts.cgroupRoot || '/sys/fs/cgroup'; + let lines; + try { + lines = fs.readFileSync(procFile, 'utf8').split('\n').filter(l => l.trim() !== ''); + } catch { + return null; + } + const v2 = lines.find(l => l.startsWith('0::')); + if (v2) { + const dir = path.join(cgroupRoot, path.posix.normalize(v2.substring(3) || '/')); + if (fs.existsSync(path.join(dir, 'memory.stat'))) { + return { dir, version: 2 }; + } + } + for (const l of lines) { + const parts = l.split(':'); + if (parts.length >= 3 && parts[1].split(',').includes('memory')) { + const dir = path.join(cgroupRoot, 'memory', path.posix.normalize(parts.slice(2).join(':') || '/')); + if (fs.existsSync(path.join(dir, 'memory.stat'))) { + return { dir, version: 1 }; + } + } + } + return null; +} + +/** + * The private memory of everything in a cgroup: what the kernel can't take back without + * killing a process. The page cache (the terminology and SQLite files) isn't counted: the + * kernel drops it when it needs the room. v2: memory.stat 'anon'; v1: 'total_rss', else 'rss'. + * @param {{dir: string, version: 2|1}} cg - from ownCgroupDir + * @returns {number} bytes, or 0 if it can't be read + */ +function readCgroupAnon(cg) { + let text; + try { + text = fs.readFileSync(path.join(cg.dir, 'memory.stat'), 'utf8'); + } catch { + return 0; + } + const values = new Map(); + for (const line of text.split('\n')) { + const i = line.indexOf(' '); + if (i > 0) { + values.set(line.substring(0, i), Number(line.substring(i + 1))); + } + } + for (const key of cg.version === 2 ? ['anon'] : ['total_rss', 'rss']) { + const v = values.get(key); + if (Number.isFinite(v) && v >= 0) { + return v; + } + } + return 0; +} + +/** + * The RSS of each process in a cgroup, from /proc//status. Processes that have gone, + * or can't be read, are left out. + * @param {{dir: string}} cg - from ownCgroupDir + * @returns {Map} pid -> bytes + */ +function readCgroupProcessRss(cg) { + const result = new Map(); + let pids; + try { + pids = fs.readFileSync(path.join(cg.dir, 'cgroup.procs'), 'utf8').split('\n').filter(Boolean).map(Number); + } catch { + return result; + } + for (const pid of pids) { + try { + const m = /^VmRSS:\s+(\d+) kB/m.exec(fs.readFileSync(`/proc/${pid}/status`, 'utf8')); + if (m) { + result.set(pid, Number(m[1]) * 1024); + } + } catch { + // gone + } + } + return result; +} + +module.exports = { readCgroupMemoryLimit, ownCgroupDir, readCgroupAnon, readCgroupProcessRss }; diff --git a/library/memory-guard.js b/library/memory-guard.js index 232f488e..aaba45c9 100644 --- a/library/memory-guard.js +++ b/library/memory-guard.js @@ -13,13 +13,22 @@ * * Both are configurable in config.server.memory: { "rejectFraction": 0.85, "abortFraction": 0.92 } * + * A master and its tx-workers (--role, cluster.md) share one cgroup, and the kernel's limit is + * on all of them together, so with a role (config.server.memory.group) the measure is the + * group's: the cgroup's private memory (memory.stat anon), read by each process itself. Every + * process refuses new requests above rejectFraction; above abortFraction only the largest + * process in the cgroup aborts its operations, so one heavy request doesn't abort the work of + * every other process. This needs a cgroup with a limit of its own (MemoryMax on the unit or + * scope, a container limit); without one each process measures its own RSS, as one does. + * * @module library/memory-guard */ const os = require('os'); const v8 = require('v8'); const vm = require('vm'); -const { readCgroupMemoryLimit } = require('./cgroup-memory'); +const path = require('path'); +const { readCgroupMemoryLimit, ownCgroupDir, readCgroupAnon, readCgroupProcessRss } = require('./cgroup-memory'); const DEFAULT_REJECT_FRACTION = 0.85; const DEFAULT_ABORT_FRACTION = 0.92; @@ -33,6 +42,9 @@ const GC_INTERVAL_MS = 5000; // next, up to this: when most of RSS is outside the JS heap (the SNOMED and LOINC cache // buffers), collecting frees little, and every full GC stops the process const GC_MAX_INTERVAL_MS = 60000; +// the group measure reads a file: the in-operation check (about ten a request) reuses one +// at most this old +const GROUP_SAMPLE_MS = 100; const state = { limit: 0, @@ -42,7 +54,10 @@ const state = { rejectAt: 0, abortAt: 0, sampleTime: 0, - sampleRss: 0 + sampleRss: 0, + // a role (master, tx-worker): the cgroup measured, from ownCgroupDir; else null + group: null, + groupWanted: false }; function fraction(v, dflt) { @@ -57,11 +72,22 @@ function fraction(v, dflt) { function configure(opts = {}) { let limit = opts.limit; let source = limit ? 'configured' : null; + let cg = null; if (!limit) { - const cg = readCgroupMemoryLimit(); + cg = readCgroupMemoryLimit(); limit = cg.limit; source = cg.source; } + state.group = null; + state.groupWanted = !!opts.group; + if (opts.group) { + // only a cgroup that is the group's own, with the limit on it + const own = opts.cgroup !== undefined ? opts.cgroup : ownCgroupDir(); + const limitFile = opts.cgroup !== undefined ? null : (cg || readCgroupMemoryLimit()).source; + if (own && (opts.cgroup !== undefined || (limitFile && path.dirname(limitFile) === own.dir)) && readCgroupAnon(own) > 0) { + state.group = own; + } + } if (!limit) { limit = os.totalmem(); source = 'os.totalmem()'; @@ -78,7 +104,7 @@ function configure(opts = {}) { function rss(fresh = false) { const now = Date.now(); if (fresh || now - state.sampleTime > SAMPLE_MS) { - state.sampleRss = process.memoryUsage.rss(); + state.sampleRss = (state.group && readCgroupAnon(state.group)) || process.memoryUsage.rss(); state.sampleTime = now; } return state.sampleRss; @@ -132,10 +158,13 @@ function overAbortThreshold(place = 'unknown') { if (state.abortAt <= 0) { return 0; } - const r = rss(true); + const r = state.group ? rss(Date.now() - state.sampleTime > GROUP_SAMPLE_MS) : rss(true); if (r <= state.abortAt) { return 0; } + if (state.group && !largestInGroup()) { + return 0; + } aborts++; const now = Date.now(); if (now - lastAbortLog > 60000) { @@ -152,12 +181,37 @@ function overAbortThreshold(place = 'unknown') { return r; } +let largest = { time: 0, value: false }; + +/** + * A role: is this the process in the cgroup with the most RSS? (Looked at for at most + * GROUP_SAMPLE_MS.) It is the one most likely to hold what took the group over, and the one + * where aborting frees most. + */ +function largestInGroup() { + const now = Date.now(); + if (now - largest.time > GROUP_SAMPLE_MS) { + const own = process.memoryUsage.rss(); + let value = true; + for (const [pid, bytes] of readCgroupProcessRss(state.group)) { + if (pid !== process.pid && bytes > own) { + value = false; + } + } + largest = { time: now, value }; + } + return largest.value; +} + function gb(bytes) { return (bytes / 1024 / 1024 / 1024).toFixed(1); } function describe() { - return `Memory limit: ${gb(state.limit)} GB (from ${state.source}); ` + + const group = !state.groupWanted ? '' : state.group + ? `, for the whole cgroup ${state.group.dir} (its private memory; only its largest process aborts)` + : ', measured as this process\'s RSS (no cgroup limit of the service\'s own to measure as a group)'; + return `Memory limit: ${gb(state.limit)} GB (from ${state.source})${group}; ` + `new requests refused above ${gb(state.rejectAt)} GB (${Math.round(state.rejectFraction * 100)}%), ` + `operations aborted above ${gb(state.abortAt)} GB (${Math.round(state.abortFraction * 100)}%)`; } @@ -204,7 +258,9 @@ function middleware(opts = {}) { const before = state.sampleRss; const heap = heapUsed(); const gc = getGc(); - if (heap < before - state.rejectAt) { + // a group: every process refusing does this, so each need only free its share + const share = state.group ? Math.max(1, readCgroupProcessRss(state.group).size) : 1; + if (heap < (before - state.rejectAt) / share) { gcWait = Math.min(gcWait * 2, gcMaxInterval); if (log) { log.info(`Memory: no forced GC - the JS heap (${gb(heap)} GB) is less than the ${gb(before - state.rejectAt)} GB over the threshold; next look in ${secs(gcWait)}s`); @@ -260,4 +316,4 @@ function middleware(opts = {}) { configure(); -module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, _state: state }; +module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, largestInGroup, _state: state }; diff --git a/tests/library/memory-guard-group.test.js b/tests/library/memory-guard-group.test.js new file mode 100644 index 00000000..a3c37ea7 --- /dev/null +++ b/tests/library/memory-guard-group.test.js @@ -0,0 +1,83 @@ +/** + * The memory guard with a role (master, tx-worker): the measure is the cgroup's private + * memory (memory.stat anon), and only the largest process in the cgroup aborts. + * A fake cgroup directory stands in for /sys/fs/cgroup/. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawn } = require('child_process'); +const memoryGuard = require('../../library/memory-guard'); + +const GB = 1024 * 1024 * 1024; + +describe('memory-guard, a group of processes in one cgroup', () => { + let dir; + let rssSpy; + let other; + + const writeCgroup = (anon, pids) => { + fs.writeFileSync(path.join(dir, 'memory.stat'), `file 123\nanon ${anon}\nshmem 0\n`); + fs.writeFileSync(path.join(dir, 'cgroup.procs'), pids.join('\n') + '\n'); + }; + const configure = () => memoryGuard.configure({ limit: 10 * GB, rejectFraction: 0.8, abortFraction: 0.9, group: true, cgroup: { dir, version: 2 } }); + + beforeAll(async () => { + // another process in the "cgroup", with an RSS of a few tens of MB + other = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); + await new Promise(resolve => setTimeout(resolve, 300)); + }); + afterAll(async () => { + await new Promise(resolve => { + other.once('exit', resolve); + other.kill(); + }); + }); + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cgroup-')); + rssSpy = jest.spyOn(process.memoryUsage, 'rss'); + }); + afterEach(() => { + rssSpy.mockRestore(); + memoryGuard.configure(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('refuses on the group measure, though this process is small', () => { + rssSpy.mockReturnValue(1024 * 1024); + writeCgroup(Math.round(8.5 * GB), [process.pid]); + configure(); + expect(memoryGuard.describe()).toMatch(/for the whole cgroup/); + expect(memoryGuard.shouldReject()).toBe(true); + writeCgroup(1 * GB, [process.pid]); + memoryGuard._state.sampleTime = 0; + expect(memoryGuard.shouldReject()).toBe(false); + }); + + test('over the abort threshold, only the largest process aborts', () => { + writeCgroup(Math.round(9.5 * GB), [process.pid, other.pid]); + configure(); + // smaller than the other process: its operations carry on + rssSpy.mockReturnValue(1024 * 1024); + expect(memoryGuard.overAbortThreshold('test')).toBe(0); + // the largest: it aborts (the answer is kept for 100 ms) + rssSpy.mockReturnValue(5 * GB); + return new Promise(resolve => setTimeout(resolve, 150)).then(() => { + expect(memoryGuard.overAbortThreshold('test')).toBeGreaterThan(9 * GB); + }); + }); + + test('without a cgroup of its own, each process measures its own RSS', () => { + memoryGuard.configure({ limit: 10 * GB, rejectFraction: 0.8, group: true, cgroup: null }); + expect(memoryGuard._state.group).toBeNull(); + expect(memoryGuard.describe()).toMatch(/measured as this process's RSS/); + rssSpy.mockReturnValue(1024 * 1024); + expect(memoryGuard.shouldReject()).toBe(false); + }); + + test('no role: no group, and the describe text is as it was', () => { + memoryGuard.configure({ limit: 10 * GB }); + expect(memoryGuard._state.group).toBeNull(); + expect(memoryGuard.describe()).toMatch(/^Memory limit: 10.0 GB \(from configured\); new requests/); + }); +}); From 8b31d29ff776111e70a9ff9f5495241c237f3962 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:40:50 -0600 Subject: [PATCH 05/23] cluster.js: a master and N tx-workers behind one port An optional launcher, for those who don't put nginx in front. It starts the master (server.js --role=master), waits until it answers /health (on a new data directory the master builds the package caches and creates the VSAC database, which processes starting together would corrupt), listens, and starts the workers (--role=tx-worker, codes tx1..txN, ports after the master's). It proxies with node http and keep-alive agents, by the rules in nginx.md: - X-Cache-Id: to the process its prefix names; if that one is down or unknown, to any tx process, which answers cache-id-unknown - $closure, /syndication, /info/ (the VSAC resync form) in any tx endpoint, and every path outside the tx endpoints: to the master - any other tx request: to the ready tx process with the fewest outstanding requests (the master included unless server.cluster.masterServesTx is false) It appends the client to X-Forwarded-For, so rate limits and logs see the client. A process that exits is started again after 1 s, doubling while it keeps failing quickly, up to a minute. SIGTERM/SIGINT: stop accepting, close idle connections, wait for the requests in progress (drainSeconds), then SIGTERM each process (SIGKILL after 30 s) and exit 0. Tests: the rules and the least-loaded choice, restarts and the drain, against fake upstream processes; and real processes (a master and two workers on a temporary data directory with a fake VSAC): 6 cache sessions through the launcher, 48 later uses, no cache-id-unknown (with cache-id routing disabled the same test fails); $closure and non-tx paths reach the master; a VSAC sync on the master reaches both workers; SIGTERM lets in-flight requests finish, stops every process and exits 0. Co-Authored-By: Claude Opus 5.5 --- cluster.js | 375 +++++++++++++++++++++++ tests/server/cluster-integration.test.js | 231 ++++++++++++++ tests/server/cluster-launcher.test.js | 187 +++++++++++ tests/utils/fake-upstream.js | 22 ++ 4 files changed, 815 insertions(+) create mode 100644 cluster.js create mode 100644 tests/server/cluster-integration.test.js create mode 100644 tests/server/cluster-launcher.test.js create mode 100644 tests/utils/fake-upstream.js diff --git a/cluster.js b/cluster.js new file mode 100644 index 00000000..54671d19 --- /dev/null +++ b/cluster.js @@ -0,0 +1,375 @@ +#!/usr/bin/env node + +// +// Copyright 2025, Health Intersections Pty Ltd (http://www.healthintersections.com.au) +// +// Licensed under BSD-3: https://opensource.org/license/bsd-3-clause +// + +/** + * An optional launcher: one master and N terminology workers on one machine, behind one port + * (see cluster.md). It does what the nginx configuration in nginx.md does, for those who + * don't run nginx in front: + * + * - starts the master (node server.js --role=master), waits until it answers /health, then + * starts the workers (--role=tx-worker), each with its own instance code and port + * - routes: a request with X-Cache-Id to the process that issued it (by the id's prefix); + * $closure, the syndication feed, the source info pages and everything that isn't a tx + * endpoint to the master; any other tx request to the tx process with the fewest + * requests outstanding + * - restarts a process that exits, waiting longer each time it fails quickly (up to a minute) + * - on SIGTERM or SIGINT, stops taking connections, lets the requests in progress finish, + * then stops the processes (each shuts down as a single server does) + * + * node cluster.js [--workers=N] [--port=P] [--master-serves-tx=false] + */ + +const http = require('http'); +const path = require('path'); +const fs = require('fs'); +const { spawn } = require('child_process'); + +const HEALTH_POLL_MS = 250; +const MAX_BACKOFF_MS = 60000; +// a process that ran this long before exiting is restarted at once +const HEALTHY_RUN_MS = 60000; +const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'trailer']; + +/** + * The settings, from the flags, then the environment, then config.json (server.cluster). + * @param {string[]} argv - process.argv.slice(2) + * @param {Object} env - process.env + * @param {Object} config - config.json + */ +function loadSettings(argv, env, config) { + const flags = {}; + for (const a of argv) { + const m = /^--([a-z-]+)=(.*)$/.exec(a); + if (m) { + flags[m[1]] = m[2]; + } + } + const cluster = (config.server && config.server.cluster) || {}; + const tx = (config.modules && config.modules.tx) || {}; + if (!tx.enabled) { + throw new Error('cluster.js runs terminology workers: the tx module must be enabled in config.json'); + } + const port = Number(flags.port || env.PORT || (config.server && config.server.port) || 3000); + const workers = Number(flags.workers || env.FHIRSMITH_WORKERS || cluster.workers || 2); + const internalPort = Number(flags['internal-port'] || cluster.internalPort || port + 1); + const masterCode = tx.instanceCode || 'm0'; + const servesTx = flags['master-serves-tx'] !== undefined ? flags['master-serves-tx'] !== 'false' : cluster.masterServesTx !== false; + const procs = [{ name: 'master', role: 'master', code: masterCode, port: internalPort, servesTx }]; + for (let i = 1; i <= workers; i++) { + const code = 'tx' + i; + if (code === masterCode) { + throw new Error(`modules.tx.instanceCode '${masterCode}' is also the code of a worker: choose another`); + } + procs.push({ name: 'worker ' + code, role: 'tx-worker', code, port: internalPort + i, servesTx: true }); + } + return { + port, + procs, + txPaths: (tx.endpoints || []).map(e => String(e.path).replace(/\/+$/, '')), + drainMs: Number(cluster.drainSeconds || 30) * 1000 + }; +} + +/** + * Where a request goes, by the rules in nginx.md. + * @param {string} url - the request target (path and query) + * @param {Object} headers - the request headers (lower case names) + * @param {string[]} txPaths - the tx endpoint paths, e.g. ['/r4', '/r5'] + * @returns {{to: 'master'}|{to: 'owner', code: string}|{to: 'tx'}} + */ +function classify(url, headers, txPaths) { + let p = String(url).split('?')[0]; + try { + p = decodeURIComponent(p); + } catch { + // leave it as it is + } + const ep = txPaths.find(e => p === e || p.startsWith(e + '/')); + if (!ep) { + return { to: 'master' }; + } + const rest = p.substring(ep.length); + // the closure tables and the syndication store are the master's; the source info pages + // carry the VSAC resync form, which writes the VSAC database + if (/^\/\$closure\/?$/.test(rest) || /^\/syndication(\/|$)/.test(rest) || rest.startsWith('/info/')) { + return { to: 'master' }; + } + const m = /^([A-Za-z0-9]{1,16})\./.exec(headers['x-cache-id'] || ''); + return m ? { to: 'owner', code: m[1] } : { to: 'tx' }; +} + +/** + * The process to send a request to, or null if none is ready. + * @param {Object} route - from classify + * @param {Object[]} procs - {code, role, servesTx, ready, outstanding} + */ +let rotation = 0; +function pickTarget(route, procs) { + if (route.to === 'master') { + return procs.find(p => p.role === 'master' && p.ready) || null; + } + if (route.to === 'owner') { + const owner = procs.find(p => p.code === route.code && p.ready && p.servesTx); + if (owner) { + return owner; + } + // the owner is down (or the code is unknown): any tx process answers cache-id-unknown, + // which the client recovers from by starting a new cache + } + const candidates = procs.filter(p => p.servesTx && p.ready); + if (candidates.length === 0) { + return null; + } + rotation = (rotation + 1) % candidates.length; + let best = null; + for (let i = 0; i < candidates.length; i++) { + const c = candidates[(rotation + i) % candidates.length]; + if (!best || c.outstanding < best.outstanding) { + best = c; + } + } + return best; +} + +function isLoopback(address) { + return address === '127.0.0.1' || address === '::1'; +} + +/** + * @param {Object} settings - from loadSettings, plus optionally + * command(proc): [executable, args] to start a process (tests), healthPath, log + */ +function createLauncher(settings) { + const log = settings.log || ((msg) => console.log(`[cluster] ${msg}`)); + const healthPath = settings.healthPath || '/health'; + const command = settings.command || ((proc) => [process.execPath, [ + ...new Set(['--expose-gc', ...process.execArgv]), path.join(__dirname, 'server.js'), + `--role=${proc.role}`, `--instance-code=${proc.code}`, `--port=${proc.port}`]]); + const procs = settings.procs.map(p => ({ + ...p, child: null, ready: false, outstanding: 0, failures: 0, startedAt: 0, restartTimer: null, + agent: new http.Agent({ keepAlive: true, maxSockets: 256 }) + })); + let server = null; + let stopping = false; + let onIdle = null; + + function totalOutstanding() { + return procs.reduce((n, p) => n + p.outstanding, 0); + } + + function startProcess(proc) { + return new Promise((resolve) => { + if (stopping) { + return resolve(false); + } + const [cmd, args] = command(proc); + proc.startedAt = Date.now(); + const child = spawn(cmd, args, { stdio: 'inherit', env: process.env }); + proc.child = child; + let poll = null; + child.on('exit', (code, signal) => { + clearTimeout(poll); + proc.ready = false; + proc.child = null; + resolve(false); + if (stopping) { + return; + } + proc.failures = Date.now() - proc.startedAt > HEALTHY_RUN_MS ? 1 : proc.failures + 1; + const delay = Math.min(MAX_BACKOFF_MS, 1000 * 2 ** (proc.failures - 1)); + log(`${proc.name} exited (${signal || code}); restarting in ${delay / 1000} s`); + proc.restartTimer = setTimeout(() => startProcess(proc), delay); + }); + const check = () => { + const req = http.get({ host: '127.0.0.1', port: proc.port, path: healthPath, agent: false }, (res) => { + res.resume(); + if (res.statusCode === 200 && proc.child === child) { + proc.ready = true; + log(`${proc.name} (${proc.code}) ready on port ${proc.port}`); + resolve(true); + } else { + poll = setTimeout(check, HEALTH_POLL_MS); + } + }); + req.on('error', () => { + if (proc.child === child) { + poll = setTimeout(check, HEALTH_POLL_MS); + } + }); + }; + poll = setTimeout(check, HEALTH_POLL_MS); + }); + } + + function proxy(req, res) { + const target = pickTarget(classify(req.url, req.headers, settings.txPaths), procs); + if (!target) { + return fail(res, 503, 'No server process is ready to take this request; try again shortly'); + } + target.outstanding++; + let finished = false; + const finish = () => { + if (!finished) { + finished = true; + target.outstanding--; + if (onIdle && totalOutstanding() === 0) { + onIdle(); + } + } + }; + const headers = { ...req.headers }; + for (const h of HOP_BY_HOP) { + delete headers[h]; + } + const client = (req.socket.remoteAddress || '').replace(/^::ffff:/, ''); + headers['x-forwarded-for'] = headers['x-forwarded-for'] ? `${headers['x-forwarded-for']}, ${client}` : client; + if (!headers['x-forwarded-proto'] || !isLoopback(client)) { + headers['x-forwarded-proto'] = 'http'; + } + const upstream = http.request({ + host: '127.0.0.1', port: target.port, method: req.method, path: req.url, headers, agent: target.agent + }, (ures) => { + const out = { ...ures.headers }; + for (const h of HOP_BY_HOP) { + delete out[h]; + } + if (stopping) { + out.connection = 'close'; + } + res.writeHead(ures.statusCode, ures.statusMessage, out); + ures.pipe(res); + ures.on('error', () => res.destroy()); + }); + upstream.on('error', (err) => { + if (!res.headersSent) { + fail(res, 502, `The ${target.name} process did not answer (${err.code || err.message})`); + } else { + res.destroy(); + } + finish(); + }); + res.on('close', () => { + if (!res.writableFinished) { + upstream.destroy(); + } + finish(); + }); + req.pipe(upstream); + } + + function fail(res, status, text) { + res.writeHead(status, { 'Content-Type': 'application/fhir+json', 'Retry-After': '5', ...(stopping ? { Connection: 'close' } : {}) }); + res.end(JSON.stringify({ + resourceType: 'OperationOutcome', + issue: [{ severity: 'error', code: 'transient', details: { text }, diagnostics: text }] + })); + } + + async function start() { + const master = procs.find(p => p.role === 'master'); + // the master first, and on its own: on a new data directory it builds the package + // caches and creates the VSAC database, which processes starting together would corrupt. + // (If it exits before it is ready, it is started again, as any process is.) + log(`starting the master (${master.code}) on port ${master.port}`); + void startProcess(master); + await new Promise(resolve => { + const t = setInterval(() => { + if (master.ready || stopping) { + clearInterval(t); + resolve(); + } + }, HEALTH_POLL_MS); + }); + if (stopping) { + return; + } + server = http.createServer(proxy); + server.keepAliveTimeout = 65000; + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(settings.port, resolve); + }); + log(`listening on port ${settings.port}; starting ${procs.length - 1} worker(s)`); + for (const p of procs) { + if (p !== master) { + void startProcess(p); + } + } + } + + /** + * Stop taking connections, wait for the requests in progress (up to drainMs), then stop + * every process (SIGTERM, then SIGKILL after 30 s). + */ + async function stop() { + if (stopping) { + return; + } + stopping = true; + for (const p of procs) { + clearTimeout(p.restartTimer); + } + if (server) { + server.close(); + server.closeIdleConnections(); + const idle = setInterval(() => server.closeIdleConnections(), 100); + if (totalOutstanding() > 0) { + log(`draining: ${totalOutstanding()} request(s) in progress`); + await new Promise(resolve => { + const timer = setTimeout(resolve, settings.drainMs); + onIdle = () => { + clearTimeout(timer); + resolve(); + }; + }); + } + clearInterval(idle); + server.closeAllConnections(); + } + await Promise.all(procs.filter(p => p.child).map(p => new Promise(resolve => { + const child = p.child; + const kill = setTimeout(() => child.kill('SIGKILL'), 30000); + child.once('exit', () => { + clearTimeout(kill); + resolve(); + }); + child.kill('SIGTERM'); + }))); + for (const p of procs) { + p.agent.destroy(); + } + log('stopped'); + } + + return { start, stop, procs, get server() { return server; } }; +} + +async function main() { + const folders = require('./library/folder-setup'); + let launcher; + try { + const config = JSON.parse(fs.readFileSync(folders.filePath('config.json'), 'utf8')); + launcher = createLauncher(loadSettings(process.argv.slice(2), process.env, config)); + } catch (error) { + console.error(`cluster.js: ${error.message}`); + process.exit(1); + } + const shutdown = () => launcher.stop().then(() => process.exit(0)); + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); + await launcher.start(); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} + +module.exports = { loadSettings, classify, pickTarget, createLauncher }; diff --git a/tests/server/cluster-integration.test.js b/tests/server/cluster-integration.test.js new file mode 100644 index 00000000..9cfd489b --- /dev/null +++ b/tests/server/cluster-integration.test.js @@ -0,0 +1,231 @@ +/** + * cluster.js with real processes: a master and two tx-workers on a temporary data + * directory, behind the launcher's port, with a fake VSAC. + * + * - $cache-control sessions started through the launcher are used on later requests, + * and every one of those requests reaches the process that holds the cache + * - $closure and the non-tx paths reach the master + * - a VSAC sync on the master is seen by both workers + * - SIGTERM: the requests in progress finish, every process stops, the launcher exits 0 + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const http = require('http'); +const { spawn } = require('child_process'); + +const ROOT = path.join(__dirname, '..', '..'); +const FRONT = 20020; +const MASTER = 20021; +const WORKERS = [20022, 20023]; +const VSAC = 20029; +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +const GENDER = 'http://hl7.org/fhir/administrative-gender'; +const vsacValueSet = (id) => ({ + resourceType: 'ValueSet', id, url: `http://cts.nlm.nih.gov/fhir/ValueSet/${id}`, version: '1', status: 'active', + compose: { include: [{ system: GENDER, concept: [{ code: 'male' }] }] } +}); + +function request(port, method, urlPath, { headers = {}, body } = {}) { + return new Promise((resolve, reject) => { + const data = body ? Buffer.from(JSON.stringify(body)) : null; + const req = http.request({ + host: '127.0.0.1', port, method, path: urlPath, agent: false, + headers: { Accept: 'application/fhir+json', ...(data ? { 'Content-Type': 'application/fhir+json' } : {}), ...headers } + }, (res) => { + let text = ''; + res.on('data', d => { text += d; }); + res.on('end', () => { + let json = null; + try { + json = JSON.parse(text); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on('error', reject); + req.end(data); + }); +} + +const isUp = (port) => request(port, 'GET', '/health').then(r => r.status === 200, () => false); +const param = (body, name) => ((body && body.parameter) || []).find(p => p.name === name); +const unknownCache = (r) => r.json && r.json.resourceType === 'OperationOutcome' && + (r.json.issue || []).some(i => ((i.details && i.details.coding) || []).some(c => c.code === 'cache-id-unknown')); + +describe('master and two tx-workers behind cluster.js', () => { + let dir; + let launcher; + let launcherExit; + let vsacServer; + const vsacList = [vsacValueSet('one')]; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-')); + // the R4 core package, if a test run has already downloaded it, saves fetching it again + const core = path.join(ROOT, 'data', 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'); + if (fs.existsSync(core)) { + fs.cpSync(core, path.join(dir, 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'), { recursive: true }); + } + fs.writeFileSync(path.join(dir, 'library.yml'), 'base:\n url: https://storage.googleapis.com/tx-fhir-org\n\nsources:\n - internal:vsac=vsac\n'); + fs.writeFileSync(path.join(dir, 'config.json'), JSON.stringify({ + hostName: 'cluster-test', + logging: { console: false }, + server: { port: FRONT, cluster: { workers: 2, internalPort: MASTER } }, + modules: { + vcl: { enabled: true }, + tx: { + enabled: true, + instanceCode: 'm0', + librarySource: path.join(dir, 'library.yml'), + endpoints: [{ path: '/r4', fhirVersion: '4.0', context: null }], + closure: { enabled: true }, + // a sync every 1.8 s + vsacCfg: { apiKey: 'test', baseUrl: `http://127.0.0.1:${VSAC}/fhir`, refreshIntervalHours: 0.0005 } + } + } + })); + + // a fake VSAC: every listing is the whole list, ?url= the one asked for + vsacServer = http.createServer((req, res) => { + const u = new URL(req.url, 'http://x'); + const url = u.searchParams.get('url'); + const list = url ? vsacList.filter(v => v.url === url) : vsacList; + res.setHeader('Content-Type', 'application/fhir+json'); + res.end(JSON.stringify({ resourceType: 'Bundle', type: 'searchset', total: list.length, entry: list.map(resource => ({ resource })), link: [] })); + }); + await new Promise(resolve => vsacServer.listen(VSAC, resolve)); + + const out = fs.openSync(path.join(dir, 'processes.log'), 'a'); + launcher = spawn(process.execPath, [path.join(ROOT, 'cluster.js')], { + cwd: ROOT, env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', out, out] + }); + launcherExit = new Promise(resolve => launcher.once('exit', (code, signal) => resolve({ code, signal }))); + const t0 = Date.now(); + for (const port of [MASTER, ...WORKERS, FRONT]) { + while (!(await isUp(port))) { + if (Date.now() - t0 > 300000 || launcher.exitCode !== null) { + throw new Error(`port ${port} did not come up; see ${path.join(dir, 'processes.log')}`); + } + await sleep(250); + } + } + // the launcher marks a worker ready a moment after it first answers + await sleep(500); + }, 330000); + + afterAll(async () => { + if (launcher && launcher.exitCode === null && launcher.signalCode === null) { + launcher.kill('SIGTERM'); + await launcherExit; + } + if (vsacServer) { + await new Promise(resolve => vsacServer.close(resolve)); + } + if (dir) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + test('each process runs what its role says', async () => { + // non-tx paths reach the master: it alone has the vcl module + const viaFront = await request(FRONT, 'GET', '/health'); + expect(Object.keys(viaFront.json.modules).sort()).toEqual(['tx', 'vcl']); + for (const port of WORKERS) { + expect(Object.keys((await request(port, 'GET', '/health')).json.modules)).toEqual(['tx']); + } + expect((await request(FRONT, 'GET', '/VCL?vcl=' + encodeURIComponent('(http://loinc.org)1-8'))).status).toBe(200); + }); + + test('$cache-control sessions started through the launcher hold on every later request', async () => { + const issuers = new Set(); + let unknown = 0; + let ok = 0; + for (let s = 0; s < 6; s++) { + const start = await request(FRONT, 'POST', '/r4/$cache-control?mode=start', { body: { resourceType: 'Parameters', parameter: [] } }); + expect(start.status).toBe(200); + const cid = param(start.json, 'cache-id').valueId; + issuers.add(cid.split('.')[0]); + const cs = { resourceType: 'CodeSystem', url: `http://example.org/cs${s}`, version: '1', status: 'active', content: 'complete', concept: [{ code: 'a' }] }; + const vs = { resourceType: 'ValueSet', url: `http://example.org/vs${s}`, version: '1', status: 'active', compose: { include: [{ system: cs.url }] } }; + // the first request leaves the resources in the cache; the later ones refer to them by url + const first = await request(FRONT, 'POST', '/r4/ValueSet/$validate-code', { + headers: { 'X-Cache-Id': cid }, + body: { resourceType: 'Parameters', parameter: [{ name: 'tx-resource', resource: cs }, { name: 'tx-resource', resource: vs }, + { name: 'url', valueUri: vs.url }, { name: 'coding', valueCoding: { system: cs.url, code: 'a' } }] } + }); + expect(param(first.json, 'result').valueBoolean).toBe(true); + for (let i = 0; i < 8; i++) { + const r = await request(FRONT, 'POST', '/r4/ValueSet/$validate-code', { + headers: { 'X-Cache-Id': cid }, + body: { resourceType: 'Parameters', parameter: [{ name: 'url', valueUri: vs.url }, { name: 'coding', valueCoding: { system: cs.url, code: 'a' } }] } + }); + if (unknownCache(r)) { + unknown++; + } else if (r.status === 200 && param(r.json, 'result').valueBoolean === true) { + ok++; + } + } + } + expect(unknown).toBe(0); + expect(ok).toBe(48); + // the sessions were spread over the tx processes, and each id names its issuer + expect(issuers.size).toBeGreaterThan(1); + for (const code of issuers) { + expect(['m0', 'tx1', 'tx2']).toContain(code); + } + }); + + test('$closure goes to the master, whatever the cache-id says', async () => { + for (const cid of ['tx1.abc', 'tx2.abc', undefined]) { + const r = await request(FRONT, 'POST', '/r4/$closure', { + headers: cid ? { 'X-Cache-Id': cid } : {}, + body: { resourceType: 'Parameters', parameter: [{ name: 'name', valueString: `t-${cid}` }] } + }); + expect(r.status).toBe(200); + expect(r.json.resourceType).toBe('ConceptMap'); + } + // a worker has no closure tables of its own, but says the service offers $closure + const direct = await request(WORKERS[0], 'POST', '/r4/$closure', { body: { resourceType: 'Parameters', parameter: [{ name: 'name', valueString: 'x' }] } }); + expect(direct.status).toBe(501); + const cap = await request(WORKERS[0], 'GET', '/r4/metadata'); + expect(cap.json.rest[0].operation.map(o => o.name)).toContain('closure'); + }); + + test('a VSAC sync on the master reaches both workers', async () => { + const validate = (port) => request(port, 'GET', '/r4/ValueSet/$validate-code?url=' + + encodeURIComponent('http://cts.nlm.nih.gov/fhir/ValueSet/two') + `&system=${encodeURIComponent(GENDER)}&code=male`) + .then(r => r.status === 200 && param(r.json, 'result') && param(r.json, 'result').valueBoolean === true); + for (const port of WORKERS) { + expect(await validate(port)).toBe(false); + } + vsacList.push(vsacValueSet('two')); + const t0 = Date.now(); + for (const port of [MASTER, ...WORKERS]) { + while (!(await validate(port))) { + if (Date.now() - t0 > 30000) { + throw new Error(`the process on ${port} never saw the new value set`); + } + await sleep(200); + } + } + }); + + test('SIGTERM: requests in progress finish, then every process stops', async () => { + const inFlight = []; + for (let i = 0; i < 10; i++) { + inFlight.push(request(FRONT, 'GET', `/r4/CodeSystem/$lookup?system=${encodeURIComponent(GENDER)}&code=male`)); + } + await sleep(50); + launcher.kill('SIGTERM'); + const results = await Promise.all(inFlight); + expect(results.map(r => r.status)).toEqual(Array(10).fill(200)); + expect(await launcherExit).toEqual({ code: 0, signal: null }); + for (const port of [FRONT, MASTER, ...WORKERS]) { + expect(await isUp(port)).toBe(false); + } + }); +}); diff --git a/tests/server/cluster-launcher.test.js b/tests/server/cluster-launcher.test.js new file mode 100644 index 00000000..62a558ee --- /dev/null +++ b/tests/server/cluster-launcher.test.js @@ -0,0 +1,187 @@ +/** + * cluster.js: the routing rules (as in nginx.md), least-loaded choice, restarts and the + * drain on stop - against fake upstream processes (tests/utils/fake-upstream.js). + */ +const http = require('http'); +const path = require('path'); +const { loadSettings, classify, pickTarget, createLauncher } = require('../../cluster'); + +const FAKE = path.join(__dirname, '..', 'utils', 'fake-upstream.js'); +const TX = ['/r4', '/r5']; +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +function get(port, urlPath, headers = {}) { + return new Promise((resolve, reject) => { + const req = http.get({ host: '127.0.0.1', port, path: urlPath, headers, agent: false }, (res) => { + let body = ''; + res.on('data', d => { body += d; }); + res.on('end', () => { + let json = null; + try { + json = JSON.parse(body); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on('error', reject); + }); +} + +async function until(fn, ms = 10000) { + const t0 = Date.now(); + while (!fn()) { + if (Date.now() - t0 > ms) { + throw new Error('timed out'); + } + await sleep(50); + } +} + +describe('routing rules', () => { + test('cache-ids go to their issuer, by the prefix', () => { + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX)).toEqual({ to: 'owner', code: 'tx2' }); + expect(classify('/r5/CodeSystem/$lookup?x=1', { 'x-cache-id': 'm0.abc' }, TX)).toEqual({ to: 'owner', code: 'm0' }); + // no prefix: any tx process (it answers cache-id-unknown) + expect(classify('/r4/ValueSet/$expand', { 'x-cache-id': '4f0c2d1e-aaaa' }, TX)).toEqual({ to: 'tx' }); + expect(classify('/r4/ValueSet/$expand', {}, TX)).toEqual({ to: 'tx' }); + }); + + test('$closure, in any FHIR version, encoded or not, goes to the master, whatever the cache-id', () => { + for (const url of ['/r4/$closure', '/r5/$closure', '/r4/%24closure', '/r4/$closure?name=x']) { + expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX)).toEqual({ to: 'master' }); + } + }); + + test('the syndication feed and the source info pages go to the master', () => { + expect(classify('/r4/syndication', {}, TX)).toEqual({ to: 'master' }); + expect(classify('/r5/syndication/archive/2', {}, TX)).toEqual({ to: 'master' }); + expect(classify('/r4/info/vsac', {}, TX)).toEqual({ to: 'master' }); + }); + + test('every path outside the tx endpoints goes to the master', () => { + for (const url of ['/', '/health', '/packages/hl7.fhir.r4.core', '/tx-reg', '/xig', '/dashboard', '/r4x/metadata', '/r']) { + expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX)).toEqual({ to: 'master' }); + } + }); + + test('least outstanding requests, among the ready tx processes', () => { + const procs = [ + { code: 'm0', role: 'master', servesTx: true, ready: true, outstanding: 3 }, + { code: 'tx1', role: 'tx-worker', servesTx: true, ready: true, outstanding: 2 }, + { code: 'tx2', role: 'tx-worker', servesTx: true, ready: true, outstanding: 1 }, + { code: 'tx3', role: 'tx-worker', servesTx: true, ready: false, outstanding: 0 } + ]; + expect(pickTarget({ to: 'tx' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'owner', code: 'tx1' }, procs).code).toBe('tx1'); + // the owner isn't ready, or doesn't exist: the least loaded + expect(pickTarget({ to: 'owner', code: 'tx3' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'owner', code: 'zz9' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'master' }, procs).code).toBe('m0'); + // a master that doesn't serve tx gets none of it + procs[0].servesTx = false; + procs[0].outstanding = 0; + expect(pickTarget({ to: 'tx' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'master' }, [procs[1]])).toBeNull(); + }); + + test('settings from flags, environment and config', () => { + const config = { server: { port: 20000, cluster: { workers: 3 } }, modules: { tx: { enabled: true, instanceCode: 'main', endpoints: [{ path: '/r4/' }] } } }; + const s = loadSettings([], {}, config); + expect(s.port).toBe(20000); + expect(s.txPaths).toEqual(['/r4']); + expect(s.procs.map(p => [p.code, p.role, p.port, p.servesTx])).toEqual([ + ['main', 'master', 20001, true], ['tx1', 'tx-worker', 20002, true], ['tx2', 'tx-worker', 20003, true], ['tx3', 'tx-worker', 20004, true]]); + const s2 = loadSettings(['--workers=1', '--port=20010', '--master-serves-tx=false'], {}, config); + expect(s2.procs.map(p => [p.code, p.port, p.servesTx])).toEqual([['main', 20011, false], ['tx1', 20012, true]]); + expect(() => loadSettings([], {}, { modules: { tx: { enabled: true, instanceCode: 'tx1' } } })).toThrow(/also the code of a worker/); + }); +}); + +describe('the launcher, with fake processes', () => { + const PORT = 20013; + let launcher; + const settings = (servesTx) => ({ + port: PORT, + txPaths: TX, + drainMs: 10000, + procs: [ + { name: 'master', role: 'master', code: 'm0', port: 20010, servesTx }, + { name: 'worker tx1', role: 'tx-worker', code: 'tx1', port: 20011, servesTx: true }, + { name: 'worker tx2', role: 'tx-worker', code: 'tx2', port: 20012, servesTx: true } + ], + command: (proc) => [process.execPath, [FAKE, proc.code, String(proc.port)]], + log: () => {} + }); + + afterEach(async () => { + if (launcher) { + await launcher.stop(); + launcher = null; + } + }); + + test('routes by the rules, and adds X-Forwarded-For', async () => { + launcher = createLauncher(settings(true)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + expect((await get(PORT, '/packages')).json.code).toBe('m0'); + expect((await get(PORT, '/r4/$closure', { 'X-Cache-Id': 'tx2.x' })).json.code).toBe('m0'); + expect((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'tx2.x' })).json.code).toBe('tx2'); + expect((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'm0.x' })).json.code).toBe('m0'); + const r = await get(PORT, '/r4/metadata', { 'X-Forwarded-For': '203.0.113.9' }); + expect(r.json.xff).toBe('203.0.113.9, 127.0.0.1'); + expect(r.json.proto).toBe('http'); + }); + + test('a tx request goes to the process with the fewest outstanding', async () => { + launcher = createLauncher(settings(false)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + // a slow request on one worker: the next requests go to the other + const slow = get(PORT, '/r4/ValueSet/$expand?delay=1500'); + await until(() => launcher.procs.some(p => p.outstanding === 1)); + const busy = launcher.procs.find(p => p.outstanding === 1).code; + for (let i = 0; i < 5; i++) { + const quick = await get(PORT, '/r4/ValueSet/$expand'); + expect(['tx1', 'tx2']).toContain(quick.json.code); + expect(quick.json.code).not.toBe(busy); + } + expect((await slow).json.code).toBe(busy); + }); + + test('a process that exits is started again', async () => { + launcher = createLauncher(settings(true)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + const tx1 = launcher.procs.find(p => p.code === 'tx1'); + const old = tx1.child; + old.kill('SIGKILL'); + await until(() => !tx1.ready); + // meanwhile its cache-ids go to another tx process, which says it doesn't know them + expect(['m0', 'tx2']).toContain((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'tx1.x' })).json.code); + await until(() => tx1.ready && tx1.child !== old); + expect((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'tx1.x' })).json.code).toBe('tx1'); + }); + + test('stop drains: no new connections, the request in progress finishes, then the processes stop', async () => { + launcher = createLauncher(settings(true)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + const children = launcher.procs.map(p => p.child); + const slow = get(PORT, '/r4/ValueSet/$expand?delay=1500'); + await until(() => launcher.procs.some(p => p.outstanding === 1)); + const stopped = launcher.stop(); + await sleep(100); + await expect(get(PORT, '/r4/metadata')).rejects.toThrow(/ECONNREFUSED/); + // the processes are still running while the request is in progress + expect(children.every(c => c.exitCode === null && c.signalCode === null)).toBe(true); + const r = await slow; + expect(r.status).toBe(200); + expect(r.headers.connection).toBe('close'); + await stopped; + expect(children.every(c => c.exitCode !== null || c.signalCode !== null)).toBe(true); + launcher = null; + }); +}); diff --git a/tests/utils/fake-upstream.js b/tests/utils/fake-upstream.js new file mode 100644 index 00000000..bf156782 --- /dev/null +++ b/tests/utils/fake-upstream.js @@ -0,0 +1,22 @@ +// A stand-in for a server process, for tests/server/cluster-launcher.test.js: +// node fake-upstream.js +// /health answers 200; anything else answers {code, url, xff, proto} as JSON, after +// ?delay=. On SIGTERM it finishes the requests it has, then exits. +const http = require('http'); +const [code, port] = process.argv.slice(2); +const server = http.createServer((req, res) => { + if (req.url === '/health') { + res.end('ok'); + return; + } + const delay = Number(new URL(req.url, 'http://x').searchParams.get('delay')) || 0; + setTimeout(() => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ code, url: req.url, xff: req.headers['x-forwarded-for'], proto: req.headers['x-forwarded-proto'] })); + }, delay); +}); +server.listen(Number(port)); +process.on('SIGTERM', () => { + server.close(() => process.exit(0)); + server.closeIdleConnections(); +}); From 124db9a14b8f62e7673dc54f54eaaf901c524672 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:41:20 -0600 Subject: [PATCH 06/23] docs: cluster.md (roles, what runs where, routing, the launcher, memory); nginx.md example cluster.md replaces #336's: the master/tx-worker roles and their flags, a table of what runs in which process, the four routing rules, cluster.js and its settings, the group memory measure, and what this arrangement doesn't do (caches and statistics are per process; the master is a single point for $closure and the non-tx modules). nginx.md gains "A master and tx workers on one machine": the existing pool, per- instance and closure upstreams, with the master as the closure instance and the default location, plus the syndication and info paths. README links to cluster.md. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- cluster.md | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++++ nginx.md | 68 ++++++++++++++++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 cluster.md diff --git a/README.md b/README.md index cb4d10d9..3179e9c4 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,9 @@ npm start The server will be available at `http://localhost:{port}` using the port specified in the config. In the production servers listed above, the server always sits behind an NGINX server which manages SSL, security, rate limiting etc. See [nginx.md](nginx.md) for configuration advice, including -running several servers behind one front door. +running several servers behind one front door. To use more than one CPU core for terminology on +one machine, see [cluster.md](cluster.md): a master and several tx workers, behind nginx or +`node cluster.js`. ## Testing diff --git a/cluster.md b/cluster.md new file mode 100644 index 00000000..d928a8be --- /dev/null +++ b/cluster.md @@ -0,0 +1,163 @@ +# A master and terminology workers on one machine + +One Node.js process runs JavaScript on one CPU core. A terminology server under load from +several IG Publisher builds at once can use more. FHIRsmith can run as one **master** and +several **tx-workers** on the same machine and data directory, with a front door that sends +each request to the right process: + +```bash +node cluster.js # the master, 2 workers, and a proxy on the configured port +``` + +or with nginx in front of separately started processes ([below](#with-nginx)). + +`node server.js` with no `--role` is unchanged: one process, every module, as before. + +* [Roles](#roles) +* [What runs where](#what-runs-where) +* [Routing](#routing) +* [The launcher (cluster.js)](#the-launcher-clusterjs) +* [With nginx](#with-nginx) +* [Memory](#memory) +* [Limits of this arrangement](#limits-of-this-arrangement) + +## Roles + +```bash +node server.js --role=master --instance-code=m0 --port=3001 +node server.js --role=tx-worker --instance-code=tx1 --port=3002 +node server.js --role=tx-worker --instance-code=tx2 --port=3003 +``` + +| Flag | Environment | Config | Meaning | +|---|---|---|---| +| `--role` | `FHIRSMITH_ROLE` | | `master` or `tx-worker`. None: an ordinary single server. | +| `--instance-code` | `FHIRSMITH_INSTANCE_CODE` | `modules.tx.instanceCode` (master only) | Prefixed onto every cache-id the process issues (see [nginx.md](nginx.md#instance-codes)). Each process needs its own, 1-16 letters and digits. A worker must be given one, since the config's is shared with the master. | +| `--port` | `PORT` | `server.port` | The port the process listens on. | + +All the processes use the same data directory and the same `config.json`. The role decides +which parts of the configuration each one acts on. + +## What runs where + +| | master | tx-worker | +|---|---|---| +| tx module (`/r4`, `/r5` and the other endpoints) | yes (may also serve tx requests) | yes | +| Every other module (packages, xig, registry, publisher, token, shl, ips, ...) | yes | no | +| `$closure` (the closure tables) | yes | no: routed to the master | +| Syndication feed (the store) | yes | no: routed to the master | +| VSAC sync, and the resync form on `/info/vsac` | yes | no: reads what the master wrote | +| Package crawler, XIG download, registry crawl, daily report, npm audit | yes | no (those modules don't run) | +| Statistics database, uptime history | yes | no: in memory only | +| `$cache-control` caches | its own, in memory | its own, in memory | +| Expansion cache, CodeSystem list refresh | its own | its own | +| Log file | `logs/server-.log` | the same file (lines interleave) | + +The master loads the terminologies like any tx server: `$closure` needs subsumption. + +**VSAC.** The master syncs, and when a sync has written its value sets it adds one to a +generation counter in the VSAC database. A worker remembers the generation its value set list +was loaded at. When a request uses the VSAC value sets, the worker reads the counter (one read +by key, at most once a second) and reloads the list in the background if it has moved, so a +sync reaches every worker about a second after it finishes. The database is in WAL mode with a +5 second busy timeout, so a worker's reload doesn't make the master's writes fail. + +**Nothing else is shared.** The workers run no module but tx, so they never read what the +package crawler, the XIG download or the registry crawl write. Each process keeps the caches +of a `folder:` or `file:` library source in its own folder +(`terminology-cache/folder-source--`), because each load wipes and rebuilds them. + +## Routing + +Whatever the front door, the rules are the same: + +1. `$closure` (in any endpoint, e.g. `/r4/$closure`), the syndication feed + (`/r4/syndication...`) and the source info pages (`/r4/info/...`) go to the **master**. +2. Every path outside the tx endpoints (`/`, `/packages`, `/xig`, `/dashboard`, `/health`, ...) + goes to the **master**. +3. A tx request with an `X-Cache-Id` goes to the process whose code prefixes the id + (`tx2.4f0c...` to `tx2`). If that process is down, or the prefix is unknown, it goes to any + tx process, which answers `cache-id-unknown`; the client starts a new cache. +4. Any other tx request goes to the tx process with the fewest requests outstanding: the + workers, and the master too unless it is set not to serve tx. + +## The launcher (cluster.js) + +`node cluster.js` reads `config.json` from the data directory and: + +* starts the master and waits until it answers `/health`. On a new data directory the master + builds the package caches and creates the VSAC database; processes starting together would + corrupt them, so the workers start only when the master is ready +* listens on the configured port, and starts the workers +* proxies each request by the rules above, over keep-alive connections to each process. It + adds the client address to `X-Forwarded-For`, so the servers' logs and rate limits see the + real client (its own address is a loopback one, which `server.trustProxy` trusts by default) +* starts again a process that exits. One that keeps failing waits longer each time: 1 s, + 2 s, 4 s ... up to a minute +* on `SIGTERM` or `SIGINT`: stops taking connections, closes idle keep-alive connections, + waits for the requests in progress (up to `drainSeconds`), then sends each process + `SIGTERM` (each shuts down as a single server does; `SIGKILL` after 30 seconds), and exits + +```json +"server": { + "port": 3000, + "cluster": { "workers": 4, "masterServesTx": true, "internalPort": 3001, "drainSeconds": 30 } +} +``` + +| Setting | Flag | Default | Meaning | +|---|---|---|---| +| `server.port` | `--port`, `PORT` | 3000 | The port clients use. | +| `server.cluster.workers` | `--workers`, `FHIRSMITH_WORKERS` | 2 | How many tx-workers. | +| `server.cluster.masterServesTx` | `--master-serves-tx=false` | true | Whether the master also takes ordinary tx requests. | +| `server.cluster.internalPort` | `--internal-port` | port + 1 | The master's port; worker *n* gets this + *n*. Firewall them. | +| `server.cluster.drainSeconds` | | 30 | How long the requests in progress get on shutdown. | +| `modules.tx.instanceCode` | | `m0` | The master's instance code. The workers are `tx1`, `tx2`, ... | + +Under systemd, the launcher and its processes are one unit and one cgroup: + +```ini +[Service] +WorkingDirectory=/opt/fhirsmith +Environment=FHIRSMITH_DATA_DIR=/var/lib/fhirsmith +ExecStart=/usr/bin/node cluster.js +KillMode=mixed +MemoryMax=16G +``` + +## With nginx + +nginx can be the front door instead: start the processes yourself (master first, workers once +it answers `/health`), and use the configuration in +[nginx.md](nginx.md#a-master-and-tx-workers-on-one-machine), which applies the same rules. + +## Memory + +With a role, the memory guard measures the whole cgroup, since the kernel's limit is on all the +processes in it together: the cgroup's private memory (`memory.stat` `anon`; cgroup v1 +`total_rss`). The page cache (the terminology and SQLite files) isn't counted: the kernel drops +it when it needs the room. + +* Above `server.memory.rejectFraction` (85%) of the limit, **every** process refuses new + requests with 503 and `Retry-After`. +* Above `abortFraction` (92%), only the process with the **largest RSS** in the cgroup aborts + its operations (422 too-costly). It is the one most likely to hold what took the group over, + and aborting there frees the most; the others carry on. + +This needs the limit on the processes' own cgroup: `MemoryMax` on the unit that runs +`cluster.js`, or a container limit. If the limit is only on an enclosing slice, or there is +none, each process measures its own RSS against the limit, as a single server does. If you +start the processes as separate systemd units, give each its own `MemoryMax`: each then guards +its own. + +Each process is a whole terminology server: it loads the library, so N workers and a master +use roughly N+1 times the memory of one server. Size the limit for that. + +## Limits of this arrangement + +* **Caches are per process.** A `$cache-control` cache lives in the process that issued it; a + restarted process loses its caches, and clients start new ones, as after any restart. +* **Statistics are per process.** `/` and `/dashboard` show the master. The workers' request + counts are not in the statistics database. +* **Everything the master runs is a single point.** While the master restarts, `$closure`, the + non-tx modules and the dashboard answer 503 through the launcher; tx requests carry on. diff --git a/nginx.md b/nginx.md index c242552f..eb7ffcb1 100644 --- a/nginx.md +++ b/nginx.md @@ -8,6 +8,7 @@ public servers (tx.fhir.org, packages.fhir.org and so on) use. * [Rate limiting](#rate-limiting) - approaches that suit terminology traffic * [Routing to multiple servers](#routing-to-multiple-servers) - running more than one instance behind a single front door * [Managing closure](#managing-closure) - where `$closure` requests go +* [A master and tx workers on one machine](#a-master-and-tx-workers-on-one-machine) - the same rules for processes started with `--role` ## General configuration @@ -397,3 +398,70 @@ the cache ends, and the table would silently change meaning. So `$closure` ignor about. An outage is better than that: clients see a failure and retry later. * **Moving the designated instance** means moving its closure database with it, then changing the upstream. + +## A master and tx workers on one machine + +On one machine, FHIRsmith can run as one master and several tx workers on the same data +directory (see [cluster.md](cluster.md)). The master runs every module, the background jobs, +`$closure` and the syndication feed; the workers run only the tx module. Start the master +first, and the workers once it answers `/health`: + +```bash +node --expose-gc server.js --role=master --instance-code=m0 --port=3001 +node --expose-gc server.js --role=tx-worker --instance-code=tx1 --port=3002 +node --expose-gc server.js --role=tx-worker --instance-code=tx2 --port=3003 +``` + +The routing is the pool above, with the master as the designated `$closure` instance and the +home of everything that isn't a tx endpoint: + +```nginx +upstream fs_master { server 127.0.0.1:3001; keepalive 16; } + +# every process that takes ordinary tx requests (leave the master out if it shouldn't); +# the zone shares least_conn's counts between nginx's worker processes +upstream fs_tx_pool { + zone fs_tx_pool 64k; + least_conn; + server 127.0.0.1:3001; + server 127.0.0.1:3002; + server 127.0.0.1:3003; + keepalive 32; +} +upstream fs_tx1 { server 127.0.0.1:3002; server 127.0.0.1:3003 backup; keepalive 16; } +upstream fs_tx2 { server 127.0.0.1:3003; server 127.0.0.1:3002 backup; keepalive 16; } + +map $http_x_cache_id $fs_tx_backend { + default fs_tx_pool; + ~^m0\. fs_master; + ~^tx1\. fs_tx1; + ~^tx2\. fs_tx2; +} + +server { + ... + # the master's parts of the tx endpoints: $closure, the syndication feed, the source + # info pages (the VSAC resync form writes the VSAC database) + location ~ ^/(r4|r5)/(\$closure|syndication(/.*)?|info/.*)$ { + include fhirsmith-proxy.conf; + proxy_set_header X-Cache-Id ""; + proxy_pass http://fs_master; + } + + # the tx endpoints: by cache-id, else the least busy + location ~ ^/(r4|r5)(/|$) { + include fhirsmith-proxy.conf; + proxy_pass http://$fs_tx_backend; + } + + # everything else: the master + location / { + include fhirsmith-proxy.conf; + proxy_pass http://fs_master; + } +} +``` + +List your own tx endpoint paths (`modules.tx.endpoints`) in the two regex locations. The +optional launcher, `node cluster.js`, starts the processes in this order and applies the same +rules without nginx (it can also sit behind nginx, as one upstream). From 9ee5293f5d5dd546424bd2714e948d4934235881 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 19:50:21 -0600 Subject: [PATCH 07/23] cluster.js: don't pass Expect on to the process Node's server has already answered a client's "Expect: 100-continue" by the time the request is proxied, so the header is dropped like the other hop-by-hop headers. Co-Authored-By: Claude Opus 5.5 --- cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.js b/cluster.js index 54671d19..3ebe3130 100644 --- a/cluster.js +++ b/cluster.js @@ -33,7 +33,7 @@ const HEALTH_POLL_MS = 250; const MAX_BACKOFF_MS = 60000; // a process that ran this long before exiting is restarted at once const HEALTHY_RUN_MS = 60000; -const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'trailer']; +const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'trailer', 'expect']; /** * The settings, from the flags, then the environment, then config.json (server.cluster). From 49f843ad95285fcadadb1671d1c28f8866f5ef56 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 18:55:52 -0600 Subject: [PATCH 08/23] $expand: hash each cache-id resource once, not on every expansion ExpansionCache.computeKey puts a content hash of every additional resource into the expansion cache key. With a cache-id those resources are the client's whole $cache-control cache, so every $expand serialised and sha256-hashed everything the IG had uploaded. Under replayed IG Publisher traffic at C=16 that was about a quarter of the main thread. CodeSystem, ValueSet and ConceptMap (CanonicalResource) now work the hash out on first use and remember it, in a non-enumerable field tied to the jsonObj it was computed from (a replaced jsonObj, or a replacement resource, is hashed again). setExpansionKeyHash lets a store that already knows the hash record it. The hash is exactly the one computeKey made, and the key is built the same way, so keys are byte-identical and expansions already cached stay valid. Remembering the hash needs a cached resource's json not to change once it is in use. One thing did change it: $compare sorted, merged and removed the compose includes of the value sets it compared, in place - cache-id resources and the server's own. It now works on copies. Tests: computeKey against the previous implementation for a mix of resources and parameters; one serialisation and hash per resource across many keys; a new hash for a replacement resource; and expand, validate-code, lookup and compare through a sealed cache-id with the cached json frozen. Co-Authored-By: Claude Opus 5.5 --- tests/tx/expansion-key-hash.test.js | 369 ++++++++++++++++++++++++++++ tx/library/canonical-resource.js | 49 ++++ tx/operation-context.js | 15 +- tx/workers/compare.js | 9 +- 4 files changed, 432 insertions(+), 10 deletions(-) create mode 100644 tests/tx/expansion-key-hash.test.js diff --git a/tests/tx/expansion-key-hash.test.js b/tests/tx/expansion-key-hash.test.js new file mode 100644 index 00000000..e510857b --- /dev/null +++ b/tests/tx/expansion-key-hash.test.js @@ -0,0 +1,369 @@ +/** + * ExpansionCache.computeKey hashes the content of every additional resource. With a + * cache-id that is the client's whole $cache-control cache, on every $expand, so each + * wrapper remembers its hash (CanonicalResource.expansionKeyHash) instead of serialising + * and hashing itself again. + * + * - the key is byte-identical to the one computeKey made before (the reference below is + * that implementation, copied), so expansions already cached stay valid; + * - a resource is hashed once however many keys it goes into; + * - a different resource, or a different jsonObj, gets its own hash; + * - and nothing writes to a cached resource's json while it is used through a cache-id, + * which is what makes remembering the hash safe. + */ + +const crypto = require('crypto'); +const request = require('supertest'); +const { ExpansionCache } = require('../../tx/operation-context'); +const { CanonicalResource } = require('../../tx/library/canonical-resource'); +const { CodeSystem } = require('../../tx/library/codesystem'); +const ValueSet = require('../../tx/library/valueset'); +const { ConceptMap } = require('../../tx/library/conceptmap'); +const { getTestApp, getTxModule, shutdownTestApp } = require('./setup'); + +// computeKey as it was before the hash was remembered - the reference the new key must match +function referenceComputeKey(valueSet, params, additionalResources) { + const keyParts = []; + const vsJson = valueSet.jsonObj || valueSet; + keyParts.push(`vs:${JSON.stringify(vsJson)}`); + if (params) { + keyParts.push(`params:` + params.hashSource()); + } + if (additionalResources && additionalResources.length > 0) { + const resourceHashes = additionalResources + .map(r => { + const json = r.jsonObj || r; + return crypto.createHash('sha256') + .update(JSON.stringify(json)) + .digest('hex') + .substring(0, 16); + }) + .sort(); + keyParts.push(`additional:${resourceHashes.join(',')}`); + } + const keyString = keyParts.join('||'); + return crypto.createHash('sha256').update(keyString).digest('hex'); +} + +const CS_URL = 'http://example.org/p6/colors'; + +function colorsCS(extra = {}) { + return { + resourceType: 'CodeSystem', url: CS_URL, version: '1.0.0', status: 'active', content: 'complete', + hierarchyMeaning: 'is-a', + property: [{ code: 'shade', type: 'code' }], + concept: [ + { code: 'red', display: 'Red', property: [{ code: 'shade', valueCode: 'warm' }], + concept: [{ code: 'crimson', display: 'Crimson' }] }, + { code: 'green', display: 'Green', property: [{ code: 'shade', valueCode: 'cool' }] }, + { code: 'blue', display: 'Blue', designation: [{ language: 'fr', value: 'Bleu' }] } + ], + ...extra + }; +} + +function supplementCS() { + return { + resourceType: 'CodeSystem', url: 'http://example.org/p6/colors-de', version: '1.0.0', status: 'active', + content: 'supplement', supplements: `${CS_URL}|1.0.0`, + concept: [{ code: 'red', designation: [{ language: 'de', value: 'Rot' }] }] + }; +} + +function enumeratedVS() { + return { + resourceType: 'ValueSet', url: 'http://example.org/p6/warm-vs', version: '1.0.0', status: 'active', + compose: { include: [{ system: CS_URL, concept: [{ code: 'red' }, { code: 'blue', display: 'Blue' }] }] } + }; +} + +function filterVS() { + return { + resourceType: 'ValueSet', url: 'http://example.org/p6/red-family', version: '1.0.0', status: 'active', + compose: { + include: [ + { system: CS_URL, filter: [{ property: 'concept', op: 'is-a', value: 'red' }] }, + { system: CS_URL, filter: [{ property: 'shade', op: '=', value: 'cool' }] } + ], + exclude: [{ system: CS_URL, concept: [{ code: 'crimson' }] }] + } + }; +} + +function importingVS() { + return { + resourceType: 'ValueSet', url: 'http://example.org/p6/all', version: '1.0.0', status: 'active', + compose: { include: [{ valueSet: ['http://example.org/p6/warm-vs|1.0.0'] }, { system: CS_URL }] } + }; +} + +function conceptMap() { + return { + resourceType: 'ConceptMap', url: 'http://example.org/p6/cm', version: '1.0.0', status: 'active', + group: [{ source: CS_URL, target: 'http://example.org/p6/other', + element: [{ code: 'red', target: [{ code: 'r', relationship: 'equivalent' }] }] }] + }; +} + +function wrapped() { + return [ + new CodeSystem(colorsCS()), + new CodeSystem(supplementCS()), + new ValueSet(enumeratedVS()), + new ValueSet(filterVS()), + new ValueSet(importingVS()), + new ConceptMap(conceptMap()) + ]; +} + +const params = (s) => ({ hashSource: () => s }); + +describe('ExpansionCache.computeKey with remembered resource hashes', () => { + const cache = new ExpansionCache(null); + + test('makes exactly the key the previous implementation made', () => { + const resources = wrapped(); + const raw = [colorsCS({ version: '2.0.0' }), enumeratedVS()]; // plain json is still accepted + const vsWrapper = new ValueSet(filterVS()); + const cases = [ + [vsWrapper, null, undefined], + [vsWrapper, null, []], + [vsWrapper, params('lang=en|count=10'), resources], + [filterVS(), params(''), resources], + [vsWrapper, params('x'), [...resources].reverse()], + [vsWrapper, params('x'), resources.slice(2)], + [vsWrapper, params('x'), raw], + [vsWrapper, params('x'), [...raw, ...resources]], + [vsWrapper, params('x'), [resources[0], resources[0]]] + ]; + for (const [vs, p, extra] of cases) { + // twice: the second time the hashes are remembered + expect(cache.computeKey(vs, p, extra)).toBe(referenceComputeKey(vs, p, extra)); + expect(cache.computeKey(vs, p, extra)).toBe(referenceComputeKey(vs, p, extra)); + } + }); + + test('a resource\'s hash is the one the key used to use', () => { + for (const r of wrapped()) { + expect(r.expansionKeyHash()).toBe( + crypto.createHash('sha256').update(JSON.stringify(r.jsonObj)).digest('hex').substring(0, 16)); + } + }); + + test('each resource is serialised and hashed once, however many keys it goes into', () => { + const resources = wrapped(); + const vs = new ValueSet(enumeratedVS()); + const stringified = new Set(); + const realStringify = JSON.stringify; + const stringify = jest.spyOn(JSON, 'stringify').mockImplementation(function (v, ...rest) { + stringified.add(v); + return realStringify.call(JSON, v, ...rest); + }); + const createHash = jest.spyOn(crypto, 'createHash'); + try { + const first = cache.computeKey(vs, params('a'), resources); + // one hash per resource, plus the key itself + expect(createHash).toHaveBeenCalledTimes(resources.length + 1); + for (const r of resources) { + expect(stringified.has(r.jsonObj)).toBe(true); + } + createHash.mockClear(); + stringify.mockClear(); + stringified.clear(); + for (let i = 0; i < 50; i++) { + expect(cache.computeKey(vs, params('a'), resources)).toBe(first); + cache.computeKey(vs, params('b' + i), [...resources].reverse()); + } + // only the key itself is hashed, and only the value set being expanded is serialised + expect(createHash).toHaveBeenCalledTimes(100); + expect(stringify).toHaveBeenCalledTimes(100); + for (const r of resources) { + expect(stringified.has(r.jsonObj)).toBe(false); + } + } finally { + stringify.mockRestore(); + createHash.mockRestore(); + } + }); + + test('a replacement resource (a new object) gets its own hash, and so a new key', () => { + const vs = new ValueSet(enumeratedVS()); + const before = wrapped(); + const keyBefore = cache.computeKey(vs, null, before); + const changed = new CodeSystem(colorsCS({ title: 'Colours, revised' })); + const after = [changed, ...before.slice(1)]; + expect(changed.expansionKeyHash()).not.toBe(before[0].expansionKeyHash()); + const keyAfter = cache.computeKey(vs, null, after); + expect(keyAfter).not.toBe(keyBefore); + expect(keyAfter).toBe(referenceComputeKey(vs, null, after)); + // an identical replacement is a new object with the same content: same hash, same key + const same = [new CodeSystem(colorsCS()), ...before.slice(1)]; + expect(cache.computeKey(vs, null, same)).toBe(keyBefore); + }); + + test('a wrapper whose jsonObj is replaced is hashed again', () => { + const cs = new CodeSystem(colorsCS()); + const h1 = cs.expansionKeyHash(); + cs.jsonObj = colorsCS({ title: 'Other' }); + expect(cs.expansionKeyHash()).not.toBe(h1); + expect(cs.expansionKeyHash()).toBe(CanonicalResource.computeExpansionKeyHash(cs.jsonObj)); + }); + + test('a store that knows the hash can record it, and it is used without hashing', () => { + const cs = new CodeSystem(colorsCS()); + const known = CanonicalResource.computeExpansionKeyHash(cs.jsonObj); + cs.setExpansionKeyHash(known); + const createHash = jest.spyOn(crypto, 'createHash'); + try { + expect(cs.expansionKeyHash()).toBe(known); + expect(createHash).not.toHaveBeenCalled(); + } finally { + createHash.mockRestore(); + } + }); + + test('the remembered hash never serialises or copies', () => { + const cs = new CodeSystem(colorsCS()); + const h = cs.expansionKeyHash(); + expect(Object.keys(cs)).not.toContain('_expansionKeyHashMemo'); + expect(JSON.stringify(cs)).not.toContain(h); + expect({ ...cs }._expansionKeyHashMemo).toBeUndefined(); + }); +}); + +// Freeze a resource's json, so that anything that writes to it while it is in use throws +// (class bodies are strict code). The one exception: compose filter elements stay +// extensible, because a code system provider remembers its analysis of a cached filter on +// the element itself under a non-enumerable symbol (cs-cs.js FILTER_ANALYSIS) - which JSON +// never sees. Their existing properties are still read-only, and the test checks nothing +// enumerable was added to them. +function freezeJson(o, filterElements, inFilter = false) { + if (!o || typeof o !== 'object' || Object.isFrozen(o)) { + return o; + } + for (const [k, v] of Object.entries(o)) { + freezeJson(v, filterElements, k === 'filter' && Array.isArray(v)); + } + if (inFilter) { + // the filter array itself + Object.freeze(o); + return o; + } + if (filterElements && filterElements.has(o)) { + for (const k of Object.keys(o)) { + Object.defineProperty(o, k, { writable: false, configurable: false }); + } + return o; + } + Object.freeze(o); + return o; +} + +function collectFilterElements(json, into) { + for (const part of ['include', 'exclude']) { + for (const inc of (json.compose && json.compose[part]) || []) { + for (const f of inc.filter || []) { + into.add(f); + } + } + } + return into; +} + +describe('cache-id resources are not written to while they are used', () => { + let app; + + beforeAll(async () => { + app = await getTestApp(); + }, 60000); + + afterAll(async () => { + await shutdownTestApp(); + }); + + function post(url, cacheId, parameter) { + return request(app) + .post(url) + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .set('x-cache-id', cacheId) + .send({ resourceType: 'Parameters', parameter }); + } + + test('expand, validate-code, lookup and compare through a cache-id leave the cached json untouched', async () => { + const start = await request(app) + .post('/tx/r5/$cache-control') + .query({ mode: 'start' }) + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .send({ resourceType: 'Parameters', parameter: [ + { name: 'sealed', valueBoolean: true }, + ...[colorsCS(), supplementCS(), enumeratedVS(), filterVS(), importingVS(), conceptMap()] + .map(resource => ({ name: 'tx-resource', resource })) + ] }); + expect(start.status).toBe(200); + const cacheId = start.body.parameter.find(p => p.name === 'cache-id').valueId; + + const endpoint = getTxModule().endpoints.find(e => e.resourceCache && e.resourceCache.has(cacheId)); + const cached = endpoint.resourceCache.get(cacheId); + expect(cached.length).toBe(6); + const filterElements = new Set(); + const before = new Map(); + for (const r of cached) { + collectFilterElements(r.jsonObj, filterElements); + before.set(r, JSON.stringify(r.jsonObj)); + freezeJson(r.jsonObj, filterElements); + r.expansionKeyHash(); + } + expect(filterElements.size).toBe(2); + + // every expansion takes the cache path, so computeKey and the cache store both run + const expansionCache = endpoint.expansionCache; + const wasForced = expansionCache.forceCaching; + expansionCache.forceCaching = true; + try { + const calls = []; + for (const url of ['http://example.org/p6/warm-vs', 'http://example.org/p6/red-family', 'http://example.org/p6/all']) { + for (let i = 0; i < 2; i++) { // the second is answered from the expansion cache + calls.push(['expand', url, await post('/tx/r5/ValueSet/$expand', cacheId, [ + { name: 'url', valueUri: url }, { name: 'includeDefinition', valueBoolean: true }, + { name: 'displayLanguage', valueCode: 'de' }])]); + } + calls.push(['expand+supplement', url, await post('/tx/r5/ValueSet/$expand', cacheId, [ + { name: 'url', valueUri: url }, { name: 'useSupplement', valueCanonical: 'http://example.org/p6/colors-de' }])]); + for (const code of ['red', 'crimson', 'green', 'nope']) { + calls.push(['validate-code', `${url}#${code}`, await post('/tx/r5/ValueSet/$validate-code', cacheId, [ + { name: 'url', valueUri: url }, { name: 'system', valueUri: CS_URL }, { name: 'code', valueCode: code }, + { name: 'display', valueString: 'Red' }])]); + } + } + calls.push(['cs validate-code', CS_URL, await post('/tx/r5/CodeSystem/$validate-code', cacheId, [ + { name: 'url', valueUri: CS_URL }, { name: 'code', valueCode: 'blue' }, { name: 'display', valueString: 'Bleu' }])]); + calls.push(['lookup', CS_URL, await post('/tx/r5/CodeSystem/$lookup', cacheId, [ + { name: 'system', valueUri: CS_URL }, { name: 'code', valueCode: 'red' }, { name: 'property', valueCode: '*' }])]); + calls.push(['compare', 'warm vs all', await post('/tx/r5/ValueSet/$compare', cacheId, [ + { name: 'thisUrl', valueUri: 'http://example.org/p6/warm-vs' }, { name: 'otherUrl', valueUri: 'http://example.org/p6/all' }])]); + calls.push(['compare', 'family vs family', await post('/tx/r5/ValueSet/$compare', cacheId, [ + { name: 'thisUrl', valueUri: 'http://example.org/p6/red-family' }, { name: 'otherUrl', valueUri: 'http://example.org/p6/red-family' }])]); + + // a write to frozen json throws, which would come back as a server error + for (const [op, what, res] of calls) { + expect({ op, what, status: res.status < 500 ? 'ok' : res.status, body: res.status < 500 ? null : res.body }) + .toEqual({ op, what, status: 'ok', body: null }); + } + const expanded = calls.filter(([op, , res]) => op === 'expand' && res.status === 200); + expect(expanded.length).toBe(6); + expect(calls.filter(([op, , res]) => op === 'validate-code' && res.status === 200).length).toBe(12); + } finally { + expansionCache.forceCaching = wasForced; + } + + for (const r of cached) { + expect(JSON.stringify(r.jsonObj)).toBe(before.get(r)); + expect(r.expansionKeyHash()).toBe(CanonicalResource.computeExpansionKeyHash(r.jsonObj)); + } + for (const f of filterElements) { + expect(Object.keys(f).sort()).toEqual(['op', 'property', 'value']); + } + }, 60000); +}); diff --git a/tx/library/canonical-resource.js b/tx/library/canonical-resource.js index db9a80b6..261378c4 100644 --- a/tx/library/canonical-resource.js +++ b/tx/library/canonical-resource.js @@ -1,3 +1,4 @@ +const crypto = require("crypto"); const {VersionUtilities, VersionPrecision} = require("../../library/version-utilities"); const {Issue} = require("./operation-outcome"); @@ -80,6 +81,54 @@ class CanonicalResource { checkContained(this.jsonObj); } + /** + * The hash ExpansionCache.computeKey puts in an expansion's cache key for this + * resource: the first 16 hex characters of the sha256 of JSON.stringify(json). Computing + * it means serialising the whole resource, and a $cache-control cache hands every + * resource it holds to every $expand that carries its cache-id, so it is worked out + * once and remembered. + * + * @param {Object} json - the resource (R5 form) + * @returns {string} + */ + static computeExpansionKeyHash(json) { + return crypto.createHash('sha256') + .update(JSON.stringify(json)) + .digest('hex') + .substring(0, 16); + } + + /** + * This resource's expansion-key hash (see computeExpansionKeyHash), computed on first + * use and then remembered. Nothing writes to jsonObj once a resource is in use (the r4/r5 conversion + * happens in the constructor, before anything can ask), and the memo is tied to the + * jsonObj it was computed from, so replacing jsonObj gets a new hash. + * @returns {string} + */ + expansionKeyHash() { + const memo = this._expansionKeyHashMemo; + if (memo && memo.json === this.jsonObj) { + return memo.hash; + } + return this.setExpansionKeyHash(CanonicalResource.computeExpansionKeyHash(this.jsonObj)); + } + + /** + * Record the expansion-key hash for a store that already knows it, so it is never + * recomputed. It must be computeExpansionKeyHash(this.jsonObj). Non-enumerable, so it never serialises + * into a response or log, and structuredClone/spread don't copy it onto a different + * resource. + * @param {string} hash + * @returns {string} the hash + */ + setExpansionKeyHash(hash) { + Object.defineProperty(this, '_expansionKeyHashMemo', { + value: { json: this.jsonObj, hash }, + enumerable: false, configurable: true, writable: true + }); + return hash; + } + get resourceType() { return this.jsonObj.resourceType; } diff --git a/tx/operation-context.js b/tx/operation-context.js index 29837db6..fd2e20d0 100644 --- a/tx/operation-context.js +++ b/tx/operation-context.js @@ -3,6 +3,7 @@ const inspector = require("inspector"); const crypto = require("crypto"); const {Languages} = require("../library/languages"); const {Issue} = require("./library/operation-outcome"); +const {CanonicalResource} = require("./library/canonical-resource"); const Logger = require("../library/logger"); const memoryGuard = require('../library/memory-guard'); @@ -651,13 +652,13 @@ class ExpansionCache { if (additionalResources && additionalResources.length > 0) { const resourceHashes = additionalResources .map(r => { - // Get the JSON object from wrapper or use directly - const json = r.jsonObj || r; - // Create a content hash for this resource - return crypto.createHash('sha256') - .update(JSON.stringify(json)) - .digest('hex') - .substring(0, 16); // Use first 16 chars for brevity + // A wrapper remembers its hash (CanonicalResource.expansionKeyHash): a cache-id's + // resources come with every $expand, and serialising and hashing them all + // each time was most of the cost of a cached request. Raw JSON is hashed the + // same way, so the key is the same either way. + return typeof r.expansionKeyHash === 'function' + ? r.expansionKeyHash() + : CanonicalResource.computeExpansionKeyHash(r.jsonObj || r); }) .sort(); keyParts.push(`additional:${resourceHashes.join(',')}`); diff --git a/tx/workers/compare.js b/tx/workers/compare.js index 0b1ca0f4..87a90cdb 100644 --- a/tx/workers/compare.js +++ b/tx/workers/compare.js @@ -172,9 +172,12 @@ class CompareWorker extends TerminologyWorker { async doCompare(txp, thisVS, otherVS) { - // ok, we have to compare the composes. we don't care about anything else - const thisC = thisVS.jsonObj.compose; - const otherC = otherVS.jsonObj.compose; + // ok, we have to compare the composes. we don't care about anything else. + // Copies: tidyIncludes merges, removes and sorts includes in place, and these value + // sets are shared - cache-id resources, and the server's own - so must not change + // (ExpansionCache.computeKey remembers a cached resource's hash, for one). + const thisC = thisVS.jsonObj.compose ? structuredClone(thisVS.jsonObj.compose) : thisVS.jsonObj.compose; + const otherC = otherVS.jsonObj.compose ? structuredClone(otherVS.jsonObj.compose) : otherVS.jsonObj.compose; if (!thisC) { return this.makeOutcome("indeterminate", `The ValueSet ${thisVS.vurl} has no compose`); } From 79d546901c6aa3d4c6fd7b5f476847577657c410 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 20:04:30 -0600 Subject: [PATCH 09/23] worker: read a supplement's canonical through CodeSystem.supplementsUrl loadSupplements read cs.jsonObj.supplements of every CodeSystem among the request's resources. A new getter, CodeSystem.supplementsUrl, returns exactly that, and loadSupplements uses it. Nothing changes for the resources the server builds today; the point is the shared resource cache in the next commit, whose stand-ins answer supplementsUrl from their index row without loading the resource (as in #336, a37c84dc). Co-Authored-By: Claude Opus 5.5 --- tx/library/codesystem.js | 9 +++++++++ tx/workers/worker.js | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tx/library/codesystem.js b/tx/library/codesystem.js index 31b543a3..07a868eb 100644 --- a/tx/library/codesystem.js +++ b/tx/library/codesystem.js @@ -739,6 +739,15 @@ class CodeSystem extends CanonicalResource { return (this.jsonObj.property || []).find(x => x.code == code && x.uri == uri); } + /** + * The canonical of the code system this one supplements (CodeSystem.supplements), + * or undefined if it isn't a supplement. + * @returns {string|undefined} + */ + get supplementsUrl() { + return this.jsonObj.supplements; + } + isSupplementFor(url, version) { if (this.jsonObj.content !== 'supplement') { return false; diff --git a/tx/workers/worker.js b/tx/workers/worker.js index 8f9f3cfd..8dd23949 100644 --- a/tx/workers/worker.js +++ b/tx/workers/worker.js @@ -354,7 +354,9 @@ class TerminologyWorker { if (resource.resourceType === 'CodeSystem' && resource instanceof CodeSystem) { const cs = resource; // Check if this code system supplements the target URL - const supplementsUrl = cs.jsonObj.supplements; + // (supplementsUrl rather than jsonObj.supplements: a resource from the shared + // resource cache answers it without being loaded - see resource-cache-sqlite.js) + const supplementsUrl = cs.supplementsUrl; if (!supplementsUrl) { continue; From a87cb62c08277488c7b19c15fcff0f4c0559115e Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 20:04:30 -0600 Subject: [PATCH 10/23] server.cluster.sharedCache: the $cache-control caches in one SQLite file every process shares Off by default. With it on (and a role), the master and the tx-workers keep their $cache-control caches in databases/resource-cache.db instead of their own memory, so any process can answer any cache-id and a restarted process still has them. tx/resource-cache-sqlite.js is the store from #336 (4656acae, dea114c4, 80d2e834, 76f37f51, 1b1c632f), cut to what tx needs: - WAL, busy_timeout 5 s, every write one BEGIN IMMEDIATE transaction, retried once and then answered 503 CACHE_WRITE_FAILED (nothing is ever dropped silently) - last_used refreshed at most every 30 s per cache, so steady use is reads only; an unchanged re-send of resources takes no write lock - the idle sweep and the size cap (server.cluster.sharedCacheMaxMB, 0 = none, over the whole file) move what they drop to tombstones in the same transaction, so any process can say what happened to a dead id (CACHE_ID_EVICTED for the cap) - a request only loads the resources it uses: get() returns stand-ins answering url, version, type, supplements and the global-supplement flag from the index - each wrapper gets the stored row's hash through setExpansionKeyHash (P6, the previous commit but one), and a stand-in answers expansionKeyHash() from the row, so expansion keys are byte-identical to the memory store's and computing one loads and hashes nothing - an id with no record is CACHE_ID_UNKNOWN_SHARED whatever its prefix: every process sees every cache and caches survive a restart, so neither "issued somewhere else" nor "restarted since" applies Not brought: the memory store's size cap, the FHIRSMITH_CLUSTER_SIZE switch, the resourceCache.store config, the Retry-After middleware and #336's computeKey change (P6 replaces it). ResourceCache and operation-context.js are untouched. Wiring, all behind values only applyRole sets: - library/role.js: server.cluster.sharedCache / cacheAffinity must be booleans; sharedCache sets modules.tx.sharedCache (and sharedCacheMaxBytes) - tx/tx.js: `this.config.sharedCache ? createSharedResourceCache(path) : new ResourceCache(this.stats)`; shutdown closes the file only when it is on - cluster.js: with sharedCache, a cache-id request goes to the least-loaded tx process; cacheAffinity: true keeps the prefix routing. Without sharedCache the routing is as before. nginx.md says the map can go. Tests: the resource-cache suite runs against both stores (the misrouted-id and size-cap cases per store), plus the store's own cases and the P6 pre-fill (same key as the memory store's resources, two hashes only: the key and a plain resource); two forked processes on one file (use, end, sweep, restart, concurrent writes, the cap across processes); and cluster.js with a master and two workers: a session started on each process is used on all three (18 of 18, no cache-id-unknown), 60 concurrent requests through the launcher, an unknown id in the shared wording, and a worker SIGKILLed and restarted still has every session. With sharedCache false the same cluster test fails (12 cache-id-unknown). Co-Authored-By: Claude Opus 5.5 --- cluster.js | 21 +- library/role.js | 16 + nginx.md | 19 + tests/library/role.test.js | 17 + tests/server/cluster-launcher.test.js | 15 + tests/server/cluster-shared-cache.test.js | 112 +++ tests/tx/resource-cache-multiprocess.test.js | 148 ++++ tests/tx/resource-cache.test.js | 502 +++++++++++- tests/utils/cluster-harness.js | 125 +++ tests/utils/resource-cache-child.js | 52 ++ translations/Messages.properties | 2 + tx/resource-cache-sqlite.js | 812 +++++++++++++++++++ tx/tx.js | 27 +- 13 files changed, 1840 insertions(+), 28 deletions(-) create mode 100644 tests/server/cluster-shared-cache.test.js create mode 100644 tests/tx/resource-cache-multiprocess.test.js create mode 100644 tests/utils/cluster-harness.js create mode 100644 tests/utils/resource-cache-child.js create mode 100644 tx/resource-cache-sqlite.js diff --git a/cluster.js b/cluster.js index 3ebe3130..daa26910 100644 --- a/cluster.js +++ b/cluster.js @@ -67,11 +67,16 @@ function loadSettings(argv, env, config) { } procs.push({ name: 'worker ' + code, role: 'tx-worker', code, port: internalPort + i, servesTx: true }); } + // the options (cluster.md, "Options"): with the shared cache, any process can answer any + // cache-id, so by default a cached request goes to the least loaded; cacheAffinity: true + // keeps sending it to the process that issued it while that one is up + const sharedCache = cluster.sharedCache === true; return { port, procs, txPaths: (tx.endpoints || []).map(e => String(e.path).replace(/\/+$/, '')), - drainMs: Number(cluster.drainSeconds || 30) * 1000 + drainMs: Number(cluster.drainSeconds || 30) * 1000, + routing: { cacheAffinity: !sharedCache || cluster.cacheAffinity === true } }; } @@ -80,9 +85,11 @@ function loadSettings(argv, env, config) { * @param {string} url - the request target (path and query) * @param {Object} headers - the request headers (lower case names) * @param {string[]} txPaths - the tx endpoint paths, e.g. ['/r4', '/r5'] + * @param {Object} [routing] - from loadSettings: cacheAffinity (default true) sends a + * request with a cache-id to the process that issued it * @returns {{to: 'master'}|{to: 'owner', code: string}|{to: 'tx'}} */ -function classify(url, headers, txPaths) { +function classify(url, headers, txPaths, routing = {}) { let p = String(url).split('?')[0]; try { p = decodeURIComponent(p); @@ -99,6 +106,9 @@ function classify(url, headers, txPaths) { if (/^\/\$closure\/?$/.test(rest) || /^\/syndication(\/|$)/.test(rest) || rest.startsWith('/info/')) { return { to: 'master' }; } + if (routing.cacheAffinity === false) { + return { to: 'tx' }; + } const m = /^([A-Za-z0-9]{1,16})\./.exec(headers['x-cache-id'] || ''); return m ? { to: 'owner', code: m[1] } : { to: 'tx' }; } @@ -118,8 +128,9 @@ function pickTarget(route, procs) { if (owner) { return owner; } - // the owner is down (or the code is unknown): any tx process answers cache-id-unknown, - // which the client recovers from by starting a new cache + // the owner is down (or the code is unknown): any tx process answers. Without the + // shared cache that is cache-id-unknown, which the client recovers from by starting a + // new cache; with it, the cache is there } const candidates = procs.filter(p => p.servesTx && p.ready); if (candidates.length === 0) { @@ -207,7 +218,7 @@ function createLauncher(settings) { } function proxy(req, res) { - const target = pickTarget(classify(req.url, req.headers, settings.txPaths), procs); + const target = pickTarget(classify(req.url, req.headers, settings.txPaths, settings.routing), procs); if (!target) { return fail(res, 503, 'No server process is ready to take this request; try again shortly'); } diff --git a/library/role.js b/library/role.js index b56b59f3..c4342a33 100644 --- a/library/role.js +++ b/library/role.js @@ -77,6 +77,22 @@ function applyRole(config, role) { throw new Error('The master needs an instance code: --instance-code, FHIRSMITH_INSTANCE_CODE or modules.tx.instanceCode'); } tx.clusterRole = role.role; + // the options in server.cluster (cluster.md, "Options"). Every process reads the same + // config.json, so they all agree; without a role these are never looked at + const options = (config.server && config.server.cluster) || {}; + for (const name of ['sharedCache', 'cacheAffinity']) { + if (options[name] !== undefined && typeof options[name] !== 'boolean') { + throw new Error(`server.cluster.${name} must be true or false`); + } + } + if (options.sharedCache === true) { + tx.sharedCache = true; + const mb = options.sharedCacheMaxMB === undefined ? 0 : Number(options.sharedCacheMaxMB); + if (!Number.isFinite(mb) || mb < 0) { + throw new Error('server.cluster.sharedCacheMaxMB must be a number of megabytes (0: no cap)'); + } + tx.sharedCacheMaxBytes = Math.round(mb * 1024 * 1024); + } if (tx.vsacCfg) { tx.vsacCfg = { ...tx.vsacCfg, cluster: role.role === 'master' ? 'leader' : 'follower' }; } diff --git a/nginx.md b/nginx.md index eb7ffcb1..9583c147 100644 --- a/nginx.md +++ b/nginx.md @@ -465,3 +465,22 @@ server { List your own tx endpoint paths (`modules.tx.endpoints`) in the two regex locations. The optional launcher, `node cluster.js`, starts the processes in this order and applies the same rules without nginx (it can also sit behind nginx, as one upstream). + +### With the shared cache + +With `server.cluster.sharedCache: true` ([cluster.md](cluster.md#options)), every process +keeps its `$cache-control` caches in one SQLite file, so any of them can answer any cache-id. +The `map` on `X-Cache-Id` and the per-worker upstreams (`fs_tx1`, `fs_tx2`) are then not +needed: the tx location goes straight to the pool. + +```nginx + location ~ ^/(r4|r5)(/|$) { + include fhirsmith-proxy.conf; + proxy_pass http://fs_tx_pool; + } +``` + +Keeping the `map` still works (the launcher's `cacheAffinity: true`). It keeps each client on +one process, whose expansion cache (which is per process) already holds the expansions it +asked for; without it, the load spreads evenly, and the first time each process expands a +value set for a client it does the work again. diff --git a/tests/library/role.test.js b/tests/library/role.test.js index 755836b1..253b8b6e 100644 --- a/tests/library/role.test.js +++ b/tests/library/role.test.js @@ -77,6 +77,23 @@ describe('applyRole', () => { expect(config.dailyReport.enabled).toBe(false); }); + test('server.cluster.sharedCache reaches tx (with a role), and nothing is set when it is off', () => { + const off = applyRole(baseConfig(), { role: 'tx-worker', instanceCode: 'tx1' }); + expect(off.modules.tx.sharedCache).toBeUndefined(); + expect(off.modules.tx.sharedCacheMaxBytes).toBeUndefined(); + const on = baseConfig(); + on.server.cluster = { sharedCache: true, sharedCacheMaxMB: 2 }; + applyRole(on, { role: 'master', instanceCode: null }); + expect(on.modules.tx.sharedCache).toBe(true); + expect(on.modules.tx.sharedCacheMaxBytes).toBe(2 * 1024 * 1024); + const bad = baseConfig(); + bad.server.cluster = { sharedCache: 'yes' }; + expect(() => applyRole(bad, { role: 'master', instanceCode: null })).toThrow(/true or false/); + const badCap = baseConfig(); + badCap.server.cluster = { sharedCache: true, sharedCacheMaxMB: -1 }; + expect(() => applyRole(badCap, { role: 'master', instanceCode: null })).toThrow(/megabytes/); + }); + test('a worker needs its own instance code, and tx must be enabled', () => { expect(() => applyRole(baseConfig(), { role: 'tx-worker', instanceCode: null })).toThrow(/its own instance code/); const noCode = baseConfig(); diff --git a/tests/server/cluster-launcher.test.js b/tests/server/cluster-launcher.test.js index 62a558ee..2b62e9f6 100644 --- a/tests/server/cluster-launcher.test.js +++ b/tests/server/cluster-launcher.test.js @@ -48,6 +48,21 @@ describe('routing rules', () => { expect(classify('/r4/ValueSet/$expand', {}, TX)).toEqual({ to: 'tx' }); }); + test('with the shared cache, a cache-id goes to any tx process, unless cacheAffinity is on', () => { + const shared = loadSettings([], {}, { server: { cluster: { sharedCache: true } }, modules: { tx: { enabled: true } } }).routing; + expect(shared).toEqual({ cacheAffinity: false }); + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, shared)).toEqual({ to: 'tx' }); + const affinity = loadSettings([], {}, { server: { cluster: { sharedCache: true, cacheAffinity: true } }, modules: { tx: { enabled: true } } }).routing; + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, affinity)).toEqual({ to: 'owner', code: 'tx2' }); + // without the shared cache, a cache-id always goes to its issuer: there is nowhere else it is + const plain = loadSettings([], {}, { server: { cluster: { cacheAffinity: false } }, modules: { tx: { enabled: true } } }).routing; + expect(plain).toEqual({ cacheAffinity: true }); + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, plain)).toEqual({ to: 'owner', code: 'tx2' }); + // the master-only paths stay the master's + expect(classify('/r4/$closure', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, shared)).toEqual({ to: 'master' }); + expect(classify('/', {}, TX, shared)).toEqual({ to: 'master' }); + }); + test('$closure, in any FHIR version, encoded or not, goes to the master, whatever the cache-id', () => { for (const url of ['/r4/$closure', '/r5/$closure', '/r4/%24closure', '/r4/$closure?name=x']) { expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX)).toEqual({ to: 'master' }); diff --git a/tests/server/cluster-shared-cache.test.js b/tests/server/cluster-shared-cache.test.js new file mode 100644 index 00000000..9d5375a5 --- /dev/null +++ b/tests/server/cluster-shared-cache.test.js @@ -0,0 +1,112 @@ +/** + * server.cluster.sharedCache with real processes: a master and two tx-workers behind + * cluster.js, sharing databases/resource-cache.db. + * + * - a $cache-control session started on one process is used on each of the others, + * directly and through the launcher (which, without cacheAffinity, spreads cached + * requests by load), with no cache-id-unknown + * - an id no process issued is CACHE_ID_UNKNOWN_SHARED + * - a worker killed and restarted by the launcher still has the sessions + */ +const { startCluster, request, param, unknownCache, pidOn, sleep } = require('../utils/cluster-harness'); + +const FRONT = 20080; +const VSAC = 20089; + +const cs = (n) => ({ resourceType: 'CodeSystem', url: `http://example.org/shared/cs${n}`, version: '1', status: 'active', content: 'complete', + concept: [{ code: 'a', display: 'A' }, { code: 'b', display: 'B' }] }); +const vs = (n) => ({ resourceType: 'ValueSet', url: `http://example.org/shared/vs${n}`, version: '1', status: 'active', + compose: { include: [{ system: cs(n).url }] } }); + +function validate(port, cid, n, code = 'a') { + return request(port, 'POST', '/r4/ValueSet/$validate-code', { + headers: { 'X-Cache-Id': cid }, + body: { resourceType: 'Parameters', parameter: [{ name: 'url', valueUri: vs(n).url }, { name: 'coding', valueCoding: { system: cs(n).url, code } }] } + }); +} + +describe('cluster.js with server.cluster.sharedCache', () => { + let cluster; + + beforeAll(async () => { + cluster = await startCluster({ front: FRONT, vsac: VSAC, cluster: { sharedCache: true } }); + }, 330000); + + afterAll(async () => { + if (cluster) { + await cluster.stop(); + } + }); + + const sessions = []; + + test('a session started on one process is used on every other one', async () => { + const all = [cluster.master, ...cluster.workers]; + let unknown = 0; + let ok = 0; + for (const [i, port] of all.entries()) { + // started (sealed, with its resources) on this process, by its own port + const start = await request(port, 'POST', '/r4/$cache-control?mode=start', { + body: { resourceType: 'Parameters', parameter: [{ name: 'sealed', valueBoolean: true }, + { name: 'tx-resource', resource: cs(i) }, { name: 'tx-resource', resource: vs(i) }] } + }); + expect(start.status).toBe(200); + const cid = param(start.json, 'cache-id').valueId; + sessions.push({ cid, n: i, issuer: port }); + for (const other of all) { + for (const code of ['a', 'b']) { + const r = await validate(other, cid, i, code); + if (unknownCache(r)) { + unknown++; + } else if (r.status === 200 && param(r.json, 'result').valueBoolean === true) { + ok++; + } + } + } + } + expect(unknown).toBe(0); + expect(ok).toBe(all.length * all.length * 2); + expect(new Set(sessions.map(s => s.cid.split('.')[0]))).toEqual(new Set(['m0', 'tx1', 'tx2'])); + }, 60000); + + test('through the launcher, cached requests go to every tx process and all find their cache', async () => { + // concurrent, so the least-loaded choice spreads them + const results = await Promise.all(Array.from({ length: 60 }, (_, k) => { + const s = sessions[k % sessions.length]; + return validate(FRONT, s.cid, s.n); + })); + expect(results.filter(unknownCache)).toHaveLength(0); + expect(results.filter(r => r.status === 200 && param(r.json, 'result').valueBoolean === true)).toHaveLength(60); + }, 60000); + + test('an id that no process issued is unknown, in the shared store\'s words', async () => { + const r = await validate(FRONT, 'tx1.00000000-0000-4000-8000-000000000000', 0); + expect(r.status).toBe(404); + expect(unknownCache(r)).toBe(true); + const text = r.json.issue[0].details.text; + expect(text).toMatch(/survive a restart/); + expect(text).not.toMatch(/restarted since/); + expect(text).not.toMatch(/reached the wrong server/); + }); + + test('a worker that is killed and restarted still has the sessions, its own included', async () => { + const port = cluster.workers[0]; + const pid = pidOn(port); + expect(pid).toBeGreaterThan(0); + process.kill(pid, 'SIGKILL'); + const t0 = Date.now(); + while (pidOn(port) === pid || pidOn(port) === null) { + if (Date.now() - t0 > 120000) { + throw new Error('the worker was not restarted'); + } + await sleep(250); + } + await cluster.waitUp([port]); + expect(pidOn(port)).not.toBe(pid); + for (const s of sessions) { + const r = await validate(port, s.cid, s.n); + expect(unknownCache(r)).toBe(false); + expect(param(r.json, 'result').valueBoolean).toBe(true); + } + }, 180000); +}); diff --git a/tests/tx/resource-cache-multiprocess.test.js b/tests/tx/resource-cache-multiprocess.test.js new file mode 100644 index 00000000..a9bcd845 --- /dev/null +++ b/tests/tx/resource-cache-multiprocess.test.js @@ -0,0 +1,148 @@ +/** + * The shared SQLite resource cache across processes: what server.cluster.sharedCache relies on. + * Two real processes open the same file; a cache started in one is used and ended + * in the other, and survives a process restart. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { fork } = require('child_process'); + +const CHILD = path.join(__dirname, '..', 'utils', 'resource-cache-child.js'); + +function startProcess(file, maxBytes = 0) { + const child = fork(CHILD, [file, String(maxBytes)], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }); + let seq = 0; + const waiting = new Map(); + let readyResolve; + const ready = new Promise(resolve => { readyResolve = resolve; }); + child.on('message', (m) => { + if (m.ready) { + readyResolve(m.ready); + return; + } + const w = waiting.get(m.seq); + waiting.delete(m.seq); + if (m.error) { + w.reject(new Error(m.error)); + } else { + w.resolve(m.result); + } + }); + const call = (op, args = {}) => new Promise((resolve, reject) => { + const s = ++seq; + waiting.set(s, { resolve, reject }); + child.send({ seq: s, op, ...args }); + }); + const stop = () => new Promise(resolve => { + child.once('exit', resolve); + child.kill(); + }); + return { ready, call, stop, pid: child.pid }; +} + +const cs = (url, version, codes) => ({ + resourceType: 'CodeSystem', url, version, status: 'active', content: 'complete', + concept: codes.map(code => ({ code, display: code.toUpperCase() })) +}); + +describe('shared SQLite resource cache across processes', () => { + let dir; + let file; + const running = []; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'resource-cache-mp-')); + file = path.join(dir, 'resource-cache.db'); + }); + + afterEach(async () => { + while (running.length) { + await running.pop().stop(); + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + const spawn = async (maxBytes = 0) => { + const p = startProcess(file, maxBytes); + running.push(p); + await p.ready; + return p; + }; + + test('A starts a cache and adds resources, B reads them, B ends it, A sees the tombstone', async () => { + const a = await spawn(); + const b = await spawn(); + expect(a.pid).not.toBe(b.pid); + + const id = await a.call('start', { resources: [cs('http://example.org/one', '1', ['a', 'b'])] }); + await a.call('add', { id, resources: [cs('http://example.org/two', '2', ['x', 'y', 'z'])] }); + + expect(await b.call('has', { id })).toBe(true); + expect(await b.call('get', { id })).toEqual([ + { cls: 'CodeSystem', url: 'http://example.org/one', version: '1', fhirVersion: 'R4', concepts: 2 }, + { cls: 'CodeSystem', url: 'http://example.org/two', version: '2', fhirVersion: 'R4', concepts: 3 } + ]); + + // B adds too, and A sees it + await b.call('add', { id, resources: [cs('http://example.org/three', '3', ['q'])] }); + expect((await a.call('get', { id })).map(r => r.url)).toEqual( + ['http://example.org/one', 'http://example.org/two', 'http://example.org/three']); + + await b.call('end', { id }); + expect(await a.call('has', { id })).toBe(false); + expect(await a.call('get', { id })).toEqual([]); + const d = await a.call('describe', { id }); + expect(d.messageId).toBe('CACHE_ID_CLOSED'); + expect(d.params[0]).toBe(id); + }); + + test('the idle sweep run by either process is seen by both, and running it twice is harmless', async () => { + const a = await spawn(); + const b = await spawn(); + const id = await a.call('start', { resources: [cs('http://example.org/one', '1', ['a'])] }); + // prune with a negative max age: everything counts as idle + await b.call('prune', { maxAge: -1000 }); + await a.call('prune', { maxAge: -1000 }); + expect(await a.call('has', { id })).toBe(false); + expect((await a.call('describe', { id })).messageId).toBe('CACHE_ID_EXPIRED'); + expect((await b.call('describe', { id })).messageId).toBe('CACHE_ID_EXPIRED'); + }); + + test('a cache survives a process restart', async () => { + const a = await spawn(); + const id = await a.call('start', { resources: [cs('http://example.org/one', '1', ['a', 'b'])], sealed: true }); + await running.pop().stop(); + + const again = await spawn(); + expect(await again.call('has', { id })).toBe(true); + expect((await again.call('get', { id })).map(r => r.url)).toEqual(['http://example.org/one']); + }); + + test('many writes from two processes at once all land', async () => { + const a = await spawn(); + const b = await spawn(); + const id = await a.call('start', {}); + const writes = []; + for (let i = 0; i < 50; i++) { + writes.push(a.call('add', { id, resources: [cs(`http://example.org/a${i}`, '1', ['c'])] })); + writes.push(b.call('add', { id, resources: [cs(`http://example.org/b${i}`, '1', ['c'])] })); + } + await Promise.all(writes); + expect(await a.call('get', { id })).toHaveLength(100); + }); + + test('the size cap is kept across processes: a write in one drops the other\'s least recently used cache', async () => { + const a = await spawn(4000); + const b = await spawn(4000); + const big = (url) => ({ ...cs(url, '1', ['a']), description: 'x'.repeat(1500) }); + const old = await b.call('start', { resources: [big('http://example.org/old')] }); + await b.call('setLastUsed', { id: old, when: Date.now() - 60000 }); + const kept = await b.call('start', { resources: [big('http://example.org/kept')] }); + const fresh = await a.call('start', { resources: [big('http://example.org/fresh')] }); + expect(await b.call('has', { id: old })).toBe(false); + expect((await b.call('describe', { id: old })).messageId).toBe('CACHE_ID_EVICTED'); + expect(await b.call('has', { id: kept })).toBe(true); + expect(await b.call('has', { id: fresh })).toBe(true); + }); +}); diff --git a/tests/tx/resource-cache.test.js b/tests/tx/resource-cache.test.js index 35844394..25c32a9c 100644 --- a/tests/tx/resource-cache.test.js +++ b/tests/tx/resource-cache.test.js @@ -4,9 +4,19 @@ * ResourceCache backs the cache-id protocol: resources sent under a cache-id on one * request must be retrievable on later requests with the same cache-id, and kept * isolated between different cache-ids. + * + * Every test here runs against both stores - the in-process memory store and the + * shared SQLite store (server.cluster.sharedCache) - because they are one interface with + * one set of semantics. The two differ only where a shared store must: an id with no + * record is CACHE_ID_UNKNOWN_SHARED whatever its prefix, and only the shared store has a + * size cap. */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); const { ResourceCache, formatDuration } = require('../../tx/operation-context'); +const { SqliteResourceCache } = require('../../tx/resource-cache-sqlite'); const CS = (url, version) => ({ resourceType: 'CodeSystem', url, version }); const VS = (url, version) => ({ resourceType: 'ValueSet', url, version }); @@ -14,11 +24,53 @@ const VS = (url, version) => ({ resourceType: 'ValueSet', url, version }); // A resource that reports a concept count, like the real wrapper classes do. const sized = (resource, n) => ({ ...resource, conceptCount: () => n }); -describe('ResourceCache', () => { +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'resource-cache-test-')); +let fileNo = 0; +const opened = []; +afterAll(() => { + for (const c of opened) { + c.close(); + } + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +// How each store is made, and how a test makes an entry look idle +const STORES = [ + { + name: 'memory', + unknownId: 'CACHE_ID_UNKNOWN', + misrouted: (named) => (named ? 'CACHE_ID_OTHER_INSTANCE' : 'CACHE_ID_OTHER_INSTANCE_UNNAMED'), + make: (opts = {}) => new ResourceCache(null, opts.maxTombstones || 1000), + setLastUsed: (cache, id, when) => { cache.cache.get(id).lastUsed = when; } + }, + { + name: 'sqlite', + // caches survive a restart here, so the message doesn't offer one as the reason + unknownId: 'CACHE_ID_UNKNOWN_SHARED', + // every process sees every cache, so a prefix naming another process says nothing + misrouted: () => 'CACHE_ID_UNKNOWN_SHARED', + make: (opts = {}) => { + const c = new SqliteResourceCache(null, { + path: path.join(tmpDir, `cache-${++fileNo}.db`), + scope: '/r4', + touchIntervalMs: 0, + ...opts + }); + opened.push(c); + return c; + }, + setLastUsed: (cache, id, when) => { + cache.db.prepare('UPDATE sessions SET last_used = ? WHERE scope = ? AND cache_id = ?').run(when, cache.scope, id); + } + } +]; + +describe.each(STORES)('ResourceCache ($name store)', (store) => { let cache; + const setLastUsed = (id, when) => store.setLastUsed(cache, id, when); beforeEach(() => { - cache = new ResourceCache(null); + cache = store.make(); }); describe('get / has on empty cache', () => { @@ -129,7 +181,7 @@ describe('ResourceCache', () => { test('removes entries older than maxAge but keeps fresh ones', () => { cache.add('old', [CS('http://old', '1')]); // Force the entry to look stale. - cache.cache.get('old').lastUsed = Date.now() - 10000; + setLastUsed('old', Date.now() - 10000); cache.add('fresh', [CS('http://fresh', '1')]); cache.prune(5000); // maxAge 5s @@ -140,7 +192,7 @@ describe('ResourceCache', () => { test('get() refreshes lastUsed so an active entry is not pruned', () => { cache.add('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10000; + setLastUsed('c1', Date.now() - 10000); cache.get('c1'); // touch -> refreshes lastUsed cache.prune(5000); expect(cache.has('c1')).toBe(true); @@ -151,10 +203,10 @@ describe('ResourceCache', () => { // the whole point: "you closed it yourself" and "it sat idle for 31 minutes" send // an investigator to completely different places (see issue #279). describe('tombstones (why a cache-id is gone)', () => { - test('an id that was never issued has no tombstone and reports CACHE_ID_UNKNOWN', () => { + test('an id that was never issued has no tombstone and reports CACHE_ID_UNKNOWN (memory) or CACHE_ID_UNKNOWN_SHARED', () => { const d = cache.describeMissing('never-seen'); expect(cache.tombstone('never-seen')).toBeNull(); - expect(d.messageId).toBe('CACHE_ID_UNKNOWN'); + expect(d.messageId).toBe(store.unknownId); expect(d.params).toEqual(['never-seen']); }); @@ -171,7 +223,7 @@ describe('ResourceCache', () => { test('prune() records the expiry with the measured idle time, not the timeout', () => { cache.add('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10 * 60 * 1000; // idle 10 min + setLastUsed('c1', Date.now() - 10 * 60 * 1000); // idle 10 min cache.prune(5 * 60 * 1000); // timeout 5 min - the sweep is late, as it is in production const t = cache.tombstone('c1'); @@ -209,7 +261,7 @@ describe('ResourceCache', () => { test('the latest fate wins when an id dies twice', () => { cache.set('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10000; + setLastUsed('c1', Date.now() - 10000); cache.prune(5000); expect(cache.tombstone('c1').reason).toBe('expired'); @@ -219,7 +271,7 @@ describe('ResourceCache', () => { }); test('tombstones are bounded: the oldest are evicted, never unbounded growth', () => { - const small = new ResourceCache(null, 3); + const small = store.make({ maxTombstones: 3 }); for (const id of ['a', 'b', 'c', 'd', 'e']) { small.set(id, [CS('http://' + id, '1')]); small.clear(id); @@ -229,13 +281,13 @@ describe('ResourceCache', () => { expect(small.tombstone('b')).toBeNull(); // evicted expect(small.tombstone('e')).not.toBeNull(); // newest kept // An evicted tombstone degrades to "never issued" rather than lying. - expect(small.describeMissing('a').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(small.describeMissing('a').messageId).toBe(store.unknownId); }); test('closing an id the server never had leaves no tombstone', () => { cache.clear('not-mine'); expect(cache.tombstone('not-mine')).toBeNull(); - expect(cache.describeMissing('not-mine').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(cache.describeMissing('not-mine').messageId).toBe(store.unknownId); }); }); @@ -282,27 +334,31 @@ describe('ResourceCache', () => { expect(ResourceCache.issuerOf(undefined)).toBeNull(); }); - test('an id from another instance is reported as misrouted, naming both instances', () => { + test('an id from another instance is reported as misrouted, naming both instances (memory store)', () => { cache.setInstanceCode('tx1'); const d = cache.describeMissing('tx2.abc'); - expect(d.messageId).toBe('CACHE_ID_OTHER_INSTANCE'); - expect(d.params).toEqual(['tx2.abc', 'tx2', 'tx1']); + expect(d.messageId).toBe(store.misrouted(true)); + if (store.name === 'memory') { + expect(d.params).toEqual(['tx2.abc', 'tx2', 'tx1']); + } }); - test('a server with no code reports a prefixed id as issued elsewhere', () => { + test('a server with no code reports a prefixed id as issued elsewhere (memory store)', () => { const d = cache.describeMissing('tx2.abc'); - expect(d.messageId).toBe('CACHE_ID_OTHER_INSTANCE_UNNAMED'); - expect(d.params).toEqual(['tx2.abc', 'tx2']); + expect(d.messageId).toBe(store.misrouted(false)); + if (store.name === 'memory') { + expect(d.params).toEqual(['tx2.abc', 'tx2']); + } }); test('an unrecorded id with our own prefix is plain unknown (e.g. issued before a restart)', () => { cache.setInstanceCode('tx1'); - expect(cache.describeMissing('tx1.abc').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(cache.describeMissing('tx1.abc').messageId).toBe(store.unknownId); }); test('an unprefixed id on a named server is plain unknown', () => { cache.setInstanceCode('tx1'); - expect(cache.describeMissing('abc').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(cache.describeMissing('abc').messageId).toBe(store.unknownId); }); test('a tombstone wins over the prefix check', () => { @@ -320,7 +376,7 @@ describe('ResourceCache', () => { describe('status / touch (mode=check)', () => { test('status reports a live cache without touching it', () => { cache.set('c1', [CS('http://a', '1'), VS('http://b', '1')], true); - cache.cache.get('c1').lastUsed = Date.now() - 90 * 1000; + setLastUsed('c1', Date.now() - 90 * 1000); const s = cache.status('c1'); expect(s.exists).toBe(true); @@ -338,7 +394,7 @@ describe('ResourceCache', () => { test('touch resets the idle clock and saves the cache from the next prune', () => { cache.add('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10000; + setLastUsed('c1', Date.now() - 10000); expect(cache.touch('c1')).toBe(true); expect(cache.status('c1').idleMs).toBeLessThan(1000); @@ -431,7 +487,7 @@ describe('ResourceCache', () => { test('prune subtracts evicted entries', () => { cache.add('old', [sized(CS('http://old', '1'), 7)]); - cache.cache.get('old').lastUsed = Date.now() - 10000; + setLastUsed('old', Date.now() - 10000); cache.add('fresh', [sized(CS('http://fresh', '1'), 4)]); cache.prune(5000); expect(cache.conceptCount()).toBe(4); @@ -466,4 +522,406 @@ describe('ResourceCache', () => { expect(cache.maxConceptCount()).toBe(20); }); }); + + describe('sealed and unsealed', () => { + test('set() records whether the cache is sealed; add() keeps an unsealed cache unsealed', () => { + cache.set('open', [CS('http://a', '1')], false); + cache.set('shut', [CS('http://a', '1')], true); + expect(cache.isSealed('open')).toBe(false); + expect(cache.isSealed('shut')).toBe(true); + expect(cache.isSealed('nope')).toBe(false); + cache.add('open', [VS('http://v', '1')]); + expect(cache.isSealed('open')).toBe(false); + expect(cache.get('open').map(r => r.url)).toEqual(['http://a', 'http://v']); + }); + + test('an empty cache made by set() exists', () => { + cache.set('empty', [], true); + expect(cache.has('empty')).toBe(true); + expect(cache.get('empty')).toEqual([]); + expect(cache.status('empty').resources).toBe(0); + }); + + test('order is kept, and a replacement keeps its place', () => { + cache.add('c1', [CS('http://a', '1'), CS('http://b', '1'), CS('http://c', '1')]); + cache.add('c1', [{ ...CS('http://b', '1'), title: 'new b' }, CS('http://d', '1')]); + const got = cache.get('c1'); + expect(got.map(r => r.url)).toEqual(['http://a', 'http://b', 'http://c', 'http://d']); + expect(got[1].title).toBe('new b'); + }); + }); + + describe('resources are marked as cached', () => { + test('get() returns resources with a non-enumerable isCached marker', () => { + cache.add('c1', [CS('http://a', '1')]); + const [r] = cache.get('c1'); + expect(r.isCached).toBe(true); + expect(Object.keys(r)).not.toContain('isCached'); + expect(JSON.stringify(r)).not.toContain('isCached'); + }); + }); + + // A cap on the total size of what the cache holds: when a write takes it over, whole + // caches are dropped, least recently used first, and remembered as `evicted`. + // only the shared store has a cap: the memory store is as it was + (store.name === 'sqlite' ? describe : describe.skip)('size cap (maxBytes)', () => { + const big = (url, n) => ({ ...CS(url, '1'), description: 'x'.repeat(n) }); + + test('drops the least recently used caches, never the one being written', () => { + cache = store.make({ maxBytes: 2500 }); + cache.set('a', [big('http://a', 1000)]); + cache.set('b', [big('http://b', 1000)]); + setLastUsed('a', Date.now() - 60000); + setLastUsed('b', Date.now() - 30000); + cache.set('c', [big('http://c', 1000)]); // over the cap: 'a' is the oldest + expect(cache.has('a')).toBe(false); + expect(cache.has('b')).toBe(true); + expect(cache.has('c')).toBe(true); + expect(cache.byteCount()).toBeLessThanOrEqual(2500); + + const t = cache.tombstone('a'); + expect(t.reason).toBe('evicted'); + expect(t.maxBytes).toBe(2500); + expect(t.idleMs).toBeGreaterThanOrEqual(60000); + const d = cache.describeMissing('a'); + expect(d.messageId).toBe('CACHE_ID_EVICTED'); + expect(d.params[0]).toBe('a'); + expect(d.params[2]).toBe('2 KB'); + expect(d.params[3]).toBe('60 seconds'); + }); + + test('growing a cache with add() can evict others too', () => { + cache = store.make({ maxBytes: 2500 }); + cache.set('a', [big('http://a', 1000)]); + setLastUsed('a', Date.now() - 60000); + cache.set('b', [big('http://b', 1000)]); + cache.add('b', [big('http://b2', 1000)]); + expect(cache.has('a')).toBe(false); + expect(cache.tombstone('a').reason).toBe('evicted'); + expect(cache.get('b')).toHaveLength(2); + }); + + test('a single cache bigger than the cap is kept (there is nothing else to drop)', () => { + cache = store.make({ maxBytes: 500 }); + cache.set('a', [big('http://a', 1000)]); + expect(cache.has('a')).toBe(true); + }); + + test('no cap, nothing is dropped', () => { + for (let i = 0; i < 20; i++) { + cache.set('c' + i, [big('http://c' + i, 1000)]); + } + expect(cache.size()).toBe(20); + }); + }); +}); + +// Behaviour that only the shared SQLite store has. +describe('SqliteResourceCache', () => { + const { CodeSystem } = require('../../tx/library/codesystem'); + const ValueSet = require('../../tx/library/valueset'); + const make = (opts = {}) => { + const c = new SqliteResourceCache(null, { path: path.join(tmpDir, `only-${++fileNo}.db`), scope: '/r4', ...opts }); + opened.push(c); + return c; + }; + const lastUsedOf = (c, id) => c.db.prepare('SELECT last_used FROM sessions WHERE scope = ? AND cache_id = ?').get(c.scope, id).last_used; + + test('wrappers come back as wrappers, with the FHIR version they arrived in', () => { + const c = make(); + const cs = new CodeSystem({ + resourceType: 'CodeSystem', url: 'http://example.org/cs', version: '1', status: 'active', content: 'complete', + concept: [{ code: 'a', display: 'A' }, { code: 'b', display: 'B', concept: [{ code: 'b1' }] }] + }, 'R4'); + const vs = new ValueSet({ + resourceType: 'ValueSet', url: 'http://example.org/vs', status: 'active', + compose: { include: [{ system: 'http://example.org/cs', filter: [{ property: 'concept', op: 'is-a', value: 'b' }] }] } + }, 'R3'); + c.set('c1', [cs, vs]); + const [cs2, vs2] = c.get('c1'); + expect(cs2).toBeInstanceOf(CodeSystem); + expect(vs2).toBeInstanceOf(ValueSet); + expect(cs2.fhirVersion).toBe('R4'); + expect(vs2.fhirVersion).toBe('R3'); + expect(cs2.jsonObj).toEqual(cs.jsonObj); + expect(vs2.jsonObj).toEqual(vs.jsonObj); + expect(cs2.conceptCount()).toBe(3); + expect(c.conceptCountFor('c1')).toBe(3); + expect(cs2.isCached).toBe(true); + // a fresh object every time: nothing is shared between requests + expect(c.get('c1')[0]).not.toBe(cs2); + }); + + describe('a request only loads the resources it uses', () => { + const { ConceptMap } = require('../../tx/library/conceptmap'); + const { ExpansionCache } = require('../../tx/operation-context'); + const mkCs = (url, version, extra = {}) => new CodeSystem({ + resourceType: 'CodeSystem', url, version, status: 'active', content: 'complete', + concept: [{ code: 'a', display: 'A' }], ...extra + }, 'R4'); + const countLoads = (c) => { + const orig = c.q.content; + const counter = { n: 0 }; + c.q.content = { get: (...args) => { counter.n++; return orig.get(...args); } }; + return counter; + }; + + test('looking through the cache by url, version and type loads nothing', () => { + const c = make(); + c.set('c1', [mkCs('http://x/a', '1'), mkCs('http://x/b', '2'), mkCs('http://x/sup', '1', { content: 'supplement', supplements: 'http://x/a' })]); + const loads = countLoads(c); + const got = c.get('c1'); + expect(got.map(r => r.url)).toEqual(['http://x/a', 'http://x/b', 'http://x/sup']); + expect(got.map(r => r.version)).toEqual(['1', '2', '1']); + expect(got.map(r => r.vurl)).toEqual(['http://x/a|1', 'http://x/b|2', 'http://x/sup|1']); + expect(got.every(r => r.resourceType === 'CodeSystem' && r instanceof CodeSystem)).toBe(true); + expect(got.some(r => r instanceof ConceptMap)).toBe(false); + expect(got.map(r => r.supplementsUrl)).toEqual([undefined, undefined, 'http://x/a']); + expect(got[2].isGlobalSupplement()).toBe(false); + expect(got.every(r => r.isCached === true)).toBe(true); + expect(loads.n).toBe(0); + + // using one loads that one only, once + expect(got[1].content).toBe('complete'); + expect(got[1].jsonObj.concept[0].code).toBe('a'); + expect(got[1].conceptCount()).toBe(1); + expect(loads.n).toBe(1); + }); + + test('a loaded stand-in behaves like the wrapper it stands for', () => { + const c = make(); + const original = mkCs('http://x/a', '1'); + c.set('c1', [original]); + const [r] = c.get('c1'); + expect(r.jsonObj).toEqual(original.jsonObj); + expect(r.fhirVersion).toBe('R4'); + expect(r.versionAlgorithm()).toEqual(original.versionAlgorithm()); + expect(JSON.parse(JSON.stringify(r)).jsonObj).toEqual(original.jsonObj); + r.sourcePackage = 'p'; + expect(r.sourcePackage).toBe('p'); + }); + + test('the expansion cache key is the same as for the resources built eagerly', () => { + const c = make(); + const resources = [mkCs('http://x/a', '1'), new ValueSet({ resourceType: 'ValueSet', url: 'http://x/vs', status: 'active', + compose: { include: [{ system: 'http://x/a' }] } }, 'R4')]; + c.set('c1', resources); + const ec = new ExpansionCache(null); + const vs = { resourceType: 'ValueSet', url: 'http://x/vs' }; + const loads = countLoads(c); + const lazyKey = ec.computeKey(vs, null, c.get('c1')); + expect(loads.n).toBe(0); + const eager = c.q.resources.all('/r4', 'c1').map(row => c._build(row)); + expect(ec.computeKey(vs, null, eager)).toBe(lazyKey); + expect(ec.computeKey(vs, null, resources)).toBe(lazyKey); + }); + + test('the stored hash pre-fills each resource\'s expansion-key hash (P6): same keys, no hashing', () => { + const crypto = require('crypto'); + const { CanonicalResource } = require('../../tx/library/canonical-resource'); + const c = make(); + const resources = [ + mkCs('http://x/a', '1'), + mkCs('http://x/sup', '1', { content: 'supplement', supplements: 'http://x/a' }), + new ValueSet({ resourceType: 'ValueSet', url: 'http://x/vs', status: 'active', + compose: { include: [{ system: 'http://x/a', filter: [{ property: 'concept', op: 'is-a', value: 'a' }] }] } }, 'R3'), + new ConceptMap({ resourceType: 'ConceptMap', url: 'http://x/cm', status: 'active', + group: [{ source: 'http://x/a', target: 'http://x/b', element: [{ code: 'a', target: [{ code: 'b', relationship: 'equivalent' }] }] }] }), + { resourceType: 'NamingSystem', url: 'http://x/ns', name: 'plain' } + ]; + // the keys the memory store's resources (the ones the client sent) make + const vs = new ValueSet({ resourceType: 'ValueSet', url: 'http://x/expand-me', status: 'active', compose: { include: [{ system: 'http://x/a' }] } }); + const ec = new ExpansionCache(null); + const params = { hashSource: () => 'p' }; + const memoryKey = ec.computeKey(vs, params, resources); + c.set('c1', resources); + + // stand-ins: the hash comes from the row - nothing is loaded and nothing hashed + // but the key itself and the plain resource + const lazy = c.get('c1'); + const loads = countLoads(c); + const createHash = jest.spyOn(crypto, 'createHash'); + let lazyKey; + try { + lazyKey = ec.computeKey(vs, params, lazy); + expect(createHash).toHaveBeenCalledTimes(2); + } finally { + createHash.mockRestore(); + } + expect(loads.n).toBe(0); + expect(lazyKey).toBe(memoryKey); + + // loaded: each wrapper was given the row's hash when it was built + for (const r of lazy.slice(0, 4)) { + void r.jsonObj; + } + expect(loads.n).toBe(4); + for (const [i, r] of lazy.entries()) { + const expected = CanonicalResource.computeExpansionKeyHash(resources[i].jsonObj || resources[i]); + if (typeof r.expansionKeyHash === 'function') { + expect(r.expansionKeyHash()).toBe(expected); + } + } + const spy = jest.spyOn(crypto, 'createHash'); + try { + expect(ec.computeKey(vs, params, lazy)).toBe(memoryKey); + expect(spy).toHaveBeenCalledTimes(2); + } finally { + spy.mockRestore(); + } + // and built eagerly from the rows + const eager = c.q.resources.all('/r4', 'c1').map(row => c._build(row)); + expect(ec.computeKey(vs, params, eager)).toBe(memoryKey); + }); + + test('a resource that has gone by the time it is used fails like a dead cache-id', () => { + const c = make(); + c.set('c1', [mkCs('http://x/a', '1')]); + const [r] = c.get('c1'); + c.clear('c1'); + let err = null; + try { + void r.jsonObj; + } catch (e) { + err = e; + } + expect(err).not.toBeNull(); + expect(err.statusCode).toBe(404); + expect(err.issueCode).toBe('cache-id-unknown'); + expect(err.message).toMatch(/was closed while this request was using it/); + }); + }); + + test('endpoints share the file but not caches', () => { + const file = path.join(tmpDir, `scopes-${++fileNo}.db`); + const r4 = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + const r5 = new SqliteResourceCache(null, { path: file, scope: '/r5' }); + opened.push(r4, r5); + r4.set('c1', [CS('http://a', '1')]); + expect(r4.has('c1')).toBe(true); + expect(r5.has('c1')).toBe(false); + expect(r5.get('c1')).toEqual([]); + r4.clear('c1'); + expect(r5.tombstone('c1')).toBeNull(); + }); + + test('last_used is refreshed at most once per touch interval', () => { + const c = make({ touchIntervalMs: 30000 }); + c.set('c1', [CS('http://a', '1')]); + const t0 = Date.now() - 10000; + c.db.prepare('UPDATE sessions SET last_used = ?').run(t0); + c.get('c1'); + expect(lastUsedOf(c, 'c1')).toBe(t0); // 10 s ago: within the interval, not written + c.add('c1', [CS('http://a', '1')]); // same content again: no write either + expect(lastUsedOf(c, 'c1')).toBe(t0); + + const t1 = Date.now() - 31000; + c.db.prepare('UPDATE sessions SET last_used = ?').run(t1); + c.get('c1'); + expect(lastUsedOf(c, 'c1')).toBeGreaterThan(Date.now() - 2000); // past the interval: refreshed + }); + + test('an explicit touch (mode=check) is not throttled', () => { + const c = make({ touchIntervalMs: 30000 }); + c.set('c1', [CS('http://a', '1')]); + c.db.prepare('UPDATE sessions SET last_used = ?').run(Date.now() - 10000); + expect(c.touch('c1')).toBe(true); + expect(c.status('c1').idleMs).toBeLessThan(1000); + }); + + test('a failed last_used refresh is ignored', () => { + const c = make({ touchIntervalMs: 0 }); + c.set('c1', [CS('http://a', '1')]); + const orig = c.q.touch; + c.q.touch = { run: () => { throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); } }; + try { + expect(c.get('c1')).toHaveLength(1); + expect(c.touch('c1')).toBe(true); + } finally { + c.q.touch = orig; + } + }); + + test('a failed write of uploaded resources is retried once, then answered 503 - never dropped silently', () => { + const c = make(); + c.set('c1', [CS('http://a', '1')]); + const orig = c.q.upsertResource; + let calls = 0; + c.q.upsertResource = { run: () => { calls++; throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); } }; + let err = null; + try { + c.add('c1', [CS('http://b', '1')]); + } catch (e) { + err = e; + } finally { + c.q.upsertResource = orig; + } + expect(calls).toBe(2); + expect(err).not.toBeNull(); + expect(err.statusCode).toBe(503); + expect(err.msgId).toBe('CACHE_WRITE_FAILED'); + // the failed transaction changed nothing + expect(c.get('c1').map(r => r.url)).toEqual(['http://a']); + }); + + test('one failure then success: the retry saves the resources', () => { + const c = make(); + c.set('c1', []); + const orig = c.q.upsertResource; + let calls = 0; + c.q.upsertResource = { run: (row) => { if (++calls === 1) { throw Object.assign(new Error('busy'), { code: 'SQLITE_BUSY' }); } return orig.run(row); } }; + try { + c.add('c1', [CS('http://b', '1')]); + } finally { + c.q.upsertResource = orig; + } + expect(c.get('c1').map(r => r.url)).toEqual(['http://b']); + }); + + test('add() does not bring back a cache that has been closed', () => { + const c = make(); + c.set('c1', [CS('http://a', '1')]); + c.clear('c1'); + c.add('c1', [CS('http://b', '1')]); + expect(c.has('c1')).toBe(false); + expect(c.tombstone('c1').reason).toBe('closed'); + }); + + test('the size cap covers every endpoint in the file', () => { + const file = path.join(tmpDir, `cap-${++fileNo}.db`); + const r4 = new SqliteResourceCache(null, { path: file, scope: '/r4', maxBytes: 2500 }); + const r5 = new SqliteResourceCache(null, { path: file, scope: '/r5', maxBytes: 2500 }); + opened.push(r4, r5); + r4.set('a', [{ ...CS('http://a', '1'), description: 'x'.repeat(1000) }]); + r4.db.prepare('UPDATE sessions SET last_used = ?').run(Date.now() - 60000); + r5.set('b', [{ ...CS('http://b', '1'), description: 'x'.repeat(1000) }]); + r5.set('c', [{ ...CS('http://c', '1'), description: 'x'.repeat(1000) }]); + expect(r4.has('a')).toBe(false); + expect(r4.tombstone('a').reason).toBe('evicted'); + expect(r5.has('b')).toBe(true); + }); + + test('a cache survives a restart (closing and reopening the file)', () => { + const file = path.join(tmpDir, `restart-${++fileNo}.db`); + const before = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + before.set('c1', [CS('http://a', '1')], true); + before.set('c2', []); + before.clear('c2'); + before.close(); + + const after = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + opened.push(after); + expect(after.has('c1')).toBe(true); + expect(after.isSealed('c1')).toBe(true); + expect(after.get('c1').map(r => r.url)).toEqual(['http://a']); + expect(after.describeMissing('c2').messageId).toBe('CACHE_ID_CLOSED'); + }); + + test('a file from a different schema version is refused, not misread', () => { + const file = path.join(tmpDir, `schema-${++fileNo}.db`); + const c = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + c.db.prepare("UPDATE meta SET value = '999' WHERE key = 'schema'").run(); + c.close(); + expect(() => new SqliteResourceCache(null, { path: file, scope: '/r4' })).toThrow(/schema version 999/); + }); }); diff --git a/tests/utils/cluster-harness.js b/tests/utils/cluster-harness.js new file mode 100644 index 00000000..db026c29 --- /dev/null +++ b/tests/utils/cluster-harness.js @@ -0,0 +1,125 @@ +// Runs `node cluster.js` on a temporary data directory, for the tests of the cluster options +// (tests/server/cluster-shared-*.test.js): a master, N tx-workers, a fake VSAC, and helpers +// to send requests to the launcher's port or to one process directly. +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const http = require('http'); +const { spawn, execFileSync } = require('child_process'); + +const ROOT = path.join(__dirname, '..', '..'); +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +function request(port, method, urlPath, { headers = {}, body } = {}) { + return new Promise((resolve, reject) => { + const data = body ? Buffer.from(JSON.stringify(body)) : null; + const req = http.request({ + host: '127.0.0.1', port, method, path: urlPath, agent: false, + headers: { Accept: 'application/fhir+json', ...(data ? { 'Content-Type': 'application/fhir+json' } : {}), ...headers } + }, (res) => { + let text = ''; + res.on('data', d => { text += d; }); + res.on('end', () => { + let json = null; + try { + json = JSON.parse(text); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on('error', reject); + req.end(data); + }); +} + +const isUp = (port) => request(port, 'GET', '/health').then(r => r.status === 200, () => false); +const param = (body, name) => ((body && body.parameter) || []).find(p => p.name === name); +const issueCodes = (r) => ((r.json && r.json.issue) || []).flatMap(i => ((i.details && i.details.coding) || []).map(c => c.code)); +const unknownCache = (r) => r.json && r.json.resourceType === 'OperationOutcome' && issueCodes(r).includes('cache-id-unknown'); + +/** The pid of the process listening on a port (Linux: ss). */ +function pidOn(port) { + const out = execFileSync('ss', ['-ltnpH', `sport = :${port}`], { encoding: 'utf8' }); + const m = /pid=(\d+)/.exec(out); + return m ? Number(m[1]) : null; +} + +/** + * Start the launcher. + * @param {Object} opts + * @param {number} opts.front - the launcher's port; the master is front+1, worker n front+1+n + * @param {number} opts.vsac - a port for the fake VSAC + * @param {number} [opts.workers=2] + * @param {Object} [opts.cluster] - more server.cluster settings (the options) + * @param {Object} [opts.tx] - more modules.tx settings + */ +async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {} }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-opt-')); + const master = front + 1; + const workerPorts = Array.from({ length: workers }, (_, i) => master + 1 + i); + // the R4 core package, if a test run has already downloaded it, saves fetching it again + const core = path.join(ROOT, 'data', 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'); + if (fs.existsSync(core)) { + fs.cpSync(core, path.join(dir, 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'), { recursive: true }); + } + fs.writeFileSync(path.join(dir, 'library.yml'), 'base:\n url: https://storage.googleapis.com/tx-fhir-org\n\nsources:\n - internal:vsac=vsac\n'); + fs.writeFileSync(path.join(dir, 'config.json'), JSON.stringify({ + hostName: 'cluster-test', + logging: { console: false }, + server: { port: front, cluster: { workers, internalPort: master, ...cluster } }, + modules: { + tx: { + enabled: true, + instanceCode: 'm0', + librarySource: path.join(dir, 'library.yml'), + endpoints: [{ path: '/r4', fhirVersion: '4.0', context: null }], + closure: { enabled: true }, + vsacCfg: { apiKey: 'test', baseUrl: `http://127.0.0.1:${vsac}/fhir`, refreshIntervalHours: 24 }, + ...tx + } + } + })); + + const vsacServer = http.createServer((req, res) => { + res.setHeader('Content-Type', 'application/fhir+json'); + res.end(JSON.stringify({ resourceType: 'Bundle', type: 'searchset', total: 0, entry: [], link: [] })); + }); + await new Promise(resolve => vsacServer.listen(vsac, resolve)); + + const out = fs.openSync(path.join(dir, 'processes.log'), 'a'); + const launcher = spawn(process.execPath, [path.join(ROOT, 'cluster.js')], { + cwd: ROOT, env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', out, out] + }); + const exited = new Promise(resolve => launcher.once('exit', (code, signal) => resolve({ code, signal }))); + + async function waitUp(ports, ms = 300000) { + const t0 = Date.now(); + for (const port of ports) { + while (!(await isUp(port))) { + if (Date.now() - t0 > ms || launcher.exitCode !== null) { + throw new Error(`port ${port} did not come up; see ${path.join(dir, 'processes.log')}`); + } + await sleep(250); + } + } + // the launcher marks a process ready a moment after it first answers + await sleep(500); + } + await waitUp([master, ...workerPorts, front]); + + async function stop() { + if (launcher.exitCode === null && launcher.signalCode === null) { + launcher.kill('SIGTERM'); + } + const result = await exited; + await new Promise(resolve => vsacServer.close(resolve)); + fs.rmSync(dir, { recursive: true, force: true }); + return result; + } + + return { dir, front, master, workers: workerPorts, launcher, exited, waitUp, stop }; +} + +module.exports = { startCluster, request, isUp, param, issueCodes, unknownCache, pidOn, sleep, ROOT }; diff --git a/tests/utils/resource-cache-child.js b/tests/utils/resource-cache-child.js new file mode 100644 index 00000000..a6e41c9c --- /dev/null +++ b/tests/utils/resource-cache-child.js @@ -0,0 +1,52 @@ +// A separate process using the shared SQLite resource cache, driven over IPC by +// tests/tx/resource-cache-multiprocess.test.js. Each message names a method of the +// cache and its arguments; the reply is the result (or the error). +const { SqliteResourceCache } = require('../../tx/resource-cache-sqlite'); +const { CodeSystem } = require('../../tx/library/codesystem'); + +const cache = new SqliteResourceCache(null, { path: process.argv[2], scope: '/r4', maxBytes: Number(process.argv[3] || 0) }); + +process.on('message', (msg) => { + try { + let result; + switch (msg.op) { + case 'start': { + const id = cache.newCacheId(); + cache.set(id, (msg.resources || []).map(r => new CodeSystem(r, 'R4')), !!msg.sealed); + result = id; + break; + } + case 'add': + cache.add(msg.id, msg.resources.map(r => new CodeSystem(r, 'R4'))); + result = true; + break; + case 'get': + result = cache.get(msg.id).map(r => ({ cls: r.constructor.name, url: r.url, version: r.version, fhirVersion: r.fhirVersion, concepts: r.conceptCount() })); + break; + case 'has': + result = cache.has(msg.id); + break; + case 'end': + cache.clear(msg.id); + result = true; + break; + case 'describe': + result = cache.describeMissing(msg.id); + break; + case 'setLastUsed': + cache.db.prepare('UPDATE sessions SET last_used = ? WHERE cache_id = ?').run(msg.when, msg.id); + result = true; + break; + case 'prune': + cache.prune(msg.maxAge); + result = true; + break; + default: + throw new Error('unknown op ' + msg.op); + } + process.send({ seq: msg.seq, result }); + } catch (e) { + process.send({ seq: msg.seq, error: e.message }); + } +}); +process.send({ ready: process.pid }); diff --git a/translations/Messages.properties b/translations/Messages.properties index 3f10c049..110bac03 100644 --- a/translations/Messages.properties +++ b/translations/Messages.properties @@ -1116,9 +1116,11 @@ Unable_to_resolve_system__value_set_has_no_includes_or_expansion = The System UR UNABLE_TO_RESOLVE_SYSTEM_SYSTEM_IS_INDETERMINATE = The code system {1} referred to from value set {0} has a grammar, and the code might be valid in it Unable_to_resolve_value_Set_ = A definition for the value Set ''{0}'' could not be found CACHE_ID_UNKNOWN = The cache ''{0}'' was never issued by this server, and this server has no record of it ever existing. Caches are created with $cache-control?mode=start, and a cache-id is only valid on the server instance and endpoint that issued it - either it was issued somewhere else, or this server has restarted since it was issued +CACHE_ID_UNKNOWN_SHARED = The cache ''{0}'' was never issued by this server for this endpoint, and this server has no record of it ever existing. Caches are created with $cache-control?mode=start, and a cache-id is only valid on the server and endpoint that issued it. This server keeps its caches on disk, shared by all its processes, where they survive a restart CACHE_ID_CLOSED = The cache ''{0}'' was closed {1} ago by the client, using $cache-control?mode=end (it had been open for {2}). It did not expire and it was not idle: something in the client released this cache while this request was still using it CACHE_ID_EXPIRED = The cache ''{0}'' expired: it went {1} without being used, and this server closes caches that have been idle for more than {2} (this one was dropped {3} ago). Every request that uses a cache resets its idle clock, so a client that keeps using a cache will not see this CACHE_ID_CLEARED = The cache ''{0}'' was dropped {1} ago, when this server cleared all of its caches +CACHE_ID_EVICTED = The cache ''{0}'' was dropped {1} ago to make room: this server holds at most {2} of client resources across all caches, and when it is full it drops the caches that have gone longest without being used (this one had been idle for {3}). Start a new cache and send the resources again CACHE_ID_OTHER_INSTANCE = The cache ''{0}'' was issued by the server instance ''{1}'', but this is instance ''{2}''. A cache only exists on the instance that issued it, so this request reached the wrong server: either the proxy in front of these servers is not routing on the cache-id, or instance ''{1}'' is unavailable CACHE_ID_OTHER_INSTANCE_UNNAMED = The cache ''{0}'' was issued by the server instance ''{1}'', but this server has no instance code, so it cannot have issued it. A cache only exists on the instance that issued it, so this request reached the wrong server: either the proxy in front of these servers is not routing on the cache-id, or instance ''{1}'' is unavailable CLOSURE_NOT_SUPPORTED = This server does not support $closure (it has to be turned on by the administrator) diff --git a/tx/resource-cache-sqlite.js b/tx/resource-cache-sqlite.js new file mode 100644 index 00000000..31627a12 --- /dev/null +++ b/tx/resource-cache-sqlite.js @@ -0,0 +1,812 @@ +// +// SqliteResourceCache - the $cache-control resource cache kept in one SQLite file +// that every server process on the machine shares. +// +// Used when server.cluster.sharedCache is on (cluster.md, "Options"): the master and the +// tx-workers then keep their $cache-control caches here instead of in their own memory, so +// any process can answer any cache-id, and caches survive a process restart. The memory +// store (ResourceCache) is the default. This class is the same interface with the same +// semantics, over a file: +// +// - Nothing about a cache's contents is held in memory between requests. Each use +// reads the rows; there is no per-process copy that could go stale when another +// process changes the cache. +// - A request only pays for the resources it uses. get() reads the list of what the +// cache holds (type, url, version - not the content) and returns a stand-in for +// each CodeSystem / ValueSet / ConceptMap that answers those from the list; the +// first time a request touches anything else on one, its JSON is read and the +// real wrapper built (see lazyResource). Most requests look up a handful of +// resources by url in a cache of tens or hundreds. +// - Every write is one short BEGIN IMMEDIATE transaction; WAL mode lets readers +// carry on while one process writes; busy_timeout makes a writer wait for another +// rather than fail. +// - last_used is refreshed at most every touchIntervalMs (default 30 s) per cache, +// not on every read, so steady use of a cache is reads only. +// - The idle sweep (prune) and the size cap (maxBytes) move the caches they drop to +// tombstones in the same transaction, so whichever process asks about a dead +// cache-id later is told what happened to it. Any process may sweep; a sweep that +// finds nothing to do (because another process just did it) does nothing. +// +// The file must be on a local disk. SQLite's locking is not reliable on network +// filesystems (NFS, SMB, most container volume drivers backed by them), and a +// broken lock there corrupts the file. +// + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const Database = require('better-sqlite3'); +const { ResourceCache, formatDuration } = require('./operation-context'); +const { Issue } = require('./library/operation-outcome'); + +const SCHEMA_VERSION = 2; +const DEFAULT_TOUCH_INTERVAL_MS = 30 * 1000; +const DEFAULT_BUSY_TIMEOUT_MS = 5000; + +/** + * Render a byte count for a human: "512 KB", "3.4 MB", "2 GB". + * @param {number} bytes + * @returns {string} + */ +function formatBytes(bytes) { + if (bytes === null || bytes === undefined || !isFinite(bytes) || bytes < 0) { + return 'an unknown size'; + } + if (bytes < 1024 * 1024) { + return `${Math.max(1, Math.round(bytes / 1024))} KB`; + } + if (bytes < 1024 * 1024 * 1024) { + return `${(bytes / (1024 * 1024)).toFixed(1).replace(/\.0$/, '')} MB`; + } + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1).replace(/\.0$/, '')} GB`; +} + +// One connection per file per process, shared by the endpoints (each has its own scope) +const connections = new Map(); // absolute path -> { db, refs } + +function openDatabase(file, busyTimeoutMs) { + const abs = path.resolve(file); + const existing = connections.get(abs); + if (existing) { + existing.refs++; + return existing.db; + } + fs.mkdirSync(path.dirname(abs), { recursive: true }); + const db = new Database(abs); + // busy_timeout first: switching to WAL and creating the tables need locks that + // another process starting at the same moment may hold + db.pragma(`busy_timeout = ${busyTimeoutMs}`); + db.pragma('journal_mode = WAL'); + // NORMAL is safe with WAL (no corruption); a power cut can lose the last + // transactions, which for a cache of client resources is acceptable + db.pragma('synchronous = NORMAL'); + db.transaction(() => { + db.exec(` + CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE IF NOT EXISTS sessions ( + scope TEXT NOT NULL, + cache_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + last_used INTEGER NOT NULL, + sealed INTEGER NOT NULL DEFAULT 0, + concepts INTEGER NOT NULL DEFAULT 0, + resources INTEGER NOT NULL DEFAULT 0, + bytes INTEGER NOT NULL DEFAULT 0, + version INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (scope, cache_id) + ); + CREATE INDEX IF NOT EXISTS sessions_last_used ON sessions (last_used); + CREATE TABLE IF NOT EXISTS resources ( + id INTEGER PRIMARY KEY, + scope TEXT NOT NULL, + cache_id TEXT NOT NULL, + key TEXT NOT NULL, + resource_type TEXT, + url TEXT, + version TEXT, + fhir_version TEXT, + kind TEXT NOT NULL, + hash TEXT NOT NULL, + json TEXT NOT NULL, + bytes INTEGER NOT NULL, + concepts INTEGER NOT NULL DEFAULT 0, + supplements TEXT, + global_supplement INTEGER, + UNIQUE (scope, cache_id, key) + ); + CREATE TABLE IF NOT EXISTS tombstones ( + scope TEXT NOT NULL, + cache_id TEXT NOT NULL, + reason TEXT NOT NULL, + at INTEGER NOT NULL, + created_at INTEGER, + last_used INTEGER, + idle_ms INTEGER, + max_age_ms INTEGER, + max_bytes INTEGER, + PRIMARY KEY (scope, cache_id) + ); + `); + const row = db.prepare("SELECT value FROM meta WHERE key = 'schema'").get(); + if (!row) { + db.prepare("INSERT INTO meta (key, value) VALUES ('schema', ?)").run(String(SCHEMA_VERSION)); + } else if (row.value !== String(SCHEMA_VERSION)) { + throw new Error(`Resource cache ${abs} has schema version ${row.value}; this server uses ${SCHEMA_VERSION}. Delete the file (it only holds client caches) and restart`); + } + }).immediate(); + connections.set(abs, { db, refs: 1 }); + return db; +} + +function releaseDatabase(file) { + const abs = path.resolve(file); + const c = connections.get(abs); + if (!c) { + return; + } + c.refs--; + if (c.refs <= 0) { + connections.delete(abs); + try { + c.db.close(); + } catch (e) { + // closing is best effort + } + } +} + +// The wrapper classes are loaded on first use: they pull in a good deal of the server +let wrappers = null; +function wrapperClass(resourceType) { + if (!wrappers) { + wrappers = { + CodeSystem: require('./library/codesystem').CodeSystem, + ValueSet: require('./library/valueset'), + ConceptMap: require('./library/conceptmap').ConceptMap + }; + } + return wrappers[resourceType] || null; +} + +/** + * Content hash of a resource's JSON, as the expansion cache key uses it + * (CanonicalResource.computeExpansionKeyHash: the first 16 hex characters of a SHA-256 + * of JSON.stringify(jsonObj)). Stored with the row, and handed to each wrapper built from + * it (setExpansionKeyHash), so a cached resource is never serialised again to be hashed + * and its expansion keys are the ones the memory store would make. + * @param {string} json + * @returns {string} + */ +function contentHash(json) { + return crypto.createHash('sha256').update(json).digest('hex').substring(0, 16); +} + +// Read from the index row, without building the resource: what callers check while +// looking through the cache for the one they want (see TerminologyWorker +// findInAdditionalResources, listVersions, loadSupplements; ExpansionCache.computeKey) +const INDEX_PROPERTIES = new Set(['url', 'version', 'vurl', 'versionedUrl', 'resourceType', 'fhirType', + 'supplementsUrl', 'isCached', 'isGlobalSupplement', 'expansionKeyHash']); + +/** + * A stand-in for a cached CodeSystem / ValueSet / ConceptMap that is only built + * when it is used. It is an instance of the wrapper class (instanceof works), and + * answers url, version, resourceType and the like from the index row. Touching + * anything else - jsonObj, a method, any other property - loads the JSON and builds + * the real wrapper, once; from then on everything goes to that wrapper, and methods + * run on it directly (bound), so a resource that is used costs the same as one + * built eagerly, and one that isn't costs nothing. + * + * @param {Function} Cls - the wrapper class + * @param {Object} row - index row: key, resource_type, url, version, supplements, global_supplement, hash + * @param {Function} load - () => the real wrapper (reads the JSON) + */ +function lazyResource(Cls, row, load) { + let real = null; + const bound = new Map(); + const target = Object.create(Cls.prototype); + const url = row.url === null ? undefined : row.url; + const version = row.version === null ? undefined : row.version; + const index = { + url, + version, + vurl: version ? url + '|' + version : url, + versionedUrl: version ? url + '|' + version : url, + resourceType: row.resource_type, + fhirType: row.resource_type, + supplementsUrl: row.supplements === null ? undefined : row.supplements, + isCached: true, + // the expansion cache key's hash of this resource (see contentHash): known from the row + expansionKeyHash: () => row.hash, + // CodeSystem.isGlobalSupplement(), worked out when it was stored (null: not known) + isGlobalSupplement: row.global_supplement === null || row.global_supplement === undefined + ? undefined : () => row.global_supplement === 1 + }; + const materialise = () => { + if (!real) { + real = load(); + } + return real; + }; + return new Proxy(target, { + get(t, prop) { + if (!real && INDEX_PROPERTIES.has(prop) && index[prop] !== undefined) { + return index[prop]; + } + if (!real && INDEX_PROPERTIES.has(prop) && prop !== 'isGlobalSupplement') { + return undefined; + } + const r = materialise(); + const v = Reflect.get(r, prop, r); + if (typeof v === 'function' && prop !== 'constructor') { + let b = bound.get(prop); + if (!b || b.fn !== v) { + b = { fn: v, bound: v.bind(r) }; + bound.set(prop, b); + } + return b.bound; + } + return v; + }, + set(t, prop, value) { + return Reflect.set(materialise(), prop, value); + }, + has(t, prop) { + return Reflect.has(materialise(), prop); + }, + deleteProperty(t, prop) { + return Reflect.deleteProperty(materialise(), prop); + }, + defineProperty(t, prop, desc) { + return Reflect.defineProperty(materialise(), prop, desc); + }, + ownKeys() { + return Reflect.ownKeys(materialise()); + }, + getOwnPropertyDescriptor(t, prop) { + const d = Reflect.getOwnPropertyDescriptor(materialise(), prop); + if (d) { + d.configurable = true; // a proxy may not report a non-configurable property its target lacks + } + return d; + } + }); +} + +function markCached(resource) { + if (resource && typeof resource === 'object') { + // the same non-enumerable marker the memory store sets (see ResourceCache.add) + Object.defineProperty(resource, 'isCached', { value: true, enumerable: false, configurable: true, writable: true }); + } +} + +class SqliteResourceCache extends ResourceCache { + /** + * @param {Object} stats - server stats (may be null) + * @param {Object} options + * @param {string} options.path - the SQLite file (local disk only) + * @param {string} [options.scope=''] - which endpoint's caches these are; endpoints + * share the file but not caches + * @param {number} [options.maxTombstones=1000] + * @param {number} [options.maxBytes=0] - size cap for the whole file, 0 = none + * @param {number} [options.touchIntervalMs=30000] - least time between last_used refreshes + * @param {number} [options.busyTimeoutMs=5000] + */ + constructor(stats, options = {}) { + super(stats, options.maxTombstones || 1000); + this.maxBytes = options.maxBytes > 0 ? options.maxBytes : 0; + if (!options.path) { + throw new Error('SqliteResourceCache needs a path'); + } + this.path = path.resolve(options.path); + this.scope = options.scope || ''; + this.touchIntervalMs = options.touchIntervalMs >= 0 ? options.touchIntervalMs : DEFAULT_TOUCH_INTERVAL_MS; + this.db = openDatabase(this.path, options.busyTimeoutMs || DEFAULT_BUSY_TIMEOUT_MS); + this.closed = false; + this._prepare(); + } + + _prepare() { + const db = this.db; + this.q = { + session: db.prepare('SELECT * FROM sessions WHERE scope = ? AND cache_id = ?'), + sessionExists: db.prepare('SELECT 1 FROM sessions WHERE scope = ? AND cache_id = ?'), + sealed: db.prepare('SELECT sealed FROM sessions WHERE scope = ? AND cache_id = ?'), + resources: db.prepare('SELECT key, resource_type, fhir_version, kind, hash, json FROM resources WHERE scope = ? AND cache_id = ? ORDER BY id'), + // the index: everything but the content (and the content only of what isn't a wrapper) + index: db.prepare(`SELECT key, resource_type, url, version, supplements, global_supplement, fhir_version, kind, hash, + CASE WHEN kind = 'wrapper' THEN NULL ELSE json END AS json + FROM resources WHERE scope = ? AND cache_id = ? ORDER BY id`), + content: db.prepare('SELECT json, fhir_version, hash FROM resources WHERE scope = ? AND cache_id = ? AND key = ?'), + resourceHash: db.prepare('SELECT hash, concepts FROM resources WHERE scope = ? AND cache_id = ? AND key = ?'), + touch: db.prepare('UPDATE sessions SET last_used = ? WHERE scope = ? AND cache_id = ? AND last_used < ?'), + insertSession: db.prepare('INSERT INTO sessions (scope, cache_id, created_at, last_used, sealed) VALUES (?, ?, ?, ?, ?)'), + upsertResource: db.prepare(` + INSERT INTO resources (scope, cache_id, key, resource_type, url, version, supplements, global_supplement, fhir_version, kind, hash, json, bytes, concepts) + VALUES (@scope, @cacheId, @key, @resourceType, @url, @version, @supplements, @globalSupplement, @fhirVersion, @kind, @hash, @json, @bytes, @concepts) + ON CONFLICT (scope, cache_id, key) DO UPDATE SET + resource_type = excluded.resource_type, url = excluded.url, version = excluded.version, supplements = excluded.supplements, + global_supplement = excluded.global_supplement, + fhir_version = excluded.fhir_version, kind = excluded.kind, hash = excluded.hash, + json = excluded.json, bytes = excluded.bytes, concepts = excluded.concepts`), + totals: db.prepare(` + UPDATE sessions SET + concepts = (SELECT COALESCE(SUM(concepts), 0) FROM resources r WHERE r.scope = sessions.scope AND r.cache_id = sessions.cache_id), + bytes = (SELECT COALESCE(SUM(bytes), 0) FROM resources r WHERE r.scope = sessions.scope AND r.cache_id = sessions.cache_id), + resources = (SELECT COUNT(*) FROM resources r WHERE r.scope = sessions.scope AND r.cache_id = sessions.cache_id), + last_used = ?, version = version + 1 + WHERE scope = ? AND cache_id = ?`), + deleteResources: db.prepare('DELETE FROM resources WHERE scope = ? AND cache_id = ?'), + deleteSession: db.prepare('DELETE FROM sessions WHERE scope = ? AND cache_id = ?'), + tombstone: db.prepare('SELECT * FROM tombstones WHERE scope = ? AND cache_id = ?'), + insertTombstone: db.prepare(` + INSERT OR REPLACE INTO tombstones (scope, cache_id, reason, at, created_at, last_used, idle_ms, max_age_ms, max_bytes) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`), + deleteTombstone: db.prepare('DELETE FROM tombstones WHERE scope = ? AND cache_id = ?'), + trimTombstones: db.prepare(` + DELETE FROM tombstones WHERE scope = ? AND rowid NOT IN + (SELECT rowid FROM tombstones WHERE scope = ? ORDER BY rowid DESC LIMIT ?)`), + tombstoneCount: db.prepare('SELECT COUNT(*) AS n FROM tombstones WHERE scope = ?'), + expired: db.prepare('SELECT * FROM sessions WHERE scope = ? AND last_used < ?'), + allSessions: db.prepare('SELECT * FROM sessions WHERE scope = ?'), + scopeTotals: db.prepare('SELECT COUNT(*) AS n, COALESCE(SUM(concepts), 0) AS concepts, COALESCE(SUM(bytes), 0) AS bytes FROM sessions WHERE scope = ?'), + fileBytes: db.prepare('SELECT COALESCE(SUM(bytes), 0) AS bytes FROM sessions'), + lru: db.prepare('SELECT * FROM sessions WHERE NOT (scope = ? AND cache_id = ?) ORDER BY last_used, created_at LIMIT 16') + }; + } + + // ---------- reads ---------- + + status(cacheId) { + const s = this.q.session.get(this.scope, cacheId); + if (!s) { + return { exists: false }; + } + const now = Date.now(); + return { + exists: true, + sealed: !!s.sealed, + resources: s.resources, + concepts: s.concepts || 0, + idleMs: Math.max(0, now - s.last_used), + ageMs: s.created_at ? now - s.created_at : null, + timeoutMs: this.idleTimeoutMs + }; + } + + has(cacheId) { + return !!this.q.sessionExists.get(this.scope, cacheId); + } + + isSealed(cacheId) { + const s = this.q.sealed.get(this.scope, cacheId); + return s ? !!s.sealed : false; + } + + /** + * The resources held under a cache-id: a fresh set of objects every call, nothing + * shared with other requests. Wrappers are stand-ins that are only built when the + * request uses them (see lazyResource). The session row and the index are read in + * one read transaction, so a concurrent write by another process is seen entirely + * or not at all. + */ + get(cacheId) { + let session = null; + let rows = []; + this.db.transaction(() => { + session = this.q.session.get(this.scope, cacheId); + if (session) { + rows = this.q.index.all(this.scope, cacheId); + } + }).deferred(); + if (!session) { + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': miss (no entry)`); + } + return []; + } + this._maybeTouch(cacheId, session.last_used); + const result = rows.map(r => this._hydrate({ ...r, cacheId })); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': hit, returning ${result.length} resource(s): ${rows.map(r => r.key).join(', ')}`); + } + return result; + } + + /** + * Rebuild one stored resource. Wrappers (CodeSystem / ValueSet / ConceptMap) are + * stored in their internal R5 form, so they are rebuilt as R5 and then given back + * the FHIR version they arrived in. + */ + _hydrate(row) { + if (row.kind === 'wrapper' && row.json === null) { + const Cls = wrapperClass(row.resource_type); + if (Cls) { + const cacheId = row.cacheId; + return lazyResource(Cls, row, () => this._loadWrapper(Cls, cacheId, row)); + } + } + return this._build(row); + } + + /** + * Build the real wrapper behind a stand-in. If the resource is no longer there + * (the client ended the cache, or it expired or was evicted, while this request + * was using it), the request fails the way a request on a dead cache-id does. + */ + _loadWrapper(Cls, cacheId, row) { + const c = this.q.content.get(this.scope, cacheId, row.key); + if (!c) { + const t = this.tombstone(cacheId); + throw new Issue('error', 'not-found', null, 'CACHE_ID_GONE_DURING_REQUEST', + `The cache '${cacheId}' ${t ? 'was ' + t.reason : 'went away'} while this request was using it, so ${row.key.replace(/\|/g, ' ')} is no longer available`, + 'cache-id-unknown', 404); + } + return this._build({ kind: 'wrapper', resource_type: row.resource_type, fhir_version: c.fhir_version, json: c.json, hash: c.hash }); + } + + _build(row) { + const json = JSON.parse(row.json); + let resource = json; + if (row.kind === 'wrapper') { + const Cls = wrapperClass(row.resource_type); + if (Cls) { + resource = new Cls(json, 'R5'); + resource.fhirVersion = row.fhir_version || 'R5'; + if (row.hash && typeof resource.setExpansionKeyHash === 'function') { + // the hash of the JSON this was stored as, which is what the memory store's + // resource (the one the client sent) hashes to + resource.setExpansionKeyHash(row.hash); + } + } + } + markCached(resource); + return resource; + } + + /** + * As for the memory store, except: + * - an id with no record at all is CACHE_ID_UNKNOWN_SHARED, whatever its prefix. The + * memory store's messages offer a restart, or a request routed to the wrong instance, + * as explanations; here every process sees every cache, and caches survive a restart, + * so neither applies. + * - an id dropped by the size cap (maxBytes) is CACHE_ID_EVICTED. + */ + describeMissing(cacheId) { + const t = this.tombstone(cacheId); + if (!t) { + return { messageId: 'CACHE_ID_UNKNOWN_SHARED', params: [cacheId] }; + } + if (t.reason === 'evicted') { + return { + messageId: 'CACHE_ID_EVICTED', + params: [cacheId, formatDuration(Date.now() - t.at), formatBytes(t.maxBytes), formatDuration(t.idleMs)] + }; + } + return super.describeMissing(cacheId); + } + + tombstone(cacheId) { + const t = this.q.tombstone.get(this.scope, cacheId); + if (!t) { + return null; + } + return { + reason: t.reason, + at: t.at, + createdAt: t.created_at, + lastUsed: t.last_used, + idleMs: t.idle_ms, + maxAgeMs: t.max_age_ms, + maxBytes: t.max_bytes + }; + } + + tombstoneCount() { + return this.q.tombstoneCount.get(this.scope).n; + } + + size() { + return this.q.scopeTotals.get(this.scope).n; + } + + conceptCount() { + return this.q.scopeTotals.get(this.scope).concepts; + } + + conceptCountFor(cacheId) { + const s = this.q.session.get(this.scope, cacheId); + return s ? (s.concepts || 0) : 0; + } + + /** + * Bytes held across the whole file (every endpoint): that is what maxBytes caps. + */ + byteCount() { + return this.q.fileBytes.get().bytes; + } + + // High-water marks are this process's view (what it saw after its own writes) + _trackMax() { + try { + const t = this.q.scopeTotals.get(this.scope); + if (t.concepts > this.maxConcepts) { + this.maxConcepts = t.concepts; + } + if (t.n > this.maxSizeValue) { + this.maxSizeValue = t.n; + } + } catch (e) { + // statistics only + } + } + + // ---------- writes ---------- + + /** + * Run a write transaction (BEGIN IMMEDIATE). If it fails it is tried once more; + * if that fails too, the request gets a 503 (the tx router adds Retry-After), so + * the client knows to retry rather than carrying on without its resources. + */ + _write(what, fn) { + const tx = this.db.transaction(fn); + try { + return tx.immediate(); + } catch (first) { + this.log.warn(`resource cache: ${what} failed (${first.code || first.message}); retrying once`); + try { + return tx.immediate(); + } catch (second) { + this.log.error(`resource cache: ${what} failed twice: ${second.message}`); + throw new Issue('error', 'transient', null, 'CACHE_WRITE_FAILED', + `The server could not save to its resource cache just now (${what}: ${second.code || second.message}). Nothing was changed; try the request again`, + null, 503); + } + } + } + + /** + * Refresh last_used, at most once per touchIntervalMs: a cache in steady use + * then costs reads, not a write per request. A failed refresh is ignored - the + * worst it can do is let a cache look idle up to one interval sooner. + */ + _maybeTouch(cacheId, lastUsed) { + const now = Date.now(); + if (now - lastUsed < this.touchIntervalMs) { + return; + } + try { + this.db.transaction(() => { + this.q.touch.run(now, this.scope, cacheId, now - this.touchIntervalMs); + }).immediate(); + } catch (e) { + // ignored, see above + } + } + + touch(cacheId) { + if (!this.has(cacheId)) { + return false; + } + try { + // an explicit keepalive ($cache-control?mode=check) is not throttled + const now = Date.now(); + this.db.transaction(() => { + this.q.touch.run(now, this.scope, cacheId, now); + }).immediate(); + } catch (e) { + // a failed refresh is ignored: the cache is still there + } + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': touched (idle clock reset)`); + } + return true; + } + + /** + * The row to store for a resource. Wrappers are stored as their (R5) jsonObj plus + * the FHIR version they came in as; anything else is stored as itself. + */ + _row(cacheId, resource) { + const isWrapper = resource && typeof resource.jsonObj === 'object' && resource.jsonObj !== null + && !!wrapperClass(resource.resourceType); + const json = JSON.stringify(isWrapper ? resource.jsonObj : resource); + return { + scope: this.scope, + cacheId, + key: this._resourceKey(resource), + resourceType: resource.resourceType || null, + url: resource.url ?? null, + version: resource.version ?? null, + supplements: isWrapper && typeof resource.jsonObj.supplements === 'string' ? resource.jsonObj.supplements : null, + globalSupplement: isWrapper && typeof resource.isGlobalSupplement === 'function' ? this._globalSupplement(resource) : null, + fhirVersion: isWrapper ? (resource.fhirVersion || 'R5') : null, + kind: isWrapper ? 'wrapper' : 'plain', + hash: contentHash(json), + json, + bytes: Buffer.byteLength(json), + concepts: this._conceptsOf(resource) + }; + } + + _globalSupplement(resource) { + try { + return resource.isGlobalSupplement() ? 1 : 0; + } catch (e) { + return null; // not known: the stand-in loads the resource to ask it + } + } + + add(cacheId, resources) { + if (!resources || resources.length === 0) return; + resources = resources.filter(r => !!r); + for (const resource of resources) { + markCached(resource); + } + // the last copy of a key wins, as it would in the memory store + const byKey = new Map(); + for (const resource of resources) { + const row = this._row(cacheId, resource); + byKey.delete(row.key); + byKey.set(row.key, row); + } + const rows = [...byKey.values()]; + + // Most requests re-send resources the cache already has, unchanged. Check + // that with reads first, so they don't take the write lock at all. + const session = this.q.session.get(this.scope, cacheId); + if (session) { + const changed = rows.some(r => { + const existing = this.q.resourceHash.get(this.scope, cacheId, r.key); + return !existing || existing.hash !== r.hash || existing.concepts !== r.concepts; + }); + if (!changed) { + this._maybeTouch(cacheId, session.last_used); + return; + } + } + + this._write('add', () => { + const now = Date.now(); + if (!this.q.sessionExists.get(this.scope, cacheId)) { + if (this.q.tombstone.get(this.scope, cacheId)) { + // Ended or expired (by another process) between the caller's check and + // now: don't bring it back from the dead. + this.log.info(`cache-id '${cacheId}': not adding ${rows.length} resource(s), the cache is gone`); + return; + } + this.q.insertSession.run(this.scope, cacheId, now, now, 0); + } + for (const row of rows) { + this.q.upsertResource.run(row); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': stored ${row.key}`); + } + } + this.q.totals.run(now, this.scope, cacheId); + this._enforceMaxBytesTx(cacheId, now); + }); + this._trackMax(); + } + + set(cacheId, resources, sealed = false) { + resources = (resources || []).filter(r => !!r); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': set (replace all, sealed=${!!sealed}) with ${resources.length} resource(s): ${resources.map(r => this._resourceKey(r)).join(', ')}`); + } + for (const resource of resources) { + markCached(resource); + } + const rows = resources.map(r => this._row(cacheId, r)); + this._write('start', () => { + const now = Date.now(); + const existing = this.q.session.get(this.scope, cacheId); + this.q.deleteResources.run(this.scope, cacheId); + this.q.deleteSession.run(this.scope, cacheId); + this.q.insertSession.run(this.scope, cacheId, existing ? existing.created_at : now, now, sealed ? 1 : 0); + for (const row of rows) { + this.q.upsertResource.run(row); + } + this.q.totals.run(now, this.scope, cacheId); + // alive again, so any record of an earlier death is now wrong + this.q.deleteTombstone.run(this.scope, cacheId); + this._enforceMaxBytesTx(cacheId, now); + }); + this._trackMax(); + } + + /** Move a session to the tombstones (inside a write transaction). */ + _bury(session, reason, now, extra = {}) { + this.q.insertTombstone.run(session.scope, session.cache_id, reason, now, session.created_at, session.last_used, + extra.idleMs ?? null, extra.maxAgeMs ?? null, extra.maxBytes ?? null); + this.q.deleteResources.run(session.scope, session.cache_id); + this.q.deleteSession.run(session.scope, session.cache_id); + this.q.trimTombstones.run(session.scope, session.scope, this.maxTombstones); + } + + /** + * The size cap covers the whole file (all endpoints): drop the least recently + * used caches until the file is under it. Never the cache being written. + */ + _enforceMaxBytesTx(keepId, now) { + if (!this.maxBytes) { + return; + } + let total = this.q.fileBytes.get().bytes; + while (total > this.maxBytes) { + const victims = this.q.lru.all(this.scope, keepId); + if (victims.length === 0) { + return; + } + for (const v of victims) { + if (total <= this.maxBytes) { + break; + } + const idleMs = now - v.last_used; + this._bury(v, 'evicted', now, { idleMs, maxBytes: this.maxBytes }); + total -= v.bytes; + this.log.info(`cache-id '${v.cache_id}': evicted (cache over its ${formatBytes(this.maxBytes)} cap; idle ${formatDuration(idleMs)})`); + } + } + } + + clear(cacheId) { + this._write('end', () => { + const s = this.q.session.get(this.scope, cacheId); + if (s) { + this._bury(s, 'closed', Date.now()); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': closed by client after ${formatDuration(Date.now() - s.created_at)} (held ${s.resources} resource(s))`); + } + } + }); + } + + clearAll() { + this._write('clear all', () => { + const now = Date.now(); + for (const s of this.q.allSessions.all(this.scope)) { + this._bury(s, 'cleared', now); + } + }); + } + + /** + * The idle sweep. One transaction: every session idle for longer than maxAge is + * tombstoned and deleted. Idempotent, so it doesn't matter which process runs + * it, or how many do. + */ + prune(maxAge = 3600000) { + if (this.stats) { + this.stats.task('Client Cache', 'Pruning'); + } + let n = 0; + try { + this.db.transaction(() => { + const now = Date.now(); + for (const s of this.q.expired.all(this.scope, now - maxAge)) { + const idleMs = now - s.last_used; + this._bury(s, 'expired', now, { idleMs, maxAgeMs: maxAge }); + n++; + if (this.logCacheOps) { + this.log.info(`cache-id '${s.cache_id}': expired after ${formatDuration(idleMs)} idle (timeout ${formatDuration(maxAge)})`); + } + } + }).immediate(); + } catch (e) { + // the next sweep will pick them up + this.log.error(`resource cache: idle sweep failed: ${e.message}`); + } + if (this.stats) { + this.stats.taskDone('Client Cache', `Pruned ${n} of ${n + this.size()} entries`); + } + } + + close() { + if (!this.closed) { + this.closed = true; + releaseDatabase(this.path); + } + } +} + +module.exports = { SqliteResourceCache, contentHash, formatBytes }; diff --git a/tx/tx.js b/tx/tx.js index 4ae9a0a8..fb0a5a5c 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -312,7 +312,9 @@ class TXModule { path: endpointPath, fhirVersion, context: context || null, - resourceCache: new ResourceCache(this.stats), + // server.cluster.sharedCache (cluster.md, "Options"; set only with a role): one SQLite + // file that every process shares. Otherwise this process's memory, as ever + resourceCache: this.config.sharedCache ? this.createSharedResourceCache(endpointPath) : new ResourceCache(this.stats), expansionCache: new ExpansionCache(this.stats, expansionCacheSize, expansionCacheMemoryThreshold) }; // Create the provider once for this endpoint @@ -1389,12 +1391,35 @@ class TXModule { /** * Shutdown the module */ + /** + * An endpoint's $cache-control cache in the shared SQLite file (databases/resource-cache.db), + * for server.cluster.sharedCache. Endpoints share the file but not caches. + * @param {string} endpointPath - e.g. /r4 + * @returns {SqliteResourceCache} + */ + createSharedResourceCache(endpointPath) { + const { SqliteResourceCache } = require('./resource-cache-sqlite'); + const folders = require('../library/folder-setup'); + const file = path.join(folders.databasesDir(), 'resource-cache.db'); + const maxBytes = this.config.sharedCacheMaxBytes || 0; + this.log.info(`Resource cache for ${endpointPath}: shared with the other processes, in ${file}` + + (maxBytes ? `, capped at ${maxBytes} bytes` : '')); + return new SqliteResourceCache(this.stats, { path: file, scope: endpointPath, maxBytes }); + } + async shutdown() { this.log.info('Shutting down TX module'); for (const timer of this.timers) { clearInterval(timer); } this.timers = []; + if (this.config && this.config.sharedCache) { + for (const ep of this.endpoints || []) { + if (ep.resourceCache) { + ep.resourceCache.close(); + } + } + } if (this.syndicationStore) { // let an induction in progress finish, so the store isn't left with a half-copied file await this.syndicationStore.whenIdle(); From 3e83e9adc7778f1765dbee41803fd3450fa65207 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 20:11:56 -0600 Subject: [PATCH 11/23] $closure: a shared mode, with version-checked writes instead of a lock across processes ClosureStore gets an option, shared, for a database that other processes answer $closure on too. Nothing sets it yet (the next commit does); without it every path is as before, and the only change to them is that ClosureWorker.add is split into workOutAdd (reads and computes) and addResult (logs and answers) around the same recordAdd call. With shared: - an add is worked out against the table as read (its id and version) and written by recordAddChecked: one BEGIN IMMEDIATE transaction that first checks the table still has that id and version. If not, nothing is written; the worker reads the table again and works the add out again from what is there now. The code lookups and subsumption tests already done are remembered across these tries (for one table id), so a retry costs reads. It waits a little, with jitter, between tries (up to 50 ms); after writeRetries (default 10) it answers the existing 429 CLOSURE_TABLE_BUSY. A busy timeout that runs out counts as a conflict. A table dropped meanwhile is 404 CLOSURE_TABLE_UNKNOWN. An add with nothing new writes nothing and needs no check. - creating checks for an existing table (409) and for maxTables (422) in the same write transaction as the insert (createTableChecked); the worker's checks before it stay, so the errors come in the same order as before - a resync reads only up to the version it reports (entriesPage's upTo), so a write by another process between its pages can't slip later entries in - the stale-table sweep deletes with one conditional statement - the database is opened with a 5 s busy timeout before switching to WAL, and the tables are created (and an old database brought up to date) in an IMMEDIATE transaction - the per-process table lock is kept: within a process calls on one table still queue, so they don't conflict with each other Read and write sets, against main's closure-store.js and workers/closure.js: an add reads the table row (id, version), its parameters (fixed for the table id), its concepts and its entry count, and writes concepts and entries at version+1 and the row's version and counts. The check covers all of it: the parameters change only with the id, and the concepts and counts only with the version. A resync reads the row and entries with version in (v, version]; create and reset read the row and the table count. Tests (tests/tx/closure-shared.test.js): a second ClosureStore on the same file writes between the module's read and its write. Three conflicting writes: the add is retried and ends as one writer doing the same adds in that order (version, entries, whole table), each pair tested for subsumption once across the tries; a concept added by the other process meanwhile is not added twice and its entries stay its own; every try conflicting gives 429 after writeRetries and writes nothing; a table dropped meanwhile is 404; a resync bounded by version; create racing another creator is 409, and the table limit holds. Co-Authored-By: Claude Opus 5.5 --- tests/tx/closure-shared.test.js | 292 ++++++++++++++++++++++++++++++++ tx/closure/closure-store.js | 110 +++++++++++- tx/workers/closure.js | 134 +++++++++++++-- 3 files changed, 522 insertions(+), 14 deletions(-) create mode 100644 tests/tx/closure-shared.test.js diff --git a/tests/tx/closure-shared.test.js b/tests/tx/closure-shared.test.js new file mode 100644 index 00000000..ceea3e1f --- /dev/null +++ b/tests/tx/closure-shared.test.js @@ -0,0 +1,292 @@ +/** + * $closure with the tables shared between processes (server.cluster.sharedClosure). + * + * One TX module answers $closure; a second ClosureStore on the same file plays the other + * processes, writing to a table at chosen moments: after the module has read the table and + * worked out its add, and before it writes. Those are real conflicts on the SQLite file, so + * this checks the version-checked write and the retry: + * - a conflict makes the add work itself out again from the table as it is now, and the + * result is what one writer doing the same adds in that order would have got + * - a concept another process added meanwhile is not added twice + * - when every try conflicts, the call gets 429 CLOSURE_TABLE_BUSY and writes nothing + * - a resync reports only up to the version it read + * - creating a table checks for an existing one and the table limit in the same + * transaction as the insert + * The same against real processes is tests/server/cluster-shared-closure.test.js. + */ + +const express = require('express'); +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const request = require('supertest'); +const TXModule = require('../../tx/tx'); +const { ClosureStore } = require('../../tx/closure/closure-store'); +const ClosureWorker = require('../../tx/workers/closure'); + +const LIBRARY = path.join(__dirname, 'fixtures', 'test-library.yaml'); +const R5 = '/tx/r5'; +const SYS = 'http://example.org/closure-shared/cs'; +const NARROWER = 'source-is-narrower-than-target'; + +// r > a > (a1, a2); r > b; and u1..u6, related to nothing +const CS = { + resourceType: 'CodeSystem', url: SYS, version: '1', status: 'active', content: 'complete', hierarchyMeaning: 'is-a', + concept: [ + { code: 'r', display: 'R', concept: [ + { code: 'a', display: 'A', concept: [{ code: 'a1', display: 'A1' }, { code: 'a2', display: 'A2' }] }, + { code: 'b', display: 'B' } + ] }, + ...[1, 2, 3, 4, 5, 6].map(i => ({ code: `u${i}`, display: `U${i}` })) + ] +}; + +const params = (...parameter) => ({ resourceType: 'Parameters', parameter }); +const name = n => ({ name: 'name', valueString: n }); +const concept = code => ({ name: 'concept', valueCoding: { system: SYS, code } }); +const init = (n) => params(name(n), { name: 'tx-resource', resource: CS }); +const n = (s, t) => `${s}>${t}:${NARROWER}`; + +function entries(cm) { + const out = []; + for (const g of cm.group || []) { + for (const e of g.element || []) { + for (const t of e.target || []) { + out.push(`${e.code}>${t.code}:${t.relationship}`); + } + } + } + return out.sort(); +} + +async function startApp(closure) { + const app = express(); + app.use(express.json()); + const txModule = new TXModule(); + await txModule.initialize({ librarySource: LIBRARY, endpoints: [{ path: R5, fhirVersion: '5.0', context: null }], + closure, sharedClosure: true }, app); + return { app, txModule }; +} + +const post = (app, body) => request(app).post(`${R5}/$closure`) + .set('Content-Type', 'application/json').set('Accept', 'application/fhir+json').send(body); + +describe('$closure on a shared database', () => { + let dir, file, app, txModule, other; + // concepts "another process" writes, with the entries a correct writer would make + const RELATED = { r: [], a: ['r'], b: ['r'], a1: ['a', 'r'], a2: ['a', 'r'] }; + + /** Another process adds a concept to a table (computing its entries as it should). */ + function otherAdds(table, code) { + const t = other.getTable(table); + const ids = new Map(other.getConcepts(t.id).map(c => [c.code, c.id])); + if (ids.has(code)) { + return t.version; + } + const ancestors = (RELATED[code] || []).filter(c => ids.has(c)); + const descendants = Object.entries(RELATED).filter(([c, up]) => up.includes(code) && ids.has(c)).map(([c]) => c); + const v = other.recordAddChecked(t, [{ key: 'k', system: SYS, code, display: code.toUpperCase() }], [ + ...ancestors.map(up => ({ source: 'k', target: ids.get(up), relationship: NARROWER })), + ...descendants.map(d => ({ source: ids.get(d), target: 'k', relationship: NARROWER })) + ]); + expect(v).toBe(t.version + 1); + return v; + } + + /** The next calls of the module's recordAddChecked each run inject() first. */ + function interfere(inject) { + const store = txModule.closureStore; + const real = store.recordAddChecked.bind(store); + let calls = 0; + store.recordAddChecked = (table, concepts, list) => { + calls++; + if (concepts.length > 0) { + inject(calls, table); + } + return real(table, concepts, list); + }; + return { calls: () => calls, restore: () => { store.recordAddChecked = real; } }; + } + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'closure-shared-')); + file = path.join(dir, 'closure.db'); + ({ app, txModule } = await startApp({ enabled: true, database: file, writeRetries: 3, maxTables: 8 })); + other = new ClosureStore({ database: file }, null, { shared: true }); + other.open(); + }, 240000); + + afterAll(async () => { + other.close(); + await txModule.shutdown(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('the module\'s store is the shared one', () => { + expect(txModule.closureStore.shared).toBe(true); + expect(txModule.closureStore.db.pragma('busy_timeout', { simple: true })).toBe(5000); + }); + + test('conflicts: the add is worked out again from the table as it is now', async () => { + const T = 'conflict-1'; + expect((await post(app, init(T))).status).toBe(200); + expect((await post(app, params(name(T), concept('a')))).body.version).toBe('1'); + + // three tries are each beaten by another process's add of an unrelated concept + const orig = ClosureWorker.prototype.memoSubsumes; + let hits = 0; + let misses = 0; + const subsumes = jest.spyOn(ClosureWorker.prototype, 'memoSubsumes').mockImplementation(async function (memo, nn, o) { + if (memo.subsumes.has(nn.system + '|' + nn.code + '|' + o.code)) { + hits++; + } else { + misses++; + } + return orig.call(this, memo, nn, o); + }); + const i = interfere((call) => { + if (call <= 3) { + otherAdds(T, `u${call}`); + } + }); + let res; + try { + res = await post(app, params(name(T), concept('a1'), concept('r'))); + } finally { + i.restore(); + } + expect(res.status).toBe(200); + expect(i.calls()).toBe(4); + // a (1), u1 u2 u3 (2-4), then this add (5) + expect(res.body.version).toBe('5'); + expect(entries(res.body)).toEqual([n('a', 'r'), n('a1', 'a'), n('a1', 'r')].sort()); + // each pair was tested once, and remembered across the four tries: a1 and r each + // against a, u1, u2, u3, and r against a1 + subsumes.mockRestore(); + expect(misses).toBe(9); + expect(hits).toBeGreaterThan(0); + + // the table is what one writer doing the same adds in the same order gets + const R = 'conflict-1-reference'; + await post(app, init(R)); + for (const code of ['a', 'u1', 'u2', 'u3']) { + await post(app, params(name(R), concept(code))); + } + const ref = await post(app, params(name(R), concept('a1'), concept('r'))); + expect(ref.body.version).toBe('5'); + expect(entries(ref.body)).toEqual(entries(res.body)); + const whole = async (t) => entries((await post(app, params(name(t), { name: 'version', valueString: '0' }))).body); + expect(await whole(T)).toEqual(await whole(R)); + }); + + test('a concept another process added meanwhile is not added twice, and its entries are its own', async () => { + const T = 'conflict-2'; + await post(app, init(T)); + await post(app, params(name(T), concept('r'))); + const i = interfere((call) => { + if (call === 1) { + otherAdds(T, 'b'); + } + }); + let res; + try { + res = await post(app, params(name(T), concept('b'), concept('a2'))); + } finally { + i.restore(); + } + expect(res.status).toBe(200); + expect(i.calls()).toBe(2); + expect(res.body.version).toBe('3'); + // b>r was the other process's add (version 2); this call added only a2 + expect(entries(res.body)).toEqual([n('a2', 'r')]); + const db = txModule.closureStore; + const t = db.getTable(T); + expect(db.getConcepts(t.id).map(c => c.code).sort()).toEqual(['a2', 'b', 'r']); + expect(entries((await post(app, params(name(T), { name: 'version', valueString: '0' }))).body)) + .toEqual([n('a2', 'r'), n('b', 'r')]); + // and each version's changes are the ones its add reported + expect(entries((await post(app, params(name(T), { name: 'version', valueString: '2' }))).body)).toEqual([n('a2', 'r')]); + }); + + test('when every try conflicts: 429 CLOSURE_TABLE_BUSY after writeRetries, and nothing of the call is written', async () => { + const T = 'conflict-3'; + await post(app, init(T)); + const i = interfere((call) => otherAdds(T, `u${call}`)); + let res; + try { + res = await post(app, params(name(T), concept('a'), concept('a1'))); + } finally { + i.restore(); + } + expect(res.status).toBe(429); + const issue = res.body.issue[0]; + expect(issue.code).toBe('too-costly'); + expect(issue.details.text).toMatch(/busy/i); + expect(i.calls()).toBe(4); // the first try and writeRetries (3) more + const db = txModule.closureStore; + const t = db.getTable(T); + expect(t.version).toBe(4); + expect(db.getConcepts(t.id).map(c => c.code).sort()).toEqual(['u1', 'u2', 'u3', 'u4']); + }); + + test('a table dropped by another process before the write: unknown, 404', async () => { + const T = 'conflict-4'; + await post(app, init(T)); + const i = interfere(() => other.dropTable(T)); + let res; + try { + res = await post(app, params(name(T), concept('a'))); + } finally { + i.restore(); + } + expect(res.status).toBe(404); + expect(txModule.closureStore.getTable(T)).toBeNull(); + }); + + test('a resync reports only up to the version it read', () => { + const T = 'bounded'; + const made = other.createTableChecked(T, [], { reset: false, maxTables: Infinity }); + const t = made.table; + other.recordAdd(t, [{ key: 'r', system: SYS, code: 'r' }], []); + const t1 = other.getTable(T); + const rId = other.getConcepts(t1.id)[0].id; + other.recordAdd(t1, [{ key: 'a', system: SYS, code: 'a' }], [{ source: 'a', target: rId, relationship: NARROWER }]); + const t2 = other.getTable(T); + other.recordAdd(t2, [{ key: 'b', system: SYS, code: 'b' }], [{ source: 'b', target: rId, relationship: NARROWER }]); + // read at version 2, while version 3 has since been written + expect(other.entriesPage(t.id, 0, 0, 100, 2).map(e => e.source_code)).toEqual(['a']); + expect(other.entriesPage(t.id, 0, 0, 100, 3).map(e => e.source_code)).toEqual(['a', 'b']); + expect(other.entriesPage(t.id, 0, 0, 100).map(e => e.source_code)).toEqual(['a', 'b']); + }); + + test('creating: an existing table and the table limit are checked in the write transaction', async () => { + // another process creates the table between this one's read and its write + const store = txModule.closureStore; + const realGet = store.getTable.bind(store); + store.getTable = (nm) => { + const t = realGet(nm); + if (nm === 'raced' && !t) { + other.createTableChecked('raced', [], { reset: false, maxTables: Infinity }); + } + return t; + }; + let res; + try { + res = await post(app, init('raced')); + } finally { + store.getTable = realGet; + } + expect(res.status).toBe(409); + + // the limit (8): fill up to it from "another process", then one more from the module + const count = store.tableCount(); + for (let k = count; k < 8; k++) { + expect(other.createTableChecked(`fill-${k}`, [], { reset: false, maxTables: 8 }).table).toBeDefined(); + } + expect(other.createTableChecked('one-too-many', [], { reset: false, maxTables: 8 })).toEqual({ refused: 'limit' }); + expect((await post(app, init('one-too-many'))).status).toBe(422); + // a reset of an existing table is not a new table + expect(other.createTableChecked('fill-7', [], { reset: true, maxTables: 8 }).table).toBeDefined(); + expect(store.tableCount()).toBe(8); + }); +}); diff --git a/tx/closure/closure-store.js b/tx/closure/closure-store.js index d4f76dfe..460a9efb 100644 --- a/tx/closure/closure-store.js +++ b/tx/closure/closure-store.js @@ -26,6 +26,18 @@ // here reads or writes an unbounded amount in one go: reads that can be large are paged // (entriesPage), and the size of what one call can write is capped by the worker. // +// Shared (server.cluster.sharedClosure, cluster.md "Options"): several processes open this +// file and any of them may answer $closure. The per-table lock is still taken, but it only +// orders the calls within one process, so correctness across processes comes from the +// writes themselves. A call computes against the table as it read it (its id and version), +// and writes in one BEGIN IMMEDIATE transaction that first checks the table is still at +// that id and version (recordAddChecked); if it isn't, nothing is written and the worker +// computes again from what is there now. Creating a table checks for an existing one and +// for the table limit in the same transaction as the insert (createTableChecked). A resync +// reads only up to the version it reports (entriesPage's upTo), so a write by another +// process between its pages can't put entries of a later version in it. Without shared, +// none of these paths is taken. +// const path = require('path'); const fs = require('fs'); @@ -40,10 +52,13 @@ class ClosureStore { /** * @param {Object} config - modules.tx.closure * @param {Object} [log] + * @param {Object} [options] + * @param {boolean} [options.shared] - other processes use the file too (see above) */ - constructor(config, log) { + constructor(config, log, options = {}) { this.config = config || {}; this.log = log || console; + this.shared = !!options.shared; this.db = null; this.path = ClosureStore.resolvePath(this.config); this.locks = new Map(); // table name -> { tail: Promise, count: holder + waiters } @@ -69,8 +84,22 @@ class ClosureStore { fs.mkdirSync(path.dirname(this.path), {recursive: true}); } this.db = new Database(this.path); + if (this.shared) { + // first: switching to WAL and creating the tables need locks another process may hold + this.db.pragma('busy_timeout = 5000'); + } this.db.pragma('journal_mode = WAL'); this.db.pragma('foreign_keys = ON'); + if (this.shared) { + // one process at a time creates the tables and brings an old database up to date + this.db.transaction(() => this.#createSchema()).immediate(); + } else { + this.#createSchema(); + } + this.#prepare(); + } + + #createSchema() { this.db.exec(` CREATE TABLE IF NOT EXISTS closure_table ( id INTEGER PRIMARY KEY, @@ -118,6 +147,9 @@ class ClosureStore { entry_count = (SELECT count(*) FROM closure_entry e WHERE e.table_id = closure_table.id); `); } + } + + #prepare() { this.stmt = { getTable: this.db.prepare('SELECT id, name, version, created, last_used FROM closure_table WHERE name = ?'), countTables: this.db.prepare('SELECT count(*) AS n FROM closure_table'), @@ -156,6 +188,22 @@ class ClosureStore { WHERE e.table_id = ? AND e.added_version > ?`), staleTables: this.db.prepare('SELECT id, name FROM closure_table WHERE last_used < ?') }; + if (this.shared) { + Object.assign(this.stmt, { + entriesPageUpTo: this.db.prepare(` + SELECT e.rowid AS rowid, + s.system AS source_system, s.code AS source_code, s.display AS source_display, + t.system AS target_system, t.code AS target_code, t.display AS target_display, + e.relationship + FROM closure_entry e + JOIN closure_concept s ON s.id = e.source_id + JOIN closure_concept t ON t.id = e.target_id + WHERE e.table_id = ? AND e.added_version > ? AND e.added_version <= ? AND e.rowid > ? + ORDER BY e.rowid + LIMIT ?`), + deleteStale: this.db.prepare('DELETE FROM closure_table WHERE id = ? AND last_used < ?') + }); + } } close() { @@ -260,7 +308,11 @@ class ClosureStore { * written. Page through with the last row's rowid as afterRowid (start with 0). * @returns {Array<{rowid, source_system, source_code, source_display, target_system, target_code, target_display, relationship}>} */ - entriesPage(tableId, version, afterRowid, limit) { + entriesPage(tableId, version, afterRowid, limit, upTo) { + if (upTo !== undefined) { + // shared: only up to the version the caller read (see the header) + return this.stmt.entriesPageUpTo.all(tableId, version, upTo, afterRowid, limit); + } return this.stmt.entriesPage.all(tableId, version, afterRowid, limit); } @@ -353,6 +405,53 @@ class ClosureStore { return version; } + /** + * Shared: create (or, with reset, replace) a table, checking in the same write + * transaction that there isn't one already (unless reset) and that the table limit + * isn't reached - so two processes creating at once can't both succeed, or go over it. + * @param {string} name + * @param {Array} params - as for createTable + * @param {Object} opts + * @param {boolean} opts.reset - replace a table of that name + * @param {number} opts.maxTables - Infinity for none + * @returns {{table: Object}|{refused: 'exists'|'limit'}} + */ + createTableChecked(name, params, { reset, maxTables }) { + return this.db.transaction(() => { + const existing = this.stmt.getTable.get(name); + if (existing && !reset) { + return { refused: 'exists' }; + } + if (!existing && this.stmt.countTables.get().n >= maxTables) { + return { refused: 'limit' }; + } + return { table: this.createTable(name, params) }; + }).immediate(); + } + + /** + * Shared: recordAdd, if the table is still the one the caller computed against - the + * same id (not reset or dropped since) and the same version (nothing added since). The + * check and the write are one BEGIN IMMEDIATE transaction, so no other process can + * write in between. + * @returns {number|null} the version after the add, or null if the table has changed + * (nothing was written) + */ + recordAddChecked(table, concepts, entries) { + if (concepts.length === 0) { + // nothing to write: what the caller read is still a true answer (it was the table + // when it read it, and nothing it was asked to add is missing from the table now) + return this.recordAdd(table, concepts, entries); + } + return this.db.transaction(() => { + const now = this.stmt.getTable.get(table.name); + if (!now || now.id !== table.id || now.version !== table.version) { + return null; + } + return this.recordAdd(table, concepts, entries); + }).immediate(); + } + /** * Every entry added after the given version. * @returns {Array<{source_system, source_code, source_display, target_system, target_code, target_display, relationship}>} @@ -395,6 +494,13 @@ class ClosureStore { const dropped = []; for (const t of stale) { await this.withLock(t.name, async () => { + if (this.shared) { + // another process may use the table between a read and a delete: one statement + if (this.db && this.stmt.deleteStale.run(t.id, cutoff).changes > 0) { + dropped.push(t.name); + } + return; + } const now = this.db ? this.stmt.getTable.get(t.name) : null; if (now && now.id === t.id && now.last_used < cutoff) { this.stmt.deleteTable.run(t.id); diff --git a/tx/workers/closure.js b/tx/workers/closure.js index b2ef1fb7..c351e2ba 100644 --- a/tx/workers/closure.js +++ b/tx/workers/closure.js @@ -35,6 +35,13 @@ // name can read, add to, or reset that table. On a public server, require UUID names // (requireUuidNames) so that a name is effectively a secret held by the client that made it. // +// Shared (server.cluster.sharedClosure): other processes answer $closure on the same tables. +// The per-table lock still orders this process's calls, but a write may find the table has +// moved on since it was read: then nothing is written, and the call is worked out again +// from the table as it is now (see add, and the header of closure-store.js). After +// writeRetries such goes, the call gets the same 429 as a busy queue. The code lookups and +// subsumption tests done before are remembered for the retries, so a retry costs reads. +// // Cost: everything a client can make this do is bounded (see LIMITS) - the tables, the // concepts and entries in a table, what one call can add, what a table's configuration // can store, and how many calls can queue for one table and for how long. The work itself @@ -68,6 +75,11 @@ const LIMITS = { const RESYNC_PAGE = 2000; // rows read from the store at a time +// shared: how many times an add is worked out again when another process changed the table +// first, and the most it waits (with jitter) before a retry +const WRITE_RETRIES = 10; +const RETRY_WAIT_MS = 50; + class ClosureWorker extends TerminologyWorker { /** * @param {OperationContext} opContext - Operation context @@ -219,7 +231,19 @@ class ClosureWorker extends TerminologyWorker { if (!table && this.store.tableCount() >= max) { throw this.closureIssue('too-costly', 'CLOSURE_TOO_MANY_TABLES', [String(max)], 'too-costly', 422); } - table = this.store.createTable(name, config); + if (this.store.shared) { + // the checks above again, in the same transaction as the write + const made = this.store.createTableChecked(name, config, { reset, maxTables: max }); + if (made.refused === 'exists') { + throw this.closureIssue('duplicate', 'CLOSURE_TABLE_EXISTS', [name], 'business-rule', 409); + } + if (made.refused === 'limit') { + throw this.closureIssue('too-costly', 'CLOSURE_TOO_MANY_TABLES', [String(max)], 'too-costly', 422); + } + table = made.table; + } else { + table = this.store.createTable(name, config); + } this.log.info(`closure table '${name}' ${reset ? 'reset' : 'created'}`); if (concepts.length === 0) { return await this.buildConceptMap(name, table.version, [], fhirVersion); @@ -297,7 +321,9 @@ class ClosureWorker extends TerminologyWorker { let after = 0; for (;;) { await this.checkAndYield('closure-resync'); - const page = this.store.entriesPage(table.id, v, after, RESYNC_PAGE); + const page = this.store.shared + ? this.store.entriesPage(table.id, v, after, RESYNC_PAGE, table.version) + : this.store.entriesPage(table.id, v, after, RESYNC_PAGE); for (const r of page) { entries.push({ source: { system: r.source_system, code: r.source_code, display: r.source_display }, @@ -320,6 +346,66 @@ class ClosureWorker extends TerminologyWorker { * system or code, or going over a limit, fails the whole call. */ async add(table, name, conceptParams, fhirVersion) { + if (this.store.shared) { + return await this.addShared(table, name, conceptParams, fhirVersion); + } + const { fresh, entries, out } = await this.workOutAdd(table, name, conceptParams, null); + const version = this.store.recordAdd(table, fresh, entries); + return await this.addResult(name, version, fresh, entries, out, fhirVersion); + } + + /** + * Shared: add, with the write checked against the table as it was read + * (recordAddChecked). If another process changed the table first, nothing was written: + * read the table again and work the add out again, up to writeRetries times, waiting a + * little (with jitter) in between; then 429, as for a busy queue. A table that is gone + * by then is unknown (404), as it would be to a call that came after the one that + * dropped it. + */ + async addShared(table, name, conceptParams, fhirVersion) { + const configured = this.closureConfig.writeRetries; + const retries = Number.isInteger(configured) && configured >= 0 ? configured : WRITE_RETRIES; + let memo = null; + for (let attempt = 0; ; attempt++) { + if (!memo || memo.tableId !== table.id) { + // what's remembered holds for one table's configuration: a reset starts again + memo = { tableId: table.id, providers: new Map(), codes: new Map(), subsumes: new Map() }; + } + const { fresh, entries, out } = await this.workOutAdd(table, name, conceptParams, memo); + let version = null; + try { + version = this.store.recordAddChecked(table, fresh, entries); + } catch (error) { + // the busy timeout ran out: another process held the write lock throughout + if (!String(error.code || '').startsWith('SQLITE_BUSY')) { + throw error; + } + } + if (version !== null) { + return await this.addResult(name, version, fresh, entries, out, fhirVersion); + } + if (attempt >= retries) { + this.log.info(`closure table '${name}': changed by another process on each of ${attempt + 1} tries; answering busy`); + throw this.closureIssue('too-costly', 'CLOSURE_TABLE_BUSY', [name], 'too-costly', 429); + } + this.log.info(`closure table '${name}': changed by another process since it was read; working the add out again (retry ${attempt + 1} of ${retries})`); + const wait = Math.random() * Math.min(RETRY_WAIT_MS, 5 * 2 ** attempt); + await this.opContext.waitFor(new Promise(resolve => setTimeout(resolve, wait)), 'closure-retry'); + table = this.store.getTable(name); + if (!table) { + throw this.closureIssue('not-found', 'CLOSURE_TABLE_UNKNOWN', [name], 'not-found', 404); + } + } + } + + /** + * Work out what adding the concepts to the table, as read, would write: the new + * concepts and the entries they make. Writes nothing. + * @param {Object} memo - shared only (else null): code lookups and subsumption tests + * already done for this table, kept across retries + * @returns {{fresh: Array, entries: Array, out: Array}} + */ + async workOutAdd(table, name, conceptParams, memo) { // replay the table's configuration: its code systems, and its version rules const stored = { resourceType: 'Parameters', parameter: this.store.getParams(table.id) }; this.setupAdditionalResources(stored); @@ -331,7 +417,7 @@ class ClosureWorker extends TerminologyWorker { for (const c of existing) { known.add(c.system + '|' + c.code); } - const providers = new Map(); + const providers = memo ? memo.providers : new Map(); const fresh = []; for (const p of conceptParams) { @@ -346,20 +432,31 @@ class ClosureWorker extends TerminologyWorker { this.seeSourceProvider(cs, coding.system); providers.set(coding.system, cs); } - const loc = await cs.locate(coding.code); - if (!loc || !loc.context) { - throw this.unknownCodeIssue(cs, coding.code, loc ? loc.message : null, txp); + const memoKey = memo ? coding.system + '|' + (coding.version || '') + '|' + coding.code : null; + let found = memo ? memo.codes.get(memoKey) : null; + if (!found) { + const loc = await cs.locate(coding.code); + if (!loc || !loc.context) { + throw this.unknownCodeIssue(cs, coding.code, loc ? loc.message : null, txp); + } + // the code system's own spelling of the code, so a case-insensitive code system + // doesn't end up with the same concept in the table twice + found = { context: loc.context, code: (await cs.code(loc.context)) || coding.code }; + if (memo) { + memo.codes.set(memoKey, found); + } } - // the code system's own spelling of the code, so a case-insensitive code system - // doesn't end up with the same concept in the table twice - const code = (await cs.code(loc.context)) || coding.code; + const code = found.code; const key = coding.system + '|' + code; if (known.has(key)) { continue; } known.add(key); + if (found.display === undefined) { + found.display = await cs.display(found.context); + } // ref: how the store refers to it until it has an id (see ClosureStore.recordAdd) - fresh.push({ key, ref: key, system: coding.system, code, display: await cs.display(loc.context), cs }); + fresh.push({ key, ref: key, system: coding.system, code, display: found.display, cs }); } const maxConcepts = this.limit('maxConceptsPerTable'); @@ -381,7 +478,7 @@ class ClosureWorker extends TerminologyWorker { continue; } await this.checkAndYield('closure-subsumes'); - const outcome = await n.cs.subsumesTest(n.code, o.code); + const outcome = memo ? await this.memoSubsumes(memo, n, o) : await n.cs.subsumesTest(n.code, o.code); let source, target, relationship; if (outcome === 'subsumes') { // the new concept is the broader one source = o; target = n; relationship = NARROWER; @@ -401,7 +498,20 @@ class ClosureWorker extends TerminologyWorker { before.push(n); } - const version = this.store.recordAdd(table, fresh, entries); + return { fresh, entries, out }; + } + + /** n.cs.subsumesTest(n.code, o.code), remembered in the memo (shared retries). */ + async memoSubsumes(memo, n, o) { + const key = n.system + '|' + n.code + '|' + o.code; + if (!memo.subsumes.has(key)) { + memo.subsumes.set(key, await n.cs.subsumesTest(n.code, o.code)); + } + return memo.subsumes.get(key); + } + + /** The response to an add, once it is written. */ + async addResult(name, version, fresh, entries, out, fhirVersion) { if (fresh.length > 0) { this.log.info(`closure table '${name}': added ${fresh.length} concept(s), ${entries.length} entr${entries.length === 1 ? 'y' : 'ies'}, now version ${version}`); } From 6c66ebc93245d25c8014602d6b4385e72bbd703c Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 20:11:56 -0600 Subject: [PATCH 12/23] server.cluster.sharedClosure: every tx process answers $closure Off by default. With it on (and a role): - library/role.js sets modules.tx.sharedClosure (a boolean, like the other options) - tx/tx.js opens the closure database in every process, the workers included, in shared mode (previous commit). Only the master runs the stale-table sweep. Without the option a worker still opens nothing and answers 501 - cluster.js sends $closure to the least-loaded tx process (whatever its cache-id); /syndication and /info stay the master's. nginx.md shows the location change Test (tests/server/cluster-shared-closure.test.js), a master and two workers on one closure.db: 3 processes x 4 clients add concepts at once, one or two per request, to one shared table and to four tables shared in threes, over a 40-concept tree. Each table then has exactly the concepts asked for, once, and exactly the entries a single writer gets for them; versions run 1..n with no gaps; every write was answered with its own version and exactly the entries stored at it; and "changes since v", asked of each process in turn, is the union of the answers after v, for every v. 12 concurrent creates of one table make one (11 x 409). Creates racing for the room left under maxTables (room + 15 of them) make exactly room, and 15 get 422. With the version check removed from recordAddChecked, the same test fails (500s from duplicate concepts). Co-Authored-By: Claude Opus 5.5 --- cluster.js | 15 +- library/role.js | 5 +- nginx.md | 19 ++ tests/library/role.test.js | 13 ++ tests/server/cluster-launcher.test.js | 17 +- tests/server/cluster-shared-closure.test.js | 236 ++++++++++++++++++++ tx/tx.js | 14 +- 7 files changed, 306 insertions(+), 13 deletions(-) create mode 100644 tests/server/cluster-shared-closure.test.js diff --git a/cluster.js b/cluster.js index daa26910..1b0db738 100644 --- a/cluster.js +++ b/cluster.js @@ -76,7 +76,7 @@ function loadSettings(argv, env, config) { procs, txPaths: (tx.endpoints || []).map(e => String(e.path).replace(/\/+$/, '')), drainMs: Number(cluster.drainSeconds || 30) * 1000, - routing: { cacheAffinity: !sharedCache || cluster.cacheAffinity === true } + routing: { cacheAffinity: !sharedCache || cluster.cacheAffinity === true, sharedClosure: cluster.sharedClosure === true } }; } @@ -86,7 +86,8 @@ function loadSettings(argv, env, config) { * @param {Object} headers - the request headers (lower case names) * @param {string[]} txPaths - the tx endpoint paths, e.g. ['/r4', '/r5'] * @param {Object} [routing] - from loadSettings: cacheAffinity (default true) sends a - * request with a cache-id to the process that issued it + * request with a cache-id to the process that issued it; sharedClosure sends $closure + * to any tx process rather than the master * @returns {{to: 'master'}|{to: 'owner', code: string}|{to: 'tx'}} */ function classify(url, headers, txPaths, routing = {}) { @@ -101,9 +102,13 @@ function classify(url, headers, txPaths, routing = {}) { return { to: 'master' }; } const rest = p.substring(ep.length); - // the closure tables and the syndication store are the master's; the source info pages - // carry the VSAC resync form, which writes the VSAC database - if (/^\/\$closure\/?$/.test(rest) || /^\/syndication(\/|$)/.test(rest) || rest.startsWith('/info/')) { + // the closure tables (unless shared) and the syndication store are the master's; the + // source info pages carry the VSAC resync form, which writes the VSAC database + if (/^\/\$closure\/?$/.test(rest)) { + // closure never uses a cache-id, so the least loaded, whatever the header says + return routing.sharedClosure ? { to: 'tx' } : { to: 'master' }; + } + if (/^\/syndication(\/|$)/.test(rest) || rest.startsWith('/info/')) { return { to: 'master' }; } if (routing.cacheAffinity === false) { diff --git a/library/role.js b/library/role.js index c4342a33..701661c4 100644 --- a/library/role.js +++ b/library/role.js @@ -80,7 +80,7 @@ function applyRole(config, role) { // the options in server.cluster (cluster.md, "Options"). Every process reads the same // config.json, so they all agree; without a role these are never looked at const options = (config.server && config.server.cluster) || {}; - for (const name of ['sharedCache', 'cacheAffinity']) { + for (const name of ['sharedCache', 'cacheAffinity', 'sharedClosure']) { if (options[name] !== undefined && typeof options[name] !== 'boolean') { throw new Error(`server.cluster.${name} must be true or false`); } @@ -93,6 +93,9 @@ function applyRole(config, role) { } tx.sharedCacheMaxBytes = Math.round(mb * 1024 * 1024); } + if (options.sharedClosure === true) { + tx.sharedClosure = true; + } if (tx.vsacCfg) { tx.vsacCfg = { ...tx.vsacCfg, cluster: role.role === 'master' ? 'leader' : 'follower' }; } diff --git a/nginx.md b/nginx.md index 9583c147..1a9b2400 100644 --- a/nginx.md +++ b/nginx.md @@ -484,3 +484,22 @@ Keeping the `map` still works (the launcher's `cacheAffinity: true`). It keeps e one process, whose expansion cache (which is per process) already holds the expansions it asked for; without it, the load spreads evenly, and the first time each process expands a value set for a client it does the work again. + +### With shared closure tables + +With `server.cluster.sharedClosure: true` ([cluster.md](cluster.md#options)), every process +answers `$closure` on the one closure database, so `$closure` can leave the master's location +and go to the pool (it never uses a cache-id): + +```nginx + location ~ ^/(r4|r5)/(syndication(/.*)?|info/.*)$ { + include fhirsmith-proxy.conf; + proxy_set_header X-Cache-Id ""; + proxy_pass http://fs_master; + } + + location ~ ^/(r4|r5)/\$closure$ { + include fhirsmith-proxy.conf; + proxy_pass http://fs_tx_pool; + } +``` diff --git a/tests/library/role.test.js b/tests/library/role.test.js index 253b8b6e..05ebc476 100644 --- a/tests/library/role.test.js +++ b/tests/library/role.test.js @@ -94,6 +94,19 @@ describe('applyRole', () => { expect(() => applyRole(badCap, { role: 'master', instanceCode: null })).toThrow(/megabytes/); }); + test('server.cluster.sharedClosure reaches tx (with a role)', () => { + const off = applyRole(baseConfig(), { role: 'tx-worker', instanceCode: 'tx1' }); + expect(off.modules.tx.sharedClosure).toBeUndefined(); + const on = baseConfig(); + on.server.cluster = { sharedClosure: true }; + applyRole(on, { role: 'tx-worker', instanceCode: 'tx1' }); + expect(on.modules.tx.sharedClosure).toBe(true); + expect(on.modules.tx.sharedCache).toBeUndefined(); + const bad = baseConfig(); + bad.server.cluster = { sharedClosure: 1 }; + expect(() => applyRole(bad, { role: 'master', instanceCode: null })).toThrow(/true or false/); + }); + test('a worker needs its own instance code, and tx must be enabled', () => { expect(() => applyRole(baseConfig(), { role: 'tx-worker', instanceCode: null })).toThrow(/its own instance code/); const noCode = baseConfig(); diff --git a/tests/server/cluster-launcher.test.js b/tests/server/cluster-launcher.test.js index 2b62e9f6..14b2effe 100644 --- a/tests/server/cluster-launcher.test.js +++ b/tests/server/cluster-launcher.test.js @@ -50,19 +50,32 @@ describe('routing rules', () => { test('with the shared cache, a cache-id goes to any tx process, unless cacheAffinity is on', () => { const shared = loadSettings([], {}, { server: { cluster: { sharedCache: true } }, modules: { tx: { enabled: true } } }).routing; - expect(shared).toEqual({ cacheAffinity: false }); + expect(shared).toEqual({ cacheAffinity: false, sharedClosure: false }); expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, shared)).toEqual({ to: 'tx' }); const affinity = loadSettings([], {}, { server: { cluster: { sharedCache: true, cacheAffinity: true } }, modules: { tx: { enabled: true } } }).routing; expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, affinity)).toEqual({ to: 'owner', code: 'tx2' }); // without the shared cache, a cache-id always goes to its issuer: there is nowhere else it is const plain = loadSettings([], {}, { server: { cluster: { cacheAffinity: false } }, modules: { tx: { enabled: true } } }).routing; - expect(plain).toEqual({ cacheAffinity: true }); + expect(plain).toEqual({ cacheAffinity: true, sharedClosure: false }); expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, plain)).toEqual({ to: 'owner', code: 'tx2' }); // the master-only paths stay the master's expect(classify('/r4/$closure', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, shared)).toEqual({ to: 'master' }); expect(classify('/', {}, TX, shared)).toEqual({ to: 'master' }); }); + test('with the shared closure tables, $closure goes to any tx process; the rest of the master\'s paths stay its', () => { + const routing = loadSettings([], {}, { server: { cluster: { sharedClosure: true } }, modules: { tx: { enabled: true } } }).routing; + expect(routing).toEqual({ cacheAffinity: true, sharedClosure: true }); + for (const url of ['/r4/$closure', '/r5/%24closure', '/r4/$closure?name=x']) { + expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX, routing)).toEqual({ to: 'tx' }); + } + expect(classify('/r4/syndication', {}, TX, routing)).toEqual({ to: 'master' }); + expect(classify('/r4/info/vsac', {}, TX, routing)).toEqual({ to: 'master' }); + expect(classify('/xig', {}, TX, routing)).toEqual({ to: 'master' }); + // and cache-ids are routed as without it + expect(classify('/r4/ValueSet/$expand', { 'x-cache-id': 'tx1.abc' }, TX, routing)).toEqual({ to: 'owner', code: 'tx1' }); + }); + test('$closure, in any FHIR version, encoded or not, goes to the master, whatever the cache-id', () => { for (const url of ['/r4/$closure', '/r5/$closure', '/r4/%24closure', '/r4/$closure?name=x']) { expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX)).toEqual({ to: 'master' }); diff --git a/tests/server/cluster-shared-closure.test.js b/tests/server/cluster-shared-closure.test.js new file mode 100644 index 00000000..d8728422 --- /dev/null +++ b/tests/server/cluster-shared-closure.test.js @@ -0,0 +1,236 @@ +/** + * server.cluster.sharedClosure with real processes: a master and two tx-workers behind + * cluster.js, all answering $closure on databases/closure.db. + * + * Contention: 3 processes x 4 clients add concepts at once - all of them to one table, and + * in fours to four more tables - each client a request at a time, one or two concepts per + * request, with overlaps between clients. Then each table is checked against a reference + * built by a single writer, and against the responses the clients were given: + * - the same concepts (none lost, none twice) and the same entries (none lost, none twice) + * - versions go up by one per write, with no gaps; every write was answered with its own + * version and exactly the entries stored at that version + * - "changes since v", asked of any process, is the union of the answers after v, for + * every v + * Also: 12 concurrent creates of one table make exactly one; creates racing for the table + * limit stop exactly at it; and $closure through the launcher reaches the workers. + */ +const path = require('path'); +const Database = require('better-sqlite3'); +const { startCluster, request, sleep } = require('../utils/cluster-harness'); +const fs = require('fs'); + +const FRONT = 20070; +const VSAC = 20079; +const MAX_TABLES = 40; +const NARROWER = 'source-is-narrower-than-target'; +const SYS = 'http://example.org/closure-contention/cs'; + +// a ternary tree of 40 concepts: c0 is the root, the parent of ci is c((i-1)/3) +const N = 40; +const parentOf = (i) => (i === 0 ? null : Math.floor((i - 1) / 3)); +function tree(i) { + const kids = []; + for (let j = 3 * i + 1; j <= 3 * i + 3 && j < N; j++) { + kids.push(tree(j)); + } + return { code: `c${i}`, display: `C${i}`, ...(kids.length ? { concept: kids } : {}) }; +} +const CS = { resourceType: 'CodeSystem', url: SYS, version: '1', status: 'active', content: 'complete', hierarchyMeaning: 'is-a', concept: [tree(0)] }; + +const params = (...parameter) => ({ resourceType: 'Parameters', parameter }); +const name = n => ({ name: 'name', valueString: n }); +const concept = code => ({ name: 'concept', valueCoding: { system: SYS, code } }); +const closure = (port, body) => request(port, 'POST', '/r4/$closure', { body }); + +function entries(cm) { + const out = []; + for (const g of (cm && cm.group) || []) { + for (const e of g.element || []) { + for (const t of e.target || []) { + out.push(`${e.code}>${t.code}`); + } + } + } + return out.sort(); +} + +// a deterministic shuffle, so a failure can be run again the same way +function rng(seed) { + let s = seed; + return () => { + s = (s * 1103515245 + 12345) % 2147483648; + return s / 2147483648; + }; +} +function shuffle(list, rand) { + const a = [...list]; + for (let i = a.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)); + [a[i], a[j]] = [a[j], a[i]]; + } + return a; +} + +describe('cluster.js with server.cluster.sharedClosure', () => { + let cluster; + let ports; + let db; + const stats = { busy: 0, retriesLogged: 0 }; + + beforeAll(async () => { + cluster = await startCluster({ + front: FRONT, vsac: VSAC, cluster: { sharedClosure: true }, + tx: { closure: { enabled: true, maxTables: MAX_TABLES } } + }); + ports = [cluster.master, ...cluster.workers]; + db = new Database(path.join(cluster.dir, 'databases', 'closure.db'), { readonly: true }); + }, 330000); + + afterAll(async () => { + if (db) { + db.close(); + } + if (cluster) { + const logs = path.join(cluster.dir, 'logs'); + const log = fs.readdirSync(logs).map(f => fs.readFileSync(path.join(logs, f), 'utf8')).join('\n'); + stats.retriesLogged = (log.match(/working the add out again/g) || []).length; + // for the record: how much contention the run produced + console.log(`shared closure: ${stats.retriesLogged} retries after a conflict, ${stats.busy} busy answers (retried by the client)`); + await cluster.stop(); + } + }); + + // one client: add its concepts a request at a time; a busy answer (429) is retried, as a + // client would. Returns what each successful add was answered with. + async function client(port, table, codes, rand) { + const answers = []; + let i = 0; + while (i < codes.length) { + const take = rand() < 0.5 ? 1 : 2; + const batch = codes.slice(i, i + take); + const r = await closure(port, params(name(table), ...batch.map(concept))); + if (r.status === 429) { + stats.busy++; + await sleep(5 + rand() * 20); + continue; + } + expect(r.status).toBe(200); + answers.push({ version: Number(r.json.version), entries: entries(r.json), codes: batch }); + i += take; + } + return answers; + } + + // the table as stored: concepts, entries by the version they were added in, the version + function stored(table) { + const t = db.prepare('SELECT id, version FROM closure_table WHERE name = ?').get(table); + const concepts = db.prepare('SELECT code, added_version FROM closure_concept WHERE table_id = ?').all(t.id); + const rows = db.prepare(`SELECT s.code AS s, t.code AS t, e.relationship AS rel, e.added_version AS v + FROM closure_entry e JOIN closure_concept s ON s.id = e.source_id JOIN closure_concept t ON t.id = e.target_id + WHERE e.table_id = ?`).all(t.id); + return { version: t.version, concepts, rows }; + } + + async function reference(table, codes) { + expect((await closure(cluster.master, params(name(table), { name: 'tx-resource', resource: CS }))).status).toBe(200); + const r = await closure(cluster.master, params(name(table), ...[...codes].sort().map(concept))); + expect(r.status).toBe(200); + return entries(r.json); + } + + async function check(table, answers, codes, ref) { + const s = stored(table); + // concepts: every one asked for, once + const asked = new Set(codes); + expect(s.concepts.map(c => c.code).sort()).toEqual([...asked].sort()); + // entries: the reference's, once each, all narrower-than in this code system + expect(s.rows.map(r => `${r.s}>${r.t}`).sort()).toEqual(ref); + expect(s.rows.every(r => r.rel === NARROWER)).toBe(true); + // versions: one per write, 1..version, no gaps + const versions = [...new Set(s.concepts.map(c => c.added_version))].sort((a, b) => a - b); + expect(versions).toEqual(Array.from({ length: s.version }, (_, i) => i + 1)); + // every write was answered with its version and exactly the entries stored at it + const byVersion = new Map(); + for (const r of s.rows) { + byVersion.set(r.v, [...(byVersion.get(r.v) || []), `${r.s}>${r.t}`]); + } + const writes = answers.filter(a => a.entries.length > 0); + expect(new Set(writes.map(a => a.version)).size).toBe(writes.length); + for (const a of writes) { + expect(a.entries).toEqual((byVersion.get(a.version) || []).sort()); + } + for (const v of byVersion.keys()) { + expect(writes.some(a => a.version === v)).toBe(true); + } + // no answer claims a version the table hasn't reached + expect(Math.max(...answers.map(a => a.version))).toBeLessThanOrEqual(s.version); + // changes since v, from any process, for every v + for (let v = 0; v <= s.version; v++) { + const r = await closure(ports[v % ports.length], params(name(table), { name: 'version', valueString: String(v) })); + expect(r.status).toBe(200); + expect(Number(r.json.version)).toBe(s.version); + const fromAnswers = writes.filter(a => a.version > v).flatMap(a => a.entries).sort(); + expect(entries(r.json)).toEqual(fromAnswers); + } + } + + test('3 processes x 4 clients adding to one table and to four others: the tables a single writer makes', async () => { + const all = Array.from({ length: N }, (_, i) => `c${i}`); + const tables = ['S', 'D0', 'D1', 'D2', 'D3']; + for (const t of tables) { + expect((await closure(ports[tables.indexOf(t) % 3], params(name(t), { name: 'tx-resource', resource: CS }))).status).toBe(200); + } + const runs = []; + const plan = []; + for (let p = 0; p < 3; p++) { + for (let k = 0; k < 4; k++) { + const c = p * 4 + k; + const rand = rng(1000 + c); + // the shared table: every concept is someone's, and each client takes 6 others too + const mine = all.filter((_, i) => i % 12 === c); + const S = shuffle([...mine, ...shuffle(all, rand).slice(0, 6)].filter((x, i, a) => a.indexOf(x) === i), rand); + // its four-way table: the three clients with this k share it + const D = shuffle(all.filter((_, i) => i % 4 === k || i % 7 === p), rand); + plan.push({ table: 'S', codes: S }, { table: `D${k}`, codes: D }); + runs.push(client(ports[p], 'S', S, rand).then(answers => ({ table: 'S', answers }))); + runs.push(client(ports[p], `D${k}`, D, rng(2000 + c)).then(answers => ({ table: `D${k}`, answers }))); + } + } + const results = await Promise.all(runs); + + for (const t of tables) { + const codes = [...new Set(plan.filter(x => x.table === t).flatMap(x => x.codes))]; + const answers = results.filter(r => r.table === t).flatMap(r => r.answers); + const ref = await reference(`${t}-reference`, codes); + expect(ref.length).toBeGreaterThan(0); + await check(t, answers, codes, ref); + } + }, 300000); + + test('12 concurrent creates of one table, on three processes: exactly one makes it', async () => { + const results = await Promise.all(Array.from({ length: 12 }, (_, i) => + closure(ports[i % 3], params(name('race'), { name: 'tx-resource', resource: CS })))); + const statuses = results.map(r => r.status).sort(); + expect(statuses.filter(s => s === 200)).toHaveLength(1); + expect(statuses.filter(s => s === 409)).toHaveLength(11); + expect(db.prepare("SELECT count(*) AS n FROM closure_table WHERE name = 'race'").get().n).toBe(1); + }, 60000); + + test('creates racing for the table limit stop exactly at it', async () => { + const before = db.prepare('SELECT count(*) AS n FROM closure_table').get().n; + const room = MAX_TABLES - before; + expect(room).toBeGreaterThan(5); + const tries = room + 15; + const results = await Promise.all(Array.from({ length: tries }, (_, i) => + closure(ports[i % 3], params(name(`limit-${i}`))))); + expect(results.filter(r => r.status === 200)).toHaveLength(room); + expect(results.filter(r => r.status === 422)).toHaveLength(15); + expect(db.prepare('SELECT count(*) AS n FROM closure_table').get().n).toBe(MAX_TABLES); + }, 60000); + + test('$closure through the launcher is answered', async () => { + // (that the launcher sends it to the least-loaded tx process is in cluster-launcher.test.js) + const rs = await Promise.all(Array.from({ length: 30 }, () => closure(FRONT, params(name('S'), { name: 'version', valueString: '0' })))); + expect(rs.every(r => r.status === 200)).toBe(true); + }, 60000); +}); diff --git a/tx/tx.js b/tx/tx.js index fb0a5a5c..b9255663 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -203,14 +203,18 @@ class TXModule { // would be worse. this.closureStore = null; // a tx-worker (cluster.md) opens neither the closure tables nor the syndication store: - // the master owns them, and $closure and the feed are routed to it + // the master owns them, and $closure and the feed are routed to it - unless + // server.cluster.sharedClosure is on (set only with a role), when every process + // answers $closure on the one database (see closure-store.js) const worker = config.clusterRole === 'tx-worker'; - if (config.closure && config.closure.enabled && !worker) { - this.closureStore = new ClosureStore(config.closure, this.log); + const sharedClosure = config.sharedClosure === true; + if (config.closure && config.closure.enabled && (!worker || sharedClosure)) { + this.closureStore = new ClosureStore(config.closure, this.log, { shared: sharedClosure }); this.closureStore.open(); - this.log.info(`$closure enabled: tables in ${this.closureStore.path}`); + this.log.info(`$closure enabled: tables in ${this.closureStore.path}${sharedClosure ? ', shared with the other processes' : ''}`); const days = config.closure.retentionDays; - if (days) { + // with the tables shared, the master alone drops the stale ones + if (days && !worker) { const prune = async () => { try { const dropped = await this.closureStore.pruneUnused(days); From 7c50d4cef866047bbec88b9e60e36596463a84cd Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 20:30:58 -0600 Subject: [PATCH 13/23] tests: count closure retries in logs/server.log only (the dated log is a copy) Co-Authored-By: Claude Opus 5.5 --- tests/server/cluster-shared-closure.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/server/cluster-shared-closure.test.js b/tests/server/cluster-shared-closure.test.js index d8728422..4ac00a80 100644 --- a/tests/server/cluster-shared-closure.test.js +++ b/tests/server/cluster-shared-closure.test.js @@ -91,8 +91,8 @@ describe('cluster.js with server.cluster.sharedClosure', () => { db.close(); } if (cluster) { - const logs = path.join(cluster.dir, 'logs'); - const log = fs.readdirSync(logs).map(f => fs.readFileSync(path.join(logs, f), 'utf8')).join('\n'); + // (the processes log to logs/server.log; the dated file is a copy) + const log = fs.readFileSync(path.join(cluster.dir, 'logs', 'server.log'), 'utf8'); stats.retriesLogged = (log.match(/working the add out again/g) || []).length; // for the record: how much contention the run produced console.log(`shared closure: ${stats.retriesLogged} retries after a conflict, ${stats.busy} busy answers (retried by the client)`); From f81442a4344ce51d9f701df99360262cd9b6cd18 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 20:30:58 -0600 Subject: [PATCH 14/23] docs: cluster.md "Options": sharedCache, cacheAffinity, sharedClosure What each option does, what it costs, and how the routing changes, with a table of the four combinations. "What runs where", "Routing" and "Limits" point to it. Co-Authored-By: Claude Opus 5.5 --- cluster.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/cluster.md b/cluster.md index d928a8be..d597827f 100644 --- a/cluster.md +++ b/cluster.md @@ -18,6 +18,7 @@ or with nginx in front of separately started processes ([below](#with-nginx)). * [Routing](#routing) * [The launcher (cluster.js)](#the-launcher-clusterjs) * [With nginx](#with-nginx) +* [Options](#options) * [Memory](#memory) * [Limits of this arrangement](#limits-of-this-arrangement) @@ -44,12 +45,12 @@ which parts of the configuration each one acts on. |---|---|---| | tx module (`/r4`, `/r5` and the other endpoints) | yes (may also serve tx requests) | yes | | Every other module (packages, xig, registry, publisher, token, shl, ips, ...) | yes | no | -| `$closure` (the closure tables) | yes | no: routed to the master | +| `$closure` (the closure tables) | yes | no: routed to the master (yes, with [`sharedClosure`](#options)) | | Syndication feed (the store) | yes | no: routed to the master | | VSAC sync, and the resync form on `/info/vsac` | yes | no: reads what the master wrote | | Package crawler, XIG download, registry crawl, daily report, npm audit | yes | no (those modules don't run) | | Statistics database, uptime history | yes | no: in memory only | -| `$cache-control` caches | its own, in memory | its own, in memory | +| `$cache-control` caches | its own, in memory (shared, with [`sharedCache`](#options)) | its own, in memory (shared, with [`sharedCache`](#options)) | | Expansion cache, CodeSystem list refresh | its own | its own | | Log file | `logs/server-.log` | the same file (lines interleave) | @@ -81,6 +82,9 @@ Whatever the front door, the rules are the same: 4. Any other tx request goes to the tx process with the fewest requests outstanding: the workers, and the master too unless it is set not to serve tx. +The [options](#options) change rules 1 and 3: with `sharedClosure`, `$closure` goes by rule 4; +with `sharedCache`, so does a request with a cache-id (unless `cacheAffinity` is on). + ## The launcher (cluster.js) `node cluster.js` reads `config.json` from the data directory and: @@ -131,6 +135,79 @@ nginx can be the front door instead: start the processes yourself (master first, it answers `/health`), and use the configuration in [nginx.md](nginx.md#a-master-and-tx-workers-on-one-machine), which applies the same rules. +## Options + +Two options in `server.cluster` change what the processes share. Both are off by default, and +both only apply with a role: `node server.js` with no role ignores them. + +```json +"server": { + "cluster": { "workers": 4, "sharedCache": true, "sharedClosure": true } +} +``` + +| Setting | Default | Meaning | +|---|---|---| +| `server.cluster.sharedCache` | false | The `$cache-control` caches go in one SQLite file, `databases/resource-cache.db`, that every process reads and writes. | +| `server.cluster.cacheAffinity` | false | With `sharedCache`: still send a request with a cache-id to the process that issued it, while it is up. | +| `server.cluster.sharedCacheMaxMB` | 0 (none) | With `sharedCache`: the most the file holds. Past it, the caches least recently used are dropped whole. | +| `server.cluster.sharedClosure` | false | Every tx process answers `$closure`, on the one closure database. | +| `modules.tx.closure.writeRetries` | 10 | With `sharedClosure`: how many times an add is worked out again when another process changed the table first. | + +### Shared cache + +Without it, a cache lives in the memory of the process that issued it, and the routing has to +send every request that uses it there. A restarted process has lost its caches, and its +clients get `cache-id-unknown` and start new ones. The load a client brings stays on one +process. + +With it, any process can answer any cache-id, and caches survive a restart of any process (or +of all of them). The launcher sends a request with a cache-id to the least-loaded tx process, +like any other. The cost is a read of the file on each use of a cache: a request reads the +list of what the cache holds, and only loads the resources it uses. Writes are short +transactions in WAL mode, so readers don't wait for them. `last_used` is written at most every +30 seconds per cache. The idle sweep (`modules.tx.cacheTimeout`) runs in every process, and +running it twice does no harm. An unknown cache-id gets a message that doesn't offer a restart +or a routing mistake as the reason, since neither can be the cause. + +`cacheAffinity: true` keeps the prefix routing on top. The expansion cache is still per +process, so a client that stays on one process finds its expansions there; without affinity, +each process does the first expansion again. The file must be on a local disk. + +### Shared closure + +Without it, the master answers every `$closure` request, and its per-table lock keeps the +calls on one table in order. + +With it, any tx process answers `$closure`, against the same `databases/closure.db`, and the +launcher sends `$closure` to the least-loaded tx process. Within a process the per-table lock +still queues the calls on a table. Across processes the writes are **checked, not locked**. A +call works out its add against the table as it read it (its version *n*). It writes in one +`BEGIN IMMEDIATE` transaction that first checks the table is still at version *n*, and was +not reset meanwhile. If it isn't, nothing is written: the call reads the table again and works +the add out again, with the code lookups and subsumption tests it already did remembered. It +waits a few milliseconds, with jitter, between tries. After `writeRetries` tries it answers +`429` (`CLOSURE_TABLE_BUSY`), as a full queue does. Creating a table checks for one of the +same name, and for the table limit, in the same transaction as the insert. A resync reports +only entries up to the version it read. The master alone drops stale tables +(`retentionDays`). + +This spreads the subsumption work of `$closure` over the processes. When many processes add +to the same table at once, some adds are worked out more than once. + +### What changes, by combination + +| `sharedCache` | `sharedClosure` | A request with a cache-id goes to | `$closure` goes to | A restarted process | +|---|---|---|---|---| +| off | off | the process that issued it (if it is down: any, which answers `cache-id-unknown`) | the master | has lost its caches | +| on | off | the least-loaded tx process (the issuer, with `cacheAffinity`) | the master | still has every cache | +| off | on | the process that issued it | the least-loaded tx process | has lost its caches | +| on | on | the least-loaded tx process (the issuer, with `cacheAffinity`) | the least-loaded tx process | still has every cache | + +The syndication feed, the `/info` pages and everything outside the tx endpoints go to the +master in every combination. For nginx, see +[nginx.md](nginx.md#with-the-shared-cache). + ## Memory With a role, the memory guard measures the whole cgroup, since the kernel's limit is on all the @@ -155,9 +232,11 @@ use roughly N+1 times the memory of one server. Size the limit for that. ## Limits of this arrangement -* **Caches are per process.** A `$cache-control` cache lives in the process that issued it; a - restarted process loses its caches, and clients start new ones, as after any restart. +* **Caches are per process** (unless [`sharedCache`](#options) is on). A `$cache-control` + cache lives in the process that issued it; a restarted process loses its caches, and + clients start new ones, as after any restart. * **Statistics are per process.** `/` and `/dashboard` show the master. The workers' request counts are not in the statistics database. -* **Everything the master runs is a single point.** While the master restarts, `$closure`, the - non-tx modules and the dashboard answer 503 through the launcher; tx requests carry on. +* **Everything the master runs is a single point.** While the master restarts, `$closure` + (unless [`sharedClosure`](#options) is on), the non-tx modules and the dashboard answer 503 + through the launcher; tx requests carry on. From 120c529f22602539102811814205d8a78de6365d Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 21:03:22 -0600 Subject: [PATCH 15/23] tx-worker: read the package caches the master built; never build or migrate one On a new package cache the master built each .valuesets-2.db with no content_hash on its rows (a package's value sets are written without one), so the next process to open the file ran the content_hash backfill: with the workers started together after the master, both backfilled hl7.terminology at once and crashed (SQLITE_BUSY / READONLY / IOERR), and a worker whose completeness check failed under the lock deleted and rebuilt the master's file. - ValueSetDatabase.upsertValueSet stores the content's hash when it isn't given one (the hash the backfill computes), so a cache is final when it is built - with --role=tx-worker the package value set and concept map caches are opened read-only. A worker never creates, rebuilds, discards or migrates one: one that is missing, incomplete or still needs a migration is waited for (up to 5 minutes, FHIRSMITH_CACHE_WAIT_MS), then the worker stops with an error saying the master builds it - cluster.js starts the workers one after another, each once the one before is ready - cluster.md: the package caches are the master's; starting the master first only spares the wait, it is not what keeps the workers from writing Tests: a cluster on a new data directory with the package's caches removed starts every process once, with no backfill and neither cache file written after the master is ready; read-only ValueSetDatabase never writes or creates the file. Co-Authored-By: Claude Opus 5.5 --- cluster.js | 17 ++-- cluster.md | 15 +++- tests/server/cluster-package-caches.test.js | 68 ++++++++++++++++ tests/tx/package-cache-readonly.test.js | 89 +++++++++++++++++++++ tests/utils/cluster-harness.js | 17 +++- tx/cm/cm-package.js | 16 +++- tx/library.js | 15 +++- tx/library/wait-for-cache.js | 43 ++++++++++ tx/tx.js | 2 + tx/vs/vs-database.js | 81 ++++++++++++++++++- tx/vs/vs-package.js | 19 ++++- 11 files changed, 361 insertions(+), 21 deletions(-) create mode 100644 tests/server/cluster-package-caches.test.js create mode 100644 tests/tx/package-cache-readonly.test.js create mode 100644 tx/library/wait-for-cache.js diff --git a/cluster.js b/cluster.js index 1b0db738..44163a84 100644 --- a/cluster.js +++ b/cluster.js @@ -12,7 +12,8 @@ * don't run nginx in front: * * - starts the master (node server.js --role=master), waits until it answers /health, then - * starts the workers (--role=tx-worker), each with its own instance code and port + * starts the workers (--role=tx-worker) one after another, each once the one before is + * ready, each with its own instance code and port * - routes: a request with X-Cache-Id to the process that issued it (by the id's prefix); * $closure, the syndication feed, the source info pages and everything that isn't a tx * endpoint to the master; any other tx request to the tx process with the fewest @@ -311,11 +312,17 @@ function createLauncher(settings) { server.listen(settings.port, resolve); }); log(`listening on port ${settings.port}; starting ${procs.length - 1} worker(s)`); - for (const p of procs) { - if (p !== master) { - void startProcess(p); + // one after another, each once the one before is ready (or has exited, and is being + // restarted): a worker only reads the caches the master built, but starting them together + // would put every worker's load (and its memory peak) at the same moment + // (in the background: start() returns once the master is ready and the port is open) + void (async () => { + for (const p of procs) { + if (p !== master && !stopping) { + await startProcess(p); + } } - } + })(); } /** diff --git a/cluster.md b/cluster.md index d597827f..5d19bf06 100644 --- a/cluster.md +++ b/cluster.md @@ -63,6 +63,15 @@ by key, at most once a second) and reloads the list in the background if it has sync reaches every worker about a second after it finishes. The database is in WAL mode with a 5 second busy timeout, so a worker's reload doesn't make the master's writes fail. +**The package caches are the master's.** The value set and concept map caches in each +package folder (`.valuesets-2.db`, `.conceptmaps-2.db`) are built, rebuilt and migrated only +by the master; it has finished every one before it answers `/health`. A worker opens them +read-only, and never creates, rebuilds or migrates one: if one is missing, incomplete or still +needs a migration (a worker started on its own, before the master), the worker waits for the +master to finish it, and after 5 minutes (`FHIRSMITH_CACHE_WAIT_MS`) stops with an error that +says so. Starting the master first is what makes the wait unnecessary, not what keeps the +workers from writing. + **Nothing else is shared.** The workers run no module but tx, so they never read what the package crawler, the XIG download or the registry crawl write. Each process keeps the caches of a `folder:` or `file:` library source in its own folder @@ -90,9 +99,9 @@ with `sharedCache`, so does a request with a cache-id (unless `cacheAffinity` is `node cluster.js` reads `config.json` from the data directory and: * starts the master and waits until it answers `/health`. On a new data directory the master - builds the package caches and creates the VSAC database; processes starting together would - corrupt them, so the workers start only when the master is ready -* listens on the configured port, and starts the workers + builds the package caches and creates the VSAC database +* listens on the configured port, and starts the workers one after another, each once the one + before answers `/health` * proxies each request by the rules above, over keep-alive connections to each process. It adds the client address to `X-Forwarded-For`, so the servers' logs and rate limits see the real client (its own address is a loopback one, which `server.trustProxy` trusts by default) diff --git a/tests/server/cluster-package-caches.test.js b/tests/server/cluster-package-caches.test.js new file mode 100644 index 00000000..f587878e --- /dev/null +++ b/tests/server/cluster-package-caches.test.js @@ -0,0 +1,68 @@ +/** + * cluster.js on a new data directory whose package has no value set or concept map cache yet + * (.valuesets-2.db and .conceptmaps-2.db removed): the master builds them before it is ready, + * the workers only read them. Every process starts once - none exits and is restarted - and + * neither cache file is written after the master is ready. + */ +const fs = require('fs'); +const path = require('path'); +const { startCluster, request } = require('../utils/cluster-harness'); + +const FRONT = 20150; +const VSAC = 20159; +const CORE = 'hl7.fhir.r4.core#4.0.1'; +const CACHES = ['.valuesets-2.db', '.conceptmaps-2.db']; + +describe('the package caches on a new data directory', () => { + let cluster; + const state = (file) => { + const st = fs.statSync(file); + return { mtimeMs: st.mtimeMs, size: st.size }; + }; + + afterAll(async () => { + if (cluster) { + await cluster.stop(); + } + }); + + test('the master builds them, the workers read them and start once', async () => { + let folder = null; + cluster = await startCluster({ + front: FRONT, vsac: VSAC, workers: 2, + sources: [`npm:${CORE}`], + prepare: (dir) => { + folder = path.join(dir, 'terminology-cache', CORE); + for (const name of CACHES) { + fs.rmSync(path.join(folder, name), { force: true }); + } + } + }); + const masterUp = cluster.upAt.get(cluster.master); + const files = CACHES.map(name => path.join(folder, name)); + const before = files.map(state); + for (const s of before) { + // written by the master, before it was ready + expect(s.mtimeMs).toBeLessThanOrEqual(masterUp); + } + + // the workers serve the package's value sets + for (const port of cluster.workers) { + const r = await request(port, 'GET', '/r4/ValueSet?url=http://hl7.org/fhir/ValueSet/administrative-gender'); + expect(r.status).toBe(200); + expect((r.json.entry || []).length).toBeGreaterThan(0); + } + + expect(files.map(state)).toEqual(before); + // no journal or WAL left by a writer + for (const file of files) { + for (const suffix of ['-journal', '-wal']) { + expect(fs.existsSync(file + suffix)).toBe(false); + } + } + const log = fs.readFileSync(path.join(cluster.dir, 'processes.log'), 'utf8'); + expect(log).not.toMatch(/exited \(/); + expect(log).not.toMatch(/Backfilling content_hash/); + expect(log).not.toMatch(/SQLITE_(BUSY|READONLY|IOERR)/); + }, 600000); +}); diff --git a/tests/tx/package-cache-readonly.test.js b/tests/tx/package-cache-readonly.test.js new file mode 100644 index 00000000..91e8e00a --- /dev/null +++ b/tests/tx/package-cache-readonly.test.js @@ -0,0 +1,89 @@ +/** + * A tx-worker's package caches (cluster.md): ValueSetDatabase with readOnly never writes the + * file - one that still needs a migration (here, rows without content_hash) fails to open, and + * opens once its builder has migrated it - and a row is never written without its hash. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const sqlite3 = require('sqlite3'); +const { ValueSetDatabase } = require('../../tx/vs/vs-database'); +const { waitForCache } = require('../../tx/library/wait-for-cache'); + +const vs = (id) => ({ resourceType: 'ValueSet', id, url: `http://example.org/vs/${id}`, version: '1', status: 'active' }); + +function run(file, sql) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(file, (err) => { + if (err) { + return reject(err); + } + db.run(sql, [], (err) => db.close(() => err ? reject(err) : resolve())); + }); + }); +} + +describe('package value set cache, read-only', () => { + let dir; + let file; + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vs-ro-')); + file = path.join(dir, '.valuesets-2.db'); + const db = new ValueSetDatabase(file); + await db.create(); + await db.upsertValueSet(vs('a')); + await db.upsertValueSet(vs('b')); + await db.markComplete(); + await db.close(); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('a value set is stored with the hash the backfill would give it', async () => { + const db = new ValueSetDatabase(file, { readOnly: true }); + const conn = await db._getReadConnection(); + const rows = await new Promise((resolve, reject) => conn.all('SELECT content, content_hash FROM valuesets', [], (e, r) => e ? reject(e) : resolve(r))); + expect(rows.length).toBe(2); + for (const row of rows) { + expect(row.content_hash).toBe(crypto.createHash('sha256').update(row.content).digest('hex')); + } + expect(await db.isComplete()).toBe(true); + expect((await db.loadAllValueSets('test')).size).toBeGreaterThan(0); + await db.close(); + }); + + test('a file that needs a backfill is not opened read-only, nor changed; it is once migrated', async () => { + await run(file, 'UPDATE valuesets SET content_hash = NULL'); + const before = fs.readFileSync(file); + const ro = new ValueSetDatabase(file, { readOnly: true }); + expect(await ro.isComplete()).toBe(false); + await expect(ro.getSetting('cache-complete')).rejects.toThrow(/not migrated yet/); + expect(fs.readFileSync(file).equals(before)).toBe(true); + await expect(ro.upsertValueSet(vs('c'))).rejects.toThrow(); + + // the builder (the master) opens it, and the backfill runs there + const rw = new ValueSetDatabase(file); + expect(await rw.isComplete()).toBe(true); + await rw.close(); + expect(await ro.isComplete()).toBe(true); + // and a write through the read-only one still fails + await expect(ro.setSetting('x', 'y')).rejects.toThrow(/SQLITE_READONLY/); + await ro.close(); + }); + + test('a missing file is not created', async () => { + const missing = path.join(dir, 'none.db'); + const ro = new ValueSetDatabase(missing, { readOnly: true }); + expect(await ro.isComplete()).toBe(false); + expect(fs.existsSync(missing)).toBe(false); + }); + + test('waitForCache: waits for the cache, and gives up with a clear error', async () => { + let n = 0; + await waitForCache(async () => ++n >= 2, 'test cache', { waitMs: 5000 }); + expect(n).toBe(2); + await expect(waitForCache(async () => false, 'test cache', { waitMs: 10 })).rejects.toThrow(/the master builds it/); + }); +}); diff --git a/tests/utils/cluster-harness.js b/tests/utils/cluster-harness.js index db026c29..eda0bd5e 100644 --- a/tests/utils/cluster-harness.js +++ b/tests/utils/cluster-harness.js @@ -54,8 +54,10 @@ function pidOn(port) { * @param {number} [opts.workers=2] * @param {Object} [opts.cluster] - more server.cluster settings (the options) * @param {Object} [opts.tx] - more modules.tx settings + * @param {string[]} [opts.sources] - more library sources (library.yml), after VSAC + * @param {function(string)} [opts.prepare] - called with the data directory before the launch */ -async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {} }) { +async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {}, sources = [], prepare = null }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-opt-')); const master = front + 1; const workerPorts = Array.from({ length: workers }, (_, i) => master + 1 + i); @@ -64,7 +66,8 @@ async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {} }) if (fs.existsSync(core)) { fs.cpSync(core, path.join(dir, 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'), { recursive: true }); } - fs.writeFileSync(path.join(dir, 'library.yml'), 'base:\n url: https://storage.googleapis.com/tx-fhir-org\n\nsources:\n - internal:vsac=vsac\n'); + fs.writeFileSync(path.join(dir, 'library.yml'), 'base:\n url: https://storage.googleapis.com/tx-fhir-org\n\nsources:\n - internal:vsac=vsac\n' + + sources.map(source => ` - ${source}\n`).join('')); fs.writeFileSync(path.join(dir, 'config.json'), JSON.stringify({ hostName: 'cluster-test', logging: { console: false }, @@ -88,12 +91,17 @@ async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {} }) }); await new Promise(resolve => vsacServer.listen(vsac, resolve)); + if (prepare) { + prepare(dir); + } const out = fs.openSync(path.join(dir, 'processes.log'), 'a'); const launcher = spawn(process.execPath, [path.join(ROOT, 'cluster.js')], { cwd: ROOT, env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', out, out] }); const exited = new Promise(resolve => launcher.once('exit', (code, signal) => resolve({ code, signal }))); + // port -> when it was first seen answering /health + const upAt = new Map(); async function waitUp(ports, ms = 300000) { const t0 = Date.now(); for (const port of ports) { @@ -103,6 +111,9 @@ async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {} }) } await sleep(250); } + if (!upAt.has(port)) { + upAt.set(port, Date.now()); + } } // the launcher marks a process ready a moment after it first answers await sleep(500); @@ -119,7 +130,7 @@ async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {} }) return result; } - return { dir, front, master, workers: workerPorts, launcher, exited, waitUp, stop }; + return { dir, front, master, workers: workerPorts, launcher, exited, waitUp, upAt, stop }; } module.exports = { startCluster, request, isUp, param, issueCodes, unknownCache, pidOn, sleep, ROOT }; diff --git a/tx/cm/cm-package.js b/tx/cm/cm-package.js index 89e7e859..ce1239d2 100644 --- a/tx/cm/cm-package.js +++ b/tx/cm/cm-package.js @@ -9,6 +9,7 @@ const {validateParameter} = require("../../library/utilities"); const {ConceptMap} = require("../library/conceptmap"); const { applyIdPrefix, stripIdPrefix } = require('../library/resource-ids'); const { keepForPinnedReference } = require('../library/tho-precedence'); +const { waitForCache } = require('../library/wait-for-cache'); /** * Package-based ConceptMap provider using shared database layer @@ -18,14 +19,19 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider { /** * @param {PackageContentLoader} packageLoader - Path to the extracted package folder + * @param {Object} [options] + * @param {boolean} [options.readOnly] - a tx-worker (cluster.md): the cache is the master's; it + * is only read (ConceptMapDatabase reads with OPEN_READONLY), waited for until the master has + * finished it, and never built or changed */ - constructor(packageLoader) { + constructor(packageLoader, options = {}) { super(); validateParameter(packageLoader, "packageLoader", PackageContentLoader); this.packageLoader = packageLoader; // -2: caches from before concept maps were checked for active content in their // narrative are rebuilt (see initialize) this.dbPath = path.join(packageLoader.packageFolder, '.conceptmaps-2.db'); + this.readOnly = !!options.readOnly; this.database = new ConceptMapDatabase(this.dbPath); this.conceptMapMap = new Map(); this.initialized = false; @@ -41,6 +47,14 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider { return; } await this.packageLoader.initialize(); + if (this.readOnly) { + await waitForCache(async () => (await this.database.exists()) && (await this.database.isComplete()), + `Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: concept map cache ${this.dbPath}`); + this.conceptMapMap = await this.database.loadAllConceptMaps(); + this.#applyIdPrefix(); + this.initialized = true; + return; + } await fs.rm(path.join(this.packageLoader.packageFolder, '.conceptmaps.db'), { force: true }); // the old cache // A cache is only used once its population finished. One that was cut short diff --git a/tx/library.js b/tx/library.js index df3700bd..eed4c310 100644 --- a/tx/library.js +++ b/tx/library.js @@ -110,6 +110,13 @@ class Library { */ localSourceTag = null; + /** + * {boolean} a tx-worker (cluster.md): the package value set and concept map caches are the + * master's. This process opens them read-only, waits for the master to have finished one, + * and never builds, rebuilds or migrates one + */ + packageCachesReadOnly = false; + /** * {SyndicationStore|null} where the content this library loads is recorded for the * syndication feed, when the server has one (see tx/syndication) @@ -852,10 +859,10 @@ class Library { this.codeSystemProviders.push(cp); let vs = null; if (!csOnly) { - vs = new PackageValueSetProvider(contentLoader); + vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await vs.initialize(); this.valueSetProviders.push(vs); - const cm = new PackageConceptMapProvider(contentLoader); + const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await cm.initialize(); this.conceptMapProviders.push(cm); } @@ -1034,10 +1041,10 @@ class Library { } this.codeSystemProviders.push(cp); - const vs = new PackageValueSetProvider(contentLoader); + const vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await vs.initialize(); this.valueSetProviders.push(vs); - const cm = new PackageConceptMapProvider(contentLoader); + const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await cm.initialize(); this.conceptMapProviders.push(cm); diff --git a/tx/library/wait-for-cache.js b/tx/library/wait-for-cache.js new file mode 100644 index 00000000..c8fcb1f1 --- /dev/null +++ b/tx/library/wait-for-cache.js @@ -0,0 +1,43 @@ +/** + * A tx-worker (cluster.md) only reads the package caches the master builds. Until the master + * has finished one - normally it has, since the workers start once it is ready - the worker + * waits for it, and gives up with a clear error rather than building it itself. + * + * @module tx/library/wait-for-cache + */ + +// how long a worker waits for the master to finish a cache, and how often it looks +const DEFAULT_WAIT_MS = 5 * 60 * 1000; +const POLL_MS = 500; + +/** + * Resolve once isReady() says true; reject if it hasn't within waitMs. + * + * @param {function(): Promise} isReady + * @param {string} what - the cache, for the messages + * @param {Object} [options] + * @param {number} [options.waitMs] - default DEFAULT_WAIT_MS, or FHIRSMITH_CACHE_WAIT_MS + * @returns {Promise} + */ +async function waitForCache(isReady, what, options = {}) { + const waitMs = options.waitMs ?? (Number(process.env.FHIRSMITH_CACHE_WAIT_MS) || DEFAULT_WAIT_MS); + const start = Date.now(); + let logged = false; + for (;;) { + if (await isReady()) { + return; + } + if (Date.now() - start >= waitMs) { + throw new Error(`${what} is not ready after ${Math.round(waitMs / 1000)} s. A tx-worker only reads it; ` + + 'the master builds it: start the master first, and the workers once it answers /health (cluster.md)'); + } + if (!logged) { + logged = true; + require('../../library/logger').getInstance().child({ module: 'tx' }) + .warn(`${what} is missing, incomplete or not yet migrated: waiting for the master to finish it`); + } + await new Promise(resolve => setTimeout(resolve, POLL_MS)); + } +} + +module.exports = { waitForCache }; diff --git a/tx/tx.js b/tx/tx.js index b9255663..4be13fdb 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -253,6 +253,8 @@ class TXModule { this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats); if (config.clusterRole) { this.library.localSourceTag = config.instanceCode; + // the master builds the package caches; a worker only reads them + this.library.packageCachesReadOnly = config.clusterRole === 'tx-worker'; } // The syndication feed: content is recorded in the store as the library loads it, from diff --git a/tx/vs/vs-database.js b/tx/vs/vs-database.js index e8f0fc5c..40907a17 100644 --- a/tx/vs/vs-database.js +++ b/tx/vs/vs-database.js @@ -48,11 +48,15 @@ class ValueSetDatabase { * @param {string} dbPath - Path to the SQLite database file * @param {Object} [options] - {shared: true} when several processes use the file (one * writing, the others reading - cluster.md): WAL, so the readers don't block the writer, - * and a wait of up to 5 s for another process's lock instead of failing at once + * and a wait of up to 5 s for another process's lock instead of failing at once. + * {readOnly: true} for a cache another process builds (a tx-worker's package caches, + * cluster.md): opened read-only, never created or migrated - a file that still needs a + * migration fails to open (see _checkFinal), to be tried again once its builder is done */ constructor(dbPath, options = {}) { this.dbPath = dbPath; this.shared = !!options.shared; + this.readOnly = !!options.readOnly; // Single read-write connection used for everything. Using a separate // OPEN_READONLY connection for reads can miss WAL-based schema changes // made through the write connection (because read-only opens can't fully @@ -205,6 +209,9 @@ class ValueSetDatabase { if (this._migrationPromise) { return this._migrationPromise; } + if (this.readOnly) { + return this._openReadOnly(); + } this._migrationPromise = new Promise((resolve, reject) => { if (this._writeDb) { this._migrateIfNeeded(this._writeDb).then(resolve).catch(reject); @@ -224,6 +231,70 @@ class ValueSetDatabase { return this._migrationPromise; } + /** + * readOnly: open the file read-only, and check it needs no migration. On failure the + * connection is closed (the next use opens it again: the builder may have replaced it) + * @returns {Promise} + * @private + */ + _openReadOnly() { + this._migrationPromise = new Promise((resolve, reject) => { + const db = new sqlite3.Database(this.dbPath, sqlite3.OPEN_READONLY, (err) => { + if (err) { + reject(new Error(`Failed to open database ${this.dbPath} read-only: ${err.message}`)); + return; + } + db.configure('busyTimeout', 5000); + this._checkFinal(db).then(() => { + this._writeDb = db; + resolve(); + }, (error) => { + db.close(() => reject(error)); + }); + }); + }); + this._migrationPromise.catch(() => { this._migrationPromise = null; }); + return this._migrationPromise; + } + + /** + * Reject unless the file is as _migrateIfNeeded leaves it: every column and table there, + * and no row without its content_hash + * @param {sqlite3.Database} db + * @returns {Promise} + * @private + */ + async _checkFinal(db) { + const all = (sql) => new Promise((res, rej) => { + db.all(sql, [], (err, rows) => err ? rej(err) : res(rows)); + }); + const columns = async (table) => (await all(`PRAGMA table_info(${table})`)).map(c => c.name); + const vsCols = await columns('valuesets'); + const pending = []; + for (const col of ['date_first_seen', 'content_hash']) { + if (!vsCols.includes(col)) { + pending.push(`valuesets.${col}`); + } + } + if (!(await columns('vsac_runs')).includes('total_updated')) { + pending.push('vsac_runs.total_updated'); + } + for (const table of ['vsac_settings', 'vsac_events']) { + if ((await columns(table)).length === 0) { + pending.push(table); + } + } + if (pending.length === 0) { + const rows = await all('SELECT COUNT(*) AS n FROM valuesets WHERE content_hash IS NULL'); + if (rows[0] && rows[0].n > 0) { + pending.push(`content_hash of ${rows[0].n} value sets`); + } + } + if (pending.length > 0) { + throw new Error(`Database ${this.dbPath} is not migrated yet (${pending.join(', ')}), and is opened read-only`); + } + } + /** * Close all database connections * @returns {Promise} @@ -547,13 +618,17 @@ class ValueSetDatabase { /** * Insert or update a single ValueSet in the database * @param {Object} valueSet - The ValueSet resource - * @param {string} [contentHash] - Optional pre-computed content hash to store + * @param {string} [contentHash] - Optional pre-computed content hash to store; by default the + * hash of the content stored, as the backfill in _migrateIfNeeded computes it, so no row is + * written without one (a row without one is backfilled by the next process to open the file) * @returns {Promise} */ async upsertValueSet(valueSet, contentHash = null) { if (!valueSet.url) { throw new Error('ValueSet must have a url property'); } + const content = JSON.stringify(valueSet); + contentHash = contentHash || crypto.createHash('sha256').update(content).digest('hex'); const db = await this._getWriteConnection(); @@ -616,7 +691,7 @@ class ValueSetDatabase { valueSet.publisher || null, valueSet.status || null, valueSet.title || null, - JSON.stringify(valueSet), + content, contentHash ], (err) => { if (err) { diff --git a/tx/vs/vs-package.js b/tx/vs/vs-package.js index 8f349344..bf2b75fb 100644 --- a/tx/vs/vs-package.js +++ b/tx/vs/vs-package.js @@ -9,6 +9,7 @@ const { VersionUtilities } = require('../../library/version-utilities'); const {validateParameter} = require("../../library/utilities"); const { applyIdPrefix, stripIdPrefix } = require('../library/resource-ids'); const { keepForPinnedReference } = require('../library/tho-precedence'); +const { waitForCache } = require('../library/wait-for-cache'); /** * Package-based ValueSet provider using shared database layer @@ -18,15 +19,19 @@ class PackageValueSetProvider extends AbstractValueSetProvider { /** * @param {PackageContentLoader} packageLoader - Path to the extracted package folder + * @param {Object} [options] + * @param {boolean} [options.readOnly] - a tx-worker (cluster.md): the cache is the master's; it + * is opened read-only, waited for until the master has finished it, and never built or changed */ - constructor(packageLoader) { + constructor(packageLoader, options = {}) { super(); validateParameter(packageLoader, "packageLoader", PackageContentLoader); this.packageLoader = packageLoader; // -2: caches from before value sets were checked for active content in their narrative // are rebuilt (see initialize) this.dbPath = path.join(packageLoader.packageFolder, '.valuesets-2.db'); - this.database = new ValueSetDatabase(this.dbPath); + this.readOnly = !!options.readOnly; + this.database = new ValueSetDatabase(this.dbPath, { readOnly: this.readOnly }); this.valueSetMap = new Map(); this.initialized = false; this.count = 0; @@ -49,6 +54,13 @@ class PackageValueSetProvider extends AbstractValueSetProvider { return; } await this.packageLoader.initialize(); + if (this.readOnly) { + await waitForCache(async () => (await this.database.exists()) && (await this.database.isComplete()), + `Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: value set cache ${this.dbPath}`); + this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage()); + this.#finishLoad(); + return; + } await fs.rm(path.join(this.packageLoader.packageFolder, '.valuesets.db'), { force: true }); // the old cache // A cache is only used once its population finished. One that was cut short @@ -77,7 +89,10 @@ class PackageValueSetProvider extends AbstractValueSetProvider { await this.#buildDatabase(); this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage()); } + this.#finishLoad(); + } + #finishLoad() { // Mark each value set as cached as it enters the store: it is retained in // memory for the process lifetime, so its compose.include.filter elements // persist and a provider may memoise resolved filter analysis on them From cade59e7d990440a70cb0e23bda370eeafabb8a1 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 21:35:33 -0600 Subject: [PATCH 16/23] Mime types: download the IANA registry to a temporary file of the process's own Every process on a data directory (a master and its tx-workers, cluster.md) refreshes media-types.xml at startup. With one temporary name for all, one process could rename another's half-written download into place; each now writes its own and renames it. Co-Authored-By: Claude Opus 5.5 --- tx/cs/cs-mimetypes.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tx/cs/cs-mimetypes.js b/tx/cs/cs-mimetypes.js index 38ebd51b..0fe142d8 100644 --- a/tx/cs/cs-mimetypes.js +++ b/tx/cs/cs-mimetypes.js @@ -783,7 +783,8 @@ class MimeTypeServicesFactory extends CodeSystemFactoryProvider { * @param {string} target */ async #refresh(target) { - const temp = target + '.tmp'; + // this process's own: several processes on one data directory (cluster.md) each refresh it + const temp = `${target}.${process.pid}.tmp`; try { if (!fs.existsSync(this.cacheFolder)) { fs.mkdirSync(this.cacheFolder, { recursive: true }); From c5826079ba6a0824de760a5df00ed6101a186ef2 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 22:02:08 -0600 Subject: [PATCH 17/23] cluster.js: close an idle connection to a process before the process does The launcher's keep-alive agents kept idle connections indefinitely. A process closes an idle connection after its keepAliveTimeout - 5 s on a server without library/keep-alive.js (65 s with it) - and a request the launcher sent down one as it closed was reset: a 502 to the client, and the owner of a cache-id seen as down. The agents now drop a connection idle for 4 s, below either. A request in progress is not affected. No retry on a reset instead: tx requests are mostly POSTs whose bodies are streamed through, so a retry would mean buffering every body, and with the idle limit below the process's no reused connection is one the process is closing. nginx.md (a master and tx workers): keep the server's keepAliveTimeout above nginx's upstream keepalive_timeout, or set keepalive_timeout 4s on servers without the fix. Test: against a process with Node's 5 s, the launcher's idle connection is gone before 4.9 s (it fails without the change: kept until the process closes it), a request at 5 s uses a new connection, and a 6 s request is not cut. Co-Authored-By: Claude Opus 5.5 --- cluster.js | 9 ++++- nginx.md | 5 +++ tests/server/cluster-launcher.test.js | 51 +++++++++++++++++++++++++++ tests/utils/fake-upstream.js | 14 ++++++-- 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/cluster.js b/cluster.js index 44163a84..3eaa727b 100644 --- a/cluster.js +++ b/cluster.js @@ -32,6 +32,11 @@ const { spawn } = require('child_process'); const HEALTH_POLL_MS = 250; const MAX_BACKOFF_MS = 60000; +// An idle keep-alive connection to a process is closed by the launcher after this long: +// below the processes' own keepAliveTimeout (65 s with library/keep-alive.js, Node's default +// of 5 s on a server without it), so the launcher never sends a request down a connection the +// process is closing at that moment (a reset, which the client would see as a 502) +const UPSTREAM_IDLE_MS = 4000; // a process that ran this long before exiting is restarted at once const HEALTHY_RUN_MS = 60000; const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'trailer', 'expect']; @@ -169,7 +174,9 @@ function createLauncher(settings) { `--role=${proc.role}`, `--instance-code=${proc.code}`, `--port=${proc.port}`]]); const procs = settings.procs.map(p => ({ ...p, child: null, ready: false, outstanding: 0, failures: 0, startedAt: 0, restartTimer: null, - agent: new http.Agent({ keepAlive: true, maxSockets: 256 }) + // timeout: a connection idle in the pool this long is closed (see UPSTREAM_IDLE_MS); a + // request in progress is not affected + agent: new http.Agent({ keepAlive: true, maxSockets: 256, timeout: settings.upstreamIdleMs || UPSTREAM_IDLE_MS }) })); let server = null; let stopping = false; diff --git a/nginx.md b/nginx.md index 1a9b2400..95df58f3 100644 --- a/nginx.md +++ b/nginx.md @@ -412,6 +412,11 @@ node --expose-gc server.js --role=tx-worker --instance-code=tx1 --port=3002 node --expose-gc server.js --role=tx-worker --instance-code=tx2 --port=3003 ``` +Keep each server's keepAliveTimeout (65 s with `library/keep-alive.js`) above nginx's upstream +`keepalive_timeout` (60 s by default); on servers without that fix (Node's 5 s), set +`keepalive_timeout 4s;` in each upstream block, or nginx reuses connections the servers are +closing, and the resets mark the owner of a cache down. + The routing is the pool above, with the master as the designated `$closure` instance and the home of everything that isn't a tx endpoint: diff --git a/tests/server/cluster-launcher.test.js b/tests/server/cluster-launcher.test.js index 14b2effe..2d8f82b8 100644 --- a/tests/server/cluster-launcher.test.js +++ b/tests/server/cluster-launcher.test.js @@ -213,3 +213,54 @@ describe('the launcher, with fake processes', () => { launcher = null; }); }); + +describe('the launcher\'s keep-alive connections to the processes', () => { + // a process that closes idle connections as Node's default does (5 s): the launcher must + // close its idle ones first, or a request sent as the process closes one is reset + const PORT = 20190; + let launcher; + afterEach(async () => { + if (launcher) { + await launcher.stop(); + launcher = null; + } + }); + + test('an idle connection is closed by the launcher before the process closes it, and a long request is not cut', async () => { + launcher = createLauncher({ + port: PORT, + txPaths: TX, + drainMs: 10000, + procs: [ + { name: 'master', role: 'master', code: 'm0', port: 20191, servesTx: false }, + { name: 'worker tx1', role: 'tx-worker', code: 'tx1', port: 20192, servesTx: true } + ], + command: (proc) => [process.execPath, [FAKE, proc.code, String(proc.port), '5000']], + log: () => {} + }); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + const tx1 = launcher.procs.find(p => p.code === 'tx1'); + const pooled = () => Object.values(tx1.agent.freeSockets).reduce((n, list) => n + list.length, 0); + + expect((await get(PORT, '/r4/ValueSet/$expand')).status).toBe(200); + await until(() => pooled() === 1); + // reused while it is fresh + const before = (await get(PORT, '/r4/connections')).json.connections; + expect((await get(PORT, '/r4/connections')).json.connections).toBe(before); + // gone from the pool before the process's 5 s are up + const idleSince = Date.now(); + await until(() => pooled() === 0, 6000); + expect(Date.now() - idleSince).toBeLessThan(4900); + // so a request at the moment the process would close it goes down a new connection + await sleep(5000 - (Date.now() - idleSince)); + const r = await get(PORT, '/r4/connections'); + expect(r.status).toBe(200); + expect(r.json.connections).toBeGreaterThan(before); + + // the idle limit doesn't apply to a request in progress + const slow = await get(PORT, '/r4/ValueSet/$expand?delay=6000'); + expect(slow.status).toBe(200); + expect(slow.json.code).toBe('tx1'); + }, 30000); +}); diff --git a/tests/utils/fake-upstream.js b/tests/utils/fake-upstream.js index bf156782..fb8114a2 100644 --- a/tests/utils/fake-upstream.js +++ b/tests/utils/fake-upstream.js @@ -1,20 +1,30 @@ // A stand-in for a server process, for tests/server/cluster-launcher.test.js: -// node fake-upstream.js +// node fake-upstream.js [keepAliveTimeout ms] // /health answers 200; anything else answers {code, url, xff, proto} as JSON, after // ?delay=. On SIGTERM it finishes the requests it has, then exits. const http = require('http'); -const [code, port] = process.argv.slice(2); +const [code, port, keepAlive] = process.argv.slice(2); const server = http.createServer((req, res) => { if (req.url === '/health') { res.end('ok'); return; } + if (req.url === '/r4/connections') { + res.end(JSON.stringify({ code, connections })); + return; + } const delay = Number(new URL(req.url, 'http://x').searchParams.get('delay')) || 0; setTimeout(() => { res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify({ code, url: req.url, xff: req.headers['x-forwarded-for'], proto: req.headers['x-forwarded-proto'] })); }, delay); }); +if (keepAlive) { + server.keepAliveTimeout = Number(keepAlive); +} +// the connections it accepted, for the test of idle keep-alive connections +let connections = 0; +server.on('connection', () => { connections++; }); server.listen(Number(port)); process.on('SIGTERM', () => { server.close(() => process.exit(0)); From df3301c0ac0915bbf4d90cdde0c5fdbccc8a66f6 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 22:21:26 -0600 Subject: [PATCH 18/23] tx-worker: read the IANA media types registry the master downloaded; never download it Every process on a data directory downloaded media-types.xml at each load (15 s timeout) and renamed it over the cached copy. With a role, only the master refreshes it now: tx.js sets library.mimeTypesReadOnly for a tx-worker, and MimeTypeServicesFactory then only reads the cached copy. The workers start once the master is ready, so the copy is fresh. A worker that finds no copy logs a warning and goes on as a failed download with no cache does (no `registered` filter); it does not download. With no role nothing changes. Since only the master writes the file, the per-process temporary name (cade59e7) is no longer needed: the temporary file is media-types.xml.tmp again, as on main. cluster.md: the registry row in "What runs where". Tests: a read-only (worker) load, directly and through Library.loadInternal, makes no axios call and leaves the cached file's mtime and the folder unchanged; with no cached copy it warns and has no registered types; a no-role/master load downloads and replaces it. Co-Authored-By: Claude Opus 5.5 --- cluster.md | 1 + tests/cs/mimetypes-cluster-role.test.js | 109 ++++++++++++++++++++++++ tx/cs/cs-mimetypes.js | 17 +++- tx/library.js | 8 +- tx/tx.js | 2 + 5 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 tests/cs/mimetypes-cluster-role.test.js diff --git a/cluster.md b/cluster.md index 5d19bf06..5d6f5cea 100644 --- a/cluster.md +++ b/cluster.md @@ -49,6 +49,7 @@ which parts of the configuration each one acts on. | Syndication feed (the store) | yes | no: routed to the master | | VSAC sync, and the resync form on `/info/vsac` | yes | no: reads what the master wrote | | Package crawler, XIG download, registry crawl, daily report, npm audit | yes | no (those modules don't run) | +| IANA media types registry (`terminology-cache/media-types.xml`), downloaded at each load | yes | no: reads the master's copy (none: a warning, and no `registered` filter) | | Statistics database, uptime history | yes | no: in memory only | | `$cache-control` caches | its own, in memory (shared, with [`sharedCache`](#options)) | its own, in memory (shared, with [`sharedCache`](#options)) | | Expansion cache, CodeSystem list refresh | its own | its own | diff --git a/tests/cs/mimetypes-cluster-role.test.js b/tests/cs/mimetypes-cluster-role.test.js new file mode 100644 index 00000000..f4ac5a2f --- /dev/null +++ b/tests/cs/mimetypes-cluster-role.test.js @@ -0,0 +1,109 @@ +/** + * The IANA media types registry in a cluster (cluster.md): a tx-worker reads the cached copy + * the master downloaded and never downloads or writes it; the master, and a server with no + * role, refresh it on every load as before. + */ +jest.mock('axios'); +const axios = require('axios'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { MimeTypeServicesFactory } = require('../../tx/cs/cs-mimetypes'); +const { Library } = require('../../tx/library'); +const { TestUtilities } = require('../test-utilities'); + +const registry = (...names) => ` + + + ${names.map(n => `${n}text/${n}`).join('\n ')} + +`; + +function fakeLog() { + const lines = []; + const add = level => msg => lines.push(`${level}: ${msg}`); + return { lines, info: add('info'), warn: add('warn'), error: add('error'), debug: add('debug') }; +} + +describe('Mime types registry and the cluster role', () => { + let dir; + let target; + const OLD = new Date('2026-01-01T00:00:00Z'); + let i18n; + + beforeAll(async () => { + i18n = await TestUtilities.loadTranslations(); + }); + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mime-role-')); + target = path.join(dir, 'media-types.xml'); + axios.get.mockReset(); + axios.get.mockResolvedValue({ data: registry('plain', 'html', 'csv') }); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function seedCache() { + fs.writeFileSync(target, registry('plain', 'html'), 'utf8'); + fs.utimesSync(target, OLD, OLD); + } + + test('read-only (a tx-worker): no download, no write, the cached copy is used', async () => { + seedCache(); + const log = fakeLog(); + const factory = new MimeTypeServicesFactory(i18n, dir, log, { readOnly: true }); + await factory.load(); + expect(axios.get).not.toHaveBeenCalled(); + expect(fs.statSync(target).mtimeMs).toBe(OLD.getTime()); + expect(fs.readdirSync(dir)).toEqual(['media-types.xml']); + expect([...factory.registeredTypes].sort()).toEqual(['text/html', 'text/plain']); + }); + + test('read-only with no cached copy: a warning, no download, no registered filter', async () => { + const log = fakeLog(); + const factory = new MimeTypeServicesFactory(i18n, dir, log, { readOnly: true }); + await factory.load(); + expect(axios.get).not.toHaveBeenCalled(); + expect(fs.readdirSync(dir)).toEqual([]); + expect(factory.registeredTypes).toBeNull(); + expect(log.lines.some(l => l.startsWith('warn: Mime types: no cached copy'))).toBe(true); + expect(log.lines.some(l => l.startsWith('warn: Mime types: no copy of the IANA media types registry'))).toBe(true); + }); + + test('no role (the default): refreshes the cached copy', async () => { + seedCache(); + const factory = new MimeTypeServicesFactory(i18n, dir, fakeLog()); + await factory.load(); + expect(axios.get).toHaveBeenCalledTimes(1); + expect(fs.statSync(target).mtimeMs).toBeGreaterThan(OLD.getTime()); + expect(fs.readdirSync(dir)).toEqual(['media-types.xml']); + expect(factory.registeredTypes.has('text/csv')).toBe(true); + }); + + describe('through the library, as tx.js sets it for each role', () => { + async function loadWith(readOnly) { + const library = new Library(null, null, fakeLog(), null); + library.cacheFolder = dir; + library.i18n = i18n; + library.mimeTypesReadOnly = readOnly; + library.registerProvider = () => {}; + await library.loadInternal('mimetypes', false, 'load'); + } + + test('tx-worker: no network call and no write', async () => { + seedCache(); + await loadWith(true); + expect(axios.get).not.toHaveBeenCalled(); + expect(fs.statSync(target).mtimeMs).toBe(OLD.getTime()); + }); + + test('master or no role: refreshes', async () => { + seedCache(); + await loadWith(false); + expect(axios.get).toHaveBeenCalledTimes(1); + expect(fs.statSync(target).mtimeMs).toBeGreaterThan(OLD.getTime()); + }); + }); +}); diff --git a/tx/cs/cs-mimetypes.js b/tx/cs/cs-mimetypes.js index 0fe142d8..d9db48f7 100644 --- a/tx/cs/cs-mimetypes.js +++ b/tx/cs/cs-mimetypes.js @@ -741,12 +741,16 @@ class MimeTypeServicesFactory extends CodeSystemFactoryProvider { * @param {I18nSupport} i18n * @param {string} cacheFolder - where the downloaded copy of the IANA registry is kept * @param {Logger} log + * @param {Object} [options] + * @param {boolean} [options.readOnly] - only read the cached copy: never download or write + * it (a tx-worker, cluster.md: the master refreshes it before the workers start) */ - constructor(i18n, cacheFolder = null, log = null) { + constructor(i18n, cacheFolder = null, log = null, options = {}) { super(i18n); this.uses = 0; this.cacheFolder = cacheFolder; this.log = log; + this.readOnly = options.readOnly === true; this.registeredTypes = null; } @@ -761,7 +765,13 @@ class MimeTypeServicesFactory extends CodeSystemFactoryProvider { return; // no cache configured: the registered filter is unavailable } const target = path.join(this.cacheFolder, IANA_MEDIA_TYPES_FILE); - await this.#refresh(target); + if (this.readOnly) { + if (!fs.existsSync(target)) { + this.log?.warn(`Mime types: no cached copy of the IANA registry at ${target} (a tx-worker reads the one the master downloads, and never downloads it)`); + } + } else { + await this.#refresh(target); + } if (fs.existsSync(target)) { try { @@ -783,8 +793,7 @@ class MimeTypeServicesFactory extends CodeSystemFactoryProvider { * @param {string} target */ async #refresh(target) { - // this process's own: several processes on one data directory (cluster.md) each refresh it - const temp = `${target}.${process.pid}.tmp`; + const temp = target + '.tmp'; try { if (!fs.existsSync(this.cacheFolder)) { fs.mkdirSync(this.cacheFolder, { recursive: true }); diff --git a/tx/library.js b/tx/library.js index eed4c310..7938f8ea 100644 --- a/tx/library.js +++ b/tx/library.js @@ -117,6 +117,12 @@ class Library { */ packageCachesReadOnly = false; + /** + * {boolean} a tx-worker (cluster.md): the cached IANA media types registry is the master's. + * This process reads it and never downloads or writes it + */ + mimeTypesReadOnly = false; + /** * {SyndicationStore|null} where the content this library loads is recorded for the * syndication feed, when the server has one (see tx/syndication) @@ -569,7 +575,7 @@ class Library { break; } case "mimetypes" : { - const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log); + const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log, { readOnly: this.mimeTypesReadOnly }); await mime.load(); this.registerProvider('internal', mime); break; diff --git a/tx/tx.js b/tx/tx.js index 4be13fdb..b215b80e 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -255,6 +255,8 @@ class TXModule { this.library.localSourceTag = config.instanceCode; // the master builds the package caches; a worker only reads them this.library.packageCachesReadOnly = config.clusterRole === 'tx-worker'; + // the master refreshes the IANA media types registry; a worker reads its copy + this.library.mimeTypesReadOnly = config.clusterRole === 'tx-worker'; } // The syndication feed: content is recorded in the store as the library loads it, from From 2d1b5f143d80b86e4974f9f5accabedde8c8b1a6 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Tue, 3 Nov 2026 22:39:50 -0600 Subject: [PATCH 19/23] tx-worker: read the master's folder:/file: source caches; never build one With a role, each process kept a folder:/file: source's caches in its own folder-source--. Since workers stopped building caches (120c529f), a worker waited for its own folder's .valuesets-2.db, which nothing built, and after 300 s stopped: a cluster with a file: source on a new data directory crash-looped. - every process in a cluster uses the master's folder, folder-source-master- - a worker never removes, creates or writes it: its providers open the caches read-only, waiting for the master like the package caches - the master builds the caches in a temporary sibling folder and renames it into place, so a worker running across a master restart never finds the folder half built - only the master (and a process with no role) prunes folder-source-* folders; no role is unchanged Test: a cluster on a new data directory with two file: sources (a CodeSystem and a ValueSet) starts every process once; the caches are in the master's folders, untouched after the master is ready, and $lookup and ValueSet search answer from each worker and through the launcher. Co-Authored-By: Claude Opus 5.5 --- cluster.md | 10 ++- tests/server/cluster-package-caches.test.js | 81 ++++++++++++++++++++- tx/library.js | 61 +++++++++++++--- tx/tx.js | 3 +- 4 files changed, 141 insertions(+), 14 deletions(-) diff --git a/cluster.md b/cluster.md index 5d6f5cea..987c9e40 100644 --- a/cluster.md +++ b/cluster.md @@ -73,10 +73,14 @@ master to finish it, and after 5 minutes (`FHIRSMITH_CACHE_WAIT_MS`) stops with says so. Starting the master first is what makes the wait unnecessary, not what keeps the workers from writing. +The same goes for the caches of a `folder:` or `file:` library source +(`terminology-cache/folder-source-master-`, one folder for every process). Each time the +master loads the library it rebuilds them in a temporary sibling folder and renames it into +place, so a running worker never finds one half built; a worker reads the new one when it next +starts. Only the master removes the folders of sources no longer loaded. + **Nothing else is shared.** The workers run no module but tx, so they never read what the -package crawler, the XIG download or the registry crawl write. Each process keeps the caches -of a `folder:` or `file:` library source in its own folder -(`terminology-cache/folder-source--`), because each load wipes and rebuilds them. +package crawler, the XIG download or the registry crawl write. ## Routing diff --git a/tests/server/cluster-package-caches.test.js b/tests/server/cluster-package-caches.test.js index f587878e..afcd6df1 100644 --- a/tests/server/cluster-package-caches.test.js +++ b/tests/server/cluster-package-caches.test.js @@ -3,10 +3,15 @@ * (.valuesets-2.db and .conceptmaps-2.db removed): the master builds them before it is ready, * the workers only read them. Every process starts once - none exits and is restarted - and * neither cache file is written after the master is ready. + * + * The same for the caches of file: library sources: every process uses the master's + * folder-source-master- folder, which the master builds before it is ready and the + * workers only read. */ const fs = require('fs'); +const os = require('os'); const path = require('path'); -const { startCluster, request } = require('../utils/cluster-harness'); +const { startCluster, request, param } = require('../utils/cluster-harness'); const FRONT = 20150; const VSAC = 20159; @@ -66,3 +71,77 @@ describe('the package caches on a new data directory', () => { expect(log).not.toMatch(/SQLITE_(BUSY|READONLY|IOERR)/); }, 600000); }); + +describe('the caches of file: sources on a new data directory', () => { + const FILE_FRONT = 20160; + const FILE_VSAC = 20169; + const SYSTEM = 'http://example.org/fhir/CodeSystem/cluster-file-source'; + const VS = 'http://example.org/fhir/ValueSet/cluster-file-source'; + let cluster; + let fixtures; + + afterAll(async () => { + if (cluster) { + await cluster.stop(); + } + if (fixtures) { + fs.rmSync(fixtures, { recursive: true, force: true }); + } + }); + + // every file under the folder-source-* folders, with its mtime and size, and the folders' + const snapshot = (cacheDir) => { + const out = {}; + for (const name of fs.readdirSync(cacheDir).filter(n => n.startsWith('folder-source-')).sort()) { + const dir = path.join(cacheDir, name); + out[name] = fs.statSync(dir).mtimeMs; + for (const file of fs.readdirSync(dir).sort()) { + const st = fs.statSync(path.join(dir, file)); + out[name + '/' + file] = [st.mtimeMs, st.size]; + } + } + return out; + }; + + test('the master builds them in its folder, the workers read it and start once', async () => { + fixtures = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-fx-')); + fs.writeFileSync(path.join(fixtures, 'cs.json'), JSON.stringify({ + resourceType: 'CodeSystem', id: 'cluster-file-source', url: SYSTEM, version: '1.0.0', + name: 'ClusterFileSource', status: 'active', content: 'complete', + concept: [{ code: 'A', display: 'Alpha' }, { code: 'B', display: 'Beta' }] + })); + fs.writeFileSync(path.join(fixtures, 'vs.json'), JSON.stringify({ + resourceType: 'ValueSet', id: 'cluster-file-source', url: VS, version: '1.0.0', + name: 'ClusterFileSource', status: 'active', compose: { include: [{ system: SYSTEM }] } + })); + cluster = await startCluster({ + front: FILE_FRONT, vsac: FILE_VSAC, workers: 2, + sources: [`file:${path.join(fixtures, 'cs.json')}`, `file:${path.join(fixtures, 'vs.json')}`] + }); + const cacheDir = path.join(cluster.dir, 'terminology-cache'); + const before = snapshot(cacheDir); + const folders = Object.keys(before).filter(k => !k.includes('/')); + // one folder per source, the master's, and nothing left from the build + expect(folders).toHaveLength(2); + for (const name of folders) { + expect(name).toMatch(/^folder-source-master-[0-9a-f]{16}$/); + expect(before[name + '/.valuesets-2.db']).toBeDefined(); + } + + const lookup = `/r4/CodeSystem/$lookup?system=${encodeURIComponent(SYSTEM)}&code=A`; + for (const port of [...cluster.workers, cluster.front, cluster.front, cluster.front]) { + const r = await request(port, 'GET', lookup); + expect(r.status).toBe(200); + expect(param(r.json, 'display').valueString).toBe('Alpha'); + const v = await request(port, 'GET', `/r4/ValueSet?url=${encodeURIComponent(VS)}`); + expect(v.status).toBe(200); + expect((v.json.entry || []).length).toBe(1); + } + + // the workers wrote nothing + expect(snapshot(cacheDir)).toEqual(before); + const log = fs.readFileSync(path.join(cluster.dir, 'processes.log'), 'utf8'); + expect(log).not.toMatch(/exited \(/); + expect(log).not.toMatch(/is not ready after/); + }, 600000); +}); diff --git a/tx/library.js b/tx/library.js index 7938f8ea..31f1e12a 100644 --- a/tx/library.js +++ b/tx/library.js @@ -104,9 +104,12 @@ class Library { totalDownloaded = 0; vsacCfg = undefined; /** - * {string|null} with several processes on one data directory (cluster.md), this one's - * instance code: its folder-source-* caches are its own, since each load wipes and - * rebuilds them, and it leaves the other processes' alone (no pruning) + * {string|null} with several processes on one data directory (cluster.md), a name every + * process in the cluster shares (the master's role): the folder-source-* caches are the + * master's, named folder-source--. The master builds each in a temporary sibling + * and renames it into place, so a worker never sees one half built; a worker + * (packageCachesReadOnly) only reads them, waiting for the master as for package caches, + * and never prunes */ localSourceTag = null; @@ -268,7 +271,8 @@ class Library { external: endMemory.external - this.startMemory.external }; - if (!this.localSourceTag) { + if (!this.packageCachesReadOnly) { + // the master, or no role: a worker leaves the master's caches alone await this.#pruneLocalSourceCaches(); } @@ -895,7 +899,7 @@ class Library { } const folderPath = this.#resolveLocalSourcePath(details); - const cacheSubdir = await this.#prepareLocalSourceCache(folderPath); + const cacheSubdir = await this.#prepareLocalSourceCache(folderPath, dir => new FolderContentLoader(folderPath, dir)); await this.#loadLocalSource(new FolderContentLoader(folderPath, cacheSubdir)); } @@ -920,7 +924,7 @@ class Library { } const filePath = this.#resolveLocalSourcePath(details); - const cacheSubdir = await this.#prepareLocalSourceCache(filePath); + const cacheSubdir = await this.#prepareLocalSourceCache(filePath, dir => new FileContentLoader(filePath, dir)); await this.#loadLocalSource(new FileContentLoader(filePath, cacheSubdir)); } @@ -940,15 +944,54 @@ class Library { * Parks the Package*Provider SQLite caches under the terminology cache rather * than polluting the user's source folder, and wipes between runs so edits to * the source are reliably picked up. + * + * In a cluster (localSourceTag set) every process uses the master's folder: + * - a worker (packageCachesReadOnly) touches nothing; its providers open the caches + * read-only, waiting for the master to have built them + * - the master builds the caches in a temporary sibling, then renames it into place, so a + * running worker never finds the folder half built + * + * @param {string} sourcePath + * @param {function(string): FolderContentLoader} makeLoader - a loader with its caches in a folder + * @returns {Promise} the cache folder * @private */ - async #prepareLocalSourceCache(sourcePath) { + async #prepareLocalSourceCache(sourcePath, makeLoader) { const hash = crypto.createHash('sha1').update(sourcePath).digest('hex').substring(0, 16); const name = LOCAL_SOURCE_CACHE_PREFIX + (this.localSourceTag ? this.localSourceTag + '-' : '') + hash; const cacheSubdir = path.join(this.cacheFolder, name); this.#localSourceCaches.add(name); - await fs.rm(cacheSubdir, { recursive: true, force: true }); - await fs.mkdir(cacheSubdir, { recursive: true }); + if (this.packageCachesReadOnly) { + return cacheSubdir; + } + if (!this.localSourceTag) { + await fs.rm(cacheSubdir, { recursive: true, force: true }); + await fs.mkdir(cacheSubdir, { recursive: true }); + return cacheSubdir; + } + const building = `${cacheSubdir}.building-${process.pid}`; + const old = `${cacheSubdir}.old-${process.pid}`; + await fs.rm(building, { recursive: true, force: true }); + await fs.rm(old, { recursive: true, force: true }); + await fs.mkdir(building, { recursive: true }); + const loader = makeLoader(building); + await loader.initialize(); + const vs = new PackageValueSetProvider(loader); + await vs.initialize(); + await vs.close(); + const cm = new PackageConceptMapProvider(loader); + await cm.initialize(); + // a directory can't be renamed over a non-empty one: move the old one aside first. A + // worker that looks in between finds no folder, and waits for it + try { + await fs.rename(cacheSubdir, old); + } catch (e) { + if (e.code !== 'ENOENT') { + throw e; + } + } + await fs.rename(building, cacheSubdir); + await fs.rm(old, { recursive: true, force: true }); return cacheSubdir; } diff --git a/tx/tx.js b/tx/tx.js index b215b80e..a2402941 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -252,7 +252,8 @@ class TXModule { this.log.info(`Loading library from: ${config.librarySource}`); this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats); if (config.clusterRole) { - this.library.localSourceTag = config.instanceCode; + // the folder:/file: source caches are the master's, under one name every process shares + this.library.localSourceTag = 'master'; // the master builds the package caches; a worker only reads them this.library.packageCachesReadOnly = config.clusterRole === 'tx-worker'; // the master refreshes the IANA media types registry; a worker reads its copy From 3e7570dce4116a902f80c6ef9af862eff149a33b Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Wed, 4 Nov 2026 10:52:55 -0600 Subject: [PATCH 20/23] Keep shared closure identities unique across resets Allocate shared table IDs from a durable counter, initialized above existing IDs, so resets and dispose/recreate cannot accept old writes or reuse retry state. Check duplicate-only adds too, and refuse a resync interrupted by reset. Existing databases keep their rows and foreign keys. Seven new regression cases fail before this change and pass after it, including reversed-hierarchy retries and reuse after reopening an empty database. Co-Authored-By: GPT-6 --- tests/tx/closure-identity.test.js | 66 +++++++++++++++++++++++++++++++ tests/tx/closure-shared.test.js | 51 ++++++++++++++++++++++++ translations/Messages.properties | 1 + tx/closure/closure-store.js | 29 +++++++++++--- tx/workers/closure.js | 11 ++++++ 5 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 tests/tx/closure-identity.test.js diff --git a/tests/tx/closure-identity.test.js b/tests/tx/closure-identity.test.js new file mode 100644 index 00000000..60d375ea --- /dev/null +++ b/tests/tx/closure-identity.test.js @@ -0,0 +1,66 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { ClosureStore } = require('../../tx/closure/closure-store'); + +describe('shared closure table identities', () => { + let dir, file; + const opened = []; + const open = (shared = true) => { + const store = new ClosureStore({ database: file }, null, { shared }); + store.open(); + opened.push(store); + return store; + }; + const config = [{ name: 'system-version', valueUri: 'http://example.org/cs|old' }]; + const concepts = [{ key: 'a', system: 'http://example.org/cs', code: 'a' }]; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'closure-identity-')); + file = path.join(dir, 'closure.db'); + }); + afterEach(() => { + for (const store of opened.splice(0)) store.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test.each(['reset', 'dispose and recreate'])('%s rejects an outstanding write to the old table', change => { + const a = open(), b = open(); + const old = a.createTableChecked('table', config, { reset: false, maxTables: 1 }).table; + if (change === 'dispose and recreate') b.dropTable('table'); + const replacement = b.createTableChecked('table', [], { reset: true, maxTables: 1 }).table; + expect(a.recordAddChecked(old, concepts, [])).toBeNull(); + expect(a.getConcepts(replacement.id)).toEqual([]); + expect(a.getParams(replacement.id)).toEqual([]); + }); + + test('a duplicate-only add still checks the snapshot it read', () => { + const a = open(), b = open(); + const table = a.createTableChecked('table', config, { reset: false, maxTables: 1 }).table; + a.recordAddChecked(table, concepts, []); + const old = a.getTable('table'); + b.createTableChecked('table', [], { reset: true, maxTables: 1 }); + expect(a.recordAddChecked(old, [], [])).toBeNull(); + expect(a.getTable('table').version).toBe(0); + }); + + test('upgrading an existing database preserves its data, and deleted IDs stay retired across reopen', () => { + const legacy = open(false); + const old = legacy.createTable('existing', config); + legacy.recordAdd(old, concepts, []); + legacy.close(); + const a = open(); + expect(a.getTable('existing').id).toBe(old.id); + expect(a.getParams(old.id)).toEqual(config); + expect(a.getConcepts(old.id).map(c => c.code)).toEqual(['a']); + const next = a.createTableChecked('new', [], { reset: false, maxTables: 2 }).table; + expect(next.id).toBeGreaterThan(old.id); + a.dropTable('existing'); + a.dropTable('new'); + a.close(); + const b = open(); + const afterRestart = b.createTableChecked('new', [], { reset: false, maxTables: 2 }).table; + expect(afterRestart.id).toBeGreaterThan(next.id); + expect(b.db.pragma('foreign_key_check')).toEqual([]); + }); +}); diff --git a/tests/tx/closure-shared.test.js b/tests/tx/closure-shared.test.js index ceea3e1f..cc28009f 100644 --- a/tests/tx/closure-shared.test.js +++ b/tests/tx/closure-shared.test.js @@ -243,6 +243,57 @@ describe('$closure on a shared database', () => { expect(txModule.closureStore.getTable(T)).toBeNull(); }); + test.each(['reset', 'dispose and recreate'])('%s during an add uses the replacement configuration on retry', async (change) => { + const T = 'replace-during-add'; + expect((await post(app, init(T))).status).toBe(200); + const replacement = { ...CS, concept: [ + { code: 'a', display: 'New A', concept: [{ code: 'r', display: 'New R' }] } + ] }; + const i = interfere((call) => { + if (call === 1) { + if (change === 'dispose and recreate') { + other.dropTable(T); + } + other.createTableChecked(T, [{ name: 'tx-resource', resource: replacement }], + { reset: true, maxTables: Infinity }); + } + }); + try { + const res = await post(app, params(name(T), concept('a'), concept('r'))); + expect(res.status).toBe(200); + // Same canonical/version and codes, but the replacement reverses their relationship. + // A retry must discard the old providers, displays and subsumption memo as well. + expect(entries(res.body)).toEqual([n('r', 'a')]); + expect(i.calls()).toBe(2); + expect(other.getConcepts(other.getTable(T).id).map(c => c.display).sort()).toEqual(['New A', 'New R']); + } finally { + i.restore(); + other.dropTable(T); + } + }); + + test('a reset during resync refuses the interrupted snapshot', async () => { + const T = 'replace-during-resync'; + await post(app, init(T)); + await post(app, params(name(T), concept('a'), concept('r'))); + const store = txModule.closureStore; + const realPage = store.entriesPage.bind(store); + const page = jest.spyOn(store, 'entriesPage').mockImplementation((...args) => { + const rows = realPage(...args); + other.createTableChecked(T, init(T).parameter.slice(1), { reset: true, maxTables: Infinity }); + return rows; + }); + try { + const res = await post(app, params(name(T), { name: 'version', valueString: '0' })); + expect(res.status).toBe(409); + expect(res.body.issue[0].code).toBe('conflict'); + expect(other.getTable(T).version).toBe(0); + } finally { + page.mockRestore(); + other.dropTable(T); + } + }); + test('a resync reports only up to the version it read', () => { const T = 'bounded'; const made = other.createTableChecked(T, [], { reset: false, maxTables: Infinity }); diff --git a/translations/Messages.properties b/translations/Messages.properties index 110bac03..6f3c7223 100644 --- a/translations/Messages.properties +++ b/translations/Messages.properties @@ -1143,6 +1143,7 @@ CLOSURE_TOO_MANY_REQUEST_CONCEPTS = A single $closure request can add at most {0 CLOSURE_TOO_MANY_ENTRIES = Adding these concepts would put more entries into the closure table ''{0}'' than this server allows, in one request or in total CLOSURE_CONFIG_TOO_LARGE = The configuration supplied for this closure table (mostly the tx-resource parameters) is larger than this server will keep, which is {0} bytes CLOSURE_TABLE_BUSY = The closure table ''{0}'' is busy with other requests. Try again shortly +CLOSURE_TABLE_CHANGED = The closure table ''{0}'' was reset while this request was reading it. Retry the request against the new table SUBSUMPTION_CS_HIERARCHY_MEANING = The CodeSystem ''{0}'' has hierarchyMeaning = {1}, so its hierarchy does not describe subsumption, and subsumption cannot be tested SUBSUMPTION_CS_NOT_COMPLETE = The CodeSystem ''{0}'' has content = {1}, so subsumption cannot be tested (subsumption can only be tested against a code system with content = complete) SUBSUMPTION_CS_NOT_FOUND = A definition for the CodeSystem ''{0}'' could not be found, so subsumption cannot be tested diff --git a/tx/closure/closure-store.js b/tx/closure/closure-store.js index 460a9efb..bc39b1dd 100644 --- a/tx/closure/closure-store.js +++ b/tx/closure/closure-store.js @@ -147,6 +147,22 @@ class ClosureStore { entry_count = (SELECT count(*) FROM closure_entry e WHERE e.table_id = closure_table.id); `); } + if (this.shared) { + // An INTEGER PRIMARY KEY can be reused after a reset/dispose. Shared calls keep + // the id across awaits, including in their retry memo and paged reads, so it + // must identify one incarnation forever. A durable counter also upgrades old + // databases without rebuilding the table and its foreign keys. This runs in + // the schema's IMMEDIATE transaction, before any shared writer can allocate. + this.db.exec(` + CREATE TABLE IF NOT EXISTS closure_table_sequence ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + last_id INTEGER NOT NULL + ); + INSERT OR IGNORE INTO closure_table_sequence VALUES (1, 0); + UPDATE closure_table_sequence SET last_id = max(last_id, + (SELECT coalesce(max(id), 0) FROM closure_table)) WHERE singleton = 1; + `); + } } #prepare() { @@ -190,6 +206,8 @@ class ClosureStore { }; if (this.shared) { Object.assign(this.stmt, { + nextTableId: this.db.prepare('UPDATE closure_table_sequence SET last_id = last_id + 1 WHERE singleton = 1 RETURNING last_id AS id'), + insertSharedTable: this.db.prepare('INSERT INTO closure_table (id, name, version, created, last_used) VALUES (?, ?, 0, ?, ?)'), entriesPageUpTo: this.db.prepare(` SELECT e.rowid AS rowid, s.system AS source_system, s.code AS source_code, s.display AS source_display, @@ -330,7 +348,9 @@ class ClosureStore { if (existing) { this.stmt.deleteTable.run(existing.id); } - const info = this.stmt.insertTable.run(name, now, now); + const info = this.shared + ? this.stmt.insertSharedTable.run(this.stmt.nextTableId.get().id, name, now, now) + : this.stmt.insertTable.run(name, now, now); const id = info.lastInsertRowid; params.forEach((p, i) => this.stmt.insertParam.run(id, i, p.name, JSON.stringify(p))); return this.stmt.getTable.get(name); @@ -438,11 +458,8 @@ class ClosureStore { * (nothing was written) */ recordAddChecked(table, concepts, entries) { - if (concepts.length === 0) { - // nothing to write: what the caller read is still a true answer (it was the table - // when it read it, and nothing it was asked to add is missing from the table now) - return this.recordAdd(table, concepts, entries); - } + // Even a duplicate-only add must check: a reset may have removed those concepts + // while this request was working, or an intervening add changed their version. return this.db.transaction(() => { const now = this.stmt.getTable.get(table.name); if (!now || now.id !== table.id || now.version !== table.version) { diff --git a/tx/workers/closure.js b/tx/workers/closure.js index c351e2ba..42922f52 100644 --- a/tx/workers/closure.js +++ b/tx/workers/closure.js @@ -336,6 +336,17 @@ class ClosureWorker extends TerminologyWorker { } after = page[page.length - 1].rowid; } + if (this.store.shared) { + // Reset/dispose deletes the old pages. Never return a truncated snapshot, or + // entries from a replacement, as though they belonged to the version we read. + const current = this.store.getTable(name); + if (!current) { + throw this.closureIssue('not-found', 'CLOSURE_TABLE_UNKNOWN', [name], 'not-found', 404); + } + if (current.id !== table.id) { + throw this.closureIssue('conflict', 'CLOSURE_TABLE_CHANGED', [name], 'business-rule', 409); + } + } return await this.buildConceptMap(name, table.version, entries, fhirVersion); } From a40ee9e0d96a54a59c002bc08c124707f8437f67 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Wed, 4 Nov 2026 10:52:55 -0600 Subject: [PATCH 21/23] Reject changed resources during shared-cache lazy loading Check a lazy resource against the hash and FHIR version read with its index. Return a retryable HTTP 409 if it changed, before new content can be used with an old expansion-cache key. The real expansion cache path fails two new before-change tests (add and set replacement). Both pass after the fix; retrying from a fresh index succeeds, and the old key remains empty. Co-Authored-By: GPT-6 --- tests/tx/resource-cache.test.js | 50 +++++++++++++++++++++++++++++++++ tx/resource-cache-sqlite.js | 10 +++++-- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/tx/resource-cache.test.js b/tests/tx/resource-cache.test.js index 25c32a9c..9ed6fea5 100644 --- a/tests/tx/resource-cache.test.js +++ b/tests/tx/resource-cache.test.js @@ -774,6 +774,56 @@ describe('SqliteResourceCache', () => { expect(ec.computeKey(vs, params, eager)).toBe(memoryKey); }); + test.each(['add', 'set'])('%s replacing a lazy resource cannot change the content behind its expansion key', async (method) => { + const { ExpandWorker } = require('../../tx/workers/expand'); + const c = make(); + const original = mkCs('http://x/a', '1'); + const replacement = mkCs('http://x/a', '1', { concept: [{ code: 'b' }] }); + c.set('c1', [original]); + const ec = new ExpansionCache(null); + ec.forceCaching = true; + const vs = { resourceType: 'ValueSet', compose: { include: [{ system: 'http://x/a' }] } }; + const resources = c.get('c1'); + const oldKey = ec.computeKey(vs, null, resources); + // Run the real cache lookup/compute/store path, with the replacement between + // computeKey and the first use of the lazy resource's JSON. + const worker = { + deadCheck() {}, log: { debug() {} }, opContext: { expansionCache: ec }, + additionalResources: resources, + async performExpansion() { + c[method]('c1', [replacement]); + return { expansion: { contains: this.additionalResources[0].jsonObj.concept } }; + } + }; + await expect(ExpandWorker.prototype.doExpand.call(worker, vs, null)) + .rejects.toMatchObject({ statusCode: 409, cause: 'conflict', msgId: 'CACHE_ID_CHANGED_DURING_REQUEST' }); + expect(ec.get(oldKey)).toBeNull(); + expect(resources[0].expansionKeyHash()).toBe(original.expansionKeyHash()); + + // Retrying from a fresh index succeeds and caches only under the new key. + worker.additionalResources = c.get('c1'); + const newKey = ec.computeKey(vs, null, worker.additionalResources); + expect(newKey).not.toBe(oldKey); + const result = await ExpandWorker.prototype.doExpand.call(worker, vs, null); + expect(result.expansion.contains).toEqual([{ code: 'b' }]); + expect(ec.get(newKey)).toEqual(result); + expect(ec.get(oldKey)).toBeNull(); + }); + + test('a loaded resource keeps its snapshot, and an unchanged lazy resource still loads after a write', () => { + const c = make(); + const original = mkCs('http://x/a', '1'); + c.set('c1', [original]); + const [loaded] = c.get('c1'); + expect(loaded.jsonObj).toEqual(original.jsonObj); + const [lazy] = c.get('c1'); + c.add('c1', [mkCs('http://x/b', '1')]); + expect(lazy.jsonObj).toEqual(original.jsonObj); + c.add('c1', [mkCs('http://x/a', '1', { concept: [{ code: 'b' }] })]); + expect(loaded.jsonObj).toEqual(original.jsonObj); + expect(loaded.expansionKeyHash()).toBe(original.expansionKeyHash()); + }); + test('a resource that has gone by the time it is used fails like a dead cache-id', () => { const c = make(); c.set('c1', [mkCs('http://x/a', '1')]); diff --git a/tx/resource-cache-sqlite.js b/tx/resource-cache-sqlite.js index 31627a12..d9bbb57f 100644 --- a/tx/resource-cache-sqlite.js +++ b/tx/resource-cache-sqlite.js @@ -386,8 +386,9 @@ class SqliteResourceCache extends ResourceCache { * The resources held under a cache-id: a fresh set of objects every call, nothing * shared with other requests. Wrappers are stand-ins that are only built when the * request uses them (see lazyResource). The session row and the index are read in - * one read transaction, so a concurrent write by another process is seen entirely - * or not at all. + * one read transaction. Later lazy reads check the indexed content hash: if another + * process replaced a resource meanwhile, this request fails so the client can retry + * from a fresh index, rather than mixing an old expansion key with new content. */ get(cacheId) { let session = null; @@ -441,6 +442,11 @@ class SqliteResourceCache extends ResourceCache { `The cache '${cacheId}' ${t ? 'was ' + t.reason : 'went away'} while this request was using it, so ${row.key.replace(/\|/g, ' ')} is no longer available`, 'cache-id-unknown', 404); } + if (c.hash !== row.hash || c.fhir_version !== row.fhir_version) { + throw new Issue('error', 'conflict', null, 'CACHE_ID_CHANGED_DURING_REQUEST', + `The cache '${cacheId}' changed while this request was using it. Retry the request`, + null, 409); + } return this._build({ kind: 'wrapper', resource_type: row.resource_type, fhir_version: c.fhir_version, json: c.json, hash: c.hash }); } From aceae5010b293f290f41ecc8e578d2f319e05d71 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Wed, 4 Nov 2026 10:52:55 -0600 Subject: [PATCH 22/23] Test VSAC database settings for both cluster roles Check actual WAL mode and busy timeout on fresh databases constructed by leader and follower providers. This guards the constructor-order conflict resolution when combining cluster-lite with VSAC on demand. Both cases pass on cluster-lite; a pre-existing WAL file cannot hide a missing shared option. Co-Authored-By: GPT-6 --- tests/vs/vs-vsac-cluster-settings.test.js | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/vs/vs-vsac-cluster-settings.test.js diff --git a/tests/vs/vs-vsac-cluster-settings.test.js b/tests/vs/vs-vsac-cluster-settings.test.js new file mode 100644 index 00000000..1e39c934 --- /dev/null +++ b/tests/vs/vs-vsac-cluster-settings.test.js @@ -0,0 +1,28 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { VSACValueSetProvider } = require('../../tx/vs/vs-vsac'); + +describe('VSAC database settings follow the cluster role', () => { + test.each(['leader', 'follower'])('%s opens a fresh database with WAL and a busy timeout', async cluster => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vsac-role-')); + const provider = new VSACValueSetProvider({ apiKey: 'test', cluster }, {}); + // No sync or external service: exercise the connection constructed by the provider, + // using an empty database unique to this test. A previously WAL-enabled file could + // otherwise hide a missing shared option in the constructor. + provider.database.dbPath = path.join(dir, 'vsac.db'); + try { + await provider.database.create(); + const db = await provider.database._getWriteConnection(); + const get = sql => new Promise((resolve, reject) => { + db.get(sql, (error, row) => error ? reject(error) : resolve(row)); + }); + expect(await get('PRAGMA journal_mode')).toEqual({ journal_mode: 'wal' }); + expect(await get('PRAGMA busy_timeout')).toEqual({ timeout: 5000 }); + expect(provider.database.shared).toBe(true); + } finally { + await provider.database.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); From d32494e217c12771ceec0aa6b4395273b488ba82 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Wed, 4 Nov 2026 11:15:10 -0600 Subject: [PATCH 23/23] Document shared cache and closure conflict responses Describe the rare client-visible 409s when a lazy resource is replaced or a closure table resets during resync, and distinguish them from internal retries of closure adds. Co-Authored-By: GPT-6 --- cluster.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cluster.md b/cluster.md index 987c9e40..88145fc3 100644 --- a/cluster.md +++ b/cluster.md @@ -184,6 +184,11 @@ transactions in WAL mode, so readers don't wait for them. `last_used` is written running it twice does no harm. An unknown cache-id gets a message that doesn't offer a restart or a routing mistake as the reason, since neither can be the cause. +If another request replaces a resource after its index was read but before its content is +loaded, the request returns `409` (`CACHE_ID_CHANGED_DURING_REQUEST`). Retry the whole +request to read the new contents together. This prevents using new content under an old +expansion-cache key; unchanged resources and additions of unrelated resources still work. + `cacheAffinity: true` keeps the prefix routing on top. The expansion cache is still per process, so a client that stays on one process finds its expansions there; without affinity, each process does the first expansion again. The file must be on a local disk. @@ -206,6 +211,12 @@ same name, and for the table limit, in the same transaction as the insert. A res only entries up to the version it read. The master alone drops stale tables (`retentionDays`). +Shared table IDs are never reused after reset or disposal, including across restarts; an +existing database is upgraded in place. An add that races a reset retries internally with +the new configuration and fresh lookup state. A resync interrupted by reset returns `409` +(`CLOSURE_TABLE_CHANGED`); retry against the new table, whose version starts at zero. If the +table was disposed without being recreated, the existing unknown-table `404` applies. + This spreads the subsumption work of `$closure` over the processes. When many processes add to the same table at once, some adds are worked out more than once.