diff --git a/README.md b/README.md index cb4d10d9..3179e9c4 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,9 @@ npm start The server will be available at `http://localhost:{port}` using the port specified in the config. In the production servers listed above, the server always sits behind an NGINX server which manages SSL, security, rate limiting etc. See [nginx.md](nginx.md) for configuration advice, including -running several servers behind one front door. +running several servers behind one front door. To use more than one CPU core for terminology on +one machine, see [cluster.md](cluster.md): a master and several tx workers, behind nginx or +`node cluster.js`. ## Testing diff --git a/cluster.js b/cluster.js new file mode 100644 index 00000000..3eaa727b --- /dev/null +++ b/cluster.js @@ -0,0 +1,405 @@ +#!/usr/bin/env node + +// +// Copyright 2025, Health Intersections Pty Ltd (http://www.healthintersections.com.au) +// +// Licensed under BSD-3: https://opensource.org/license/bsd-3-clause +// + +/** + * An optional launcher: one master and N terminology workers on one machine, behind one port + * (see cluster.md). It does what the nginx configuration in nginx.md does, for those who + * don't run nginx in front: + * + * - starts the master (node server.js --role=master), waits until it answers /health, then + * starts the workers (--role=tx-worker) one after another, each once the one before is + * ready, each with its own instance code and port + * - routes: a request with X-Cache-Id to the process that issued it (by the id's prefix); + * $closure, the syndication feed, the source info pages and everything that isn't a tx + * endpoint to the master; any other tx request to the tx process with the fewest + * requests outstanding + * - restarts a process that exits, waiting longer each time it fails quickly (up to a minute) + * - on SIGTERM or SIGINT, stops taking connections, lets the requests in progress finish, + * then stops the processes (each shuts down as a single server does) + * + * node cluster.js [--workers=N] [--port=P] [--master-serves-tx=false] + */ + +const http = require('http'); +const path = require('path'); +const fs = require('fs'); +const { spawn } = require('child_process'); + +const HEALTH_POLL_MS = 250; +const MAX_BACKOFF_MS = 60000; +// An idle keep-alive connection to a process is closed by the launcher after this long: +// below the processes' own keepAliveTimeout (65 s with library/keep-alive.js, Node's default +// of 5 s on a server without it), so the launcher never sends a request down a connection the +// process is closing at that moment (a reset, which the client would see as a 502) +const UPSTREAM_IDLE_MS = 4000; +// a process that ran this long before exiting is restarted at once +const HEALTHY_RUN_MS = 60000; +const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'trailer', 'expect']; + +/** + * The settings, from the flags, then the environment, then config.json (server.cluster). + * @param {string[]} argv - process.argv.slice(2) + * @param {Object} env - process.env + * @param {Object} config - config.json + */ +function loadSettings(argv, env, config) { + const flags = {}; + for (const a of argv) { + const m = /^--([a-z-]+)=(.*)$/.exec(a); + if (m) { + flags[m[1]] = m[2]; + } + } + const cluster = (config.server && config.server.cluster) || {}; + const tx = (config.modules && config.modules.tx) || {}; + if (!tx.enabled) { + throw new Error('cluster.js runs terminology workers: the tx module must be enabled in config.json'); + } + const port = Number(flags.port || env.PORT || (config.server && config.server.port) || 3000); + const workers = Number(flags.workers || env.FHIRSMITH_WORKERS || cluster.workers || 2); + const internalPort = Number(flags['internal-port'] || cluster.internalPort || port + 1); + const masterCode = tx.instanceCode || 'm0'; + const servesTx = flags['master-serves-tx'] !== undefined ? flags['master-serves-tx'] !== 'false' : cluster.masterServesTx !== false; + const procs = [{ name: 'master', role: 'master', code: masterCode, port: internalPort, servesTx }]; + for (let i = 1; i <= workers; i++) { + const code = 'tx' + i; + if (code === masterCode) { + throw new Error(`modules.tx.instanceCode '${masterCode}' is also the code of a worker: choose another`); + } + procs.push({ name: 'worker ' + code, role: 'tx-worker', code, port: internalPort + i, servesTx: true }); + } + // the options (cluster.md, "Options"): with the shared cache, any process can answer any + // cache-id, so by default a cached request goes to the least loaded; cacheAffinity: true + // keeps sending it to the process that issued it while that one is up + const sharedCache = cluster.sharedCache === true; + return { + port, + procs, + txPaths: (tx.endpoints || []).map(e => String(e.path).replace(/\/+$/, '')), + drainMs: Number(cluster.drainSeconds || 30) * 1000, + routing: { cacheAffinity: !sharedCache || cluster.cacheAffinity === true, sharedClosure: cluster.sharedClosure === true } + }; +} + +/** + * Where a request goes, by the rules in nginx.md. + * @param {string} url - the request target (path and query) + * @param {Object} headers - the request headers (lower case names) + * @param {string[]} txPaths - the tx endpoint paths, e.g. ['/r4', '/r5'] + * @param {Object} [routing] - from loadSettings: cacheAffinity (default true) sends a + * request with a cache-id to the process that issued it; sharedClosure sends $closure + * to any tx process rather than the master + * @returns {{to: 'master'}|{to: 'owner', code: string}|{to: 'tx'}} + */ +function classify(url, headers, txPaths, routing = {}) { + let p = String(url).split('?')[0]; + try { + p = decodeURIComponent(p); + } catch { + // leave it as it is + } + const ep = txPaths.find(e => p === e || p.startsWith(e + '/')); + if (!ep) { + return { to: 'master' }; + } + const rest = p.substring(ep.length); + // the closure tables (unless shared) and the syndication store are the master's; the + // source info pages carry the VSAC resync form, which writes the VSAC database + if (/^\/\$closure\/?$/.test(rest)) { + // closure never uses a cache-id, so the least loaded, whatever the header says + return routing.sharedClosure ? { to: 'tx' } : { to: 'master' }; + } + if (/^\/syndication(\/|$)/.test(rest) || rest.startsWith('/info/')) { + return { to: 'master' }; + } + if (routing.cacheAffinity === false) { + return { to: 'tx' }; + } + const m = /^([A-Za-z0-9]{1,16})\./.exec(headers['x-cache-id'] || ''); + return m ? { to: 'owner', code: m[1] } : { to: 'tx' }; +} + +/** + * The process to send a request to, or null if none is ready. + * @param {Object} route - from classify + * @param {Object[]} procs - {code, role, servesTx, ready, outstanding} + */ +let rotation = 0; +function pickTarget(route, procs) { + if (route.to === 'master') { + return procs.find(p => p.role === 'master' && p.ready) || null; + } + if (route.to === 'owner') { + const owner = procs.find(p => p.code === route.code && p.ready && p.servesTx); + if (owner) { + return owner; + } + // the owner is down (or the code is unknown): any tx process answers. Without the + // shared cache that is cache-id-unknown, which the client recovers from by starting a + // new cache; with it, the cache is there + } + const candidates = procs.filter(p => p.servesTx && p.ready); + if (candidates.length === 0) { + return null; + } + rotation = (rotation + 1) % candidates.length; + let best = null; + for (let i = 0; i < candidates.length; i++) { + const c = candidates[(rotation + i) % candidates.length]; + if (!best || c.outstanding < best.outstanding) { + best = c; + } + } + return best; +} + +function isLoopback(address) { + return address === '127.0.0.1' || address === '::1'; +} + +/** + * @param {Object} settings - from loadSettings, plus optionally + * command(proc): [executable, args] to start a process (tests), healthPath, log + */ +function createLauncher(settings) { + const log = settings.log || ((msg) => console.log(`[cluster] ${msg}`)); + const healthPath = settings.healthPath || '/health'; + const command = settings.command || ((proc) => [process.execPath, [ + ...new Set(['--expose-gc', ...process.execArgv]), path.join(__dirname, 'server.js'), + `--role=${proc.role}`, `--instance-code=${proc.code}`, `--port=${proc.port}`]]); + const procs = settings.procs.map(p => ({ + ...p, child: null, ready: false, outstanding: 0, failures: 0, startedAt: 0, restartTimer: null, + // timeout: a connection idle in the pool this long is closed (see UPSTREAM_IDLE_MS); a + // request in progress is not affected + agent: new http.Agent({ keepAlive: true, maxSockets: 256, timeout: settings.upstreamIdleMs || UPSTREAM_IDLE_MS }) + })); + let server = null; + let stopping = false; + let onIdle = null; + + function totalOutstanding() { + return procs.reduce((n, p) => n + p.outstanding, 0); + } + + function startProcess(proc) { + return new Promise((resolve) => { + if (stopping) { + return resolve(false); + } + const [cmd, args] = command(proc); + proc.startedAt = Date.now(); + const child = spawn(cmd, args, { stdio: 'inherit', env: process.env }); + proc.child = child; + let poll = null; + child.on('exit', (code, signal) => { + clearTimeout(poll); + proc.ready = false; + proc.child = null; + resolve(false); + if (stopping) { + return; + } + proc.failures = Date.now() - proc.startedAt > HEALTHY_RUN_MS ? 1 : proc.failures + 1; + const delay = Math.min(MAX_BACKOFF_MS, 1000 * 2 ** (proc.failures - 1)); + log(`${proc.name} exited (${signal || code}); restarting in ${delay / 1000} s`); + proc.restartTimer = setTimeout(() => startProcess(proc), delay); + }); + const check = () => { + const req = http.get({ host: '127.0.0.1', port: proc.port, path: healthPath, agent: false }, (res) => { + res.resume(); + if (res.statusCode === 200 && proc.child === child) { + proc.ready = true; + log(`${proc.name} (${proc.code}) ready on port ${proc.port}`); + resolve(true); + } else { + poll = setTimeout(check, HEALTH_POLL_MS); + } + }); + req.on('error', () => { + if (proc.child === child) { + poll = setTimeout(check, HEALTH_POLL_MS); + } + }); + }; + poll = setTimeout(check, HEALTH_POLL_MS); + }); + } + + function proxy(req, res) { + const target = pickTarget(classify(req.url, req.headers, settings.txPaths, settings.routing), procs); + if (!target) { + return fail(res, 503, 'No server process is ready to take this request; try again shortly'); + } + target.outstanding++; + let finished = false; + const finish = () => { + if (!finished) { + finished = true; + target.outstanding--; + if (onIdle && totalOutstanding() === 0) { + onIdle(); + } + } + }; + const headers = { ...req.headers }; + for (const h of HOP_BY_HOP) { + delete headers[h]; + } + const client = (req.socket.remoteAddress || '').replace(/^::ffff:/, ''); + headers['x-forwarded-for'] = headers['x-forwarded-for'] ? `${headers['x-forwarded-for']}, ${client}` : client; + if (!headers['x-forwarded-proto'] || !isLoopback(client)) { + headers['x-forwarded-proto'] = 'http'; + } + const upstream = http.request({ + host: '127.0.0.1', port: target.port, method: req.method, path: req.url, headers, agent: target.agent + }, (ures) => { + const out = { ...ures.headers }; + for (const h of HOP_BY_HOP) { + delete out[h]; + } + if (stopping) { + out.connection = 'close'; + } + res.writeHead(ures.statusCode, ures.statusMessage, out); + ures.pipe(res); + ures.on('error', () => res.destroy()); + }); + upstream.on('error', (err) => { + if (!res.headersSent) { + fail(res, 502, `The ${target.name} process did not answer (${err.code || err.message})`); + } else { + res.destroy(); + } + finish(); + }); + res.on('close', () => { + if (!res.writableFinished) { + upstream.destroy(); + } + finish(); + }); + req.pipe(upstream); + } + + function fail(res, status, text) { + res.writeHead(status, { 'Content-Type': 'application/fhir+json', 'Retry-After': '5', ...(stopping ? { Connection: 'close' } : {}) }); + res.end(JSON.stringify({ + resourceType: 'OperationOutcome', + issue: [{ severity: 'error', code: 'transient', details: { text }, diagnostics: text }] + })); + } + + async function start() { + const master = procs.find(p => p.role === 'master'); + // the master first, and on its own: on a new data directory it builds the package + // caches and creates the VSAC database, which processes starting together would corrupt. + // (If it exits before it is ready, it is started again, as any process is.) + log(`starting the master (${master.code}) on port ${master.port}`); + void startProcess(master); + await new Promise(resolve => { + const t = setInterval(() => { + if (master.ready || stopping) { + clearInterval(t); + resolve(); + } + }, HEALTH_POLL_MS); + }); + if (stopping) { + return; + } + server = http.createServer(proxy); + server.keepAliveTimeout = 65000; + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(settings.port, resolve); + }); + log(`listening on port ${settings.port}; starting ${procs.length - 1} worker(s)`); + // one after another, each once the one before is ready (or has exited, and is being + // restarted): a worker only reads the caches the master built, but starting them together + // would put every worker's load (and its memory peak) at the same moment + // (in the background: start() returns once the master is ready and the port is open) + void (async () => { + for (const p of procs) { + if (p !== master && !stopping) { + await startProcess(p); + } + } + })(); + } + + /** + * Stop taking connections, wait for the requests in progress (up to drainMs), then stop + * every process (SIGTERM, then SIGKILL after 30 s). + */ + async function stop() { + if (stopping) { + return; + } + stopping = true; + for (const p of procs) { + clearTimeout(p.restartTimer); + } + if (server) { + server.close(); + server.closeIdleConnections(); + const idle = setInterval(() => server.closeIdleConnections(), 100); + if (totalOutstanding() > 0) { + log(`draining: ${totalOutstanding()} request(s) in progress`); + await new Promise(resolve => { + const timer = setTimeout(resolve, settings.drainMs); + onIdle = () => { + clearTimeout(timer); + resolve(); + }; + }); + } + clearInterval(idle); + server.closeAllConnections(); + } + await Promise.all(procs.filter(p => p.child).map(p => new Promise(resolve => { + const child = p.child; + const kill = setTimeout(() => child.kill('SIGKILL'), 30000); + child.once('exit', () => { + clearTimeout(kill); + resolve(); + }); + child.kill('SIGTERM'); + }))); + for (const p of procs) { + p.agent.destroy(); + } + log('stopped'); + } + + return { start, stop, procs, get server() { return server; } }; +} + +async function main() { + const folders = require('./library/folder-setup'); + let launcher; + try { + const config = JSON.parse(fs.readFileSync(folders.filePath('config.json'), 'utf8')); + launcher = createLauncher(loadSettings(process.argv.slice(2), process.env, config)); + } catch (error) { + console.error(`cluster.js: ${error.message}`); + process.exit(1); + } + const shutdown = () => launcher.stop().then(() => process.exit(0)); + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); + await launcher.start(); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} + +module.exports = { loadSettings, classify, pickTarget, createLauncher }; diff --git a/cluster.md b/cluster.md new file mode 100644 index 00000000..88145fc3 --- /dev/null +++ b/cluster.md @@ -0,0 +1,267 @@ +# A master and terminology workers on one machine + +One Node.js process runs JavaScript on one CPU core. A terminology server under load from +several IG Publisher builds at once can use more. FHIRsmith can run as one **master** and +several **tx-workers** on the same machine and data directory, with a front door that sends +each request to the right process: + +```bash +node cluster.js # the master, 2 workers, and a proxy on the configured port +``` + +or with nginx in front of separately started processes ([below](#with-nginx)). + +`node server.js` with no `--role` is unchanged: one process, every module, as before. + +* [Roles](#roles) +* [What runs where](#what-runs-where) +* [Routing](#routing) +* [The launcher (cluster.js)](#the-launcher-clusterjs) +* [With nginx](#with-nginx) +* [Options](#options) +* [Memory](#memory) +* [Limits of this arrangement](#limits-of-this-arrangement) + +## Roles + +```bash +node server.js --role=master --instance-code=m0 --port=3001 +node server.js --role=tx-worker --instance-code=tx1 --port=3002 +node server.js --role=tx-worker --instance-code=tx2 --port=3003 +``` + +| Flag | Environment | Config | Meaning | +|---|---|---|---| +| `--role` | `FHIRSMITH_ROLE` | | `master` or `tx-worker`. None: an ordinary single server. | +| `--instance-code` | `FHIRSMITH_INSTANCE_CODE` | `modules.tx.instanceCode` (master only) | Prefixed onto every cache-id the process issues (see [nginx.md](nginx.md#instance-codes)). Each process needs its own, 1-16 letters and digits. A worker must be given one, since the config's is shared with the master. | +| `--port` | `PORT` | `server.port` | The port the process listens on. | + +All the processes use the same data directory and the same `config.json`. The role decides +which parts of the configuration each one acts on. + +## What runs where + +| | master | tx-worker | +|---|---|---| +| tx module (`/r4`, `/r5` and the other endpoints) | yes (may also serve tx requests) | yes | +| Every other module (packages, xig, registry, publisher, token, shl, ips, ...) | yes | no | +| `$closure` (the closure tables) | yes | no: routed to the master (yes, with [`sharedClosure`](#options)) | +| Syndication feed (the store) | yes | no: routed to the master | +| VSAC sync, and the resync form on `/info/vsac` | yes | no: reads what the master wrote | +| Package crawler, XIG download, registry crawl, daily report, npm audit | yes | no (those modules don't run) | +| IANA media types registry (`terminology-cache/media-types.xml`), downloaded at each load | yes | no: reads the master's copy (none: a warning, and no `registered` filter) | +| Statistics database, uptime history | yes | no: in memory only | +| `$cache-control` caches | its own, in memory (shared, with [`sharedCache`](#options)) | its own, in memory (shared, with [`sharedCache`](#options)) | +| Expansion cache, CodeSystem list refresh | its own | its own | +| Log file | `logs/server-.log` | the same file (lines interleave) | + +The master loads the terminologies like any tx server: `$closure` needs subsumption. + +**VSAC.** The master syncs, and when a sync has written its value sets it adds one to a +generation counter in the VSAC database. A worker remembers the generation its value set list +was loaded at. When a request uses the VSAC value sets, the worker reads the counter (one read +by key, at most once a second) and reloads the list in the background if it has moved, so a +sync reaches every worker about a second after it finishes. The database is in WAL mode with a +5 second busy timeout, so a worker's reload doesn't make the master's writes fail. + +**The package caches are the master's.** The value set and concept map caches in each +package folder (`.valuesets-2.db`, `.conceptmaps-2.db`) are built, rebuilt and migrated only +by the master; it has finished every one before it answers `/health`. A worker opens them +read-only, and never creates, rebuilds or migrates one: if one is missing, incomplete or still +needs a migration (a worker started on its own, before the master), the worker waits for the +master to finish it, and after 5 minutes (`FHIRSMITH_CACHE_WAIT_MS`) stops with an error that +says so. Starting the master first is what makes the wait unnecessary, not what keeps the +workers from writing. + +The same goes for the caches of a `folder:` or `file:` library source +(`terminology-cache/folder-source-master-`, one folder for every process). Each time the +master loads the library it rebuilds them in a temporary sibling folder and renames it into +place, so a running worker never finds one half built; a worker reads the new one when it next +starts. Only the master removes the folders of sources no longer loaded. + +**Nothing else is shared.** The workers run no module but tx, so they never read what the +package crawler, the XIG download or the registry crawl write. + +## Routing + +Whatever the front door, the rules are the same: + +1. `$closure` (in any endpoint, e.g. `/r4/$closure`), the syndication feed + (`/r4/syndication...`) and the source info pages (`/r4/info/...`) go to the **master**. +2. Every path outside the tx endpoints (`/`, `/packages`, `/xig`, `/dashboard`, `/health`, ...) + goes to the **master**. +3. A tx request with an `X-Cache-Id` goes to the process whose code prefixes the id + (`tx2.4f0c...` to `tx2`). If that process is down, or the prefix is unknown, it goes to any + tx process, which answers `cache-id-unknown`; the client starts a new cache. +4. Any other tx request goes to the tx process with the fewest requests outstanding: the + workers, and the master too unless it is set not to serve tx. + +The [options](#options) change rules 1 and 3: with `sharedClosure`, `$closure` goes by rule 4; +with `sharedCache`, so does a request with a cache-id (unless `cacheAffinity` is on). + +## The launcher (cluster.js) + +`node cluster.js` reads `config.json` from the data directory and: + +* starts the master and waits until it answers `/health`. On a new data directory the master + builds the package caches and creates the VSAC database +* listens on the configured port, and starts the workers one after another, each once the one + before answers `/health` +* proxies each request by the rules above, over keep-alive connections to each process. It + adds the client address to `X-Forwarded-For`, so the servers' logs and rate limits see the + real client (its own address is a loopback one, which `server.trustProxy` trusts by default) +* starts again a process that exits. One that keeps failing waits longer each time: 1 s, + 2 s, 4 s ... up to a minute +* on `SIGTERM` or `SIGINT`: stops taking connections, closes idle keep-alive connections, + waits for the requests in progress (up to `drainSeconds`), then sends each process + `SIGTERM` (each shuts down as a single server does; `SIGKILL` after 30 seconds), and exits + +```json +"server": { + "port": 3000, + "cluster": { "workers": 4, "masterServesTx": true, "internalPort": 3001, "drainSeconds": 30 } +} +``` + +| Setting | Flag | Default | Meaning | +|---|---|---|---| +| `server.port` | `--port`, `PORT` | 3000 | The port clients use. | +| `server.cluster.workers` | `--workers`, `FHIRSMITH_WORKERS` | 2 | How many tx-workers. | +| `server.cluster.masterServesTx` | `--master-serves-tx=false` | true | Whether the master also takes ordinary tx requests. | +| `server.cluster.internalPort` | `--internal-port` | port + 1 | The master's port; worker *n* gets this + *n*. Firewall them. | +| `server.cluster.drainSeconds` | | 30 | How long the requests in progress get on shutdown. | +| `modules.tx.instanceCode` | | `m0` | The master's instance code. The workers are `tx1`, `tx2`, ... | + +Under systemd, the launcher and its processes are one unit and one cgroup: + +```ini +[Service] +WorkingDirectory=/opt/fhirsmith +Environment=FHIRSMITH_DATA_DIR=/var/lib/fhirsmith +ExecStart=/usr/bin/node cluster.js +KillMode=mixed +MemoryMax=16G +``` + +## With nginx + +nginx can be the front door instead: start the processes yourself (master first, workers once +it answers `/health`), and use the configuration in +[nginx.md](nginx.md#a-master-and-tx-workers-on-one-machine), which applies the same rules. + +## Options + +Two options in `server.cluster` change what the processes share. Both are off by default, and +both only apply with a role: `node server.js` with no role ignores them. + +```json +"server": { + "cluster": { "workers": 4, "sharedCache": true, "sharedClosure": true } +} +``` + +| Setting | Default | Meaning | +|---|---|---| +| `server.cluster.sharedCache` | false | The `$cache-control` caches go in one SQLite file, `databases/resource-cache.db`, that every process reads and writes. | +| `server.cluster.cacheAffinity` | false | With `sharedCache`: still send a request with a cache-id to the process that issued it, while it is up. | +| `server.cluster.sharedCacheMaxMB` | 0 (none) | With `sharedCache`: the most the file holds. Past it, the caches least recently used are dropped whole. | +| `server.cluster.sharedClosure` | false | Every tx process answers `$closure`, on the one closure database. | +| `modules.tx.closure.writeRetries` | 10 | With `sharedClosure`: how many times an add is worked out again when another process changed the table first. | + +### Shared cache + +Without it, a cache lives in the memory of the process that issued it, and the routing has to +send every request that uses it there. A restarted process has lost its caches, and its +clients get `cache-id-unknown` and start new ones. The load a client brings stays on one +process. + +With it, any process can answer any cache-id, and caches survive a restart of any process (or +of all of them). The launcher sends a request with a cache-id to the least-loaded tx process, +like any other. The cost is a read of the file on each use of a cache: a request reads the +list of what the cache holds, and only loads the resources it uses. Writes are short +transactions in WAL mode, so readers don't wait for them. `last_used` is written at most every +30 seconds per cache. The idle sweep (`modules.tx.cacheTimeout`) runs in every process, and +running it twice does no harm. An unknown cache-id gets a message that doesn't offer a restart +or a routing mistake as the reason, since neither can be the cause. + +If another request replaces a resource after its index was read but before its content is +loaded, the request returns `409` (`CACHE_ID_CHANGED_DURING_REQUEST`). Retry the whole +request to read the new contents together. This prevents using new content under an old +expansion-cache key; unchanged resources and additions of unrelated resources still work. + +`cacheAffinity: true` keeps the prefix routing on top. The expansion cache is still per +process, so a client that stays on one process finds its expansions there; without affinity, +each process does the first expansion again. The file must be on a local disk. + +### Shared closure + +Without it, the master answers every `$closure` request, and its per-table lock keeps the +calls on one table in order. + +With it, any tx process answers `$closure`, against the same `databases/closure.db`, and the +launcher sends `$closure` to the least-loaded tx process. Within a process the per-table lock +still queues the calls on a table. Across processes the writes are **checked, not locked**. A +call works out its add against the table as it read it (its version *n*). It writes in one +`BEGIN IMMEDIATE` transaction that first checks the table is still at version *n*, and was +not reset meanwhile. If it isn't, nothing is written: the call reads the table again and works +the add out again, with the code lookups and subsumption tests it already did remembered. It +waits a few milliseconds, with jitter, between tries. After `writeRetries` tries it answers +`429` (`CLOSURE_TABLE_BUSY`), as a full queue does. Creating a table checks for one of the +same name, and for the table limit, in the same transaction as the insert. A resync reports +only entries up to the version it read. The master alone drops stale tables +(`retentionDays`). + +Shared table IDs are never reused after reset or disposal, including across restarts; an +existing database is upgraded in place. An add that races a reset retries internally with +the new configuration and fresh lookup state. A resync interrupted by reset returns `409` +(`CLOSURE_TABLE_CHANGED`); retry against the new table, whose version starts at zero. If the +table was disposed without being recreated, the existing unknown-table `404` applies. + +This spreads the subsumption work of `$closure` over the processes. When many processes add +to the same table at once, some adds are worked out more than once. + +### What changes, by combination + +| `sharedCache` | `sharedClosure` | A request with a cache-id goes to | `$closure` goes to | A restarted process | +|---|---|---|---|---| +| off | off | the process that issued it (if it is down: any, which answers `cache-id-unknown`) | the master | has lost its caches | +| on | off | the least-loaded tx process (the issuer, with `cacheAffinity`) | the master | still has every cache | +| off | on | the process that issued it | the least-loaded tx process | has lost its caches | +| on | on | the least-loaded tx process (the issuer, with `cacheAffinity`) | the least-loaded tx process | still has every cache | + +The syndication feed, the `/info` pages and everything outside the tx endpoints go to the +master in every combination. For nginx, see +[nginx.md](nginx.md#with-the-shared-cache). + +## Memory + +With a role, the memory guard measures the whole cgroup, since the kernel's limit is on all the +processes in it together: the cgroup's private memory (`memory.stat` `anon`; cgroup v1 +`total_rss`). The page cache (the terminology and SQLite files) isn't counted: the kernel drops +it when it needs the room. + +* Above `server.memory.rejectFraction` (85%) of the limit, **every** process refuses new + requests with 503 and `Retry-After`. +* Above `abortFraction` (92%), only the process with the **largest RSS** in the cgroup aborts + its operations (422 too-costly). It is the one most likely to hold what took the group over, + and aborting there frees the most; the others carry on. + +This needs the limit on the processes' own cgroup: `MemoryMax` on the unit that runs +`cluster.js`, or a container limit. If the limit is only on an enclosing slice, or there is +none, each process measures its own RSS against the limit, as a single server does. If you +start the processes as separate systemd units, give each its own `MemoryMax`: each then guards +its own. + +Each process is a whole terminology server: it loads the library, so N workers and a master +use roughly N+1 times the memory of one server. Size the limit for that. + +## Limits of this arrangement + +* **Caches are per process** (unless [`sharedCache`](#options) is on). A `$cache-control` + cache lives in the process that issued it; a restarted process loses its caches, and + clients start new ones, as after any restart. +* **Statistics are per process.** `/` and `/dashboard` show the master. The workers' request + counts are not in the statistics database. +* **Everything the master runs is a single point.** While the master restarts, `$closure` + (unless [`sharedClosure`](#options) is on), the non-tx modules and the dashboard answer 503 + through the launcher; tx requests carry on. diff --git a/library/cgroup-memory.js b/library/cgroup-memory.js index dd8b0ee2..f46a8d09 100644 --- a/library/cgroup-memory.js +++ b/library/cgroup-memory.js @@ -110,4 +110,95 @@ function readCgroupMemoryLimit(opts = {}) { return { limit: 0, source: null }; } -module.exports = { readCgroupMemoryLimit }; +/** + * The directory of this process's own cgroup (memory controller), and whether it is v2 or + * v1. null when there is none to be found (not Linux, or no /proc). + * @param {Object} [opts] - procFile, cgroupRoot (tests) + * @returns {{dir: string, version: 2|1}|null} + */ +function ownCgroupDir(opts = {}) { + const procFile = opts.procFile || '/proc/self/cgroup'; + const cgroupRoot = opts.cgroupRoot || '/sys/fs/cgroup'; + let lines; + try { + lines = fs.readFileSync(procFile, 'utf8').split('\n').filter(l => l.trim() !== ''); + } catch { + return null; + } + const v2 = lines.find(l => l.startsWith('0::')); + if (v2) { + const dir = path.join(cgroupRoot, path.posix.normalize(v2.substring(3) || '/')); + if (fs.existsSync(path.join(dir, 'memory.stat'))) { + return { dir, version: 2 }; + } + } + for (const l of lines) { + const parts = l.split(':'); + if (parts.length >= 3 && parts[1].split(',').includes('memory')) { + const dir = path.join(cgroupRoot, 'memory', path.posix.normalize(parts.slice(2).join(':') || '/')); + if (fs.existsSync(path.join(dir, 'memory.stat'))) { + return { dir, version: 1 }; + } + } + } + return null; +} + +/** + * The private memory of everything in a cgroup: what the kernel can't take back without + * killing a process. The page cache (the terminology and SQLite files) isn't counted: the + * kernel drops it when it needs the room. v2: memory.stat 'anon'; v1: 'total_rss', else 'rss'. + * @param {{dir: string, version: 2|1}} cg - from ownCgroupDir + * @returns {number} bytes, or 0 if it can't be read + */ +function readCgroupAnon(cg) { + let text; + try { + text = fs.readFileSync(path.join(cg.dir, 'memory.stat'), 'utf8'); + } catch { + return 0; + } + const values = new Map(); + for (const line of text.split('\n')) { + const i = line.indexOf(' '); + if (i > 0) { + values.set(line.substring(0, i), Number(line.substring(i + 1))); + } + } + for (const key of cg.version === 2 ? ['anon'] : ['total_rss', 'rss']) { + const v = values.get(key); + if (Number.isFinite(v) && v >= 0) { + return v; + } + } + return 0; +} + +/** + * The RSS of each process in a cgroup, from /proc//status. Processes that have gone, + * or can't be read, are left out. + * @param {{dir: string}} cg - from ownCgroupDir + * @returns {Map} pid -> bytes + */ +function readCgroupProcessRss(cg) { + const result = new Map(); + let pids; + try { + pids = fs.readFileSync(path.join(cg.dir, 'cgroup.procs'), 'utf8').split('\n').filter(Boolean).map(Number); + } catch { + return result; + } + for (const pid of pids) { + try { + const m = /^VmRSS:\s+(\d+) kB/m.exec(fs.readFileSync(`/proc/${pid}/status`, 'utf8')); + if (m) { + result.set(pid, Number(m[1]) * 1024); + } + } catch { + // gone + } + } + return result; +} + +module.exports = { readCgroupMemoryLimit, ownCgroupDir, readCgroupAnon, readCgroupProcessRss }; diff --git a/library/memory-guard.js b/library/memory-guard.js index 232f488e..aaba45c9 100644 --- a/library/memory-guard.js +++ b/library/memory-guard.js @@ -13,13 +13,22 @@ * * Both are configurable in config.server.memory: { "rejectFraction": 0.85, "abortFraction": 0.92 } * + * A master and its tx-workers (--role, cluster.md) share one cgroup, and the kernel's limit is + * on all of them together, so with a role (config.server.memory.group) the measure is the + * group's: the cgroup's private memory (memory.stat anon), read by each process itself. Every + * process refuses new requests above rejectFraction; above abortFraction only the largest + * process in the cgroup aborts its operations, so one heavy request doesn't abort the work of + * every other process. This needs a cgroup with a limit of its own (MemoryMax on the unit or + * scope, a container limit); without one each process measures its own RSS, as one does. + * * @module library/memory-guard */ const os = require('os'); const v8 = require('v8'); const vm = require('vm'); -const { readCgroupMemoryLimit } = require('./cgroup-memory'); +const path = require('path'); +const { readCgroupMemoryLimit, ownCgroupDir, readCgroupAnon, readCgroupProcessRss } = require('./cgroup-memory'); const DEFAULT_REJECT_FRACTION = 0.85; const DEFAULT_ABORT_FRACTION = 0.92; @@ -33,6 +42,9 @@ const GC_INTERVAL_MS = 5000; // next, up to this: when most of RSS is outside the JS heap (the SNOMED and LOINC cache // buffers), collecting frees little, and every full GC stops the process const GC_MAX_INTERVAL_MS = 60000; +// the group measure reads a file: the in-operation check (about ten a request) reuses one +// at most this old +const GROUP_SAMPLE_MS = 100; const state = { limit: 0, @@ -42,7 +54,10 @@ const state = { rejectAt: 0, abortAt: 0, sampleTime: 0, - sampleRss: 0 + sampleRss: 0, + // a role (master, tx-worker): the cgroup measured, from ownCgroupDir; else null + group: null, + groupWanted: false }; function fraction(v, dflt) { @@ -57,11 +72,22 @@ function fraction(v, dflt) { function configure(opts = {}) { let limit = opts.limit; let source = limit ? 'configured' : null; + let cg = null; if (!limit) { - const cg = readCgroupMemoryLimit(); + cg = readCgroupMemoryLimit(); limit = cg.limit; source = cg.source; } + state.group = null; + state.groupWanted = !!opts.group; + if (opts.group) { + // only a cgroup that is the group's own, with the limit on it + const own = opts.cgroup !== undefined ? opts.cgroup : ownCgroupDir(); + const limitFile = opts.cgroup !== undefined ? null : (cg || readCgroupMemoryLimit()).source; + if (own && (opts.cgroup !== undefined || (limitFile && path.dirname(limitFile) === own.dir)) && readCgroupAnon(own) > 0) { + state.group = own; + } + } if (!limit) { limit = os.totalmem(); source = 'os.totalmem()'; @@ -78,7 +104,7 @@ function configure(opts = {}) { function rss(fresh = false) { const now = Date.now(); if (fresh || now - state.sampleTime > SAMPLE_MS) { - state.sampleRss = process.memoryUsage.rss(); + state.sampleRss = (state.group && readCgroupAnon(state.group)) || process.memoryUsage.rss(); state.sampleTime = now; } return state.sampleRss; @@ -132,10 +158,13 @@ function overAbortThreshold(place = 'unknown') { if (state.abortAt <= 0) { return 0; } - const r = rss(true); + const r = state.group ? rss(Date.now() - state.sampleTime > GROUP_SAMPLE_MS) : rss(true); if (r <= state.abortAt) { return 0; } + if (state.group && !largestInGroup()) { + return 0; + } aborts++; const now = Date.now(); if (now - lastAbortLog > 60000) { @@ -152,12 +181,37 @@ function overAbortThreshold(place = 'unknown') { return r; } +let largest = { time: 0, value: false }; + +/** + * A role: is this the process in the cgroup with the most RSS? (Looked at for at most + * GROUP_SAMPLE_MS.) It is the one most likely to hold what took the group over, and the one + * where aborting frees most. + */ +function largestInGroup() { + const now = Date.now(); + if (now - largest.time > GROUP_SAMPLE_MS) { + const own = process.memoryUsage.rss(); + let value = true; + for (const [pid, bytes] of readCgroupProcessRss(state.group)) { + if (pid !== process.pid && bytes > own) { + value = false; + } + } + largest = { time: now, value }; + } + return largest.value; +} + function gb(bytes) { return (bytes / 1024 / 1024 / 1024).toFixed(1); } function describe() { - return `Memory limit: ${gb(state.limit)} GB (from ${state.source}); ` + + const group = !state.groupWanted ? '' : state.group + ? `, for the whole cgroup ${state.group.dir} (its private memory; only its largest process aborts)` + : ', measured as this process\'s RSS (no cgroup limit of the service\'s own to measure as a group)'; + return `Memory limit: ${gb(state.limit)} GB (from ${state.source})${group}; ` + `new requests refused above ${gb(state.rejectAt)} GB (${Math.round(state.rejectFraction * 100)}%), ` + `operations aborted above ${gb(state.abortAt)} GB (${Math.round(state.abortFraction * 100)}%)`; } @@ -204,7 +258,9 @@ function middleware(opts = {}) { const before = state.sampleRss; const heap = heapUsed(); const gc = getGc(); - if (heap < before - state.rejectAt) { + // a group: every process refusing does this, so each need only free its share + const share = state.group ? Math.max(1, readCgroupProcessRss(state.group).size) : 1; + if (heap < (before - state.rejectAt) / share) { gcWait = Math.min(gcWait * 2, gcMaxInterval); if (log) { log.info(`Memory: no forced GC - the JS heap (${gb(heap)} GB) is less than the ${gb(before - state.rejectAt)} GB over the threshold; next look in ${secs(gcWait)}s`); @@ -260,4 +316,4 @@ function middleware(opts = {}) { configure(); -module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, _state: state }; +module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, largestInGroup, _state: state }; diff --git a/library/role.js b/library/role.js new file mode 100644 index 00000000..701661c4 --- /dev/null +++ b/library/role.js @@ -0,0 +1,126 @@ +/** + * Process roles, for running one master and several terminology workers on one data + * directory (see cluster.md). + * + * node server.js no role: everything exactly as before + * node server.js --role=master --instance-code=m0 --port=3001 + * node server.js --role=tx-worker --instance-code=tx1 --port=3002 + * + * The role also comes from FHIRSMITH_ROLE and the instance code from FHIRSMITH_INSTANCE_CODE + * (flags win). With no role, nothing here changes the configuration: server.js only calls + * applyRole when parseRole returns a role. + * + * - master: every configured module, every background job, $closure and the syndication + * feed. The VSAC provider syncs and, after each sync, bumps a generation counter that the + * workers watch. + * - tx-worker: the tx module only. No background jobs, no statistics database, no $closure + * or syndication store (those requests go to the master), and the VSAC provider only + * reads what the master's sync wrote. + * + * @module library/role + */ + +const ROLES = ['master', 'tx-worker']; +const FLAGS = ['role', 'instance-code', 'port']; + +/** + * The role this process was started with, or null for none. + * @param {string[]} argv - process.argv.slice(2) + * @param {Object} env - process.env + * @returns {{role: string, instanceCode: string|null, port: number|null}|null} + */ +function parseRole(argv = [], env = {}) { + const flags = {}; + for (let i = 0; i < argv.length; i++) { + const m = /^--([a-z-]+)(?:=(.*))?$/.exec(argv[i]); + if (m && FLAGS.includes(m[1])) { + flags[m[1]] = m[2] !== undefined ? m[2] : argv[++i]; + } + } + const role = flags.role || env.FHIRSMITH_ROLE || null; + if (!role) { + if (flags['instance-code'] !== undefined || flags.port !== undefined) { + throw new Error('--instance-code and --port are only used with --role (master or tx-worker)'); + } + return null; + } + if (!ROLES.includes(role)) { + throw new Error(`Unknown role '${role}': use master or tx-worker`); + } + let port = null; + if (flags.port !== undefined) { + port = Number(flags.port); + if (!Number.isInteger(port) || port <= 0 || port > 65535) { + throw new Error(`--port '${flags.port}' is not a port number`); + } + } + return { role, instanceCode: flags['instance-code'] || env.FHIRSMITH_INSTANCE_CODE || null, port }; +} + +/** + * Change the configuration (in place) for the role. + * @param {Object} config - the parsed config.json + * @param {{role: string, instanceCode: string|null}} role - from parseRole + * @returns {Object} config + */ +function applyRole(config, role) { + const tx = config.modules && config.modules.tx; + if (!tx || !tx.enabled) { + throw new Error(`--role=${role.role} needs the tx module enabled in config.json`); + } + if (role.role === 'tx-worker' && !role.instanceCode) { + // the config's code is the master's: every process needs its own + throw new Error('A tx-worker needs its own instance code: --instance-code or FHIRSMITH_INSTANCE_CODE'); + } + tx.instanceCode = role.instanceCode || tx.instanceCode; + if (!tx.instanceCode) { + throw new Error('The master needs an instance code: --instance-code, FHIRSMITH_INSTANCE_CODE or modules.tx.instanceCode'); + } + tx.clusterRole = role.role; + // the options in server.cluster (cluster.md, "Options"). Every process reads the same + // config.json, so they all agree; without a role these are never looked at + const options = (config.server && config.server.cluster) || {}; + for (const name of ['sharedCache', 'cacheAffinity', 'sharedClosure']) { + if (options[name] !== undefined && typeof options[name] !== 'boolean') { + throw new Error(`server.cluster.${name} must be true or false`); + } + } + if (options.sharedCache === true) { + tx.sharedCache = true; + const mb = options.sharedCacheMaxMB === undefined ? 0 : Number(options.sharedCacheMaxMB); + if (!Number.isFinite(mb) || mb < 0) { + throw new Error('server.cluster.sharedCacheMaxMB must be a number of megabytes (0: no cap)'); + } + tx.sharedCacheMaxBytes = Math.round(mb * 1024 * 1024); + } + if (options.sharedClosure === true) { + tx.sharedClosure = true; + } + if (tx.vsacCfg) { + tx.vsacCfg = { ...tx.vsacCfg, cluster: role.role === 'master' ? 'leader' : 'follower' }; + } + config.server = config.server || {}; + // the memory limit is the whole group's (master and workers in one cgroup) + config.server.memory = { ...(config.server.memory || {}), group: true }; + if (role.role === 'tx-worker') { + for (const [name, mod] of Object.entries(config.modules)) { + if (name !== 'tx' && mod) { + mod.enabled = false; + } + } + // the master keeps the statistics database, runs the npm audit and sends the report + config.stats = { ...(config.stats || {}), enabled: false }; + config.npmAudit = { ...(config.npmAudit || {}), enabled: false }; + config.dailyReport = { ...(config.dailyReport || {}), enabled: false }; + } + return config; +} + +function describeRole(role, config) { + const code = config.modules.tx.instanceCode; + return role.role === 'master' + ? `Role: master (instance code ${code}): all modules, background jobs, $closure` + : `Role: tx-worker (instance code ${code}): the tx module only, no background jobs`; +} + +module.exports = { parseRole, applyRole, describeRole, ROLES }; diff --git a/nginx.md b/nginx.md index c242552f..95df58f3 100644 --- a/nginx.md +++ b/nginx.md @@ -8,6 +8,7 @@ public servers (tx.fhir.org, packages.fhir.org and so on) use. * [Rate limiting](#rate-limiting) - approaches that suit terminology traffic * [Routing to multiple servers](#routing-to-multiple-servers) - running more than one instance behind a single front door * [Managing closure](#managing-closure) - where `$closure` requests go +* [A master and tx workers on one machine](#a-master-and-tx-workers-on-one-machine) - the same rules for processes started with `--role` ## General configuration @@ -397,3 +398,113 @@ the cache ends, and the table would silently change meaning. So `$closure` ignor about. An outage is better than that: clients see a failure and retry later. * **Moving the designated instance** means moving its closure database with it, then changing the upstream. + +## A master and tx workers on one machine + +On one machine, FHIRsmith can run as one master and several tx workers on the same data +directory (see [cluster.md](cluster.md)). The master runs every module, the background jobs, +`$closure` and the syndication feed; the workers run only the tx module. Start the master +first, and the workers once it answers `/health`: + +```bash +node --expose-gc server.js --role=master --instance-code=m0 --port=3001 +node --expose-gc server.js --role=tx-worker --instance-code=tx1 --port=3002 +node --expose-gc server.js --role=tx-worker --instance-code=tx2 --port=3003 +``` + +Keep each server's keepAliveTimeout (65 s with `library/keep-alive.js`) above nginx's upstream +`keepalive_timeout` (60 s by default); on servers without that fix (Node's 5 s), set +`keepalive_timeout 4s;` in each upstream block, or nginx reuses connections the servers are +closing, and the resets mark the owner of a cache down. + +The routing is the pool above, with the master as the designated `$closure` instance and the +home of everything that isn't a tx endpoint: + +```nginx +upstream fs_master { server 127.0.0.1:3001; keepalive 16; } + +# every process that takes ordinary tx requests (leave the master out if it shouldn't); +# the zone shares least_conn's counts between nginx's worker processes +upstream fs_tx_pool { + zone fs_tx_pool 64k; + least_conn; + server 127.0.0.1:3001; + server 127.0.0.1:3002; + server 127.0.0.1:3003; + keepalive 32; +} +upstream fs_tx1 { server 127.0.0.1:3002; server 127.0.0.1:3003 backup; keepalive 16; } +upstream fs_tx2 { server 127.0.0.1:3003; server 127.0.0.1:3002 backup; keepalive 16; } + +map $http_x_cache_id $fs_tx_backend { + default fs_tx_pool; + ~^m0\. fs_master; + ~^tx1\. fs_tx1; + ~^tx2\. fs_tx2; +} + +server { + ... + # the master's parts of the tx endpoints: $closure, the syndication feed, the source + # info pages (the VSAC resync form writes the VSAC database) + location ~ ^/(r4|r5)/(\$closure|syndication(/.*)?|info/.*)$ { + include fhirsmith-proxy.conf; + proxy_set_header X-Cache-Id ""; + proxy_pass http://fs_master; + } + + # the tx endpoints: by cache-id, else the least busy + location ~ ^/(r4|r5)(/|$) { + include fhirsmith-proxy.conf; + proxy_pass http://$fs_tx_backend; + } + + # everything else: the master + location / { + include fhirsmith-proxy.conf; + proxy_pass http://fs_master; + } +} +``` + +List your own tx endpoint paths (`modules.tx.endpoints`) in the two regex locations. The +optional launcher, `node cluster.js`, starts the processes in this order and applies the same +rules without nginx (it can also sit behind nginx, as one upstream). + +### With the shared cache + +With `server.cluster.sharedCache: true` ([cluster.md](cluster.md#options)), every process +keeps its `$cache-control` caches in one SQLite file, so any of them can answer any cache-id. +The `map` on `X-Cache-Id` and the per-worker upstreams (`fs_tx1`, `fs_tx2`) are then not +needed: the tx location goes straight to the pool. + +```nginx + location ~ ^/(r4|r5)(/|$) { + include fhirsmith-proxy.conf; + proxy_pass http://fs_tx_pool; + } +``` + +Keeping the `map` still works (the launcher's `cacheAffinity: true`). It keeps each client on +one process, whose expansion cache (which is per process) already holds the expansions it +asked for; without it, the load spreads evenly, and the first time each process expands a +value set for a client it does the work again. + +### With shared closure tables + +With `server.cluster.sharedClosure: true` ([cluster.md](cluster.md#options)), every process +answers `$closure` on the one closure database, so `$closure` can leave the master's location +and go to the pool (it never uses a cache-id): + +```nginx + location ~ ^/(r4|r5)/(syndication(/.*)?|info/.*)$ { + include fhirsmith-proxy.conf; + proxy_set_header X-Cache-Id ""; + proxy_pass http://fs_master; + } + + location ~ ^/(r4|r5)/\$closure$ { + include fhirsmith-proxy.conf; + proxy_pass http://fs_tx_pool; + } +``` diff --git a/server.js b/server.js index cc724cd9..8caec884 100644 --- a/server.js +++ b/server.js @@ -33,6 +33,20 @@ try { process.exit(1); } +// --role=master|tx-worker (cluster.md). With no role this is null and the configuration is +// untouched: every `if (role)` below is skipped +const { parseRole, applyRole, describeRole } = require('./library/role'); +let role; +try { + role = parseRole(process.argv.slice(2), process.env); + if (role) { + applyRole(config, role); + } +} catch (error) { + console.error(error.message); + process.exit(1); +} + const Logger = require('./library/logger'); const serverLog = Logger.getInstance(config.logging || {}).child({ module: 'server' }); const packageJson = require('./package.json'); @@ -46,6 +60,9 @@ serverLog.info(`Node.js ${process.version} on ${os.type()} ${os.release()} (${os serverLog.info(`Memory: ${freeMemGB} GB free / ${totalMemGB} GB total`); memoryGuard.configure(config.server.memory || {}); serverLog.info(memoryGuard.describe()); +if (role) { + serverLog.info(describeRole(role, config)); +} serverLog.info(`Data directory: ${folders.dataDir()}`); serverLog.info(`========================================`); @@ -87,7 +104,7 @@ const app = express(); const DEFAULT_TRUST_PROXY = 'loopback, linklocal, uniquelocal'; app.set('trust proxy', config.server.trustProxy ?? DEFAULT_TRUST_PROXY); -const PORT = process.env.PORT || config.server.port || 3000; +const PORT = (role && role.port) || process.env.PORT || config.server.port || 3000; // Middleware // Refuse new requests (503) while memory is near the limit - first, so a refused request diff --git a/tests/cs/mimetypes-cluster-role.test.js b/tests/cs/mimetypes-cluster-role.test.js new file mode 100644 index 00000000..f4ac5a2f --- /dev/null +++ b/tests/cs/mimetypes-cluster-role.test.js @@ -0,0 +1,109 @@ +/** + * The IANA media types registry in a cluster (cluster.md): a tx-worker reads the cached copy + * the master downloaded and never downloads or writes it; the master, and a server with no + * role, refresh it on every load as before. + */ +jest.mock('axios'); +const axios = require('axios'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { MimeTypeServicesFactory } = require('../../tx/cs/cs-mimetypes'); +const { Library } = require('../../tx/library'); +const { TestUtilities } = require('../test-utilities'); + +const registry = (...names) => ` + + + ${names.map(n => `${n}text/${n}`).join('\n ')} + +`; + +function fakeLog() { + const lines = []; + const add = level => msg => lines.push(`${level}: ${msg}`); + return { lines, info: add('info'), warn: add('warn'), error: add('error'), debug: add('debug') }; +} + +describe('Mime types registry and the cluster role', () => { + let dir; + let target; + const OLD = new Date('2026-01-01T00:00:00Z'); + let i18n; + + beforeAll(async () => { + i18n = await TestUtilities.loadTranslations(); + }); + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mime-role-')); + target = path.join(dir, 'media-types.xml'); + axios.get.mockReset(); + axios.get.mockResolvedValue({ data: registry('plain', 'html', 'csv') }); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function seedCache() { + fs.writeFileSync(target, registry('plain', 'html'), 'utf8'); + fs.utimesSync(target, OLD, OLD); + } + + test('read-only (a tx-worker): no download, no write, the cached copy is used', async () => { + seedCache(); + const log = fakeLog(); + const factory = new MimeTypeServicesFactory(i18n, dir, log, { readOnly: true }); + await factory.load(); + expect(axios.get).not.toHaveBeenCalled(); + expect(fs.statSync(target).mtimeMs).toBe(OLD.getTime()); + expect(fs.readdirSync(dir)).toEqual(['media-types.xml']); + expect([...factory.registeredTypes].sort()).toEqual(['text/html', 'text/plain']); + }); + + test('read-only with no cached copy: a warning, no download, no registered filter', async () => { + const log = fakeLog(); + const factory = new MimeTypeServicesFactory(i18n, dir, log, { readOnly: true }); + await factory.load(); + expect(axios.get).not.toHaveBeenCalled(); + expect(fs.readdirSync(dir)).toEqual([]); + expect(factory.registeredTypes).toBeNull(); + expect(log.lines.some(l => l.startsWith('warn: Mime types: no cached copy'))).toBe(true); + expect(log.lines.some(l => l.startsWith('warn: Mime types: no copy of the IANA media types registry'))).toBe(true); + }); + + test('no role (the default): refreshes the cached copy', async () => { + seedCache(); + const factory = new MimeTypeServicesFactory(i18n, dir, fakeLog()); + await factory.load(); + expect(axios.get).toHaveBeenCalledTimes(1); + expect(fs.statSync(target).mtimeMs).toBeGreaterThan(OLD.getTime()); + expect(fs.readdirSync(dir)).toEqual(['media-types.xml']); + expect(factory.registeredTypes.has('text/csv')).toBe(true); + }); + + describe('through the library, as tx.js sets it for each role', () => { + async function loadWith(readOnly) { + const library = new Library(null, null, fakeLog(), null); + library.cacheFolder = dir; + library.i18n = i18n; + library.mimeTypesReadOnly = readOnly; + library.registerProvider = () => {}; + await library.loadInternal('mimetypes', false, 'load'); + } + + test('tx-worker: no network call and no write', async () => { + seedCache(); + await loadWith(true); + expect(axios.get).not.toHaveBeenCalled(); + expect(fs.statSync(target).mtimeMs).toBe(OLD.getTime()); + }); + + test('master or no role: refreshes', async () => { + seedCache(); + await loadWith(false); + expect(axios.get).toHaveBeenCalledTimes(1); + expect(fs.statSync(target).mtimeMs).toBeGreaterThan(OLD.getTime()); + }); + }); +}); diff --git a/tests/library/memory-guard-group.test.js b/tests/library/memory-guard-group.test.js new file mode 100644 index 00000000..a3c37ea7 --- /dev/null +++ b/tests/library/memory-guard-group.test.js @@ -0,0 +1,83 @@ +/** + * The memory guard with a role (master, tx-worker): the measure is the cgroup's private + * memory (memory.stat anon), and only the largest process in the cgroup aborts. + * A fake cgroup directory stands in for /sys/fs/cgroup/. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawn } = require('child_process'); +const memoryGuard = require('../../library/memory-guard'); + +const GB = 1024 * 1024 * 1024; + +describe('memory-guard, a group of processes in one cgroup', () => { + let dir; + let rssSpy; + let other; + + const writeCgroup = (anon, pids) => { + fs.writeFileSync(path.join(dir, 'memory.stat'), `file 123\nanon ${anon}\nshmem 0\n`); + fs.writeFileSync(path.join(dir, 'cgroup.procs'), pids.join('\n') + '\n'); + }; + const configure = () => memoryGuard.configure({ limit: 10 * GB, rejectFraction: 0.8, abortFraction: 0.9, group: true, cgroup: { dir, version: 2 } }); + + beforeAll(async () => { + // another process in the "cgroup", with an RSS of a few tens of MB + other = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); + await new Promise(resolve => setTimeout(resolve, 300)); + }); + afterAll(async () => { + await new Promise(resolve => { + other.once('exit', resolve); + other.kill(); + }); + }); + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cgroup-')); + rssSpy = jest.spyOn(process.memoryUsage, 'rss'); + }); + afterEach(() => { + rssSpy.mockRestore(); + memoryGuard.configure(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('refuses on the group measure, though this process is small', () => { + rssSpy.mockReturnValue(1024 * 1024); + writeCgroup(Math.round(8.5 * GB), [process.pid]); + configure(); + expect(memoryGuard.describe()).toMatch(/for the whole cgroup/); + expect(memoryGuard.shouldReject()).toBe(true); + writeCgroup(1 * GB, [process.pid]); + memoryGuard._state.sampleTime = 0; + expect(memoryGuard.shouldReject()).toBe(false); + }); + + test('over the abort threshold, only the largest process aborts', () => { + writeCgroup(Math.round(9.5 * GB), [process.pid, other.pid]); + configure(); + // smaller than the other process: its operations carry on + rssSpy.mockReturnValue(1024 * 1024); + expect(memoryGuard.overAbortThreshold('test')).toBe(0); + // the largest: it aborts (the answer is kept for 100 ms) + rssSpy.mockReturnValue(5 * GB); + return new Promise(resolve => setTimeout(resolve, 150)).then(() => { + expect(memoryGuard.overAbortThreshold('test')).toBeGreaterThan(9 * GB); + }); + }); + + test('without a cgroup of its own, each process measures its own RSS', () => { + memoryGuard.configure({ limit: 10 * GB, rejectFraction: 0.8, group: true, cgroup: null }); + expect(memoryGuard._state.group).toBeNull(); + expect(memoryGuard.describe()).toMatch(/measured as this process's RSS/); + rssSpy.mockReturnValue(1024 * 1024); + expect(memoryGuard.shouldReject()).toBe(false); + }); + + test('no role: no group, and the describe text is as it was', () => { + memoryGuard.configure({ limit: 10 * GB }); + expect(memoryGuard._state.group).toBeNull(); + expect(memoryGuard.describe()).toMatch(/^Memory limit: 10.0 GB \(from configured\); new requests/); + }); +}); diff --git a/tests/library/role.test.js b/tests/library/role.test.js new file mode 100644 index 00000000..05ebc476 --- /dev/null +++ b/tests/library/role.test.js @@ -0,0 +1,119 @@ +const { parseRole, applyRole } = require('../../library/role'); + +const baseConfig = () => ({ + server: { port: 3000, memory: { rejectFraction: 0.8 } }, + stats: { intervalMinutes: 5 }, + modules: { + tx: { enabled: true, instanceCode: 'm0', vsacCfg: { apiKey: 'k' }, closure: { enabled: true } }, + packages: { enabled: true }, + xig: { enabled: true }, + registry: { enabled: false } + } +}); + +describe('parseRole', () => { + test('no flags and no environment: no role', () => { + expect(parseRole([], {})).toBeNull(); + // other arguments and an instance code in the environment don't make a role + expect(parseRole(['--inspect', 'foo'], { FHIRSMITH_INSTANCE_CODE: 'tx1', PORT: '4000' })).toBeNull(); + }); + + test('flags, in either form', () => { + expect(parseRole(['--role=tx-worker', '--instance-code=tx1', '--port=20001'], {})) + .toEqual({ role: 'tx-worker', instanceCode: 'tx1', port: 20001 }); + expect(parseRole(['--role', 'master', '--port', '20000'], {})) + .toEqual({ role: 'master', instanceCode: null, port: 20000 }); + }); + + test('the environment, with the flags winning', () => { + expect(parseRole([], { FHIRSMITH_ROLE: 'tx-worker', FHIRSMITH_INSTANCE_CODE: 'tx2' })) + .toEqual({ role: 'tx-worker', instanceCode: 'tx2', port: null }); + expect(parseRole(['--role=master', '--instance-code=a'], { FHIRSMITH_ROLE: 'tx-worker', FHIRSMITH_INSTANCE_CODE: 'b' })) + .toEqual({ role: 'master', instanceCode: 'a', port: null }); + }); + + test('mistakes are refused', () => { + expect(() => parseRole(['--role=worker'], {})).toThrow(/Unknown role/); + expect(() => parseRole(['--role=master', '--port=x'], {})).toThrow(/not a port/); + expect(() => parseRole(['--port=3001'], {})).toThrow(/only used with --role/); + expect(() => parseRole(['--instance-code=tx1'], {})).toThrow(/only used with --role/); + }); +}); + +describe('applyRole', () => { + test('without a role, server.js never calls it: the configuration is untouched', () => { + // what server.js does: applyRole only when parseRole returns a role + const config = baseConfig(); + const before = JSON.stringify(config); + const role = parseRole([], {}); + if (role) { + applyRole(config, role); + } + expect(JSON.stringify(config)).toBe(before); + }); + + test('master: every module, the VSAC sync, the group memory measure', () => { + const config = applyRole(baseConfig(), { role: 'master', instanceCode: null }); + expect(config.modules.tx.instanceCode).toBe('m0'); + expect(config.modules.tx.clusterRole).toBe('master'); + expect(config.modules.tx.vsacCfg).toEqual({ apiKey: 'k', cluster: 'leader' }); + expect(config.modules.packages.enabled).toBe(true); + expect(config.modules.xig.enabled).toBe(true); + expect(config.stats.enabled).toBeUndefined(); + expect(config.server.memory).toEqual({ rejectFraction: 0.8, group: true }); + }); + + test('tx-worker: the tx module only, no background jobs, its own instance code', () => { + const config = applyRole(baseConfig(), { role: 'tx-worker', instanceCode: 'tx1' }); + expect(config.modules.tx.enabled).toBe(true); + expect(config.modules.tx.instanceCode).toBe('tx1'); + expect(config.modules.tx.clusterRole).toBe('tx-worker'); + expect(config.modules.tx.vsacCfg.cluster).toBe('follower'); + expect(config.modules.packages.enabled).toBe(false); + expect(config.modules.xig.enabled).toBe(false); + expect(config.modules.registry.enabled).toBe(false); + expect(config.stats).toEqual({ intervalMinutes: 5, enabled: false }); + expect(config.npmAudit.enabled).toBe(false); + expect(config.dailyReport.enabled).toBe(false); + }); + + test('server.cluster.sharedCache reaches tx (with a role), and nothing is set when it is off', () => { + const off = applyRole(baseConfig(), { role: 'tx-worker', instanceCode: 'tx1' }); + expect(off.modules.tx.sharedCache).toBeUndefined(); + expect(off.modules.tx.sharedCacheMaxBytes).toBeUndefined(); + const on = baseConfig(); + on.server.cluster = { sharedCache: true, sharedCacheMaxMB: 2 }; + applyRole(on, { role: 'master', instanceCode: null }); + expect(on.modules.tx.sharedCache).toBe(true); + expect(on.modules.tx.sharedCacheMaxBytes).toBe(2 * 1024 * 1024); + const bad = baseConfig(); + bad.server.cluster = { sharedCache: 'yes' }; + expect(() => applyRole(bad, { role: 'master', instanceCode: null })).toThrow(/true or false/); + const badCap = baseConfig(); + badCap.server.cluster = { sharedCache: true, sharedCacheMaxMB: -1 }; + expect(() => applyRole(badCap, { role: 'master', instanceCode: null })).toThrow(/megabytes/); + }); + + test('server.cluster.sharedClosure reaches tx (with a role)', () => { + const off = applyRole(baseConfig(), { role: 'tx-worker', instanceCode: 'tx1' }); + expect(off.modules.tx.sharedClosure).toBeUndefined(); + const on = baseConfig(); + on.server.cluster = { sharedClosure: true }; + applyRole(on, { role: 'tx-worker', instanceCode: 'tx1' }); + expect(on.modules.tx.sharedClosure).toBe(true); + expect(on.modules.tx.sharedCache).toBeUndefined(); + const bad = baseConfig(); + bad.server.cluster = { sharedClosure: 1 }; + expect(() => applyRole(bad, { role: 'master', instanceCode: null })).toThrow(/true or false/); + }); + + test('a worker needs its own instance code, and tx must be enabled', () => { + expect(() => applyRole(baseConfig(), { role: 'tx-worker', instanceCode: null })).toThrow(/its own instance code/); + const noCode = baseConfig(); + delete noCode.modules.tx.instanceCode; + expect(() => applyRole(noCode, { role: 'master', instanceCode: null })).toThrow(/needs an instance code/); + const noTx = baseConfig(); + noTx.modules.tx.enabled = false; + expect(() => applyRole(noTx, { role: 'master', instanceCode: 'm0' })).toThrow(/tx module enabled/); + }); +}); diff --git a/tests/server/cluster-integration.test.js b/tests/server/cluster-integration.test.js new file mode 100644 index 00000000..9cfd489b --- /dev/null +++ b/tests/server/cluster-integration.test.js @@ -0,0 +1,231 @@ +/** + * cluster.js with real processes: a master and two tx-workers on a temporary data + * directory, behind the launcher's port, with a fake VSAC. + * + * - $cache-control sessions started through the launcher are used on later requests, + * and every one of those requests reaches the process that holds the cache + * - $closure and the non-tx paths reach the master + * - a VSAC sync on the master is seen by both workers + * - SIGTERM: the requests in progress finish, every process stops, the launcher exits 0 + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const http = require('http'); +const { spawn } = require('child_process'); + +const ROOT = path.join(__dirname, '..', '..'); +const FRONT = 20020; +const MASTER = 20021; +const WORKERS = [20022, 20023]; +const VSAC = 20029; +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +const GENDER = 'http://hl7.org/fhir/administrative-gender'; +const vsacValueSet = (id) => ({ + resourceType: 'ValueSet', id, url: `http://cts.nlm.nih.gov/fhir/ValueSet/${id}`, version: '1', status: 'active', + compose: { include: [{ system: GENDER, concept: [{ code: 'male' }] }] } +}); + +function request(port, method, urlPath, { headers = {}, body } = {}) { + return new Promise((resolve, reject) => { + const data = body ? Buffer.from(JSON.stringify(body)) : null; + const req = http.request({ + host: '127.0.0.1', port, method, path: urlPath, agent: false, + headers: { Accept: 'application/fhir+json', ...(data ? { 'Content-Type': 'application/fhir+json' } : {}), ...headers } + }, (res) => { + let text = ''; + res.on('data', d => { text += d; }); + res.on('end', () => { + let json = null; + try { + json = JSON.parse(text); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on('error', reject); + req.end(data); + }); +} + +const isUp = (port) => request(port, 'GET', '/health').then(r => r.status === 200, () => false); +const param = (body, name) => ((body && body.parameter) || []).find(p => p.name === name); +const unknownCache = (r) => r.json && r.json.resourceType === 'OperationOutcome' && + (r.json.issue || []).some(i => ((i.details && i.details.coding) || []).some(c => c.code === 'cache-id-unknown')); + +describe('master and two tx-workers behind cluster.js', () => { + let dir; + let launcher; + let launcherExit; + let vsacServer; + const vsacList = [vsacValueSet('one')]; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-')); + // the R4 core package, if a test run has already downloaded it, saves fetching it again + const core = path.join(ROOT, 'data', 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'); + if (fs.existsSync(core)) { + fs.cpSync(core, path.join(dir, 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'), { recursive: true }); + } + fs.writeFileSync(path.join(dir, 'library.yml'), 'base:\n url: https://storage.googleapis.com/tx-fhir-org\n\nsources:\n - internal:vsac=vsac\n'); + fs.writeFileSync(path.join(dir, 'config.json'), JSON.stringify({ + hostName: 'cluster-test', + logging: { console: false }, + server: { port: FRONT, cluster: { workers: 2, internalPort: MASTER } }, + modules: { + vcl: { enabled: true }, + tx: { + enabled: true, + instanceCode: 'm0', + librarySource: path.join(dir, 'library.yml'), + endpoints: [{ path: '/r4', fhirVersion: '4.0', context: null }], + closure: { enabled: true }, + // a sync every 1.8 s + vsacCfg: { apiKey: 'test', baseUrl: `http://127.0.0.1:${VSAC}/fhir`, refreshIntervalHours: 0.0005 } + } + } + })); + + // a fake VSAC: every listing is the whole list, ?url= the one asked for + vsacServer = http.createServer((req, res) => { + const u = new URL(req.url, 'http://x'); + const url = u.searchParams.get('url'); + const list = url ? vsacList.filter(v => v.url === url) : vsacList; + res.setHeader('Content-Type', 'application/fhir+json'); + res.end(JSON.stringify({ resourceType: 'Bundle', type: 'searchset', total: list.length, entry: list.map(resource => ({ resource })), link: [] })); + }); + await new Promise(resolve => vsacServer.listen(VSAC, resolve)); + + const out = fs.openSync(path.join(dir, 'processes.log'), 'a'); + launcher = spawn(process.execPath, [path.join(ROOT, 'cluster.js')], { + cwd: ROOT, env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', out, out] + }); + launcherExit = new Promise(resolve => launcher.once('exit', (code, signal) => resolve({ code, signal }))); + const t0 = Date.now(); + for (const port of [MASTER, ...WORKERS, FRONT]) { + while (!(await isUp(port))) { + if (Date.now() - t0 > 300000 || launcher.exitCode !== null) { + throw new Error(`port ${port} did not come up; see ${path.join(dir, 'processes.log')}`); + } + await sleep(250); + } + } + // the launcher marks a worker ready a moment after it first answers + await sleep(500); + }, 330000); + + afterAll(async () => { + if (launcher && launcher.exitCode === null && launcher.signalCode === null) { + launcher.kill('SIGTERM'); + await launcherExit; + } + if (vsacServer) { + await new Promise(resolve => vsacServer.close(resolve)); + } + if (dir) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + test('each process runs what its role says', async () => { + // non-tx paths reach the master: it alone has the vcl module + const viaFront = await request(FRONT, 'GET', '/health'); + expect(Object.keys(viaFront.json.modules).sort()).toEqual(['tx', 'vcl']); + for (const port of WORKERS) { + expect(Object.keys((await request(port, 'GET', '/health')).json.modules)).toEqual(['tx']); + } + expect((await request(FRONT, 'GET', '/VCL?vcl=' + encodeURIComponent('(http://loinc.org)1-8'))).status).toBe(200); + }); + + test('$cache-control sessions started through the launcher hold on every later request', async () => { + const issuers = new Set(); + let unknown = 0; + let ok = 0; + for (let s = 0; s < 6; s++) { + const start = await request(FRONT, 'POST', '/r4/$cache-control?mode=start', { body: { resourceType: 'Parameters', parameter: [] } }); + expect(start.status).toBe(200); + const cid = param(start.json, 'cache-id').valueId; + issuers.add(cid.split('.')[0]); + const cs = { resourceType: 'CodeSystem', url: `http://example.org/cs${s}`, version: '1', status: 'active', content: 'complete', concept: [{ code: 'a' }] }; + const vs = { resourceType: 'ValueSet', url: `http://example.org/vs${s}`, version: '1', status: 'active', compose: { include: [{ system: cs.url }] } }; + // the first request leaves the resources in the cache; the later ones refer to them by url + const first = await request(FRONT, 'POST', '/r4/ValueSet/$validate-code', { + headers: { 'X-Cache-Id': cid }, + body: { resourceType: 'Parameters', parameter: [{ name: 'tx-resource', resource: cs }, { name: 'tx-resource', resource: vs }, + { name: 'url', valueUri: vs.url }, { name: 'coding', valueCoding: { system: cs.url, code: 'a' } }] } + }); + expect(param(first.json, 'result').valueBoolean).toBe(true); + for (let i = 0; i < 8; i++) { + const r = await request(FRONT, 'POST', '/r4/ValueSet/$validate-code', { + headers: { 'X-Cache-Id': cid }, + body: { resourceType: 'Parameters', parameter: [{ name: 'url', valueUri: vs.url }, { name: 'coding', valueCoding: { system: cs.url, code: 'a' } }] } + }); + if (unknownCache(r)) { + unknown++; + } else if (r.status === 200 && param(r.json, 'result').valueBoolean === true) { + ok++; + } + } + } + expect(unknown).toBe(0); + expect(ok).toBe(48); + // the sessions were spread over the tx processes, and each id names its issuer + expect(issuers.size).toBeGreaterThan(1); + for (const code of issuers) { + expect(['m0', 'tx1', 'tx2']).toContain(code); + } + }); + + test('$closure goes to the master, whatever the cache-id says', async () => { + for (const cid of ['tx1.abc', 'tx2.abc', undefined]) { + const r = await request(FRONT, 'POST', '/r4/$closure', { + headers: cid ? { 'X-Cache-Id': cid } : {}, + body: { resourceType: 'Parameters', parameter: [{ name: 'name', valueString: `t-${cid}` }] } + }); + expect(r.status).toBe(200); + expect(r.json.resourceType).toBe('ConceptMap'); + } + // a worker has no closure tables of its own, but says the service offers $closure + const direct = await request(WORKERS[0], 'POST', '/r4/$closure', { body: { resourceType: 'Parameters', parameter: [{ name: 'name', valueString: 'x' }] } }); + expect(direct.status).toBe(501); + const cap = await request(WORKERS[0], 'GET', '/r4/metadata'); + expect(cap.json.rest[0].operation.map(o => o.name)).toContain('closure'); + }); + + test('a VSAC sync on the master reaches both workers', async () => { + const validate = (port) => request(port, 'GET', '/r4/ValueSet/$validate-code?url=' + + encodeURIComponent('http://cts.nlm.nih.gov/fhir/ValueSet/two') + `&system=${encodeURIComponent(GENDER)}&code=male`) + .then(r => r.status === 200 && param(r.json, 'result') && param(r.json, 'result').valueBoolean === true); + for (const port of WORKERS) { + expect(await validate(port)).toBe(false); + } + vsacList.push(vsacValueSet('two')); + const t0 = Date.now(); + for (const port of [MASTER, ...WORKERS]) { + while (!(await validate(port))) { + if (Date.now() - t0 > 30000) { + throw new Error(`the process on ${port} never saw the new value set`); + } + await sleep(200); + } + } + }); + + test('SIGTERM: requests in progress finish, then every process stops', async () => { + const inFlight = []; + for (let i = 0; i < 10; i++) { + inFlight.push(request(FRONT, 'GET', `/r4/CodeSystem/$lookup?system=${encodeURIComponent(GENDER)}&code=male`)); + } + await sleep(50); + launcher.kill('SIGTERM'); + const results = await Promise.all(inFlight); + expect(results.map(r => r.status)).toEqual(Array(10).fill(200)); + expect(await launcherExit).toEqual({ code: 0, signal: null }); + for (const port of [FRONT, MASTER, ...WORKERS]) { + expect(await isUp(port)).toBe(false); + } + }); +}); diff --git a/tests/server/cluster-launcher.test.js b/tests/server/cluster-launcher.test.js new file mode 100644 index 00000000..2d8f82b8 --- /dev/null +++ b/tests/server/cluster-launcher.test.js @@ -0,0 +1,266 @@ +/** + * cluster.js: the routing rules (as in nginx.md), least-loaded choice, restarts and the + * drain on stop - against fake upstream processes (tests/utils/fake-upstream.js). + */ +const http = require('http'); +const path = require('path'); +const { loadSettings, classify, pickTarget, createLauncher } = require('../../cluster'); + +const FAKE = path.join(__dirname, '..', 'utils', 'fake-upstream.js'); +const TX = ['/r4', '/r5']; +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +function get(port, urlPath, headers = {}) { + return new Promise((resolve, reject) => { + const req = http.get({ host: '127.0.0.1', port, path: urlPath, headers, agent: false }, (res) => { + let body = ''; + res.on('data', d => { body += d; }); + res.on('end', () => { + let json = null; + try { + json = JSON.parse(body); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on('error', reject); + }); +} + +async function until(fn, ms = 10000) { + const t0 = Date.now(); + while (!fn()) { + if (Date.now() - t0 > ms) { + throw new Error('timed out'); + } + await sleep(50); + } +} + +describe('routing rules', () => { + test('cache-ids go to their issuer, by the prefix', () => { + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX)).toEqual({ to: 'owner', code: 'tx2' }); + expect(classify('/r5/CodeSystem/$lookup?x=1', { 'x-cache-id': 'm0.abc' }, TX)).toEqual({ to: 'owner', code: 'm0' }); + // no prefix: any tx process (it answers cache-id-unknown) + expect(classify('/r4/ValueSet/$expand', { 'x-cache-id': '4f0c2d1e-aaaa' }, TX)).toEqual({ to: 'tx' }); + expect(classify('/r4/ValueSet/$expand', {}, TX)).toEqual({ to: 'tx' }); + }); + + test('with the shared cache, a cache-id goes to any tx process, unless cacheAffinity is on', () => { + const shared = loadSettings([], {}, { server: { cluster: { sharedCache: true } }, modules: { tx: { enabled: true } } }).routing; + expect(shared).toEqual({ cacheAffinity: false, sharedClosure: false }); + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, shared)).toEqual({ to: 'tx' }); + const affinity = loadSettings([], {}, { server: { cluster: { sharedCache: true, cacheAffinity: true } }, modules: { tx: { enabled: true } } }).routing; + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, affinity)).toEqual({ to: 'owner', code: 'tx2' }); + // without the shared cache, a cache-id always goes to its issuer: there is nowhere else it is + const plain = loadSettings([], {}, { server: { cluster: { cacheAffinity: false } }, modules: { tx: { enabled: true } } }).routing; + expect(plain).toEqual({ cacheAffinity: true, sharedClosure: false }); + expect(classify('/r4/ValueSet/$validate-code', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, plain)).toEqual({ to: 'owner', code: 'tx2' }); + // the master-only paths stay the master's + expect(classify('/r4/$closure', { 'x-cache-id': 'tx2.4f0c2d1e' }, TX, shared)).toEqual({ to: 'master' }); + expect(classify('/', {}, TX, shared)).toEqual({ to: 'master' }); + }); + + test('with the shared closure tables, $closure goes to any tx process; the rest of the master\'s paths stay its', () => { + const routing = loadSettings([], {}, { server: { cluster: { sharedClosure: true } }, modules: { tx: { enabled: true } } }).routing; + expect(routing).toEqual({ cacheAffinity: true, sharedClosure: true }); + for (const url of ['/r4/$closure', '/r5/%24closure', '/r4/$closure?name=x']) { + expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX, routing)).toEqual({ to: 'tx' }); + } + expect(classify('/r4/syndication', {}, TX, routing)).toEqual({ to: 'master' }); + expect(classify('/r4/info/vsac', {}, TX, routing)).toEqual({ to: 'master' }); + expect(classify('/xig', {}, TX, routing)).toEqual({ to: 'master' }); + // and cache-ids are routed as without it + expect(classify('/r4/ValueSet/$expand', { 'x-cache-id': 'tx1.abc' }, TX, routing)).toEqual({ to: 'owner', code: 'tx1' }); + }); + + test('$closure, in any FHIR version, encoded or not, goes to the master, whatever the cache-id', () => { + for (const url of ['/r4/$closure', '/r5/$closure', '/r4/%24closure', '/r4/$closure?name=x']) { + expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX)).toEqual({ to: 'master' }); + } + }); + + test('the syndication feed and the source info pages go to the master', () => { + expect(classify('/r4/syndication', {}, TX)).toEqual({ to: 'master' }); + expect(classify('/r5/syndication/archive/2', {}, TX)).toEqual({ to: 'master' }); + expect(classify('/r4/info/vsac', {}, TX)).toEqual({ to: 'master' }); + }); + + test('every path outside the tx endpoints goes to the master', () => { + for (const url of ['/', '/health', '/packages/hl7.fhir.r4.core', '/tx-reg', '/xig', '/dashboard', '/r4x/metadata', '/r']) { + expect(classify(url, { 'x-cache-id': 'tx1.abc' }, TX)).toEqual({ to: 'master' }); + } + }); + + test('least outstanding requests, among the ready tx processes', () => { + const procs = [ + { code: 'm0', role: 'master', servesTx: true, ready: true, outstanding: 3 }, + { code: 'tx1', role: 'tx-worker', servesTx: true, ready: true, outstanding: 2 }, + { code: 'tx2', role: 'tx-worker', servesTx: true, ready: true, outstanding: 1 }, + { code: 'tx3', role: 'tx-worker', servesTx: true, ready: false, outstanding: 0 } + ]; + expect(pickTarget({ to: 'tx' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'owner', code: 'tx1' }, procs).code).toBe('tx1'); + // the owner isn't ready, or doesn't exist: the least loaded + expect(pickTarget({ to: 'owner', code: 'tx3' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'owner', code: 'zz9' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'master' }, procs).code).toBe('m0'); + // a master that doesn't serve tx gets none of it + procs[0].servesTx = false; + procs[0].outstanding = 0; + expect(pickTarget({ to: 'tx' }, procs).code).toBe('tx2'); + expect(pickTarget({ to: 'master' }, [procs[1]])).toBeNull(); + }); + + test('settings from flags, environment and config', () => { + const config = { server: { port: 20000, cluster: { workers: 3 } }, modules: { tx: { enabled: true, instanceCode: 'main', endpoints: [{ path: '/r4/' }] } } }; + const s = loadSettings([], {}, config); + expect(s.port).toBe(20000); + expect(s.txPaths).toEqual(['/r4']); + expect(s.procs.map(p => [p.code, p.role, p.port, p.servesTx])).toEqual([ + ['main', 'master', 20001, true], ['tx1', 'tx-worker', 20002, true], ['tx2', 'tx-worker', 20003, true], ['tx3', 'tx-worker', 20004, true]]); + const s2 = loadSettings(['--workers=1', '--port=20010', '--master-serves-tx=false'], {}, config); + expect(s2.procs.map(p => [p.code, p.port, p.servesTx])).toEqual([['main', 20011, false], ['tx1', 20012, true]]); + expect(() => loadSettings([], {}, { modules: { tx: { enabled: true, instanceCode: 'tx1' } } })).toThrow(/also the code of a worker/); + }); +}); + +describe('the launcher, with fake processes', () => { + const PORT = 20013; + let launcher; + const settings = (servesTx) => ({ + port: PORT, + txPaths: TX, + drainMs: 10000, + procs: [ + { name: 'master', role: 'master', code: 'm0', port: 20010, servesTx }, + { name: 'worker tx1', role: 'tx-worker', code: 'tx1', port: 20011, servesTx: true }, + { name: 'worker tx2', role: 'tx-worker', code: 'tx2', port: 20012, servesTx: true } + ], + command: (proc) => [process.execPath, [FAKE, proc.code, String(proc.port)]], + log: () => {} + }); + + afterEach(async () => { + if (launcher) { + await launcher.stop(); + launcher = null; + } + }); + + test('routes by the rules, and adds X-Forwarded-For', async () => { + launcher = createLauncher(settings(true)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + expect((await get(PORT, '/packages')).json.code).toBe('m0'); + expect((await get(PORT, '/r4/$closure', { 'X-Cache-Id': 'tx2.x' })).json.code).toBe('m0'); + expect((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'tx2.x' })).json.code).toBe('tx2'); + expect((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'm0.x' })).json.code).toBe('m0'); + const r = await get(PORT, '/r4/metadata', { 'X-Forwarded-For': '203.0.113.9' }); + expect(r.json.xff).toBe('203.0.113.9, 127.0.0.1'); + expect(r.json.proto).toBe('http'); + }); + + test('a tx request goes to the process with the fewest outstanding', async () => { + launcher = createLauncher(settings(false)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + // a slow request on one worker: the next requests go to the other + const slow = get(PORT, '/r4/ValueSet/$expand?delay=1500'); + await until(() => launcher.procs.some(p => p.outstanding === 1)); + const busy = launcher.procs.find(p => p.outstanding === 1).code; + for (let i = 0; i < 5; i++) { + const quick = await get(PORT, '/r4/ValueSet/$expand'); + expect(['tx1', 'tx2']).toContain(quick.json.code); + expect(quick.json.code).not.toBe(busy); + } + expect((await slow).json.code).toBe(busy); + }); + + test('a process that exits is started again', async () => { + launcher = createLauncher(settings(true)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + const tx1 = launcher.procs.find(p => p.code === 'tx1'); + const old = tx1.child; + old.kill('SIGKILL'); + await until(() => !tx1.ready); + // meanwhile its cache-ids go to another tx process, which says it doesn't know them + expect(['m0', 'tx2']).toContain((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'tx1.x' })).json.code); + await until(() => tx1.ready && tx1.child !== old); + expect((await get(PORT, '/r4/metadata', { 'X-Cache-Id': 'tx1.x' })).json.code).toBe('tx1'); + }); + + test('stop drains: no new connections, the request in progress finishes, then the processes stop', async () => { + launcher = createLauncher(settings(true)); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + const children = launcher.procs.map(p => p.child); + const slow = get(PORT, '/r4/ValueSet/$expand?delay=1500'); + await until(() => launcher.procs.some(p => p.outstanding === 1)); + const stopped = launcher.stop(); + await sleep(100); + await expect(get(PORT, '/r4/metadata')).rejects.toThrow(/ECONNREFUSED/); + // the processes are still running while the request is in progress + expect(children.every(c => c.exitCode === null && c.signalCode === null)).toBe(true); + const r = await slow; + expect(r.status).toBe(200); + expect(r.headers.connection).toBe('close'); + await stopped; + expect(children.every(c => c.exitCode !== null || c.signalCode !== null)).toBe(true); + launcher = null; + }); +}); + +describe('the launcher\'s keep-alive connections to the processes', () => { + // a process that closes idle connections as Node's default does (5 s): the launcher must + // close its idle ones first, or a request sent as the process closes one is reset + const PORT = 20190; + let launcher; + afterEach(async () => { + if (launcher) { + await launcher.stop(); + launcher = null; + } + }); + + test('an idle connection is closed by the launcher before the process closes it, and a long request is not cut', async () => { + launcher = createLauncher({ + port: PORT, + txPaths: TX, + drainMs: 10000, + procs: [ + { name: 'master', role: 'master', code: 'm0', port: 20191, servesTx: false }, + { name: 'worker tx1', role: 'tx-worker', code: 'tx1', port: 20192, servesTx: true } + ], + command: (proc) => [process.execPath, [FAKE, proc.code, String(proc.port), '5000']], + log: () => {} + }); + await launcher.start(); + await until(() => launcher.procs.every(p => p.ready)); + const tx1 = launcher.procs.find(p => p.code === 'tx1'); + const pooled = () => Object.values(tx1.agent.freeSockets).reduce((n, list) => n + list.length, 0); + + expect((await get(PORT, '/r4/ValueSet/$expand')).status).toBe(200); + await until(() => pooled() === 1); + // reused while it is fresh + const before = (await get(PORT, '/r4/connections')).json.connections; + expect((await get(PORT, '/r4/connections')).json.connections).toBe(before); + // gone from the pool before the process's 5 s are up + const idleSince = Date.now(); + await until(() => pooled() === 0, 6000); + expect(Date.now() - idleSince).toBeLessThan(4900); + // so a request at the moment the process would close it goes down a new connection + await sleep(5000 - (Date.now() - idleSince)); + const r = await get(PORT, '/r4/connections'); + expect(r.status).toBe(200); + expect(r.json.connections).toBeGreaterThan(before); + + // the idle limit doesn't apply to a request in progress + const slow = await get(PORT, '/r4/ValueSet/$expand?delay=6000'); + expect(slow.status).toBe(200); + expect(slow.json.code).toBe('tx1'); + }, 30000); +}); diff --git a/tests/server/cluster-package-caches.test.js b/tests/server/cluster-package-caches.test.js new file mode 100644 index 00000000..afcd6df1 --- /dev/null +++ b/tests/server/cluster-package-caches.test.js @@ -0,0 +1,147 @@ +/** + * cluster.js on a new data directory whose package has no value set or concept map cache yet + * (.valuesets-2.db and .conceptmaps-2.db removed): the master builds them before it is ready, + * the workers only read them. Every process starts once - none exits and is restarted - and + * neither cache file is written after the master is ready. + * + * The same for the caches of file: library sources: every process uses the master's + * folder-source-master- folder, which the master builds before it is ready and the + * workers only read. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { startCluster, request, param } = require('../utils/cluster-harness'); + +const FRONT = 20150; +const VSAC = 20159; +const CORE = 'hl7.fhir.r4.core#4.0.1'; +const CACHES = ['.valuesets-2.db', '.conceptmaps-2.db']; + +describe('the package caches on a new data directory', () => { + let cluster; + const state = (file) => { + const st = fs.statSync(file); + return { mtimeMs: st.mtimeMs, size: st.size }; + }; + + afterAll(async () => { + if (cluster) { + await cluster.stop(); + } + }); + + test('the master builds them, the workers read them and start once', async () => { + let folder = null; + cluster = await startCluster({ + front: FRONT, vsac: VSAC, workers: 2, + sources: [`npm:${CORE}`], + prepare: (dir) => { + folder = path.join(dir, 'terminology-cache', CORE); + for (const name of CACHES) { + fs.rmSync(path.join(folder, name), { force: true }); + } + } + }); + const masterUp = cluster.upAt.get(cluster.master); + const files = CACHES.map(name => path.join(folder, name)); + const before = files.map(state); + for (const s of before) { + // written by the master, before it was ready + expect(s.mtimeMs).toBeLessThanOrEqual(masterUp); + } + + // the workers serve the package's value sets + for (const port of cluster.workers) { + const r = await request(port, 'GET', '/r4/ValueSet?url=http://hl7.org/fhir/ValueSet/administrative-gender'); + expect(r.status).toBe(200); + expect((r.json.entry || []).length).toBeGreaterThan(0); + } + + expect(files.map(state)).toEqual(before); + // no journal or WAL left by a writer + for (const file of files) { + for (const suffix of ['-journal', '-wal']) { + expect(fs.existsSync(file + suffix)).toBe(false); + } + } + const log = fs.readFileSync(path.join(cluster.dir, 'processes.log'), 'utf8'); + expect(log).not.toMatch(/exited \(/); + expect(log).not.toMatch(/Backfilling content_hash/); + expect(log).not.toMatch(/SQLITE_(BUSY|READONLY|IOERR)/); + }, 600000); +}); + +describe('the caches of file: sources on a new data directory', () => { + const FILE_FRONT = 20160; + const FILE_VSAC = 20169; + const SYSTEM = 'http://example.org/fhir/CodeSystem/cluster-file-source'; + const VS = 'http://example.org/fhir/ValueSet/cluster-file-source'; + let cluster; + let fixtures; + + afterAll(async () => { + if (cluster) { + await cluster.stop(); + } + if (fixtures) { + fs.rmSync(fixtures, { recursive: true, force: true }); + } + }); + + // every file under the folder-source-* folders, with its mtime and size, and the folders' + const snapshot = (cacheDir) => { + const out = {}; + for (const name of fs.readdirSync(cacheDir).filter(n => n.startsWith('folder-source-')).sort()) { + const dir = path.join(cacheDir, name); + out[name] = fs.statSync(dir).mtimeMs; + for (const file of fs.readdirSync(dir).sort()) { + const st = fs.statSync(path.join(dir, file)); + out[name + '/' + file] = [st.mtimeMs, st.size]; + } + } + return out; + }; + + test('the master builds them in its folder, the workers read it and start once', async () => { + fixtures = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-fx-')); + fs.writeFileSync(path.join(fixtures, 'cs.json'), JSON.stringify({ + resourceType: 'CodeSystem', id: 'cluster-file-source', url: SYSTEM, version: '1.0.0', + name: 'ClusterFileSource', status: 'active', content: 'complete', + concept: [{ code: 'A', display: 'Alpha' }, { code: 'B', display: 'Beta' }] + })); + fs.writeFileSync(path.join(fixtures, 'vs.json'), JSON.stringify({ + resourceType: 'ValueSet', id: 'cluster-file-source', url: VS, version: '1.0.0', + name: 'ClusterFileSource', status: 'active', compose: { include: [{ system: SYSTEM }] } + })); + cluster = await startCluster({ + front: FILE_FRONT, vsac: FILE_VSAC, workers: 2, + sources: [`file:${path.join(fixtures, 'cs.json')}`, `file:${path.join(fixtures, 'vs.json')}`] + }); + const cacheDir = path.join(cluster.dir, 'terminology-cache'); + const before = snapshot(cacheDir); + const folders = Object.keys(before).filter(k => !k.includes('/')); + // one folder per source, the master's, and nothing left from the build + expect(folders).toHaveLength(2); + for (const name of folders) { + expect(name).toMatch(/^folder-source-master-[0-9a-f]{16}$/); + expect(before[name + '/.valuesets-2.db']).toBeDefined(); + } + + const lookup = `/r4/CodeSystem/$lookup?system=${encodeURIComponent(SYSTEM)}&code=A`; + for (const port of [...cluster.workers, cluster.front, cluster.front, cluster.front]) { + const r = await request(port, 'GET', lookup); + expect(r.status).toBe(200); + expect(param(r.json, 'display').valueString).toBe('Alpha'); + const v = await request(port, 'GET', `/r4/ValueSet?url=${encodeURIComponent(VS)}`); + expect(v.status).toBe(200); + expect((v.json.entry || []).length).toBe(1); + } + + // the workers wrote nothing + expect(snapshot(cacheDir)).toEqual(before); + const log = fs.readFileSync(path.join(cluster.dir, 'processes.log'), 'utf8'); + expect(log).not.toMatch(/exited \(/); + expect(log).not.toMatch(/is not ready after/); + }, 600000); +}); diff --git a/tests/server/cluster-shared-cache.test.js b/tests/server/cluster-shared-cache.test.js new file mode 100644 index 00000000..9d5375a5 --- /dev/null +++ b/tests/server/cluster-shared-cache.test.js @@ -0,0 +1,112 @@ +/** + * server.cluster.sharedCache with real processes: a master and two tx-workers behind + * cluster.js, sharing databases/resource-cache.db. + * + * - a $cache-control session started on one process is used on each of the others, + * directly and through the launcher (which, without cacheAffinity, spreads cached + * requests by load), with no cache-id-unknown + * - an id no process issued is CACHE_ID_UNKNOWN_SHARED + * - a worker killed and restarted by the launcher still has the sessions + */ +const { startCluster, request, param, unknownCache, pidOn, sleep } = require('../utils/cluster-harness'); + +const FRONT = 20080; +const VSAC = 20089; + +const cs = (n) => ({ resourceType: 'CodeSystem', url: `http://example.org/shared/cs${n}`, version: '1', status: 'active', content: 'complete', + concept: [{ code: 'a', display: 'A' }, { code: 'b', display: 'B' }] }); +const vs = (n) => ({ resourceType: 'ValueSet', url: `http://example.org/shared/vs${n}`, version: '1', status: 'active', + compose: { include: [{ system: cs(n).url }] } }); + +function validate(port, cid, n, code = 'a') { + return request(port, 'POST', '/r4/ValueSet/$validate-code', { + headers: { 'X-Cache-Id': cid }, + body: { resourceType: 'Parameters', parameter: [{ name: 'url', valueUri: vs(n).url }, { name: 'coding', valueCoding: { system: cs(n).url, code } }] } + }); +} + +describe('cluster.js with server.cluster.sharedCache', () => { + let cluster; + + beforeAll(async () => { + cluster = await startCluster({ front: FRONT, vsac: VSAC, cluster: { sharedCache: true } }); + }, 330000); + + afterAll(async () => { + if (cluster) { + await cluster.stop(); + } + }); + + const sessions = []; + + test('a session started on one process is used on every other one', async () => { + const all = [cluster.master, ...cluster.workers]; + let unknown = 0; + let ok = 0; + for (const [i, port] of all.entries()) { + // started (sealed, with its resources) on this process, by its own port + const start = await request(port, 'POST', '/r4/$cache-control?mode=start', { + body: { resourceType: 'Parameters', parameter: [{ name: 'sealed', valueBoolean: true }, + { name: 'tx-resource', resource: cs(i) }, { name: 'tx-resource', resource: vs(i) }] } + }); + expect(start.status).toBe(200); + const cid = param(start.json, 'cache-id').valueId; + sessions.push({ cid, n: i, issuer: port }); + for (const other of all) { + for (const code of ['a', 'b']) { + const r = await validate(other, cid, i, code); + if (unknownCache(r)) { + unknown++; + } else if (r.status === 200 && param(r.json, 'result').valueBoolean === true) { + ok++; + } + } + } + } + expect(unknown).toBe(0); + expect(ok).toBe(all.length * all.length * 2); + expect(new Set(sessions.map(s => s.cid.split('.')[0]))).toEqual(new Set(['m0', 'tx1', 'tx2'])); + }, 60000); + + test('through the launcher, cached requests go to every tx process and all find their cache', async () => { + // concurrent, so the least-loaded choice spreads them + const results = await Promise.all(Array.from({ length: 60 }, (_, k) => { + const s = sessions[k % sessions.length]; + return validate(FRONT, s.cid, s.n); + })); + expect(results.filter(unknownCache)).toHaveLength(0); + expect(results.filter(r => r.status === 200 && param(r.json, 'result').valueBoolean === true)).toHaveLength(60); + }, 60000); + + test('an id that no process issued is unknown, in the shared store\'s words', async () => { + const r = await validate(FRONT, 'tx1.00000000-0000-4000-8000-000000000000', 0); + expect(r.status).toBe(404); + expect(unknownCache(r)).toBe(true); + const text = r.json.issue[0].details.text; + expect(text).toMatch(/survive a restart/); + expect(text).not.toMatch(/restarted since/); + expect(text).not.toMatch(/reached the wrong server/); + }); + + test('a worker that is killed and restarted still has the sessions, its own included', async () => { + const port = cluster.workers[0]; + const pid = pidOn(port); + expect(pid).toBeGreaterThan(0); + process.kill(pid, 'SIGKILL'); + const t0 = Date.now(); + while (pidOn(port) === pid || pidOn(port) === null) { + if (Date.now() - t0 > 120000) { + throw new Error('the worker was not restarted'); + } + await sleep(250); + } + await cluster.waitUp([port]); + expect(pidOn(port)).not.toBe(pid); + for (const s of sessions) { + const r = await validate(port, s.cid, s.n); + expect(unknownCache(r)).toBe(false); + expect(param(r.json, 'result').valueBoolean).toBe(true); + } + }, 180000); +}); diff --git a/tests/server/cluster-shared-closure.test.js b/tests/server/cluster-shared-closure.test.js new file mode 100644 index 00000000..4ac00a80 --- /dev/null +++ b/tests/server/cluster-shared-closure.test.js @@ -0,0 +1,236 @@ +/** + * server.cluster.sharedClosure with real processes: a master and two tx-workers behind + * cluster.js, all answering $closure on databases/closure.db. + * + * Contention: 3 processes x 4 clients add concepts at once - all of them to one table, and + * in fours to four more tables - each client a request at a time, one or two concepts per + * request, with overlaps between clients. Then each table is checked against a reference + * built by a single writer, and against the responses the clients were given: + * - the same concepts (none lost, none twice) and the same entries (none lost, none twice) + * - versions go up by one per write, with no gaps; every write was answered with its own + * version and exactly the entries stored at that version + * - "changes since v", asked of any process, is the union of the answers after v, for + * every v + * Also: 12 concurrent creates of one table make exactly one; creates racing for the table + * limit stop exactly at it; and $closure through the launcher reaches the workers. + */ +const path = require('path'); +const Database = require('better-sqlite3'); +const { startCluster, request, sleep } = require('../utils/cluster-harness'); +const fs = require('fs'); + +const FRONT = 20070; +const VSAC = 20079; +const MAX_TABLES = 40; +const NARROWER = 'source-is-narrower-than-target'; +const SYS = 'http://example.org/closure-contention/cs'; + +// a ternary tree of 40 concepts: c0 is the root, the parent of ci is c((i-1)/3) +const N = 40; +const parentOf = (i) => (i === 0 ? null : Math.floor((i - 1) / 3)); +function tree(i) { + const kids = []; + for (let j = 3 * i + 1; j <= 3 * i + 3 && j < N; j++) { + kids.push(tree(j)); + } + return { code: `c${i}`, display: `C${i}`, ...(kids.length ? { concept: kids } : {}) }; +} +const CS = { resourceType: 'CodeSystem', url: SYS, version: '1', status: 'active', content: 'complete', hierarchyMeaning: 'is-a', concept: [tree(0)] }; + +const params = (...parameter) => ({ resourceType: 'Parameters', parameter }); +const name = n => ({ name: 'name', valueString: n }); +const concept = code => ({ name: 'concept', valueCoding: { system: SYS, code } }); +const closure = (port, body) => request(port, 'POST', '/r4/$closure', { body }); + +function entries(cm) { + const out = []; + for (const g of (cm && cm.group) || []) { + for (const e of g.element || []) { + for (const t of e.target || []) { + out.push(`${e.code}>${t.code}`); + } + } + } + return out.sort(); +} + +// a deterministic shuffle, so a failure can be run again the same way +function rng(seed) { + let s = seed; + return () => { + s = (s * 1103515245 + 12345) % 2147483648; + return s / 2147483648; + }; +} +function shuffle(list, rand) { + const a = [...list]; + for (let i = a.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)); + [a[i], a[j]] = [a[j], a[i]]; + } + return a; +} + +describe('cluster.js with server.cluster.sharedClosure', () => { + let cluster; + let ports; + let db; + const stats = { busy: 0, retriesLogged: 0 }; + + beforeAll(async () => { + cluster = await startCluster({ + front: FRONT, vsac: VSAC, cluster: { sharedClosure: true }, + tx: { closure: { enabled: true, maxTables: MAX_TABLES } } + }); + ports = [cluster.master, ...cluster.workers]; + db = new Database(path.join(cluster.dir, 'databases', 'closure.db'), { readonly: true }); + }, 330000); + + afterAll(async () => { + if (db) { + db.close(); + } + if (cluster) { + // (the processes log to logs/server.log; the dated file is a copy) + const log = fs.readFileSync(path.join(cluster.dir, 'logs', 'server.log'), 'utf8'); + stats.retriesLogged = (log.match(/working the add out again/g) || []).length; + // for the record: how much contention the run produced + console.log(`shared closure: ${stats.retriesLogged} retries after a conflict, ${stats.busy} busy answers (retried by the client)`); + await cluster.stop(); + } + }); + + // one client: add its concepts a request at a time; a busy answer (429) is retried, as a + // client would. Returns what each successful add was answered with. + async function client(port, table, codes, rand) { + const answers = []; + let i = 0; + while (i < codes.length) { + const take = rand() < 0.5 ? 1 : 2; + const batch = codes.slice(i, i + take); + const r = await closure(port, params(name(table), ...batch.map(concept))); + if (r.status === 429) { + stats.busy++; + await sleep(5 + rand() * 20); + continue; + } + expect(r.status).toBe(200); + answers.push({ version: Number(r.json.version), entries: entries(r.json), codes: batch }); + i += take; + } + return answers; + } + + // the table as stored: concepts, entries by the version they were added in, the version + function stored(table) { + const t = db.prepare('SELECT id, version FROM closure_table WHERE name = ?').get(table); + const concepts = db.prepare('SELECT code, added_version FROM closure_concept WHERE table_id = ?').all(t.id); + const rows = db.prepare(`SELECT s.code AS s, t.code AS t, e.relationship AS rel, e.added_version AS v + FROM closure_entry e JOIN closure_concept s ON s.id = e.source_id JOIN closure_concept t ON t.id = e.target_id + WHERE e.table_id = ?`).all(t.id); + return { version: t.version, concepts, rows }; + } + + async function reference(table, codes) { + expect((await closure(cluster.master, params(name(table), { name: 'tx-resource', resource: CS }))).status).toBe(200); + const r = await closure(cluster.master, params(name(table), ...[...codes].sort().map(concept))); + expect(r.status).toBe(200); + return entries(r.json); + } + + async function check(table, answers, codes, ref) { + const s = stored(table); + // concepts: every one asked for, once + const asked = new Set(codes); + expect(s.concepts.map(c => c.code).sort()).toEqual([...asked].sort()); + // entries: the reference's, once each, all narrower-than in this code system + expect(s.rows.map(r => `${r.s}>${r.t}`).sort()).toEqual(ref); + expect(s.rows.every(r => r.rel === NARROWER)).toBe(true); + // versions: one per write, 1..version, no gaps + const versions = [...new Set(s.concepts.map(c => c.added_version))].sort((a, b) => a - b); + expect(versions).toEqual(Array.from({ length: s.version }, (_, i) => i + 1)); + // every write was answered with its version and exactly the entries stored at it + const byVersion = new Map(); + for (const r of s.rows) { + byVersion.set(r.v, [...(byVersion.get(r.v) || []), `${r.s}>${r.t}`]); + } + const writes = answers.filter(a => a.entries.length > 0); + expect(new Set(writes.map(a => a.version)).size).toBe(writes.length); + for (const a of writes) { + expect(a.entries).toEqual((byVersion.get(a.version) || []).sort()); + } + for (const v of byVersion.keys()) { + expect(writes.some(a => a.version === v)).toBe(true); + } + // no answer claims a version the table hasn't reached + expect(Math.max(...answers.map(a => a.version))).toBeLessThanOrEqual(s.version); + // changes since v, from any process, for every v + for (let v = 0; v <= s.version; v++) { + const r = await closure(ports[v % ports.length], params(name(table), { name: 'version', valueString: String(v) })); + expect(r.status).toBe(200); + expect(Number(r.json.version)).toBe(s.version); + const fromAnswers = writes.filter(a => a.version > v).flatMap(a => a.entries).sort(); + expect(entries(r.json)).toEqual(fromAnswers); + } + } + + test('3 processes x 4 clients adding to one table and to four others: the tables a single writer makes', async () => { + const all = Array.from({ length: N }, (_, i) => `c${i}`); + const tables = ['S', 'D0', 'D1', 'D2', 'D3']; + for (const t of tables) { + expect((await closure(ports[tables.indexOf(t) % 3], params(name(t), { name: 'tx-resource', resource: CS }))).status).toBe(200); + } + const runs = []; + const plan = []; + for (let p = 0; p < 3; p++) { + for (let k = 0; k < 4; k++) { + const c = p * 4 + k; + const rand = rng(1000 + c); + // the shared table: every concept is someone's, and each client takes 6 others too + const mine = all.filter((_, i) => i % 12 === c); + const S = shuffle([...mine, ...shuffle(all, rand).slice(0, 6)].filter((x, i, a) => a.indexOf(x) === i), rand); + // its four-way table: the three clients with this k share it + const D = shuffle(all.filter((_, i) => i % 4 === k || i % 7 === p), rand); + plan.push({ table: 'S', codes: S }, { table: `D${k}`, codes: D }); + runs.push(client(ports[p], 'S', S, rand).then(answers => ({ table: 'S', answers }))); + runs.push(client(ports[p], `D${k}`, D, rng(2000 + c)).then(answers => ({ table: `D${k}`, answers }))); + } + } + const results = await Promise.all(runs); + + for (const t of tables) { + const codes = [...new Set(plan.filter(x => x.table === t).flatMap(x => x.codes))]; + const answers = results.filter(r => r.table === t).flatMap(r => r.answers); + const ref = await reference(`${t}-reference`, codes); + expect(ref.length).toBeGreaterThan(0); + await check(t, answers, codes, ref); + } + }, 300000); + + test('12 concurrent creates of one table, on three processes: exactly one makes it', async () => { + const results = await Promise.all(Array.from({ length: 12 }, (_, i) => + closure(ports[i % 3], params(name('race'), { name: 'tx-resource', resource: CS })))); + const statuses = results.map(r => r.status).sort(); + expect(statuses.filter(s => s === 200)).toHaveLength(1); + expect(statuses.filter(s => s === 409)).toHaveLength(11); + expect(db.prepare("SELECT count(*) AS n FROM closure_table WHERE name = 'race'").get().n).toBe(1); + }, 60000); + + test('creates racing for the table limit stop exactly at it', async () => { + const before = db.prepare('SELECT count(*) AS n FROM closure_table').get().n; + const room = MAX_TABLES - before; + expect(room).toBeGreaterThan(5); + const tries = room + 15; + const results = await Promise.all(Array.from({ length: tries }, (_, i) => + closure(ports[i % 3], params(name(`limit-${i}`))))); + expect(results.filter(r => r.status === 200)).toHaveLength(room); + expect(results.filter(r => r.status === 422)).toHaveLength(15); + expect(db.prepare('SELECT count(*) AS n FROM closure_table').get().n).toBe(MAX_TABLES); + }, 60000); + + test('$closure through the launcher is answered', async () => { + // (that the launcher sends it to the least-loaded tx process is in cluster-launcher.test.js) + const rs = await Promise.all(Array.from({ length: 30 }, () => closure(FRONT, params(name('S'), { name: 'version', valueString: '0' })))); + expect(rs.every(r => r.status === 200)).toBe(true); + }, 60000); +}); diff --git a/tests/tx/closure-identity.test.js b/tests/tx/closure-identity.test.js new file mode 100644 index 00000000..60d375ea --- /dev/null +++ b/tests/tx/closure-identity.test.js @@ -0,0 +1,66 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { ClosureStore } = require('../../tx/closure/closure-store'); + +describe('shared closure table identities', () => { + let dir, file; + const opened = []; + const open = (shared = true) => { + const store = new ClosureStore({ database: file }, null, { shared }); + store.open(); + opened.push(store); + return store; + }; + const config = [{ name: 'system-version', valueUri: 'http://example.org/cs|old' }]; + const concepts = [{ key: 'a', system: 'http://example.org/cs', code: 'a' }]; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'closure-identity-')); + file = path.join(dir, 'closure.db'); + }); + afterEach(() => { + for (const store of opened.splice(0)) store.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test.each(['reset', 'dispose and recreate'])('%s rejects an outstanding write to the old table', change => { + const a = open(), b = open(); + const old = a.createTableChecked('table', config, { reset: false, maxTables: 1 }).table; + if (change === 'dispose and recreate') b.dropTable('table'); + const replacement = b.createTableChecked('table', [], { reset: true, maxTables: 1 }).table; + expect(a.recordAddChecked(old, concepts, [])).toBeNull(); + expect(a.getConcepts(replacement.id)).toEqual([]); + expect(a.getParams(replacement.id)).toEqual([]); + }); + + test('a duplicate-only add still checks the snapshot it read', () => { + const a = open(), b = open(); + const table = a.createTableChecked('table', config, { reset: false, maxTables: 1 }).table; + a.recordAddChecked(table, concepts, []); + const old = a.getTable('table'); + b.createTableChecked('table', [], { reset: true, maxTables: 1 }); + expect(a.recordAddChecked(old, [], [])).toBeNull(); + expect(a.getTable('table').version).toBe(0); + }); + + test('upgrading an existing database preserves its data, and deleted IDs stay retired across reopen', () => { + const legacy = open(false); + const old = legacy.createTable('existing', config); + legacy.recordAdd(old, concepts, []); + legacy.close(); + const a = open(); + expect(a.getTable('existing').id).toBe(old.id); + expect(a.getParams(old.id)).toEqual(config); + expect(a.getConcepts(old.id).map(c => c.code)).toEqual(['a']); + const next = a.createTableChecked('new', [], { reset: false, maxTables: 2 }).table; + expect(next.id).toBeGreaterThan(old.id); + a.dropTable('existing'); + a.dropTable('new'); + a.close(); + const b = open(); + const afterRestart = b.createTableChecked('new', [], { reset: false, maxTables: 2 }).table; + expect(afterRestart.id).toBeGreaterThan(next.id); + expect(b.db.pragma('foreign_key_check')).toEqual([]); + }); +}); diff --git a/tests/tx/closure-shared.test.js b/tests/tx/closure-shared.test.js new file mode 100644 index 00000000..cc28009f --- /dev/null +++ b/tests/tx/closure-shared.test.js @@ -0,0 +1,343 @@ +/** + * $closure with the tables shared between processes (server.cluster.sharedClosure). + * + * One TX module answers $closure; a second ClosureStore on the same file plays the other + * processes, writing to a table at chosen moments: after the module has read the table and + * worked out its add, and before it writes. Those are real conflicts on the SQLite file, so + * this checks the version-checked write and the retry: + * - a conflict makes the add work itself out again from the table as it is now, and the + * result is what one writer doing the same adds in that order would have got + * - a concept another process added meanwhile is not added twice + * - when every try conflicts, the call gets 429 CLOSURE_TABLE_BUSY and writes nothing + * - a resync reports only up to the version it read + * - creating a table checks for an existing one and the table limit in the same + * transaction as the insert + * The same against real processes is tests/server/cluster-shared-closure.test.js. + */ + +const express = require('express'); +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const request = require('supertest'); +const TXModule = require('../../tx/tx'); +const { ClosureStore } = require('../../tx/closure/closure-store'); +const ClosureWorker = require('../../tx/workers/closure'); + +const LIBRARY = path.join(__dirname, 'fixtures', 'test-library.yaml'); +const R5 = '/tx/r5'; +const SYS = 'http://example.org/closure-shared/cs'; +const NARROWER = 'source-is-narrower-than-target'; + +// r > a > (a1, a2); r > b; and u1..u6, related to nothing +const CS = { + resourceType: 'CodeSystem', url: SYS, version: '1', status: 'active', content: 'complete', hierarchyMeaning: 'is-a', + concept: [ + { code: 'r', display: 'R', concept: [ + { code: 'a', display: 'A', concept: [{ code: 'a1', display: 'A1' }, { code: 'a2', display: 'A2' }] }, + { code: 'b', display: 'B' } + ] }, + ...[1, 2, 3, 4, 5, 6].map(i => ({ code: `u${i}`, display: `U${i}` })) + ] +}; + +const params = (...parameter) => ({ resourceType: 'Parameters', parameter }); +const name = n => ({ name: 'name', valueString: n }); +const concept = code => ({ name: 'concept', valueCoding: { system: SYS, code } }); +const init = (n) => params(name(n), { name: 'tx-resource', resource: CS }); +const n = (s, t) => `${s}>${t}:${NARROWER}`; + +function entries(cm) { + const out = []; + for (const g of cm.group || []) { + for (const e of g.element || []) { + for (const t of e.target || []) { + out.push(`${e.code}>${t.code}:${t.relationship}`); + } + } + } + return out.sort(); +} + +async function startApp(closure) { + const app = express(); + app.use(express.json()); + const txModule = new TXModule(); + await txModule.initialize({ librarySource: LIBRARY, endpoints: [{ path: R5, fhirVersion: '5.0', context: null }], + closure, sharedClosure: true }, app); + return { app, txModule }; +} + +const post = (app, body) => request(app).post(`${R5}/$closure`) + .set('Content-Type', 'application/json').set('Accept', 'application/fhir+json').send(body); + +describe('$closure on a shared database', () => { + let dir, file, app, txModule, other; + // concepts "another process" writes, with the entries a correct writer would make + const RELATED = { r: [], a: ['r'], b: ['r'], a1: ['a', 'r'], a2: ['a', 'r'] }; + + /** Another process adds a concept to a table (computing its entries as it should). */ + function otherAdds(table, code) { + const t = other.getTable(table); + const ids = new Map(other.getConcepts(t.id).map(c => [c.code, c.id])); + if (ids.has(code)) { + return t.version; + } + const ancestors = (RELATED[code] || []).filter(c => ids.has(c)); + const descendants = Object.entries(RELATED).filter(([c, up]) => up.includes(code) && ids.has(c)).map(([c]) => c); + const v = other.recordAddChecked(t, [{ key: 'k', system: SYS, code, display: code.toUpperCase() }], [ + ...ancestors.map(up => ({ source: 'k', target: ids.get(up), relationship: NARROWER })), + ...descendants.map(d => ({ source: ids.get(d), target: 'k', relationship: NARROWER })) + ]); + expect(v).toBe(t.version + 1); + return v; + } + + /** The next calls of the module's recordAddChecked each run inject() first. */ + function interfere(inject) { + const store = txModule.closureStore; + const real = store.recordAddChecked.bind(store); + let calls = 0; + store.recordAddChecked = (table, concepts, list) => { + calls++; + if (concepts.length > 0) { + inject(calls, table); + } + return real(table, concepts, list); + }; + return { calls: () => calls, restore: () => { store.recordAddChecked = real; } }; + } + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'closure-shared-')); + file = path.join(dir, 'closure.db'); + ({ app, txModule } = await startApp({ enabled: true, database: file, writeRetries: 3, maxTables: 8 })); + other = new ClosureStore({ database: file }, null, { shared: true }); + other.open(); + }, 240000); + + afterAll(async () => { + other.close(); + await txModule.shutdown(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('the module\'s store is the shared one', () => { + expect(txModule.closureStore.shared).toBe(true); + expect(txModule.closureStore.db.pragma('busy_timeout', { simple: true })).toBe(5000); + }); + + test('conflicts: the add is worked out again from the table as it is now', async () => { + const T = 'conflict-1'; + expect((await post(app, init(T))).status).toBe(200); + expect((await post(app, params(name(T), concept('a')))).body.version).toBe('1'); + + // three tries are each beaten by another process's add of an unrelated concept + const orig = ClosureWorker.prototype.memoSubsumes; + let hits = 0; + let misses = 0; + const subsumes = jest.spyOn(ClosureWorker.prototype, 'memoSubsumes').mockImplementation(async function (memo, nn, o) { + if (memo.subsumes.has(nn.system + '|' + nn.code + '|' + o.code)) { + hits++; + } else { + misses++; + } + return orig.call(this, memo, nn, o); + }); + const i = interfere((call) => { + if (call <= 3) { + otherAdds(T, `u${call}`); + } + }); + let res; + try { + res = await post(app, params(name(T), concept('a1'), concept('r'))); + } finally { + i.restore(); + } + expect(res.status).toBe(200); + expect(i.calls()).toBe(4); + // a (1), u1 u2 u3 (2-4), then this add (5) + expect(res.body.version).toBe('5'); + expect(entries(res.body)).toEqual([n('a', 'r'), n('a1', 'a'), n('a1', 'r')].sort()); + // each pair was tested once, and remembered across the four tries: a1 and r each + // against a, u1, u2, u3, and r against a1 + subsumes.mockRestore(); + expect(misses).toBe(9); + expect(hits).toBeGreaterThan(0); + + // the table is what one writer doing the same adds in the same order gets + const R = 'conflict-1-reference'; + await post(app, init(R)); + for (const code of ['a', 'u1', 'u2', 'u3']) { + await post(app, params(name(R), concept(code))); + } + const ref = await post(app, params(name(R), concept('a1'), concept('r'))); + expect(ref.body.version).toBe('5'); + expect(entries(ref.body)).toEqual(entries(res.body)); + const whole = async (t) => entries((await post(app, params(name(t), { name: 'version', valueString: '0' }))).body); + expect(await whole(T)).toEqual(await whole(R)); + }); + + test('a concept another process added meanwhile is not added twice, and its entries are its own', async () => { + const T = 'conflict-2'; + await post(app, init(T)); + await post(app, params(name(T), concept('r'))); + const i = interfere((call) => { + if (call === 1) { + otherAdds(T, 'b'); + } + }); + let res; + try { + res = await post(app, params(name(T), concept('b'), concept('a2'))); + } finally { + i.restore(); + } + expect(res.status).toBe(200); + expect(i.calls()).toBe(2); + expect(res.body.version).toBe('3'); + // b>r was the other process's add (version 2); this call added only a2 + expect(entries(res.body)).toEqual([n('a2', 'r')]); + const db = txModule.closureStore; + const t = db.getTable(T); + expect(db.getConcepts(t.id).map(c => c.code).sort()).toEqual(['a2', 'b', 'r']); + expect(entries((await post(app, params(name(T), { name: 'version', valueString: '0' }))).body)) + .toEqual([n('a2', 'r'), n('b', 'r')]); + // and each version's changes are the ones its add reported + expect(entries((await post(app, params(name(T), { name: 'version', valueString: '2' }))).body)).toEqual([n('a2', 'r')]); + }); + + test('when every try conflicts: 429 CLOSURE_TABLE_BUSY after writeRetries, and nothing of the call is written', async () => { + const T = 'conflict-3'; + await post(app, init(T)); + const i = interfere((call) => otherAdds(T, `u${call}`)); + let res; + try { + res = await post(app, params(name(T), concept('a'), concept('a1'))); + } finally { + i.restore(); + } + expect(res.status).toBe(429); + const issue = res.body.issue[0]; + expect(issue.code).toBe('too-costly'); + expect(issue.details.text).toMatch(/busy/i); + expect(i.calls()).toBe(4); // the first try and writeRetries (3) more + const db = txModule.closureStore; + const t = db.getTable(T); + expect(t.version).toBe(4); + expect(db.getConcepts(t.id).map(c => c.code).sort()).toEqual(['u1', 'u2', 'u3', 'u4']); + }); + + test('a table dropped by another process before the write: unknown, 404', async () => { + const T = 'conflict-4'; + await post(app, init(T)); + const i = interfere(() => other.dropTable(T)); + let res; + try { + res = await post(app, params(name(T), concept('a'))); + } finally { + i.restore(); + } + expect(res.status).toBe(404); + expect(txModule.closureStore.getTable(T)).toBeNull(); + }); + + test.each(['reset', 'dispose and recreate'])('%s during an add uses the replacement configuration on retry', async (change) => { + const T = 'replace-during-add'; + expect((await post(app, init(T))).status).toBe(200); + const replacement = { ...CS, concept: [ + { code: 'a', display: 'New A', concept: [{ code: 'r', display: 'New R' }] } + ] }; + const i = interfere((call) => { + if (call === 1) { + if (change === 'dispose and recreate') { + other.dropTable(T); + } + other.createTableChecked(T, [{ name: 'tx-resource', resource: replacement }], + { reset: true, maxTables: Infinity }); + } + }); + try { + const res = await post(app, params(name(T), concept('a'), concept('r'))); + expect(res.status).toBe(200); + // Same canonical/version and codes, but the replacement reverses their relationship. + // A retry must discard the old providers, displays and subsumption memo as well. + expect(entries(res.body)).toEqual([n('r', 'a')]); + expect(i.calls()).toBe(2); + expect(other.getConcepts(other.getTable(T).id).map(c => c.display).sort()).toEqual(['New A', 'New R']); + } finally { + i.restore(); + other.dropTable(T); + } + }); + + test('a reset during resync refuses the interrupted snapshot', async () => { + const T = 'replace-during-resync'; + await post(app, init(T)); + await post(app, params(name(T), concept('a'), concept('r'))); + const store = txModule.closureStore; + const realPage = store.entriesPage.bind(store); + const page = jest.spyOn(store, 'entriesPage').mockImplementation((...args) => { + const rows = realPage(...args); + other.createTableChecked(T, init(T).parameter.slice(1), { reset: true, maxTables: Infinity }); + return rows; + }); + try { + const res = await post(app, params(name(T), { name: 'version', valueString: '0' })); + expect(res.status).toBe(409); + expect(res.body.issue[0].code).toBe('conflict'); + expect(other.getTable(T).version).toBe(0); + } finally { + page.mockRestore(); + other.dropTable(T); + } + }); + + test('a resync reports only up to the version it read', () => { + const T = 'bounded'; + const made = other.createTableChecked(T, [], { reset: false, maxTables: Infinity }); + const t = made.table; + other.recordAdd(t, [{ key: 'r', system: SYS, code: 'r' }], []); + const t1 = other.getTable(T); + const rId = other.getConcepts(t1.id)[0].id; + other.recordAdd(t1, [{ key: 'a', system: SYS, code: 'a' }], [{ source: 'a', target: rId, relationship: NARROWER }]); + const t2 = other.getTable(T); + other.recordAdd(t2, [{ key: 'b', system: SYS, code: 'b' }], [{ source: 'b', target: rId, relationship: NARROWER }]); + // read at version 2, while version 3 has since been written + expect(other.entriesPage(t.id, 0, 0, 100, 2).map(e => e.source_code)).toEqual(['a']); + expect(other.entriesPage(t.id, 0, 0, 100, 3).map(e => e.source_code)).toEqual(['a', 'b']); + expect(other.entriesPage(t.id, 0, 0, 100).map(e => e.source_code)).toEqual(['a', 'b']); + }); + + test('creating: an existing table and the table limit are checked in the write transaction', async () => { + // another process creates the table between this one's read and its write + const store = txModule.closureStore; + const realGet = store.getTable.bind(store); + store.getTable = (nm) => { + const t = realGet(nm); + if (nm === 'raced' && !t) { + other.createTableChecked('raced', [], { reset: false, maxTables: Infinity }); + } + return t; + }; + let res; + try { + res = await post(app, init('raced')); + } finally { + store.getTable = realGet; + } + expect(res.status).toBe(409); + + // the limit (8): fill up to it from "another process", then one more from the module + const count = store.tableCount(); + for (let k = count; k < 8; k++) { + expect(other.createTableChecked(`fill-${k}`, [], { reset: false, maxTables: 8 }).table).toBeDefined(); + } + expect(other.createTableChecked('one-too-many', [], { reset: false, maxTables: 8 })).toEqual({ refused: 'limit' }); + expect((await post(app, init('one-too-many'))).status).toBe(422); + // a reset of an existing table is not a new table + expect(other.createTableChecked('fill-7', [], { reset: true, maxTables: 8 }).table).toBeDefined(); + expect(store.tableCount()).toBe(8); + }); +}); diff --git a/tests/tx/expansion-key-hash.test.js b/tests/tx/expansion-key-hash.test.js new file mode 100644 index 00000000..e510857b --- /dev/null +++ b/tests/tx/expansion-key-hash.test.js @@ -0,0 +1,369 @@ +/** + * ExpansionCache.computeKey hashes the content of every additional resource. With a + * cache-id that is the client's whole $cache-control cache, on every $expand, so each + * wrapper remembers its hash (CanonicalResource.expansionKeyHash) instead of serialising + * and hashing itself again. + * + * - the key is byte-identical to the one computeKey made before (the reference below is + * that implementation, copied), so expansions already cached stay valid; + * - a resource is hashed once however many keys it goes into; + * - a different resource, or a different jsonObj, gets its own hash; + * - and nothing writes to a cached resource's json while it is used through a cache-id, + * which is what makes remembering the hash safe. + */ + +const crypto = require('crypto'); +const request = require('supertest'); +const { ExpansionCache } = require('../../tx/operation-context'); +const { CanonicalResource } = require('../../tx/library/canonical-resource'); +const { CodeSystem } = require('../../tx/library/codesystem'); +const ValueSet = require('../../tx/library/valueset'); +const { ConceptMap } = require('../../tx/library/conceptmap'); +const { getTestApp, getTxModule, shutdownTestApp } = require('./setup'); + +// computeKey as it was before the hash was remembered - the reference the new key must match +function referenceComputeKey(valueSet, params, additionalResources) { + const keyParts = []; + const vsJson = valueSet.jsonObj || valueSet; + keyParts.push(`vs:${JSON.stringify(vsJson)}`); + if (params) { + keyParts.push(`params:` + params.hashSource()); + } + if (additionalResources && additionalResources.length > 0) { + const resourceHashes = additionalResources + .map(r => { + const json = r.jsonObj || r; + return crypto.createHash('sha256') + .update(JSON.stringify(json)) + .digest('hex') + .substring(0, 16); + }) + .sort(); + keyParts.push(`additional:${resourceHashes.join(',')}`); + } + const keyString = keyParts.join('||'); + return crypto.createHash('sha256').update(keyString).digest('hex'); +} + +const CS_URL = 'http://example.org/p6/colors'; + +function colorsCS(extra = {}) { + return { + resourceType: 'CodeSystem', url: CS_URL, version: '1.0.0', status: 'active', content: 'complete', + hierarchyMeaning: 'is-a', + property: [{ code: 'shade', type: 'code' }], + concept: [ + { code: 'red', display: 'Red', property: [{ code: 'shade', valueCode: 'warm' }], + concept: [{ code: 'crimson', display: 'Crimson' }] }, + { code: 'green', display: 'Green', property: [{ code: 'shade', valueCode: 'cool' }] }, + { code: 'blue', display: 'Blue', designation: [{ language: 'fr', value: 'Bleu' }] } + ], + ...extra + }; +} + +function supplementCS() { + return { + resourceType: 'CodeSystem', url: 'http://example.org/p6/colors-de', version: '1.0.0', status: 'active', + content: 'supplement', supplements: `${CS_URL}|1.0.0`, + concept: [{ code: 'red', designation: [{ language: 'de', value: 'Rot' }] }] + }; +} + +function enumeratedVS() { + return { + resourceType: 'ValueSet', url: 'http://example.org/p6/warm-vs', version: '1.0.0', status: 'active', + compose: { include: [{ system: CS_URL, concept: [{ code: 'red' }, { code: 'blue', display: 'Blue' }] }] } + }; +} + +function filterVS() { + return { + resourceType: 'ValueSet', url: 'http://example.org/p6/red-family', version: '1.0.0', status: 'active', + compose: { + include: [ + { system: CS_URL, filter: [{ property: 'concept', op: 'is-a', value: 'red' }] }, + { system: CS_URL, filter: [{ property: 'shade', op: '=', value: 'cool' }] } + ], + exclude: [{ system: CS_URL, concept: [{ code: 'crimson' }] }] + } + }; +} + +function importingVS() { + return { + resourceType: 'ValueSet', url: 'http://example.org/p6/all', version: '1.0.0', status: 'active', + compose: { include: [{ valueSet: ['http://example.org/p6/warm-vs|1.0.0'] }, { system: CS_URL }] } + }; +} + +function conceptMap() { + return { + resourceType: 'ConceptMap', url: 'http://example.org/p6/cm', version: '1.0.0', status: 'active', + group: [{ source: CS_URL, target: 'http://example.org/p6/other', + element: [{ code: 'red', target: [{ code: 'r', relationship: 'equivalent' }] }] }] + }; +} + +function wrapped() { + return [ + new CodeSystem(colorsCS()), + new CodeSystem(supplementCS()), + new ValueSet(enumeratedVS()), + new ValueSet(filterVS()), + new ValueSet(importingVS()), + new ConceptMap(conceptMap()) + ]; +} + +const params = (s) => ({ hashSource: () => s }); + +describe('ExpansionCache.computeKey with remembered resource hashes', () => { + const cache = new ExpansionCache(null); + + test('makes exactly the key the previous implementation made', () => { + const resources = wrapped(); + const raw = [colorsCS({ version: '2.0.0' }), enumeratedVS()]; // plain json is still accepted + const vsWrapper = new ValueSet(filterVS()); + const cases = [ + [vsWrapper, null, undefined], + [vsWrapper, null, []], + [vsWrapper, params('lang=en|count=10'), resources], + [filterVS(), params(''), resources], + [vsWrapper, params('x'), [...resources].reverse()], + [vsWrapper, params('x'), resources.slice(2)], + [vsWrapper, params('x'), raw], + [vsWrapper, params('x'), [...raw, ...resources]], + [vsWrapper, params('x'), [resources[0], resources[0]]] + ]; + for (const [vs, p, extra] of cases) { + // twice: the second time the hashes are remembered + expect(cache.computeKey(vs, p, extra)).toBe(referenceComputeKey(vs, p, extra)); + expect(cache.computeKey(vs, p, extra)).toBe(referenceComputeKey(vs, p, extra)); + } + }); + + test('a resource\'s hash is the one the key used to use', () => { + for (const r of wrapped()) { + expect(r.expansionKeyHash()).toBe( + crypto.createHash('sha256').update(JSON.stringify(r.jsonObj)).digest('hex').substring(0, 16)); + } + }); + + test('each resource is serialised and hashed once, however many keys it goes into', () => { + const resources = wrapped(); + const vs = new ValueSet(enumeratedVS()); + const stringified = new Set(); + const realStringify = JSON.stringify; + const stringify = jest.spyOn(JSON, 'stringify').mockImplementation(function (v, ...rest) { + stringified.add(v); + return realStringify.call(JSON, v, ...rest); + }); + const createHash = jest.spyOn(crypto, 'createHash'); + try { + const first = cache.computeKey(vs, params('a'), resources); + // one hash per resource, plus the key itself + expect(createHash).toHaveBeenCalledTimes(resources.length + 1); + for (const r of resources) { + expect(stringified.has(r.jsonObj)).toBe(true); + } + createHash.mockClear(); + stringify.mockClear(); + stringified.clear(); + for (let i = 0; i < 50; i++) { + expect(cache.computeKey(vs, params('a'), resources)).toBe(first); + cache.computeKey(vs, params('b' + i), [...resources].reverse()); + } + // only the key itself is hashed, and only the value set being expanded is serialised + expect(createHash).toHaveBeenCalledTimes(100); + expect(stringify).toHaveBeenCalledTimes(100); + for (const r of resources) { + expect(stringified.has(r.jsonObj)).toBe(false); + } + } finally { + stringify.mockRestore(); + createHash.mockRestore(); + } + }); + + test('a replacement resource (a new object) gets its own hash, and so a new key', () => { + const vs = new ValueSet(enumeratedVS()); + const before = wrapped(); + const keyBefore = cache.computeKey(vs, null, before); + const changed = new CodeSystem(colorsCS({ title: 'Colours, revised' })); + const after = [changed, ...before.slice(1)]; + expect(changed.expansionKeyHash()).not.toBe(before[0].expansionKeyHash()); + const keyAfter = cache.computeKey(vs, null, after); + expect(keyAfter).not.toBe(keyBefore); + expect(keyAfter).toBe(referenceComputeKey(vs, null, after)); + // an identical replacement is a new object with the same content: same hash, same key + const same = [new CodeSystem(colorsCS()), ...before.slice(1)]; + expect(cache.computeKey(vs, null, same)).toBe(keyBefore); + }); + + test('a wrapper whose jsonObj is replaced is hashed again', () => { + const cs = new CodeSystem(colorsCS()); + const h1 = cs.expansionKeyHash(); + cs.jsonObj = colorsCS({ title: 'Other' }); + expect(cs.expansionKeyHash()).not.toBe(h1); + expect(cs.expansionKeyHash()).toBe(CanonicalResource.computeExpansionKeyHash(cs.jsonObj)); + }); + + test('a store that knows the hash can record it, and it is used without hashing', () => { + const cs = new CodeSystem(colorsCS()); + const known = CanonicalResource.computeExpansionKeyHash(cs.jsonObj); + cs.setExpansionKeyHash(known); + const createHash = jest.spyOn(crypto, 'createHash'); + try { + expect(cs.expansionKeyHash()).toBe(known); + expect(createHash).not.toHaveBeenCalled(); + } finally { + createHash.mockRestore(); + } + }); + + test('the remembered hash never serialises or copies', () => { + const cs = new CodeSystem(colorsCS()); + const h = cs.expansionKeyHash(); + expect(Object.keys(cs)).not.toContain('_expansionKeyHashMemo'); + expect(JSON.stringify(cs)).not.toContain(h); + expect({ ...cs }._expansionKeyHashMemo).toBeUndefined(); + }); +}); + +// Freeze a resource's json, so that anything that writes to it while it is in use throws +// (class bodies are strict code). The one exception: compose filter elements stay +// extensible, because a code system provider remembers its analysis of a cached filter on +// the element itself under a non-enumerable symbol (cs-cs.js FILTER_ANALYSIS) - which JSON +// never sees. Their existing properties are still read-only, and the test checks nothing +// enumerable was added to them. +function freezeJson(o, filterElements, inFilter = false) { + if (!o || typeof o !== 'object' || Object.isFrozen(o)) { + return o; + } + for (const [k, v] of Object.entries(o)) { + freezeJson(v, filterElements, k === 'filter' && Array.isArray(v)); + } + if (inFilter) { + // the filter array itself + Object.freeze(o); + return o; + } + if (filterElements && filterElements.has(o)) { + for (const k of Object.keys(o)) { + Object.defineProperty(o, k, { writable: false, configurable: false }); + } + return o; + } + Object.freeze(o); + return o; +} + +function collectFilterElements(json, into) { + for (const part of ['include', 'exclude']) { + for (const inc of (json.compose && json.compose[part]) || []) { + for (const f of inc.filter || []) { + into.add(f); + } + } + } + return into; +} + +describe('cache-id resources are not written to while they are used', () => { + let app; + + beforeAll(async () => { + app = await getTestApp(); + }, 60000); + + afterAll(async () => { + await shutdownTestApp(); + }); + + function post(url, cacheId, parameter) { + return request(app) + .post(url) + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .set('x-cache-id', cacheId) + .send({ resourceType: 'Parameters', parameter }); + } + + test('expand, validate-code, lookup and compare through a cache-id leave the cached json untouched', async () => { + const start = await request(app) + .post('/tx/r5/$cache-control') + .query({ mode: 'start' }) + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .send({ resourceType: 'Parameters', parameter: [ + { name: 'sealed', valueBoolean: true }, + ...[colorsCS(), supplementCS(), enumeratedVS(), filterVS(), importingVS(), conceptMap()] + .map(resource => ({ name: 'tx-resource', resource })) + ] }); + expect(start.status).toBe(200); + const cacheId = start.body.parameter.find(p => p.name === 'cache-id').valueId; + + const endpoint = getTxModule().endpoints.find(e => e.resourceCache && e.resourceCache.has(cacheId)); + const cached = endpoint.resourceCache.get(cacheId); + expect(cached.length).toBe(6); + const filterElements = new Set(); + const before = new Map(); + for (const r of cached) { + collectFilterElements(r.jsonObj, filterElements); + before.set(r, JSON.stringify(r.jsonObj)); + freezeJson(r.jsonObj, filterElements); + r.expansionKeyHash(); + } + expect(filterElements.size).toBe(2); + + // every expansion takes the cache path, so computeKey and the cache store both run + const expansionCache = endpoint.expansionCache; + const wasForced = expansionCache.forceCaching; + expansionCache.forceCaching = true; + try { + const calls = []; + for (const url of ['http://example.org/p6/warm-vs', 'http://example.org/p6/red-family', 'http://example.org/p6/all']) { + for (let i = 0; i < 2; i++) { // the second is answered from the expansion cache + calls.push(['expand', url, await post('/tx/r5/ValueSet/$expand', cacheId, [ + { name: 'url', valueUri: url }, { name: 'includeDefinition', valueBoolean: true }, + { name: 'displayLanguage', valueCode: 'de' }])]); + } + calls.push(['expand+supplement', url, await post('/tx/r5/ValueSet/$expand', cacheId, [ + { name: 'url', valueUri: url }, { name: 'useSupplement', valueCanonical: 'http://example.org/p6/colors-de' }])]); + for (const code of ['red', 'crimson', 'green', 'nope']) { + calls.push(['validate-code', `${url}#${code}`, await post('/tx/r5/ValueSet/$validate-code', cacheId, [ + { name: 'url', valueUri: url }, { name: 'system', valueUri: CS_URL }, { name: 'code', valueCode: code }, + { name: 'display', valueString: 'Red' }])]); + } + } + calls.push(['cs validate-code', CS_URL, await post('/tx/r5/CodeSystem/$validate-code', cacheId, [ + { name: 'url', valueUri: CS_URL }, { name: 'code', valueCode: 'blue' }, { name: 'display', valueString: 'Bleu' }])]); + calls.push(['lookup', CS_URL, await post('/tx/r5/CodeSystem/$lookup', cacheId, [ + { name: 'system', valueUri: CS_URL }, { name: 'code', valueCode: 'red' }, { name: 'property', valueCode: '*' }])]); + calls.push(['compare', 'warm vs all', await post('/tx/r5/ValueSet/$compare', cacheId, [ + { name: 'thisUrl', valueUri: 'http://example.org/p6/warm-vs' }, { name: 'otherUrl', valueUri: 'http://example.org/p6/all' }])]); + calls.push(['compare', 'family vs family', await post('/tx/r5/ValueSet/$compare', cacheId, [ + { name: 'thisUrl', valueUri: 'http://example.org/p6/red-family' }, { name: 'otherUrl', valueUri: 'http://example.org/p6/red-family' }])]); + + // a write to frozen json throws, which would come back as a server error + for (const [op, what, res] of calls) { + expect({ op, what, status: res.status < 500 ? 'ok' : res.status, body: res.status < 500 ? null : res.body }) + .toEqual({ op, what, status: 'ok', body: null }); + } + const expanded = calls.filter(([op, , res]) => op === 'expand' && res.status === 200); + expect(expanded.length).toBe(6); + expect(calls.filter(([op, , res]) => op === 'validate-code' && res.status === 200).length).toBe(12); + } finally { + expansionCache.forceCaching = wasForced; + } + + for (const r of cached) { + expect(JSON.stringify(r.jsonObj)).toBe(before.get(r)); + expect(r.expansionKeyHash()).toBe(CanonicalResource.computeExpansionKeyHash(r.jsonObj)); + } + for (const f of filterElements) { + expect(Object.keys(f).sort()).toEqual(['op', 'property', 'value']); + } + }, 60000); +}); diff --git a/tests/tx/package-cache-readonly.test.js b/tests/tx/package-cache-readonly.test.js new file mode 100644 index 00000000..91e8e00a --- /dev/null +++ b/tests/tx/package-cache-readonly.test.js @@ -0,0 +1,89 @@ +/** + * A tx-worker's package caches (cluster.md): ValueSetDatabase with readOnly never writes the + * file - one that still needs a migration (here, rows without content_hash) fails to open, and + * opens once its builder has migrated it - and a row is never written without its hash. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const sqlite3 = require('sqlite3'); +const { ValueSetDatabase } = require('../../tx/vs/vs-database'); +const { waitForCache } = require('../../tx/library/wait-for-cache'); + +const vs = (id) => ({ resourceType: 'ValueSet', id, url: `http://example.org/vs/${id}`, version: '1', status: 'active' }); + +function run(file, sql) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(file, (err) => { + if (err) { + return reject(err); + } + db.run(sql, [], (err) => db.close(() => err ? reject(err) : resolve())); + }); + }); +} + +describe('package value set cache, read-only', () => { + let dir; + let file; + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vs-ro-')); + file = path.join(dir, '.valuesets-2.db'); + const db = new ValueSetDatabase(file); + await db.create(); + await db.upsertValueSet(vs('a')); + await db.upsertValueSet(vs('b')); + await db.markComplete(); + await db.close(); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('a value set is stored with the hash the backfill would give it', async () => { + const db = new ValueSetDatabase(file, { readOnly: true }); + const conn = await db._getReadConnection(); + const rows = await new Promise((resolve, reject) => conn.all('SELECT content, content_hash FROM valuesets', [], (e, r) => e ? reject(e) : resolve(r))); + expect(rows.length).toBe(2); + for (const row of rows) { + expect(row.content_hash).toBe(crypto.createHash('sha256').update(row.content).digest('hex')); + } + expect(await db.isComplete()).toBe(true); + expect((await db.loadAllValueSets('test')).size).toBeGreaterThan(0); + await db.close(); + }); + + test('a file that needs a backfill is not opened read-only, nor changed; it is once migrated', async () => { + await run(file, 'UPDATE valuesets SET content_hash = NULL'); + const before = fs.readFileSync(file); + const ro = new ValueSetDatabase(file, { readOnly: true }); + expect(await ro.isComplete()).toBe(false); + await expect(ro.getSetting('cache-complete')).rejects.toThrow(/not migrated yet/); + expect(fs.readFileSync(file).equals(before)).toBe(true); + await expect(ro.upsertValueSet(vs('c'))).rejects.toThrow(); + + // the builder (the master) opens it, and the backfill runs there + const rw = new ValueSetDatabase(file); + expect(await rw.isComplete()).toBe(true); + await rw.close(); + expect(await ro.isComplete()).toBe(true); + // and a write through the read-only one still fails + await expect(ro.setSetting('x', 'y')).rejects.toThrow(/SQLITE_READONLY/); + await ro.close(); + }); + + test('a missing file is not created', async () => { + const missing = path.join(dir, 'none.db'); + const ro = new ValueSetDatabase(missing, { readOnly: true }); + expect(await ro.isComplete()).toBe(false); + expect(fs.existsSync(missing)).toBe(false); + }); + + test('waitForCache: waits for the cache, and gives up with a clear error', async () => { + let n = 0; + await waitForCache(async () => ++n >= 2, 'test cache', { waitMs: 5000 }); + expect(n).toBe(2); + await expect(waitForCache(async () => false, 'test cache', { waitMs: 10 })).rejects.toThrow(/the master builds it/); + }); +}); diff --git a/tests/tx/resource-cache-multiprocess.test.js b/tests/tx/resource-cache-multiprocess.test.js new file mode 100644 index 00000000..a9bcd845 --- /dev/null +++ b/tests/tx/resource-cache-multiprocess.test.js @@ -0,0 +1,148 @@ +/** + * The shared SQLite resource cache across processes: what server.cluster.sharedCache relies on. + * Two real processes open the same file; a cache started in one is used and ended + * in the other, and survives a process restart. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { fork } = require('child_process'); + +const CHILD = path.join(__dirname, '..', 'utils', 'resource-cache-child.js'); + +function startProcess(file, maxBytes = 0) { + const child = fork(CHILD, [file, String(maxBytes)], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }); + let seq = 0; + const waiting = new Map(); + let readyResolve; + const ready = new Promise(resolve => { readyResolve = resolve; }); + child.on('message', (m) => { + if (m.ready) { + readyResolve(m.ready); + return; + } + const w = waiting.get(m.seq); + waiting.delete(m.seq); + if (m.error) { + w.reject(new Error(m.error)); + } else { + w.resolve(m.result); + } + }); + const call = (op, args = {}) => new Promise((resolve, reject) => { + const s = ++seq; + waiting.set(s, { resolve, reject }); + child.send({ seq: s, op, ...args }); + }); + const stop = () => new Promise(resolve => { + child.once('exit', resolve); + child.kill(); + }); + return { ready, call, stop, pid: child.pid }; +} + +const cs = (url, version, codes) => ({ + resourceType: 'CodeSystem', url, version, status: 'active', content: 'complete', + concept: codes.map(code => ({ code, display: code.toUpperCase() })) +}); + +describe('shared SQLite resource cache across processes', () => { + let dir; + let file; + const running = []; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'resource-cache-mp-')); + file = path.join(dir, 'resource-cache.db'); + }); + + afterEach(async () => { + while (running.length) { + await running.pop().stop(); + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + const spawn = async (maxBytes = 0) => { + const p = startProcess(file, maxBytes); + running.push(p); + await p.ready; + return p; + }; + + test('A starts a cache and adds resources, B reads them, B ends it, A sees the tombstone', async () => { + const a = await spawn(); + const b = await spawn(); + expect(a.pid).not.toBe(b.pid); + + const id = await a.call('start', { resources: [cs('http://example.org/one', '1', ['a', 'b'])] }); + await a.call('add', { id, resources: [cs('http://example.org/two', '2', ['x', 'y', 'z'])] }); + + expect(await b.call('has', { id })).toBe(true); + expect(await b.call('get', { id })).toEqual([ + { cls: 'CodeSystem', url: 'http://example.org/one', version: '1', fhirVersion: 'R4', concepts: 2 }, + { cls: 'CodeSystem', url: 'http://example.org/two', version: '2', fhirVersion: 'R4', concepts: 3 } + ]); + + // B adds too, and A sees it + await b.call('add', { id, resources: [cs('http://example.org/three', '3', ['q'])] }); + expect((await a.call('get', { id })).map(r => r.url)).toEqual( + ['http://example.org/one', 'http://example.org/two', 'http://example.org/three']); + + await b.call('end', { id }); + expect(await a.call('has', { id })).toBe(false); + expect(await a.call('get', { id })).toEqual([]); + const d = await a.call('describe', { id }); + expect(d.messageId).toBe('CACHE_ID_CLOSED'); + expect(d.params[0]).toBe(id); + }); + + test('the idle sweep run by either process is seen by both, and running it twice is harmless', async () => { + const a = await spawn(); + const b = await spawn(); + const id = await a.call('start', { resources: [cs('http://example.org/one', '1', ['a'])] }); + // prune with a negative max age: everything counts as idle + await b.call('prune', { maxAge: -1000 }); + await a.call('prune', { maxAge: -1000 }); + expect(await a.call('has', { id })).toBe(false); + expect((await a.call('describe', { id })).messageId).toBe('CACHE_ID_EXPIRED'); + expect((await b.call('describe', { id })).messageId).toBe('CACHE_ID_EXPIRED'); + }); + + test('a cache survives a process restart', async () => { + const a = await spawn(); + const id = await a.call('start', { resources: [cs('http://example.org/one', '1', ['a', 'b'])], sealed: true }); + await running.pop().stop(); + + const again = await spawn(); + expect(await again.call('has', { id })).toBe(true); + expect((await again.call('get', { id })).map(r => r.url)).toEqual(['http://example.org/one']); + }); + + test('many writes from two processes at once all land', async () => { + const a = await spawn(); + const b = await spawn(); + const id = await a.call('start', {}); + const writes = []; + for (let i = 0; i < 50; i++) { + writes.push(a.call('add', { id, resources: [cs(`http://example.org/a${i}`, '1', ['c'])] })); + writes.push(b.call('add', { id, resources: [cs(`http://example.org/b${i}`, '1', ['c'])] })); + } + await Promise.all(writes); + expect(await a.call('get', { id })).toHaveLength(100); + }); + + test('the size cap is kept across processes: a write in one drops the other\'s least recently used cache', async () => { + const a = await spawn(4000); + const b = await spawn(4000); + const big = (url) => ({ ...cs(url, '1', ['a']), description: 'x'.repeat(1500) }); + const old = await b.call('start', { resources: [big('http://example.org/old')] }); + await b.call('setLastUsed', { id: old, when: Date.now() - 60000 }); + const kept = await b.call('start', { resources: [big('http://example.org/kept')] }); + const fresh = await a.call('start', { resources: [big('http://example.org/fresh')] }); + expect(await b.call('has', { id: old })).toBe(false); + expect((await b.call('describe', { id: old })).messageId).toBe('CACHE_ID_EVICTED'); + expect(await b.call('has', { id: kept })).toBe(true); + expect(await b.call('has', { id: fresh })).toBe(true); + }); +}); diff --git a/tests/tx/resource-cache.test.js b/tests/tx/resource-cache.test.js index 35844394..9ed6fea5 100644 --- a/tests/tx/resource-cache.test.js +++ b/tests/tx/resource-cache.test.js @@ -4,9 +4,19 @@ * ResourceCache backs the cache-id protocol: resources sent under a cache-id on one * request must be retrievable on later requests with the same cache-id, and kept * isolated between different cache-ids. + * + * Every test here runs against both stores - the in-process memory store and the + * shared SQLite store (server.cluster.sharedCache) - because they are one interface with + * one set of semantics. The two differ only where a shared store must: an id with no + * record is CACHE_ID_UNKNOWN_SHARED whatever its prefix, and only the shared store has a + * size cap. */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); const { ResourceCache, formatDuration } = require('../../tx/operation-context'); +const { SqliteResourceCache } = require('../../tx/resource-cache-sqlite'); const CS = (url, version) => ({ resourceType: 'CodeSystem', url, version }); const VS = (url, version) => ({ resourceType: 'ValueSet', url, version }); @@ -14,11 +24,53 @@ const VS = (url, version) => ({ resourceType: 'ValueSet', url, version }); // A resource that reports a concept count, like the real wrapper classes do. const sized = (resource, n) => ({ ...resource, conceptCount: () => n }); -describe('ResourceCache', () => { +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'resource-cache-test-')); +let fileNo = 0; +const opened = []; +afterAll(() => { + for (const c of opened) { + c.close(); + } + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +// How each store is made, and how a test makes an entry look idle +const STORES = [ + { + name: 'memory', + unknownId: 'CACHE_ID_UNKNOWN', + misrouted: (named) => (named ? 'CACHE_ID_OTHER_INSTANCE' : 'CACHE_ID_OTHER_INSTANCE_UNNAMED'), + make: (opts = {}) => new ResourceCache(null, opts.maxTombstones || 1000), + setLastUsed: (cache, id, when) => { cache.cache.get(id).lastUsed = when; } + }, + { + name: 'sqlite', + // caches survive a restart here, so the message doesn't offer one as the reason + unknownId: 'CACHE_ID_UNKNOWN_SHARED', + // every process sees every cache, so a prefix naming another process says nothing + misrouted: () => 'CACHE_ID_UNKNOWN_SHARED', + make: (opts = {}) => { + const c = new SqliteResourceCache(null, { + path: path.join(tmpDir, `cache-${++fileNo}.db`), + scope: '/r4', + touchIntervalMs: 0, + ...opts + }); + opened.push(c); + return c; + }, + setLastUsed: (cache, id, when) => { + cache.db.prepare('UPDATE sessions SET last_used = ? WHERE scope = ? AND cache_id = ?').run(when, cache.scope, id); + } + } +]; + +describe.each(STORES)('ResourceCache ($name store)', (store) => { let cache; + const setLastUsed = (id, when) => store.setLastUsed(cache, id, when); beforeEach(() => { - cache = new ResourceCache(null); + cache = store.make(); }); describe('get / has on empty cache', () => { @@ -129,7 +181,7 @@ describe('ResourceCache', () => { test('removes entries older than maxAge but keeps fresh ones', () => { cache.add('old', [CS('http://old', '1')]); // Force the entry to look stale. - cache.cache.get('old').lastUsed = Date.now() - 10000; + setLastUsed('old', Date.now() - 10000); cache.add('fresh', [CS('http://fresh', '1')]); cache.prune(5000); // maxAge 5s @@ -140,7 +192,7 @@ describe('ResourceCache', () => { test('get() refreshes lastUsed so an active entry is not pruned', () => { cache.add('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10000; + setLastUsed('c1', Date.now() - 10000); cache.get('c1'); // touch -> refreshes lastUsed cache.prune(5000); expect(cache.has('c1')).toBe(true); @@ -151,10 +203,10 @@ describe('ResourceCache', () => { // the whole point: "you closed it yourself" and "it sat idle for 31 minutes" send // an investigator to completely different places (see issue #279). describe('tombstones (why a cache-id is gone)', () => { - test('an id that was never issued has no tombstone and reports CACHE_ID_UNKNOWN', () => { + test('an id that was never issued has no tombstone and reports CACHE_ID_UNKNOWN (memory) or CACHE_ID_UNKNOWN_SHARED', () => { const d = cache.describeMissing('never-seen'); expect(cache.tombstone('never-seen')).toBeNull(); - expect(d.messageId).toBe('CACHE_ID_UNKNOWN'); + expect(d.messageId).toBe(store.unknownId); expect(d.params).toEqual(['never-seen']); }); @@ -171,7 +223,7 @@ describe('ResourceCache', () => { test('prune() records the expiry with the measured idle time, not the timeout', () => { cache.add('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10 * 60 * 1000; // idle 10 min + setLastUsed('c1', Date.now() - 10 * 60 * 1000); // idle 10 min cache.prune(5 * 60 * 1000); // timeout 5 min - the sweep is late, as it is in production const t = cache.tombstone('c1'); @@ -209,7 +261,7 @@ describe('ResourceCache', () => { test('the latest fate wins when an id dies twice', () => { cache.set('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10000; + setLastUsed('c1', Date.now() - 10000); cache.prune(5000); expect(cache.tombstone('c1').reason).toBe('expired'); @@ -219,7 +271,7 @@ describe('ResourceCache', () => { }); test('tombstones are bounded: the oldest are evicted, never unbounded growth', () => { - const small = new ResourceCache(null, 3); + const small = store.make({ maxTombstones: 3 }); for (const id of ['a', 'b', 'c', 'd', 'e']) { small.set(id, [CS('http://' + id, '1')]); small.clear(id); @@ -229,13 +281,13 @@ describe('ResourceCache', () => { expect(small.tombstone('b')).toBeNull(); // evicted expect(small.tombstone('e')).not.toBeNull(); // newest kept // An evicted tombstone degrades to "never issued" rather than lying. - expect(small.describeMissing('a').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(small.describeMissing('a').messageId).toBe(store.unknownId); }); test('closing an id the server never had leaves no tombstone', () => { cache.clear('not-mine'); expect(cache.tombstone('not-mine')).toBeNull(); - expect(cache.describeMissing('not-mine').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(cache.describeMissing('not-mine').messageId).toBe(store.unknownId); }); }); @@ -282,27 +334,31 @@ describe('ResourceCache', () => { expect(ResourceCache.issuerOf(undefined)).toBeNull(); }); - test('an id from another instance is reported as misrouted, naming both instances', () => { + test('an id from another instance is reported as misrouted, naming both instances (memory store)', () => { cache.setInstanceCode('tx1'); const d = cache.describeMissing('tx2.abc'); - expect(d.messageId).toBe('CACHE_ID_OTHER_INSTANCE'); - expect(d.params).toEqual(['tx2.abc', 'tx2', 'tx1']); + expect(d.messageId).toBe(store.misrouted(true)); + if (store.name === 'memory') { + expect(d.params).toEqual(['tx2.abc', 'tx2', 'tx1']); + } }); - test('a server with no code reports a prefixed id as issued elsewhere', () => { + test('a server with no code reports a prefixed id as issued elsewhere (memory store)', () => { const d = cache.describeMissing('tx2.abc'); - expect(d.messageId).toBe('CACHE_ID_OTHER_INSTANCE_UNNAMED'); - expect(d.params).toEqual(['tx2.abc', 'tx2']); + expect(d.messageId).toBe(store.misrouted(false)); + if (store.name === 'memory') { + expect(d.params).toEqual(['tx2.abc', 'tx2']); + } }); test('an unrecorded id with our own prefix is plain unknown (e.g. issued before a restart)', () => { cache.setInstanceCode('tx1'); - expect(cache.describeMissing('tx1.abc').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(cache.describeMissing('tx1.abc').messageId).toBe(store.unknownId); }); test('an unprefixed id on a named server is plain unknown', () => { cache.setInstanceCode('tx1'); - expect(cache.describeMissing('abc').messageId).toBe('CACHE_ID_UNKNOWN'); + expect(cache.describeMissing('abc').messageId).toBe(store.unknownId); }); test('a tombstone wins over the prefix check', () => { @@ -320,7 +376,7 @@ describe('ResourceCache', () => { describe('status / touch (mode=check)', () => { test('status reports a live cache without touching it', () => { cache.set('c1', [CS('http://a', '1'), VS('http://b', '1')], true); - cache.cache.get('c1').lastUsed = Date.now() - 90 * 1000; + setLastUsed('c1', Date.now() - 90 * 1000); const s = cache.status('c1'); expect(s.exists).toBe(true); @@ -338,7 +394,7 @@ describe('ResourceCache', () => { test('touch resets the idle clock and saves the cache from the next prune', () => { cache.add('c1', [CS('http://a', '1')]); - cache.cache.get('c1').lastUsed = Date.now() - 10000; + setLastUsed('c1', Date.now() - 10000); expect(cache.touch('c1')).toBe(true); expect(cache.status('c1').idleMs).toBeLessThan(1000); @@ -431,7 +487,7 @@ describe('ResourceCache', () => { test('prune subtracts evicted entries', () => { cache.add('old', [sized(CS('http://old', '1'), 7)]); - cache.cache.get('old').lastUsed = Date.now() - 10000; + setLastUsed('old', Date.now() - 10000); cache.add('fresh', [sized(CS('http://fresh', '1'), 4)]); cache.prune(5000); expect(cache.conceptCount()).toBe(4); @@ -466,4 +522,456 @@ describe('ResourceCache', () => { expect(cache.maxConceptCount()).toBe(20); }); }); + + describe('sealed and unsealed', () => { + test('set() records whether the cache is sealed; add() keeps an unsealed cache unsealed', () => { + cache.set('open', [CS('http://a', '1')], false); + cache.set('shut', [CS('http://a', '1')], true); + expect(cache.isSealed('open')).toBe(false); + expect(cache.isSealed('shut')).toBe(true); + expect(cache.isSealed('nope')).toBe(false); + cache.add('open', [VS('http://v', '1')]); + expect(cache.isSealed('open')).toBe(false); + expect(cache.get('open').map(r => r.url)).toEqual(['http://a', 'http://v']); + }); + + test('an empty cache made by set() exists', () => { + cache.set('empty', [], true); + expect(cache.has('empty')).toBe(true); + expect(cache.get('empty')).toEqual([]); + expect(cache.status('empty').resources).toBe(0); + }); + + test('order is kept, and a replacement keeps its place', () => { + cache.add('c1', [CS('http://a', '1'), CS('http://b', '1'), CS('http://c', '1')]); + cache.add('c1', [{ ...CS('http://b', '1'), title: 'new b' }, CS('http://d', '1')]); + const got = cache.get('c1'); + expect(got.map(r => r.url)).toEqual(['http://a', 'http://b', 'http://c', 'http://d']); + expect(got[1].title).toBe('new b'); + }); + }); + + describe('resources are marked as cached', () => { + test('get() returns resources with a non-enumerable isCached marker', () => { + cache.add('c1', [CS('http://a', '1')]); + const [r] = cache.get('c1'); + expect(r.isCached).toBe(true); + expect(Object.keys(r)).not.toContain('isCached'); + expect(JSON.stringify(r)).not.toContain('isCached'); + }); + }); + + // A cap on the total size of what the cache holds: when a write takes it over, whole + // caches are dropped, least recently used first, and remembered as `evicted`. + // only the shared store has a cap: the memory store is as it was + (store.name === 'sqlite' ? describe : describe.skip)('size cap (maxBytes)', () => { + const big = (url, n) => ({ ...CS(url, '1'), description: 'x'.repeat(n) }); + + test('drops the least recently used caches, never the one being written', () => { + cache = store.make({ maxBytes: 2500 }); + cache.set('a', [big('http://a', 1000)]); + cache.set('b', [big('http://b', 1000)]); + setLastUsed('a', Date.now() - 60000); + setLastUsed('b', Date.now() - 30000); + cache.set('c', [big('http://c', 1000)]); // over the cap: 'a' is the oldest + expect(cache.has('a')).toBe(false); + expect(cache.has('b')).toBe(true); + expect(cache.has('c')).toBe(true); + expect(cache.byteCount()).toBeLessThanOrEqual(2500); + + const t = cache.tombstone('a'); + expect(t.reason).toBe('evicted'); + expect(t.maxBytes).toBe(2500); + expect(t.idleMs).toBeGreaterThanOrEqual(60000); + const d = cache.describeMissing('a'); + expect(d.messageId).toBe('CACHE_ID_EVICTED'); + expect(d.params[0]).toBe('a'); + expect(d.params[2]).toBe('2 KB'); + expect(d.params[3]).toBe('60 seconds'); + }); + + test('growing a cache with add() can evict others too', () => { + cache = store.make({ maxBytes: 2500 }); + cache.set('a', [big('http://a', 1000)]); + setLastUsed('a', Date.now() - 60000); + cache.set('b', [big('http://b', 1000)]); + cache.add('b', [big('http://b2', 1000)]); + expect(cache.has('a')).toBe(false); + expect(cache.tombstone('a').reason).toBe('evicted'); + expect(cache.get('b')).toHaveLength(2); + }); + + test('a single cache bigger than the cap is kept (there is nothing else to drop)', () => { + cache = store.make({ maxBytes: 500 }); + cache.set('a', [big('http://a', 1000)]); + expect(cache.has('a')).toBe(true); + }); + + test('no cap, nothing is dropped', () => { + for (let i = 0; i < 20; i++) { + cache.set('c' + i, [big('http://c' + i, 1000)]); + } + expect(cache.size()).toBe(20); + }); + }); +}); + +// Behaviour that only the shared SQLite store has. +describe('SqliteResourceCache', () => { + const { CodeSystem } = require('../../tx/library/codesystem'); + const ValueSet = require('../../tx/library/valueset'); + const make = (opts = {}) => { + const c = new SqliteResourceCache(null, { path: path.join(tmpDir, `only-${++fileNo}.db`), scope: '/r4', ...opts }); + opened.push(c); + return c; + }; + const lastUsedOf = (c, id) => c.db.prepare('SELECT last_used FROM sessions WHERE scope = ? AND cache_id = ?').get(c.scope, id).last_used; + + test('wrappers come back as wrappers, with the FHIR version they arrived in', () => { + const c = make(); + const cs = new CodeSystem({ + resourceType: 'CodeSystem', url: 'http://example.org/cs', version: '1', status: 'active', content: 'complete', + concept: [{ code: 'a', display: 'A' }, { code: 'b', display: 'B', concept: [{ code: 'b1' }] }] + }, 'R4'); + const vs = new ValueSet({ + resourceType: 'ValueSet', url: 'http://example.org/vs', status: 'active', + compose: { include: [{ system: 'http://example.org/cs', filter: [{ property: 'concept', op: 'is-a', value: 'b' }] }] } + }, 'R3'); + c.set('c1', [cs, vs]); + const [cs2, vs2] = c.get('c1'); + expect(cs2).toBeInstanceOf(CodeSystem); + expect(vs2).toBeInstanceOf(ValueSet); + expect(cs2.fhirVersion).toBe('R4'); + expect(vs2.fhirVersion).toBe('R3'); + expect(cs2.jsonObj).toEqual(cs.jsonObj); + expect(vs2.jsonObj).toEqual(vs.jsonObj); + expect(cs2.conceptCount()).toBe(3); + expect(c.conceptCountFor('c1')).toBe(3); + expect(cs2.isCached).toBe(true); + // a fresh object every time: nothing is shared between requests + expect(c.get('c1')[0]).not.toBe(cs2); + }); + + describe('a request only loads the resources it uses', () => { + const { ConceptMap } = require('../../tx/library/conceptmap'); + const { ExpansionCache } = require('../../tx/operation-context'); + const mkCs = (url, version, extra = {}) => new CodeSystem({ + resourceType: 'CodeSystem', url, version, status: 'active', content: 'complete', + concept: [{ code: 'a', display: 'A' }], ...extra + }, 'R4'); + const countLoads = (c) => { + const orig = c.q.content; + const counter = { n: 0 }; + c.q.content = { get: (...args) => { counter.n++; return orig.get(...args); } }; + return counter; + }; + + test('looking through the cache by url, version and type loads nothing', () => { + const c = make(); + c.set('c1', [mkCs('http://x/a', '1'), mkCs('http://x/b', '2'), mkCs('http://x/sup', '1', { content: 'supplement', supplements: 'http://x/a' })]); + const loads = countLoads(c); + const got = c.get('c1'); + expect(got.map(r => r.url)).toEqual(['http://x/a', 'http://x/b', 'http://x/sup']); + expect(got.map(r => r.version)).toEqual(['1', '2', '1']); + expect(got.map(r => r.vurl)).toEqual(['http://x/a|1', 'http://x/b|2', 'http://x/sup|1']); + expect(got.every(r => r.resourceType === 'CodeSystem' && r instanceof CodeSystem)).toBe(true); + expect(got.some(r => r instanceof ConceptMap)).toBe(false); + expect(got.map(r => r.supplementsUrl)).toEqual([undefined, undefined, 'http://x/a']); + expect(got[2].isGlobalSupplement()).toBe(false); + expect(got.every(r => r.isCached === true)).toBe(true); + expect(loads.n).toBe(0); + + // using one loads that one only, once + expect(got[1].content).toBe('complete'); + expect(got[1].jsonObj.concept[0].code).toBe('a'); + expect(got[1].conceptCount()).toBe(1); + expect(loads.n).toBe(1); + }); + + test('a loaded stand-in behaves like the wrapper it stands for', () => { + const c = make(); + const original = mkCs('http://x/a', '1'); + c.set('c1', [original]); + const [r] = c.get('c1'); + expect(r.jsonObj).toEqual(original.jsonObj); + expect(r.fhirVersion).toBe('R4'); + expect(r.versionAlgorithm()).toEqual(original.versionAlgorithm()); + expect(JSON.parse(JSON.stringify(r)).jsonObj).toEqual(original.jsonObj); + r.sourcePackage = 'p'; + expect(r.sourcePackage).toBe('p'); + }); + + test('the expansion cache key is the same as for the resources built eagerly', () => { + const c = make(); + const resources = [mkCs('http://x/a', '1'), new ValueSet({ resourceType: 'ValueSet', url: 'http://x/vs', status: 'active', + compose: { include: [{ system: 'http://x/a' }] } }, 'R4')]; + c.set('c1', resources); + const ec = new ExpansionCache(null); + const vs = { resourceType: 'ValueSet', url: 'http://x/vs' }; + const loads = countLoads(c); + const lazyKey = ec.computeKey(vs, null, c.get('c1')); + expect(loads.n).toBe(0); + const eager = c.q.resources.all('/r4', 'c1').map(row => c._build(row)); + expect(ec.computeKey(vs, null, eager)).toBe(lazyKey); + expect(ec.computeKey(vs, null, resources)).toBe(lazyKey); + }); + + test('the stored hash pre-fills each resource\'s expansion-key hash (P6): same keys, no hashing', () => { + const crypto = require('crypto'); + const { CanonicalResource } = require('../../tx/library/canonical-resource'); + const c = make(); + const resources = [ + mkCs('http://x/a', '1'), + mkCs('http://x/sup', '1', { content: 'supplement', supplements: 'http://x/a' }), + new ValueSet({ resourceType: 'ValueSet', url: 'http://x/vs', status: 'active', + compose: { include: [{ system: 'http://x/a', filter: [{ property: 'concept', op: 'is-a', value: 'a' }] }] } }, 'R3'), + new ConceptMap({ resourceType: 'ConceptMap', url: 'http://x/cm', status: 'active', + group: [{ source: 'http://x/a', target: 'http://x/b', element: [{ code: 'a', target: [{ code: 'b', relationship: 'equivalent' }] }] }] }), + { resourceType: 'NamingSystem', url: 'http://x/ns', name: 'plain' } + ]; + // the keys the memory store's resources (the ones the client sent) make + const vs = new ValueSet({ resourceType: 'ValueSet', url: 'http://x/expand-me', status: 'active', compose: { include: [{ system: 'http://x/a' }] } }); + const ec = new ExpansionCache(null); + const params = { hashSource: () => 'p' }; + const memoryKey = ec.computeKey(vs, params, resources); + c.set('c1', resources); + + // stand-ins: the hash comes from the row - nothing is loaded and nothing hashed + // but the key itself and the plain resource + const lazy = c.get('c1'); + const loads = countLoads(c); + const createHash = jest.spyOn(crypto, 'createHash'); + let lazyKey; + try { + lazyKey = ec.computeKey(vs, params, lazy); + expect(createHash).toHaveBeenCalledTimes(2); + } finally { + createHash.mockRestore(); + } + expect(loads.n).toBe(0); + expect(lazyKey).toBe(memoryKey); + + // loaded: each wrapper was given the row's hash when it was built + for (const r of lazy.slice(0, 4)) { + void r.jsonObj; + } + expect(loads.n).toBe(4); + for (const [i, r] of lazy.entries()) { + const expected = CanonicalResource.computeExpansionKeyHash(resources[i].jsonObj || resources[i]); + if (typeof r.expansionKeyHash === 'function') { + expect(r.expansionKeyHash()).toBe(expected); + } + } + const spy = jest.spyOn(crypto, 'createHash'); + try { + expect(ec.computeKey(vs, params, lazy)).toBe(memoryKey); + expect(spy).toHaveBeenCalledTimes(2); + } finally { + spy.mockRestore(); + } + // and built eagerly from the rows + const eager = c.q.resources.all('/r4', 'c1').map(row => c._build(row)); + expect(ec.computeKey(vs, params, eager)).toBe(memoryKey); + }); + + test.each(['add', 'set'])('%s replacing a lazy resource cannot change the content behind its expansion key', async (method) => { + const { ExpandWorker } = require('../../tx/workers/expand'); + const c = make(); + const original = mkCs('http://x/a', '1'); + const replacement = mkCs('http://x/a', '1', { concept: [{ code: 'b' }] }); + c.set('c1', [original]); + const ec = new ExpansionCache(null); + ec.forceCaching = true; + const vs = { resourceType: 'ValueSet', compose: { include: [{ system: 'http://x/a' }] } }; + const resources = c.get('c1'); + const oldKey = ec.computeKey(vs, null, resources); + // Run the real cache lookup/compute/store path, with the replacement between + // computeKey and the first use of the lazy resource's JSON. + const worker = { + deadCheck() {}, log: { debug() {} }, opContext: { expansionCache: ec }, + additionalResources: resources, + async performExpansion() { + c[method]('c1', [replacement]); + return { expansion: { contains: this.additionalResources[0].jsonObj.concept } }; + } + }; + await expect(ExpandWorker.prototype.doExpand.call(worker, vs, null)) + .rejects.toMatchObject({ statusCode: 409, cause: 'conflict', msgId: 'CACHE_ID_CHANGED_DURING_REQUEST' }); + expect(ec.get(oldKey)).toBeNull(); + expect(resources[0].expansionKeyHash()).toBe(original.expansionKeyHash()); + + // Retrying from a fresh index succeeds and caches only under the new key. + worker.additionalResources = c.get('c1'); + const newKey = ec.computeKey(vs, null, worker.additionalResources); + expect(newKey).not.toBe(oldKey); + const result = await ExpandWorker.prototype.doExpand.call(worker, vs, null); + expect(result.expansion.contains).toEqual([{ code: 'b' }]); + expect(ec.get(newKey)).toEqual(result); + expect(ec.get(oldKey)).toBeNull(); + }); + + test('a loaded resource keeps its snapshot, and an unchanged lazy resource still loads after a write', () => { + const c = make(); + const original = mkCs('http://x/a', '1'); + c.set('c1', [original]); + const [loaded] = c.get('c1'); + expect(loaded.jsonObj).toEqual(original.jsonObj); + const [lazy] = c.get('c1'); + c.add('c1', [mkCs('http://x/b', '1')]); + expect(lazy.jsonObj).toEqual(original.jsonObj); + c.add('c1', [mkCs('http://x/a', '1', { concept: [{ code: 'b' }] })]); + expect(loaded.jsonObj).toEqual(original.jsonObj); + expect(loaded.expansionKeyHash()).toBe(original.expansionKeyHash()); + }); + + test('a resource that has gone by the time it is used fails like a dead cache-id', () => { + const c = make(); + c.set('c1', [mkCs('http://x/a', '1')]); + const [r] = c.get('c1'); + c.clear('c1'); + let err = null; + try { + void r.jsonObj; + } catch (e) { + err = e; + } + expect(err).not.toBeNull(); + expect(err.statusCode).toBe(404); + expect(err.issueCode).toBe('cache-id-unknown'); + expect(err.message).toMatch(/was closed while this request was using it/); + }); + }); + + test('endpoints share the file but not caches', () => { + const file = path.join(tmpDir, `scopes-${++fileNo}.db`); + const r4 = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + const r5 = new SqliteResourceCache(null, { path: file, scope: '/r5' }); + opened.push(r4, r5); + r4.set('c1', [CS('http://a', '1')]); + expect(r4.has('c1')).toBe(true); + expect(r5.has('c1')).toBe(false); + expect(r5.get('c1')).toEqual([]); + r4.clear('c1'); + expect(r5.tombstone('c1')).toBeNull(); + }); + + test('last_used is refreshed at most once per touch interval', () => { + const c = make({ touchIntervalMs: 30000 }); + c.set('c1', [CS('http://a', '1')]); + const t0 = Date.now() - 10000; + c.db.prepare('UPDATE sessions SET last_used = ?').run(t0); + c.get('c1'); + expect(lastUsedOf(c, 'c1')).toBe(t0); // 10 s ago: within the interval, not written + c.add('c1', [CS('http://a', '1')]); // same content again: no write either + expect(lastUsedOf(c, 'c1')).toBe(t0); + + const t1 = Date.now() - 31000; + c.db.prepare('UPDATE sessions SET last_used = ?').run(t1); + c.get('c1'); + expect(lastUsedOf(c, 'c1')).toBeGreaterThan(Date.now() - 2000); // past the interval: refreshed + }); + + test('an explicit touch (mode=check) is not throttled', () => { + const c = make({ touchIntervalMs: 30000 }); + c.set('c1', [CS('http://a', '1')]); + c.db.prepare('UPDATE sessions SET last_used = ?').run(Date.now() - 10000); + expect(c.touch('c1')).toBe(true); + expect(c.status('c1').idleMs).toBeLessThan(1000); + }); + + test('a failed last_used refresh is ignored', () => { + const c = make({ touchIntervalMs: 0 }); + c.set('c1', [CS('http://a', '1')]); + const orig = c.q.touch; + c.q.touch = { run: () => { throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); } }; + try { + expect(c.get('c1')).toHaveLength(1); + expect(c.touch('c1')).toBe(true); + } finally { + c.q.touch = orig; + } + }); + + test('a failed write of uploaded resources is retried once, then answered 503 - never dropped silently', () => { + const c = make(); + c.set('c1', [CS('http://a', '1')]); + const orig = c.q.upsertResource; + let calls = 0; + c.q.upsertResource = { run: () => { calls++; throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); } }; + let err = null; + try { + c.add('c1', [CS('http://b', '1')]); + } catch (e) { + err = e; + } finally { + c.q.upsertResource = orig; + } + expect(calls).toBe(2); + expect(err).not.toBeNull(); + expect(err.statusCode).toBe(503); + expect(err.msgId).toBe('CACHE_WRITE_FAILED'); + // the failed transaction changed nothing + expect(c.get('c1').map(r => r.url)).toEqual(['http://a']); + }); + + test('one failure then success: the retry saves the resources', () => { + const c = make(); + c.set('c1', []); + const orig = c.q.upsertResource; + let calls = 0; + c.q.upsertResource = { run: (row) => { if (++calls === 1) { throw Object.assign(new Error('busy'), { code: 'SQLITE_BUSY' }); } return orig.run(row); } }; + try { + c.add('c1', [CS('http://b', '1')]); + } finally { + c.q.upsertResource = orig; + } + expect(c.get('c1').map(r => r.url)).toEqual(['http://b']); + }); + + test('add() does not bring back a cache that has been closed', () => { + const c = make(); + c.set('c1', [CS('http://a', '1')]); + c.clear('c1'); + c.add('c1', [CS('http://b', '1')]); + expect(c.has('c1')).toBe(false); + expect(c.tombstone('c1').reason).toBe('closed'); + }); + + test('the size cap covers every endpoint in the file', () => { + const file = path.join(tmpDir, `cap-${++fileNo}.db`); + const r4 = new SqliteResourceCache(null, { path: file, scope: '/r4', maxBytes: 2500 }); + const r5 = new SqliteResourceCache(null, { path: file, scope: '/r5', maxBytes: 2500 }); + opened.push(r4, r5); + r4.set('a', [{ ...CS('http://a', '1'), description: 'x'.repeat(1000) }]); + r4.db.prepare('UPDATE sessions SET last_used = ?').run(Date.now() - 60000); + r5.set('b', [{ ...CS('http://b', '1'), description: 'x'.repeat(1000) }]); + r5.set('c', [{ ...CS('http://c', '1'), description: 'x'.repeat(1000) }]); + expect(r4.has('a')).toBe(false); + expect(r4.tombstone('a').reason).toBe('evicted'); + expect(r5.has('b')).toBe(true); + }); + + test('a cache survives a restart (closing and reopening the file)', () => { + const file = path.join(tmpDir, `restart-${++fileNo}.db`); + const before = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + before.set('c1', [CS('http://a', '1')], true); + before.set('c2', []); + before.clear('c2'); + before.close(); + + const after = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + opened.push(after); + expect(after.has('c1')).toBe(true); + expect(after.isSealed('c1')).toBe(true); + expect(after.get('c1').map(r => r.url)).toEqual(['http://a']); + expect(after.describeMissing('c2').messageId).toBe('CACHE_ID_CLOSED'); + }); + + test('a file from a different schema version is refused, not misread', () => { + const file = path.join(tmpDir, `schema-${++fileNo}.db`); + const c = new SqliteResourceCache(null, { path: file, scope: '/r4' }); + c.db.prepare("UPDATE meta SET value = '999' WHERE key = 'schema'").run(); + c.close(); + expect(() => new SqliteResourceCache(null, { path: file, scope: '/r4' })).toThrow(/schema version 999/); + }); }); diff --git a/tests/tx/vsac-generation.test.js b/tests/tx/vsac-generation.test.js new file mode 100644 index 00000000..5d18843c --- /dev/null +++ b/tests/tx/vsac-generation.test.js @@ -0,0 +1,130 @@ +/** + * VSAC with --role (cluster.md): the master's provider (leader) syncs and bumps the + * database's generation; a worker's (follower) never syncs, and when a request uses the + * value sets it compares the generation with the one it loaded (at most once a second), + * reloading only when it differs. Two real processes on one VSAC database. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { fork } = require('child_process'); + +const CHILD = path.join(__dirname, '..', 'utils', 'vsac-child.js'); +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +const vs = (id, version) => ({ + resourceType: 'ValueSet', id, url: `http://cts.nlm.nih.gov/fhir/ValueSet/${id}`, version, status: 'active', + compose: { include: [{ system: 'http://loinc.org', concept: [{ code: '1-8' }] }] } +}); + +describe('VSAC value sets, master and worker', () => { + let dir; + const children = []; + + const spawn = (role) => { + const child = fork(CHILD, [role], { env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', 'inherit', 'inherit', 'ipc'] }); + children.push(child); + let seq = 0; + const waiting = new Map(); + let readyResolve; + const ready = new Promise(resolve => { readyResolve = resolve; }); + child.on('message', (m) => { + if (m.ready) { + readyResolve(); + return; + } + const w = waiting.get(m.seq); + if (m.error) { + w.reject(new Error(m.error)); + } else { + w.resolve(m.result); + } + }); + const call = (op, args = {}) => new Promise((resolve, reject) => { + const s = ++seq; + waiting.set(s, { resolve, reject }); + child.send({ seq: s, op, ...args }); + }); + return ready.then(() => ({ call })); + }; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vsac-gen-')); + }); + afterEach(async () => { + for (const c of children.splice(0)) { + if (c.exitCode === null && c.signalCode === null) { + await new Promise(resolve => { + c.once('exit', resolve); + c.kill(); + }); + } + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('the worker picks up a sync on use, once, and only after it', async () => { + // the master's first sync (of an empty VSAC) takes the generation to 1 + const a = await spawn('leader'); + expect(await a.call('init')).toBe('leader'); + const b = await spawn('follower'); + expect(await b.call('init')).toBe('follower'); + expect(await b.call('catchUpNow')).toBe(1); + + const url = 'http://cts.nlm.nih.gov/fhir/ValueSet/one'; + expect(await b.call('fetch', { url, version: '20250101' })).toBeNull(); + + expect(await a.call('sync', { valueSets: [vs('one', '20250101')] })).toBe(2); + expect(await a.call('fetch', { url, version: '20250101' })).toBe('20250101'); + + // the worker notices on use, within a second or so (the check is throttled, and the + // reload runs in the background, so the request that notices still has the old list) + const before = await b.call('counts'); + let seen = null; + const t0 = Date.now(); + while (Date.now() - t0 < 5000 && !(seen = await b.call('fetch', { url, version: '20250101' }))) { + await sleep(50); + } + expect(seen).toBe('20250101'); + expect(Date.now() - t0).toBeLessThan(2500); + const after = await b.call('counts'); + expect(after.loaded).toBe(2); + expect(after.reloads - before.reloads).toBe(1); + + // nothing changes: many uses over 1.5 s cost at most two generation reads, and no reload + const quiet = await b.call('counts'); + const t1 = Date.now(); + let uses = 0; + while (Date.now() - t1 < 1500) { + await b.call('fetch', { url, version: '20250101' }); + uses++; + } + const end = await b.call('counts'); + expect(uses).toBeGreaterThan(20); + expect(end.generationReads - quiet.generationReads).toBeLessThanOrEqual(2); + expect(end.reloads).toBe(quiet.reloads); + }); + + test('a resync of one value set on the master reaches the worker', async () => { + const a = await spawn('leader'); + await a.call('init'); + const b = await spawn('follower'); + await b.call('init'); + await a.call('sync', { valueSets: [vs('one', '1')] }); + expect(await b.call('catchUpNow')).toBe(2); + const url = 'http://cts.nlm.nih.gov/fhir/ValueSet/one'; + expect(await a.call('resync', { url, valueSets: [vs('one', '1'), vs('one', '2')] })).toBe(3); + expect(await b.call('catchUpNow')).toBe(3); + expect(await b.call('fetch', { url, version: '2' })).toBe('2'); + }); + + test('a worker started before the master has made the database leaves it alone', async () => { + const b = await spawn('follower'); + expect(await b.call('init')).toBe('follower'); + expect(await b.call('catchUpNow')).toBeNull(); + expect(fs.existsSync(path.join(dir, 'terminology-cache', 'vsac', 'vsac-valuesets.db'))).toBe(false); + const a = await spawn('leader'); + await a.call('init'); + expect(await b.call('catchUpNow')).toBe(1); + }); +}); diff --git a/tests/utils/cluster-harness.js b/tests/utils/cluster-harness.js new file mode 100644 index 00000000..eda0bd5e --- /dev/null +++ b/tests/utils/cluster-harness.js @@ -0,0 +1,136 @@ +// Runs `node cluster.js` on a temporary data directory, for the tests of the cluster options +// (tests/server/cluster-shared-*.test.js): a master, N tx-workers, a fake VSAC, and helpers +// to send requests to the launcher's port or to one process directly. +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const http = require('http'); +const { spawn, execFileSync } = require('child_process'); + +const ROOT = path.join(__dirname, '..', '..'); +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +function request(port, method, urlPath, { headers = {}, body } = {}) { + return new Promise((resolve, reject) => { + const data = body ? Buffer.from(JSON.stringify(body)) : null; + const req = http.request({ + host: '127.0.0.1', port, method, path: urlPath, agent: false, + headers: { Accept: 'application/fhir+json', ...(data ? { 'Content-Type': 'application/fhir+json' } : {}), ...headers } + }, (res) => { + let text = ''; + res.on('data', d => { text += d; }); + res.on('end', () => { + let json = null; + try { + json = JSON.parse(text); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, json }); + }); + }); + req.on('error', reject); + req.end(data); + }); +} + +const isUp = (port) => request(port, 'GET', '/health').then(r => r.status === 200, () => false); +const param = (body, name) => ((body && body.parameter) || []).find(p => p.name === name); +const issueCodes = (r) => ((r.json && r.json.issue) || []).flatMap(i => ((i.details && i.details.coding) || []).map(c => c.code)); +const unknownCache = (r) => r.json && r.json.resourceType === 'OperationOutcome' && issueCodes(r).includes('cache-id-unknown'); + +/** The pid of the process listening on a port (Linux: ss). */ +function pidOn(port) { + const out = execFileSync('ss', ['-ltnpH', `sport = :${port}`], { encoding: 'utf8' }); + const m = /pid=(\d+)/.exec(out); + return m ? Number(m[1]) : null; +} + +/** + * Start the launcher. + * @param {Object} opts + * @param {number} opts.front - the launcher's port; the master is front+1, worker n front+1+n + * @param {number} opts.vsac - a port for the fake VSAC + * @param {number} [opts.workers=2] + * @param {Object} [opts.cluster] - more server.cluster settings (the options) + * @param {Object} [opts.tx] - more modules.tx settings + * @param {string[]} [opts.sources] - more library sources (library.yml), after VSAC + * @param {function(string)} [opts.prepare] - called with the data directory before the launch + */ +async function startCluster({ front, vsac, workers = 2, cluster = {}, tx = {}, sources = [], prepare = null }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cluster-opt-')); + const master = front + 1; + const workerPorts = Array.from({ length: workers }, (_, i) => master + 1 + i); + // the R4 core package, if a test run has already downloaded it, saves fetching it again + const core = path.join(ROOT, 'data', 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'); + if (fs.existsSync(core)) { + fs.cpSync(core, path.join(dir, 'terminology-cache', 'hl7.fhir.r4.core#4.0.1'), { recursive: true }); + } + fs.writeFileSync(path.join(dir, 'library.yml'), 'base:\n url: https://storage.googleapis.com/tx-fhir-org\n\nsources:\n - internal:vsac=vsac\n' + + sources.map(source => ` - ${source}\n`).join('')); + fs.writeFileSync(path.join(dir, 'config.json'), JSON.stringify({ + hostName: 'cluster-test', + logging: { console: false }, + server: { port: front, cluster: { workers, internalPort: master, ...cluster } }, + modules: { + tx: { + enabled: true, + instanceCode: 'm0', + librarySource: path.join(dir, 'library.yml'), + endpoints: [{ path: '/r4', fhirVersion: '4.0', context: null }], + closure: { enabled: true }, + vsacCfg: { apiKey: 'test', baseUrl: `http://127.0.0.1:${vsac}/fhir`, refreshIntervalHours: 24 }, + ...tx + } + } + })); + + const vsacServer = http.createServer((req, res) => { + res.setHeader('Content-Type', 'application/fhir+json'); + res.end(JSON.stringify({ resourceType: 'Bundle', type: 'searchset', total: 0, entry: [], link: [] })); + }); + await new Promise(resolve => vsacServer.listen(vsac, resolve)); + + if (prepare) { + prepare(dir); + } + const out = fs.openSync(path.join(dir, 'processes.log'), 'a'); + const launcher = spawn(process.execPath, [path.join(ROOT, 'cluster.js')], { + cwd: ROOT, env: { ...process.env, FHIRSMITH_DATA_DIR: dir }, stdio: ['ignore', out, out] + }); + const exited = new Promise(resolve => launcher.once('exit', (code, signal) => resolve({ code, signal }))); + + // port -> when it was first seen answering /health + const upAt = new Map(); + async function waitUp(ports, ms = 300000) { + const t0 = Date.now(); + for (const port of ports) { + while (!(await isUp(port))) { + if (Date.now() - t0 > ms || launcher.exitCode !== null) { + throw new Error(`port ${port} did not come up; see ${path.join(dir, 'processes.log')}`); + } + await sleep(250); + } + if (!upAt.has(port)) { + upAt.set(port, Date.now()); + } + } + // the launcher marks a process ready a moment after it first answers + await sleep(500); + } + await waitUp([master, ...workerPorts, front]); + + async function stop() { + if (launcher.exitCode === null && launcher.signalCode === null) { + launcher.kill('SIGTERM'); + } + const result = await exited; + await new Promise(resolve => vsacServer.close(resolve)); + fs.rmSync(dir, { recursive: true, force: true }); + return result; + } + + return { dir, front, master, workers: workerPorts, launcher, exited, waitUp, upAt, stop }; +} + +module.exports = { startCluster, request, isUp, param, issueCodes, unknownCache, pidOn, sleep, ROOT }; diff --git a/tests/utils/fake-upstream.js b/tests/utils/fake-upstream.js new file mode 100644 index 00000000..fb8114a2 --- /dev/null +++ b/tests/utils/fake-upstream.js @@ -0,0 +1,32 @@ +// A stand-in for a server process, for tests/server/cluster-launcher.test.js: +// node fake-upstream.js [keepAliveTimeout ms] +// /health answers 200; anything else answers {code, url, xff, proto} as JSON, after +// ?delay=. On SIGTERM it finishes the requests it has, then exits. +const http = require('http'); +const [code, port, keepAlive] = process.argv.slice(2); +const server = http.createServer((req, res) => { + if (req.url === '/health') { + res.end('ok'); + return; + } + if (req.url === '/r4/connections') { + res.end(JSON.stringify({ code, connections })); + return; + } + const delay = Number(new URL(req.url, 'http://x').searchParams.get('delay')) || 0; + setTimeout(() => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ code, url: req.url, xff: req.headers['x-forwarded-for'], proto: req.headers['x-forwarded-proto'] })); + }, delay); +}); +if (keepAlive) { + server.keepAliveTimeout = Number(keepAlive); +} +// the connections it accepted, for the test of idle keep-alive connections +let connections = 0; +server.on('connection', () => { connections++; }); +server.listen(Number(port)); +process.on('SIGTERM', () => { + server.close(() => process.exit(0)); + server.closeIdleConnections(); +}); diff --git a/tests/utils/resource-cache-child.js b/tests/utils/resource-cache-child.js new file mode 100644 index 00000000..a6e41c9c --- /dev/null +++ b/tests/utils/resource-cache-child.js @@ -0,0 +1,52 @@ +// A separate process using the shared SQLite resource cache, driven over IPC by +// tests/tx/resource-cache-multiprocess.test.js. Each message names a method of the +// cache and its arguments; the reply is the result (or the error). +const { SqliteResourceCache } = require('../../tx/resource-cache-sqlite'); +const { CodeSystem } = require('../../tx/library/codesystem'); + +const cache = new SqliteResourceCache(null, { path: process.argv[2], scope: '/r4', maxBytes: Number(process.argv[3] || 0) }); + +process.on('message', (msg) => { + try { + let result; + switch (msg.op) { + case 'start': { + const id = cache.newCacheId(); + cache.set(id, (msg.resources || []).map(r => new CodeSystem(r, 'R4')), !!msg.sealed); + result = id; + break; + } + case 'add': + cache.add(msg.id, msg.resources.map(r => new CodeSystem(r, 'R4'))); + result = true; + break; + case 'get': + result = cache.get(msg.id).map(r => ({ cls: r.constructor.name, url: r.url, version: r.version, fhirVersion: r.fhirVersion, concepts: r.conceptCount() })); + break; + case 'has': + result = cache.has(msg.id); + break; + case 'end': + cache.clear(msg.id); + result = true; + break; + case 'describe': + result = cache.describeMissing(msg.id); + break; + case 'setLastUsed': + cache.db.prepare('UPDATE sessions SET last_used = ? WHERE cache_id = ?').run(msg.when, msg.id); + result = true; + break; + case 'prune': + cache.prune(msg.maxAge); + result = true; + break; + default: + throw new Error('unknown op ' + msg.op); + } + process.send({ seq: msg.seq, result }); + } catch (e) { + process.send({ seq: msg.seq, error: e.message }); + } +}); +process.send({ ready: process.pid }); diff --git a/tests/utils/vsac-child.js b/tests/utils/vsac-child.js new file mode 100644 index 00000000..b6ebd86c --- /dev/null +++ b/tests/utils/vsac-child.js @@ -0,0 +1,73 @@ +// A process with the VSAC value set provider, for tests/tx/vsac-generation.test.js: +// FHIRSMITH_DATA_DIR= node vsac-child.js leader|follower +// Its "VSAC" is a list of value sets the test sends: the provider's real refresh runs +// against it (_fetchBundle answers from the list), so a sync writes the database and +// bumps its generation exactly as a real one does. +const { VSACValueSetProvider } = require('../../tx/vs/vs-vsac'); + +const stats = { addTask() {}, task() {}, taskDone() {}, taskError() {} }; +const provider = new VSACValueSetProvider({ apiKey: 'test', cluster: process.argv[2] }, stats); +provider.log = { error() {} }; +let upstream = []; +provider._scanLastUpdated = async () => 0; +provider._fetchBundle = async (url) => { + const m = /[?&]url=([^&]+)/.exec(url); + const list = m ? upstream.filter(vs => vs.url === decodeURIComponent(m[1])) : upstream; + return { resourceType: 'Bundle', total: list.length, entry: list.map(resource => ({ resource })), link: [] }; +}; +// count the generation reads and the reloads, to show the check is throttled +let generationReads = 0; +const getGeneration = provider.database.getGeneration.bind(provider.database); +provider.database.getGeneration = async () => { + generationReads++; + return getGeneration(); +}; +let reloads = 0; +const reload = provider._reloadMap.bind(provider); +provider._reloadMap = async () => { + reloads++; + return reload(); +}; +const origLog = console.log; +console.log = () => {}; + +process.on('message', async (msg) => { + try { + let result; + switch (msg.op) { + case 'init': + await provider.initialize(); + provider.stopRefreshTimer(); + result = provider.cluster; + break; + case 'sync': + upstream = msg.valueSets; + await provider.refreshValueSets(); + result = await provider.database.getGeneration(); + break; + case 'resync': + upstream = msg.valueSets; + await provider.resyncValueSet(msg.url); + result = await provider.database.getGeneration(); + break; + case 'fetch': { + const vs = await provider.fetchValueSet(msg.url, msg.version || ''); + result = vs ? vs.version : null; + break; + } + case 'catchUpNow': + provider.generationCheckedAt = 0; + await provider.catchUp(); + result = provider.loadedGeneration; + break; + case 'counts': + result = { generationReads, reloads, loaded: provider.loadedGeneration }; + break; + } + process.send({ seq: msg.seq, result }); + } catch (e) { + origLog(e); + process.send({ seq: msg.seq, error: e.message }); + } +}); +process.send({ ready: true }); diff --git a/tests/vs/vs-vsac-cluster-settings.test.js b/tests/vs/vs-vsac-cluster-settings.test.js new file mode 100644 index 00000000..1e39c934 --- /dev/null +++ b/tests/vs/vs-vsac-cluster-settings.test.js @@ -0,0 +1,28 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { VSACValueSetProvider } = require('../../tx/vs/vs-vsac'); + +describe('VSAC database settings follow the cluster role', () => { + test.each(['leader', 'follower'])('%s opens a fresh database with WAL and a busy timeout', async cluster => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vsac-role-')); + const provider = new VSACValueSetProvider({ apiKey: 'test', cluster }, {}); + // No sync or external service: exercise the connection constructed by the provider, + // using an empty database unique to this test. A previously WAL-enabled file could + // otherwise hide a missing shared option in the constructor. + provider.database.dbPath = path.join(dir, 'vsac.db'); + try { + await provider.database.create(); + const db = await provider.database._getWriteConnection(); + const get = sql => new Promise((resolve, reject) => { + db.get(sql, (error, row) => error ? reject(error) : resolve(row)); + }); + expect(await get('PRAGMA journal_mode')).toEqual({ journal_mode: 'wal' }); + expect(await get('PRAGMA busy_timeout')).toEqual({ timeout: 5000 }); + expect(provider.database.shared).toBe(true); + } finally { + await provider.database.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/translations/Messages.properties b/translations/Messages.properties index 3f10c049..6f3c7223 100644 --- a/translations/Messages.properties +++ b/translations/Messages.properties @@ -1116,9 +1116,11 @@ Unable_to_resolve_system__value_set_has_no_includes_or_expansion = The System UR UNABLE_TO_RESOLVE_SYSTEM_SYSTEM_IS_INDETERMINATE = The code system {1} referred to from value set {0} has a grammar, and the code might be valid in it Unable_to_resolve_value_Set_ = A definition for the value Set ''{0}'' could not be found CACHE_ID_UNKNOWN = The cache ''{0}'' was never issued by this server, and this server has no record of it ever existing. Caches are created with $cache-control?mode=start, and a cache-id is only valid on the server instance and endpoint that issued it - either it was issued somewhere else, or this server has restarted since it was issued +CACHE_ID_UNKNOWN_SHARED = The cache ''{0}'' was never issued by this server for this endpoint, and this server has no record of it ever existing. Caches are created with $cache-control?mode=start, and a cache-id is only valid on the server and endpoint that issued it. This server keeps its caches on disk, shared by all its processes, where they survive a restart CACHE_ID_CLOSED = The cache ''{0}'' was closed {1} ago by the client, using $cache-control?mode=end (it had been open for {2}). It did not expire and it was not idle: something in the client released this cache while this request was still using it CACHE_ID_EXPIRED = The cache ''{0}'' expired: it went {1} without being used, and this server closes caches that have been idle for more than {2} (this one was dropped {3} ago). Every request that uses a cache resets its idle clock, so a client that keeps using a cache will not see this CACHE_ID_CLEARED = The cache ''{0}'' was dropped {1} ago, when this server cleared all of its caches +CACHE_ID_EVICTED = The cache ''{0}'' was dropped {1} ago to make room: this server holds at most {2} of client resources across all caches, and when it is full it drops the caches that have gone longest without being used (this one had been idle for {3}). Start a new cache and send the resources again CACHE_ID_OTHER_INSTANCE = The cache ''{0}'' was issued by the server instance ''{1}'', but this is instance ''{2}''. A cache only exists on the instance that issued it, so this request reached the wrong server: either the proxy in front of these servers is not routing on the cache-id, or instance ''{1}'' is unavailable CACHE_ID_OTHER_INSTANCE_UNNAMED = The cache ''{0}'' was issued by the server instance ''{1}'', but this server has no instance code, so it cannot have issued it. A cache only exists on the instance that issued it, so this request reached the wrong server: either the proxy in front of these servers is not routing on the cache-id, or instance ''{1}'' is unavailable CLOSURE_NOT_SUPPORTED = This server does not support $closure (it has to be turned on by the administrator) @@ -1141,6 +1143,7 @@ CLOSURE_TOO_MANY_REQUEST_CONCEPTS = A single $closure request can add at most {0 CLOSURE_TOO_MANY_ENTRIES = Adding these concepts would put more entries into the closure table ''{0}'' than this server allows, in one request or in total CLOSURE_CONFIG_TOO_LARGE = The configuration supplied for this closure table (mostly the tx-resource parameters) is larger than this server will keep, which is {0} bytes CLOSURE_TABLE_BUSY = The closure table ''{0}'' is busy with other requests. Try again shortly +CLOSURE_TABLE_CHANGED = The closure table ''{0}'' was reset while this request was reading it. Retry the request against the new table SUBSUMPTION_CS_HIERARCHY_MEANING = The CodeSystem ''{0}'' has hierarchyMeaning = {1}, so its hierarchy does not describe subsumption, and subsumption cannot be tested SUBSUMPTION_CS_NOT_COMPLETE = The CodeSystem ''{0}'' has content = {1}, so subsumption cannot be tested (subsumption can only be tested against a code system with content = complete) SUBSUMPTION_CS_NOT_FOUND = A definition for the CodeSystem ''{0}'' could not be found, so subsumption cannot be tested diff --git a/tx/closure/closure-store.js b/tx/closure/closure-store.js index d4f76dfe..bc39b1dd 100644 --- a/tx/closure/closure-store.js +++ b/tx/closure/closure-store.js @@ -26,6 +26,18 @@ // here reads or writes an unbounded amount in one go: reads that can be large are paged // (entriesPage), and the size of what one call can write is capped by the worker. // +// Shared (server.cluster.sharedClosure, cluster.md "Options"): several processes open this +// file and any of them may answer $closure. The per-table lock is still taken, but it only +// orders the calls within one process, so correctness across processes comes from the +// writes themselves. A call computes against the table as it read it (its id and version), +// and writes in one BEGIN IMMEDIATE transaction that first checks the table is still at +// that id and version (recordAddChecked); if it isn't, nothing is written and the worker +// computes again from what is there now. Creating a table checks for an existing one and +// for the table limit in the same transaction as the insert (createTableChecked). A resync +// reads only up to the version it reports (entriesPage's upTo), so a write by another +// process between its pages can't put entries of a later version in it. Without shared, +// none of these paths is taken. +// const path = require('path'); const fs = require('fs'); @@ -40,10 +52,13 @@ class ClosureStore { /** * @param {Object} config - modules.tx.closure * @param {Object} [log] + * @param {Object} [options] + * @param {boolean} [options.shared] - other processes use the file too (see above) */ - constructor(config, log) { + constructor(config, log, options = {}) { this.config = config || {}; this.log = log || console; + this.shared = !!options.shared; this.db = null; this.path = ClosureStore.resolvePath(this.config); this.locks = new Map(); // table name -> { tail: Promise, count: holder + waiters } @@ -69,8 +84,22 @@ class ClosureStore { fs.mkdirSync(path.dirname(this.path), {recursive: true}); } this.db = new Database(this.path); + if (this.shared) { + // first: switching to WAL and creating the tables need locks another process may hold + this.db.pragma('busy_timeout = 5000'); + } this.db.pragma('journal_mode = WAL'); this.db.pragma('foreign_keys = ON'); + if (this.shared) { + // one process at a time creates the tables and brings an old database up to date + this.db.transaction(() => this.#createSchema()).immediate(); + } else { + this.#createSchema(); + } + this.#prepare(); + } + + #createSchema() { this.db.exec(` CREATE TABLE IF NOT EXISTS closure_table ( id INTEGER PRIMARY KEY, @@ -118,6 +147,25 @@ class ClosureStore { entry_count = (SELECT count(*) FROM closure_entry e WHERE e.table_id = closure_table.id); `); } + if (this.shared) { + // An INTEGER PRIMARY KEY can be reused after a reset/dispose. Shared calls keep + // the id across awaits, including in their retry memo and paged reads, so it + // must identify one incarnation forever. A durable counter also upgrades old + // databases without rebuilding the table and its foreign keys. This runs in + // the schema's IMMEDIATE transaction, before any shared writer can allocate. + this.db.exec(` + CREATE TABLE IF NOT EXISTS closure_table_sequence ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + last_id INTEGER NOT NULL + ); + INSERT OR IGNORE INTO closure_table_sequence VALUES (1, 0); + UPDATE closure_table_sequence SET last_id = max(last_id, + (SELECT coalesce(max(id), 0) FROM closure_table)) WHERE singleton = 1; + `); + } + } + + #prepare() { this.stmt = { getTable: this.db.prepare('SELECT id, name, version, created, last_used FROM closure_table WHERE name = ?'), countTables: this.db.prepare('SELECT count(*) AS n FROM closure_table'), @@ -156,6 +204,24 @@ class ClosureStore { WHERE e.table_id = ? AND e.added_version > ?`), staleTables: this.db.prepare('SELECT id, name FROM closure_table WHERE last_used < ?') }; + if (this.shared) { + Object.assign(this.stmt, { + nextTableId: this.db.prepare('UPDATE closure_table_sequence SET last_id = last_id + 1 WHERE singleton = 1 RETURNING last_id AS id'), + insertSharedTable: this.db.prepare('INSERT INTO closure_table (id, name, version, created, last_used) VALUES (?, ?, 0, ?, ?)'), + entriesPageUpTo: this.db.prepare(` + SELECT e.rowid AS rowid, + s.system AS source_system, s.code AS source_code, s.display AS source_display, + t.system AS target_system, t.code AS target_code, t.display AS target_display, + e.relationship + FROM closure_entry e + JOIN closure_concept s ON s.id = e.source_id + JOIN closure_concept t ON t.id = e.target_id + WHERE e.table_id = ? AND e.added_version > ? AND e.added_version <= ? AND e.rowid > ? + ORDER BY e.rowid + LIMIT ?`), + deleteStale: this.db.prepare('DELETE FROM closure_table WHERE id = ? AND last_used < ?') + }); + } } close() { @@ -260,7 +326,11 @@ class ClosureStore { * written. Page through with the last row's rowid as afterRowid (start with 0). * @returns {Array<{rowid, source_system, source_code, source_display, target_system, target_code, target_display, relationship}>} */ - entriesPage(tableId, version, afterRowid, limit) { + entriesPage(tableId, version, afterRowid, limit, upTo) { + if (upTo !== undefined) { + // shared: only up to the version the caller read (see the header) + return this.stmt.entriesPageUpTo.all(tableId, version, upTo, afterRowid, limit); + } return this.stmt.entriesPage.all(tableId, version, afterRowid, limit); } @@ -278,7 +348,9 @@ class ClosureStore { if (existing) { this.stmt.deleteTable.run(existing.id); } - const info = this.stmt.insertTable.run(name, now, now); + const info = this.shared + ? this.stmt.insertSharedTable.run(this.stmt.nextTableId.get().id, name, now, now) + : this.stmt.insertTable.run(name, now, now); const id = info.lastInsertRowid; params.forEach((p, i) => this.stmt.insertParam.run(id, i, p.name, JSON.stringify(p))); return this.stmt.getTable.get(name); @@ -353,6 +425,50 @@ class ClosureStore { return version; } + /** + * Shared: create (or, with reset, replace) a table, checking in the same write + * transaction that there isn't one already (unless reset) and that the table limit + * isn't reached - so two processes creating at once can't both succeed, or go over it. + * @param {string} name + * @param {Array} params - as for createTable + * @param {Object} opts + * @param {boolean} opts.reset - replace a table of that name + * @param {number} opts.maxTables - Infinity for none + * @returns {{table: Object}|{refused: 'exists'|'limit'}} + */ + createTableChecked(name, params, { reset, maxTables }) { + return this.db.transaction(() => { + const existing = this.stmt.getTable.get(name); + if (existing && !reset) { + return { refused: 'exists' }; + } + if (!existing && this.stmt.countTables.get().n >= maxTables) { + return { refused: 'limit' }; + } + return { table: this.createTable(name, params) }; + }).immediate(); + } + + /** + * Shared: recordAdd, if the table is still the one the caller computed against - the + * same id (not reset or dropped since) and the same version (nothing added since). The + * check and the write are one BEGIN IMMEDIATE transaction, so no other process can + * write in between. + * @returns {number|null} the version after the add, or null if the table has changed + * (nothing was written) + */ + recordAddChecked(table, concepts, entries) { + // Even a duplicate-only add must check: a reset may have removed those concepts + // while this request was working, or an intervening add changed their version. + return this.db.transaction(() => { + const now = this.stmt.getTable.get(table.name); + if (!now || now.id !== table.id || now.version !== table.version) { + return null; + } + return this.recordAdd(table, concepts, entries); + }).immediate(); + } + /** * Every entry added after the given version. * @returns {Array<{source_system, source_code, source_display, target_system, target_code, target_display, relationship}>} @@ -395,6 +511,13 @@ class ClosureStore { const dropped = []; for (const t of stale) { await this.withLock(t.name, async () => { + if (this.shared) { + // another process may use the table between a read and a delete: one statement + if (this.db && this.stmt.deleteStale.run(t.id, cutoff).changes > 0) { + dropped.push(t.name); + } + return; + } const now = this.db ? this.stmt.getTable.get(t.name) : null; if (now && now.id === t.id && now.last_used < cutoff) { this.stmt.deleteTable.run(t.id); diff --git a/tx/cm/cm-package.js b/tx/cm/cm-package.js index 89e7e859..ce1239d2 100644 --- a/tx/cm/cm-package.js +++ b/tx/cm/cm-package.js @@ -9,6 +9,7 @@ const {validateParameter} = require("../../library/utilities"); const {ConceptMap} = require("../library/conceptmap"); const { applyIdPrefix, stripIdPrefix } = require('../library/resource-ids'); const { keepForPinnedReference } = require('../library/tho-precedence'); +const { waitForCache } = require('../library/wait-for-cache'); /** * Package-based ConceptMap provider using shared database layer @@ -18,14 +19,19 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider { /** * @param {PackageContentLoader} packageLoader - Path to the extracted package folder + * @param {Object} [options] + * @param {boolean} [options.readOnly] - a tx-worker (cluster.md): the cache is the master's; it + * is only read (ConceptMapDatabase reads with OPEN_READONLY), waited for until the master has + * finished it, and never built or changed */ - constructor(packageLoader) { + constructor(packageLoader, options = {}) { super(); validateParameter(packageLoader, "packageLoader", PackageContentLoader); this.packageLoader = packageLoader; // -2: caches from before concept maps were checked for active content in their // narrative are rebuilt (see initialize) this.dbPath = path.join(packageLoader.packageFolder, '.conceptmaps-2.db'); + this.readOnly = !!options.readOnly; this.database = new ConceptMapDatabase(this.dbPath); this.conceptMapMap = new Map(); this.initialized = false; @@ -41,6 +47,14 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider { return; } await this.packageLoader.initialize(); + if (this.readOnly) { + await waitForCache(async () => (await this.database.exists()) && (await this.database.isComplete()), + `Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: concept map cache ${this.dbPath}`); + this.conceptMapMap = await this.database.loadAllConceptMaps(); + this.#applyIdPrefix(); + this.initialized = true; + return; + } await fs.rm(path.join(this.packageLoader.packageFolder, '.conceptmaps.db'), { force: true }); // the old cache // A cache is only used once its population finished. One that was cut short diff --git a/tx/cs/cs-mimetypes.js b/tx/cs/cs-mimetypes.js index 38ebd51b..d9db48f7 100644 --- a/tx/cs/cs-mimetypes.js +++ b/tx/cs/cs-mimetypes.js @@ -741,12 +741,16 @@ class MimeTypeServicesFactory extends CodeSystemFactoryProvider { * @param {I18nSupport} i18n * @param {string} cacheFolder - where the downloaded copy of the IANA registry is kept * @param {Logger} log + * @param {Object} [options] + * @param {boolean} [options.readOnly] - only read the cached copy: never download or write + * it (a tx-worker, cluster.md: the master refreshes it before the workers start) */ - constructor(i18n, cacheFolder = null, log = null) { + constructor(i18n, cacheFolder = null, log = null, options = {}) { super(i18n); this.uses = 0; this.cacheFolder = cacheFolder; this.log = log; + this.readOnly = options.readOnly === true; this.registeredTypes = null; } @@ -761,7 +765,13 @@ class MimeTypeServicesFactory extends CodeSystemFactoryProvider { return; // no cache configured: the registered filter is unavailable } const target = path.join(this.cacheFolder, IANA_MEDIA_TYPES_FILE); - await this.#refresh(target); + if (this.readOnly) { + if (!fs.existsSync(target)) { + this.log?.warn(`Mime types: no cached copy of the IANA registry at ${target} (a tx-worker reads the one the master downloads, and never downloads it)`); + } + } else { + await this.#refresh(target); + } if (fs.existsSync(target)) { try { diff --git a/tx/library.js b/tx/library.js index 5098a813..31f1e12a 100644 --- a/tx/library.js +++ b/tx/library.js @@ -103,6 +103,28 @@ class Library { lastTime = null; totalDownloaded = 0; vsacCfg = undefined; + /** + * {string|null} with several processes on one data directory (cluster.md), a name every + * process in the cluster shares (the master's role): the folder-source-* caches are the + * master's, named folder-source--. The master builds each in a temporary sibling + * and renames it into place, so a worker never sees one half built; a worker + * (packageCachesReadOnly) only reads them, waiting for the master as for package caches, + * and never prunes + */ + localSourceTag = null; + + /** + * {boolean} a tx-worker (cluster.md): the package value set and concept map caches are the + * master's. This process opens them read-only, waits for the master to have finished one, + * and never builds, rebuilds or migrates one + */ + packageCachesReadOnly = false; + + /** + * {boolean} a tx-worker (cluster.md): the cached IANA media types registry is the master's. + * This process reads it and never downloads or writes it + */ + mimeTypesReadOnly = false; /** * {SyndicationStore|null} where the content this library loads is recorded for the @@ -249,7 +271,10 @@ class Library { external: endMemory.external - this.startMemory.external }; - await this.#pruneLocalSourceCaches(); + if (!this.packageCachesReadOnly) { + // the master, or no role: a worker leaves the master's caches alone + await this.#pruneLocalSourceCaches(); + } this.log.info(`Loading Time: ${(totalTime / 1000).toLocaleString()}s`); this.log.info(`Memory Used: ${(memoryIncrease.rss / 1024 / 1024).toFixed(2)} MB`); @@ -554,7 +579,7 @@ class Library { break; } case "mimetypes" : { - const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log); + const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log, { readOnly: this.mimeTypesReadOnly }); await mime.load(); this.registerProvider('internal', mime); break; @@ -844,10 +869,10 @@ class Library { this.codeSystemProviders.push(cp); let vs = null; if (!csOnly) { - vs = new PackageValueSetProvider(contentLoader); + vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await vs.initialize(); this.valueSetProviders.push(vs); - const cm = new PackageConceptMapProvider(contentLoader); + const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await cm.initialize(); this.conceptMapProviders.push(cm); } @@ -874,7 +899,7 @@ class Library { } const folderPath = this.#resolveLocalSourcePath(details); - const cacheSubdir = await this.#prepareLocalSourceCache(folderPath); + const cacheSubdir = await this.#prepareLocalSourceCache(folderPath, dir => new FolderContentLoader(folderPath, dir)); await this.#loadLocalSource(new FolderContentLoader(folderPath, cacheSubdir)); } @@ -899,7 +924,7 @@ class Library { } const filePath = this.#resolveLocalSourcePath(details); - const cacheSubdir = await this.#prepareLocalSourceCache(filePath); + const cacheSubdir = await this.#prepareLocalSourceCache(filePath, dir => new FileContentLoader(filePath, dir)); await this.#loadLocalSource(new FileContentLoader(filePath, cacheSubdir)); } @@ -919,15 +944,54 @@ class Library { * Parks the Package*Provider SQLite caches under the terminology cache rather * than polluting the user's source folder, and wipes between runs so edits to * the source are reliably picked up. + * + * In a cluster (localSourceTag set) every process uses the master's folder: + * - a worker (packageCachesReadOnly) touches nothing; its providers open the caches + * read-only, waiting for the master to have built them + * - the master builds the caches in a temporary sibling, then renames it into place, so a + * running worker never finds the folder half built + * + * @param {string} sourcePath + * @param {function(string): FolderContentLoader} makeLoader - a loader with its caches in a folder + * @returns {Promise} the cache folder * @private */ - async #prepareLocalSourceCache(sourcePath) { + async #prepareLocalSourceCache(sourcePath, makeLoader) { const hash = crypto.createHash('sha1').update(sourcePath).digest('hex').substring(0, 16); - const name = LOCAL_SOURCE_CACHE_PREFIX + hash; + const name = LOCAL_SOURCE_CACHE_PREFIX + (this.localSourceTag ? this.localSourceTag + '-' : '') + hash; const cacheSubdir = path.join(this.cacheFolder, name); this.#localSourceCaches.add(name); - await fs.rm(cacheSubdir, { recursive: true, force: true }); - await fs.mkdir(cacheSubdir, { recursive: true }); + if (this.packageCachesReadOnly) { + return cacheSubdir; + } + if (!this.localSourceTag) { + await fs.rm(cacheSubdir, { recursive: true, force: true }); + await fs.mkdir(cacheSubdir, { recursive: true }); + return cacheSubdir; + } + const building = `${cacheSubdir}.building-${process.pid}`; + const old = `${cacheSubdir}.old-${process.pid}`; + await fs.rm(building, { recursive: true, force: true }); + await fs.rm(old, { recursive: true, force: true }); + await fs.mkdir(building, { recursive: true }); + const loader = makeLoader(building); + await loader.initialize(); + const vs = new PackageValueSetProvider(loader); + await vs.initialize(); + await vs.close(); + const cm = new PackageConceptMapProvider(loader); + await cm.initialize(); + // a directory can't be renamed over a non-empty one: move the old one aside first. A + // worker that looks in between finds no folder, and waits for it + try { + await fs.rename(cacheSubdir, old); + } catch (e) { + if (e.code !== 'ENOENT') { + throw e; + } + } + await fs.rename(building, cacheSubdir); + await fs.rm(old, { recursive: true, force: true }); return cacheSubdir; } @@ -1026,10 +1090,10 @@ class Library { } this.codeSystemProviders.push(cp); - const vs = new PackageValueSetProvider(contentLoader); + const vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await vs.initialize(); this.valueSetProviders.push(vs); - const cm = new PackageConceptMapProvider(contentLoader); + const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly }); await cm.initialize(); this.conceptMapProviders.push(cm); diff --git a/tx/library/canonical-resource.js b/tx/library/canonical-resource.js index db9a80b6..261378c4 100644 --- a/tx/library/canonical-resource.js +++ b/tx/library/canonical-resource.js @@ -1,3 +1,4 @@ +const crypto = require("crypto"); const {VersionUtilities, VersionPrecision} = require("../../library/version-utilities"); const {Issue} = require("./operation-outcome"); @@ -80,6 +81,54 @@ class CanonicalResource { checkContained(this.jsonObj); } + /** + * The hash ExpansionCache.computeKey puts in an expansion's cache key for this + * resource: the first 16 hex characters of the sha256 of JSON.stringify(json). Computing + * it means serialising the whole resource, and a $cache-control cache hands every + * resource it holds to every $expand that carries its cache-id, so it is worked out + * once and remembered. + * + * @param {Object} json - the resource (R5 form) + * @returns {string} + */ + static computeExpansionKeyHash(json) { + return crypto.createHash('sha256') + .update(JSON.stringify(json)) + .digest('hex') + .substring(0, 16); + } + + /** + * This resource's expansion-key hash (see computeExpansionKeyHash), computed on first + * use and then remembered. Nothing writes to jsonObj once a resource is in use (the r4/r5 conversion + * happens in the constructor, before anything can ask), and the memo is tied to the + * jsonObj it was computed from, so replacing jsonObj gets a new hash. + * @returns {string} + */ + expansionKeyHash() { + const memo = this._expansionKeyHashMemo; + if (memo && memo.json === this.jsonObj) { + return memo.hash; + } + return this.setExpansionKeyHash(CanonicalResource.computeExpansionKeyHash(this.jsonObj)); + } + + /** + * Record the expansion-key hash for a store that already knows it, so it is never + * recomputed. It must be computeExpansionKeyHash(this.jsonObj). Non-enumerable, so it never serialises + * into a response or log, and structuredClone/spread don't copy it onto a different + * resource. + * @param {string} hash + * @returns {string} the hash + */ + setExpansionKeyHash(hash) { + Object.defineProperty(this, '_expansionKeyHashMemo', { + value: { json: this.jsonObj, hash }, + enumerable: false, configurable: true, writable: true + }); + return hash; + } + get resourceType() { return this.jsonObj.resourceType; } diff --git a/tx/library/codesystem.js b/tx/library/codesystem.js index 31b543a3..07a868eb 100644 --- a/tx/library/codesystem.js +++ b/tx/library/codesystem.js @@ -739,6 +739,15 @@ class CodeSystem extends CanonicalResource { return (this.jsonObj.property || []).find(x => x.code == code && x.uri == uri); } + /** + * The canonical of the code system this one supplements (CodeSystem.supplements), + * or undefined if it isn't a supplement. + * @returns {string|undefined} + */ + get supplementsUrl() { + return this.jsonObj.supplements; + } + isSupplementFor(url, version) { if (this.jsonObj.content !== 'supplement') { return false; diff --git a/tx/library/wait-for-cache.js b/tx/library/wait-for-cache.js new file mode 100644 index 00000000..c8fcb1f1 --- /dev/null +++ b/tx/library/wait-for-cache.js @@ -0,0 +1,43 @@ +/** + * A tx-worker (cluster.md) only reads the package caches the master builds. Until the master + * has finished one - normally it has, since the workers start once it is ready - the worker + * waits for it, and gives up with a clear error rather than building it itself. + * + * @module tx/library/wait-for-cache + */ + +// how long a worker waits for the master to finish a cache, and how often it looks +const DEFAULT_WAIT_MS = 5 * 60 * 1000; +const POLL_MS = 500; + +/** + * Resolve once isReady() says true; reject if it hasn't within waitMs. + * + * @param {function(): Promise} isReady + * @param {string} what - the cache, for the messages + * @param {Object} [options] + * @param {number} [options.waitMs] - default DEFAULT_WAIT_MS, or FHIRSMITH_CACHE_WAIT_MS + * @returns {Promise} + */ +async function waitForCache(isReady, what, options = {}) { + const waitMs = options.waitMs ?? (Number(process.env.FHIRSMITH_CACHE_WAIT_MS) || DEFAULT_WAIT_MS); + const start = Date.now(); + let logged = false; + for (;;) { + if (await isReady()) { + return; + } + if (Date.now() - start >= waitMs) { + throw new Error(`${what} is not ready after ${Math.round(waitMs / 1000)} s. A tx-worker only reads it; ` + + 'the master builds it: start the master first, and the workers once it answers /health (cluster.md)'); + } + if (!logged) { + logged = true; + require('../../library/logger').getInstance().child({ module: 'tx' }) + .warn(`${what} is missing, incomplete or not yet migrated: waiting for the master to finish it`); + } + await new Promise(resolve => setTimeout(resolve, POLL_MS)); + } +} + +module.exports = { waitForCache }; diff --git a/tx/operation-context.js b/tx/operation-context.js index 29837db6..fd2e20d0 100644 --- a/tx/operation-context.js +++ b/tx/operation-context.js @@ -3,6 +3,7 @@ const inspector = require("inspector"); const crypto = require("crypto"); const {Languages} = require("../library/languages"); const {Issue} = require("./library/operation-outcome"); +const {CanonicalResource} = require("./library/canonical-resource"); const Logger = require("../library/logger"); const memoryGuard = require('../library/memory-guard'); @@ -651,13 +652,13 @@ class ExpansionCache { if (additionalResources && additionalResources.length > 0) { const resourceHashes = additionalResources .map(r => { - // Get the JSON object from wrapper or use directly - const json = r.jsonObj || r; - // Create a content hash for this resource - return crypto.createHash('sha256') - .update(JSON.stringify(json)) - .digest('hex') - .substring(0, 16); // Use first 16 chars for brevity + // A wrapper remembers its hash (CanonicalResource.expansionKeyHash): a cache-id's + // resources come with every $expand, and serialising and hashing them all + // each time was most of the cost of a cached request. Raw JSON is hashed the + // same way, so the key is the same either way. + return typeof r.expansionKeyHash === 'function' + ? r.expansionKeyHash() + : CanonicalResource.computeExpansionKeyHash(r.jsonObj || r); }) .sort(); keyParts.push(`additional:${resourceHashes.join(',')}`); diff --git a/tx/resource-cache-sqlite.js b/tx/resource-cache-sqlite.js new file mode 100644 index 00000000..d9bbb57f --- /dev/null +++ b/tx/resource-cache-sqlite.js @@ -0,0 +1,818 @@ +// +// SqliteResourceCache - the $cache-control resource cache kept in one SQLite file +// that every server process on the machine shares. +// +// Used when server.cluster.sharedCache is on (cluster.md, "Options"): the master and the +// tx-workers then keep their $cache-control caches here instead of in their own memory, so +// any process can answer any cache-id, and caches survive a process restart. The memory +// store (ResourceCache) is the default. This class is the same interface with the same +// semantics, over a file: +// +// - Nothing about a cache's contents is held in memory between requests. Each use +// reads the rows; there is no per-process copy that could go stale when another +// process changes the cache. +// - A request only pays for the resources it uses. get() reads the list of what the +// cache holds (type, url, version - not the content) and returns a stand-in for +// each CodeSystem / ValueSet / ConceptMap that answers those from the list; the +// first time a request touches anything else on one, its JSON is read and the +// real wrapper built (see lazyResource). Most requests look up a handful of +// resources by url in a cache of tens or hundreds. +// - Every write is one short BEGIN IMMEDIATE transaction; WAL mode lets readers +// carry on while one process writes; busy_timeout makes a writer wait for another +// rather than fail. +// - last_used is refreshed at most every touchIntervalMs (default 30 s) per cache, +// not on every read, so steady use of a cache is reads only. +// - The idle sweep (prune) and the size cap (maxBytes) move the caches they drop to +// tombstones in the same transaction, so whichever process asks about a dead +// cache-id later is told what happened to it. Any process may sweep; a sweep that +// finds nothing to do (because another process just did it) does nothing. +// +// The file must be on a local disk. SQLite's locking is not reliable on network +// filesystems (NFS, SMB, most container volume drivers backed by them), and a +// broken lock there corrupts the file. +// + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const Database = require('better-sqlite3'); +const { ResourceCache, formatDuration } = require('./operation-context'); +const { Issue } = require('./library/operation-outcome'); + +const SCHEMA_VERSION = 2; +const DEFAULT_TOUCH_INTERVAL_MS = 30 * 1000; +const DEFAULT_BUSY_TIMEOUT_MS = 5000; + +/** + * Render a byte count for a human: "512 KB", "3.4 MB", "2 GB". + * @param {number} bytes + * @returns {string} + */ +function formatBytes(bytes) { + if (bytes === null || bytes === undefined || !isFinite(bytes) || bytes < 0) { + return 'an unknown size'; + } + if (bytes < 1024 * 1024) { + return `${Math.max(1, Math.round(bytes / 1024))} KB`; + } + if (bytes < 1024 * 1024 * 1024) { + return `${(bytes / (1024 * 1024)).toFixed(1).replace(/\.0$/, '')} MB`; + } + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1).replace(/\.0$/, '')} GB`; +} + +// One connection per file per process, shared by the endpoints (each has its own scope) +const connections = new Map(); // absolute path -> { db, refs } + +function openDatabase(file, busyTimeoutMs) { + const abs = path.resolve(file); + const existing = connections.get(abs); + if (existing) { + existing.refs++; + return existing.db; + } + fs.mkdirSync(path.dirname(abs), { recursive: true }); + const db = new Database(abs); + // busy_timeout first: switching to WAL and creating the tables need locks that + // another process starting at the same moment may hold + db.pragma(`busy_timeout = ${busyTimeoutMs}`); + db.pragma('journal_mode = WAL'); + // NORMAL is safe with WAL (no corruption); a power cut can lose the last + // transactions, which for a cache of client resources is acceptable + db.pragma('synchronous = NORMAL'); + db.transaction(() => { + db.exec(` + CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE IF NOT EXISTS sessions ( + scope TEXT NOT NULL, + cache_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + last_used INTEGER NOT NULL, + sealed INTEGER NOT NULL DEFAULT 0, + concepts INTEGER NOT NULL DEFAULT 0, + resources INTEGER NOT NULL DEFAULT 0, + bytes INTEGER NOT NULL DEFAULT 0, + version INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (scope, cache_id) + ); + CREATE INDEX IF NOT EXISTS sessions_last_used ON sessions (last_used); + CREATE TABLE IF NOT EXISTS resources ( + id INTEGER PRIMARY KEY, + scope TEXT NOT NULL, + cache_id TEXT NOT NULL, + key TEXT NOT NULL, + resource_type TEXT, + url TEXT, + version TEXT, + fhir_version TEXT, + kind TEXT NOT NULL, + hash TEXT NOT NULL, + json TEXT NOT NULL, + bytes INTEGER NOT NULL, + concepts INTEGER NOT NULL DEFAULT 0, + supplements TEXT, + global_supplement INTEGER, + UNIQUE (scope, cache_id, key) + ); + CREATE TABLE IF NOT EXISTS tombstones ( + scope TEXT NOT NULL, + cache_id TEXT NOT NULL, + reason TEXT NOT NULL, + at INTEGER NOT NULL, + created_at INTEGER, + last_used INTEGER, + idle_ms INTEGER, + max_age_ms INTEGER, + max_bytes INTEGER, + PRIMARY KEY (scope, cache_id) + ); + `); + const row = db.prepare("SELECT value FROM meta WHERE key = 'schema'").get(); + if (!row) { + db.prepare("INSERT INTO meta (key, value) VALUES ('schema', ?)").run(String(SCHEMA_VERSION)); + } else if (row.value !== String(SCHEMA_VERSION)) { + throw new Error(`Resource cache ${abs} has schema version ${row.value}; this server uses ${SCHEMA_VERSION}. Delete the file (it only holds client caches) and restart`); + } + }).immediate(); + connections.set(abs, { db, refs: 1 }); + return db; +} + +function releaseDatabase(file) { + const abs = path.resolve(file); + const c = connections.get(abs); + if (!c) { + return; + } + c.refs--; + if (c.refs <= 0) { + connections.delete(abs); + try { + c.db.close(); + } catch (e) { + // closing is best effort + } + } +} + +// The wrapper classes are loaded on first use: they pull in a good deal of the server +let wrappers = null; +function wrapperClass(resourceType) { + if (!wrappers) { + wrappers = { + CodeSystem: require('./library/codesystem').CodeSystem, + ValueSet: require('./library/valueset'), + ConceptMap: require('./library/conceptmap').ConceptMap + }; + } + return wrappers[resourceType] || null; +} + +/** + * Content hash of a resource's JSON, as the expansion cache key uses it + * (CanonicalResource.computeExpansionKeyHash: the first 16 hex characters of a SHA-256 + * of JSON.stringify(jsonObj)). Stored with the row, and handed to each wrapper built from + * it (setExpansionKeyHash), so a cached resource is never serialised again to be hashed + * and its expansion keys are the ones the memory store would make. + * @param {string} json + * @returns {string} + */ +function contentHash(json) { + return crypto.createHash('sha256').update(json).digest('hex').substring(0, 16); +} + +// Read from the index row, without building the resource: what callers check while +// looking through the cache for the one they want (see TerminologyWorker +// findInAdditionalResources, listVersions, loadSupplements; ExpansionCache.computeKey) +const INDEX_PROPERTIES = new Set(['url', 'version', 'vurl', 'versionedUrl', 'resourceType', 'fhirType', + 'supplementsUrl', 'isCached', 'isGlobalSupplement', 'expansionKeyHash']); + +/** + * A stand-in for a cached CodeSystem / ValueSet / ConceptMap that is only built + * when it is used. It is an instance of the wrapper class (instanceof works), and + * answers url, version, resourceType and the like from the index row. Touching + * anything else - jsonObj, a method, any other property - loads the JSON and builds + * the real wrapper, once; from then on everything goes to that wrapper, and methods + * run on it directly (bound), so a resource that is used costs the same as one + * built eagerly, and one that isn't costs nothing. + * + * @param {Function} Cls - the wrapper class + * @param {Object} row - index row: key, resource_type, url, version, supplements, global_supplement, hash + * @param {Function} load - () => the real wrapper (reads the JSON) + */ +function lazyResource(Cls, row, load) { + let real = null; + const bound = new Map(); + const target = Object.create(Cls.prototype); + const url = row.url === null ? undefined : row.url; + const version = row.version === null ? undefined : row.version; + const index = { + url, + version, + vurl: version ? url + '|' + version : url, + versionedUrl: version ? url + '|' + version : url, + resourceType: row.resource_type, + fhirType: row.resource_type, + supplementsUrl: row.supplements === null ? undefined : row.supplements, + isCached: true, + // the expansion cache key's hash of this resource (see contentHash): known from the row + expansionKeyHash: () => row.hash, + // CodeSystem.isGlobalSupplement(), worked out when it was stored (null: not known) + isGlobalSupplement: row.global_supplement === null || row.global_supplement === undefined + ? undefined : () => row.global_supplement === 1 + }; + const materialise = () => { + if (!real) { + real = load(); + } + return real; + }; + return new Proxy(target, { + get(t, prop) { + if (!real && INDEX_PROPERTIES.has(prop) && index[prop] !== undefined) { + return index[prop]; + } + if (!real && INDEX_PROPERTIES.has(prop) && prop !== 'isGlobalSupplement') { + return undefined; + } + const r = materialise(); + const v = Reflect.get(r, prop, r); + if (typeof v === 'function' && prop !== 'constructor') { + let b = bound.get(prop); + if (!b || b.fn !== v) { + b = { fn: v, bound: v.bind(r) }; + bound.set(prop, b); + } + return b.bound; + } + return v; + }, + set(t, prop, value) { + return Reflect.set(materialise(), prop, value); + }, + has(t, prop) { + return Reflect.has(materialise(), prop); + }, + deleteProperty(t, prop) { + return Reflect.deleteProperty(materialise(), prop); + }, + defineProperty(t, prop, desc) { + return Reflect.defineProperty(materialise(), prop, desc); + }, + ownKeys() { + return Reflect.ownKeys(materialise()); + }, + getOwnPropertyDescriptor(t, prop) { + const d = Reflect.getOwnPropertyDescriptor(materialise(), prop); + if (d) { + d.configurable = true; // a proxy may not report a non-configurable property its target lacks + } + return d; + } + }); +} + +function markCached(resource) { + if (resource && typeof resource === 'object') { + // the same non-enumerable marker the memory store sets (see ResourceCache.add) + Object.defineProperty(resource, 'isCached', { value: true, enumerable: false, configurable: true, writable: true }); + } +} + +class SqliteResourceCache extends ResourceCache { + /** + * @param {Object} stats - server stats (may be null) + * @param {Object} options + * @param {string} options.path - the SQLite file (local disk only) + * @param {string} [options.scope=''] - which endpoint's caches these are; endpoints + * share the file but not caches + * @param {number} [options.maxTombstones=1000] + * @param {number} [options.maxBytes=0] - size cap for the whole file, 0 = none + * @param {number} [options.touchIntervalMs=30000] - least time between last_used refreshes + * @param {number} [options.busyTimeoutMs=5000] + */ + constructor(stats, options = {}) { + super(stats, options.maxTombstones || 1000); + this.maxBytes = options.maxBytes > 0 ? options.maxBytes : 0; + if (!options.path) { + throw new Error('SqliteResourceCache needs a path'); + } + this.path = path.resolve(options.path); + this.scope = options.scope || ''; + this.touchIntervalMs = options.touchIntervalMs >= 0 ? options.touchIntervalMs : DEFAULT_TOUCH_INTERVAL_MS; + this.db = openDatabase(this.path, options.busyTimeoutMs || DEFAULT_BUSY_TIMEOUT_MS); + this.closed = false; + this._prepare(); + } + + _prepare() { + const db = this.db; + this.q = { + session: db.prepare('SELECT * FROM sessions WHERE scope = ? AND cache_id = ?'), + sessionExists: db.prepare('SELECT 1 FROM sessions WHERE scope = ? AND cache_id = ?'), + sealed: db.prepare('SELECT sealed FROM sessions WHERE scope = ? AND cache_id = ?'), + resources: db.prepare('SELECT key, resource_type, fhir_version, kind, hash, json FROM resources WHERE scope = ? AND cache_id = ? ORDER BY id'), + // the index: everything but the content (and the content only of what isn't a wrapper) + index: db.prepare(`SELECT key, resource_type, url, version, supplements, global_supplement, fhir_version, kind, hash, + CASE WHEN kind = 'wrapper' THEN NULL ELSE json END AS json + FROM resources WHERE scope = ? AND cache_id = ? ORDER BY id`), + content: db.prepare('SELECT json, fhir_version, hash FROM resources WHERE scope = ? AND cache_id = ? AND key = ?'), + resourceHash: db.prepare('SELECT hash, concepts FROM resources WHERE scope = ? AND cache_id = ? AND key = ?'), + touch: db.prepare('UPDATE sessions SET last_used = ? WHERE scope = ? AND cache_id = ? AND last_used < ?'), + insertSession: db.prepare('INSERT INTO sessions (scope, cache_id, created_at, last_used, sealed) VALUES (?, ?, ?, ?, ?)'), + upsertResource: db.prepare(` + INSERT INTO resources (scope, cache_id, key, resource_type, url, version, supplements, global_supplement, fhir_version, kind, hash, json, bytes, concepts) + VALUES (@scope, @cacheId, @key, @resourceType, @url, @version, @supplements, @globalSupplement, @fhirVersion, @kind, @hash, @json, @bytes, @concepts) + ON CONFLICT (scope, cache_id, key) DO UPDATE SET + resource_type = excluded.resource_type, url = excluded.url, version = excluded.version, supplements = excluded.supplements, + global_supplement = excluded.global_supplement, + fhir_version = excluded.fhir_version, kind = excluded.kind, hash = excluded.hash, + json = excluded.json, bytes = excluded.bytes, concepts = excluded.concepts`), + totals: db.prepare(` + UPDATE sessions SET + concepts = (SELECT COALESCE(SUM(concepts), 0) FROM resources r WHERE r.scope = sessions.scope AND r.cache_id = sessions.cache_id), + bytes = (SELECT COALESCE(SUM(bytes), 0) FROM resources r WHERE r.scope = sessions.scope AND r.cache_id = sessions.cache_id), + resources = (SELECT COUNT(*) FROM resources r WHERE r.scope = sessions.scope AND r.cache_id = sessions.cache_id), + last_used = ?, version = version + 1 + WHERE scope = ? AND cache_id = ?`), + deleteResources: db.prepare('DELETE FROM resources WHERE scope = ? AND cache_id = ?'), + deleteSession: db.prepare('DELETE FROM sessions WHERE scope = ? AND cache_id = ?'), + tombstone: db.prepare('SELECT * FROM tombstones WHERE scope = ? AND cache_id = ?'), + insertTombstone: db.prepare(` + INSERT OR REPLACE INTO tombstones (scope, cache_id, reason, at, created_at, last_used, idle_ms, max_age_ms, max_bytes) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`), + deleteTombstone: db.prepare('DELETE FROM tombstones WHERE scope = ? AND cache_id = ?'), + trimTombstones: db.prepare(` + DELETE FROM tombstones WHERE scope = ? AND rowid NOT IN + (SELECT rowid FROM tombstones WHERE scope = ? ORDER BY rowid DESC LIMIT ?)`), + tombstoneCount: db.prepare('SELECT COUNT(*) AS n FROM tombstones WHERE scope = ?'), + expired: db.prepare('SELECT * FROM sessions WHERE scope = ? AND last_used < ?'), + allSessions: db.prepare('SELECT * FROM sessions WHERE scope = ?'), + scopeTotals: db.prepare('SELECT COUNT(*) AS n, COALESCE(SUM(concepts), 0) AS concepts, COALESCE(SUM(bytes), 0) AS bytes FROM sessions WHERE scope = ?'), + fileBytes: db.prepare('SELECT COALESCE(SUM(bytes), 0) AS bytes FROM sessions'), + lru: db.prepare('SELECT * FROM sessions WHERE NOT (scope = ? AND cache_id = ?) ORDER BY last_used, created_at LIMIT 16') + }; + } + + // ---------- reads ---------- + + status(cacheId) { + const s = this.q.session.get(this.scope, cacheId); + if (!s) { + return { exists: false }; + } + const now = Date.now(); + return { + exists: true, + sealed: !!s.sealed, + resources: s.resources, + concepts: s.concepts || 0, + idleMs: Math.max(0, now - s.last_used), + ageMs: s.created_at ? now - s.created_at : null, + timeoutMs: this.idleTimeoutMs + }; + } + + has(cacheId) { + return !!this.q.sessionExists.get(this.scope, cacheId); + } + + isSealed(cacheId) { + const s = this.q.sealed.get(this.scope, cacheId); + return s ? !!s.sealed : false; + } + + /** + * The resources held under a cache-id: a fresh set of objects every call, nothing + * shared with other requests. Wrappers are stand-ins that are only built when the + * request uses them (see lazyResource). The session row and the index are read in + * one read transaction. Later lazy reads check the indexed content hash: if another + * process replaced a resource meanwhile, this request fails so the client can retry + * from a fresh index, rather than mixing an old expansion key with new content. + */ + get(cacheId) { + let session = null; + let rows = []; + this.db.transaction(() => { + session = this.q.session.get(this.scope, cacheId); + if (session) { + rows = this.q.index.all(this.scope, cacheId); + } + }).deferred(); + if (!session) { + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': miss (no entry)`); + } + return []; + } + this._maybeTouch(cacheId, session.last_used); + const result = rows.map(r => this._hydrate({ ...r, cacheId })); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': hit, returning ${result.length} resource(s): ${rows.map(r => r.key).join(', ')}`); + } + return result; + } + + /** + * Rebuild one stored resource. Wrappers (CodeSystem / ValueSet / ConceptMap) are + * stored in their internal R5 form, so they are rebuilt as R5 and then given back + * the FHIR version they arrived in. + */ + _hydrate(row) { + if (row.kind === 'wrapper' && row.json === null) { + const Cls = wrapperClass(row.resource_type); + if (Cls) { + const cacheId = row.cacheId; + return lazyResource(Cls, row, () => this._loadWrapper(Cls, cacheId, row)); + } + } + return this._build(row); + } + + /** + * Build the real wrapper behind a stand-in. If the resource is no longer there + * (the client ended the cache, or it expired or was evicted, while this request + * was using it), the request fails the way a request on a dead cache-id does. + */ + _loadWrapper(Cls, cacheId, row) { + const c = this.q.content.get(this.scope, cacheId, row.key); + if (!c) { + const t = this.tombstone(cacheId); + throw new Issue('error', 'not-found', null, 'CACHE_ID_GONE_DURING_REQUEST', + `The cache '${cacheId}' ${t ? 'was ' + t.reason : 'went away'} while this request was using it, so ${row.key.replace(/\|/g, ' ')} is no longer available`, + 'cache-id-unknown', 404); + } + if (c.hash !== row.hash || c.fhir_version !== row.fhir_version) { + throw new Issue('error', 'conflict', null, 'CACHE_ID_CHANGED_DURING_REQUEST', + `The cache '${cacheId}' changed while this request was using it. Retry the request`, + null, 409); + } + return this._build({ kind: 'wrapper', resource_type: row.resource_type, fhir_version: c.fhir_version, json: c.json, hash: c.hash }); + } + + _build(row) { + const json = JSON.parse(row.json); + let resource = json; + if (row.kind === 'wrapper') { + const Cls = wrapperClass(row.resource_type); + if (Cls) { + resource = new Cls(json, 'R5'); + resource.fhirVersion = row.fhir_version || 'R5'; + if (row.hash && typeof resource.setExpansionKeyHash === 'function') { + // the hash of the JSON this was stored as, which is what the memory store's + // resource (the one the client sent) hashes to + resource.setExpansionKeyHash(row.hash); + } + } + } + markCached(resource); + return resource; + } + + /** + * As for the memory store, except: + * - an id with no record at all is CACHE_ID_UNKNOWN_SHARED, whatever its prefix. The + * memory store's messages offer a restart, or a request routed to the wrong instance, + * as explanations; here every process sees every cache, and caches survive a restart, + * so neither applies. + * - an id dropped by the size cap (maxBytes) is CACHE_ID_EVICTED. + */ + describeMissing(cacheId) { + const t = this.tombstone(cacheId); + if (!t) { + return { messageId: 'CACHE_ID_UNKNOWN_SHARED', params: [cacheId] }; + } + if (t.reason === 'evicted') { + return { + messageId: 'CACHE_ID_EVICTED', + params: [cacheId, formatDuration(Date.now() - t.at), formatBytes(t.maxBytes), formatDuration(t.idleMs)] + }; + } + return super.describeMissing(cacheId); + } + + tombstone(cacheId) { + const t = this.q.tombstone.get(this.scope, cacheId); + if (!t) { + return null; + } + return { + reason: t.reason, + at: t.at, + createdAt: t.created_at, + lastUsed: t.last_used, + idleMs: t.idle_ms, + maxAgeMs: t.max_age_ms, + maxBytes: t.max_bytes + }; + } + + tombstoneCount() { + return this.q.tombstoneCount.get(this.scope).n; + } + + size() { + return this.q.scopeTotals.get(this.scope).n; + } + + conceptCount() { + return this.q.scopeTotals.get(this.scope).concepts; + } + + conceptCountFor(cacheId) { + const s = this.q.session.get(this.scope, cacheId); + return s ? (s.concepts || 0) : 0; + } + + /** + * Bytes held across the whole file (every endpoint): that is what maxBytes caps. + */ + byteCount() { + return this.q.fileBytes.get().bytes; + } + + // High-water marks are this process's view (what it saw after its own writes) + _trackMax() { + try { + const t = this.q.scopeTotals.get(this.scope); + if (t.concepts > this.maxConcepts) { + this.maxConcepts = t.concepts; + } + if (t.n > this.maxSizeValue) { + this.maxSizeValue = t.n; + } + } catch (e) { + // statistics only + } + } + + // ---------- writes ---------- + + /** + * Run a write transaction (BEGIN IMMEDIATE). If it fails it is tried once more; + * if that fails too, the request gets a 503 (the tx router adds Retry-After), so + * the client knows to retry rather than carrying on without its resources. + */ + _write(what, fn) { + const tx = this.db.transaction(fn); + try { + return tx.immediate(); + } catch (first) { + this.log.warn(`resource cache: ${what} failed (${first.code || first.message}); retrying once`); + try { + return tx.immediate(); + } catch (second) { + this.log.error(`resource cache: ${what} failed twice: ${second.message}`); + throw new Issue('error', 'transient', null, 'CACHE_WRITE_FAILED', + `The server could not save to its resource cache just now (${what}: ${second.code || second.message}). Nothing was changed; try the request again`, + null, 503); + } + } + } + + /** + * Refresh last_used, at most once per touchIntervalMs: a cache in steady use + * then costs reads, not a write per request. A failed refresh is ignored - the + * worst it can do is let a cache look idle up to one interval sooner. + */ + _maybeTouch(cacheId, lastUsed) { + const now = Date.now(); + if (now - lastUsed < this.touchIntervalMs) { + return; + } + try { + this.db.transaction(() => { + this.q.touch.run(now, this.scope, cacheId, now - this.touchIntervalMs); + }).immediate(); + } catch (e) { + // ignored, see above + } + } + + touch(cacheId) { + if (!this.has(cacheId)) { + return false; + } + try { + // an explicit keepalive ($cache-control?mode=check) is not throttled + const now = Date.now(); + this.db.transaction(() => { + this.q.touch.run(now, this.scope, cacheId, now); + }).immediate(); + } catch (e) { + // a failed refresh is ignored: the cache is still there + } + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': touched (idle clock reset)`); + } + return true; + } + + /** + * The row to store for a resource. Wrappers are stored as their (R5) jsonObj plus + * the FHIR version they came in as; anything else is stored as itself. + */ + _row(cacheId, resource) { + const isWrapper = resource && typeof resource.jsonObj === 'object' && resource.jsonObj !== null + && !!wrapperClass(resource.resourceType); + const json = JSON.stringify(isWrapper ? resource.jsonObj : resource); + return { + scope: this.scope, + cacheId, + key: this._resourceKey(resource), + resourceType: resource.resourceType || null, + url: resource.url ?? null, + version: resource.version ?? null, + supplements: isWrapper && typeof resource.jsonObj.supplements === 'string' ? resource.jsonObj.supplements : null, + globalSupplement: isWrapper && typeof resource.isGlobalSupplement === 'function' ? this._globalSupplement(resource) : null, + fhirVersion: isWrapper ? (resource.fhirVersion || 'R5') : null, + kind: isWrapper ? 'wrapper' : 'plain', + hash: contentHash(json), + json, + bytes: Buffer.byteLength(json), + concepts: this._conceptsOf(resource) + }; + } + + _globalSupplement(resource) { + try { + return resource.isGlobalSupplement() ? 1 : 0; + } catch (e) { + return null; // not known: the stand-in loads the resource to ask it + } + } + + add(cacheId, resources) { + if (!resources || resources.length === 0) return; + resources = resources.filter(r => !!r); + for (const resource of resources) { + markCached(resource); + } + // the last copy of a key wins, as it would in the memory store + const byKey = new Map(); + for (const resource of resources) { + const row = this._row(cacheId, resource); + byKey.delete(row.key); + byKey.set(row.key, row); + } + const rows = [...byKey.values()]; + + // Most requests re-send resources the cache already has, unchanged. Check + // that with reads first, so they don't take the write lock at all. + const session = this.q.session.get(this.scope, cacheId); + if (session) { + const changed = rows.some(r => { + const existing = this.q.resourceHash.get(this.scope, cacheId, r.key); + return !existing || existing.hash !== r.hash || existing.concepts !== r.concepts; + }); + if (!changed) { + this._maybeTouch(cacheId, session.last_used); + return; + } + } + + this._write('add', () => { + const now = Date.now(); + if (!this.q.sessionExists.get(this.scope, cacheId)) { + if (this.q.tombstone.get(this.scope, cacheId)) { + // Ended or expired (by another process) between the caller's check and + // now: don't bring it back from the dead. + this.log.info(`cache-id '${cacheId}': not adding ${rows.length} resource(s), the cache is gone`); + return; + } + this.q.insertSession.run(this.scope, cacheId, now, now, 0); + } + for (const row of rows) { + this.q.upsertResource.run(row); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': stored ${row.key}`); + } + } + this.q.totals.run(now, this.scope, cacheId); + this._enforceMaxBytesTx(cacheId, now); + }); + this._trackMax(); + } + + set(cacheId, resources, sealed = false) { + resources = (resources || []).filter(r => !!r); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': set (replace all, sealed=${!!sealed}) with ${resources.length} resource(s): ${resources.map(r => this._resourceKey(r)).join(', ')}`); + } + for (const resource of resources) { + markCached(resource); + } + const rows = resources.map(r => this._row(cacheId, r)); + this._write('start', () => { + const now = Date.now(); + const existing = this.q.session.get(this.scope, cacheId); + this.q.deleteResources.run(this.scope, cacheId); + this.q.deleteSession.run(this.scope, cacheId); + this.q.insertSession.run(this.scope, cacheId, existing ? existing.created_at : now, now, sealed ? 1 : 0); + for (const row of rows) { + this.q.upsertResource.run(row); + } + this.q.totals.run(now, this.scope, cacheId); + // alive again, so any record of an earlier death is now wrong + this.q.deleteTombstone.run(this.scope, cacheId); + this._enforceMaxBytesTx(cacheId, now); + }); + this._trackMax(); + } + + /** Move a session to the tombstones (inside a write transaction). */ + _bury(session, reason, now, extra = {}) { + this.q.insertTombstone.run(session.scope, session.cache_id, reason, now, session.created_at, session.last_used, + extra.idleMs ?? null, extra.maxAgeMs ?? null, extra.maxBytes ?? null); + this.q.deleteResources.run(session.scope, session.cache_id); + this.q.deleteSession.run(session.scope, session.cache_id); + this.q.trimTombstones.run(session.scope, session.scope, this.maxTombstones); + } + + /** + * The size cap covers the whole file (all endpoints): drop the least recently + * used caches until the file is under it. Never the cache being written. + */ + _enforceMaxBytesTx(keepId, now) { + if (!this.maxBytes) { + return; + } + let total = this.q.fileBytes.get().bytes; + while (total > this.maxBytes) { + const victims = this.q.lru.all(this.scope, keepId); + if (victims.length === 0) { + return; + } + for (const v of victims) { + if (total <= this.maxBytes) { + break; + } + const idleMs = now - v.last_used; + this._bury(v, 'evicted', now, { idleMs, maxBytes: this.maxBytes }); + total -= v.bytes; + this.log.info(`cache-id '${v.cache_id}': evicted (cache over its ${formatBytes(this.maxBytes)} cap; idle ${formatDuration(idleMs)})`); + } + } + } + + clear(cacheId) { + this._write('end', () => { + const s = this.q.session.get(this.scope, cacheId); + if (s) { + this._bury(s, 'closed', Date.now()); + if (this.logCacheOps) { + this.log.info(`cache-id '${cacheId}': closed by client after ${formatDuration(Date.now() - s.created_at)} (held ${s.resources} resource(s))`); + } + } + }); + } + + clearAll() { + this._write('clear all', () => { + const now = Date.now(); + for (const s of this.q.allSessions.all(this.scope)) { + this._bury(s, 'cleared', now); + } + }); + } + + /** + * The idle sweep. One transaction: every session idle for longer than maxAge is + * tombstoned and deleted. Idempotent, so it doesn't matter which process runs + * it, or how many do. + */ + prune(maxAge = 3600000) { + if (this.stats) { + this.stats.task('Client Cache', 'Pruning'); + } + let n = 0; + try { + this.db.transaction(() => { + const now = Date.now(); + for (const s of this.q.expired.all(this.scope, now - maxAge)) { + const idleMs = now - s.last_used; + this._bury(s, 'expired', now, { idleMs, maxAgeMs: maxAge }); + n++; + if (this.logCacheOps) { + this.log.info(`cache-id '${s.cache_id}': expired after ${formatDuration(idleMs)} idle (timeout ${formatDuration(maxAge)})`); + } + } + }).immediate(); + } catch (e) { + // the next sweep will pick them up + this.log.error(`resource cache: idle sweep failed: ${e.message}`); + } + if (this.stats) { + this.stats.taskDone('Client Cache', `Pruned ${n} of ${n + this.size()} entries`); + } + } + + close() { + if (!this.closed) { + this.closed = true; + releaseDatabase(this.path); + } + } +} + +module.exports = { SqliteResourceCache, contentHash, formatBytes }; diff --git a/tx/tx.js b/tx/tx.js index 09d5ec42..a2402941 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -202,12 +202,19 @@ class TXModule { // the database can't be opened, that stops startup - silently running without it // would be worse. this.closureStore = null; - if (config.closure && config.closure.enabled) { - this.closureStore = new ClosureStore(config.closure, this.log); + // a tx-worker (cluster.md) opens neither the closure tables nor the syndication store: + // the master owns them, and $closure and the feed are routed to it - unless + // server.cluster.sharedClosure is on (set only with a role), when every process + // answers $closure on the one database (see closure-store.js) + const worker = config.clusterRole === 'tx-worker'; + const sharedClosure = config.sharedClosure === true; + if (config.closure && config.closure.enabled && (!worker || sharedClosure)) { + this.closureStore = new ClosureStore(config.closure, this.log, { shared: sharedClosure }); this.closureStore.open(); - this.log.info(`$closure enabled: tables in ${this.closureStore.path}`); + this.log.info(`$closure enabled: tables in ${this.closureStore.path}${sharedClosure ? ', shared with the other processes' : ''}`); const days = config.closure.retentionDays; - if (days) { + // with the tables shared, the master alone drops the stale ones + if (days && !worker) { const prune = async () => { try { const dropped = await this.closureStore.pruneUnused(days); @@ -225,7 +232,7 @@ class TXModule { // Initialize metadata handler with config this.metadataHandler = new MetadataHandler({ - closure: this.closureStore != null, + closure: this.closureStore != null || (worker && !!(config.closure && config.closure.enabled)), baseUrl: config.baseUrl, serverVersion: packageJson.version, txVersion: packageJson.txVersion, @@ -244,13 +251,21 @@ class TXModule { // Load the library from YAML this.log.info(`Loading library from: ${config.librarySource}`); this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats); + if (config.clusterRole) { + // the folder:/file: source caches are the master's, under one name every process shares + this.library.localSourceTag = 'master'; + // the master builds the package caches; a worker only reads them + this.library.packageCachesReadOnly = config.clusterRole === 'tx-worker'; + // the master refreshes the IANA media types registry; a worker reads its copy + this.library.mimeTypesReadOnly = config.clusterRole === 'tx-worker'; + } // The syndication feed: content is recorded in the store as the library loads it, from // the time the administrator turns it on (see tx/syndication). One store serves every // endpoint, since the library is shared. If it is turned on and the store can't be // opened, that stops startup. this.syndicationStore = null; - if (config.syndication && config.syndication.enabled) { + if (config.syndication && config.syndication.enabled && !worker) { this.syndicationStore = new SyndicationStore(path.join(this.library.cacheFolder, 'syndication'), this.log).open(); this.library.syndication = this.syndicationStore; this.log.info(`Syndication feed enabled: store in ${this.syndicationStore.folder}`); @@ -306,7 +321,9 @@ class TXModule { path: endpointPath, fhirVersion, context: context || null, - resourceCache: new ResourceCache(this.stats), + // server.cluster.sharedCache (cluster.md, "Options"; set only with a role): one SQLite + // file that every process shares. Otherwise this process's memory, as ever + resourceCache: this.config.sharedCache ? this.createSharedResourceCache(endpointPath) : new ResourceCache(this.stats), expansionCache: new ExpansionCache(this.stats, expansionCacheSize, expansionCacheMemoryThreshold) }; // Create the provider once for this endpoint @@ -1383,12 +1400,35 @@ class TXModule { /** * Shutdown the module */ + /** + * An endpoint's $cache-control cache in the shared SQLite file (databases/resource-cache.db), + * for server.cluster.sharedCache. Endpoints share the file but not caches. + * @param {string} endpointPath - e.g. /r4 + * @returns {SqliteResourceCache} + */ + createSharedResourceCache(endpointPath) { + const { SqliteResourceCache } = require('./resource-cache-sqlite'); + const folders = require('../library/folder-setup'); + const file = path.join(folders.databasesDir(), 'resource-cache.db'); + const maxBytes = this.config.sharedCacheMaxBytes || 0; + this.log.info(`Resource cache for ${endpointPath}: shared with the other processes, in ${file}` + + (maxBytes ? `, capped at ${maxBytes} bytes` : '')); + return new SqliteResourceCache(this.stats, { path: file, scope: endpointPath, maxBytes }); + } + async shutdown() { this.log.info('Shutting down TX module'); for (const timer of this.timers) { clearInterval(timer); } this.timers = []; + if (this.config && this.config.sharedCache) { + for (const ep of this.endpoints || []) { + if (ep.resourceCache) { + ep.resourceCache.close(); + } + } + } if (this.syndicationStore) { // let an induction in progress finish, so the store isn't left with a half-copied file await this.syndicationStore.whenIdle(); diff --git a/tx/vs/vs-database.js b/tx/vs/vs-database.js index 5ebeae88..40907a17 100644 --- a/tx/vs/vs-database.js +++ b/tx/vs/vs-database.js @@ -6,6 +6,9 @@ const ValueSet = require("../library/valueset"); // Settings key written once a package cache has been fully populated (see isComplete) const CACHE_COMPLETE_KEY = 'cache-complete'; +// A counter in vsac_settings that the VSAC sync bumps when it has written (cluster.md): +// a process that loaded the value set list at generation n is up to date while it says n +const GENERATION_KEY = 'generation'; // Columns that can be returned directly without parsing JSON const INDEXED_COLUMNS = ['id', 'url', 'version', 'date', 'description', 'name', 'publisher', 'status', 'title']; @@ -43,9 +46,17 @@ class ValueSetDatabase { /** * @param {string} dbPath - Path to the SQLite database file + * @param {Object} [options] - {shared: true} when several processes use the file (one + * writing, the others reading - cluster.md): WAL, so the readers don't block the writer, + * and a wait of up to 5 s for another process's lock instead of failing at once. + * {readOnly: true} for a cache another process builds (a tx-worker's package caches, + * cluster.md): opened read-only, never created or migrated - a file that still needs a + * migration fails to open (see _checkFinal), to be tried again once its builder is done */ - constructor(dbPath) { + constructor(dbPath, options = {}) { this.dbPath = dbPath; + this.shared = !!options.shared; + this.readOnly = !!options.readOnly; // Single read-write connection used for everything. Using a separate // OPEN_READONLY connection for reads can miss WAL-based schema changes // made through the write connection (because read-only opens can't fully @@ -76,6 +87,10 @@ class ValueSetDatabase { }); return (async () => { + if (this.shared) { + db.configure('busyTimeout', 5000); + await all('PRAGMA journal_mode = WAL'); + } const cols = await all("PRAGMA table_info(valuesets)"); const hasDateFirstSeen = cols.some(c => c.name === 'date_first_seen'); const hasContentHash = cols.some(c => c.name === 'content_hash'); @@ -194,6 +209,9 @@ class ValueSetDatabase { if (this._migrationPromise) { return this._migrationPromise; } + if (this.readOnly) { + return this._openReadOnly(); + } this._migrationPromise = new Promise((resolve, reject) => { if (this._writeDb) { this._migrateIfNeeded(this._writeDb).then(resolve).catch(reject); @@ -213,6 +231,70 @@ class ValueSetDatabase { return this._migrationPromise; } + /** + * readOnly: open the file read-only, and check it needs no migration. On failure the + * connection is closed (the next use opens it again: the builder may have replaced it) + * @returns {Promise} + * @private + */ + _openReadOnly() { + this._migrationPromise = new Promise((resolve, reject) => { + const db = new sqlite3.Database(this.dbPath, sqlite3.OPEN_READONLY, (err) => { + if (err) { + reject(new Error(`Failed to open database ${this.dbPath} read-only: ${err.message}`)); + return; + } + db.configure('busyTimeout', 5000); + this._checkFinal(db).then(() => { + this._writeDb = db; + resolve(); + }, (error) => { + db.close(() => reject(error)); + }); + }); + }); + this._migrationPromise.catch(() => { this._migrationPromise = null; }); + return this._migrationPromise; + } + + /** + * Reject unless the file is as _migrateIfNeeded leaves it: every column and table there, + * and no row without its content_hash + * @param {sqlite3.Database} db + * @returns {Promise} + * @private + */ + async _checkFinal(db) { + const all = (sql) => new Promise((res, rej) => { + db.all(sql, [], (err, rows) => err ? rej(err) : res(rows)); + }); + const columns = async (table) => (await all(`PRAGMA table_info(${table})`)).map(c => c.name); + const vsCols = await columns('valuesets'); + const pending = []; + for (const col of ['date_first_seen', 'content_hash']) { + if (!vsCols.includes(col)) { + pending.push(`valuesets.${col}`); + } + } + if (!(await columns('vsac_runs')).includes('total_updated')) { + pending.push('vsac_runs.total_updated'); + } + for (const table of ['vsac_settings', 'vsac_events']) { + if ((await columns(table)).length === 0) { + pending.push(table); + } + } + if (pending.length === 0) { + const rows = await all('SELECT COUNT(*) AS n FROM valuesets WHERE content_hash IS NULL'); + if (rows[0] && rows[0].n > 0) { + pending.push(`content_hash of ${rows[0].n} value sets`); + } + } + if (pending.length > 0) { + throw new Error(`Database ${this.dbPath} is not migrated yet (${pending.join(', ')}), and is opened read-only`); + } + } + /** * Close all database connections * @returns {Promise} @@ -470,6 +552,28 @@ class ValueSetDatabase { }); } + /** + * The generation (see GENERATION_KEY): 0 until a sync has bumped it. One read by key. + * @returns {Promise} + */ + async getGeneration() { + const value = await this.getSetting(GENERATION_KEY); + return value === null ? 0 : Number(value); + } + + /** + * Add one to the generation, in one statement. + * @returns {Promise} + */ + async bumpGeneration() { + const db = await this._getWriteConnection(); + return new Promise((resolve, reject) => { + db.run(`INSERT INTO vsac_settings (key, value) VALUES (?, '1') + ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1`, + [GENERATION_KEY], err => err ? reject(err) : resolve()); + }); + } + /** * Set a setting value in the vsac_settings table * @param {string} key - The setting key @@ -514,13 +618,17 @@ class ValueSetDatabase { /** * Insert or update a single ValueSet in the database * @param {Object} valueSet - The ValueSet resource - * @param {string} [contentHash] - Optional pre-computed content hash to store + * @param {string} [contentHash] - Optional pre-computed content hash to store; by default the + * hash of the content stored, as the backfill in _migrateIfNeeded computes it, so no row is + * written without one (a row without one is backfilled by the next process to open the file) * @returns {Promise} */ async upsertValueSet(valueSet, contentHash = null) { if (!valueSet.url) { throw new Error('ValueSet must have a url property'); } + const content = JSON.stringify(valueSet); + contentHash = contentHash || crypto.createHash('sha256').update(content).digest('hex'); const db = await this._getWriteConnection(); @@ -583,7 +691,7 @@ class ValueSetDatabase { valueSet.publisher || null, valueSet.status || null, valueSet.title || null, - JSON.stringify(valueSet), + content, contentHash ], (err) => { if (err) { diff --git a/tx/vs/vs-package.js b/tx/vs/vs-package.js index 8f349344..bf2b75fb 100644 --- a/tx/vs/vs-package.js +++ b/tx/vs/vs-package.js @@ -9,6 +9,7 @@ const { VersionUtilities } = require('../../library/version-utilities'); const {validateParameter} = require("../../library/utilities"); const { applyIdPrefix, stripIdPrefix } = require('../library/resource-ids'); const { keepForPinnedReference } = require('../library/tho-precedence'); +const { waitForCache } = require('../library/wait-for-cache'); /** * Package-based ValueSet provider using shared database layer @@ -18,15 +19,19 @@ class PackageValueSetProvider extends AbstractValueSetProvider { /** * @param {PackageContentLoader} packageLoader - Path to the extracted package folder + * @param {Object} [options] + * @param {boolean} [options.readOnly] - a tx-worker (cluster.md): the cache is the master's; it + * is opened read-only, waited for until the master has finished it, and never built or changed */ - constructor(packageLoader) { + constructor(packageLoader, options = {}) { super(); validateParameter(packageLoader, "packageLoader", PackageContentLoader); this.packageLoader = packageLoader; // -2: caches from before value sets were checked for active content in their narrative // are rebuilt (see initialize) this.dbPath = path.join(packageLoader.packageFolder, '.valuesets-2.db'); - this.database = new ValueSetDatabase(this.dbPath); + this.readOnly = !!options.readOnly; + this.database = new ValueSetDatabase(this.dbPath, { readOnly: this.readOnly }); this.valueSetMap = new Map(); this.initialized = false; this.count = 0; @@ -49,6 +54,13 @@ class PackageValueSetProvider extends AbstractValueSetProvider { return; } await this.packageLoader.initialize(); + if (this.readOnly) { + await waitForCache(async () => (await this.database.exists()) && (await this.database.isComplete()), + `Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: value set cache ${this.dbPath}`); + this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage()); + this.#finishLoad(); + return; + } await fs.rm(path.join(this.packageLoader.packageFolder, '.valuesets.db'), { force: true }); // the old cache // A cache is only used once its population finished. One that was cut short @@ -77,7 +89,10 @@ class PackageValueSetProvider extends AbstractValueSetProvider { await this.#buildDatabase(); this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage()); } + this.#finishLoad(); + } + #finishLoad() { // Mark each value set as cached as it enters the store: it is retained in // memory for the process lifetime, so its compose.include.filter elements // persist and a provider may memoise resolved filter analysis on them diff --git a/tx/vs/vs-vsac.js b/tx/vs/vs-vsac.js index 337a3573..dd2616c4 100644 --- a/tx/vs/vs-vsac.js +++ b/tx/vs/vs-vsac.js @@ -11,6 +11,10 @@ const { applyIdPrefix, stripIdPrefix } = require("../library/resource-ids"); // Persisted watermark for the phase-1b _lastUpdated scan. const VSAC_LAST_UPDATED_KEY = 'vsac_last_updated_date'; +// A follower (cluster.md) compares its generation with the database's at most this often, +// when a request uses the value sets +const GENERATION_CHECK_MS = 1000; + // Canonical URL prefix for VSAC value sets, so operators can enter a bare OID. const VSAC_VALUESET_URL_PREFIX = 'http://cts.nlm.nih.gov/fhir/ValueSet/'; @@ -30,6 +34,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { * @param {number} [config.timeoutMs=120000] - HTTP request timeout in milliseconds * @param {string} [config.resyncPassword] - If set, enables the operator "resync a ValueSet" * form on the /info page, gated by this password. If unset, the form is not offered. + * @param {string} [config.cluster] - set by --role (library/role.js), else absent: 'leader' + * (the master) syncs, and bumps the database's generation after each sync; 'follower' (a + * tx-worker) never syncs, and reloads its list when the generation moves (catchUp) */ constructor(config, stats) { super(); @@ -52,8 +59,13 @@ class VSACValueSetProvider extends AbstractValueSetProvider { this.refreshIntervalHours = config.refreshIntervalHours || 24; this.dbPath = path.join(this.cacheFolder, 'vsac-valuesets.db'); - this.database = new ValueSetDatabase(this.dbPath); + this.cluster = config.cluster || null; + this.database = new ValueSetDatabase(this.dbPath, { shared: !!this.cluster }); this.valueSetMap = new Map(); + // a follower: the generation its map was loaded at, and when it last compared + this.loadedGeneration = null; + this.generationCheckedAt = 0; + this.catchingUp = null; this.initialized = false; this.refreshTimer = null; this.isRefreshing = false; @@ -88,6 +100,14 @@ class VSACValueSetProvider extends AbstractValueSetProvider { if (this.initialized) { return; } + if (this.cluster === 'follower') { + // the master creates the database and syncs; this process only reads + if (await this.database.exists()) { + await this._reloadMap(); + } + this.initialized = true; + return; + } this.stats.addTask('VSAC Sync', `${this.refreshIntervalHours} hours`); // Create database if it doesn't exist @@ -235,6 +255,10 @@ class VSACValueSetProvider extends AbstractValueSetProvider { tracking.count++; } + // the followers reload when they see the generation move + if (this.cluster) { + await this.database.bumpGeneration(); + } // Reload map with fresh data await this._reloadMap(); @@ -260,6 +284,10 @@ class VSACValueSetProvider extends AbstractValueSetProvider { debugLog(error, 'Error during VSAC refresh:'); this.stats.taskError('VSAC Sync', `Error (${error.message})`); await this.database.failRun(runId, error.message); + if (this.cluster) { + // what the run wrote before it failed is in the database + await this.database.bumpGeneration(); + } throw error; } finally { this.isRefreshing = false; @@ -428,6 +456,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { * @private */ async _reloadMap() { + // the generation before the list: a sync finishing meanwhile makes the next check load + // again, never the reverse + const generation = this.cluster ? await this.database.getGeneration() : null; const newMap = await this.database.loadAllValueSets(this.sourcePackage()); for (const vs of newMap.values()) { if (vs.jsonObj.compose) { @@ -448,6 +479,40 @@ class VSACValueSetProvider extends AbstractValueSetProvider { } // Atomic replacement of the map this.valueSetMap = newMap; + if (this.cluster) { + this.loadedGeneration = generation; + } + } + + /** + * A follower: pick up what the master's sync wrote. At most every GENERATION_CHECK_MS, + * read the database's generation; if it isn't the one the map was loaded at, reload the + * map. Runs in the background (requests meanwhile use the map they have) and never + * rejects. Returns the promise of the check, for tests. + * @returns {Promise} + */ + catchUp() { + const now = Date.now(); + if (this.catchingUp || now - this.generationCheckedAt < GENERATION_CHECK_MS) { + return this.catchingUp || Promise.resolve(); + } + this.generationCheckedAt = now; + this.catchingUp = (async () => { + // not before the master has made it: opening it would create an empty file + if (this.loadedGeneration === null && !(await this.database.exists())) { + return; + } + const generation = await this.database.getGeneration(); + if (generation !== this.loadedGeneration) { + console.log(`VSAC: the database is at generation ${generation}, this process at ${this.loadedGeneration}; reloading`); + await this._reloadMap(); + } + })().catch((error) => { + debugLog(error, 'VSAC: catching up with the database failed'); + }).finally(() => { + this.catchingUp = null; + }); + return this.catchingUp; } /** @@ -458,6 +523,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { */ async fetchValueSet(url, version) { await this.initialize(); + if (this.cluster === 'follower') { + void this.catchUp(); + } this._validateFetchParams(url, version); // Try exact match first: url|version @@ -490,6 +558,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { } async fetchValueSetById(id) { + if (this.cluster === 'follower') { + void this.catchUp(); + } // the map is keyed by the value set's own id (and its url); the value set itself // carries the prefixed id, which is how a url key that happens to match is excluded const vs = this.valueSetMap.get(stripIdPrefix(id, this.spaceId)); @@ -502,6 +573,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { */ async searchValueSets(searchParams, elements) { await this.initialize(); + if (this.cluster === 'follower') { + void this.catchUp(); + } this._validateSearchParams(searchParams); if (searchParams.length === 0) { @@ -703,7 +777,8 @@ class VSACValueSetProvider extends AbstractValueSetProvider { * Whether the on-demand resync form is enabled (a resyncPassword is configured). */ _resyncEnabled() { - return !!this.resyncPassword; + // a follower doesn't write: the info pages are routed to the master + return !!this.resyncPassword && this.cluster !== 'follower'; } /** @@ -731,6 +806,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider { async resyncValueSet(url) { const tracking = { totalFetched: 0, totalNew: 0, totalUpdated: 0, count: 0, newCount: 0 }; await this.processContentAndHistory(url, tracking, 1); + if (this.cluster) { + await this.database.bumpGeneration(); + } await this._reloadMap(); return tracking.totalFetched; } diff --git a/tx/workers/closure.js b/tx/workers/closure.js index b2ef1fb7..42922f52 100644 --- a/tx/workers/closure.js +++ b/tx/workers/closure.js @@ -35,6 +35,13 @@ // name can read, add to, or reset that table. On a public server, require UUID names // (requireUuidNames) so that a name is effectively a secret held by the client that made it. // +// Shared (server.cluster.sharedClosure): other processes answer $closure on the same tables. +// The per-table lock still orders this process's calls, but a write may find the table has +// moved on since it was read: then nothing is written, and the call is worked out again +// from the table as it is now (see add, and the header of closure-store.js). After +// writeRetries such goes, the call gets the same 429 as a busy queue. The code lookups and +// subsumption tests done before are remembered for the retries, so a retry costs reads. +// // Cost: everything a client can make this do is bounded (see LIMITS) - the tables, the // concepts and entries in a table, what one call can add, what a table's configuration // can store, and how many calls can queue for one table and for how long. The work itself @@ -68,6 +75,11 @@ const LIMITS = { const RESYNC_PAGE = 2000; // rows read from the store at a time +// shared: how many times an add is worked out again when another process changed the table +// first, and the most it waits (with jitter) before a retry +const WRITE_RETRIES = 10; +const RETRY_WAIT_MS = 50; + class ClosureWorker extends TerminologyWorker { /** * @param {OperationContext} opContext - Operation context @@ -219,7 +231,19 @@ class ClosureWorker extends TerminologyWorker { if (!table && this.store.tableCount() >= max) { throw this.closureIssue('too-costly', 'CLOSURE_TOO_MANY_TABLES', [String(max)], 'too-costly', 422); } - table = this.store.createTable(name, config); + if (this.store.shared) { + // the checks above again, in the same transaction as the write + const made = this.store.createTableChecked(name, config, { reset, maxTables: max }); + if (made.refused === 'exists') { + throw this.closureIssue('duplicate', 'CLOSURE_TABLE_EXISTS', [name], 'business-rule', 409); + } + if (made.refused === 'limit') { + throw this.closureIssue('too-costly', 'CLOSURE_TOO_MANY_TABLES', [String(max)], 'too-costly', 422); + } + table = made.table; + } else { + table = this.store.createTable(name, config); + } this.log.info(`closure table '${name}' ${reset ? 'reset' : 'created'}`); if (concepts.length === 0) { return await this.buildConceptMap(name, table.version, [], fhirVersion); @@ -297,7 +321,9 @@ class ClosureWorker extends TerminologyWorker { let after = 0; for (;;) { await this.checkAndYield('closure-resync'); - const page = this.store.entriesPage(table.id, v, after, RESYNC_PAGE); + const page = this.store.shared + ? this.store.entriesPage(table.id, v, after, RESYNC_PAGE, table.version) + : this.store.entriesPage(table.id, v, after, RESYNC_PAGE); for (const r of page) { entries.push({ source: { system: r.source_system, code: r.source_code, display: r.source_display }, @@ -310,6 +336,17 @@ class ClosureWorker extends TerminologyWorker { } after = page[page.length - 1].rowid; } + if (this.store.shared) { + // Reset/dispose deletes the old pages. Never return a truncated snapshot, or + // entries from a replacement, as though they belonged to the version we read. + const current = this.store.getTable(name); + if (!current) { + throw this.closureIssue('not-found', 'CLOSURE_TABLE_UNKNOWN', [name], 'not-found', 404); + } + if (current.id !== table.id) { + throw this.closureIssue('conflict', 'CLOSURE_TABLE_CHANGED', [name], 'business-rule', 409); + } + } return await this.buildConceptMap(name, table.version, entries, fhirVersion); } @@ -320,6 +357,66 @@ class ClosureWorker extends TerminologyWorker { * system or code, or going over a limit, fails the whole call. */ async add(table, name, conceptParams, fhirVersion) { + if (this.store.shared) { + return await this.addShared(table, name, conceptParams, fhirVersion); + } + const { fresh, entries, out } = await this.workOutAdd(table, name, conceptParams, null); + const version = this.store.recordAdd(table, fresh, entries); + return await this.addResult(name, version, fresh, entries, out, fhirVersion); + } + + /** + * Shared: add, with the write checked against the table as it was read + * (recordAddChecked). If another process changed the table first, nothing was written: + * read the table again and work the add out again, up to writeRetries times, waiting a + * little (with jitter) in between; then 429, as for a busy queue. A table that is gone + * by then is unknown (404), as it would be to a call that came after the one that + * dropped it. + */ + async addShared(table, name, conceptParams, fhirVersion) { + const configured = this.closureConfig.writeRetries; + const retries = Number.isInteger(configured) && configured >= 0 ? configured : WRITE_RETRIES; + let memo = null; + for (let attempt = 0; ; attempt++) { + if (!memo || memo.tableId !== table.id) { + // what's remembered holds for one table's configuration: a reset starts again + memo = { tableId: table.id, providers: new Map(), codes: new Map(), subsumes: new Map() }; + } + const { fresh, entries, out } = await this.workOutAdd(table, name, conceptParams, memo); + let version = null; + try { + version = this.store.recordAddChecked(table, fresh, entries); + } catch (error) { + // the busy timeout ran out: another process held the write lock throughout + if (!String(error.code || '').startsWith('SQLITE_BUSY')) { + throw error; + } + } + if (version !== null) { + return await this.addResult(name, version, fresh, entries, out, fhirVersion); + } + if (attempt >= retries) { + this.log.info(`closure table '${name}': changed by another process on each of ${attempt + 1} tries; answering busy`); + throw this.closureIssue('too-costly', 'CLOSURE_TABLE_BUSY', [name], 'too-costly', 429); + } + this.log.info(`closure table '${name}': changed by another process since it was read; working the add out again (retry ${attempt + 1} of ${retries})`); + const wait = Math.random() * Math.min(RETRY_WAIT_MS, 5 * 2 ** attempt); + await this.opContext.waitFor(new Promise(resolve => setTimeout(resolve, wait)), 'closure-retry'); + table = this.store.getTable(name); + if (!table) { + throw this.closureIssue('not-found', 'CLOSURE_TABLE_UNKNOWN', [name], 'not-found', 404); + } + } + } + + /** + * Work out what adding the concepts to the table, as read, would write: the new + * concepts and the entries they make. Writes nothing. + * @param {Object} memo - shared only (else null): code lookups and subsumption tests + * already done for this table, kept across retries + * @returns {{fresh: Array, entries: Array, out: Array}} + */ + async workOutAdd(table, name, conceptParams, memo) { // replay the table's configuration: its code systems, and its version rules const stored = { resourceType: 'Parameters', parameter: this.store.getParams(table.id) }; this.setupAdditionalResources(stored); @@ -331,7 +428,7 @@ class ClosureWorker extends TerminologyWorker { for (const c of existing) { known.add(c.system + '|' + c.code); } - const providers = new Map(); + const providers = memo ? memo.providers : new Map(); const fresh = []; for (const p of conceptParams) { @@ -346,20 +443,31 @@ class ClosureWorker extends TerminologyWorker { this.seeSourceProvider(cs, coding.system); providers.set(coding.system, cs); } - const loc = await cs.locate(coding.code); - if (!loc || !loc.context) { - throw this.unknownCodeIssue(cs, coding.code, loc ? loc.message : null, txp); + const memoKey = memo ? coding.system + '|' + (coding.version || '') + '|' + coding.code : null; + let found = memo ? memo.codes.get(memoKey) : null; + if (!found) { + const loc = await cs.locate(coding.code); + if (!loc || !loc.context) { + throw this.unknownCodeIssue(cs, coding.code, loc ? loc.message : null, txp); + } + // the code system's own spelling of the code, so a case-insensitive code system + // doesn't end up with the same concept in the table twice + found = { context: loc.context, code: (await cs.code(loc.context)) || coding.code }; + if (memo) { + memo.codes.set(memoKey, found); + } } - // the code system's own spelling of the code, so a case-insensitive code system - // doesn't end up with the same concept in the table twice - const code = (await cs.code(loc.context)) || coding.code; + const code = found.code; const key = coding.system + '|' + code; if (known.has(key)) { continue; } known.add(key); + if (found.display === undefined) { + found.display = await cs.display(found.context); + } // ref: how the store refers to it until it has an id (see ClosureStore.recordAdd) - fresh.push({ key, ref: key, system: coding.system, code, display: await cs.display(loc.context), cs }); + fresh.push({ key, ref: key, system: coding.system, code, display: found.display, cs }); } const maxConcepts = this.limit('maxConceptsPerTable'); @@ -381,7 +489,7 @@ class ClosureWorker extends TerminologyWorker { continue; } await this.checkAndYield('closure-subsumes'); - const outcome = await n.cs.subsumesTest(n.code, o.code); + const outcome = memo ? await this.memoSubsumes(memo, n, o) : await n.cs.subsumesTest(n.code, o.code); let source, target, relationship; if (outcome === 'subsumes') { // the new concept is the broader one source = o; target = n; relationship = NARROWER; @@ -401,7 +509,20 @@ class ClosureWorker extends TerminologyWorker { before.push(n); } - const version = this.store.recordAdd(table, fresh, entries); + return { fresh, entries, out }; + } + + /** n.cs.subsumesTest(n.code, o.code), remembered in the memo (shared retries). */ + async memoSubsumes(memo, n, o) { + const key = n.system + '|' + n.code + '|' + o.code; + if (!memo.subsumes.has(key)) { + memo.subsumes.set(key, await n.cs.subsumesTest(n.code, o.code)); + } + return memo.subsumes.get(key); + } + + /** The response to an add, once it is written. */ + async addResult(name, version, fresh, entries, out, fhirVersion) { if (fresh.length > 0) { this.log.info(`closure table '${name}': added ${fresh.length} concept(s), ${entries.length} entr${entries.length === 1 ? 'y' : 'ies'}, now version ${version}`); } diff --git a/tx/workers/compare.js b/tx/workers/compare.js index 0b1ca0f4..87a90cdb 100644 --- a/tx/workers/compare.js +++ b/tx/workers/compare.js @@ -172,9 +172,12 @@ class CompareWorker extends TerminologyWorker { async doCompare(txp, thisVS, otherVS) { - // ok, we have to compare the composes. we don't care about anything else - const thisC = thisVS.jsonObj.compose; - const otherC = otherVS.jsonObj.compose; + // ok, we have to compare the composes. we don't care about anything else. + // Copies: tidyIncludes merges, removes and sorts includes in place, and these value + // sets are shared - cache-id resources, and the server's own - so must not change + // (ExpansionCache.computeKey remembers a cached resource's hash, for one). + const thisC = thisVS.jsonObj.compose ? structuredClone(thisVS.jsonObj.compose) : thisVS.jsonObj.compose; + const otherC = otherVS.jsonObj.compose ? structuredClone(otherVS.jsonObj.compose) : otherVS.jsonObj.compose; if (!thisC) { return this.makeOutcome("indeterminate", `The ValueSet ${thisVS.vurl} has no compose`); } diff --git a/tx/workers/worker.js b/tx/workers/worker.js index 8f9f3cfd..8dd23949 100644 --- a/tx/workers/worker.js +++ b/tx/workers/worker.js @@ -354,7 +354,9 @@ class TerminologyWorker { if (resource.resourceType === 'CodeSystem' && resource instanceof CodeSystem) { const cs = resource; // Check if this code system supplements the target URL - const supplementsUrl = cs.jsonObj.supplements; + // (supplementsUrl rather than jsonObj.supplements: a resource from the shared + // resource cache answers it without being loaded - see resource-cache-sqlite.js) + const supplementsUrl = cs.supplementsUrl; if (!supplementsUrl) { continue;