diff --git a/config-template.json b/config-template.json index de2602ff..331f338a 100644 --- a/config-template.json +++ b/config-template.json @@ -252,6 +252,12 @@ // (and a "Library" item in the tx nav bar), anything else gets the YAML file itself. // Off by default - the file names every database, cache and package the server runs. "publishLibrarySource": false, + // How SNOMED CT cache files are loaded (see tx/README.md, "Loading SNOMED CT"). Optional; + // without it every edition is read into the process, as before. "load": "read" or "mmap" + // for every edition; "prewarm": editions to map, then read through once in the background at startup; + // "read": editions to read into the process anyway. Entries are cache file names, edition + // version URIs or edition URIs + // "snomed": { "load": "mmap", "prewarm": ["sct_intl_20250201.cache", "sct_us_20250901b.cache"], "read": [] }, "cacheTimeout": 30, // Optional. A short code (1-16 letters and digits) naming this server instance. Every // cache-id the server issues then starts with ".", so a proxy in front of diff --git a/library/memory-guard.js b/library/memory-guard.js index 232f488e..2fa760ab 100644 --- a/library/memory-guard.js +++ b/library/memory-guard.js @@ -16,6 +16,7 @@ * @module library/memory-guard */ +const fs = require('fs'); const os = require('os'); const v8 = require('v8'); const vm = require('vm'); @@ -33,6 +34,13 @@ const GC_INTERVAL_MS = 5000; // next, up to this: when most of RSS is outside the JS heap (the SNOMED and LOINC cache // buffers), collecting frees little, and every full GC stops the process const GC_MAX_INTERVAL_MS = 60000; +// RssFile (/proc/self/status) is read at most this often. The in-operation check runs many +// times a request, and a read of /proc/self/status costs twice what the RSS itself does; +// between reads, a fresh RSS less the last RssFile is a close enough estimate +const FILE_SAMPLE_MS = 200; +// the in-operation check reuses a measure at most this old. It runs about ten times a request, +// and RSS moves little in 10 ms +const ABORT_SAMPLE_MS = 10; const state = { limit: 0, @@ -42,7 +50,11 @@ const state = { rejectAt: 0, abortAt: 0, sampleTime: 0, - sampleRss: 0 + sampleRss: 0, + // set by measureExcludingFileBacked() + excludeFileBacked: false, + fileTime: 0, + fileRss: 0 // the last RssFile read (see fileBackedRss) }; function fraction(v, dflt) { @@ -75,10 +87,42 @@ function configure(opts = {}) { state.sampleTime = 0; } +/** + * Measure RSS less its file-backed pages (Linux only). With SNOMED CT cache files memory-mapped + * ("snomed": {"load": "mmap"} in config.json) RSS includes every mapped page that has been read, which the kernel drops + * and re-reads as it needs to - counting them would refuse requests on a server that is fine. + */ +function measureExcludingFileBacked() { + state.excludeFileBacked = process.platform === 'linux'; + state.sampleTime = 0; +} + +/** + * RssFile as last read, read again if that was FILE_SAMPLE_MS ago or more, or if the sample + * has been dropped (sampleTime 0: configure, measureExcludingFileBacked, a test). + */ +function fileBackedRss() { + const now = Date.now(); + if (state.sampleTime === 0 || now - state.fileTime >= FILE_SAMPLE_MS) { + state.fileRss = rssFileBytes(); + state.fileTime = now; + } + return state.fileRss; +} + +function rssFileBytes() { + try { + const m = /^RssFile:\s+(\d+) kB/m.exec(fs.readFileSync('/proc/self/status', 'utf8')); + return m ? Number(m[1]) * 1024 : 0; + } catch { + return 0; + } +} + function rss(fresh = false) { const now = Date.now(); if (fresh || now - state.sampleTime > SAMPLE_MS) { - state.sampleRss = process.memoryUsage.rss(); + state.sampleRss = process.memoryUsage.rss() - (state.excludeFileBacked ? fileBackedRss() : 0); state.sampleTime = now; } return state.sampleRss; @@ -123,7 +167,9 @@ let lastAbortLog = 0; let abortLog = null; /** - * Current RSS in bytes if running operations should be aborted, else 0. Always a fresh read. + * Current RSS in bytes if running operations should be aborted, else 0. A measure at most + * ABORT_SAMPLE_MS old (less the file-backed part as read at most FILE_SAMPLE_MS ago), or fresh + * if the sample was dropped (configure, measureExcludingFileBacked). * A trip is written to the server log (at most once a minute, with a running count) - the * caller's own message only goes to the aborted operation's diagnostics. * @param {string} [place] - where the check was made, for the log @@ -132,7 +178,7 @@ function overAbortThreshold(place = 'unknown') { if (state.abortAt <= 0) { return 0; } - const r = rss(true); + const r = rss(Date.now() - state.sampleTime > ABORT_SAMPLE_MS); if (r <= state.abortAt) { return 0; } @@ -158,6 +204,7 @@ function gb(bytes) { function describe() { return `Memory limit: ${gb(state.limit)} GB (from ${state.source}); ` + + (state.excludeFileBacked ? 'measured as RSS less file-backed pages; ' : '') + `new requests refused above ${gb(state.rejectAt)} GB (${Math.round(state.rejectFraction * 100)}%), ` + `operations aborted above ${gb(state.abortAt)} GB (${Math.round(state.abortFraction * 100)}%)`; } @@ -260,4 +307,4 @@ function middleware(opts = {}) { configure(); -module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, _state: state }; +module.exports = { configure, measureExcludingFileBacked, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, _state: state }; diff --git a/package-lock.json b/package-lock.json index 9ff86470..2800b80a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,6 +71,9 @@ }, "engines": { "node": ">=24" + }, + "optionalDependencies": { + "koffi": "^3.3.6" } }, "node_modules/@babel/code-frame": { @@ -1522,6 +1525,326 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@koromix/koffi-android-arm64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-arm64/-/koffi-android-arm64-3.3.6.tgz", + "integrity": "sha512-wxsPuEMDrqOR5RUdABYFNDTabhh/xwDetCUDI+/CN93QF1T7Yeq218kvDWmYT/g1wV2kU1aYdRO4D7cH9PDskw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-android-x64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-x64/-/koffi-android-x64-3.3.6.tgz", + "integrity": "sha512-fHZE6ejYLmAEe2eou2uoG61HUtolY45RhQEsTKAKNI5eKSW7Z2Jh8eP5EE+3CVQSbzbaMeHXRTcgEQGk5uEF5Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-arm64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-arm64/-/koffi-darwin-arm64-3.3.6.tgz", + "integrity": "sha512-++V4qscXff/Ya5cE6vOPOomcxk3T5JQu81Uc0fvoso3VVELcoHt2/dz9E0FY0EPyQNIRcWOVg7CgvsPCKId9lg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-x64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-x64/-/koffi-darwin-x64-3.3.6.tgz", + "integrity": "sha512-eOyit0eyfkmdXCNlOkG8cNtpQUx+dikr1QsozJdwm7/zH1u7KP5dRtmsaGUMLSUY7cVYSRTUStkwHBrHjBtuzw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-arm64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-arm64/-/koffi-freebsd-arm64-3.3.6.tgz", + "integrity": "sha512-7teGsq2BukkLZ60xxq4KyPxeJct51nV1G/MfNj8IoMTqlpd35w4aiIIoFWVwLL3j3DdT95Ye/FpVY4C7ZEuMHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-ia32": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-ia32/-/koffi-freebsd-ia32-3.3.6.tgz", + "integrity": "sha512-WA7qEAMxTWS2WSFDceETVJTFrZgRwNvfG+FJpCC2XkJXftHwOhv8/X6d5HUEXh361sd7HwCHoT1wypL0nvI7Mw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-x64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-x64/-/koffi-freebsd-x64-3.3.6.tgz", + "integrity": "sha512-OQ/eBhGefbtxcPs5VDkK9mwxoIeSwrIriOT1/JHHA/tbNAXdC0GiAYsBw82T4L4r2u0TuJ0CbgN3HrulzdPPig==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm/-/koffi-linux-arm-3.3.6.tgz", + "integrity": "sha512-0uo15z9L7GsAEGvPCVKcXzCtRJCyghCYDrWO2L3vXk4G5lgk+OBvWS7RmlDZ99X/A0oDiL3vLCplGaBCEZZ8Eg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm64/-/koffi-linux-arm64-3.3.6.tgz", + "integrity": "sha512-8LuJcdH74wTg7JcZjnRnb3z+qWPKcEDSv8HRPkWTVD56TFm0CXZXgl/WJ4maDbWem6ZoXxc87a8nQEA+wABzNA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ia32": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ia32/-/koffi-linux-ia32-3.3.6.tgz", + "integrity": "sha512-aTOjTMdy2Q6JqNA9KkT2OaCs77d0TzEG5KipfinzDyiV9DkYyUCvFykIY5GDHCUKXyJi+xy1IfuUxeL64wzzBQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-loong64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-loong64/-/koffi-linux-loong64-3.3.6.tgz", + "integrity": "sha512-m33Zjrhj2Atpkcr0We61V2Zjm8wFb79QOtaOfzttmNFQMtryYpWZ4CBxx4gLKvcrR7NcK3GV/D9Qi2M0ppWK7A==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ppc64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ppc64/-/koffi-linux-ppc64-3.3.6.tgz", + "integrity": "sha512-L1+1cjH5GtNtv7TYWy469jsDberOOwCfE96h0AWFKrVHycDn8hTKH5fEA/er42R04lZiNWbiC0+sMb50enYRGw==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-riscv64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-riscv64/-/koffi-linux-riscv64-3.3.6.tgz", + "integrity": "sha512-VnzoNRg3MfTdGXLDtUrBj4wJnZD3buX6LajGrvYm5Mf1U132bzQwxP1ryZhs8Q0Cl1pTsN2fIovjNwpwE7Hs3A==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-x64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-x64/-/koffi-linux-x64-3.3.6.tgz", + "integrity": "sha512-RTRbVTZ1rf1GFowSsbuU2acYoRq66ZwJGh+rTgau1Cat3muv2884p0FRH6UKfwlgWkpjIHJ7N8N0liM7Oe2eAQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-arm64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-arm64/-/koffi-openbsd-arm64-3.3.6.tgz", + "integrity": "sha512-eXHyRZw/BACrBIuntkqlNTa8p8i9KobtREJjhxldsVoEPRQTREnhBq5IbqbvXesY89MwFGn4lHmzG1x5ueV63A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-ia32": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-ia32/-/koffi-openbsd-ia32-3.3.6.tgz", + "integrity": "sha512-xbJYOFY0IAIx3nyNdYad5g2f8cNAOY8OBcjPDYRlc7gkqQiPNER9ZA9g+M2LuVf48i65ZCn11XaWDSQymDJJBg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-x64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-x64/-/koffi-openbsd-x64-3.3.6.tgz", + "integrity": "sha512-bNL4uIxUBhA93naaL3dWwXdLgQGAss6krDbwljO1uS3iJ+8K7KSo8EGbJfRYUf1L+ctaHTJuDwgZ1XTSEC9FQQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-arm64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-arm64/-/koffi-win32-arm64-3.3.6.tgz", + "integrity": "sha512-WDVsrvWAaHbA7xiCMXX3PBmfDf4+VnfY67ryRAW86QNW8109W9WPS3YVGGD2YoPswDmSZsMBVzzZCEFUrLdJuA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-ia32": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-ia32/-/koffi-win32-ia32-3.3.6.tgz", + "integrity": "sha512-6hwsUGC7FCICiLdmEQwTK0+MOLmuraqWuVsUCBkdSCL6k62LQLScuTn2dzONFvN/H5EXq0zzX4ddgtq2abL/xQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-x64": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-x64/-/koffi-win32-x64-3.3.6.tgz", + "integrity": "sha512-g9KC6VKQLe88Tjgeubk3yhu0CjtqSeRq4XjWFWxUq4rsP0/wfLUBoeISvG60HoPmhYQp+U7hVd8Ms+amOMHDvA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, "node_modules/@lhncbc/ucum-lhc": { "version": "7.1.10", "resolved": "https://registry.npmjs.org/@lhncbc/ucum-lhc/-/ucum-lhc-7.1.10.tgz", @@ -6632,6 +6955,39 @@ "json-buffer": "3.0.1" } }, + "node_modules/koffi": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-3.3.6.tgz", + "integrity": "sha512-pPqCZg8o4laLvzxySGEju/T1Fd+eCJTVnJmcH6h1u98HFRoBQ9EokNuBSj3NxIQdzkEPRxPnHKjhBVkLeB+X0g==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "funding": { + "url": "https://liberapay.com/Koromix" + }, + "optionalDependencies": { + "@koromix/koffi-android-arm64": "3.3.6", + "@koromix/koffi-android-x64": "3.3.6", + "@koromix/koffi-darwin-arm64": "3.3.6", + "@koromix/koffi-darwin-x64": "3.3.6", + "@koromix/koffi-freebsd-arm64": "3.3.6", + "@koromix/koffi-freebsd-ia32": "3.3.6", + "@koromix/koffi-freebsd-x64": "3.3.6", + "@koromix/koffi-linux-arm": "3.3.6", + "@koromix/koffi-linux-arm64": "3.3.6", + "@koromix/koffi-linux-ia32": "3.3.6", + "@koromix/koffi-linux-loong64": "3.3.6", + "@koromix/koffi-linux-ppc64": "3.3.6", + "@koromix/koffi-linux-riscv64": "3.3.6", + "@koromix/koffi-linux-x64": "3.3.6", + "@koromix/koffi-openbsd-arm64": "3.3.6", + "@koromix/koffi-openbsd-ia32": "3.3.6", + "@koromix/koffi-openbsd-x64": "3.3.6", + "@koromix/koffi-win32-arm64": "3.3.6", + "@koromix/koffi-win32-ia32": "3.3.6", + "@koromix/koffi-win32-x64": "3.3.6" + } + }, "node_modules/kuler": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", diff --git a/package.json b/package.json index d306d5d9..3fdcdadf 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,9 @@ "winston-daily-rotate-file": "^4.7.1", "yaml": "^2.8.2" }, + "optionalDependencies": { + "koffi": "^3.3.6" + }, "devDependencies": { "@types/jest": "^30.0.0", "@typescript-eslint/eslint-plugin": "^8.54.0", diff --git a/tests/library/memory-guard.test.js b/tests/library/memory-guard.test.js index c2da3984..f07803cf 100644 --- a/tests/library/memory-guard.test.js +++ b/tests/library/memory-guard.test.js @@ -148,4 +148,97 @@ describe('memory-guard', () => { mw({ path: '/r4/x' }, fakeRes(), () => {}); expect(log.warn.mock.calls.some(c => /accepting requests again after \d+s \(1 refused\)/.test(c[0]))).toBe(true); }); + + (process.platform === 'linux' ? test : test.skip)('with mapped files, file-backed pages are not counted', () => { + const fs = require('fs'); + const realRead = fs.readFileSync; + const readSpy = jest.spyOn(fs, 'readFileSync').mockImplementation((file, ...rest) => + file === '/proc/self/status' ? `VmRSS:\t1 kB\nRssFile:\t${4 * 1024 * 1024} kB\n` : realRead(file, ...rest)); + try { + setRss(9 * GB); // 4 GB of it file-backed + expect(memoryGuard.shouldReject()).toBe(true); + memoryGuard.measureExcludingFileBacked(); + expect(memoryGuard.shouldReject()).toBe(false); + expect(memoryGuard.overAbortThreshold('test')).toBe(0); + } finally { + memoryGuard._state.excludeFileBacked = false; + readSpy.mockRestore(); + } + }); + + (process.platform === 'linux' ? test : test.skip)('with mapped files, the in-operation check reads RssFile at most once per window, and RSS every 10 ms', () => { + const fs = require('fs'); + const realRead = fs.readFileSync; + let reads = 0; + let rssFileKb = 2 * 1024 * 1024; + const readSpy = jest.spyOn(fs, 'readFileSync').mockImplementation((file, ...rest) => { + if (file !== '/proc/self/status') { + return realRead(file, ...rest); + } + reads++; + return `VmRSS:\t1 kB\nRssFile:\t${rssFileKb} kB\n`; + }); + let now = 1000000; + const nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); + try { + memoryGuard.measureExcludingFileBacked(); + setRss(10.5 * GB); + expect(memoryGuard.overAbortThreshold('test')).toBe(0); // 8.5 GB counts + expect(reads).toBe(1); + // many checks inside the window: one read, while a rise in RSS is seen within 10 ms + for (let i = 0; i < 1000; i++) { + now += 0.1; + expect(memoryGuard.overAbortThreshold('test')).toBe(0); + } + rssSpy.mockReturnValue(11.5 * GB); + now += 10.5; + expect(memoryGuard.overAbortThreshold('test')).toBe(9.5 * GB); + expect(reads).toBe(1); + expect(rssSpy.mock.calls.length).toBeGreaterThanOrEqual(10); + expect(rssSpy.mock.calls.length).toBeLessThanOrEqual(12); + // once the window has passed, RssFile is read again + rssFileKb = 3 * 1024 * 1024; + now += 200; + expect(memoryGuard.overAbortThreshold('test')).toBe(0); // 8.5 GB counts + expect(reads).toBe(2); + } finally { + nowSpy.mockRestore(); + memoryGuard._state.excludeFileBacked = false; + readSpy.mockRestore(); + } + }); + + test('many checks inside 10 ms read RSS once; a dropped sample is measured again at once', () => { + let now = 1000000; + const nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); + try { + setRss(9.5 * GB); + rssSpy.mockClear(); + for (let i = 0; i < 100; i++) { + now += 0.09; + expect(memoryGuard.overAbortThreshold('test')).toBe(9.5 * GB); + } + expect(rssSpy).toHaveBeenCalledTimes(1); + // a fall inside the 10 ms is not seen yet... + rssSpy.mockReturnValue(7 * GB); + expect(memoryGuard.overAbortThreshold('test')).toBe(9.5 * GB); + expect(rssSpy).toHaveBeenCalledTimes(1); + // ...but configure drops the sample, and the next check reads afresh + memoryGuard.configure({ limit: 10 * GB, rejectFraction: 0.8, abortFraction: 0.9 }); + expect(memoryGuard.overAbortThreshold('test')).toBe(0); + expect(rssSpy).toHaveBeenCalledTimes(2); + // as do measureExcludingFileBacked and any other drop of the sample + rssSpy.mockReturnValue(9.5 * GB); + memoryGuard.measureExcludingFileBacked(); + memoryGuard._state.excludeFileBacked = false; + expect(memoryGuard.overAbortThreshold('test')).toBe(9.5 * GB); + expect(rssSpy).toHaveBeenCalledTimes(3); + rssSpy.mockReturnValue(7 * GB); + memoryGuard._state.sampleTime = 0; + expect(memoryGuard.overAbortThreshold('test')).toBe(0); + expect(rssSpy).toHaveBeenCalledTimes(4); + } finally { + nowSpy.mockRestore(); + } + }); }); diff --git a/tests/tx/snomed-load-config.test.js b/tests/tx/snomed-load-config.test.js new file mode 100644 index 00000000..a42b77dc --- /dev/null +++ b/tests/tx/snomed-load-config.test.js @@ -0,0 +1,299 @@ +/** + * How each SNOMED CT cache file is loaded: the tx module's "snomed" block in config.json + * (tx/sct/load-config.js), and the library applying it. + */ +const { SnomedLoadConfig } = require('../../tx/sct/load-config'); +const { SnomedFileReader, SnomedConceptList } = require('../../tx/sct/structures'); +const { SnomedServices } = require('../../tx/cs/cs-snomed'); +const { mappingAvailable, _setMmapForTesting } = require('../../tx/sct/mmap'); +const { Library } = require('../../tx/library'); +const folders = require('../../library/folder-setup'); +const memoryGuard = require('../../library/memory-guard'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +const INTL = 'http://snomed.info/sct/900000000000207008'; +const US = 'http://snomed.info/sct/731000124108'; +const intl25 = ['/data/sct_intl_20250201.cache', `${INTL}/version/20250201`]; +const intl26 = ['/data/Snomed_Intl_20260701.cache', `${INTL}/version/20260701`]; +const us = ['/data/sct_us_20250901b.cache', `${US}/version/20250901`]; +const uk = ['/data/sct_uk_20230412.cache', 'http://snomed.info/sct/83821000000107/version/20230412']; + +function config(block) { + const warnings = []; + return { cfg: new SnomedLoadConfig(block, (m) => warnings.push(m)), warnings }; +} + +describe('SnomedLoadConfig', () => { + test('no block: everything is read, as before', () => { + const { cfg, warnings } = config(undefined); + expect([intl25, us, uk].map(f => cfg.modeFor(...f))).toEqual(['read', 'read', 'read']); + expect(cfg.mapsAnything()).toBe(false); + cfg.checkUnused(); + expect(warnings).toEqual([]); + }); + + test('load mmap maps everything', () => { + const { cfg } = config({ load: 'mmap' }); + expect([intl25, us, uk].map(f => cfg.modeFor(...f))).toEqual(['mmap', 'mmap', 'mmap']); + expect(cfg.mapsAnything()).toBe(true); + }); + + test('entries match a file name, a version URI, or an edition URI (every version)', () => { + const { cfg, warnings } = config({ + load: 'mmap', + read: ['sct_us_20250901b.cache'], + prewarm: [`${INTL}/version/20250201`, 'http://snomed.info/sct/83821000000107'] + }); + expect(cfg.modeFor(...us)).toBe('read'); + expect(cfg.modeFor(...intl25)).toBe('prewarm'); + expect(cfg.modeFor(...intl26)).toBe('mmap'); + expect(cfg.modeFor(...uk)).toBe('prewarm'); + cfg.checkUnused(); + expect(warnings).toEqual([]); + }); + + test('an edition URI matches every version of it', () => { + const { cfg } = config({ load: 'mmap', read: [INTL] }); + expect(cfg.modeFor(...intl25)).toBe('read'); + expect(cfg.modeFor(...intl26)).toBe('read'); + expect(cfg.modeFor(...us)).toBe('mmap'); + }); + + test('"read" wins over "prewarm", which wins over "load"', () => { + const { cfg, warnings } = config({ load: 'read', read: ['sct_uk_20230412.cache'], prewarm: ['sct_uk_20230412.cache', `${US}/version/20250901`] }); + expect(cfg.modeFor(...uk)).toBe('read'); + expect(cfg.modeFor(...us)).toBe('prewarm'); // prewarm maps the file even when load is read + expect(cfg.modeFor(...intl25)).toBe('read'); + expect(cfg.mapsAnything()).toBe(true); + expect(warnings).toEqual([expect.stringMatching(/sct_uk_20230412.cache is in both "prewarm" and "read"/)]); + }); + + test('bad values are warned about and ignored', () => { + const { cfg, warnings } = config({ load: 'fast', read: 'sct_us_20250901b.cache', prewarm: [3] }); + expect(cfg.load).toBe('read'); + expect(cfg.modeFor(...us)).toBe('read'); + expect(warnings).toHaveLength(3); + expect(warnings[0]).toMatch(/snomed.load must be "read" or "mmap"/); + expect(warnings[1]).toMatch(/snomed.prewarm must be a list/); + expect(warnings[2]).toMatch(/snomed.read must be a list/); + }); + + test('entries that match no loaded edition are warned about', () => { + const { cfg, warnings } = config({ load: 'mmap', read: ['sct_us_20250901.cache'], prewarm: [US] }); + cfg.modeFor(...us); + cfg.modeFor(...intl25); + cfg.checkUnused(); + expect(warnings).toEqual([expect.stringMatching(/snomed.read: "sct_us_20250901.cache" matches no SNOMED CT edition/)]); + }); +}); + +describe('SnomedFileReader.versionUri', () => { + test('reads the edition version URI from the header', () => { + const file = path.resolve(__dirname, '../../tx/data/snomed-testing.cache'); + expect(SnomedFileReader.versionUri(file)).toMatch(/^http:\/\/snomed\.info\/x?sct\/\d+\/version\/\d{8}$/); + }); +}); + +const CACHE = path.resolve(__dirname, '../../tx/data/snomed-testing.cache'); +const CAN_MAP = mappingAvailable(); +const NO_MAP = ' (skipped: memory mapping is not available on this platform - needs 64-bit Linux/macOS/BSD with koffi)'; +const testIfMapping = CAN_MAP ? test : test.skip; +const ifMapping = (name) => CAN_MAP ? name : name + NO_MAP; + +/** Whether this process has filePath mapped, from /proc/self/maps (Linux only). */ +function inProcMaps(filePath) { + return fs.readFileSync('/proc/self/maps', 'utf8').split('\n').some(l => l.endsWith(' ' + fs.realpathSync(filePath))); +} + +// Fails every mmap() call the way libc does (MAP_FAILED, errno set) +const FAILING_MMAP = { + koffi: { address: () => 0xFFFFFFFFFFFFFFFFn, errno: () => 12 }, + mmap: () => ({}) +}; + +async function readCache(loadMode, log) { + const reader = new SnomedFileReader(CACHE, loadMode, log); + const data = await reader.loadSnomedData(); + return { reader, data, services: new SnomedServices(data) }; +} + +/** Everything the services report about one concept. */ +function describeConcept(sct, index) { + const c = sct.concepts.getConcept(index); + const descriptions = c.descriptions === 0 ? [] : sct.refs.getReferences(c.descriptions).map(i => { + const d = sct.descriptions.getDescription(i); + return { id: String(d.id), term: sct.strings.getEntry(d.iDesc), active: d.active, kind: d.kind, lang: d.lang }; + }); + const rels = (ref) => ref === 0 ? [] : sct.refs.getReferences(ref).map(i => { + const r = sct.relationships.getRelationship(i); + return [String(sct.concepts.getConceptId(r.source)), String(sct.concepts.getConceptId(r.target)), + String(sct.concepts.getConceptId(r.relType)), r.group, r.active]; + }); + return { + id: String(sct.concepts.getConceptId(index)), + display: sct.getDisplayName(index), + active: sct.isActive(index), + descriptions, + outbound: rels(c.outbounds), + inbound: rels(c.inbounds), + parents: sct.getConceptParents(index).map(p => String(sct.concepts.getConceptId(p))) + }; +} + +describe('SnomedFileReader: memory mapping', () => { + afterEach(() => _setMmapForTesting()); + + test('load read: the buffer is not a mapping', async () => { + const { reader } = await readCache('read'); + expect(reader.mapped).toBe(false); + }); + + testIfMapping(ifMapping('load mmap: the buffer really is a mapping of the file'), async () => { + const before = process.platform === 'linux' && inProcMaps(CACHE); + const { reader } = await readCache('mmap'); + expect(reader.mapped).toBe(true); + if (process.platform === 'linux') { + expect(before).toBe(false); + expect(inProcMaps(CACHE)).toBe(true); // the kernel lists the file among this process's mappings + } + }); + + testIfMapping(ifMapping('prewarm: mapped, and the background read finishes'), async () => { + const { reader } = await readCache('prewarm'); + expect(reader.mapped).toBe(true); + expect(typeof await reader.prewarmed).toBe('number'); + }); + + testIfMapping(ifMapping('a mapped load gives the same data as a read load'), async () => { + const read = await readCache('read'); + const mapped = await readCache('mmap'); + expect(read.reader.mapped).toBe(false); + expect(mapped.reader.mapped).toBe(true); + + // the structures, byte for byte, and everything else in the header + const sections = ['strings', 'refs', 'desc', 'words', 'stems', 'concept', 'rel', 'refSetIndex', 'refSetMembers', 'descRef']; + const rest = (d) => Object.fromEntries(Object.entries(d).filter(([k]) => !sections.includes(k))); + for (const k of sections) { + expect(mapped.data[k].length).toBe(read.data[k].length); + expect(Buffer.compare(mapped.data[k], read.data[k])).toBe(0); + } + expect(rest(mapped.data)).toEqual(rest(read.data)); + + // and what the services make of them: counts, and every concept's descriptions and relationships + for (const what of ['concepts', 'descriptions', 'relationships', 'words', 'stems', 'refSetIndex']) { + expect(mapped.services[what].count()).toBe(read.services[what].count()); + } + const n = read.services.concepts.count(); + expect(n).toBeGreaterThan(1000); + let withRels = 0; + for (let i = 0; i < n; i++) { + const index = i * SnomedConceptList.CONCEPT_SIZE; + const a = describeConcept(read.services, index); + expect(describeConcept(mapped.services, index)).toEqual(a); + if (a.descriptions.length && a.outbound.length) withRels++; + } + expect(withRels).toBeGreaterThan(100); // the comparison covered real content, not empty concepts + }, 60000); + + test.each([ + ['mapping not available', () => _setMmapForTesting(null, 'simulated'), /memory mapping is not available here \(simulated\); reading .*snomed-testing.cache into memory/], + ['mmap() failing', () => _setMmapForTesting(FAILING_MMAP), /could not map .*snomed-testing.cache \(errno 12\); reading it into memory/] + ])('%s: falls back to reading, mapped is false, and the log says why', async (name, simulate, message) => { + simulate(); + const log = { warn: jest.fn() }; + const { reader, services } = await readCache('mmap', log); + expect(reader.mapped).toBe(false); + expect(services.concepts.count()).toBeGreaterThan(0); + expect(log.warn).toHaveBeenCalledTimes(1); + expect(log.warn).toHaveBeenCalledWith(expect.stringMatching(message)); + }); + + test('with no logger, the fallback warning goes to the console', async () => { + _setMmapForTesting(null, 'simulated'); + const spy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const { reader } = await readCache('mmap'); + expect(reader.mapped).toBe(false); + expect(spy).toHaveBeenCalledWith(expect.stringMatching(/memory mapping is not available here \(simulated\)/)); + } finally { + spy.mockRestore(); + } + }); +}); + +describe('Library: SNOMED CT load mode from config', () => { + const COMMITTED = path.resolve(__dirname, '../../tx/data/snomed-testing.cache'); + const SYSTEM = 'http://snomed.info/sct'; + let tmpDir; + let yamlPath; + + beforeAll(() => { + const target = path.join(folders.subDir('terminology-cache'), 'snomed-testing.cache'); + if (!fs.existsSync(target)) { + fs.copyFileSync(COMMITTED, target); + } + }); + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lib-sct-load-')); + yamlPath = path.join(tmpDir, 'library.yml'); + fs.writeFileSync(yamlPath, 'base:\n url: https://storage.googleapis.com/tx-fhir-org\nsources:\n - snomed:snomed-testing.cache\n'); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + memoryGuard._state.excludeFileBacked = false; + }); + + async function load(block) { + const log = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() }; + const library = new Library(yamlPath, undefined, log, { addStat: jest.fn() }, block); + await library.load(); + return { library, log }; + } + + const version = () => SnomedFileReader.versionUri(COMMITTED); + + test.each([ + ['no block', undefined, 'read'], + ['load mmap', { load: 'mmap' }, 'mmap'], + ['load mmap, file read', { load: 'mmap', read: ['snomed-testing.cache'] }, 'read'], + ['load read, version prewarmed', { prewarm: ['VERSION'] }, 'prewarm'] + ])('%s: loaded with mode %#', async (name, block, expected) => { + if (block && block.prewarm) block = { ...block, prewarm: [version()] }; + const { library, log } = await load(block); + const factory = library.codeSystemFactories.get(SYSTEM); + expect(factory.loadMode).toBe(expected); + expect(factory.mapped).toBe(expected !== 'read' && CAN_MAP); + expect(factory.snomedServices.concepts.count()).toBeGreaterThan(0); + expect(memoryGuard._state.excludeFileBacked).toBe(factory.mapped && process.platform === 'linux'); + if (expected !== 'read') { + expect(log.info).toHaveBeenCalledWith(expect.stringMatching( + CAN_MAP ? /snomed-testing.cache: load \w+, memory-mapped$/ : /snomed-testing.cache: load \w+, not mapped, read into memory$/)); + } + if (CAN_MAP) expect(log.warn).not.toHaveBeenCalled(); + }, 30000); + + test('load mmap where mapping fails: read instead, and the server log says so', async () => { + _setMmapForTesting(FAILING_MMAP); + try { + const { library, log } = await load({ load: 'mmap' }); + const factory = library.codeSystemFactories.get(SYSTEM); + expect(factory.loadMode).toBe('mmap'); + expect(factory.mapped).toBe(false); + expect(factory.snomedServices.concepts.count()).toBeGreaterThan(0); + expect(memoryGuard._state.excludeFileBacked).toBe(false); + expect(log.warn).toHaveBeenCalledWith(expect.stringMatching(/could not map .*snomed-testing.cache \(errno 12\); reading it into memory/)); + expect(log.info).toHaveBeenCalledWith(expect.stringMatching(/snomed-testing.cache: load mmap, not mapped, read into memory$/)); + } finally { + _setMmapForTesting(); + } + }, 30000); + + test('an entry that matches nothing is warned about', async () => { + const { log } = await load({ load: 'mmap', read: ['sct_us_20250901b.cache'] }); + expect(log.warn).toHaveBeenCalledWith(expect.stringMatching(/"sct_us_20250901b.cache" matches no SNOMED CT edition/)); + }, 30000); +}); diff --git a/tx/README.md b/tx/README.md index 493cba89..47b2de82 100644 --- a/tx/README.md +++ b/tx/README.md @@ -258,6 +258,52 @@ actually returns (metadata, read and search, `$expand`, `$validate-code`, `$look `$subsumes`, `$batch-validate-code`, `$versions`, `$cache-control`, an error) against the schemas, using ajv. So a new parameter needs adding to openapi-operations.js. +## Loading SNOMED CT + +By default each SNOMED CT cache file is read into the server process, at about its file size +(0.6–1.2 GB an edition). A file can instead be memory-mapped read-only: + +- the edition loads in milliseconds; +- only the pages that requests touch become resident, as page cache that the kernel can drop + and read again; +- every server process on the machine that maps the same file shares those pages. + +The first request to touch a part of a mapped edition waits for the disk, unless that edition +is *prewarmed*: mapped, then read through once in the background at startup, so its pages are +already in the page cache when requests arrive. Startup doesn't wait for it. + +This is set per server, in the tx module's `snomed` block in `config.json`: + +```json +"tx": { + "snomed": { + "load": "mmap", + "prewarm": ["http://snomed.info/sct/900000000000207008/version/20250201", + "sct_us_20250901b.cache"], + "read": [] + } +} +``` + +- **`load`** is `"read"` (the default) or `"mmap"`, for every edition not named below. +- **`prewarm`** lists editions to prewarm (map, then read through once in the background at + startup), even when `load` is `"read"`. +- **`read`** lists editions to read into the process even when `load` is `"mmap"`. +- An entry is any of: + - a cache file name (`sct_us_20250901b.cache`) + - an edition version URI (`http://snomed.info/sct/731000124108/version/20250901`) + - an edition URI (`http://snomed.info/sct/731000124108`), which covers every loaded version + of that edition +- `read` wins over `prewarm`. +- Entries that match no loaded edition are warned about at startup. +- With no `snomed` block, everything is read, as before. + +Mapping uses the optional `koffi` dependency. Where it is not available (Windows, 32-bit +platforms, or koffi not installed), a mapped edition is read instead, with a warning in the +server log. The log says, for each edition that was to be mapped, whether it really was. When +any edition is mapped, the memory guard leaves file-backed pages out of the memory it measures +(Linux). + ## Library Configuration The library - the source to load - is configured using a YAML file. @@ -408,6 +454,9 @@ Common edition identifiers: The file is built by importing SNOMED CT (see [documentation](importers/readme.md)) +How the cache files are loaded is a deployment choice, so it is set in `config.json` (the +tx module's `snomed` block), not here - see [Loading SNOMED CT](#loading-snomed-ct). + #### `cpt` - CPT (Current Procedural Terminology) diff --git a/tx/cs/cs-snomed.js b/tx/cs/cs-snomed.js index 858e0843..7a70ae91 100644 --- a/tx/cs/cs-snomed.js +++ b/tx/cs/cs-snomed.js @@ -4307,9 +4307,12 @@ class SnomedProvider extends BaseCSServices { * Factory for creating SNOMED services and providers */ class SnomedServicesFactory extends CodeSystemFactoryProvider { - constructor(i18n, filePath) { + constructor(i18n, filePath, loadMode = 'read', log = null) { super(i18n); this.filePath = filePath; + this.loadMode = loadMode; // 'read', 'mmap' or 'prewarm' - see SnomedFileReader + this.log = log; + this.mapped = false; // set by load(): whether the cache file really is memory-mapped this.uses = 0; this._loaded = false; this._sharedData = null; @@ -4564,8 +4567,9 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { } async load() { - const reader = new SnomedFileReader(this.filePath); + const reader = new SnomedFileReader(this.filePath, this.loadMode, this.log); this._sharedData = await reader.loadSnomedData(); + this.mapped = reader.mapped; this.snomedServices = new SnomedServices(this._sharedData); this._loaded = true; } diff --git a/tx/library.js b/tx/library.js index 1e769d40..1be32107 100644 --- a/tx/library.js +++ b/tx/library.js @@ -22,6 +22,8 @@ const {RxNormServicesFactory} = require("./cs/cs-rxnorm"); const {NdcServicesFactory} = require("./cs/cs-ndc"); const {UniiServicesFactory} = require("./cs/cs-unii"); const {SnomedServicesFactory} = require("./cs/cs-snomed"); +const {SnomedFileReader} = require("./sct/structures"); +const {SnomedLoadConfig} = require("./sct/load-config"); const {CPTServicesFactory} = require("./cs/cs-cpt"); const {OMOPServicesFactory} = require("./cs/cs-omop"); const {ICD11ServicesFactory} = require("./cs/cs-icd11"); @@ -34,6 +36,7 @@ const {ListCodeSystemProvider} = require("./cs/cs-provider-list"); const { Provider } = require("./provider"); const {I18nSupport} = require("../library/i18nsupport"); const folders = require('../library/folder-setup'); +const memoryGuard = require('../library/memory-guard'); const {VSACValueSetProvider} = require("./vs/vs-vsac"); const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory } = require('./ocl/cs-ocl'); const { OCLValueSetProvider } = require('./ocl/vs-ocl'); @@ -112,11 +115,20 @@ class Library { } } - constructor(configFile, vsacCfg, log, stats) { + /** + * @param {string} configFile - the library yml + * @param {object} vsacCfg + * @param {object} log + * @param {object} stats + * @param {object} [snomedCfg] - the tx module's "snomed" block: how SNOMED CT cache files are + * loaded (see tx/sct/load-config.js) + */ + constructor(configFile, vsacCfg, log, stats, snomedCfg = undefined) { this.configFile = configFile; this.vsacCfg = vsacCfg; this.log = log; this.stats = stats; + this.snomedLoad = new SnomedLoadConfig(snomedCfg, (msg) => (this.log ? this.log.warn(msg) : console.warn(msg))); // Only synchronous initialization here this.codeSystemFactories = new Map(); @@ -215,6 +227,7 @@ class Library { throw error; } } + this.snomedLoad.checkUnused(); this.log.info('Loading Packages'); this.#logPackagesHeader(); @@ -654,8 +667,18 @@ class Library { if (mode === "fetch" || mode === "npm") { return; } - const sct = new SnomedServicesFactory(this.i18n, sctFN); + const loadMode = this.snomedLoad.modeFor(sctFN, SnomedFileReader.versionUri(sctFN)); + const sct = new SnomedServicesFactory(this.i18n, sctFN, loadMode, this.log); await sct.load(); + if (loadMode !== 'read') { + // if it could not be mapped, mapFile has already said why, through this.log + this.log.info(`SNOMED CT ${path.basename(sctFN)}: load ${loadMode}, ` + + (sct.mapped ? 'memory-mapped' : 'not mapped, read into memory')); + } + if (sct.mapped && !memoryGuard._state.excludeFileBacked) { + memoryGuard.measureExcludingFileBacked(); + this.log.info(`SNOMED CT cache files are memory-mapped. ${memoryGuard.describe()}`); + } this.registerProvider(sctFN, sct, isDefault); this.#syndicateBinary('snomed', sctFN, sct); } diff --git a/tx/sct/load-config.js b/tx/sct/load-config.js new file mode 100644 index 00000000..73b23f22 --- /dev/null +++ b/tx/sct/load-config.js @@ -0,0 +1,107 @@ +'use strict'; + +/** + * How each SNOMED CT cache file is loaded, from the tx module's "snomed" block in config.json: + * + * "snomed": { + * "load": "read" | "mmap", // the default for every edition (absent: "read") + * "prewarm": [ ... ], // map these, then read them through once in the background at startup + * "read": [ ... ] // read these into the process even when load is "mmap" + * } + * + * An entry names a cache file (e.g. "sct_us_20250901b.cache"), an edition version URI + * ("http://snomed.info/sct/731000124108/version/20250901"), or an edition URI + * ("http://snomed.info/sct/731000124108", every loaded version of the edition). "read" wins + * over "prewarm", which wins over "load". + * + * read - the file is read into this process (private memory, as before) + * mmap - the file is mapped read-only; pages are read from disk when first touched, and + * shared with every other process that maps the same file + * prewarm - mapped, and read through once in the background at startup so its pages are + * already in the shared page cache when the first requests arrive + */ + +const path = require('path'); + +const MODES = ['read', 'mmap']; + +class SnomedLoadConfig { + /** + * @param {object} [block] - the "snomed" block of the tx module config + * @param {function(string)} [warn] + */ + constructor(block, warn = () => {}) { + this.warn = warn; + block = block || {}; + this.load = 'read'; + if (block.load !== undefined) { + if (MODES.includes(block.load)) { + this.load = block.load; + } else { + warn(`snomed.load must be "read" or "mmap", not ${JSON.stringify(block.load)}; using "read"`); + } + } + this.lists = {}; + for (const name of ['prewarm', 'read']) { + const v = block[name]; + if (v === undefined) { + this.lists[name] = []; + } else if (Array.isArray(v) && v.every(e => typeof e === 'string' && e.trim())) { + this.lists[name] = v.map(e => e.trim()); + } else { + warn(`snomed.${name} must be a list of cache file names or edition URIs; ignored`); + this.lists[name] = []; + } + } + for (const e of this.lists.prewarm) { + if (this.lists.read.includes(e)) { + warn(`snomed: ${e} is in both "prewarm" and "read"; it will be read`); + } + } + this.used = new Set(); + } + + /** True if any edition can end up mapped. */ + mapsAnything() { + return this.load === 'mmap' || this.lists.prewarm.length > 0; + } + + static #matches(entry, fileName, versionUri) { + if (entry === fileName) return true; + if (!versionUri) return false; + return entry === versionUri || versionUri.startsWith(entry.replace(/\/+$/, '') + '/version/'); + } + + #find(list, fileName, versionUri) { + const hit = this.lists[list].filter(e => SnomedLoadConfig.#matches(e, fileName, versionUri)); + hit.forEach(e => this.used.add(e)); + return hit.length > 0; + } + + /** + * @param {string} filePath - the cache file + * @param {string|null} versionUri - its edition version URI, e.g. http://snomed.info/sct/900000000000207008/version/20250201 + * @returns {'read'|'mmap'|'prewarm'} + */ + modeFor(filePath, versionUri) { + const fileName = path.basename(filePath); + const read = this.#find('read', fileName, versionUri); + const prewarm = this.#find('prewarm', fileName, versionUri); + if (read) return 'read'; + if (prewarm) return 'prewarm'; + return this.load; + } + + /** Warn about entries that matched no loaded edition. Call once every edition is loaded. */ + checkUnused() { + for (const list of ['prewarm', 'read']) { + for (const e of this.lists[list]) { + if (!this.used.has(e)) { + this.warn(`snomed.${list}: "${e}" matches no SNOMED CT edition in the library`); + } + } + } + } +} + +module.exports = { SnomedLoadConfig }; diff --git a/tx/sct/mmap.js b/tx/sct/mmap.js new file mode 100644 index 00000000..4419c79a --- /dev/null +++ b/tx/sct/mmap.js @@ -0,0 +1,109 @@ +'use strict'; + +/** + * Read-only memory mapping of SNOMED CT cache files ("snomed": {"load": "mmap"} in config.json). + * + * Node has no mmap of its own, so this calls the C library's mmap() through koffi (an FFI + * module that ships prebuilt binaries for Linux glibc and musl, macOS and the BSDs, so + * nothing is compiled at install time). The mapping is never unmapped: a loaded edition + * stays loaded for the life of the process. + * + * mapFile() returns null where mapping is not available (Windows, koffi missing, mmap + * failing), and the caller then reads the file into memory as before. It says why through the + * warn function it is given (the server log), or console.warn where there is none. + */ + +const fs = require('fs'); + +const PROT_READ = 1; +const MAP_PRIVATE = 2; // same value on Linux, macOS and the BSDs +const MAP_FAILED = 0xFFFFFFFFFFFFFFFFn; + +let libcMmap; // undefined: not tried yet; null: not available +let unavailable = null; // why libcMmap is null + +function getMmap() { + if (libcMmap === undefined) { + libcMmap = null; + if (process.platform === 'win32' || !process.arch.endsWith('64')) { + unavailable = `not supported on ${process.platform}/${process.arch}`; + } else { + try { + const koffi = require('koffi'); + const mmap = koffi.load(null).func('void *mmap(void *addr, size_t len, int prot, int flags, int fd, int64_t off)'); + libcMmap = { koffi, mmap }; + } catch (e) { + unavailable = e.message; + } + } + } + return libcMmap; +} + +/** True if mapFile() can map files on this platform (it may still fail for a given file). */ +function mappingAvailable() { + return getMmap() !== null; +} + +/** + * @param {string} filePath + * @param {function(string)} [warn] - where to say why the file was not mapped + * @returns {Buffer|null} a read-only Buffer over the whole file, or null + */ +function mapFile(filePath, warn = (msg) => console.warn(msg)) { + const lib = getMmap(); + if (!lib) { + warn(`SNOMED CT: memory mapping is not available here (${unavailable}); reading ${filePath} into memory`); + return null; + } + const fd = fs.openSync(filePath, 'r'); + try { + const size = fs.fstatSync(fd).size; + if (size === 0) return null; + const ptr = lib.mmap(null, size, PROT_READ, MAP_PRIVATE, fd, 0); + if (ptr === null || lib.koffi.address(ptr) === MAP_FAILED) { + warn(`SNOMED CT: could not map ${filePath} (errno ${lib.koffi.errno()}); reading it into memory`); + return null; + } + return Buffer.from(lib.koffi.view(ptr, size)); + } finally { + fs.closeSync(fd); // the mapping keeps its own reference to the file + } +} + +/** + * Read a file through once, in the background, so its pages are in the page cache before + * requests need them (madvise(MADV_WILLNEED) is only a hint, and on Linux it read almost + * nothing of a mapped file). The reads go through one small buffer, so this costs no process + * memory; the pages land in the page cache that every mapping of the file shares. + * + * @param {string} filePath + * @returns {Promise} milliseconds taken + */ +async function prewarmFile(filePath) { + const start = Date.now(); + const buf = Buffer.allocUnsafe(4 * 1024 * 1024); + const fh = await fs.promises.open(filePath, 'r'); + try { + let pos = 0; + for (;;) { + const { bytesRead } = await fh.read(buf, 0, buf.length, pos); + if (bytesRead === 0) break; + pos += bytesRead; + } + } finally { + await fh.close(); + } + return Date.now() - start; +} + +/** + * For tests: replace the mmap binding ({koffi, mmap}, or null with a reason for "not + * available"), or with no arguments go back to the real one. + */ +function _setMmapForTesting(lib, reason = 'disabled by a test') { + libcMmap = arguments.length === 0 ? undefined : lib; + unavailable = lib ? null : reason; +} + +module.exports = { mapFile, prewarmFile, mappingAvailable, _setMmapForTesting }; diff --git a/tx/sct/structures.js b/tx/sct/structures.js index 89bddf67..256f941d 100644 --- a/tx/sct/structures.js +++ b/tx/sct/structures.js @@ -1,9 +1,9 @@ const fs = require('fs'); +const { mapFile, prewarmFile } = require('./mmap'); // The ArrayBuffers of cache files loaded by SnomedFileReader. A structure uses its section of -// one in place rather than copying it: nothing writes to a loaded structure, and the reader's -// sections (kept in the factory's shared data) would hold the whole file anyway. Any other -// buffer is copied, so the structure owns its bytes. +// one in place (nothing writes to a loaded structure, and the file stays loaded for the life +// of the process); any other buffer is copied, so the structure owns its bytes. const fileBuffers = new WeakSet(); function adopt(buffer) { @@ -1342,16 +1342,58 @@ class SnomedReferenceSetIndex { class SnomedFileReader { - constructor(filePath) { + /** + * @param {string} filePath + * @param {string} [loadMode] 'read' (default): read the file into memory. 'mmap': map it + * read-only, so its pages are file-backed and shared with the page cache. 'prewarm': mapped, + * then read through once in the background at startup, so its pages are already in the + * page cache when requests arrive; startup doesn't wait for it. Where mapping is not + * available (see mmap.js) the file is read, and `mapped` stays false. + * @param {object} [log] - a logger with warn(); says why a file that was to be mapped was read + */ + constructor(filePath, loadMode = 'read', log = null) { this.filePath = filePath; + this.loadMode = loadMode; + this.log = log; this.buffer = null; + this.mapped = false; // true once load() has the buffer as a memory mapping of the file this.offset = 0; } + /** + * The edition version URI in a cache file's header, without loading the file. + * @param {string} filePath + * @returns {string|null} + */ + static versionUri(filePath) { + const head = Buffer.alloc(1024); + const fd = fs.openSync(filePath, 'r'); + let n; + try { + n = fs.readSync(fd, head, 0, head.length, 0); + } finally { + fs.closeSync(fd); + } + const reader = new SnomedFileReader(filePath); + reader.buffer = head.subarray(0, n); + try { + reader.readString(); // cache version + return reader.readString(); + } catch { + return null; + } + } + async load() { - // Read entire file into buffer - this.buffer = await fs.promises.readFile(this.filePath); + const warn = this.log ? (msg) => this.log.warn(msg) : undefined; + const mapped = this.loadMode !== 'read' ? mapFile(this.filePath, warn) : null; + this.mapped = mapped !== null; + this.buffer = mapped || await fs.promises.readFile(this.filePath); fileBuffers.add(this.buffer.buffer); + if (mapped && this.loadMode === 'prewarm') { + // not awaited: the server starts while the file is read into the page cache + this.prewarmed = prewarmFile(this.filePath).catch(() => null); + } this.offset = 0; } diff --git a/tx/tx.js b/tx/tx.js index 09d5ec42..a63f7e20 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -243,7 +243,7 @@ class TXModule { // Load the library from YAML this.log.info(`Loading library from: ${config.librarySource}`); - this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats); + this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats, config.snomed); // The syndication feed: content is recorded in the store as the library loads it, from // the time the administrator turns it on (see tx/syndication). One store serves every