diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f57d4e..3e0f78bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Extension-codes supplements: a code in a code space the code system reserves for users (`codesystem-reservedCodeSpace`, a regular expression) is a valid code, whether or not anything defines it; with nothing defining it, its display is the code space's `default-display`, or 'User-assigned', and (as a placeholder) another display only gets an information issue. An `extension-codes` supplement in scope defines such codes, with their displays: a global one as if the code system defined them (a different display is an error, or a warning with lenient display validation), one that is referenced as a display that isn't the only correct one (a different display is a warning). Codes from supplements are only in an expansion when the value set lists them. ISO 3166's user-assigned codes work the same way: an extension-codes supplement's display replaces the 'User-assigned' placeholder. So do ICF's qualified (post-coordinated) codes and SNOMED CT post-coordinated expressions (matched as written, or in their minimal rendering): an extension-codes supplement's display is the code's display, and the generated one stays valid +- Reserved code spaces: an expansion of all of a code system that reserves code spaces (`codesystem-reservedCodeSpace`) is marked unclosed (`valueset-unclosed`, with the reason), since `$validate-code` accepts codes in those spaces that the expansion can't list +- `$cache-control` mode=start also returns the issued cache-id in the `X-Cache-Id` response header (the body is unchanged), so a request log taken at the proxy (nginx `$sent_http_x_cache_id`) can tie each cache-id to the request that created it, for replay - IPS: a Structure view (after Narrative) showing a document's logical structure as an expandable tree: what the Composition refers to, then its sections, each with its entries and what else it refers to, and each resource referred to expanding into its own references. A reference that doesn't resolve in the document is `not-in-document`; a resource already shown isn't expanded again ### Changed diff --git a/Dockerfile b/Dockerfile index 82a25c2a..f8d33ca9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,4 +44,5 @@ LABEL org.opencontainers.image.title="FHIRsmith" \ # Expose port and define command EXPOSE 3000 -CMD ["node", "server.js"] \ No newline at end of file +# --expose-gc: the memory guard forces a GC while it refuses requests (library/memory-guard.js) +CMD ["node", "--expose-gc", "server.js"] \ No newline at end of file diff --git a/ips/analysis.js b/ips/analysis.js index d8cd082c..8362dd5c 100644 --- a/ips/analysis.js +++ b/ips/analysis.js @@ -25,13 +25,15 @@ * modifier a modifier extension, or implicitRules (maroon) * status inactive, entered in error, unknown, or missing status (navy) * subject a subject other than the document's, or a focus (dark green) - * attention a future date, a Patient's link, a Composition that isn't final + * attention a date in the future where one isn't expected, a Patient's link, a + * Composition that isn't final * note anything else (a section's emptyReason, a section's author) * * @module ips/analysis */ const { structureTree, flatten, isObject, text } = require('./structure'); +const { typeOf } = require('./terminology'); // the resources whose status (or lifecycleStatus) is required, as the element that holds it const STATUS_ELEMENTS = { @@ -45,11 +47,59 @@ const STATUS_ELEMENTS = { // extensions not worth mentioning const IGNORED_EXTENSIONS = new Set(['http://hl7.org/fhir/StructureDefinition/textLink']); -// resources where a date in the future is to be expected -const FUTURE_OK = new Set(['Appointment', 'AppointmentResponse']); +// ---- dates in the future ---- +// +// Only elements the (R4) type model says are dates (date, dateTime, instant) are looked at, +// so a string or an identifier that happens to look like a date isn't one. Some dates are +// about the future by their nature, wherever they are; some resources are about the future +// - always, or while their status says the thing hasn't happened yet. + +// the statuses of a request (or plan) that is over: its dates should be past +const CLOSED = new Set(['completed', 'revoked', 'cancelled', 'entered-in-error', 'stopped', 'failed', 'rejected']); +const open = (r) => !CLOSED.has(r.status); + +// resources whose dates may be in the future, by type: a function of the resource that says +// whether they may (for that resource as it is) +const FUTURE_INTENT = { + // about the future, whatever their status + Appointment: () => true, + AppointmentResponse: () => true, + ImmunizationRecommendation: () => true, + Schedule: () => true, + Slot: () => true, + // requests and plans, until they are over + ServiceRequest: open, + MedicationRequest: open, + DeviceRequest: open, + NutritionOrder: open, + CommunicationRequest: open, + SupplyRequest: open, + VisionPrescription: open, + CarePlan: open, + Task: open, + Goal: (r) => !CLOSED.has(r.lifecycleStatus), + // events, while they are still to come + Procedure: (r) => r.status === 'preparation', + Encounter: (r) => r.status === 'planned', + MedicationStatement: (r) => r.status === 'intended', + MedicationUsage: (r) => r.status === 'intended', + DeviceUseStatement: (r) => r.status === 'intended', + DeviceUsage: (r) => r.status === 'intended' +}; + +/** Whether a resource's dates may be in the future. */ +function mayBeFuture(r) { + const f = FUTURE_INTENT[r.resourceType]; + return !!f && f(r); +} + +// dates that are about the future by their nature, wherever they are: the end of a period +// (a name, an address, an identifier valid until; a prescription's validity), an expiry, +// a due date. By element name, or by the type they are in (a Timing is a schedule) +const FUTURE_ELEMENTS = new Set(['end', 'expirationDate', 'dueDate']); +const FUTURE_TYPES = new Set(['Timing']); +const DATE_TYPES = new Set(['date', 'dateTime', 'instant']); -// keys whose values are dates even when they are just a year -const DATE_KEY = /date|time|instant|period|start|end|issued|recorded|onset|abatement|occurrence|effective|authored|when|birth|deceased|lastUpdated|created/i; const DATE_VALUE = /^(\d{4})(?:-(\d\d)(?:-(\d\d)(?:T(\d\d):(\d\d)(?::(\d\d)(?:\.\d+)?)?(Z|[+-]\d\d:\d\d)?)?)?)?$/; /** The earliest moment a date/dateTime/instant can mean, as ms, or null if it isn't one. */ @@ -107,17 +157,49 @@ function extensionsIn(node, prefix, skip) { return out; } -/** The dates after now in a resource or section, as [path, value]. */ -function futureDates(node, prefix, skip, now) { +/** + * The dates after now in a resource or section, as [path, value]: the elements whose type is + * date, dateTime or instant, walking the resource with the type model - but not those about + * the future by their nature (FUTURE_ELEMENTS, FUTURE_TYPES), nor inside extensions (what + * their dates mean isn't known), nor elements the model doesn't know. + * @param {Object} node - the resource, or a section + * @param {string} modelPath - the model's path for node: the resource type, or Composition.section + * @param {string} prefix - the path shown for node + * @param {string[]} skip - keys of node not to look in (a Composition's sections) + * @param {number} now - in ms + */ +function futureDates(node, modelPath, prefix, skip, now) { const out = []; - walk(node, prefix, skip, (k, v, at, path) => { - if (typeof v === 'string' && k !== 'div' && (v.includes('-') || DATE_KEY.test(k))) { - const t = earliest(v); - if (t !== null && t > now) { - out.push([path, v]); + const go = (obj, model, shown, top) => { + for (const [k, v] of Object.entries(obj)) { + if (v === null || k === 'resourceType' || k === 'contained' || k === 'extension' || k === 'modifierExtension' || + k.startsWith('_') || (top && skip.includes(k)) || FUTURE_ELEMENTS.has(k)) { + continue; + } + const resolved = typeOf(`${model}.${k}`); + if (!resolved || FUTURE_TYPES.has(resolved.type)) { + continue; } + const items = Array.isArray(v) ? v : [v]; + items.forEach((item, i) => { + const path = `${shown}.${k}${Array.isArray(v) ? `[${i}]` : ''}`; + if (DATE_TYPES.has(resolved.type)) { + const t = typeof item === 'string' ? earliest(item) : null; + if (t !== null && t > now) { + out.push([path, item]); + } + } else if (isObject(item)) { + if (resolved.type === 'BackboneElement' || resolved.type === 'Element') { + go(item, resolved.path, path, false); + } else if (/^[A-Z]/.test(resolved.type) && !['Resource', 'Narrative', 'Extension'].includes(resolved.type)) { + // a datatype: its own elements are defined on the type + go(item, resolved.type, path, false); + } + } + }); } - }); + }; + go(node, modelPath, prefix, true); return out; } @@ -163,7 +245,7 @@ function sameReference(a, fromA, b, fromB, resolve) { } /** The findings common to resources and sections: extensions, narrative, focus, future dates. */ -async function common(findings, thing, prefix, skip, options, cache) { +async function common(findings, thing, modelPath, prefix, skip, options, cache) { for (const [kind, url, path] of extensionsIn(thing, prefix, skip)) { if (!cache.has(url)) { let def = null; @@ -183,8 +265,8 @@ async function common(findings, thing, prefix, skip, options, cache) { if (focus.length) { findings.push({ kind: 'subject', text: `Has a focus: ${focus.map(referenceWords).join(', ')}` }); } - if (!FUTURE_OK.has(thing.resourceType)) { - for (const [path, v] of futureDates(thing, prefix, skip, options.now)) { + if (!mayBeFuture(thing)) { + for (const [path, v] of futureDates(thing, modelPath, prefix, skip, options.now)) { findings.push({ kind: 'attention', text: `A date in the future: ${path} = ${v}` }); } } @@ -239,7 +321,7 @@ async function analyseResource(node, ctx) { findings.push({ kind: 'attention', text: `Linked${text(link.type) ? ` (${link.type})` : ''} to ${referenceWords(link.other)}${target ? '' : ' - not in the document'}` }); } } - await common(findings, r, prefix, skip, ctx.options, ctx.cache); + await common(findings, r, type || 'Resource', prefix, skip, ctx.options, ctx.cache); return finish(findings, narrativeOf(r)); } @@ -259,7 +341,7 @@ async function analyseSection(node, ctx) { (parent.length ? ` (${parent.map(referenceWords).join(', ')})` : ' (none)') }); } } - await common(findings, sec, 'section', ['section'], ctx.options, ctx.cache); + await common(findings, sec, 'Composition.section', 'section', ['section'], ctx.options, ctx.cache); return finish(findings, narrativeOf(sec)); } @@ -328,4 +410,4 @@ const analysisStep = { } }; -module.exports = { analyseDocument, analysisStep, extensionsIn, futureDates, earliest, STATUS_ELEMENTS }; +module.exports = { analyseDocument, analysisStep, extensionsIn, futureDates, mayBeFuture, earliest, STATUS_ELEMENTS, FUTURE_INTENT }; diff --git a/ips/readme.md b/ips/readme.md index 1d95a818..41430224 100644 --- a/ips/readme.md +++ b/ips/readme.md @@ -214,8 +214,18 @@ Composition itself without its sections), keeps a list of findings, by the node' status where the resource requires one (`lifecycleStatus` for a Goal); - in dark green: a `subject` (or `patient`) that isn't the document's - not the same resource in the document, nor the same reference, nor the same identifier - and any `focus`; -- dates in the future (except in an Appointment or AppointmentResponse): a date with no time - zone is in the future only if it is everywhere; +- dates in the future, where one isn't expected. Only elements the (R4) type model says are + dates (date, dateTime, instant) are looked at - so not an identifier or a note that looks + like a date - and not inside extensions. Some dates are about the future by their nature and + are never mentioned: the end of a period (a name's, address's or identifier's validity, a + prescription's), expiry dates, due dates, and schedules (Timing). Some resources may have + future dates: Appointment, AppointmentResponse, ImmunizationRecommendation, Schedule and + Slot always; requests and plans (ServiceRequest, MedicationRequest, DeviceRequest, + NutritionOrder, CommunicationRequest, SupplyRequest, VisionPrescription, CarePlan, Task, and + a Goal by its lifecycleStatus) until they are over (completed, revoked, cancelled, + entered-in-error, stopped, failed or rejected); and events still to come - a Procedure in + preparation, a planned Encounter, an intended MedicationStatement or DeviceUseStatement. A + date with no time zone is in the future only if it is everywhere; - a Patient's `link`s; a Composition whose status isn't `final`; - for a section: its `emptyReason`, and an author other than that of what contains it (the Composition, or the containing section - which may have inherited its own); diff --git a/ips/render.js b/ips/render.js index 2f4eb9a4..ad00cd89 100644 --- a/ips/render.js +++ b/ips/render.js @@ -880,7 +880,7 @@ function renderAnalysis(row, options = {}) { 'extensions (with their definitions, from the validator, where it knows them; textLink is left out), modifier extensions and implicitRules, ' + 'records that are inactive, entered in error, of unknown status, or missing a status they should have, ' + 'a subject that is not the document\'s, or a focus, ' + - 'dates in the future (except in Appointments), a Patient\'s links, and a Composition that is not final, ' + + 'dates in the future where none is expected (not period ends, expiry or due dates, schedules, nor appointments, requests and plans that are not over, or events still to come), a Patient\'s links, and a Composition that is not final, ' + 'a section\'s emptyReason or an author other than its container\'s, and whether there is narrative. A resource with nothing else to say is All OK. References to resources that are not in the document are left out.

'; return h; } diff --git a/ips/terminology.js b/ips/terminology.js index 73ea04bc..b96e83d6 100644 --- a/ips/terminology.js +++ b/ips/terminology.js @@ -391,5 +391,5 @@ function terminologyOf(bundle) { module.exports = { terminologyOf, conceptsOf, systemName, describeVersion, sctEdition, linkForSystem, linkForCode, isExternalSystem, isIgnored, - sectionRank, SECTION_ORDER, IGNORED, SYSTEM_NAMES + sectionRank, SECTION_ORDER, IGNORED, SYSTEM_NAMES, typeOf }; diff --git a/library/memory-guard.js b/library/memory-guard.js index 8623207d..232f488e 100644 --- a/library/memory-guard.js +++ b/library/memory-guard.js @@ -17,12 +17,22 @@ */ const os = require('os'); +const v8 = require('v8'); +const vm = require('vm'); const { readCgroupMemoryLimit } = require('./cgroup-memory'); const DEFAULT_REJECT_FRACTION = 0.85; const DEFAULT_ABORT_FRACTION = 0.92; // RSS is cheap to read, but the gate runs on every request; a sample this old is fine const SAMPLE_MS = 250; +// While refusing, force a full GC at most this often. Refusing requests removes the +// allocation activity that would otherwise trigger one, so without this RSS might never +// fall back below the threshold and the server would stay shut. +const GC_INTERVAL_MS = 5000; +// A forced GC that leaves the server still over the threshold doubles the wait before the +// next, up to this: when most of RSS is outside the JS heap (the SNOMED and LOINC cache +// buffers), collecting frees little, and every full GC stops the process +const GC_MAX_INTERVAL_MS = 60000; const state = { limit: 0, @@ -79,6 +89,35 @@ function shouldReject() { return state.rejectAt > 0 && rss() > state.rejectAt; } +let gcFn; + +/** + * global.gc if node was started with --expose-gc, which is how it should be started (the + * Dockerfile and npm start do; add it to the node command of a systemd unit). Otherwise + * expose it at runtime: the flag can be set late, and a fresh context then has gc - this + * works, but Node does not promise it will keep working. null if neither works. + */ +function getGc() { + if (gcFn !== undefined) { + return gcFn; + } + gcFn = null; + if (typeof global.gc === 'function') { + gcFn = global.gc; + } else { + try { + v8.setFlagsFromString('--expose-gc'); + const fn = vm.runInNewContext('gc'); + if (typeof fn === 'function') { + gcFn = fn; + } + } catch { + // no gc available - the guard still works, it just recovers more slowly + } + } + return gcFn; +} + let aborts = 0; let lastAbortLog = 0; let abortLog = null; @@ -125,23 +164,86 @@ function describe() { /** * Express middleware: 503 + Retry-After while RSS is over the reject threshold. Install it - * before the body parsers so a refused request costs nothing. - * @param {object} [opts] - { log, exempt: [path prefixes never refused], retryAfter (seconds) } + * before the body parsers so a refused request costs nothing. While refusing, it forces a full + * GC now and then and re-measures - every gcInterval ms, backing off to gcMaxInterval while + * the GCs don't bring it back under, and not at all while the JS heap is too small for one to + * help. It logs when refusing starts (then once a minute, with how long and how many) and + * when it stops. + * @param {object} [opts] - { log, exempt: [path prefixes never refused], retryAfter (seconds), + * gcInterval (ms), gcMaxInterval (ms), heapUsed (function returning bytes, for tests) } */ function middleware(opts = {}) { const exempt = opts.exempt || []; const retryAfter = String(opts.retryAfter || 30); const log = opts.log; - let refused = 0; + const gcInterval = opts.gcInterval || GC_INTERVAL_MS; + const gcMaxInterval = Math.max(opts.gcMaxInterval || GC_MAX_INTERVAL_MS, gcInterval); + const heapUsed = opts.heapUsed || (() => process.memoryUsage().heapUsed); + let gcWait = gcInterval; // the current wait between forced GCs (backs off, see below) + let refusingSince = 0; // when the current spell of refusing began (0 = not refusing) + let refused = 0; // refused in the current spell let lastLog = 0; + let lastGc = 0; + + const secs = (ms) => Math.round(ms / 1000); + return (req, res, next) => { - if (!shouldReject() || exempt.some(p => req.path === p || req.path.startsWith(p + '/'))) { + if (exempt.some(p => req.path === p || req.path.startsWith(p + '/'))) { return next(); } - refused++; + let over = shouldReject(); const now = Date.now(); + + // Refusing starves the GC of the allocation that would normally trigger it, so while + // refusing, force a full collection now and then and re-measure. Only when it could help: + // a GC frees heap and nothing else, so if the whole JS heap is smaller than the amount + // over the threshold, collecting can't get the server back under it. And a GC that + // leaves the server still over doubles the wait before the next one + if (over && now - lastGc >= gcWait) { + lastGc = now; + const before = state.sampleRss; + const heap = heapUsed(); + const gc = getGc(); + if (heap < before - state.rejectAt) { + gcWait = Math.min(gcWait * 2, gcMaxInterval); + if (log) { + log.info(`Memory: no forced GC - the JS heap (${gb(heap)} GB) is less than the ${gb(before - state.rejectAt)} GB over the threshold; next look in ${secs(gcWait)}s`); + } + } else if (gc) { + try { + gc(); + } catch { + // ignore + } + const after = rss(true); + over = after > state.rejectAt; + gcWait = over ? Math.min(gcWait * 2, gcMaxInterval) : gcInterval; + if (log) { + log.info(`Memory: forced GC while refusing - ${gb(before)} GB -> ${gb(after)} GB` + + (over ? `; next in ${secs(gcWait)}s` : '')); + } + } + } + + if (!over) { + gcWait = gcInterval; + if (refusingSince) { + if (log) { + log.warn(`Memory recovered: ${gb(state.sampleRss)} GB of ${gb(state.limit)} GB - accepting requests again after ${secs(now - refusingSince)}s (${refused} refused)`); + } + refusingSince = 0; + refused = 0; + } + return next(); + } + + if (!refusingSince) { + refusingSince = now; + lastLog = 0; + } + refused++; if (log && now - lastLog > 60000) { - log.warn(`Memory high: ${gb(state.sampleRss)} GB of ${gb(state.limit)} GB - refusing new requests with 503 (${refused} refused so far)`); + log.warn(`Memory high: ${gb(state.sampleRss)} GB of ${gb(state.limit)} GB - refusing new requests with 503 for ${secs(now - refusingSince)}s (${refused} refused)`); lastLog = now; } const text = `Server is under memory pressure (${gb(state.sampleRss)} GB of ${gb(state.limit)} GB); try again shortly`; @@ -158,4 +260,4 @@ function middleware(opts = {}) { configure(); -module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, _state: state }; +module.exports = { configure, shouldReject, overAbortThreshold, describe, middleware, gb, getGc, _state: state }; diff --git a/package-lock.json b/package-lock.json index f796ffed..9ff86470 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "express-rate-limit": "^7.4.1", "express-session": "^1.19.0", "fast-xml-parser": "^5.3.4", - "fhir-validator-wrapper": "1.6.2", + "fhir-validator-wrapper": "1.6.3", "fhirpath": "^5.2.1", "fs-extra": "^11.3.3", "ini": "^6.0.0", @@ -4767,9 +4767,9 @@ "license": "MIT" }, "node_modules/fhir-validator-wrapper": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/fhir-validator-wrapper/-/fhir-validator-wrapper-1.6.2.tgz", - "integrity": "sha512-Ws7Duj/aW3WUY0Rr/Oli01QkUfUwH56x6qHZkvC0vcmBYBYZTGcenRYk88fkM2EjBqf992t95VaKaOw7M/qp9w==", + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/fhir-validator-wrapper/-/fhir-validator-wrapper-1.6.3.tgz", + "integrity": "sha512-bcHr79Zw0X5l/6RIj4yWlOuiwjt+kvDLHxiMpeQW9b1N8K1xiDQ+srIzgeggPNS0EzwbB1zogMfgDeZi7WCsvQ==", "license": "Apache-2.0", "engines": { "node": ">=12.0.0" diff --git a/package.json b/package.json index b028e2da..d306d5d9 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,8 @@ "node": ">=24" }, "scripts": { - "start": "node server.js", - "dev": "node --watch server.js", + "start": "node --expose-gc server.js", + "dev": "node --expose-gc --watch server.js", "test": "jest", "test:watch": "jest --watch", "test:coverage": "jest --coverage", @@ -48,7 +48,7 @@ "express-rate-limit": "^7.4.1", "express-session": "^1.19.0", "fast-xml-parser": "^5.3.4", - "fhir-validator-wrapper": "1.6.2", + "fhir-validator-wrapper": "1.6.3", "fhirpath": "^5.2.1", "fs-extra": "^11.3.3", "ini": "^6.0.0", diff --git a/packages/package-crawler.js b/packages/package-crawler.js index c905d0e4..85e01c74 100644 --- a/packages/package-crawler.js +++ b/packages/package-crawler.js @@ -72,6 +72,11 @@ function isLoopbackAddress(ip) { } } +// A mirror file is written as .tgz.tmp- and then renamed into place +const MIRROR_TMP_MARKER = '.tmp-'; +// A temporary mirror file older than this was left by a crawler that stopped mid-write +const MIRROR_TMP_MAX_AGE_MS = 60 * 60 * 1000; + class PackageCrawler { log; packages = new Set(); @@ -122,6 +127,11 @@ class PackageCrawler { }; this.log.info('Running web crawler for packages using master URL: '+ this.config.masterUrl); + try { + this.cleanMirrorTempFiles(); + } catch (e) { + this.log.warn(`Could not clean the package mirror: ${e.message}`); + } this.stats.task('Package Crawler', 'Running'); try { @@ -786,6 +796,55 @@ class PackageCrawler { }); } + // Runs fn inside a database transaction: all of its writes commit, or none do. The crawler + // stores one package at a time, so its transactions never nest. The connection is shared + // with the web side, so a write it makes meanwhile (a download count) joins the transaction. + async transaction(fn) { + await this.run('BEGIN IMMEDIATE'); + try { + const result = await fn(); + await this.run('COMMIT'); + return result; + } catch (e) { + try { + await this.run('ROLLBACK'); + } catch (rollbackError) { + this.log?.error?.(`Rollback failed: ${rollbackError.message}`); + } + throw e; + } + } + + // Removes mirror files that a crawler stopped before renaming into place. Only files old + // enough that nothing can still be writing them: the mirror may be shared, and in a + // container the pid in the name is no guide. + cleanMirrorTempFiles(maxAgeMs = MIRROR_TMP_MAX_AGE_MS) { + if (!this.config.mirrorPath || !fs.existsSync(this.config.mirrorPath)) { + return 0; + } + const cutoff = Date.now() - maxAgeMs; + let removed = 0; + for (const name of fs.readdirSync(this.config.mirrorPath)) { + if (!name.includes(MIRROR_TMP_MARKER)) { + continue; + } + const file = path.join(this.config.mirrorPath, name); + try { + const stat = fs.statSync(file); + if (stat.isFile() && stat.mtimeMs < cutoff) { + fs.rmSync(file, { force: true }); + removed++; + } + } catch (e) { + this.log?.warn?.(`Could not remove stale mirror file ${name}: ${e.message}`); + } + } + if (removed > 0) { + this.log?.info?.(`Removed ${removed} stale temporary file(s) from the package mirror`); + } + return removed; + } + // Point Packages.CurrentVersion at the most recent version of id async refreshCurrentVersion(id) { const latest = await new Promise((resolve, reject) => { @@ -905,7 +964,8 @@ class PackageCrawler { // The mirror file has to be complete before the registry advertises the package (the // download redirect points at it), and a replacement must not touch the file served for // the copy we hold unless it commits. So: write the new file beside its target first; - // a new package's file is put in place before the commit, a replacement's after it. + // a new package's file is put in place before the transaction, a replacement's at the + // end of it, just before COMMIT, so a failed rename rolls the replacement back. // Rename within a directory is atomic, so readers see the old file or the whole new one. let filepath = null; let tmpPath = null; @@ -916,20 +976,24 @@ class PackageCrawler { if (path.dirname(path.resolve(filepath)) !== path.resolve(this.config.mirrorPath)) { throw new Error(`Package ${id}#${version} rejected: its mirror file name is not a plain file name`); } - tmpPath = `${filepath}.tmp-${process.pid}`; - fs.writeFileSync(tmpPath, packageBuffer); + tmpPath = `${filepath}${MIRROR_TMP_MARKER}${process.pid}`; } const replacing = check.action === 'replace'; let placed = false; try { - if (tmpPath && !replacing) { - fs.renameSync(tmpPath, filepath); - placed = true; - } - await this.commit(packageBuffer, npmPackage, date, guid, id, version, canonical, urls, check.keys); - if (tmpPath && replacing) { - fs.renameSync(tmpPath, filepath); + if (tmpPath) { + fs.writeFileSync(tmpPath, packageBuffer); + if (!replacing) { + fs.renameSync(tmpPath, filepath); + placed = true; + } } + await this.transaction(async () => { + await this.commit(packageBuffer, npmPackage, date, guid, id, version, canonical, urls, check.keys); + if (tmpPath && replacing) { + fs.renameSync(tmpPath, filepath); + } + }); } catch (e) { if (tmpPath) { fs.rmSync(tmpPath, { force: true }); diff --git a/tests/cs/cs-cs-property-hierarchy.test.js b/tests/cs/cs-cs-property-hierarchy.test.js index 94fe2e4d..7c6eaaab 100644 --- a/tests/cs/cs-cs-property-hierarchy.test.js +++ b/tests/cs/cs-cs-property-hierarchy.test.js @@ -248,6 +248,11 @@ describe('subsumption over a property-stated hierarchy', () => { expect(await filterCodes(withUri(), 'is-not-a', 'leaf')).toEqual(['mid', 'other', 'root', 'shared']); }); + test('descendent-leaf takes the descendants that have no children', async () => { + expect(await filterCodes(withUri(), 'descendent-leaf', 'root')).toEqual(['leaf', 'shared']); + expect(await filterCodes(withUri(), 'descendent-leaf', 'leaf')).toEqual([]); + }); + test('child-of takes only the direct children', async () => { expect(await filterCodes(withUri(), 'child-of', 'root')).toEqual(['mid', 'shared']); }); diff --git a/tests/cs/cs-cs.test.js b/tests/cs/cs-cs.test.js index 1f4b3747..a3380c22 100644 --- a/tests/cs/cs-cs.test.js +++ b/tests/cs/cs-cs.test.js @@ -1425,6 +1425,7 @@ describe('FHIR CodeSystem Provider', () => { expect(await simpleProvider.doesFilter('concept', 'is-a', 'code1')).toBe(true); expect(await simpleProvider.doesFilter('code', 'descendent-of', 'code2')).toBe(true); expect(await simpleProvider.doesFilter('concept', 'is-not-a', 'code1')).toBe(true); + expect(await simpleProvider.doesFilter('concept', 'descendent-leaf', 'code2')).toBe(true); expect(await simpleProvider.doesFilter('concept', 'child-of', 'code2')).toBe(true); expect(await simpleProvider.doesFilter('code', 'in', 'code1,code2')).toBe(true); expect(await simpleProvider.doesFilter('code', '=', 'code1')).toBe(true); @@ -1597,6 +1598,23 @@ describe('FHIR CodeSystem Provider', () => { expect(results.findConceptByCode('code2b')).toBeDefined(); }); + test('should filter by descendent-leaf relationship', async () => { + // R5: the descendants with no children - not code2 itself, and not code2a, which has children + const results = await simpleProvider.filter(filterContext, true, 'code', 'descendent-leaf', 'code2'); + expect(results.size()).toBe(3); + + expect(results.findConceptByCode('code2aI')).toBeDefined(); + expect(results.findConceptByCode('code2aII')).toBeDefined(); + expect(results.findConceptByCode('code2b')).toBeDefined(); + expect(results.findConceptByCode('code2')).toBeNull(); + expect(results.findConceptByCode('code2a')).toBeNull(); + }); + + test('descendent-leaf of a leaf is empty', async () => { + const results = await simpleProvider.filter(filterContext, true, 'concept', 'descendent-leaf', 'code1'); + expect(results.size()).toBe(0); + }); + test('should filter by is-not-a relationship', async () => { const results = await simpleProvider.filter(filterContext, true, 'concept', 'is-not-a', 'code2'); expect(results.size()).toBe(2); // code1 and code3 (not descendants of code2) diff --git a/tests/cs/cs-extension-codes.test.js b/tests/cs/cs-extension-codes.test.js index 9870082c..c6bb4eea 100644 --- a/tests/cs/cs-extension-codes.test.js +++ b/tests/cs/cs-extension-codes.test.js @@ -126,6 +126,12 @@ describe('Extension-codes supplements', () => { expect(designations.designations.map(d => d.value).sort()).toEqual(['Another A1', 'Code A1']); }); + test('a code system with reserved code spaces is not closed: an expansion of all of it can not list their codes', () => { + expect(factory.build(opContext, [], baseCS()).openCodeSpaceReason()).toMatch(/reserves code spaces/); + const plain = new CodeSystem({resourceType: 'CodeSystem', url: BASE, status: 'active', content: 'complete', concept: [{code: 'A1'}]}); + expect(factory.build(opContext, [], plain).openCodeSpaceReason()).toBeNull(); + }); + test('codes from supplements are not iterated with the code system', async () => { const supp = supplement('http://example.org/global', true, [{code: 'XG1', display: 'Global Code One'}]); const provider = factory.build(opContext, [supp], baseCS()); diff --git a/tests/cs/cs-loinc.test.js b/tests/cs/cs-loinc.test.js index 7e3ca537..44931935 100644 --- a/tests/cs/cs-loinc.test.js +++ b/tests/cs/cs-loinc.test.js @@ -967,6 +967,44 @@ describe('LOINC Provider', () => { return codes; } + async function filterCodes(property, operator, value) { + const codes = []; + const filterContext = await provider.getPrepContext(true); + await provider.filter(filterContext, true, property, operator, value); + const filters = await provider.executeFilters(filterContext); + while (await provider.filterMore(filterContext, filters[0])) { + codes.push(await provider.code(await provider.filterConcept(filterContext, filters[0]))); + } + return codes; + } + + // LOINC's own name for the hierarchy filter, used by its implicit value sets + // (http://loinc.org/vs/[partcode]): ancestor = X is the codes X subsumes, not X itself + test('ancestor = a part is descendent-of the part', async () => { + expect(await provider.doesFilter('ancestor', '=', 'LP7786-9')).toBe(true); + const codes = await filterCodes('ancestor', '=', 'LP7786-9'); + expect(codes.length).toBeGreaterThan(1); + expect(codes).not.toContain('LP7786-9'); + expect(codes).toEqual(await descendantsOf('LP7786-9')); + }); + + test('ancestor in two parts is the union of their descendants', async () => { + expect(await provider.doesFilter('ancestor', 'in', 'LP7786-9,LP14082-9')).toBe(true); + const a = await descendantsOf('LP7786-9'); + const b = await descendantsOf('LP14082-9'); + expect(b.length).toBeGreaterThan(0); + const codes = await filterCodes('ancestor', 'in', 'LP7786-9, LP14082-9'); + expect(new Set(codes)).toEqual(new Set([...a, ...b])); + expect(codes.length).toBe(new Set(codes).size); + }); + + test('the implicit value set for a part expands to its descendants', async () => { + const vs = await factory.buildKnownValueSet('http://loinc.org/vs/LP7786-9', null); + expect(vs.compose.include[0].filter).toEqual([{ property: 'ancestor', op: '=', value: 'LP7786-9' }]); + const f = vs.compose.include[0].filter[0]; + expect(await provider.doesFilter(f.property, f.op, f.value)).toBe(true); + }); + test('should locate codes within filters', async () => { const testCase = expectedResults.filters.relationships[0]; // Use first relationship filter const filterContext = await provider.getPrepContext(false); @@ -1109,6 +1147,71 @@ describe('LOINC Provider', () => { expect(mappingProps(params).map(p => p.value.code)).toEqual(['10828004']); }); + // LP14355-9 Creatinine is listed under LP31398-8 Renal function on two paths in + // ComponentHierarchyBySystem. It has that parent once, and is that part's child once + test('a part listed twice under the same parent has it as parent once', async () => { + const params = []; + await provider.extendLookup('LP14355-9', ['parent'], params); + const parents = params.filter(p => p.name === 'property' && p.part[0].valueCode === 'parent') + .map(p => p.part[1].valueCode); + expect(parents).toEqual(['LP31398-8']); + + const rows = await new Promise((resolve, reject) => { + const db = new sqlite3.Database(testDbPath, sqlite3.OPEN_READONLY); + db.all(`SELECT COUNT(*) AS n FROM Relationships r, RelationshipTypes t, Codes s, Codes p + WHERE r.RelationshipTypeKey = t.RelationshipTypeKey AND t.Description = 'parent' + AND r.SourceKey = s.CodeKey AND s.Code = 'LP14355-9' + AND r.TargetKey = p.CodeKey AND p.Code = 'LP31398-8'`, (err, result) => { + db.close(); + if (err) reject(err); else resolve(result); + }); + }); + expect(rows[0].n).toBe(1); + + const parent = await provider.locate('LP31398-8'); + const iter = await provider.iterator(parent.context); + const children = []; + for (let c = await provider.nextContext(iter); c; c = await provider.nextContext(iter)) { + children.push(c.code); + } + expect(children.filter(c => c === 'LP14355-9')).toEqual(['LP14355-9']); + }); + + // The LinguisticVariant files' axis columns translate the term's Parts. Some languages + // (ko-KR, most of fr-CA) have nothing else, so they are loaded as PartName designations on + // the Parts, and every language that has a variant file is then available + test('every linguistic variant language is loaded', async () => { + const dir = path.resolve(__dirname, '../../tx/data/loinc/AccessoryFiles/LinguisticVariants'); + const fileLangs = fs.readdirSync(dir) + .map(f => f.match(/^([a-z]{2})([A-Z]{2})\d+LinguisticVariant\.csv$/)) + .filter(m => m) + .map(m => m[1] + '-' + m[2]) + .sort(); + expect(fileLangs).toContain('ko-KR'); + const rows = await new Promise((resolve, reject) => { + const db = new sqlite3.Database(testDbPath, sqlite3.OPEN_READONLY); + db.all(`SELECT DISTINCT Languages.Code AS Lang FROM Descriptions, Languages + WHERE Descriptions.LanguageKey = Languages.LanguageKey`, (err, result) => { + db.close(); + if (err) reject(err); else resolve(result); + }); + }); + const loaded = rows.map(r => r.Lang); + expect(fileLangs.filter(l => !loaded.includes(l))).toEqual([]); + }); + + test('a Part has the translations of its axis as PartName designations', async () => { + // LP7753-9 Qn is the SCALE_TYP of 11702-8 (zh-CN 定量型) and of 10391-1 (ko-KR 정량) + const designations = new Designations(opContext.i18n.languageDefinitions); + await provider.designations('LP7753-9', designations); + const partNames = designations.designations + .filter(d => d.use && d.use.code === 'PartName') + .map(d => d.language.code + ' ' + d.value); + expect(partNames).toEqual(expect.arrayContaining(['zh-CN 定量型', 'ko-KR 정량'])); + // and the display is not changed + expect(await provider.display('LP7753-9')).toBe('Qn'); + }); + test('an implicit map is offered for a code it has a mapping for', async () => { let maps = []; await factory.findImplicitConceptMaps(maps, 'http://loinc.org', null, 'http://snomed.info/sct', 'LP14355-9'); @@ -1213,10 +1316,15 @@ describe('LOINC Provider', () => { expect(codes).not.toContain('100865-5'); // has a method expect(codes).not.toContain('LP190769-2'); // a Part + // the two filters come back as one set, their intersection - so its size is the + // include's size (count=0 is answered without walking it) + expect(sets.length).toBe(1); + expect(await provider.filterSize(filterContext, sets[0])).toBe(298); + // membership checks agree with the expansion expect(await provider.filterLocate(filterContext, sets[0], '100007-4')).toBeTruthy(); expect(await provider.filterLocate(filterContext, sets[0], '100865-5')).toBeNull(); - expect(await provider.filterLocate(filterContext, sets[1], 'LP190769-2')).toBeNull(); + expect(await provider.filterLocate(filterContext, sets[0], 'LP190769-2')).toBeNull(); }); test('exists only takes true or false', async () => { diff --git a/tests/cs/cs-rxnorm-filters.test.js b/tests/cs/cs-rxnorm-filters.test.js new file mode 100644 index 00000000..2d7c775e --- /dev/null +++ b/tests/cs/cs-rxnorm-filters.test.js @@ -0,0 +1,258 @@ +/** + * RxNorm filters, totals, status and $lookup properties (bug report R1-R5) + * + * Runs against the RxNorm subset in tx/data/rxnorm, imported once into a scratch database. + * The expected numbers were counted from the RRF files directly, not from the provider. + * + * R1 - filters in one include are ANDed (the SAB filter used to be dropped: its parameter + * shared a name with the SAB the whole query is about) + * R2 - results and totals are concepts, not atoms + * R3 - concept (or code) = / in + * R4 - relationship filters take in (the union), and only RxNorm's own relationships count + * R5 - $lookup returns TTY, STY, SAB, and the relationships when asked for + * and - a concept whose display atom is obsolete (SUPPRESS = O) is inactive + */ + +const fs = require('fs'); +const sqlite3 = require('sqlite3'); +const path = require('path'); +const { RxNormImporter } = require('../../tx/importers/import-rxnorm.module'); +const { RxNormServicesFactory, RxNormConcept } = require('../../tx/cs/cs-rxnorm'); +const { OperationContext } = require('../../tx/operation-context'); +const { TestUtilities } = require('../test-utilities'); +const folders = require('../../library/folder-setup'); + + +describe('RxNorm filters, totals, status and lookup', () => { + const sourceDir = path.resolve(__dirname, '../../tx/data/rxnorm'); + const dbPath = folders.ensureFilePath('rxnorm-filters-testing.db'); + let factory; + let provider; + + beforeAll(async () => { + // Progress to stderr (SUPPRESS_LOGS silences console in CI): if setup is slow or stalls in + // the pipeline, the log shows which step and how far it got + const t0 = Date.now(); + let lastOp = null; + const step = (msg) => process.stderr.write(`[cs-rxnorm-filters] ${((Date.now() - t0) / 1000).toFixed(1)}s ${msg}\n`); + step(`start: ${dbPath}, rss ${Math.round(process.memoryUsage.rss() / 1e6)} MB`); + if (fs.existsSync(dbPath)) { + fs.unlinkSync(dbPath); + } + const importer = new RxNormImporter(sourceDir, dbPath, 'TEST-2025-08-24', { + verbose: false, createStems: false, + progressCallback: (n, op) => { + if (op && op !== lastOp) { + lastOp = op; + step(`${op} (${n} rows so far)`); + } + } + }); + await importer.import(); + step('imported'); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + step('translations loaded'); + factory = new RxNormServicesFactory(opContext.i18n, dbPath); + provider = await factory.build(opContext, []); + step(`provider ready, rss ${Math.round(process.memoryUsage.rss() / 1e6)} MB`); + }, 600000); // tests/setup.js sets 30s for everything; this import takes far longer in CI + + afterAll(() => { + if (provider) { + provider.close(); + } + }); + + // run the filters together, as one include does, and return the set + async function filterSet(...filters) { + const ctxt = await provider.getPrepContext(true); + for (const [prop, op, value] of filters) { + await provider.filter(ctxt, true, prop, op, value); + } + const sets = await provider.executeFilters(ctxt); + return { ctxt, set: sets[0] }; + } + + async function codes(...filters) { + const { ctxt, set } = await filterSet(...filters); + const result = []; + while (await provider.filterMore(ctxt, set)) { + result.push((await provider.filterConcept(ctxt, set)).code); + } + return result; + } + + async function size(...filters) { + const { ctxt, set } = await filterSet(...filters); + return provider.filterSize(ctxt, set); + } + + describe('R1: filters in one include are ANDed', () => { + test('STY and SAB together give their intersection', async () => { + expect(await size(['STY', '=', 'T121'])).toBe(3123); + expect(await size(['SAB', '=', 'USP'])).toBe(403); + expect(await size(['STY', '=', 'T121'], ['SAB', '=', 'USP'])).toBe(169); + }); + + test('two filters of the same kind are both applied', async () => { + // T121 and T109 together: concepts that have both semantic types + const both = await codes(['STY', '=', 'T121'], ['STY', '=', 'T109']); + const t121 = new Set(await codes(['STY', '=', 'T121'])); + const t109 = new Set(await codes(['STY', '=', 'T109'])); + expect(both.length).toBeLessThan(Math.min(t121.size, t109.size)); + expect(both.every(c => t121.has(c) && t109.has(c))).toBe(true); + }); + + test('a value with a quote in it is matched as written', async () => { + expect(await size(['SAB', '=', "O'Brien"])).toBe(0); + }); + }); + + describe('R2: results and totals are concepts, not atoms', () => { + test('STY = T200 counts each concept once (36,375 atoms)', async () => { + const result = await codes(['STY', '=', 'T200']); + expect(result.length).toBe(25295); + expect(new Set(result).size).toBe(25295); + expect(await size(['STY', '=', 'T200'])).toBe(25295); + }); + + test('TTY in SCD,PSN: a concept with both atoms is one result (8,456 atoms)', async () => { + const result = await codes(['TTY', 'in', 'SCD,PSN']); + expect(result.length).toBe(6069); + expect(new Set(result).size).toBe(6069); + }); + + test('the code system total counts concepts', async () => { + expect(await provider.totalCount()).toBe(28488); + const iter = await provider.iterator(null); + let n = 0; + const seen = new Set(); + for (let c = await provider.nextContext(iter); c; c = await provider.nextContext(iter)) { + n++; + seen.add(c.code); + } + expect(n).toBe(28488); + expect(seen.size).toBe(28488); + }); + + test('the display is the main atom, not a prescribable name or tall man synonym', async () => { + // 197589 has SCD, PSN and TMSY atoms + const { ctxt, set } = await filterSet(['TTY', 'in', 'PSN,TMSY']); + const found = await provider.filterLocate(ctxt, set, '197589'); + expect(found).toBeInstanceOf(RxNormConcept); + expect(found.display).toBe('diazepam 10 MG Oral Tablet'); + const located = await provider.locate('197589'); + expect(await provider.display(located.context)).toBe('diazepam 10 MG Oral Tablet'); + expect(located.context.others).toContain('diazePAM 10 MG Oral Tablet'); + }); + }); + + describe('R3: concept in', () => { + test('concept in lists the codes', async () => { + expect((await codes(['concept', 'in', '197589, 161,8163'])).sort()).toEqual(['161', '197589', '8163']); + }); + + test('code is accepted the same as concept', async () => { + expect(await provider.doesFilter('code', 'in', '161,8163')).toBe(true); + expect((await codes(['code', 'in', '161,8163'])).sort()).toEqual(['161', '8163']); + expect(await codes(['concept', '=', '161'])).toEqual(['161']); + }); + + test('an unknown code is just not there', async () => { + expect(await codes(['concept', 'in', '161,999999999'])).toEqual(['161']); + }); + }); + + describe('R4: relationship in', () => { + test('has_ingredient in two ingredients is the union of the two = filters', async () => { + const acetaminophen = await codes(['has_ingredient', '=', 'CUI:161']); + const phenylephrine = await codes(['has_ingredient', '=', 'CUI:8163']); + expect(acetaminophen.length).toBe(962); + expect(phenylephrine.length).toBe(983); + const union = await codes(['has_ingredient', 'in', 'CUI:161,CUI:8163']); + expect(union.length).toBe(1784); + expect(new Set(union)).toEqual(new Set([...acetaminophen, ...phenylephrine])); + }); + + test('an AUI: value follows atom-level rows, from any source', async () => { + // Atom-level RXNREL rows leave RXCUI1 and RXCUI2 empty, and most come from sources other + // than RxNorm. The test data has none, so add one: MMSL's diazepam ingredient atom (101186, + // concept 3322) is the ingredient of its diazepam 10 MG oral tablet atom (8241449, 197589) + await new Promise((resolve, reject) => { + const db = new sqlite3.Database(dbPath); + db.run(`INSERT INTO RXNREL (RXCUI1, RXAUI1, REL, RXCUI2, RXAUI2, RELA, SAB) + VALUES ('', '101186', 'RO', '', '8241449', 'has_ingredient', 'MMSL')`, + err => db.close(() => err ? reject(err) : resolve())); + }); + expect(await codes(['has_ingredient', '=', 'AUI:101186'])).toEqual(['197589']); + // and with CUI: values, which only use RxNorm's own concept-level rows + const both = await codes(['has_ingredient', 'in', 'AUI:101186,CUI:161']); + expect(both).toContain('197589'); + expect(both.length).toBe(963); + }); + + test('a relationship value must be CUI: or AUI:', async () => { + expect(await provider.doesFilter('has_ingredient', 'in', 'CUI:161,CUI:8163')).toBe(true); + expect(await provider.doesFilter('has_ingredient', 'in', 'CUI:161,8163')).toBe(false); + }); + + test('relationships combine with other filters', async () => { + expect(await size(['has_ingredient', 'in', 'CUI:161,CUI:8163'], ['STY', '=', 'T121'])).toBe(0); + }); + }); + + describe('obsolete concepts are inactive', () => { + test('SUPPRESS = O on the display atom', async () => { + const obsolete = (await provider.locate('102152')).context; + expect(await provider.isInactive(obsolete)).toBe(true); + expect(await provider.getStatus(obsolete)).toBe('obsolete'); + const current = (await provider.locate('197589')).context; + expect(await provider.isInactive(current)).toBe(false); + expect(await provider.getStatus(current)).toBeNull(); + }); + + test('concepts from a filter know their status', async () => { + const { ctxt, set } = await filterSet(['concept', 'in', '102152,197589']); + const status = {}; + while (await provider.filterMore(ctxt, set)) { + const c = await provider.filterConcept(ctxt, set); + status[c.code] = await provider.isInactive(c); + } + expect(status).toEqual({ '102152': true, '197589': false }); + }); + }); + + describe('R5: $lookup properties', () => { + const props = (params, name) => params.filter(p => p.name === 'property' && p.part[0].valueCode === name).map(p => p.part[1].valueCode); + + test('TTY, STY and SAB by default, not the relationships', async () => { + const params = []; + await provider.extendLookup('197589', [], params); + expect(props(params, 'TTY')).toEqual(['PSN', 'SCD', 'TMSY']); + expect(props(params, 'STY')).toEqual(['T200']); + expect(props(params, 'SAB')).toEqual(['GS', 'MMSL', 'MMX', 'MTHSPL', 'NDDF', 'RXNORM', 'SNOMEDCT_US', 'VANDF']); + expect(props(params, 'has_tradename')).toEqual([]); + }); + + test('* adds RxNorm\'s relationships, valued with the bare RXCUI', async () => { + const params = []; + await provider.extendLookup('197589', ['*'], params); + expect(props(params, 'TTY')).toEqual(['PSN', 'SCD', 'TMSY']); + expect(props(params, 'consists_of')).toEqual(['315796']); + expect(props(params, 'has_dose_form')).toEqual(['317541']); + expect(props(params, 'has_tradename')).toEqual(['104701', '200933', '200934', '200935', '200936']); + expect(props(params, 'isa')).toEqual(['1155861', '1155862', '371786']); + }); + + test('named properties: only those', async () => { + const params = []; + await provider.extendLookup('197589', ['has_dose_form', 'STY'], params); + expect(params.map(p => p.part[0].valueCode).sort()).toEqual(['STY', 'has_dose_form']); + }); + + test('a relationship property value finds the code again as a filter', async () => { + // 197589 has_dose_form 317541 (as a lookup property) <=> has_dose_form = CUI:317541 (as a filter) + expect(await codes(['has_dose_form', '=', 'CUI:317541'])).toContain('197589'); + }); + }); +}); diff --git a/tests/cs/cs-snomed-ecl.test.js b/tests/cs/cs-snomed-ecl.test.js index 1861b113..587bfea2 100644 --- a/tests/cs/cs-snomed-ecl.test.js +++ b/tests/cs/cs-snomed-ecl.test.js @@ -335,6 +335,21 @@ describe('ECL Validator Test Suite', () => { test('an unknown description filter field is a syntax error', () => { expect(() => snomedServices.filterECL('< 64572001 {{ D foo = 1 }}', true, opContext)).toThrow('Unknown description filter field'); }); + + // ECL 2.2 makes the D optional: {{ term = ... }} is a description filter (S7) + test('the D is optional', () => { + for (const body of ['term = "lobe liv"', 'term = wild:"*lobe*"', 'typeId = 900000000000003001 |Fully specified name|, term = "body structure"', + 'language = en', 'term = ("caudate" "quadrate")', 'active = true']) { + const withD = codes(`<< 10200004 {{ D ${body} }}`); + expect(withD.length).toBeGreaterThan(0); + expect(codes(`<< 10200004 {{ ${body} }}`)).toEqual(withD); + expect(codes(`<< 10200004 {{ d ${body} }}`)).toEqual(withD); + } + expect(codes('<< 10200004 {{ term = "liver" }} {{ term = "lobe" }}')).toEqual(codes('<< 10200004 {{ D term = "liver" }} {{ D term = "lobe" }}')); + // an unknown field is still an error, and so is anything that is not a field name + expect(() => snomedServices.filterECL('< 64572001 {{ foo = 1 }}', true, opContext)).toThrow('Unknown description filter field'); + expect(() => snomedServices.filterECL('< 64572001 {{ 123 }}', true, opContext)).toThrow('Expected M, D, C, + or a description filter after {{'); + }); }); describe('ECL != and reversed (R) attributes', () => { diff --git a/tests/cs/cs-snomed.test.js b/tests/cs/cs-snomed.test.js index 5b099254..cb913c53 100644 --- a/tests/cs/cs-snomed.test.js +++ b/tests/cs/cs-snomed.test.js @@ -28,6 +28,7 @@ const {SnomedImporter} = require("../../tx/importers/import-sct.module"); const { OperationContext } = require('../../tx/operation-context'); const {Designations} = require("../../tx/library/designations"); const {TestUtilities} = require("../test-utilities"); +const { Languages } = require('../../library/languages'); const folders = require('../../library/folder-setup'); // Shared cache file paths and utilities. @@ -1242,4 +1243,731 @@ if (foundFiles === 0) { } else { console.log('\n✅ All prerequisites met'); //process.exit(0); -} \ No newline at end of file +} +// Inactive relationship rows are imported but ignored altogether (agreed with the SNOMED +// leads, Nov 2026). The test edition's TEST1000003 relationship file carries two inactive +// rows on 50325005 |Alcoholic fatty liver|: a finding site of 64033007 |Kidney structure| +// (group 1) and an is-a to 51292008 |Hepatorenal syndrome|. Neither may be visible. +describe('SNOMED CT inactive relationships are ignored', () => { + const FATTY_LIVER = '50325005'; + const KIDNEY = '64033007'; + const HEPATORENAL = '51292008'; + const FINDING_SITE = '363698007'; + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + const ref = (code) => { + const r = provider.sct.concepts.findConcept(BigInt(code)); + expect(r.found).toBe(true); + return r.index; + }; + + async function filterCodes(prop, op, value) { + const fc = await provider.getPrepContext(true); + await provider.filter(fc, true, prop, op, value); + const [set] = await provider.executeFilters(fc); + const codes = []; + while (await provider.filterMore(fc, set)) { + codes.push(await provider.code(await provider.filterConcept(fc, set))); + } + return codes; + } + + async function filterHas(prop, op, value, code) { + const fc = await provider.getPrepContext(false); + await provider.filter(fc, false, prop, op, value); + const [set] = await provider.executeFilters(fc); + const located = await provider.filterLocate(fc, set, code); + const { context } = await provider.locate(code); + const checked = await provider.filterCheck(fc, set, context); + return { located: located !== null && typeof located !== 'string', checked }; + } + + async function lookupProps(code, props) { + const params = []; + await provider.extendLookup(code, props, params); + return params.filter(p => p.name === 'property').map(p => p.part.find(x => x.name === 'code').valueCode + '=' + p.part.find(x => x.name === 'value').valueCode); + } + + test('the test edition carries the inactive rows (otherwise rebuild the cache from tx/data/snomed)', () => { + const sct = provider.sct; + // the raw outbound list: getConceptRelationships() leaves inactive rows out + const outbounds = sct.concepts.getConcept(ref(FATTY_LIVER)).outbounds; + const rels = sct.refs.getReferences(outbounds).map(i => sct.relationships.getRelationship(i)); + const inactive = rels.filter(r => !r.active).map(r => sct.getConceptId(r.relType) + '=' + sct.getConceptId(r.target)).sort(); + expect(inactive).toEqual([FINDING_SITE + '=' + KIDNEY, '116680003=' + HEPATORENAL].sort()); + expect(sct.getConceptRelationships(ref(FATTY_LIVER)).every(i => sct.relationships.isActive(i))).toBe(true); + }); + + test('a property filter does not match an inactive relationship (expand)', async () => { + const codes = await filterCodes(FINDING_SITE, '=', KIDNEY); + expect(codes).not.toContain(FATTY_LIVER); + expect(codes).toHaveLength(9); + }); + + test('a property filter does not match an inactive relationship (validate)', async () => { + expect(await filterHas(FINDING_SITE, '=', KIDNEY, FATTY_LIVER)).toEqual({ located: false, checked: false }); + expect(await filterHas(FINDING_SITE, '=', '10200004', FATTY_LIVER)).toEqual({ located: true, checked: true }); + }); + + test('$lookup lists neither the inactive finding site nor the inactive parent', async () => { + const props = await lookupProps(FATTY_LIVER, [FINDING_SITE, 'parent']); + expect(props.sort()).toEqual([FINDING_SITE + '=10200004', 'parent=197321007', 'parent=41309000']); + }); + + test('$lookup does not list a child through an inactive is-a', async () => { + const props = await lookupProps(HEPATORENAL, ['child']); + expect(props).not.toContain('child=' + FATTY_LIVER); + expect(props).toHaveLength(4); + }); + + test('is-a, ECL and subsumption ignore an inactive is-a', async () => { + expect(await filterCodes('concept', 'is-a', HEPATORENAL)).not.toContain(FATTY_LIVER); + expect(await filterCodes('constraint', '=', '<< ' + HEPATORENAL)).not.toContain(FATTY_LIVER); + expect(await provider.subsumesTest(HEPATORENAL, FATTY_LIVER)).toBe('not-subsumed'); + expect((await provider.locateIsA(FATTY_LIVER, HEPATORENAL)).context).toBeFalsy(); + }); + + test('an ECL refinement does not match an inactive relationship', async () => { + const codes = await filterCodes('constraint', '=', '< 404684003 : ' + FINDING_SITE + ' = ' + KIDNEY); + expect(codes).not.toContain(FATTY_LIVER); + expect(codes).toHaveLength(9); + }); +}); + +// concept in A,B,C means any of the listed concepts (S3). Plain concepts were pushed into +// set.members as bare concept indexes while everything reading set.members expects member +// records ({ref}), so each came out as 4294967295 and $validate-code never matched. +// "code" is accepted as the same property as "concept": value sets in production use it. +describe.each(['concept', 'code'])('SNOMED CT %s in filter', (prop) => { + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + async function inCodes(value) { + const fc = await provider.getPrepContext(true); + expect(await provider.doesFilter(prop, 'in', value)).toBe(true); + await provider.filter(fc, true, prop, 'in', value); + const [set] = await provider.executeFilters(fc); + const codes = []; + while (await provider.filterMore(fc, set)) { + codes.push(await provider.code(await provider.filterConcept(fc, set))); + } + return codes; + } + + async function inHas(value, code) { + const fc = await provider.getPrepContext(false); + await provider.filter(fc, false, prop, 'in', value); + const [set] = await provider.executeFilters(fc); + const located = await provider.filterLocate(fc, set, code); + const { context } = await provider.locate(code); + return { located: located !== null && typeof located !== 'string', checked: await provider.filterCheck(fc, set, context) }; + } + + test('expands to every listed concept', async () => { + expect(await inCodes('10200004,50325005,51292008')).toEqual(['10200004', '50325005', '51292008']); + }); + + test('ignores spaces, and lists a concept named twice once', async () => { + expect(await inCodes('50325005, 51292008,50325005')).toEqual(['50325005', '51292008']); + }); + + test('a listed concept is in the set, an unlisted one is not', async () => { + expect(await inHas('10200004,50325005,51292008', '51292008')).toEqual({ located: true, checked: true }); + expect(await inHas('10200004,50325005,51292008', '64033007')).toEqual({ located: false, checked: false }); + }); + + test('a listed reference set stands for its members, alongside plain concepts', async () => { + const refset = await inCodes('723264001'); // Lateralizable body structure reference set + expect(refset.length).toBeGreaterThan(0); + expect(refset).toContain('64033007'); + const mixed = await inCodes('723264001,51292008'); + expect(mixed).toEqual([...refset, '51292008']); + expect(await inHas('723264001,51292008', '64033007')).toEqual({ located: true, checked: true }); + }); +}); + +// "code" is the same filter property as "concept" for the other concept operations too +describe('SNOMED CT code filter property', () => { + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + async function codes(prop, op, value) { + const fc = await provider.getPrepContext(true); + await provider.filter(fc, true, prop, op, value); + const [set] = await provider.executeFilters(fc); + const out = []; + while (await provider.filterMore(fc, set)) { + out.push(await provider.code(await provider.filterConcept(fc, set))); + } + return out.sort(); + } + + test.each(['=', 'is-a', 'descendent-of', 'child-of', 'generalizes'])('code %s gives the same result as concept %s', async (op) => { + expect(await provider.doesFilter('code', op, '51292008')).toBe(true); + const viaCode = await codes('code', op, '51292008'); + expect(viaCode.length).toBeGreaterThan(0); + expect(viaCode).toEqual(await codes('concept', op, '51292008')); + }); +}); + +// The R5 hierarchy filters descendent-leaf (S4) and is-not-a (S5); both used to fail with +// "Unsupported filter operation". 16763008 |Hemorrhage of liver| has 12 descendants, 8 of +// them leaves; 78171005 |Traumatic hemorrhage of liver| is one of them, with 3 descendants. +describe('SNOMED CT descendent-leaf and is-not-a filters', () => { + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + // the codes passing every filter: the first is iterated, the rest checked + async function codes(...filters) { + const fc = await provider.getPrepContext(true); + for (const [op, value] of filters) { + expect(await provider.doesFilter('concept', op, value)).toBe(true); + await provider.filter(fc, true, 'concept', op, value); + } + const [first, ...rest] = await provider.executeFilters(fc); + const out = []; + while (await provider.filterMore(fc, first)) { + const c = await provider.filterConcept(fc, first); + let ok = true; + for (const r of rest) { + ok = ok && await provider.filterCheck(fc, r, c); + } + if (ok) { + out.push(await provider.code(c)); + } + } + return out.sort(); + } + + async function located(op, value, code) { + const fc = await provider.getPrepContext(false); + await provider.filter(fc, false, 'concept', op, value); + const [set] = await provider.executeFilters(fc); + const r = await provider.filterLocate(fc, set, code); + return r !== null && typeof r !== 'string'; + } + + test('descendent-leaf takes only the descendants with no children', async () => { + expect(await codes(['descendent-leaf', '16763008'])).toEqual(['111371005', '197342009', '197364000', '206242003', + '21580006', '262798009', '405559000', '735677007']); + expect(await located('descendent-leaf', '16763008', '206242003')).toBe(true); + expect(await located('descendent-leaf', '16763008', '78171005')).toBe(false); // has children + expect(await located('descendent-leaf', '16763008', '16763008')).toBe(false); // the concept itself + }); + + test('descendent-leaf of a leaf is empty', async () => { + expect(await codes(['descendent-leaf', '206242003'])).toEqual([]); + }); + + test('is-not-a takes out the concept and its descendants', async () => { + expect(await codes(['is-a', '16763008'], ['is-not-a', '78171005'])).toEqual(['111371005', '16763008', '197342009', + '197364000', '262796008', '262797004', '262798009', '405559000', '735677007']); + expect(await located('is-not-a', '78171005', '78171005')).toBe(false); + expect(await located('is-not-a', '78171005', '206242003')).toBe(false); + expect(await located('is-not-a', '78171005', '16763008')).toBe(true); + }); + + test('is-not-a is the complement of is-a', async () => { + const isA = await codes(['is-a', '78171005']); + const notA = await codes(['is-not-a', '78171005']); + expect(isA.length + notA.length).toBe(provider.sct.concepts.count()); + expect(notA.filter(c => isA.includes(c))).toEqual([]); + }); +}); + +// display regex (S6): a concept matches when it has an active description (FSN or synonym, +// not a text definition), in one language, whose whole term matches the pattern, +// case-sensitively. The language is the first one the request named that the edition has, +// else the edition's default. The test edition is English, plus Dutch descriptions for a few +// hematoma concepts in its TEST1000003 extension (with a Dutch language reference set). Iterating it scans the description table (yielding as it goes), and a cached value +// set's filter keeps the result, as for ECL. +describe('SNOMED CT display regex filter', () => { + const HEMATOMA = ['206242003', '21580006', '262796008', '262797004', '262798009', '35566002', '405559000', '431674004']; + let factory; + let provider; + let langDefs; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + langDefs = await TestUtilities.loadLanguageDefinitions(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(langDefs)); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + const langs = (code) => code ? Languages.fromAcceptLanguage(code, langDefs, true) : null; + + async function regexCodes(pattern, { lang = null, fc = null, prop = 'display', op = 'regex' } = {}) { + const prep = await provider.getPrepContext(true, langs(lang)); + await provider.filter(prep, true, prop, op, pattern, fc); + const [set] = await provider.executeFilters(prep); + const out = []; + while (await provider.filterMore(prep, set)) { + out.push(await provider.code(await provider.filterConcept(prep, set))); + } + return { codes: out.sort(), set }; + } + + async function regexHas(pattern, code) { + const prep = await provider.getPrepContext(false, null); + await provider.filter(prep, false, 'display', 'regex', pattern); + const [set] = await provider.executeFilters(prep); + const located = await provider.filterLocate(prep, set, code); + const { context } = await provider.locate(code); + return { located: located !== null && typeof located !== 'string', checked: await provider.filterCheck(prep, set, context) }; + } + + test('is supported, but not for a pattern that does not compile', async () => { + expect(await provider.doesFilter('display', 'regex', '.*[Hh]ematoma.*')).toBe(true); + expect(await provider.doesFilter('display', 'regex', '(unclosed')).toBe(false); + }); + + test('matches every concept with a matching active description', async () => { + expect((await regexCodes('.*[Hh]ematoma.*')).codes).toEqual(HEMATOMA); + }); + + test('matches the whole term, case-sensitively', async () => { + // only 35566002 has a description that ends in "Hematoma" with a capital H + expect((await regexCodes('.*Hematoma')).codes).toEqual(['35566002']); + expect((await regexCodes('Hematoma')).codes).toEqual(['35566002']); + expect((await regexCodes('hematoma')).codes).toEqual([]); + expect((await regexCodes('(?i)hematoma')).codes).toEqual(['35566002']); + }); + + test('matches synonyms, not only the preferred term', async () => { + // "Haematoma" is a synonym of 35566002 (GB spelling); its preferred term is "Hematoma" + const sct = provider.sct; + const r = sct.concepts.findConcept(35566002n); + expect(sct.getDisplayName(r.index)).toBe('Hematoma'); + expect((await regexCodes('Haematoma')).codes).toEqual(['35566002']); + }); + + test('ignores inactive descriptions', async () => { + // 35566002 has the inactive synonyms "Hematoma, NOS" and "Haematoma, NOS" + expect((await regexCodes('H(a)?ematoma, NOS')).codes).toEqual([]); + }); + + test('validates one concept by its own descriptions', async () => { + expect(await regexHas('.*[Hh]ematoma.*', '262798009')).toEqual({ located: true, checked: true }); + expect(await regexHas('.*[Hh]ematoma.*', '16763008')).toEqual({ located: false, checked: false }); + }); + + test('a language the edition does not have falls back to the default', async () => { + expect((await regexCodes('.*[Hh]ematoma.*', { lang: 'de' })).codes).toEqual(HEMATOMA); + expect((await regexCodes('.*[Hh]ematoma.*', { lang: 'en-US' })).codes).toEqual(HEMATOMA); + }); + + test('a language the edition has: only its descriptions are searched', async () => { + const DUTCH = ['262796008', '262797004', '262798009', '35566002']; + expect((await regexCodes('.*hematoom.*', { lang: 'nl' })).codes).toEqual(DUTCH); + expect((await regexCodes('.*hematoom.*', { lang: 'nl-BE' })).codes).toEqual(DUTCH); + expect((await regexCodes('.*[Hh]ematoma.*', { lang: 'nl' })).codes).toEqual([]); + expect((await regexCodes('.*hematoom.*')).codes).toEqual([]); // English by default + // an acceptable (not preferred) Dutch synonym counts too + expect((await regexCodes('bloeduitstorting', { lang: 'nl' })).codes).toEqual(['35566002']); + // the first requested language the edition has wins + expect((await regexCodes('.*hematoom.*', { lang: 'de, nl;q=0.8, en;q=0.5' })).codes).toEqual(DUTCH); + const ecl = await regexCodes('* {{ D term = wild:"*hematoom*", language = nl }}', { prop: 'constraint', op: '=' }); + expect(ecl.codes).toEqual(DUTCH); + }); + + test('a text definition is not a display', async () => { + // 35566002 |Hematoma| has no definition in the test edition; use one that does + const sct = provider.sct; + const definitionKind = sct.concepts.findConcept(900000000000550004n).index; + let defined = null; + for (let i = 0; i < sct.concepts.count() && !defined; i++) { + const ref = sct.concepts.getConceptByCount(i).index; + const c = sct.concepts.getConcept(ref); + for (const di of (c.descriptions ? sct.refs.getReferences(c.descriptions) : [])) { + const d = sct.descriptions.getDescription(di); + if (d.active && d.kind === definitionKind) { + defined = { code: String(sct.getConceptId(ref)), term: sct.strings.getEntry(d.iDesc) }; + break; + } + } + } + expect(defined).not.toBeNull(); + const escaped = defined.term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + expect((await regexCodes(escaped)).codes).not.toContain(defined.code); + }); + + test('a cached value set filter keeps the scan result', async () => { + const fc = {}; + const first = await regexCodes('.*[Hh]ematoma.*', { fc }); + expect(first.codes).toEqual(HEMATOMA); + const prep = await provider.getPrepContext(true, null); + await provider.filter(prep, true, 'display', 'regex', '.*[Hh]ematoma.*', fc); + const [set] = await provider.executeFilters(prep); + expect(set.populationDone).toBe(true); // no scan needed + expect((await regexCodes('.*[Hh]ematoma.*', { fc })).codes).toEqual(HEMATOMA); + }); + + test('gives the same concepts as the equivalent ECL description filter', async () => { + const ecl = await regexCodes('* {{ D term = wild:"*hematoma*", language = en }}', { prop: 'constraint', op: '=' }); + expect(ecl.codes).toEqual(HEMATOMA); + }); +}); + +// Concrete values (S8). The test edition's International subset has integer-valued drug +// strengths and counts; its TEST1000003 extension adds three test data attributes with +// string, boolean and decimal values on 329238006, 776168003 and 1204474000. +describe('SNOMED CT concrete values', () => { + const NICKNAME = '1000701000003104'; + const FISH = '1000711000003102'; + const POTENCY = '1000721000003107'; + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + async function ecl(expression) { + const fc = await provider.getPrepContext(true); + await provider.filter(fc, true, 'constraint', '=', expression); + const [set] = await provider.executeFilters(fc); + const out = []; + while (await provider.filterMore(fc, set)) { + out.push(await provider.code(await provider.filterConcept(fc, set))); + } + return out.sort(); + } + + async function lookupValues(code, props) { + const params = []; + await provider.extendLookup(code, props, params); + const out = {}; + for (const p of params.filter(p => p.name === 'property')) { + const value = p.part.find(x => x.name === 'value'); + const key = Object.keys(value).find(k => k !== 'name'); + out[p.part.find(x => x.name === 'code').valueCode] = { [key]: value[key] }; + } + return out; + } + + test('$lookup returns concrete values, typed by the MRCM range or else by how they are written', async () => { + expect(await lookupValues('329238006', ['1142135004', '1142136003', '1142139005', NICKNAME, FISH, POTENCY])).toEqual({ + '1142135004': { valueDecimal: 4000 }, // dec() in the MRCM, though written #4000 + '1142136003': { valueDecimal: 1 }, + '1142139005': { valueInteger: 1 }, // int() in the MRCM + [NICKNAME]: { valueString: 'Sunshine in a capsule' }, + [FISH]: { valueBoolean: true }, + [POTENCY]: { valueDecimal: 2.5 } // no MRCM rule: #2.5 is written as a decimal + }); + expect(await lookupValues('1204474000', [FISH, POTENCY])).toEqual({ [FISH]: { valueBoolean: false }, [POTENCY]: { valueDecimal: 0.75 } }); + }); + + test('$lookup only returns the concrete values asked for, when properties are named', async () => { + expect(Object.keys(await lookupValues('329238006', [FISH]))).toEqual([FISH]); + }); + + test('ECL compares decimals by value, and decimal literals against integer values', async () => { + expect(await ecl(`< 763158003 : ${POTENCY} >= #0.75`)).toEqual(['1204474000', '329238006']); + expect(await ecl(`< 763158003 : ${POTENCY} = #2.50`)).toEqual(['329238006']); + expect(await ecl(`< 763158003 : ${POTENCY} < #1.0`)).toEqual(['1204474000']); + expect(await ecl('< 763158003 : 1142138002 > #599.5')).toEqual(['1204474000']); + }); + + test('ECL string concrete values: =, !=, a set, wild and match', async () => { + expect(await ecl(`< 763158003 : ${NICKNAME} = "Barium milkshake"`)).toEqual(['1204474000']); + expect(await ecl(`< 763158003 : ${NICKNAME} != "Barium milkshake"`)).toEqual(['329238006']); + expect(await ecl(`< 763158003 : ${NICKNAME} = ("Barium milkshake" "Sunshine in a capsule")`)).toEqual(['1204474000', '329238006']); + expect(await ecl(`< 763158003 : ${NICKNAME} = wild:"*capsule"`)).toEqual(['329238006']); + expect(await ecl(`< 763158003 : ${NICKNAME} = match:"barium"`)).toEqual(['1204474000']); + }); + + test('ECL boolean concrete values', async () => { + expect(await ecl(`< 763158003 : ${FISH} = true`)).toEqual(['329238006', '776168003']); + expect(await ecl(`< 763158003 : ${FISH} = false`)).toEqual(['1204474000']); + expect(await ecl(`< 763158003 : ${FISH} != true`)).toEqual(['1204474000']); + }); + + test('a value of the wrong type never matches', async () => { + expect(await ecl('< 763158003 : 1142135004 = "4000"')).toEqual([]); + expect(await ecl(`< 763158003 : ${FISH} = #1`)).toEqual([]); + expect(await ecl(`< 763158003 : ${NICKNAME} = true`)).toEqual([]); + }); + + test('concrete values in the same group', async () => { + expect(await ecl('< 763158003 : { 1142135004 = #4000, 1142136003 = #1 }')).toEqual(['329238006']); + expect(await ecl('< 763158003 : { 1142135004 = #4000, 1142136003 = #2 }')).toEqual([]); + }); +}); + +// Map reference sets have members (S9). The importer used to skip any reference set file +// whose header mentioned "map" (every map refset: mapTarget, mapGroup, ...), so ECL ^ and +// ?fhir_vs=refset/ found nothing in them. The test edition's TEST1000003 extension has a +// simple map (1000891000003103, two members) and an extended map (1000901000003102: three +// active rows for two concepts, plus an inactive row for 405559000). +describe('SNOMED CT map reference sets', () => { + const SIMPLE = '1000891000003103'; + const EXTENDED = '1000901000003102'; + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + async function codes(prop, op, value) { + const fc = await provider.getPrepContext(true); + await provider.filter(fc, true, prop, op, value); + const [set] = await provider.executeFilters(fc); + const out = []; + while (await provider.filterMore(fc, set)) { + out.push(await provider.code(await provider.filterConcept(fc, set))); + } + return out.sort(); + } + + async function located(prop, op, value, code) { + const fc = await provider.getPrepContext(false); + await provider.filter(fc, false, prop, op, value); + const [set] = await provider.executeFilters(fc); + const r = await provider.filterLocate(fc, set, code); + return r !== null && typeof r !== 'string'; + } + + test('a simple map reference set has its members', async () => { + expect(await codes('constraint', '=', `^ ${SIMPLE}`)).toEqual(['16763008', '262796008']); + expect(await codes('concept', 'in', SIMPLE)).toEqual(['16763008', '262796008']); + }); + + test('an extended map: each member concept once, inactive rows left out', async () => { + expect(await codes('constraint', '=', `^ ${EXTENDED}`)).toEqual(['262797004', '262798009']); + expect(await codes('concept', 'in', EXTENDED)).toEqual(['262797004', '262798009']); + expect(await located('constraint', '=', `^ ${EXTENDED}`, '262797004')).toBe(true); + expect(await located('constraint', '=', `^ ${EXTENDED}`, '405559000')).toBe(false); // inactive row + expect(await located('concept', 'in', EXTENDED, '262798009')).toBe(true); + }); + + test('member filters can select on the map fields', async () => { + expect(await codes('constraint', '=', `^ ${EXTENDED} {{ M mapTarget = "Y92.9" }}`)).toEqual(['262797004']); + expect(await codes('constraint', '=', `^ ${SIMPLE} {{ M mapTarget = "K76.89" }}`)).toEqual(['16763008']); + }); +}); + +// Implicit value set urls (S10): the ecl/ form, and the edition and version forms, which +// used to compare against an undefined edition and so never matched. +describe('SNOMED CT implicit value set urls', () => { + let factory; + let provider; + const HEMORRHAGE = ['111371005', '16763008', '197342009', '197364000', '206242003', '21580006', '262796008', + '262797004', '262798009', '405559000', '431674004', '735677007', '78171005']; + const VERSION = 'http://snomed.info/xsct/31000003106/version/20250909'; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + // the concepts the value set's (single) include selects + async function members(vs) { + const include = vs.compose.include[0]; + const fc = await provider.getPrepContext(true); + for (const f of include.filter) { + await provider.filter(fc, true, f.property, f.op, f.value); + } + const [set] = await provider.executeFilters(fc); + const out = []; + while (await provider.filterMore(fc, set)) { + out.push(await provider.code(await provider.filterConcept(fc, set))); + } + return out.sort(); + } + + test('the three forms are told apart, for the sct and xsct bases', () => { + expect(factory.implicitValueSetForm('http://snomed.info/sct?fhir_vs=isa/16763008').form).toBe('generic'); + expect(factory.implicitValueSetForm('http://snomed.info/sct/31000003106?fhir_vs=isa/16763008').form).toBe('edition'); + expect(factory.implicitValueSetForm('http://snomed.info/xsct/31000003106?fhir_vs').form).toBe('edition'); + expect(factory.implicitValueSetForm('http://snomed.info/sct/31000003106/version/20250909?fhir_vs').form).toBe('version'); + expect(factory.implicitValueSetForm(`${VERSION}?fhir_vs=refset/723264001`).form).toBe('version'); + expect(factory.implicitValueSetForm('http://snomed.info/sct/731000124108?fhir_vs')).toBeNull(); + expect(factory.implicitValueSetForm('http://snomed.info/sct/31000003106/version/20240101?fhir_vs')).toBeNull(); + expect(factory.implicitValueSetForm('http://snomed.info/sct?fhir_vsx')).toBeNull(); + }); + + test('ecl/ takes a URL-encoded expression, or one already decoded', async () => { + for (const url of ['http://snomed.info/sct?fhir_vs=ecl/%3C%3C%2016763008', 'http://snomed.info/sct?fhir_vs=ecl/<< 16763008']) { + const vs = await factory.buildKnownValueSet(url, null); + expect(vs.url).toBe(url); + expect(vs.compose.include[0].filter).toEqual([{ property: 'constraint', op: '=', value: '<< 16763008' }]); + expect(await members(vs)).toEqual(HEMORRHAGE); + } + expect(await factory.buildKnownValueSet('http://snomed.info/sct?fhir_vs=ecl/', null)).toBeNull(); + }); + + test('the edition and version forms pin the include to this version; the generic form does not', async () => { + const generic = await factory.buildKnownValueSet('http://snomed.info/sct?fhir_vs=isa/16763008', null); + expect(generic.compose.include[0].version).toBeUndefined(); + for (const url of ['http://snomed.info/sct/31000003106?fhir_vs=isa/16763008', 'http://snomed.info/sct/31000003106/version/20250909?fhir_vs=isa/16763008']) { + const vs = await factory.buildKnownValueSet(url, null); + expect(vs.compose.include[0].version).toBe(VERSION); + expect(await members(vs)).toEqual(HEMORRHAGE); + } + // an implicit value set's edition and version are in its url; it has no version of its own + expect(generic.version).toBeUndefined(); + }); +}); + +// $lookup property=* (S12): every property the SNOMED CT on FHIR page defines - including +// moduleId (as the page names it), sufficientlyDefined and semanticTag - except the normal +// forms, which are computed and have to be asked for by name. +describe('SNOMED CT $lookup properties', () => { + let factory; + let provider; + + beforeAll(async () => { + const cacheFilePath = globalCacheFilePath || findAvailableCacheFile(); + const opContext = new OperationContext('en', await TestUtilities.loadTranslations(await TestUtilities.loadLanguageDefinitions())); + factory = new SnomedServicesFactory(opContext.i18n, cacheFilePath); + provider = await factory.build(opContext, []); + }); + + afterAll(() => { + if (provider) { + provider.sct.close(); + } + }); + + async function props(code, asked) { + const params = []; + const { context } = await provider.locate(code); + await provider.extendLookup(context, asked, params); + const out = {}; + for (const p of params.filter(p => p.name === 'property')) { + const value = p.part.find(x => x.name === 'value'); + (out[p.part.find(x => x.name === 'code').valueCode] = out[p.part.find(x => x.name === 'code').valueCode] || []) + .push(value[Object.keys(value).find(k => k !== 'name')]); + } + return out; + } + + test('* adds sufficientlyDefined and semanticTag to the default properties, but not the normal forms', async () => { + const all = await props('367430006', ['*']); + const def = await props('367430006', null); + expect(all.sufficientlyDefined).toEqual([true]); + expect(all.semanticTag).toEqual(['procedure']); + expect(all.normalForm).toBeUndefined(); + expect(all.normalFormTerse).toBeUndefined(); + const rest = { ...all }; + delete rest.sufficientlyDefined; + delete rest.semanticTag; + expect(rest).toEqual(def); + expect(def.sufficientlyDefined).toBeUndefined(); + }); + + test('the module is moduleId, and asking for "module" still works', async () => { + expect((await props('367430006', null)).moduleId).toEqual(['900000000000207008']); + expect(await props('367430006', ['module'])).toEqual({ moduleId: ['900000000000207008'] }); + expect((await props('367430006', null)).module).toBeUndefined(); + }); + + test('sufficientlyDefined is false for a primitive concept', async () => { + const sct = provider.sct; + let primitive = null; + for (let i = 0; i < sct.concepts.count() && !primitive; i++) { + const ref = sct.concepts.getConceptByCount(i).index; + if (sct.isActive(ref) && sct.isPrimitive(ref)) { + primitive = String(sct.getConceptId(ref)); + } + } + expect((await props(primitive, ['sufficientlyDefined'])).sufficientlyDefined).toEqual([false]); + }); + + test('the normal forms, when asked for by name, for a concept and an expression', async () => { + expect(await props('367430006', ['normalFormTerse'])).toEqual({ + normalFormTerse: ['387713003:{260686004=257903006,405813007=(118632007:{272741003=182353008})}'] + }); + expect((await props('367430006', ['normalForm'])).normalForm).toEqual([ + '387713003|Surgical procedure|:{260686004|Method|=257903006|Repair - action|,405813007|Procedure site - Direct|=(118632007|Structure of tendon within hand|:{272741003|Laterality|=182353008|Side|})}']); + const expr = await props('367430006:{405813007=(85562004:272741003=24028007)}', ['normalFormTerse']); + expect(expr.normalFormTerse).toEqual(['387713003:{260686004=257903006,405813007=(118632007:{272741003=182353008})},{405813007=(85562004:{272741003=24028007})}']); + }); +}); diff --git a/tests/ips/analysis.test.js b/tests/ips/analysis.test.js index f6999a01..8bcaace6 100644 --- a/tests/ips/analysis.test.js +++ b/tests/ips/analysis.test.js @@ -96,13 +96,52 @@ describe('the analysis', () => { entry({ resourceType: 'Condition', id: 'a', subject: { reference: 'urn:uuid:p' }, onsetDateTime: '2027-01', recordedDate: '2026-10-31', note: [{ text: '2030-01-01' }] }), entry({ resourceType: 'Appointment', id: 'b', start: '2027-02-01T10:00:00Z' }), entry({ resourceType: 'Immunization', id: 'c', status: 'completed', patient: { reference: 'urn:uuid:p' }, occurrenceDateTime: '2026-11-01T09:00:00+10:00', protocolApplied: [{ doseNumberString: '2028' }] })), { now: NOW }); - expect(texts(out.items.e3)).toEqual(['A date in the future: Condition.onsetDateTime = 2027-01', 'A date in the future: Condition.note[0].text = 2030-01-01', 'No narrative']); + // a note's text is not a date, whatever it looks like + expect(texts(out.items.e3)).toEqual(['A date in the future: Condition.onsetDateTime = 2027-01', 'No narrative']); expect(kinds(out.items.e3)[0]).toBe('attention'); expect(kinds(out.items.e4)).toEqual(['ok', 'narrative']); // 9am in Brisbane is 23:00 UTC the day before: not the future; a dose number of 2028 is not a date expect(kinds(out.items.e5)).toEqual(['ok', 'narrative']); }); + test('dates about the future by their nature: period ends, expiry and due dates, schedules; and only real dates', async () => { + const d = doc(section(['a', 'b', 'c']), + entry({ resourceType: 'Immunization', id: 'a', status: 'completed', patient: { reference: 'urn:uuid:p' }, occurrenceDateTime: '2026-10-01', + expirationDate: '2027-06-30', identifier: [{ value: '2030-05', period: { start: '2026-01-01', end: '2031-01-01' } }], + extension: [{ url: 'http://x.org/next', valueDateTime: '2027-01-01' }] }), + entry({ resourceType: 'MedicationStatement', id: 'b', status: 'active', subject: { reference: 'urn:uuid:p' }, + effectivePeriod: { start: '2026-01-01', end: '2027-12-31' }, dosage: [{ timing: { event: ['2027-01-01'], repeat: { boundsPeriod: { start: '2027-01-01' } } } }] }), + entry({ resourceType: 'Goal', id: 'c', lifecycleStatus: 'completed', subject: { reference: 'urn:uuid:p' }, startDate: '2027-02-01', target: [{ dueDate: '2027-03-01' }] })); + d.entry[1].resource.name = [{ family: 'X', period: { end: '2030-01-01' } }]; + const out = await analyseDocument(d, { now: NOW }); + expect(kinds(out.items.e1)[0]).toBe('ok'); + expect(kinds(out.items.e3)).toEqual(['extension', 'narrative']); + expect(kinds(out.items.e4)).toEqual(['ok', 'narrative']); + // a completed goal that starts in the future; its due date is a due date + expect(texts(out.items.e5)).toEqual(['A date in the future: Goal.startDate = 2027-02-01', 'No narrative']); + }); + + test('requests, plans and events still to come may be in the future; once over, they may not', async () => { + const out = await analyseDocument(doc(section(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']), + entry({ resourceType: 'ServiceRequest', id: 'a', status: 'active', subject: { reference: 'urn:uuid:p' }, occurrenceDateTime: '2027-01-01' }), + entry({ resourceType: 'ServiceRequest', id: 'b', status: 'completed', subject: { reference: 'urn:uuid:p' }, occurrenceDateTime: '2027-01-01' }), + entry({ resourceType: 'Procedure', id: 'c', status: 'preparation', subject: { reference: 'urn:uuid:p' }, performedDateTime: '2027-01-01' }), + entry({ resourceType: 'Procedure', id: 'd', status: 'completed', subject: { reference: 'urn:uuid:p' }, performedDateTime: '2027-01-01' }), + entry({ resourceType: 'CarePlan', id: 'e', status: 'active', subject: { reference: 'urn:uuid:p' }, period: { start: '2027-01-01' } }), + entry({ resourceType: 'Goal', id: 'f', lifecycleStatus: 'active', subject: { reference: 'urn:uuid:p' }, startDate: '2027-01-01' }), + entry({ resourceType: 'ImmunizationRecommendation', id: 'g', patient: { reference: 'urn:uuid:p' }, date: '2027-01-01' }), + entry({ resourceType: 'Encounter', id: 'h', status: 'planned', subject: { reference: 'urn:uuid:p' }, period: { start: '2027-01-01' } }), + entry({ resourceType: 'MedicationStatement', id: 'i', status: 'intended', subject: { reference: 'urn:uuid:p' }, effectiveDateTime: '2027-01-01' })), { now: NOW }); + const future = (k) => out.items[k].filter(f => f.kind === 'attention').map(f => f.text); + expect(future('e3')).toEqual([]); + expect(future('e4')).toEqual(['A date in the future: ServiceRequest.occurrenceDateTime = 2027-01-01']); + expect(future('e5')).toEqual([]); + expect(future('e6')).toEqual(['A date in the future: Procedure.performedDateTime = 2027-01-01']); + for (const k of ['e7', 'e8', 'e9', 'e10', 'e11']) { + expect(future(k)).toEqual([]); + } + }); + test('a date with no time zone is in the future only if it is everywhere', () => { expect(earliest('2026-11-01')).toBe(Date.UTC(2026, 10, 1) - 14 * 3600000); expect(earliest('2026-11-01T10:00:00Z')).toBe(Date.UTC(2026, 10, 1, 10)); diff --git a/tests/library/memory-guard.test.js b/tests/library/memory-guard.test.js index fc9d6869..c2da3984 100644 --- a/tests/library/memory-guard.test.js +++ b/tests/library/memory-guard.test.js @@ -78,4 +78,74 @@ describe('memory-guard', () => { setRss(9.5 * GB); expect(memoryGuard.overAbortThreshold('test')).toBe(9.5 * GB); }); + + test('a gc function is available even without --expose-gc', () => { + expect(typeof memoryGuard.getGc()).toBe('function'); + }); + + test('while refusing, a forced GC that brings memory down lets the request through', () => { + setRss(8.5 * GB); + const gc = memoryGuard.getGc(); + expect(gc).toBeTruthy(); + // first request: over threshold, GC runs at once; pretend it freed enough + rssSpy.mockImplementation(() => 7 * GB); + memoryGuard._state.sampleRss = 8.5 * GB; + memoryGuard._state.sampleTime = Date.now(); // the cached sample says "over" + const next = jest.fn(); + memoryGuard.middleware({ gcInterval: 1000, heapUsed: () => 2 * GB })({ path: '/r4/x' }, fakeRes(), next); + expect(next).toHaveBeenCalled(); + }); + + test('no forced GC when the JS heap is smaller than the amount over the threshold', () => { + // 8.5 GB of RSS against an 8 GB threshold, but only 0.2 GB of it is JS heap: collecting + // can't get under the threshold, so don't stop the process to try + const log = { info: jest.fn(), warn: jest.fn() }; + setRss(8.5 * GB); + const res = fakeRes(); + memoryGuard.middleware({ log, gcInterval: 1000, heapUsed: () => 0.2 * GB })({ path: '/r4/x' }, res, () => {}); + expect(res.statusCode).toBe(503); + expect(log.info.mock.calls.some(c => /no forced GC - the JS heap/.test(c[0]))).toBe(true); + expect(log.info.mock.calls.some(c => /forced GC while refusing/.test(c[0]))).toBe(false); + }); + + test('a forced GC that leaves memory over the threshold backs off, up to the maximum', () => { + const log = { info: jest.fn(), warn: jest.fn() }; + let now = 1000000; + const nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); + try { + setRss(8.5 * GB); // and it stays there: the GCs free nothing + const mw = memoryGuard.middleware({ log, gcInterval: 1000, gcMaxInterval: 4000, heapUsed: () => 2 * GB }); + const gcs = () => log.info.mock.calls.filter(c => /forced GC while refusing/.test(c[0])).length; + const at = (t) => { now = 1000000 + t; memoryGuard._state.sampleTime = 0; mw({ path: '/r4/x' }, fakeRes(), () => {}); }; + at(0); // GC, still over: next in 2s + expect(gcs()).toBe(1); + at(1500); // too soon + expect(gcs()).toBe(1); + at(2000); // GC: next in 4s + at(5000); // too soon + expect(gcs()).toBe(2); + at(6000); // GC: next in 4s, the maximum + at(10000); // GC + expect(gcs()).toBe(4); + // recovering resets the wait: the next spell of refusing starts at gcInterval again + setRss(7 * GB); + at(10500); + setRss(8.5 * GB); + at(11000); // 1s after the last GC: allowed again + expect(gcs()).toBe(5); + } finally { + nowSpy.mockRestore(); + } + }); + + test('logs when refusing starts and when it stops', () => { + const log = { info: jest.fn(), warn: jest.fn() }; + const mw = memoryGuard.middleware({ log, gcInterval: 60000 }); + setRss(8.5 * GB); + mw({ path: '/r4/x' }, fakeRes(), () => {}); + expect(log.warn.mock.calls.some(c => /refusing new requests/.test(c[0]))).toBe(true); + setRss(7 * GB); + mw({ path: '/r4/x' }, fakeRes(), () => {}); + expect(log.warn.mock.calls.some(c => /accepting requests again after \d+s \(1 refused\)/.test(c[0]))).toBe(true); + }); }); diff --git a/tests/packages/package-crawler-versions.test.js b/tests/packages/package-crawler-versions.test.js index 46bd2aed..b5e75498 100644 --- a/tests/packages/package-crawler-versions.test.js +++ b/tests/packages/package-crawler-versions.test.js @@ -157,6 +157,76 @@ describe('crawler: one copy of each id#version', () => { expect(files.filter(f => f.includes('.tmp-'))).toEqual([]); }); + const tmpFiles = () => fs.readdirSync(path.join(env.tmp, 'mirror')).filter(f => f.includes('.tmp-')); + + test('a failed mirror write leaves no partial file, and stores nothing', async () => { + const write = fs.writeFileSync; + const spy = jest.spyOn(fs, 'writeFileSync').mockImplementationOnce((file, data) => { + write(file, data.subarray(0, 10)); // a partial write, then the disk fills + throw new Error('ENOSPC: no space left on device'); + }); + try { + await feedItem(env.crawler, 'ex.w#1.0.0', direct('ex.w', '1.0.0'), tarball('ex.w', '1.0.0')); + } finally { + spy.mockRestore(); + } + expect(fs.readdirSync(path.join(env.tmp, 'mirror'))).not.toContain('ex.w-1.0.0.tgz'); + expect(tmpFiles()).toEqual([]); + expect(await rows('ex.w')).toEqual([]); + }); + + test('a new package whose commit fails part way leaves no rows and no file', async () => { + env.crawler.insertRelatedData = jest.fn(async () => { throw new Error('insert failed'); }); + await feedItem(env.crawler, 'ex.t#1.0.0', direct('ex.t', '1.0.0'), tarball('ex.t', '1.0.0')); + expect(await rows('ex.t')).toEqual([]); + expect(await env.all("SELECT Id FROM Packages WHERE Id = 'ex.t'")).toEqual([]); + expect(fs.readdirSync(path.join(env.tmp, 'mirror'))).not.toContain('ex.t-1.0.0.tgz'); + expect(tmpFiles()).toEqual([]); + }); + + test('a replacement whose commit fails part way leaves the old row and file', async () => { + const old = tarball('ex.u', '1.0.0', 'simplifier'); + await feedItem(env.crawler, 'ex.u#1.0.0', simplifier('ex.u', '1.0.0'), old); + const before = await rows('ex.u'); + // fails after replaceVersion has already updated the PackageVersions row + env.crawler.insertRelatedData = jest.fn(async () => { throw new Error('insert failed'); }); + await feedItem(env.crawler, 'ex.u#1.0.0', direct('ex.u', '1.0.0'), tarball('ex.u', '1.0.0', 'direct')); + expect(await rows('ex.u')).toEqual(before); + expect(await env.all(`SELECT Version FROM PackageFHIRVersions WHERE PackageVersionKey = ${before[0].PackageVersionKey}`)).toEqual([{ Version: '4.0.1' }]); + expect(mirror('ex.u', '1.0.0')).toEqual(old); + expect(tmpFiles()).toEqual([]); + }); + + test('a replacement whose file cannot be put in place is rolled back', async () => { + const old = tarball('ex.v', '1.0.0', 'simplifier'); + await feedItem(env.crawler, 'ex.v#1.0.0', simplifier('ex.v', '1.0.0'), old); + const before = await rows('ex.v'); + const spy = jest.spyOn(fs, 'renameSync').mockImplementationOnce(() => { throw new Error('EIO: rename failed'); }); + try { + await feedItem(env.crawler, 'ex.v#1.0.0', direct('ex.v', '1.0.0'), tarball('ex.v', '1.0.0', 'direct')); + } finally { + spy.mockRestore(); + } + expect(await rows('ex.v')).toEqual(before); + expect(mirror('ex.v', '1.0.0')).toEqual(old); + expect(tmpFiles()).toEqual([]); + }); + + test('stale temporary mirror files are cleaned up, recent ones and packages are left', () => { + const dir = path.join(env.tmp, 'mirror'); + const stale = path.join(dir, 'ex.x-1.0.0.tgz.tmp-1'); + const recent = path.join(dir, 'ex.y-1.0.0.tgz.tmp-1'); + const pkg = path.join(dir, 'ex.z-1.0.0.tgz'); + for (const f of [stale, recent, pkg]) { + fs.writeFileSync(f, 'x'); + } + const old = new Date(Date.now() - 2 * 60 * 60 * 1000); + fs.utimesSync(stale, old, old); + fs.utimesSync(pkg, old, old); + expect(env.crawler.cleanMirrorTempFiles()).toBe(1); + expect(fs.readdirSync(dir).sort()).toEqual(['ex.y-1.0.0.tgz.tmp-1', 'ex.z-1.0.0.tgz']); + }); + test('a second direct copy from another GUID is an error, and ignored', async () => { const buf = tarball('ex.c', '1.0.0', 'first'); await feedItem(env.crawler, 'ex.c#1.0.0', direct('ex.c', '1.0.0'), buf); diff --git a/tests/tx/cache-control.test.js b/tests/tx/cache-control.test.js index 19b9adfd..d5573dbf 100644 --- a/tests/tx/cache-control.test.js +++ b/tests/tx/cache-control.test.js @@ -50,6 +50,17 @@ describe('$cache-control routing (scaffolding)', () => { expect(p.valueId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); }); + test('mode=start echoes the issued cache-id in the X-Cache-Id response header', async () => { + const res = await request(app) + .post(BASE) + .query({ mode: 'start' }) + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .send(emptyBody); + const p = (res.body.parameter || []).find(x => x.name === 'cache-id'); + expect(res.headers['x-cache-id']).toBe(p.valueId); + }); + test('GET mode=start resolves (browsable, like the other operations)', async () => { const res = await request(app) .get(BASE) diff --git a/tests/tx/expand-designation-param.test.js b/tests/tx/expand-designation-param.test.js new file mode 100644 index 00000000..e84ae045 --- /dev/null +++ b/tests/tx/expand-designation-param.test.js @@ -0,0 +1,45 @@ +/** + * The $expand designation parameter (L3): a token, system|code, or a bare code that matches a + * designation's language or its use code from any system. A bare language code used to match + * nothing, so designation=nl-NL dropped every designation + */ + +const { ValueSetExpander } = require('../../tx/workers/expand'); + +function expander(designations) { + const worker = { internalLimit: 1000, externalLimit: 1000 }; + return new ValueSetExpander(worker, { hasDesignations: designations.length > 0, designations }); +} + +const nl = { language: { code: 'nl-NL' }, value: 'Glucose [massa/volume] in serum of plasma' }; +const fr = { language: { code: 'fr-FR' }, value: 'Glucose [Masse/Volume] Sérum/Plasma ; Numérique' }; +const de = { language: { code: 'de-DE' }, value: 'Glucose [Masse/Vol] in Serum oder Plasma' }; +const shortName = { use: { system: 'http://loinc.org', code: 'SHORTNAME' }, value: 'Glucose SerPl-mCnc' }; + +describe('$expand designation parameter', () => { + test('bare language codes keep those languages only', () => { + const exp = expander(['nl-NL', 'fr-FR']); + expect([nl, fr, de, shortName].filter(d => exp.useDesignation(d))).toEqual([nl, fr]); + }); + + test('a bare code also matches a use code, from any system', () => { + const exp = expander(['SHORTNAME']); + expect([nl, fr, de, shortName].filter(d => exp.useDesignation(d))).toEqual([shortName]); + }); + + test('|code is the same as a bare code', () => { + const exp = expander(['|de-DE']); + expect([nl, fr, de, shortName].filter(d => exp.useDesignation(d))).toEqual([de]); + }); + + test('system|code still works, for languages and uses', () => { + const exp = expander(['urn:ietf:bcp:47|fr-FR', 'http://loinc.org|SHORTNAME']); + expect([nl, fr, de, shortName].filter(d => exp.useDesignation(d))).toEqual([fr, shortName]); + expect(expander(['http://other.org|SHORTNAME']).useDesignation(shortName)).toBe(false); + }); + + test('no designation parameter keeps them all', () => { + const exp = expander([]); + expect([nl, fr, de, shortName].every(d => exp.useDesignation(d))).toBe(true); + }); +}); diff --git a/tests/tx/expand-exclude-version.test.js b/tests/tx/expand-exclude-version.test.js new file mode 100644 index 00000000..9f92127f --- /dev/null +++ b/tests/tx/expand-exclude-version.test.js @@ -0,0 +1,89 @@ +/** + * S11: include one version of a code system, exclude another whole version + * (e.g. US Edition minus International). The excluded version's codes used to + * be listed with iteratorAll(), which hierarchical providers (SNOMED CT, LOINC, + * ...) refuse ("Must override") - so the expansion failed with a 500. Now the + * excluded version is asked about each included code instead (locate()). + */ + +const { ValueSetExpander } = require('../../tx/workers/expand'); + +const SYSTEM = 'http://example.org/cs'; + +// a provider that can locate codes but can't list them +function fakeProvider(version, codes) { + return { + system: () => SYSTEM, + version: () => version, + iteratorAll: async () => { throw new Error('Must override'); }, + locate: async (code) => codes.includes(code) ? { context: { code }, message: null } : { context: null, message: undefined } + }; +} + +function makeExpander(excludedCs) { + const worker = { + internalLimit: 1000, + externalLimit: 1000, + checkAndYield: async () => {}, + findCodeSystem: async () => excludedCs, + checkSupplements: () => {}, + opContext: { log: () => {} } + }; + const exp = new ValueSetExpander(worker, { versionsMatch: true }); + exp.checkResourceCanonicalStatus = () => {}; + exp.excludeSpecialCase = true; + return exp; +} + +describe('Exclude a whole other version of the included code system', () => { + test('does not list the excluded version, and excludes the codes it knows', async () => { + const v1 = fakeProvider('1', ['a', 'b']); + const exp = makeExpander(v1); + const expansion = {}; + await exp.excludeCodes({ system: SYSTEM, version: '1' }, 'ValueSet.compose.exclude[0]', { url: 'http://example.org/vs', vurl: 'http://example.org/vs' }, null, expansion, false, false); + + expect(await exp.isExcluded(SYSTEM, '2', 'a')).toBe(true); + expect(await exp.isExcluded(SYSTEM, '2', 'b')).toBe(true); + expect(await exp.isExcluded(SYSTEM, '2', 'c')).toBe(false); + expect(await exp.isExcluded('http://example.org/other', '2', 'a')).toBe(false); + expect(expansion.parameter.find(p => p.name === 'used-codesystem').valueUri).toBe(SYSTEM + '|1'); + }); + + test('a provider that fails to locate a code does not exclude it', async () => { + const v1 = fakeProvider('1', []); + v1.locate = async () => { throw new Error('boom'); }; + const exp = makeExpander(v1); + await exp.excludeCodes({ system: SYSTEM, version: '1' }, 'x', { url: 'u', vurl: 'u' }, null, {}, false, false); + expect(await exp.isExcluded(SYSTEM, '2', 'a')).toBe(false); + }); + + test('outside the special case, a whole-system exclude still excludes by system', async () => { + const v1 = fakeProvider('1', []); + const exp = makeExpander(v1); + exp.excludeSpecialCase = false; + exp.params = { versionsMatch: false }; + await exp.excludeCodes({ system: SYSTEM }, 'x', { url: 'u', vurl: 'u' }, null, {}, false, false); + expect(await exp.isExcluded(SYSTEM, '1', 'anything')).toBe(true); + }); + + test('a system + value set exclude walks the value set, not the code system', async () => { + const cs = fakeProvider('1', ['a', 'b', 'c']); + cs.isNotClosed = () => false; + cs.listSupplements = () => []; + const exp = makeExpander(cs); + exp.excludeSpecialCase = false; + exp.params = { versionsMatch: false }; + exp.doingVersion = false; + exp.worker.pinValueSet = u => u; + exp.worker.findValueSet = async () => ({ isContained: true }); + exp.worker.makeVurl = () => null; + exp.worker.deadCheck = () => {}; + exp.expandValueSet = async () => ({ expansion: { contains: [ + { system: SYSTEM, code: 'a' }, { system: SYSTEM, code: 'z' }, { system: 'http://example.org/other', code: 'b' }] } }); + await exp.excludeCodes({ system: SYSTEM, valueSet: ['http://example.org/vs2'] }, 'x', { url: 'u', vurl: 'u' }, null, {}, false, false); + expect(await exp.isExcluded(SYSTEM, '1', 'a')).toBe(true); + expect(await exp.isExcluded(SYSTEM, '1', 'b')).toBe(false); // in the value set, but another system + expect(await exp.isExcluded(SYSTEM, '1', 'c')).toBe(false); // in the code system, not the value set + expect(await exp.isExcluded(SYSTEM, '1', 'z')).toBe(false); // in the value set, not the code system + }); +}); diff --git a/tests/tx/implicit-valuesets.test.js b/tests/tx/implicit-valuesets.test.js new file mode 100644 index 00000000..62bc91b1 --- /dev/null +++ b/tests/tx/implicit-valuesets.test.js @@ -0,0 +1,104 @@ +// Which loaded version an implicit value set url is drawn from (S10): the generic form from +// the default version, the edition form from the latest version of that edition, the version +// form from that version - whatever order the versions were registered in. +const { chooseImplicitValueSetFactory } = require('../../tx/library/implicit-valuesets'); + +// A stand-in for a SNOMED CT factory: recognises the three forms for its edition and version +function edition(editionId, date) { + const version = `http://snomed.info/sct/${editionId}/version/${date}`; + return { + label: `${editionId}/${date}`, + system: () => 'http://snomed.info/sct', + version: () => version, + implicitValueSetForm(url) { + const q = url.indexOf('?fhir_vs'); + if (q === -1) return null; + const base = url.substring(0, q); + if (base === 'http://snomed.info/sct') return { form: 'generic' }; + if (base === `http://snomed.info/sct/${editionId}`) return { form: 'edition' }; + if (base === version) return { form: 'version' }; + return null; + } + }; +} + +describe('choosing the version for an implicit value set', () => { + const intl = edition('900000000000207008', '20260701'); + const us2023 = edition('731000124108', '20230301'); + const us2025 = edition('731000124108', '20250901'); + const us2024 = edition('731000124108', '20240301'); + const other = { system: () => 'http://loinc.org', version: () => '2.83' }; // no implicitValueSetForm + + // registered out of date order, and each factory under more than one key, as the library does + const registered = [us2024, us2025, other, intl, us2023, us2024, intl]; + const choose = (url, defaultFactory = intl) => chooseImplicitValueSetFactory(registered, () => defaultFactory, url); + + test('the generic form comes from the default version', () => { + expect(choose('http://snomed.info/sct?fhir_vs=isa/73211009')).toBe(intl); + expect(choose('http://snomed.info/sct?fhir_vs', us2023)).toBe(us2023); + }); + + test('the edition form comes from the latest version of that edition', () => { + expect(choose('http://snomed.info/sct/731000124108?fhir_vs=isa/73211009')).toBe(us2025); + expect(choose('http://snomed.info/sct/900000000000207008?fhir_vs')).toBe(intl); + }); + + test('the version form comes from that version only', () => { + expect(choose('http://snomed.info/sct/731000124108/version/20230301?fhir_vs=isa/73211009')).toBe(us2023); + expect(choose('http://snomed.info/sct/731000124108/version/20240301?fhir_vs')).toBe(us2024); + }); + + test('the generic form follows the version the request asks for (system-version and the like)', () => { + const asked = (v) => () => v; + const url = 'http://snomed.info/sct?fhir_vs=isa/73211009'; + expect(chooseImplicitValueSetFactory(registered, () => intl, url, asked('http://snomed.info/sct/731000124108/version/20240301'))).toBe(us2024); + // an edition with no version: the latest version of it + expect(chooseImplicitValueSetFactory(registered, () => intl, url, asked('http://snomed.info/sct/731000124108'))).toBe(us2025); + // a version that is not loaded: the default, and building the value set reports what is missing + expect(chooseImplicitValueSetFactory(registered, () => intl, url, asked('http://snomed.info/sct/83821000000107'))).toBe(intl); + expect(chooseImplicitValueSetFactory(registered, () => intl, url, asked(null))).toBe(intl); + }); + + test('the requested version does not override an edition or version named in the url', () => { + const askedIntl = () => 'http://snomed.info/sct/900000000000207008'; + expect(chooseImplicitValueSetFactory(registered, () => intl, 'http://snomed.info/sct/731000124108?fhir_vs', askedIntl)).toBe(us2025); + expect(chooseImplicitValueSetFactory(registered, () => intl, 'http://snomed.info/sct/731000124108/version/20230301?fhir_vs', askedIntl)).toBe(us2023); + }); + + test('an edition or version that is not loaded, or a url that is not an implicit value set, chooses nothing', () => { + expect(choose('http://snomed.info/sct/83821000000107?fhir_vs')).toBeNull(); + expect(choose('http://snomed.info/sct/731000124108/version/20990101?fhir_vs')).toBeNull(); + expect(choose('http://hl7.org/fhir/ValueSet/example')).toBeNull(); + }); +}); + +describe('TxParameters.requestedSystemVersion', () => { + const { TxParameters } = require('../../tx/params'); + const { TestUtilities } = require('../test-utilities'); + let langDefs; + let i18n; + beforeAll(async () => { + langDefs = await TestUtilities.loadLanguageDefinitions(); + i18n = await TestUtilities.loadTranslations(langDefs); + }); + const params = () => new TxParameters(langDefs, i18n, false); + const SCT = 'http://snomed.info/sct'; + + test('force-system-version wins over system-version, which wins over check-system-version', () => { + const p = params(); + p.seeVersionRule(SCT + '|http://snomed.info/sct/731000124108', false, 'check'); + expect(p.requestedSystemVersion(SCT)).toBe('http://snomed.info/sct/731000124108'); + p.seeVersionRule(SCT + '|http://snomed.info/sct/900000000000207008', false, 'default'); + expect(p.requestedSystemVersion(SCT)).toBe('http://snomed.info/sct/900000000000207008'); + p.seeVersionRule(SCT + '|http://snomed.info/sct/83821000000107', false, 'override'); + expect(p.requestedSystemVersion(SCT)).toBe('http://snomed.info/sct/83821000000107'); + }); + + test('none asked for, another system, or a value set version rule: null', () => { + const p = params(); + expect(p.requestedSystemVersion(SCT)).toBeNull(); + p.seeVersionRule('http://loinc.org|2.83', false, 'default'); + p.seeVersionRule(SCT + '|20250901', true, 'default'); // default-valueset-version + expect(p.requestedSystemVersion(SCT)).toBeNull(); + }); +}); diff --git a/tests/tx/test-cases.test.js b/tests/tx/test-cases.test.js index 7e39d6bd..4c81c65d 100644 --- a/tests/tx/test-cases.test.js +++ b/tests/tx/test-cases.test.js @@ -907,6 +907,14 @@ describe('language', () => { await runTest({"suite":"language","test":"language-echo-en-designation"}, "4.0"); }); + it("language-echo-en-designation-bare" + 'R5', async () => { + await runTest({"suite":"language","test":"language-echo-en-designation-bare"}, "5.0"); + }); + + it("language-echo-en-designation-bare" + 'R4', async () => { + await runTest({"suite":"language","test":"language-echo-en-designation-bare"}, "4.0"); + }); + it("language-echo-en-designations" + 'R5', async () => { await runTest({"suite":"language","test":"language-echo-en-designations"}, "5.0"); }); @@ -1123,7 +1131,7 @@ describe('language2', () => { }); describe('global-lang-pack', () => { - // Global language packs: a CodeSystem supplement marked with the codesystem-supplement-type extension, value global-lang-pack, is used for the code system it supplements whether or not it is referenced; any other supplement is only used when it is referenced + // Global language packs: a language pack (codesystem-supplement-type = lang-pack) marked with codesystem-supplement-global = true is used for the code system it supplements whether or not it is referenced; any other supplement is only used when it is referenced it("glp-cs-validate-de" + 'R5', async () => { await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-de"}, "5.0"); @@ -1181,6 +1189,14 @@ describe('global-lang-pack', () => { await runTest({"suite":"global-lang-pack","test":"glp-lookup-de"}, "4.0"); }); + it("glp-lookup-nl-unreferenced" + 'R5', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-lookup-nl-unreferenced"}, "5.0"); + }); + + it("glp-lookup-nl-unreferenced" + 'R4', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-lookup-nl-unreferenced"}, "4.0"); + }); + it("glp-lookup-other" + 'R5', async () => { await runTest({"suite":"global-lang-pack","test":"glp-lookup-other"}, "5.0"); }); @@ -1205,6 +1221,280 @@ describe('global-lang-pack', () => { await runTest({"suite":"global-lang-pack","test":"glp-expand-en"}, "4.0"); }); + it("glp-expand-nl-unreferenced" + 'R5', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-expand-nl-unreferenced"}, "5.0"); + }); + + it("glp-expand-nl-unreferenced" + 'R4', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-expand-nl-unreferenced"}, "4.0"); + }); + + it("glp-own-expand-de" + 'R5', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-expand-de"}, "5.0"); + }); + + it("glp-own-expand-de" + 'R4', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-expand-de"}, "4.0"); + }); + + it("glp-own-lookup-de" + 'R5', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-lookup-de"}, "5.0"); + }); + + it("glp-own-lookup-de" + 'R4', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-lookup-de"}, "4.0"); + }); + + it("glp-own-validate-pack-display-de" + 'R5', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-validate-pack-display-de"}, "5.0"); + }); + + it("glp-own-validate-pack-display-de" + 'R4', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-validate-pack-display-de"}, "4.0"); + }); + +}); + +describe('extension-codes', () => { + // Extension-codes supplements: a code system reserves a code space (codesystem-reservedCodeSpace) for users to define their own codes, and a CodeSystem supplement of type extension-codes defines codes in it, with displays. A global one (codesystem-supplement-global) is used whether or not it is referenced, as if the code system defined the codes; one that is not global is only used when it is referenced. A code in the reserved code space that nothing in scope defines is valid, with the code space's default display (default-display, or User-assigned) + + it("ec-lookup-global" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-global"}, "5.0"); + }); + + it("ec-lookup-global" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-global"}, "4.0"); + }); + + it("ec-lookup-local-referenced" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-local-referenced"}, "5.0"); + }); + + it("ec-lookup-local-referenced" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-local-referenced"}, "4.0"); + }); + + it("ec-lookup-local-unreferenced" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-local-unreferenced"}, "5.0"); + }); + + it("ec-lookup-local-unreferenced" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-local-unreferenced"}, "4.0"); + }); + + it("ec-lookup-undefined" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-undefined"}, "5.0"); + }); + + it("ec-lookup-undefined" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-undefined"}, "4.0"); + }); + + it("ec-lookup-default-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-default-display"}, "5.0"); + }); + + it("ec-lookup-default-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-default-display"}, "4.0"); + }); + + it("ec-validate-global" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global"}, "5.0"); + }); + + it("ec-validate-global" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global"}, "4.0"); + }); + + it("ec-validate-global-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-display"}, "5.0"); + }); + + it("ec-validate-global-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-display"}, "4.0"); + }); + + it("ec-validate-global-wrong-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-validate-global-wrong-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-wrong-display"}, "4.0"); + }); + + it("ec-validate-global-wrong-display-lenient" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-wrong-display-lenient"}, "5.0"); + }); + + it("ec-validate-global-wrong-display-lenient" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-wrong-display-lenient"}, "4.0"); + }); + + it("ec-validate-local-referenced" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced"}, "5.0"); + }); + + it("ec-validate-local-referenced" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced"}, "4.0"); + }); + + it("ec-validate-local-referenced-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced-display"}, "5.0"); + }); + + it("ec-validate-local-referenced-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced-display"}, "4.0"); + }); + + it("ec-validate-local-referenced-wrong-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-validate-local-referenced-wrong-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced-wrong-display"}, "4.0"); + }); + + it("ec-validate-local-unreferenced" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-unreferenced"}, "5.0"); + }); + + it("ec-validate-local-unreferenced" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-unreferenced"}, "4.0"); + }); + + it("ec-validate-undefined" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-undefined"}, "5.0"); + }); + + it("ec-validate-undefined" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-undefined"}, "4.0"); + }); + + it("ec-validate-default-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-default-display"}, "5.0"); + }); + + it("ec-validate-default-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-default-display"}, "4.0"); + }); + + it("ec-validate-outside" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-outside"}, "5.0"); + }); + + it("ec-validate-outside" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-outside"}, "4.0"); + }); + + it("ec-validate-supplement-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-supplement-display"}, "5.0"); + }); + + it("ec-validate-supplement-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-supplement-display"}, "4.0"); + }); + + it("ec-expand-enumerated" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-enumerated"}, "5.0"); + }); + + it("ec-expand-enumerated" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-enumerated"}, "4.0"); + }); + + it("ec-expand-enumerated-referenced" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-enumerated-referenced"}, "5.0"); + }); + + it("ec-expand-enumerated-referenced" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-enumerated-referenced"}, "4.0"); + }); + + it("ec-expand-all" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-all"}, "5.0"); + }); + + it("ec-expand-all" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-all"}, "4.0"); + }); + + it("ec-validate-global-referenced-wrong-display" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-referenced-wrong-display"}, "5.0"); + }); + + it("ec-validate-global-referenced-wrong-display" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-referenced-wrong-display"}, "4.0"); + }); + + it("ec-validate-global-referenced-wrong-display-lenient" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-referenced-wrong-display-lenient"}, "5.0"); + }); + + it("ec-validate-global-referenced-wrong-display-lenient" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-referenced-wrong-display-lenient"}, "4.0"); + }); + + it("ec-lookup-global-referenced" + 'R5', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-global-referenced"}, "5.0"); + }); + + it("ec-lookup-global-referenced" + 'R4', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-global-referenced"}, "4.0"); + }); + +}); + +describe('extension-codes-3166', () => { + // Extension-codes supplements for ISO 3166, whose user-assigned codes are its reserved code space. In the tx.fhir.org mode, since no general suite tests ISO 3166 + + it("ec-3166-validate-global" + 'R5', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-global"}, "5.0"); + }); + + it("ec-3166-validate-global" + 'R4', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-global"}, "4.0"); + }); + + it("ec-3166-validate-global-wrong-display" + 'R5', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-3166-validate-global-wrong-display" + 'R4', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-global-wrong-display"}, "4.0"); + }); + + it("ec-3166-validate-local-referenced" + 'R5', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-local-referenced"}, "5.0"); + }); + + it("ec-3166-validate-local-referenced" + 'R4', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-local-referenced"}, "4.0"); + }); + + it("ec-3166-validate-local-referenced-wrong-display" + 'R5', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-3166-validate-local-referenced-wrong-display" + 'R4', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-local-referenced-wrong-display"}, "4.0"); + }); + + it("ec-3166-expand" + 'R5', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-expand"}, "5.0"); + }); + + it("ec-3166-expand" + 'R4', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-expand"}, "4.0"); + }); + + it("ec-3166-expand-referenced" + 'R5', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-expand-referenced"}, "5.0"); + }); + + it("ec-3166-expand-referenced" + 'R4', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-expand-referenced"}, "4.0"); + }); + }); describe('extensions', () => { @@ -1306,6 +1596,54 @@ describe('extensions', () => { await runTest({"suite":"extensions","test":"validate-code-inactive"}, "4.0"); }); + it("lookup-supplements-hierarchy-child" + 'R5', async () => { + await runTest({"suite":"extensions","test":"lookup-supplements-hierarchy-child"}, "5.0"); + }); + + it("lookup-supplements-hierarchy-child" + 'R4', async () => { + await runTest({"suite":"extensions","test":"lookup-supplements-hierarchy-child"}, "4.0"); + }); + + it("lookup-supplements-hierarchy-grandchild" + 'R5', async () => { + await runTest({"suite":"extensions","test":"lookup-supplements-hierarchy-grandchild"}, "5.0"); + }); + + it("lookup-supplements-hierarchy-grandchild" + 'R4', async () => { + await runTest({"suite":"extensions","test":"lookup-supplements-hierarchy-grandchild"}, "4.0"); + }); + + it("validate-supplements-grandchild-wrong-display" + 'R5', async () => { + await runTest({"suite":"extensions","test":"validate-supplements-grandchild-wrong-display"}, "5.0"); + }); + + it("validate-supplements-grandchild-wrong-display" + 'R4', async () => { + await runTest({"suite":"extensions","test":"validate-supplements-grandchild-wrong-display"}, "4.0"); + }); + + it("validate-supplement-nolang-display" + 'R5', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-display"}, "5.0"); + }); + + it("validate-supplement-nolang-display" + 'R4', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-display"}, "4.0"); + }); + + it("validate-supplement-nolang-designation" + 'R5', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-designation"}, "5.0"); + }); + + it("validate-supplement-nolang-designation" + 'R4', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-designation"}, "4.0"); + }); + + it("validate-supplement-nolang-other-lang" + 'R5', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-other-lang"}, "5.0"); + }); + + it("validate-supplement-nolang-other-lang" + 'R4', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-other-lang"}, "4.0"); + }); + }); describe('validation', () => { @@ -4807,6 +5145,14 @@ describe('translate2', () => { await runTest({"suite":"translate2","test":"translate-6b"}, "4.0"); }); + it("translate-reverse-r5+" + 'R5', async () => { + await runTest({"suite":"translate2","test":"translate-reverse-r5+"}, "5.0"); + }); + + it("translate-reverse-r5+-a" + 'R5', async () => { + await runTest({"suite":"translate2","test":"translate-reverse-r5+-a"}, "5.0"); + }); + it("translate-reverse-r4" + 'R4', async () => { await runTest({"suite":"translate2","test":"translate-reverse-r4"}, "4.0"); }); @@ -5529,6 +5875,30 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-lookup-part-mappings"}, "4.0"); }); + it("loinc-lookup-part-parent" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-part-parent"}, "5.0"); + }); + + it("loinc-lookup-part-parent" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-part-parent"}, "4.0"); + }); + + it("loinc-lookup-part-languages" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-part-languages"}, "5.0"); + }); + + it("loinc-lookup-part-languages" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-part-languages"}, "4.0"); + }); + + it("loinc-lookup-code-languages" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-code-languages"}, "5.0"); + }); + + it("loinc-lookup-code-languages" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-code-languages"}, "4.0"); + }); + it("loinc-lookup-answer-race" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-lookup-answer-race"}, "5.0"); }); @@ -5633,6 +6003,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-all-limited"}, "4.0"); }); + it("loinc-expand-all-limited-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-all-limited-total"}, "5.0"); + }); + + it("loinc-expand-all-limited-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-all-limited-total"}, "4.0"); + }); + it("loinc-expand-enum-bad" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-enum-bad"}, "5.0"); }); @@ -5649,6 +6027,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-status"}, "4.0"); }); + it("loinc-expand-status-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-status-total"}, "5.0"); + }); + + it("loinc-expand-status-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-status-total"}, "4.0"); + }); + it("loinc-expand-parent" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-parent"}, "5.0"); }); @@ -5665,6 +6051,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-class-regex"}, "4.0"); }); + it("loinc-expand-class-regex-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-regex-total"}, "5.0"); + }); + + it("loinc-expand-class-regex-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-regex-total"}, "4.0"); + }); + it("loinc-expand-prop-component" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-component"}, "5.0"); }); @@ -5681,6 +6075,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-method"}, "4.0"); }); + it("loinc-expand-prop-method-part" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-method-part"}, "5.0"); + }); + + it("loinc-expand-prop-method-part" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-method-part"}, "4.0"); + }); + it("loinc-expand-prop-component-str" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-component-str"}, "5.0"); }); @@ -5697,6 +6099,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-order-obs"}, "4.0"); }); + it("loinc-expand-prop-order-obs-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-order-obs-total"}, "5.0"); + }); + + it("loinc-expand-prop-order-obs-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-order-obs-total"}, "4.0"); + }); + it("loinc-expand-concept-is-a" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-concept-is-a"}, "5.0"); }); @@ -5705,6 +6115,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-concept-is-a"}, "4.0"); }); + it("loinc-expand-implicit-part" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-implicit-part"}, "5.0"); + }); + + it("loinc-expand-implicit-part" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-implicit-part"}, "4.0"); + }); + it("loinc-expand-copyright" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-copyright"}, "5.0"); }); @@ -5713,6 +6131,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-copyright"}, "4.0"); }); + it("loinc-expand-copyright-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-copyright-total"}, "5.0"); + }); + + it("loinc-expand-copyright-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-copyright-total"}, "4.0"); + }); + it("loinc-expand-scale-type" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-scale-type"}, "5.0"); }); @@ -5721,6 +6147,30 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-scale-type"}, "4.0"); }); + it("loinc-expand-scale-type-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-total"}, "5.0"); + }); + + it("loinc-expand-scale-type-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-total"}, "4.0"); + }); + + it("loinc-expand-scale-type-part" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-part"}, "5.0"); + }); + + it("loinc-expand-scale-type-part" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-part"}, "4.0"); + }); + + it("loinc-expand-scale-type-part-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-part-total"}, "5.0"); + }); + + it("loinc-expand-scale-type-part-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-part-total"}, "4.0"); + }); + it("loinc-validate-enum-good" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-validate-enum-good"}, "5.0"); }); @@ -5801,6 +6251,86 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-bad"}, "4.0"); }); + it("loinc-validate-filter-scale-type-part-good" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-part-good"}, "5.0"); + }); + + it("loinc-validate-filter-scale-type-part-good" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-part-good"}, "4.0"); + }); + + it("loinc-validate-filter-scale-type-part-bad" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-part-bad"}, "5.0"); + }); + + it("loinc-validate-filter-scale-type-part-bad" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-part-bad"}, "4.0"); + }); + + it("loinc-expand-class-scale" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-scale"}, "5.0"); + }); + + it("loinc-expand-class-scale" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-scale"}, "4.0"); + }); + + it("loinc-expand-class-scale-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-scale-total"}, "5.0"); + }); + + it("loinc-expand-class-scale-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-scale-total"}, "4.0"); + }); + + it("loinc-expand-class-part-scale" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-part-scale"}, "5.0"); + }); + + it("loinc-expand-class-part-scale" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-part-scale"}, "4.0"); + }); + + it("loinc-expand-class-part-scale-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-part-scale-total"}, "5.0"); + }); + + it("loinc-expand-class-part-scale-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-part-scale-total"}, "4.0"); + }); + + it("loinc-validate-filter-class-scale-good" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-good"}, "5.0"); + }); + + it("loinc-validate-filter-class-scale-good" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-good"}, "4.0"); + }); + + it("loinc-validate-filter-class-scale-bad-scale" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-bad-scale"}, "5.0"); + }); + + it("loinc-validate-filter-class-scale-bad-scale" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-bad-scale"}, "4.0"); + }); + + it("loinc-validate-filter-class-scale-bad-class" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-bad-class"}, "5.0"); + }); + + it("loinc-validate-filter-class-scale-bad-class" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-bad-class"}, "4.0"); + }); + + it("loinc-validate-filter-class-part-scale-good" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-part-scale-good"}, "5.0"); + }); + + it("loinc-validate-filter-class-part-scale-good" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-part-scale-good"}, "4.0"); + }); + it("loinc-expand-list-request-parameters" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-expand-list-request-parameters"}, "5.0"); }); @@ -5857,6 +6387,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-filter-dockind-request-parameters"}, "4.0"); }); + it("loinc-expand-prop-dockind-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-dockind-total"}, "5.0"); + }); + + it("loinc-expand-prop-dockind-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-dockind-total"}, "4.0"); + }); + it("loinc-validate-filter-dockind-type-good" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-validate-filter-dockind-type-good"}, "5.0"); }); @@ -5881,6 +6419,14 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-filter-no-method"}, "4.0"); }); + it("loinc-expand-filter-no-method-total" + 'R5', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-filter-no-method-total"}, "5.0"); + }); + + it("loinc-expand-filter-no-method-total" + 'R4', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-filter-no-method-total"}, "4.0"); + }); + it("loinc-validate-filter-no-method-good" + 'R5', async () => { await runTest({"suite":"loinc","test":"loinc-validate-filter-no-method-good"}, "5.0"); }); @@ -6070,6 +6616,22 @@ describe('snomed', () => { await runTest({"suite":"snomed","test":"snomed-expand-isa"}, "4.0"); }); + it("snomed-expand-include-exclude" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-include-exclude"}, "5.0"); + }); + + it("snomed-expand-include-exclude" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-include-exclude"}, "4.0"); + }); + + it("snomed-expand-include-exclude-multi" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-include-exclude-multi"}, "5.0"); + }); + + it("snomed-expand-include-exclude-multi" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-include-exclude-multi"}, "4.0"); + }); + it("snomed-expand-count-all" + 'R5', async () => { await runTest({"suite":"snomed","test":"snomed-expand-count-all"}, "5.0"); }); @@ -6762,16 +7324,301 @@ describe('snomed', () => { await runTest({"suite":"snomed","test":"snomed-translate-loinc-system"}, "5.0"); }); - it("snomed-translate-loinc-system" + 'R4', async () => { - await runTest({"suite":"snomed","test":"snomed-translate-loinc-system"}, "4.0"); + it("snomed-translate-loinc-system" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-translate-loinc-system"}, "4.0"); + }); + + it("snomed-translate-loinc-system-reverse" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-translate-loinc-system-reverse"}, "5.0"); + }); + + it("snomed-translate-loinc-system-reverse" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-translate-loinc-system-reverse"}, "4.0"); + }); + + it("snomed-expand-property-inactive-rel" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-property-inactive-rel"}, "5.0"); + }); + + it("snomed-expand-property-inactive-rel" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-property-inactive-rel"}, "4.0"); + }); + + it("snomed-lookup-inactive-rel" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-inactive-rel"}, "5.0"); + }); + + it("snomed-lookup-inactive-rel" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-inactive-rel"}, "4.0"); + }); + + it("snomed-lookup-inactive-isa-child" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-inactive-isa-child"}, "5.0"); + }); + + it("snomed-lookup-inactive-isa-child" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-inactive-isa-child"}, "4.0"); + }); + + it("snomed-lookup-concrete" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-concrete"}, "5.0"); + }); + + it("snomed-lookup-concrete" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-concrete"}, "4.0"); + }); + + it("snomed-lookup-all-properties" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-all-properties"}, "5.0"); + }); + + it("snomed-lookup-all-properties" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-all-properties"}, "4.0"); + }); + + it("snomed-lookup-normal-form" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-normal-form"}, "5.0"); + }); + + it("snomed-lookup-normal-form" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-normal-form"}, "4.0"); + }); + + it("snomed-validate-implied-refset-map" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-implied-refset-map"}, "5.0"); + }); + + it("snomed-validate-implied-refset-map" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-implied-refset-map"}, "4.0"); + }); + + it("snomed-expand-implicit-ecl" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-ecl"}, "5.0"); + }); + + it("snomed-expand-implicit-ecl" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-ecl"}, "4.0"); + }); + + it("snomed-expand-implicit-edition" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-edition"}, "5.0"); + }); + + it("snomed-expand-implicit-edition" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-edition"}, "4.0"); + }); + + it("snomed-expand-implicit-version" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-version"}, "5.0"); + }); + + it("snomed-expand-implicit-version" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-version"}, "4.0"); + }); + + it("snomed-expand-implicit-system-version" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-system-version"}, "5.0"); + }); + + it("snomed-expand-implicit-system-version" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-system-version"}, "4.0"); + }); + + it("snomed-expand-isa-inactive-isa" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-isa-inactive-isa"}, "5.0"); + }); + + it("snomed-expand-isa-inactive-isa" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-isa-inactive-isa"}, "4.0"); + }); + + it("subsumes-inactive-isa" + 'R5', async () => { + await runTest({"suite":"snomed","test":"subsumes-inactive-isa"}, "5.0"); + }); + + it("subsumes-inactive-isa" + 'R4', async () => { + await runTest({"suite":"snomed","test":"subsumes-inactive-isa"}, "4.0"); + }); + + it("snomed-expand-concept-in" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-concept-in"}, "5.0"); + }); + + it("snomed-expand-concept-in" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-concept-in"}, "4.0"); + }); + + it("snomed-validate-concept-in" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-concept-in"}, "5.0"); + }); + + it("snomed-validate-concept-in" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-concept-in"}, "4.0"); + }); + + it("snomed-expand-code-in" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-code-in"}, "5.0"); + }); + + it("snomed-expand-code-in" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-code-in"}, "4.0"); + }); + + it("snomed-validate-code-in" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-code-in"}, "5.0"); + }); + + it("snomed-validate-code-in" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-code-in"}, "4.0"); + }); + + it("snomed-expand-descendent-leaf" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-descendent-leaf"}, "5.0"); + }); + + it("snomed-expand-descendent-leaf" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-descendent-leaf"}, "4.0"); + }); + + it("snomed-expand-is-not-a" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-is-not-a"}, "5.0"); + }); + + it("snomed-expand-is-not-a" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-is-not-a"}, "4.0"); + }); + + it("snomed-expand-display-regex" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-display-regex"}, "5.0"); + }); + + it("snomed-expand-display-regex" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-display-regex"}, "4.0"); + }); + + it("snomed-validate-display-regex" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-display-regex"}, "5.0"); + }); + + it("snomed-validate-display-regex" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-display-regex"}, "4.0"); + }); + + it("snomed-expand-display-regex-nl" + 'R5', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-display-regex-nl"}, "5.0"); + }); + + it("snomed-expand-display-regex-nl" + 'R4', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-display-regex-nl"}, "4.0"); + }); + +}); + +describe('sct-ec', () => { + // Extension-codes supplements for the SNOMED CT test edition: a supplement gives some post-coordinated expressions human displays. In the snomed mode, and a suite of its own, since its global supplement would change the snomed suite's other tests + + it("ec-sct-lookup-global" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-global"}, "5.0"); + }); + + it("ec-sct-lookup-global" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-global"}, "4.0"); + }); + + it("ec-sct-lookup-local-referenced" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-local-referenced"}, "5.0"); + }); + + it("ec-sct-lookup-local-referenced" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-local-referenced"}, "4.0"); + }); + + it("ec-sct-lookup-local-unreferenced" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-local-unreferenced"}, "5.0"); + }); + + it("ec-sct-lookup-local-unreferenced" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-local-unreferenced"}, "4.0"); + }); + + it("ec-sct-validate-global" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global"}, "5.0"); + }); + + it("ec-sct-validate-global" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global"}, "4.0"); + }); + + it("ec-sct-validate-global-display" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-display"}, "5.0"); + }); + + it("ec-sct-validate-global-display" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-display"}, "4.0"); + }); + + it("ec-sct-validate-global-generated-display" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-generated-display"}, "5.0"); + }); + + it("ec-sct-validate-global-generated-display" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-generated-display"}, "4.0"); + }); + + it("ec-sct-validate-global-wrong-display" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-sct-validate-global-wrong-display" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-wrong-display"}, "4.0"); + }); + + it("ec-sct-validate-global-wrong-display-lenient" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-wrong-display-lenient"}, "5.0"); + }); + + it("ec-sct-validate-global-wrong-display-lenient" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-wrong-display-lenient"}, "4.0"); + }); + + it("ec-sct-validate-local-referenced" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-referenced"}, "5.0"); + }); + + it("ec-sct-validate-local-referenced" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-referenced"}, "4.0"); + }); + + it("ec-sct-validate-local-referenced-wrong-display" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-sct-validate-local-referenced-wrong-display" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-referenced-wrong-display"}, "4.0"); + }); + + it("ec-sct-validate-local-unreferenced" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-unreferenced"}, "5.0"); + }); + + it("ec-sct-validate-local-unreferenced" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-unreferenced"}, "4.0"); + }); + + it("ec-sct-expand" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-expand"}, "5.0"); + }); + + it("ec-sct-expand" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-expand"}, "4.0"); }); - it("snomed-translate-loinc-system-reverse" + 'R5', async () => { - await runTest({"suite":"snomed","test":"snomed-translate-loinc-system-reverse"}, "5.0"); + it("ec-sct-expand-referenced" + 'R5', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-expand-referenced"}, "5.0"); }); - it("snomed-translate-loinc-system-reverse" + 'R4', async () => { - await runTest({"suite":"snomed","test":"snomed-translate-loinc-system-reverse"}, "4.0"); + it("ec-sct-expand-referenced" + 'R4', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-expand-referenced"}, "4.0"); }); }); @@ -7659,6 +8506,14 @@ describe('sct-ecl', () => { await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term"}, "4.0"); }); + it("snomed-expand-ecl-description-filter-term-no-d" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term-no-d"}, "5.0"); + }); + + it("snomed-expand-ecl-description-filter-term-no-d" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term-no-d"}, "4.0"); + }); + it("snomed-expand-ecl-description-filter-term-set" + 'R5', async () => { await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term-set"}, "5.0"); }); @@ -8483,6 +9338,158 @@ describe('sct-ecl', () => { await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-dotted"}, "4.0"); }); + it("snomed-expand-ecl-refinement-inactive-rel" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-refinement-inactive-rel"}, "5.0"); + }); + + it("snomed-expand-ecl-refinement-inactive-rel" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-refinement-inactive-rel"}, "4.0"); + }); + + it("snomed-expand-ecl-descendants-inactive-isa" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-descendants-inactive-isa"}, "5.0"); + }); + + it("snomed-expand-ecl-descendants-inactive-isa" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-descendants-inactive-isa"}, "4.0"); + }); + + it("snomed-expand-ecl-description-filter-wild-all-language" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-wild-all-language"}, "5.0"); + }); + + it("snomed-expand-ecl-description-filter-wild-all-language" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-wild-all-language"}, "4.0"); + }); + + it("snomed-expand-ecl-description-filter-wild-nl" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-wild-nl"}, "5.0"); + }); + + it("snomed-expand-ecl-description-filter-wild-nl" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-wild-nl"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-decimal-literal" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal-literal"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-decimal-literal" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal-literal"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-decimal" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-decimal" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-decimal-equals" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal-equals"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-decimal-equals" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal-equals"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-grouped" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-grouped"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-grouped" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-grouped"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-type-mismatch" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-type-mismatch"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-type-mismatch" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-type-mismatch"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-string" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-string" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-string-not-equals" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string-not-equals"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-string-not-equals" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string-not-equals"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-string-wild" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string-wild"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-string-wild" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string-wild"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-boolean-true" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-boolean-true"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-boolean-true" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-boolean-true"}, "4.0"); + }); + + it("snomed-expand-ecl-concrete-boolean-false" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-boolean-false"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-boolean-false" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-boolean-false"}, "4.0"); + }); + + it("snomed-validate-ecl-concrete-code-in" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-validate-ecl-concrete-code-in"}, "5.0"); + }); + + it("snomed-validate-ecl-concrete-code-in" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-validate-ecl-concrete-code-in"}, "4.0"); + }); + + it("snomed-validate-ecl-concrete-code-out" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-validate-ecl-concrete-code-out"}, "5.0"); + }); + + it("snomed-validate-ecl-concrete-code-out" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-validate-ecl-concrete-code-out"}, "4.0"); + }); + + it("snomed-expand-ecl-memberOf-simple-map" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-memberOf-simple-map"}, "5.0"); + }); + + it("snomed-expand-ecl-memberOf-simple-map" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-memberOf-simple-map"}, "4.0"); + }); + + it("snomed-expand-ecl-memberOf-extended-map" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-memberOf-extended-map"}, "5.0"); + }); + + it("snomed-expand-ecl-memberOf-extended-map" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-memberOf-extended-map"}, "4.0"); + }); + + it("snomed-expand-ecl-member-filter-map-target" + 'R5', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-member-filter-map-target"}, "5.0"); + }); + + it("snomed-expand-ecl-member-filter-map-target" + 'R4', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-member-filter-map-target"}, "4.0"); + }); + }); describe('batch', () => { @@ -8504,6 +9511,30 @@ describe('batch', () => { await runTest({"suite":"batch","test":"batch-validate-bad"}, "4.0"); }); + it("cs-batch-validate" + 'R5', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate"}, "5.0"); + }); + + it("cs-batch-validate" + 'R4', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate"}, "4.0"); + }); + + it("cs-batch-validate-bad" + 'R5', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate-bad"}, "5.0"); + }); + + it("cs-batch-validate-bad" + 'R4', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate-bad"}, "4.0"); + }); + + it("cs-batch-validate-codings" + 'R5', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate-codings"}, "5.0"); + }); + + it("cs-batch-validate-codings" + 'R4', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate-codings"}, "4.0"); + }); + }); describe('omop', () => { @@ -12382,6 +13413,115 @@ describe('icf', () => { }); +describe('icf-ec', () => { + // Extension-codes supplements for ICF (2001a), whose qualified (post-coordinated) codes are a grammar: a supplement gives some of them human displays. In the icf mode, and a suite of its own, since its global supplement would change the icf suite's other tests + + it("ec-icf-lookup-global" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-global"}, "5.0"); + }); + + it("ec-icf-lookup-global" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-global"}, "4.0"); + }); + + it("ec-icf-lookup-local-referenced" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-local-referenced"}, "5.0"); + }); + + it("ec-icf-lookup-local-referenced" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-local-referenced"}, "4.0"); + }); + + it("ec-icf-lookup-local-unreferenced" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-local-unreferenced"}, "5.0"); + }); + + it("ec-icf-lookup-local-unreferenced" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-local-unreferenced"}, "4.0"); + }); + + it("ec-icf-validate-global" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global"}, "5.0"); + }); + + it("ec-icf-validate-global" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global"}, "4.0"); + }); + + it("ec-icf-validate-global-display" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-display"}, "5.0"); + }); + + it("ec-icf-validate-global-display" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-display"}, "4.0"); + }); + + it("ec-icf-validate-global-generated-display" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-generated-display"}, "5.0"); + }); + + it("ec-icf-validate-global-generated-display" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-generated-display"}, "4.0"); + }); + + it("ec-icf-validate-global-wrong-display" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-icf-validate-global-wrong-display" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-wrong-display"}, "4.0"); + }); + + it("ec-icf-validate-global-wrong-display-lenient" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-wrong-display-lenient"}, "5.0"); + }); + + it("ec-icf-validate-global-wrong-display-lenient" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-wrong-display-lenient"}, "4.0"); + }); + + it("ec-icf-validate-local-referenced" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-referenced"}, "5.0"); + }); + + it("ec-icf-validate-local-referenced" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-referenced"}, "4.0"); + }); + + it("ec-icf-validate-local-referenced-wrong-display" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-icf-validate-local-referenced-wrong-display" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-referenced-wrong-display"}, "4.0"); + }); + + it("ec-icf-validate-local-unreferenced" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-unreferenced"}, "5.0"); + }); + + it("ec-icf-validate-local-unreferenced" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-unreferenced"}, "4.0"); + }); + + it("ec-icf-expand" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-expand"}, "5.0"); + }); + + it("ec-icf-expand" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-expand"}, "4.0"); + }); + + it("ec-icf-expand-referenced" + 'R5', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-expand-referenced"}, "5.0"); + }); + + it("ec-icf-expand-referenced" + 'R4', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-expand-referenced"}, "4.0"); + }); + +}); + describe('cached (forced caching)', () => { beforeAll(() => { setForcedCaching(true); }); afterAll(() => { setForcedCaching(false); }); @@ -12845,6 +13985,10 @@ describe('language', () => { await runTest({"suite":"language","test":"language-echo-en-designation"}, "5.0"); }); + it("language-echo-en-designation-bare" + 'R5-cached', async () => { + await runTest({"suite":"language","test":"language-echo-en-designation-bare"}, "5.0"); + }); + it("language-echo-en-designations" + 'R5-cached', async () => { await runTest({"suite":"language","test":"language-echo-en-designations"}, "5.0"); }); @@ -12934,69 +14078,215 @@ describe('language2', () => { await runTest({"suite":"language2","test":"validation-wrong-en-ende"}, "5.0"); }); - it("validation-wrong-en-none" + 'R5-cached', async () => { - await runTest({"suite":"language2","test":"validation-wrong-en-none"}, "5.0"); + it("validation-wrong-en-none" + 'R5-cached', async () => { + await runTest({"suite":"language2","test":"validation-wrong-en-none"}, "5.0"); + }); + + it("validation-wrong-none-en" + 'R5-cached', async () => { + await runTest({"suite":"language2","test":"validation-wrong-none-en"}, "5.0"); + }); + + it("validation-wrong-none-ende-N" + 'R5-cached', async () => { + await runTest({"suite":"language2","test":"validation-wrong-none-ende-N"}, "5.0"); + }); + + it("validation-wrong-none-ende" + 'R5-cached', async () => { + await runTest({"suite":"language2","test":"validation-wrong-none-ende"}, "5.0"); + }); + + it("validation-wrong-none-none" + 'R5-cached', async () => { + await runTest({"suite":"language2","test":"validation-wrong-none-none"}, "5.0"); + }); + +}); + +describe('global-lang-pack', () => { + // Global language packs: a language pack (codesystem-supplement-type = lang-pack) marked with codesystem-supplement-global = true is used for the code system it supplements whether or not it is referenced; any other supplement is only used when it is referenced + + it("glp-cs-validate-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-de"}, "5.0"); + }); + + it("glp-validate-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-validate-de"}, "5.0"); + }); + + it("glp-validate-nl-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-validate-nl-unreferenced"}, "5.0"); + }); + + it("glp-validate-nl-referenced" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-validate-nl-referenced"}, "5.0"); + }); + + it("glp-cs-validate-nl-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-nl-unreferenced"}, "5.0"); + }); + + it("glp-cs-validate-nl-referenced" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-nl-referenced"}, "5.0"); + }); + + it("glp-lookup-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-lookup-de"}, "5.0"); + }); + + it("glp-lookup-nl-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-lookup-nl-unreferenced"}, "5.0"); + }); + + it("glp-lookup-other" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-lookup-other"}, "5.0"); + }); + + it("glp-expand-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-expand-de"}, "5.0"); + }); + + it("glp-expand-en" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-expand-en"}, "5.0"); + }); + + it("glp-expand-nl-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-expand-nl-unreferenced"}, "5.0"); + }); + + it("glp-own-expand-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-expand-de"}, "5.0"); + }); + + it("glp-own-lookup-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-lookup-de"}, "5.0"); + }); + + it("glp-own-validate-pack-display-de" + 'R5-cached', async () => { + await runTest({"suite":"global-lang-pack","test":"glp-own-validate-pack-display-de"}, "5.0"); + }); + +}); + +describe('extension-codes', () => { + // Extension-codes supplements: a code system reserves a code space (codesystem-reservedCodeSpace) for users to define their own codes, and a CodeSystem supplement of type extension-codes defines codes in it, with displays. A global one (codesystem-supplement-global) is used whether or not it is referenced, as if the code system defined the codes; one that is not global is only used when it is referenced. A code in the reserved code space that nothing in scope defines is valid, with the code space's default display (default-display, or User-assigned) + + it("ec-lookup-global" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-global"}, "5.0"); + }); + + it("ec-lookup-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-local-referenced"}, "5.0"); + }); + + it("ec-lookup-local-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-local-unreferenced"}, "5.0"); + }); + + it("ec-lookup-undefined" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-undefined"}, "5.0"); + }); + + it("ec-lookup-default-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-default-display"}, "5.0"); + }); + + it("ec-validate-global" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global"}, "5.0"); + }); + + it("ec-validate-global-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-display"}, "5.0"); + }); + + it("ec-validate-global-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-validate-global-wrong-display-lenient" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-wrong-display-lenient"}, "5.0"); + }); + + it("ec-validate-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced"}, "5.0"); + }); + + it("ec-validate-local-referenced-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced-display"}, "5.0"); + }); + + it("ec-validate-local-referenced-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-validate-local-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-local-unreferenced"}, "5.0"); }); - it("validation-wrong-none-en" + 'R5-cached', async () => { - await runTest({"suite":"language2","test":"validation-wrong-none-en"}, "5.0"); + it("ec-validate-undefined" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-undefined"}, "5.0"); }); - it("validation-wrong-none-ende-N" + 'R5-cached', async () => { - await runTest({"suite":"language2","test":"validation-wrong-none-ende-N"}, "5.0"); + it("ec-validate-default-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-default-display"}, "5.0"); }); - it("validation-wrong-none-ende" + 'R5-cached', async () => { - await runTest({"suite":"language2","test":"validation-wrong-none-ende"}, "5.0"); + it("ec-validate-outside" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-outside"}, "5.0"); }); - it("validation-wrong-none-none" + 'R5-cached', async () => { - await runTest({"suite":"language2","test":"validation-wrong-none-none"}, "5.0"); + it("ec-validate-supplement-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-supplement-display"}, "5.0"); }); -}); + it("ec-expand-enumerated" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-enumerated"}, "5.0"); + }); -describe('global-lang-pack', () => { - // Global language packs: a CodeSystem supplement marked with the codesystem-supplement-type extension, value global-lang-pack, is used for the code system it supplements whether or not it is referenced; any other supplement is only used when it is referenced + it("ec-expand-enumerated-referenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-enumerated-referenced"}, "5.0"); + }); - it("glp-cs-validate-de" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-de"}, "5.0"); + it("ec-expand-all" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-expand-all"}, "5.0"); }); - it("glp-validate-de" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-validate-de"}, "5.0"); + it("ec-validate-global-referenced-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-referenced-wrong-display"}, "5.0"); }); - it("glp-validate-nl-unreferenced" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-validate-nl-unreferenced"}, "5.0"); + it("ec-validate-global-referenced-wrong-display-lenient" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-validate-global-referenced-wrong-display-lenient"}, "5.0"); }); - it("glp-validate-nl-referenced" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-validate-nl-referenced"}, "5.0"); + it("ec-lookup-global-referenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes","test":"ec-lookup-global-referenced"}, "5.0"); }); - it("glp-cs-validate-nl-unreferenced" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-nl-unreferenced"}, "5.0"); +}); + +describe('extension-codes-3166', () => { + // Extension-codes supplements for ISO 3166, whose user-assigned codes are its reserved code space. In the tx.fhir.org mode, since no general suite tests ISO 3166 + + it("ec-3166-validate-global" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-global"}, "5.0"); }); - it("glp-cs-validate-nl-referenced" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-cs-validate-nl-referenced"}, "5.0"); + it("ec-3166-validate-global-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-global-wrong-display"}, "5.0"); }); - it("glp-lookup-de" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-lookup-de"}, "5.0"); + it("ec-3166-validate-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-local-referenced"}, "5.0"); }); - it("glp-lookup-other" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-lookup-other"}, "5.0"); + it("ec-3166-validate-local-referenced-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-validate-local-referenced-wrong-display"}, "5.0"); }); - it("glp-expand-de" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-expand-de"}, "5.0"); + it("ec-3166-expand" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-expand"}, "5.0"); }); - it("glp-expand-en" + 'R5-cached', async () => { - await runTest({"suite":"global-lang-pack","test":"glp-expand-en"}, "5.0"); + it("ec-3166-expand-referenced" + 'R5-cached', async () => { + await runTest({"suite":"extension-codes-3166","test":"ec-3166-expand-referenced"}, "5.0"); }); }); @@ -13052,6 +14342,30 @@ describe('extensions', () => { await runTest({"suite":"extensions","test":"validate-code-inactive"}, "5.0"); }); + it("lookup-supplements-hierarchy-child" + 'R5-cached', async () => { + await runTest({"suite":"extensions","test":"lookup-supplements-hierarchy-child"}, "5.0"); + }); + + it("lookup-supplements-hierarchy-grandchild" + 'R5-cached', async () => { + await runTest({"suite":"extensions","test":"lookup-supplements-hierarchy-grandchild"}, "5.0"); + }); + + it("validate-supplements-grandchild-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"extensions","test":"validate-supplements-grandchild-wrong-display"}, "5.0"); + }); + + it("validate-supplement-nolang-display" + 'R5-cached', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-display"}, "5.0"); + }); + + it("validate-supplement-nolang-designation" + 'R5-cached', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-designation"}, "5.0"); + }); + + it("validate-supplement-nolang-other-lang" + 'R5-cached', async () => { + await runTest({"suite":"extensions","test":"validate-supplement-nolang-other-lang"}, "5.0"); + }); + }); describe('validation', () => { @@ -14841,6 +16155,14 @@ describe('translate2', () => { await runTest({"suite":"translate2","test":"translate-6b"}, "5.0"); }); + it("translate-reverse-r5+" + 'R5-cached', async () => { + await runTest({"suite":"translate2","test":"translate-reverse-r5+"}, "5.0"); + }); + + it("translate-reverse-r5+-a" + 'R5-cached', async () => { + await runTest({"suite":"translate2","test":"translate-reverse-r5+-a"}, "5.0"); + }); + it("translate-reverse-r4" + 'R4-cached', async () => { await runTest({"suite":"translate2","test":"translate-reverse-r4"}, "4.0"); }); @@ -15223,6 +16545,18 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-lookup-part-mappings"}, "5.0"); }); + it("loinc-lookup-part-parent" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-part-parent"}, "5.0"); + }); + + it("loinc-lookup-part-languages" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-part-languages"}, "5.0"); + }); + + it("loinc-lookup-code-languages" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-lookup-code-languages"}, "5.0"); + }); + it("loinc-lookup-answer-race" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-lookup-answer-race"}, "5.0"); }); @@ -15275,6 +16609,10 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-all-limited"}, "5.0"); }); + it("loinc-expand-all-limited-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-all-limited-total"}, "5.0"); + }); + it("loinc-expand-enum-bad" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-enum-bad"}, "5.0"); }); @@ -15283,6 +16621,10 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-status"}, "5.0"); }); + it("loinc-expand-status-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-status-total"}, "5.0"); + }); + it("loinc-expand-parent" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-parent"}, "5.0"); }); @@ -15291,6 +16633,10 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-class-regex"}, "5.0"); }); + it("loinc-expand-class-regex-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-regex-total"}, "5.0"); + }); + it("loinc-expand-prop-component" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-component"}, "5.0"); }); @@ -15299,6 +16645,10 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-method"}, "5.0"); }); + it("loinc-expand-prop-method-part" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-method-part"}, "5.0"); + }); + it("loinc-expand-prop-component-str" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-component-str"}, "5.0"); }); @@ -15307,18 +16657,42 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-prop-order-obs"}, "5.0"); }); + it("loinc-expand-prop-order-obs-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-order-obs-total"}, "5.0"); + }); + it("loinc-expand-concept-is-a" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-concept-is-a"}, "5.0"); }); + it("loinc-expand-implicit-part" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-implicit-part"}, "5.0"); + }); + it("loinc-expand-copyright" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-copyright"}, "5.0"); }); + it("loinc-expand-copyright-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-copyright-total"}, "5.0"); + }); + it("loinc-expand-scale-type" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-scale-type"}, "5.0"); }); + it("loinc-expand-scale-type-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-total"}, "5.0"); + }); + + it("loinc-expand-scale-type-part" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-part"}, "5.0"); + }); + + it("loinc-expand-scale-type-part-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-scale-type-part-total"}, "5.0"); + }); + it("loinc-validate-enum-good" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-validate-enum-good"}, "5.0"); }); @@ -15359,6 +16733,46 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-bad"}, "5.0"); }); + it("loinc-validate-filter-scale-type-part-good" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-part-good"}, "5.0"); + }); + + it("loinc-validate-filter-scale-type-part-bad" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-scale-type-part-bad"}, "5.0"); + }); + + it("loinc-expand-class-scale" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-scale"}, "5.0"); + }); + + it("loinc-expand-class-scale-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-scale-total"}, "5.0"); + }); + + it("loinc-expand-class-part-scale" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-part-scale"}, "5.0"); + }); + + it("loinc-expand-class-part-scale-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-class-part-scale-total"}, "5.0"); + }); + + it("loinc-validate-filter-class-scale-good" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-good"}, "5.0"); + }); + + it("loinc-validate-filter-class-scale-bad-scale" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-bad-scale"}, "5.0"); + }); + + it("loinc-validate-filter-class-scale-bad-class" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-scale-bad-class"}, "5.0"); + }); + + it("loinc-validate-filter-class-part-scale-good" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-validate-filter-class-part-scale-good"}, "5.0"); + }); + it("loinc-expand-list-request-parameters" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-expand-list-request-parameters"}, "5.0"); }); @@ -15387,6 +16801,10 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-filter-dockind-request-parameters"}, "5.0"); }); + it("loinc-expand-prop-dockind-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-prop-dockind-total"}, "5.0"); + }); + it("loinc-validate-filter-dockind-type-good" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-validate-filter-dockind-type-good"}, "5.0"); }); @@ -15399,6 +16817,10 @@ describe('loinc', () => { await runTest({"suite":"loinc","test":"loinc-expand-filter-no-method"}, "5.0"); }); + it("loinc-expand-filter-no-method-total" + 'R5-cached', async () => { + await runTest({"suite":"loinc","test":"loinc-expand-filter-no-method-total"}, "5.0"); + }); + it("loinc-validate-filter-no-method-good" + 'R5-cached', async () => { await runTest({"suite":"loinc","test":"loinc-validate-filter-no-method-good"}, "5.0"); }); @@ -15496,6 +16918,14 @@ describe('snomed', () => { await runTest({"suite":"snomed","test":"snomed-expand-isa"}, "5.0"); }); + it("snomed-expand-include-exclude" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-include-exclude"}, "5.0"); + }); + + it("snomed-expand-include-exclude-multi" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-include-exclude-multi"}, "5.0"); + }); + it("snomed-expand-count-all" + 'R5-cached', async () => { await runTest({"suite":"snomed","test":"snomed-expand-count-all"}, "5.0"); }); @@ -15848,6 +17278,151 @@ describe('snomed', () => { await runTest({"suite":"snomed","test":"snomed-translate-loinc-system-reverse"}, "5.0"); }); + it("snomed-expand-property-inactive-rel" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-property-inactive-rel"}, "5.0"); + }); + + it("snomed-lookup-inactive-rel" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-inactive-rel"}, "5.0"); + }); + + it("snomed-lookup-inactive-isa-child" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-inactive-isa-child"}, "5.0"); + }); + + it("snomed-lookup-concrete" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-concrete"}, "5.0"); + }); + + it("snomed-lookup-all-properties" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-all-properties"}, "5.0"); + }); + + it("snomed-lookup-normal-form" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-lookup-normal-form"}, "5.0"); + }); + + it("snomed-validate-implied-refset-map" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-implied-refset-map"}, "5.0"); + }); + + it("snomed-expand-implicit-ecl" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-ecl"}, "5.0"); + }); + + it("snomed-expand-implicit-edition" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-edition"}, "5.0"); + }); + + it("snomed-expand-implicit-version" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-version"}, "5.0"); + }); + + it("snomed-expand-implicit-system-version" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-implicit-system-version"}, "5.0"); + }); + + it("snomed-expand-isa-inactive-isa" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-isa-inactive-isa"}, "5.0"); + }); + + it("subsumes-inactive-isa" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"subsumes-inactive-isa"}, "5.0"); + }); + + it("snomed-expand-concept-in" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-concept-in"}, "5.0"); + }); + + it("snomed-validate-concept-in" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-concept-in"}, "5.0"); + }); + + it("snomed-expand-code-in" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-code-in"}, "5.0"); + }); + + it("snomed-validate-code-in" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-code-in"}, "5.0"); + }); + + it("snomed-expand-descendent-leaf" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-descendent-leaf"}, "5.0"); + }); + + it("snomed-expand-is-not-a" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-is-not-a"}, "5.0"); + }); + + it("snomed-expand-display-regex" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-display-regex"}, "5.0"); + }); + + it("snomed-validate-display-regex" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-validate-display-regex"}, "5.0"); + }); + + it("snomed-expand-display-regex-nl" + 'R5-cached', async () => { + await runTest({"suite":"snomed","test":"snomed-expand-display-regex-nl"}, "5.0"); + }); + +}); + +describe('sct-ec', () => { + // Extension-codes supplements for the SNOMED CT test edition: a supplement gives some post-coordinated expressions human displays. In the snomed mode, and a suite of its own, since its global supplement would change the snomed suite's other tests + + it("ec-sct-lookup-global" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-global"}, "5.0"); + }); + + it("ec-sct-lookup-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-local-referenced"}, "5.0"); + }); + + it("ec-sct-lookup-local-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-lookup-local-unreferenced"}, "5.0"); + }); + + it("ec-sct-validate-global" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global"}, "5.0"); + }); + + it("ec-sct-validate-global-display" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-display"}, "5.0"); + }); + + it("ec-sct-validate-global-generated-display" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-generated-display"}, "5.0"); + }); + + it("ec-sct-validate-global-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-sct-validate-global-wrong-display-lenient" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-global-wrong-display-lenient"}, "5.0"); + }); + + it("ec-sct-validate-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-referenced"}, "5.0"); + }); + + it("ec-sct-validate-local-referenced-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-sct-validate-local-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-validate-local-unreferenced"}, "5.0"); + }); + + it("ec-sct-expand" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-expand"}, "5.0"); + }); + + it("ec-sct-expand-referenced" + 'R5-cached', async () => { + await runTest({"suite":"sct-ec","test":"ec-sct-expand-referenced"}, "5.0"); + }); + }); describe('sct-ecl', () => { @@ -16293,6 +17868,10 @@ describe('sct-ecl', () => { await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term"}, "5.0"); }); + it("snomed-expand-ecl-description-filter-term-no-d" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term-no-d"}, "5.0"); + }); + it("snomed-expand-ecl-description-filter-term-set" + 'R5-cached', async () => { await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-term-set"}, "5.0"); }); @@ -16705,6 +18284,82 @@ describe('sct-ecl', () => { await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-dotted"}, "5.0"); }); + it("snomed-expand-ecl-refinement-inactive-rel" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-refinement-inactive-rel"}, "5.0"); + }); + + it("snomed-expand-ecl-descendants-inactive-isa" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-descendants-inactive-isa"}, "5.0"); + }); + + it("snomed-expand-ecl-description-filter-wild-all-language" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-wild-all-language"}, "5.0"); + }); + + it("snomed-expand-ecl-description-filter-wild-nl" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-description-filter-wild-nl"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-decimal-literal" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal-literal"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-decimal" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-decimal-equals" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-decimal-equals"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-grouped" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-grouped"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-type-mismatch" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-type-mismatch"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-string" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-string-not-equals" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string-not-equals"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-string-wild" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-string-wild"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-boolean-true" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-boolean-true"}, "5.0"); + }); + + it("snomed-expand-ecl-concrete-boolean-false" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-concrete-boolean-false"}, "5.0"); + }); + + it("snomed-validate-ecl-concrete-code-in" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-validate-ecl-concrete-code-in"}, "5.0"); + }); + + it("snomed-validate-ecl-concrete-code-out" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-validate-ecl-concrete-code-out"}, "5.0"); + }); + + it("snomed-expand-ecl-memberOf-simple-map" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-memberOf-simple-map"}, "5.0"); + }); + + it("snomed-expand-ecl-memberOf-extended-map" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-memberOf-extended-map"}, "5.0"); + }); + + it("snomed-expand-ecl-member-filter-map-target" + 'R5-cached', async () => { + await runTest({"suite":"sct-ecl","test":"snomed-expand-ecl-member-filter-map-target"}, "5.0"); + }); + }); describe('batch', () => { @@ -16718,6 +18373,18 @@ describe('batch', () => { await runTest({"suite":"batch","test":"batch-validate-bad"}, "5.0"); }); + it("cs-batch-validate" + 'R5-cached', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate"}, "5.0"); + }); + + it("cs-batch-validate-bad" + 'R5-cached', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate-bad"}, "5.0"); + }); + + it("cs-batch-validate-codings" + 'R5-cached', async () => { + await runTest({"suite":"batch","test":"cs-batch-validate-codings"}, "5.0"); + }); + }); describe('omop', () => { @@ -18696,6 +20363,63 @@ describe('icf', () => { }); +describe('icf-ec', () => { + // Extension-codes supplements for ICF (2001a), whose qualified (post-coordinated) codes are a grammar: a supplement gives some of them human displays. In the icf mode, and a suite of its own, since its global supplement would change the icf suite's other tests + + it("ec-icf-lookup-global" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-global"}, "5.0"); + }); + + it("ec-icf-lookup-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-local-referenced"}, "5.0"); + }); + + it("ec-icf-lookup-local-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-lookup-local-unreferenced"}, "5.0"); + }); + + it("ec-icf-validate-global" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global"}, "5.0"); + }); + + it("ec-icf-validate-global-display" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-display"}, "5.0"); + }); + + it("ec-icf-validate-global-generated-display" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-generated-display"}, "5.0"); + }); + + it("ec-icf-validate-global-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-wrong-display"}, "5.0"); + }); + + it("ec-icf-validate-global-wrong-display-lenient" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-global-wrong-display-lenient"}, "5.0"); + }); + + it("ec-icf-validate-local-referenced" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-referenced"}, "5.0"); + }); + + it("ec-icf-validate-local-referenced-wrong-display" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-referenced-wrong-display"}, "5.0"); + }); + + it("ec-icf-validate-local-unreferenced" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-validate-local-unreferenced"}, "5.0"); + }); + + it("ec-icf-expand" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-expand"}, "5.0"); + }); + + it("ec-icf-expand-referenced" + 'R5-cached', async () => { + await runTest({"suite":"icf-ec","test":"ec-icf-expand-referenced"}, "5.0"); + }); + +}); + }); }); diff --git a/tx/cs/cs-api.js b/tx/cs/cs-api.js index b3795410..e1496903 100644 --- a/tx/cs/cs-api.js +++ b/tx/cs/cs-api.js @@ -224,6 +224,17 @@ class CodeSystemProvider { return false; } + /** + * Why an expansion of all of the code system is not closed even though every code it defines + * is in it - e.g. it reserves code spaces for users to define their own codes + * (codesystem-reservedCodeSpace), and those codes can't be listed - or null when it is closed + * (as far as this is concerned: see isNotClosed() for grammars) + * @returns {string|null} + */ + openCodeSpaceReason() { + return null; + } + /** * returns false if the code system cannot be iterated / enumerated in any * meaningful sense - as distinct from isNotClosed(), which marks a code system @@ -694,10 +705,13 @@ class CodeSystemProvider { * it's only use is to be passed back to the CodeSystem provider so it can make use of it to organise the filter process * * @param {boolean} iterate true if the conceptSets that result from this will be iterated, and false if they'll be used to locate a single code + * @param {Languages|null} languages the languages the request named (TxParameters.requestedLanguages()), or null if it named none; + * for filters whose meaning depends on language, such as a regex on the display * @returns {FilterExecutionContext} filter * **/ - async getPrepContext(iterate) { return new FilterExecutionContext(iterate); } + // eslint-disable-next-line no-unused-vars + async getPrepContext(iterate, languages = null) { return new FilterExecutionContext(iterate); } /** diff --git a/tx/cs/cs-api.md b/tx/cs/cs-api.md index da57c309..a1d7999e 100644 --- a/tx/cs/cs-api.md +++ b/tx/cs/cs-api.md @@ -63,6 +63,7 @@ These methods describe the code system as a whole. All are synchronous. | `sourcePackage()` | `null` | The NPM package that contributed this code system, if known. | | `propertyDefinitions()` | `null` | The set of defined properties; override to expose them. | | `isNotClosed()` | `false` | Return `true` for grammar-based systems (e.g. UCUM) that cannot be fully enumerated. | +| `openCodeSpaceReason()` | `null` | Why an expansion of all of the code system isn't closed although it lists every code the system defines - e.g. the system reserves code spaces for user-defined codes (`codesystem-reservedCodeSpace`). The expansion is marked unclosed with this as the reason. | | `isCaseSensitive()` | `true` | Whether code comparisons are case-sensitive. | | `hasParents()` | `false` | Whether the code system has a concept hierarchy. | | `specialEnumeration()` | `null` | Override to nominate a substitute enumeration (used by UCUM). | diff --git a/tx/cs/cs-cs.js b/tx/cs/cs-cs.js index 37b14bfb..0f78291e 100644 --- a/tx/cs/cs-cs.js +++ b/tx/cs/cs-cs.js @@ -197,6 +197,16 @@ function providerClassFor(url) { } class FhirCodeSystemProvider extends BaseCSServices { + /** + * A code system that reserves code spaces for users (codesystem-reservedCodeSpace) has codes that + * aren't listed in an expansion of all of it + * @returns {string|null} + */ + openCodeSpaceReason() { + return this.codeSystem.reservedCodeSpaces().length > 0 ? + "The code system '" + this.system() + "' reserves code spaces for users to define their own codes, and those codes are not in the expansion" : null; + } + /** * The provider for a CodeSystem resource: a FhirCodeSystemProvider, or the subclass * registered for its url. Always use this rather than the constructor. @@ -1192,7 +1202,7 @@ class FhirCodeSystemProvider extends BaseCSServices { } // Supported hierarchy filters if ((prop === 'concept' || prop === 'code') && - ['is-a', 'descendent-of', 'is-not-a', 'child-of', 'in', '=', 'regex'].includes(op)) { + ['is-a', 'descendent-of', 'descendent-leaf', 'is-not-a', 'child-of', 'in', '=', 'regex'].includes(op)) { return true; } @@ -1510,7 +1520,7 @@ class FhirCodeSystemProvider extends BaseCSServices { */ _isCacheableFilter(prop, op) { if (prop === 'concept' || prop === 'code') { - return op === 'is-a' || op === 'descendent-of' || op === 'is-not-a' || op === 'regex'; + return op === 'is-a' || op === 'descendent-of' || op === 'descendent-leaf' || op === 'is-not-a' || op === 'regex'; } if (prop === 'child' && op === 'exists') { return true; @@ -1549,6 +1559,19 @@ class FhirCodeSystemProvider extends BaseCSServices { // Find all descendants of the specified code const includeRoot = (op === 'is-a'); await this._addDescendants(results, value, includeRoot); + } else if (op === 'descendent-leaf') { + // R5: the descendants that have no children of their own + if (this.codeSystem.getConceptByCode(value)) { + for (const code of this.codeSystem.getDescendants(value)) { + this.opContext.deadCheck('cs:conceptFilter:descendent-leaf'); + if (code !== value && this.codeSystem.getChildren(code).length === 0) { + const concept = this.codeSystem.getConceptByCode(code); + if (concept) { + results.add(concept, 0); + } + } + } + } } else if (op === 'child-of') { // Find all descendants of the specified code await this._addChildren(results, value); diff --git a/tx/cs/cs-loinc.js b/tx/cs/cs-loinc.js index 21bf39d5..1b898ccf 100644 --- a/tx/cs/cs-loinc.js +++ b/tx/cs/cs-loinc.js @@ -124,6 +124,9 @@ class LoincFilterHolder { } hasKey(key) { + if (this.keySet) { + return this.keySet.has(key); + } // Binary search since keys should be sorted let l = 0; let r = this.keys.length - 1; @@ -478,8 +481,11 @@ class LoincServices extends BaseCSServices { async #addRelationshipProperties(ctxt, props, params) { return new Promise((resolve, reject) => { + // DISTINCT: databases imported before the importer de-duplicated relationships have the + // same row more than once - a part listed twice under the same parent in + // ComponentHierarchyBySystem (on two paths) has two parent rows const sql = ` - SELECT RelationshipTypes.Description as Relationship, Codes.Code, Codes.Description as Value + SELECT DISTINCT RelationshipTypes.Description as Relationship, Codes.Code, Codes.Description as Value FROM Relationships, RelationshipTypes, Codes WHERE Relationships.SourceKey = ? AND Relationships.RelationshipTypeKey = RelationshipTypes.RelationshipTypeKey @@ -802,6 +808,13 @@ class LoincServices extends BaseCSServices { return true; } + // LOINC's own hierarchy filter: ancestor = part (or in parts), the codes the part subsumes + // in the multi-axial hierarchy. LOINC's implicit value sets (http://loinc.org/vs/[partcode]) + // are defined with it + if (prop === 'ancestor' && ['=', 'in'].includes(op)) { + return true; + } + // code filters (VSAC workaround) if (prop === 'code' && ['is-a', 'descendent-of', '='].includes(op)) { return true; @@ -1033,6 +1046,14 @@ class LoincServices extends BaseCSServices { else if (prop === 'concept' && ['is-a', 'descendent-of'].includes(op)) { ({ sql, lsql } = this.#hierarchyFilterSql(op, value)); } + // ancestor = X is LOINC's name for descendent-of X: "Ancestor includes parents transitively", + // and the part itself is not its own ancestor. in is the union over the parts + else if (prop === 'ancestor' && op === '=') { + ({ sql, lsql } = this.#hierarchyFilterSql('descendent-of', value)); + } + else if (prop === 'ancestor' && op === 'in') { + ({ sql, lsql } = this.#hierarchyFilterSql('descendent-of', null, this.#commaListOfCodes(value))); + } // Concept equal filter (workaround for VSAC misuse) else if (prop === 'concept' && op === '=') { sql = `SELECT CodeKey as Key FROM Codes @@ -1109,14 +1130,15 @@ class LoincServices extends BaseCSServices { * the file as a graph would, for example, put Glucose in Serum or Plasma under Urinalysis, * because Protein appears under both. */ - #hierarchyFilterSql(op, value) { - const code = this.#sqlWrapString(value); - const descendants = `SELECT DescendentKey as Key FROM Closure - WHERE AncestorKey IN (SELECT CodeKey FROM Codes WHERE Code = '${code}')`; + #hierarchyFilterSql(op, value, codeList = null) { + // one code, or (codeList) an already quoted, comma separated list of them + const codes = codeList || `'${this.#sqlWrapString(value)}'`; + const descendants = `SELECT DISTINCT DescendentKey as Key FROM Closure + WHERE AncestorKey IN (SELECT CodeKey FROM Codes WHERE Code IN (${codes}))`; const set = op === 'is-a' ? `${descendants} UNION - SELECT CodeKey as Key FROM Codes WHERE Code = '${code}'` + SELECT CodeKey as Key FROM Codes WHERE Code IN (${codes})` : descendants; return { sql: `${set} @@ -1173,8 +1195,19 @@ class LoincServices extends BaseCSServices { } async executeFilters(filterContext) { - - return filterContext.filters; + const filters = filterContext.filters; + if (filters.length <= 1) { + return filters; + } + // Every filter is already a list of keys in memory, so the include is their intersection, + // as one set: its size is the include's size (count=0 needs no walk, and so no limit), and + // the expander walks only the codes that are in it. In the first filter's order + const others = filters.slice(1).map(f => new Set(f.keys)); + const set = new LoincFilterHolder(); + set.keys = filters[0].keys.filter(k => others.every(o => o.has(k))); + set.keySet = new Set(set.keys); + set.lsql = filters.map(f => f.lsql).find(l => l) || 'intersection'; + return [set]; } async filterSize(filterContext, set) { @@ -1528,8 +1561,9 @@ class LoincServicesFactory extends CodeSystemFactoryProvider { } return new Promise((resolve, reject) => { + // DISTINCT, for databases imported with duplicate rows (see #addRelationshipProperties) const sql = ` - SELECT SourceKey, TargetKey FROM Relationships + SELECT DISTINCT SourceKey, TargetKey FROM Relationships WHERE RelationshipTypeKey = ${childRelKey} `; diff --git a/tx/cs/cs-rxnorm.js b/tx/cs/cs-rxnorm.js index 62f9bdcf..a02cbde3 100644 --- a/tx/cs/cs-rxnorm.js +++ b/tx/cs/cs-rxnorm.js @@ -1,8 +1,8 @@ const sqlite3 = require('sqlite3').verbose(); const assert = require('assert'); const { CodeSystem } = require('../library/codesystem'); -const { CodeSystemProvider, CodeSystemFactoryProvider } = require('./cs-api'); -const {Designations} = require("../library/designations"); +const { CodeSystemFactoryProvider } = require('./cs-api'); +const { BaseCSServices } = require('./cs-base'); const {validateArrayParameter, formatDateMMDDYYYY} = require("../../library/utilities"); // Context for RxNorm concepts @@ -12,9 +12,41 @@ class RxNormConcept { this.display = display; this.others = []; // Array of alternative displays (SY terms, etc.) this.archived = false; + this.suppress = null; // SUPPRESS of the display atom, when known: N, E, O (obsolete) or Y + } + + get obsolete() { + return this.suppress === 'O'; + } +} + +// Which RXNORM atom gives a concept its display: the atom with its main term type (IN, SCD, +// BN...), or failing that a prescribable name, an entry term, or a tall man synonym. SY atoms +// are never the display. As an SQL expression that is lowest for the best atom +const DISPLAY_RANK_SQL = `CASE TTY WHEN 'PSN' THEN 1 WHEN 'ET' THEN 2 WHEN 'TMSY' THEN 3 WHEN 'SY' THEN 4 ELSE 0 END`; +function displayRank(tty) { + switch (tty) { + case 'PSN': return 1; + case 'ET': return 2; + case 'TMSY': return 3; + case 'SY': return 4; + default: return 0; } } +// RxNorm's own relationships are the only ones used, for now (filters and $lookup). RXNREL +// holds every source's (MTHSPL, VANDF...), and they are all imported +const RELATIONSHIP_SAB = 'RXNORM'; + +// indexes the provider needs beyond what older imports made (see also import-rxnorm.module.js) +const RXNORM_LOAD_INDEXES = [ + 'CREATE INDEX IF NOT EXISTS idx_rxnstems_cui_stem ON RXNSTEMS(CUI, stem)', + 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rxcui2 ON RXNREL(RXCUI2, SAB)', + 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela_rxcui1 ON RXNREL(RELA, RXCUI1)', + 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela_rxaui1 ON RXNREL(RELA, RXAUI1)', + 'CREATE INDEX IF NOT EXISTS idx_rxnsty_rxcui ON RXNSTY(RXCUI)' +]; + // Filter holder for query building and iteration class RxNormFilterHolder { constructor() { @@ -68,7 +100,7 @@ class RxNormIteratorContext { } } -class RxNormServices extends CodeSystemProvider { +class RxNormServices extends BaseCSServices { constructor(opContext, supplements, db, sharedData, isNCI = false) { super(opContext, supplements); this.db = db; @@ -162,19 +194,7 @@ class RxNormServices extends CodeSystemProvider { if (ctxt && ctxt.archived) { return 'archived'; } - - // Check suppress flag - return new Promise((resolve, reject) => { - const sql = `SELECT suppress FROM rxnconso WHERE ${this.getCodeField()} = ? AND SAB = ? AND TTY <> 'SY'`; - - this.db.get(sql, [ctxt.code, this.getSAB()], (err, row) => { - if (err) { - reject(err); - } else { - resolve(row ? row.suppress === '1' ? 'suppressed' : null : null); - } - }); - }); + return (await this.#isObsolete(ctxt)) ? 'obsolete' : null; } async isInactive(context) { @@ -184,19 +204,24 @@ class RxNormServices extends CodeSystemProvider { if (ctxt && ctxt.archived) { return true; } + return this.#isObsolete(ctxt); + } - // Check suppress flag - return new Promise((resolve, reject) => { - const sql = `SELECT suppress FROM rxnconso WHERE ${this.getCodeField()} = ? AND SAB = ? AND TTY <> 'SY'`; - - this.db.get(sql, [ctxt.code, this.getSAB()], (err, row) => { - if (err) { - reject(err); - } else { - resolve(row ? row.suppress === '1' : false); - } - }); - }); + /** + * A concept is obsolete (inactive) when its display atom is: SUPPRESS = O + * @private + */ + async #isObsolete(ctxt) { + if (!ctxt) { + return false; + } + if (ctxt.suppress == null) { + const row = await this.#dbGet( + `SELECT SUPPRESS, MIN(${DISPLAY_RANK_SQL}) AS rnk FROM rxnconso WHERE ${this.getCodeField()} = ? AND SAB = ? AND TTY <> 'SY'`, + [ctxt.code, this.getSAB()]); + ctxt.suppress = row && row.SUPPRESS ? row.SUPPRESS : 'N'; + } + return ctxt.obsolete; } async isDeprecated(context) { @@ -248,7 +273,7 @@ class RxNormServices extends CodeSystemProvider { if (!code) return { context: null, message: 'Empty code' }; return new Promise((resolve, reject) => { - let sql = `SELECT STR, TTY FROM rxnconso WHERE ${this.getCodeField()} = ? AND SAB = ?`; + let sql = `SELECT STR, TTY, SUPPRESS FROM rxnconso WHERE ${this.getCodeField()} = ? AND SAB = ?`; this.db.all(sql, [code, this.getSAB()], (err, rows) => { if (err) { @@ -285,24 +310,68 @@ class RxNormServices extends CodeSystemProvider { const concept = new RxNormConcept(code); concept.archived = archived; + // the display is the best ranked atom (see DISPLAY_RANK_SQL), the rest are other names + let best = null; for (const row of rows) { - if (row.TTY === 'SY' || concept.display && concept.display) { - concept.others.push(row.STR.trim()); - } else { - concept.display = row.STR.trim(); + if (row.TTY !== 'SY' && (!best || displayRank(row.TTY) < displayRank(best.TTY))) { + best = row; + } + } + if (best) { + concept.display = best.STR.trim(); + if (!archived) { + concept.suppress = best.SUPPRESS || 'N'; + } + } + for (const row of rows) { + const str = row.STR.trim(); + if (row !== best && str !== concept.display && !concept.others.includes(str)) { + concept.others.push(str); } } return concept; } + /** + * The query for a set of concepts: one row per concept, with its display, whatever atoms + * the conditions matched on. Every condition is about the concept (its code), not the atom + * @param {string} conditions - SQL 'AND ...' conditions on rxnconso. + * @param {string} [from] - more tables to join (the text search stems) + * @returns {string} + * @private + */ + #conceptQuery(conditions, from = '') { + const cf = this.getCodeField(); + return `SELECT rxnconso.${cf} AS ${cf}, STR, SUPPRESS, MIN(${DISPLAY_RANK_SQL}) AS rnk FROM rxnconso${from} ` + + `WHERE SAB = $sab AND TTY <> 'SY' ${conditions} GROUP BY rxnconso.${cf}`; + } + + #conceptFromRow(row) { + const concept = new RxNormConcept(row[this.getCodeField()], row.STR); + concept.suppress = row.SUPPRESS || 'N'; + return concept; + } + + #dbGet(sql, params) { + return new Promise((resolve, reject) => { + this.db.get(sql, params, (err, row) => err ? reject(err) : resolve(row)); + }); + } + + #dbAll(sql, params) { + return new Promise((resolve, reject) => { + this.db.all(sql, params, (err, rows) => err ? reject(err) : resolve(rows)); + }); + } + // Iterator methods async iterator(context) { if (!context) { // Iterate all codes - const query = `SELECT ${this.getCodeField()}, STR FROM rxnconso WHERE SAB = ? AND TTY <> 'SY' ORDER BY ${this.getCodeField()}`; + const query = this.#conceptQuery('') + ` ORDER BY rxnconso.${this.getCodeField()}`; return new RxNormIteratorContext(query, { sab: this.getSAB() }, this.totalCodeCount); } else { // No hierarchical iteration for specific contexts in this implementation. The expander @@ -326,8 +395,7 @@ class RxNormServices extends CodeSystemProvider { const row = iteratorContext.results[iteratorContext.cursor]; iteratorContext.next(); - const concept = new RxNormConcept(row[this.getCodeField()], row.STR); - return concept; + return this.#conceptFromRow(row); } async #executeIterator(iteratorContext) { @@ -345,36 +413,42 @@ class RxNormServices extends CodeSystemProvider { } // Filter support - async doesFilter(prop, op, value) { - - - let propUC = prop.toUpperCase(); + // + // Every filter is a condition on the concept (rxnconso. IN ...), not on the atom, + // so a concept with several atoms (SCD + PSN + TMSY...) is one result, whichever atoms matched. + // Filters in one include are ANDed. TTY, STY, SAB, concept (or code), and the relationships + // (REL or RELA, with CUI: or AUI: values) all take = or in; in is the union - // TTY filters - if (propUC === 'TTY' && ['=', 'in'].includes(op)) { - return true; + /** + * @returns {string[]} the values of an = or in filter + * @private + */ + static #filterValues(op, value) { + if (op === 'in') { + return (value || '').split(',').map(v => v.trim()).filter(v => v); } + return value ? [value.trim()] : []; + } - // STY filter - if (propUC === 'STY' && op === '=') { - return true; - } + static #isRelValue(v) { + return v.startsWith('CUI:') || v.startsWith('AUI:'); + } - // SAB filter - if (propUC === 'SAB' && op === '=') { - return true; + async doesFilter(prop, op, value) { + if (!['=', 'in'].includes(op)) { + return false; } - - // Relationship filters (REL values like 'SY', 'RN', etc.) - if (this.rels.includes(prop) && op === '=' && (value.startsWith('CUI:') || value.startsWith('AUI:'))) { - return true; + const values = RxNormServices.#filterValues(op, value); + if (values.length === 0) { + return false; } - - // Relationship type filters (RELA values) - if (this.reltypes.includes(prop) && op === '=' && (value.startsWith('CUI:') || value.startsWith('AUI:'))) { + const propUC = prop.toUpperCase(); + if (['TTY', 'STY', 'SAB'].includes(propUC) || prop === 'concept' || prop === 'code') { return true; } - + if (this.rels.includes(prop) || this.reltypes.includes(prop)) { + return values.every(v => RxNormServices.#isRelValue(v)); + } return false; } @@ -384,55 +458,47 @@ class RxNormServices extends CodeSystemProvider { } async filter(filterContext, forIteration, prop, op, value) { - - const filter = new RxNormFilterHolder(); - let propUC = prop.toUpperCase(); + const propUC = prop.toUpperCase(); + const cf = this.getCodeField(); + const values = ['=', 'in'].includes(op) ? RxNormServices.#filterValues(op, value) : []; + // each filter gets its own parameter names, so two filters of the same kind in one + // include don't overwrite each other's values (or the SAB the whole query is about) + const pfx = `f${filterContext.filters.length}_`; + const params = {}; + const list = (name, vals) => vals.map((v, i) => { + params[`${pfx}${name}${i}`] = v; + return `$${pfx}${name}${i}`; + }).join(', '); let sql = ''; - let params = {}; - - if (op === 'in' && propUC === 'TTY') { - const values = value.split(',').map(v => v.trim()).filter(v => v); - const placeholders = values.map((_, i) => `$tty${i}`).join(','); - sql = `AND TTY IN (${placeholders})`; - values.forEach((val, i) => { - params[`tty${i}`] = this.#sqlWrapString(val); - }); - } else if (op === '=') { - if (propUC === 'STY') { - sql = `AND ${this.getCodeField()} IN (SELECT RXCUI FROM rxnsty WHERE TUI = $sty)`; - params.sty = this.#sqlWrapString(value); + if (values.length > 0) { + if (propUC === 'TTY') { + sql = `AND rxnconso.${cf} IN (SELECT ${cf} FROM rxnconso WHERE SAB = $sab AND TTY IN (${list('tty', values)}))`; + } else if (propUC === 'STY') { + sql = `AND rxnconso.${cf} IN (SELECT RXCUI FROM rxnsty WHERE TUI IN (${list('sty', values)}))`; } else if (propUC === 'SAB') { - sql = `AND ${this.getCodeField()} IN (SELECT ${this.getCodeField()} FROM rxnconso WHERE SAB = $sab)`; - params.sab = this.#sqlWrapString(value); - } else if (propUC === 'TTY') { - sql = `AND TTY = $tty`; - params.tty = this.#sqlWrapString(value); - } else if (this.rels.includes(prop)) { - if (value.startsWith('CUI:')) { - const cui = value.substring(4); - sql = `AND (${this.getCodeField()} IN (SELECT ${this.getCodeField()} FROM rxnconso WHERE RXCUI IN (SELECT RXCUI2 FROM rxnrel WHERE REL = $rel AND RXCUI1 = $cui2)))`; - params.rel = this.#sqlWrapString(prop); - params.cui2 = this.#sqlWrapString(cui); - } else if (value.startsWith('AUI:')) { - const aui = value.substring(4); - sql = `AND (${this.getCodeField()} IN (SELECT ${this.getCodeField()} FROM rxnconso WHERE RXAUI IN (SELECT RXAUI2 FROM rxnrel WHERE REL = $rel AND RXAUI1 = $aui2)))`; - params.rel = this.#sqlWrapString(prop); - params.aui2 = this.#sqlWrapString(aui); + sql = `AND rxnconso.${cf} IN (SELECT ${cf} FROM rxnconso WHERE SAB IN (${list('sab', values)}))`; + } else if (prop === 'concept' || prop === 'code') { + sql = `AND rxnconso.${cf} IN (${list('code', values)})`; + } else if ((this.rels.includes(prop) || this.reltypes.includes(prop)) && values.every(v => RxNormServices.#isRelValue(v))) { + // RXNREL rows read 'RXCUI2 RXCUI1': has_ingredient = CUI:161 is the concepts + // that have acetaminophen as an ingredient. Concept-level rows: RxNorm's own (SAB=RXNORM) + const field = this.rels.includes(prop) ? 'REL' : 'RELA'; + params[`${pfx}rel`] = prop; + const cuis = values.filter(v => v.startsWith('CUI:')).map(v => v.substring(4)); + const auis = values.filter(v => v.startsWith('AUI:')).map(v => v.substring(4)); + const targets = []; + if (cuis.length > 0) { + params[`${pfx}rsab`] = RELATIONSHIP_SAB; + targets.push(`rxnconso.${cf} IN (SELECT RXCUI2 FROM rxnrel WHERE ${field} = $${pfx}rel AND SAB = $${pfx}rsab AND RXCUI1 IN (${list('cui', cuis)}))`); } - } else if (this.reltypes.includes(prop)) { - if (value.startsWith('CUI:')) { - const cui = value.substring(4); - sql = `AND (${this.getCodeField()} IN (SELECT ${this.getCodeField()} FROM rxnconso WHERE RXCUI IN (SELECT RXCUI2 FROM rxnrel WHERE RELA = $rela AND RXCUI1 = $cui2)))`; - params.rela = this.#sqlWrapString(prop); - params.cui2 = this.#sqlWrapString(cui); - } else if (value.startsWith('AUI:')) { - const aui = value.substring(4); - sql = `AND (${this.getCodeField()} IN (SELECT ${this.getCodeField()} FROM rxnconso WHERE RXAUI IN (SELECT RXAUI2 FROM rxnrel WHERE RELA = $rela AND RXAUI1 = $aui2)))`; - params.rela = this.#sqlWrapString(prop); - params.aui2 = this.#sqlWrapString(aui); + if (auis.length > 0) { + // Atom-level rows leave RXCUI2 empty, and most come from other sources (MMSL, MTHSPL...): + // the target atom's concept comes from rxnconso, and every source's rows count + targets.push(`rxnconso.${cf} IN (SELECT ${cf} FROM rxnconso WHERE RXAUI IN (SELECT RXAUI2 FROM rxnrel WHERE ${field} = $${pfx}rel AND RXAUI1 IN (${list('aui', auis)})))`); } + sql = `AND (${targets.join(' OR ')})`; } } @@ -475,9 +541,11 @@ class RxNormServices extends CodeSystemProvider { } else { // Not in the stem table, so match the token against the text itself. Left as a // plain (non-text) filter so that executeFilters does not join RXNSTEMS for it. - // Unindexed, but it only ever narrows rows the other tokens have already found, - // and the tokens are alphanumeric, so there are no LIKE wildcards to escape. - rxnormFilter.sql = ` AND STR LIKE $text${i}`; + // Unindexed, but it only ever narrows concepts the other tokens have already found, + // and the tokens are alphanumeric, so there are no LIKE wildcards to escape. Any of + // the concept's RxNorm names can match, as for the stems + const cf = this.getCodeField(); + rxnormFilter.sql = ` AND EXISTS (SELECT 1 FROM rxnconso t WHERE t.${cf} = rxnconso.${cf} AND t.SAB = $sab AND t.STR LIKE $text${i})`; rxnormFilter.params[`text${i}`] = '%' + ((filter.words && filter.words[i]) || stem) + '%'; } @@ -530,12 +598,13 @@ class RxNormServices extends CodeSystemProvider { } } - const fullQuery = `SELECT ${this.getCodeField()}, STR ${sql2} WHERE SAB = $sab AND TTY <> 'SY' ${sql1}`; allParams.sab = this.getSAB(); // Create a single filter holder with the combined query const combinedFilter = new RxNormFilterHolder(); - combinedFilter.sql = fullQuery; + combinedFilter.conditions = sql1; + combinedFilter.from = sql2.substring('FROM rxnconso'.length); + combinedFilter.sql = this.#conceptQuery(sql1, combinedFilter.from); combinedFilter.params = allParams; return [combinedFilter]; @@ -550,8 +619,6 @@ class RxNormServices extends CodeSystemProvider { } async filterMore(filterContext, set) { - - if (!set.executed) { await this.#executeFilter(set); } @@ -560,8 +627,6 @@ class RxNormServices extends CodeSystemProvider { } async filterConcept(filterContext, set) { - - if (!set.executed) { await this.#executeFilter(set); } @@ -572,36 +637,18 @@ class RxNormServices extends CodeSystemProvider { const row = set.results[set.cursor]; set.cursor++; - - const concept = new RxNormConcept(row[this.getCodeField()], row.STR); - return concept; + return this.#conceptFromRow(row); } async filterLocate(filterContext, set, code) { - - - return new Promise((resolve, reject) => { - // Build query to check if code exists in filter - const checkQuery = `SELECT ${this.getCodeField()}, STR FROM rxnconso WHERE SAB = $sab AND TTY <> 'SY' AND ${this.getCodeField()} = $code ${set.sql.replace(/SELECT.*?FROM rxnconso/, '').replace(/WHERE SAB = \$sab AND TTY <> 'SY'/, '')}`; - - const params = { ...set.params, code }; - - this.db.get(checkQuery, this.#buildParamArray(checkQuery, params), (err, row) => { - if (err) { - reject(err); - } else if (!row) { - resolve(null); - } else { - const concept = new RxNormConcept(row[this.getCodeField()], row.STR); - resolve(concept); - } - }); - }); + const cf = this.getCodeField(); + const sql = this.#conceptQuery((set.conditions || '') + ` AND rxnconso.${cf} = $code`, set.from || ''); + const params = { ...set.params, code }; + const row = await this.#dbGet(sql, this.#buildParamArray(sql, params)); + return row ? this.#conceptFromRow(row) : null; } async filterCheck(filterContext, set, concept) { - - if (!(concept instanceof RxNormConcept)) { return false; } @@ -609,8 +656,10 @@ class RxNormServices extends CodeSystemProvider { if (!set.executed) { await this.#executeFilter(set); } - - return set.results.some(row => row[this.getCodeField()] === concept.code); + if (!set.codes) { + set.codes = new Set(set.results.map(row => row[this.getCodeField()])); + } + return set.codes.has(concept.code); } async #executeFilter(filter) { @@ -653,10 +702,6 @@ class RxNormServices extends CodeSystemProvider { return paramArray; } - #sqlWrapString(str) { - return str.replace(/'/g, "''"); - } - // Subsumption testing async subsumesTest(codeA, codeB) { await this.#ensureContext(codeA); @@ -665,11 +710,14 @@ class RxNormServices extends CodeSystemProvider { } // Extension for lookup operation + // + // TTY, STY and SAB are returned unless other properties are asked for; the relationships + // (one property per REL/RELA, valued with the bare RXCUI of the other concept, RxNorm's + // own relationships only) only when asked for, by name or with * async extendLookup(ctxt, props, params) { validateArrayParameter(props, 'props', String); validateArrayParameter(params, 'params', Object); - if (typeof ctxt === 'string') { const located = await this.locate(ctxt); if (!located.context) { @@ -681,36 +729,43 @@ class RxNormServices extends CodeSystemProvider { if (!(ctxt instanceof RxNormConcept)) { throw new Error('Invalid context for RxNorm lookup'); } - - // Set abstract status - params.abstract = false; - - // Add designations - const designations = new Designations(this.opContext.i18n.languageDefinitions); - await this.designations(ctxt, designations); - for (const designation of designations) { - this.#addProperty(params, 'designation', 'display', designation.value, designation.language); + if (ctxt.archived) { + return; } - } - #addProperty(params, type, name, value, language = null) { - if (!params.parameter) { - params.parameter = []; + const cf = this.getCodeField(); + if (this._hasProp(props, 'TTY', true)) { + const rows = await this.#dbAll(`SELECT DISTINCT TTY FROM rxnconso WHERE ${cf} = ? AND SAB = ? ORDER BY TTY`, [ctxt.code, this.getSAB()]); + for (const row of rows) { + this._addCodeProperty(params, 'property', 'TTY', row.TTY); + } } - - const property = { - name: type, - part: [ - { name: 'code', valueCode: name }, - { name: 'value', valueString: value } - ] - }; - - if (language) { - property.part.push({ name: 'language', valueCode: language }); + if (this._hasProp(props, 'STY', true)) { + const rows = await this.#dbAll('SELECT DISTINCT TUI FROM rxnsty WHERE RXCUI = ? ORDER BY TUI', [ctxt.code]); + for (const row of rows) { + this._addCodeProperty(params, 'property', 'STY', row.TUI); + } + } + if (this._hasProp(props, 'SAB', true)) { + const rows = await this.#dbAll(`SELECT DISTINCT SAB FROM rxnconso WHERE ${cf} = ? ORDER BY SAB`, [ctxt.code]); + for (const row of rows) { + this._addCodeProperty(params, 'property', 'SAB', row.SAB); + } + } + if (props && props.length > 0) { + const rows = await this.#dbAll( + 'SELECT DISTINCT REL, RELA, RXCUI1 FROM rxnrel WHERE RXCUI2 = ? AND SAB = ? ORDER BY RELA, REL, RXCUI1', + [ctxt.code, RELATIONSHIP_SAB]); + const seen = new Set(); + for (const row of rows) { + const name = row.RELA || row.REL; + const key = name + '|' + row.RXCUI1; + if (name && row.RXCUI1 && !seen.has(key) && this._hasProp(props, name, false)) { + seen.add(key); + this._addCodeProperty(params, 'property', name, row.RXCUI1); + } + } } - - params.parameter.push(property); } versionAlgorithm() { @@ -750,10 +805,13 @@ class RxNormTypeServicesFactory extends CodeSystemFactoryProvider { const db = new sqlite3.Database(this.dbPath); try { - await new Promise((resolve, reject) => { - db.run(`CREATE INDEX IF NOT EXISTS idx_rxnstems_cui_stem ON RXNSTEMS(CUI, stem)`, - err => err ? reject(err) : resolve()); - }); + // indexes the importer makes now, for databases imported before it did. A one-off cost + // the first time an older database is loaded + for (const sql of RXNORM_LOAD_INDEXES) { + await new Promise((resolve, reject) => { + db.run(sql, err => err ? reject(err) : resolve()); + }); + } this._sharedData = { version: '', @@ -773,7 +831,8 @@ class RxNormTypeServicesFactory extends CodeSystemFactoryProvider { // Get total count const sab = this.isNCI ? 'NCI' : 'RXNORM'; - this._sharedData.totalCodeCount = await this.#getCount(db, `SELECT COUNT(RXCUI) FROM rxnconso WHERE SAB = ? AND TTY <> 'SY'`, [sab]); + // concepts, not atoms + this._sharedData.totalCodeCount = await this.#getCount(db, `SELECT COUNT(DISTINCT RXCUI) FROM rxnconso WHERE SAB = ? AND TTY <> 'SY'`, [sab]); } finally { db.close(); diff --git a/tx/cs/cs-snomed.js b/tx/cs/cs-snomed.js index 232ce853..858e0843 100644 --- a/tx/cs/cs-snomed.js +++ b/tx/cs/cs-snomed.js @@ -19,6 +19,7 @@ const {debugLog} = require("../operation-context"); const {editionName, editionCode} = require("../sct/editions"); const {SnomedTextIndex, rankMatches, filterTokens} = require("../sct/text-index"); const Logger = require("../../library/logger"); +const regexUtilities = require("../../library/regex-utilities"); // Symbol key under which a resolved ECL filter analysis is memoised directly on // the ValueSet compose.include.filter element (`fc`). Symbol-keyed and defined @@ -30,6 +31,7 @@ const Logger = require("../../library/logger"); // _cacheEcl): a wildcard's materialised form depends on forIteration and is not // shared - it simply recomputes. const SNOMED_FILTER_ANALYSIS = Symbol('snomedFilterAnalysis'); +const SNOMED_DISPLAY_REGEX = Symbol('snomedDisplayRegex'); /** * Alternate identifier schemes (sct2_Identifier) that FHIRsmith knows how to present: @@ -58,6 +60,8 @@ const SCT_LANGUAGE_CODES = { 9: 'cs' }; +const SCT_LANGUAGE_BYTES = Object.fromEntries(Object.entries(SCT_LANGUAGE_CODES).map(([b, c]) => [c, Number(b)])); + // ECL dialect aliases (ECL specification, appendix "Dialect aliases") to the // language reference set they denote. Used by {{ D dialect = en-gb }}. const ECL_DIALECT_ALIASES = { @@ -146,8 +150,9 @@ class SnomedFilterContext { } class SnomedPrep { - constructor() { + constructor(languages = null) { this.filters = []; + this.languages = languages; // the languages the request named, or null } } @@ -659,6 +664,9 @@ class SnomedServices { } } + // The concept's active outbound relationships. Inactive rows are imported (they are in the + // cache) but ignored altogether, as agreed with the SNOMED leads: a retired row no longer + // describes the concept, so no filter, lookup or refinement may see it. getConceptRelationships(reference) { try { const concept = this.concepts.getConcept(reference); @@ -666,7 +674,7 @@ class SnomedServices { if (relRef === 0) return []; - return this.refs.getReferences(relRef) || []; + return (this.refs.getReferences(relRef) || []).filter(relIdx => this.relationships.isActive(relIdx)); } catch (error) { return []; } @@ -839,6 +847,35 @@ class SnomedServices { return result; } + // descendent-leaf (R5): the concept's descendants that have no children of their own + filterDescendentLeaf(id) { + const result = new SnomedFilterContext(); + const conceptResult = this.concepts.findConcept(id); + + if (!conceptResult.found) { + throw new Error(`The SNOMED CT Concept ${id} is not known`); + } + + result.descendants = this.getConceptDescendants(conceptResult.index) + .filter(ref => this.getConceptChildren(ref).length === 0); + return result; + } + + // is-not-a (R5): every concept except the concept itself and its descendants - the + // same population as the ECL * MINUS << id + filterIsNotA(id, opContext = null) { + const result = new SnomedFilterContext(); + const conceptResult = this.concepts.findConcept(id); + + if (!conceptResult.found) { + throw new Error(`The SNOMED CT Concept ${id} is not known`); + } + + const excluded = new Set([conceptResult.index, ...this.getConceptDescendants(conceptResult.index)]); + result.descendants = this._eclEnumerateAllConcepts(opContext).filter(ref => !excluded.has(ref)); + return result; + } + filterChildOf(id = true) { const result = new SnomedFilterContext(); const conceptResult = this.concepts.findConcept(id); @@ -886,10 +923,25 @@ class SnomedServices { } + // concept in A,B,C: any of the listed concepts; a listed reference set stands for its + // members. Members are kept as reference set member records ({ref}), which is what every + // consumer of set.members reads, so a plain concept is wrapped the same way; and each + // concept appears once, however many times it is named or reached. filterIn(idList) { const result = new SnomedFilterContext(); - let members = []; + const members = []; + const seen = new Set(); + const add = (member) => { + if (!seen.has(member.ref)) { + seen.add(member.ref); + members.push(member); + } + }; for (let id of idList.split(',')) { + id = id.trim(); + if (!id) { + continue; + } const conceptResult = this.concepts.findConcept(id); if (!conceptResult.found) { @@ -898,9 +950,11 @@ class SnomedServices { const refSetIndex = this.getConceptRefSet(conceptResult.index, false); if (refSetIndex === 0) { - members.push(conceptResult.index); + add({ ref: conceptResult.index }); } else { - members.push(...this.refSetMembers.getMembers(refSetIndex)); + for (const member of this.refSetMembers.getMembers(refSetIndex) || []) { + add(member); + } } } result.members = members; @@ -3009,15 +3063,42 @@ class SnomedProvider extends BaseCSServices { return SnomedExpressionContext.fromReference(key); } + // The FHIR value for a concrete value in $lookup. A number is typed by the attribute's MRCM + // range where it has one - so #4000 for a dec() attribute is a decimal, as the concept model + // defines it - and otherwise by how it is written (#4000 an integer, #2.5 a decimal). + #concreteLookupValue(cv) { + const c = cv.concrete; + switch (c.kind) { + case 'string': + return { valueString: c.value }; + case 'boolean': + return { valueBoolean: c.value }; + case 'number': { + let type = c.decimal ? 'dec' : 'int'; + const services = this.sct.expressionServices; + const constraint = services && services.mrcmAttributeRanges ? services.mrcmAttributeRanges().get(cv.relType) : null; + const range = constraint ? services.parseConcreteRange(constraint) : null; + if (range && (range.type === 'dec' || range.type === 'int')) { + type = range.type; + } + return type === 'int' && Number.isInteger(c.value) ? { valueInteger: c.value } : { valueDecimal: c.value }; + } + default: + return null; + } + } + async extendLookup(context, props, params) { const ctxt = await this.#ensureContext(context); if (ctxt) { if (!(ctxt instanceof SnomedExpressionContext) || ctxt.expression?.concepts.length == 1) { - // Only what was asked for, if anything was (see _hasProp): effectiveTime, module, + // Only what was asked for, if anything was (see _hasProp): effectiveTime, moduleId, // parent, child and the defining relationships are returned by default but are not // important enough to return when the client named the properties it wants. // The mappings (alternate identifiers such as the LOINC code) are returned by - // default too, and when asked for by name. + // default too, and when asked for by name. property=* asks for every property the + // SNOMED CT on FHIR page defines (adding sufficientlyDefined and semanticTag) except + // the normal forms, which are computed and have to be asked for by name. if (this._hasProp(props, 'effectiveTime', true)) { const time = this.sct.concepts.getConcept(ctxt.getReference()).effectiveTime; // Pascal TDateTime epoch (1899-12-30) computed in UTC so the formatted @@ -3047,11 +3128,35 @@ class SnomedProvider extends BaseCSServices { } } - if (this._hasProp(props, 'module', true)) { + // moduleId, as the SNOMED CT on FHIR page names it ("module" is still understood) + if (this._hasProp(props, 'moduleId', true) || SnomedProvider.#askedFor(props, 'module')) { const moduleId = this.sct.concepts.getModuleId(ctxt.getReference()); if (moduleId) { const code = this.sct.getConceptId(moduleId); - this._addCodeProperty(params, 'property', 'module', code, null, this.sct.getDisplayName(moduleId)); + this._addCodeProperty(params, 'property', 'moduleId', code, null, this.sct.getDisplayName(moduleId)); + } + } + + // returned for property=* or by name, not by default + if (this._hasProp(props, 'sufficientlyDefined', false)) { + params.push({ + name: 'property', + part: [ + {name: 'code', valueCode: 'sufficientlyDefined'}, + {name: 'value', valueBoolean: !this.sct.isPrimitive(ctxt.getReference())} + ] + }); + } + if (this._hasProp(props, 'semanticTag', false)) { + const tag = this.#semanticTag(ctxt.getReference()); + if (tag) { + params.push({ + name: 'property', + part: [ + {name: 'code', valueCode: 'semanticTag'}, + {name: 'value', valueString: tag} + ] + }); } } @@ -3072,8 +3177,28 @@ class SnomedProvider extends BaseCSServices { } } + // concrete values (strengths, counts, ...): not relationships to concepts, so held apart + for (const cv of (this.sct.concreteValues && this.sct.concreteValues.get(ctxt.getReference())) || []) { + const relType = this.sct.getConceptId(cv.relType); + if (this._hasProp(props, relType, true)) { + const value = this.#concreteLookupValue(cv); + if (value) { + params.push({ + name: 'property', + part: [ + {name: 'code', valueCode: relType}, + {name: 'value', ...value}, + {name: 'code-display', valueString: this.sct.getDisplayName(cv.relType)} + ] + }); + } + } + } + this.#addMappingProperties(ctxt, props, params); } + // the normal forms: only when asked for by name (not by default, not by property=*) + this.#addNormalFormProperties(ctxt, props, params); if (ctxt instanceof SnomedExpressionContext) { // ignore concepts for now, but list refinements and refinement groups for (const refinement of ctxt.expression.refinements) { @@ -3088,6 +3213,63 @@ class SnomedProvider extends BaseCSServices { } } + // Whether props names this property explicitly (property=* does not count) + static #askedFor(props, name) { + const lower = name.toLowerCase(); + return !!props && props.some(p => p && p.toLowerCase() === lower); + } + + // The semantic tag: the bracketed suffix of the concept's active fully specified name + #semanticTag(reference) { + const K = this.sct._displayConstants(); + const concept = this.sct.concepts.getConcept(reference); + const descIndices = concept.descriptions ? (this.sct.refs.getReferences(concept.descriptions) || []) : []; + for (const descIndex of descIndices) { + const d = this.sct.descriptions.getDescription(descIndex); + if (d.active && d.kind === K.fsn) { + const m = /\(([^()]+)\)\s*$/.exec(this.sct.strings.getEntry(d.iDesc)); + if (m) { + return m[1]; + } + } + } + return null; + } + + // normalForm (with terms) and normalFormTerse (without), for a concept or an expression + #addNormalFormProperties(ctxt, props, params) { + const wantFull = SnomedProvider.#askedFor(props, 'normalForm'); + const wantTerse = SnomedProvider.#askedFor(props, 'normalFormTerse'); + if (!wantFull && !wantTerse) { + return; + } + const services = this.sct.expressionServices; + let nf; + if (ctxt instanceof SnomedExpressionContext && ctxt.isComplex()) { + nf = services.normaliseExpression(ctxt.expression); + } else { + nf = services.createNormalForm(ctxt.getReference()); + } + if (wantFull) { + params.push({ + name: 'property', + part: [ + {name: 'code', valueCode: 'normalForm'}, + {name: 'value', valueString: services.renderExpression(nf, SnomedServicesRenderOption.FillMissing)} + ] + }); + } + if (wantTerse) { + params.push({ + name: 'property', + part: [ + {name: 'code', valueCode: 'normalFormTerse'}, + {name: 'value', valueString: services.renderExpression(nf, SnomedServicesRenderOption.Minimal)} + ] + }); + } + } + /** * The concept's alternate identifiers in the schemes we know (IDENTIFIER_SCHEMES), as * Coding-valued properties named for the scheme - e.g. loinc = http://loinc.org#2160-0. @@ -3131,15 +3313,16 @@ class SnomedProvider extends BaseCSServices { // Filter support async doesFilter(prop, op, value) { + prop = SnomedProvider.#filterProperty(prop); if (prop === 'concept') { const id = this.sct.stringToIdOrZero(value); - if (id !== 0n && ['=', 'is-a', 'descendent-of', 'in', 'generalizes', 'child-of'].includes(op)) { + if (id !== 0n && ['=', 'is-a', 'descendent-of', 'in', 'generalizes', 'child-of', 'descendent-leaf', 'is-not-a'].includes(op)) { return this.sct.conceptExists(value); } if (op === 'in' && value.includes(',')) { let ok = true; for (const idStr of value.split(',')) { - const id = this.sct.stringToIdOrZero(idStr); + const id = this.sct.stringToIdOrZero(idStr.trim()); if (id === 0n) { ok = false; break; @@ -3152,6 +3335,15 @@ class SnomedProvider extends BaseCSServices { return op === '=' && ['true', 'false'].includes(value); } + if (prop === 'display' && op === 'regex') { + try { + SnomedProvider.#displayRegex(value); + return true; + } catch (e) { + return false; + } + } + if (prop === 'moduleId') { const id = this.sct.stringToIdOrZero(value); return id !== 0n && op === '='; @@ -3179,9 +3371,8 @@ class SnomedProvider extends BaseCSServices { } // eslint-disable-next-line no-unused-vars - async getPrepContext(iterate) { - - return new SnomedPrep(); // Simple filter context + async getPrepContext(iterate, languages = null) { + return new SnomedPrep(languages); } // Cache key pinning resolved analysis to the SNOMED edition url + version. @@ -3233,7 +3424,84 @@ class SnomedProvider extends BaseCSServices { }); } + // Value sets in production say "code" where SNOMED defines "concept" (users don't tell + // them apart), so the two are the same filter property here, as they are for code + // systems defined as resources (cs-cs.js). + static #filterProperty(prop) { + return prop === 'code' ? 'concept' : prop; + } + + // display regex: matches a concept with an active description (FSN or synonym - a text + // definition is not a display), in one language, whose whole term matches the pattern + // (anchored, as for regex filters on code systems defined as resources), case-sensitively. RE2, so no catastrophic backtracking. The language is + // the first one the request named that the edition has descriptions in, else the + // edition's default. + static #displayRegex(pattern) { + return regexUtilities.compile('^(?:' + pattern + ')$'); + } + + #filterLanguage(languages) { + if (!this.sct._activeDescLanguages) { + this.sct._activeDescLanguages = this.sct.descriptions.activeLanguages(); + } + const present = this.sct._activeDescLanguages; + for (const lang of languages || []) { + const code = SCT_LANGUAGE_BYTES[(lang.language || '').toLowerCase()]; + if (code !== undefined && present.has(code)) { + return code; + } + } + return this.sct.defaultLanguage || 1; + } + + // the concept index of 900000000000550004 |Definition|, or -1 + #definitionKind() { + if (this.sct._definitionKind === undefined) { + const r = this.sct.concepts.findConcept(900000000000550004n); + this.sct._definitionKind = r.found ? r.index : -1; + } + return this.sct._definitionKind; + } + + #displayMatches(set, reference) { + const definition = this.#definitionKind(); + const concept = this.sct.concepts.getConcept(reference); + const descIndices = concept.descriptions ? (this.sct.refs.getReferences(concept.descriptions) || []) : []; + for (const descIndex of descIndices) { + const d = this.sct.descriptions.getDescription(descIndex); + if (d.active && d.lang === set.displayLang && d.kind !== definition && set.displayRegex.test(this.sct.strings.getEntry(d.iDesc))) { + return true; + } + } + return false; + } + + // The scan for an iterated display regex: every active description in the language, + // a chunk at a time, yielding between chunks so the server keeps serving meanwhile. + async #populateDisplayRegex(set) { + if (set.populationDone) { + return; + } + const CHUNK = 10000; + const found = new Set(); + const test = (term) => set.displayRegex.test(term); + const definition = this.#definitionKind(); + let count = Infinity; + for (let start = 0; start < count; start += CHUNK) { + if (this.opContext) { + await this.opContext.checkAndYield('snomed:displayRegex'); + } + count = this.sct.descriptions.matchActiveTerms(start, start + CHUNK, set.displayLang, this.sct.strings, test, found, definition); + } + set.descendants = [...found].sort((a, b) => a - b); + set.populationDone = true; + if (set.displayCache) { + set.displayCache.map.set(set.displayCache.key, set.descendants); + } + } + async filter(filterContext, forIteration, prop, op, value, fc) { + prop = SnomedProvider.#filterProperty(prop); if (prop === 'concept') { const id = this.sct.stringToIdOrZero(value); @@ -3262,6 +3530,14 @@ class SnomedProvider extends BaseCSServices { filterContext.filters.push(this.sct.filterGeneralizes(id, false)); return null; } + case 'descendent-leaf': { + filterContext.filters.push(this.sct.filterDescendentLeaf(id)); + return null; + } + case 'is-not-a': { + filterContext.filters.push(this.sct.filterIsNotA(id, this.opContext)); + return null; + } case 'in': { filterContext.filters.push(this.sct.filterIn(value)); return null; @@ -3271,6 +3547,31 @@ class SnomedProvider extends BaseCSServices { } } + if (prop === 'display' && op === 'regex') { + const result = new SnomedFilterContext(); + result.displayRegex = SnomedProvider.#displayRegex(value); + result.displayLang = this.#filterLanguage(filterContext.languages); + // Like ECL, a cached value set's filter keeps what it resolved to: here, per edition + // and language, the concepts the scan found, so the scan is paid once. + if (fc) { + let map = fc[SNOMED_DISPLAY_REGEX]; + if (!map) { + map = new Map(); + Object.defineProperty(fc, SNOMED_DISPLAY_REGEX, { value: map, enumerable: false, configurable: true, writable: true }); + } + const key = this._snomedFilterCacheKey() + '|' + result.displayLang; + const cached = map.get(key); + if (cached) { + result.descendants = cached; + result.populationDone = true; + } else { + result.displayCache = { map, key }; + } + } + filterContext.filters.push(result); + return null; + } + if (prop === 'inactive') { if (value !== 'true' && value !== 'false') { throw new Error(`Invalid filter value: ${value}`); @@ -3383,6 +3684,9 @@ class SnomedProvider extends BaseCSServices { } async filterSize(filterContext, set) { + if (set.displayRegex) { + await this.#populateDisplayRegex(set); + } // A wildcard (`*`) set is lazy - it carries the eclWildcard flag rather than a // materialised descendant list - so count the active concepts it denotes. if (set.eclWildcard && (!set.descendants || set.descendants.length === 0)) { @@ -3401,6 +3705,9 @@ class SnomedProvider extends BaseCSServices { async filterMore(filterContext, set) { set.cursor = set.cursor || 0; + if (set.displayRegex) { + await this.#populateDisplayRegex(set); + } this.#ensurePopulated(set); const size = await this.filterSize(filterContext, set); return set.cursor < size; @@ -3459,6 +3766,8 @@ class SnomedProvider extends BaseCSServices { let concept = this.sct.concepts.getConcept(reference); let moduleId = this.sct.concepts.getModuleId(concept.index); found = moduleId === set.moduleId; + } else if (set.displayRegex) { + found = this.#displayMatches(set, reference); } else if (set.propProp || set.propValue) { found = false; const relationships = this.sct.getConceptRelationships(reference); @@ -3508,6 +3817,10 @@ class SnomedProvider extends BaseCSServices { return this.sct.concepts.getModuleId(reference) === set.moduleId; } + if (set.displayRegex) { + return this.#displayMatches(set, reference); + } + if (set.propProp || set.propValue) { const relationships = this.sct.getConceptRelationships(reference); for (let relationshipRef of relationships) { @@ -4028,39 +4341,86 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { * http://snomed.info/sct?fhir_vs=refset – list of reference sets * http://snomed.info/sct?fhir_vs=refset/ – members of a reference set * http://snomed.info/sct?fhir_vs=isa/ – concept and descendants + * http://snomed.info/sct?fhir_vs=ecl/ – the concepts an ECL expression selects * * The URL may optionally include edition and/or version segments: * http://snomed.info/sct/?fhir_vs... * http://snomed.info/sct//version/?fhir_vs... * + * Which edition and version the value set is drawn from is the provider's choice, made + * with implicitValueSetForm(): the generic form from the default edition, the edition + * form from the latest version of that edition, the version form from that version. + * For the edition and version forms, the value set's include names this version, so that + * it is expanded against it; the generic form leaves the version to the request. + * * @param {string} url - The ValueSet URL to resolve * @returns {object|null} A FHIR ValueSet JSON object, or null if the URL is not recognised */ async buildKnownValueSet(url, version) { - if (!url.startsWith("http://snomed.info/sct")) { + if (!url.startsWith('http://snomed.info/')) { return null; } + await this.#ensureLoaded(); if (version != null && !this.version().startsWith(version)) { return null; } + const form = this.implicitValueSetForm(url); + if (!form) { + return null; + } + const id = form.query; + const vs = await this.#buildImplicitValueSet(url, id); + if (vs && form.form !== 'generic') { + for (const include of vs.compose.include) { + include.version = this.version(); + } + } + return vs; + } - const URI_SNOMED = 'http://snomed.info/sct'; - - // Extract the query portion (?fhir_vs...) if this is a recognised SNOMED implicit VS URL - let id = null; - const qIdx = url.indexOf('?'); + /** + * Whether url is an implicit value set of this edition, and in which form: + * generic - http://snomed.info/sct?fhir_vs... (any edition) + * edition - http://snomed.info/sct/?fhir_vs... (this edition, no version given) + * version - http://snomed.info/sct//version/?fhir_vs... (this version) + * The edition and version forms are also recognised under this version's own base + * (http://snomed.info/xsct/... for the test edition). + * + * @param {string} url + * @returns {{form: string, query: string}|null} + */ + implicitValueSetForm(url) { + if (!url || !this._sharedData) { + return null; + } + const qIdx = url.indexOf('?fhir_vs'); if (qIdx === -1) { return null; } - - if (url.startsWith('http://snomed.info/sct?fhir_vs') || - url.startsWith(`http://snomed.info/sct/${this.edition}?fhir_vs`) || - url.startsWith(`http://snomed.info/sct/${this.edition}/version/${this.version()}?fhir_vs`)) { - id = url.substring(qIdx); - } else { + const base = url.substring(0, qIdx); + const query = url.substring(qIdx); + if (query !== '?fhir_vs' && !query.startsWith('?fhir_vs=')) { return null; } + const versionUri = this.version(); + const edition = this._sharedData.edition; + const date = this._sharedData.version; + const editionBases = new Set([`http://snomed.info/sct/${edition}`, this.getPartialVersion()]); + const versionBases = new Set([versionUri, `http://snomed.info/sct/${edition}/version/${date}`]); + if (base === 'http://snomed.info/sct') { + return { form: 'generic', query }; + } + if (editionBases.has(base)) { + return { form: 'edition', query }; + } + if (versionBases.has(base)) { + return { form: 'version', query }; + } + return null; + } + async #buildImplicitValueSet(url, id) { + const URI_SNOMED = 'http://snomed.info/sct'; const now = new Date().toISOString(); if (id === '?fhir_vs=refset') { @@ -4074,7 +4434,6 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { resourceType: 'ValueSet', url, status: 'active', - version: this.versionDate, name: 'SNOMEDCTReferenceSetList', title: 'SNOMED CT Reference Set List', description: 'Reference Sets defined in this SNOMED-CT version', @@ -4094,7 +4453,6 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { resourceType: 'ValueSet', url, status: 'active', - version: this.versionDate, name: 'ALLSNOMEDCT', title: 'SNOMED CT Reference Set (All of SNOMED CT)', description: 'SNOMED CT Reference Set (All of SNOMED CT)', @@ -4121,7 +4479,6 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { resourceType: 'ValueSet', url, status: 'active', - version: this.versionDate, name: 'SNOMEDCTRefSet' + refsetId, title: 'SNOMED CT Reference Set ' + refsetId, description: this.snomedServices.getDisplayName(ref.index), @@ -4139,6 +4496,38 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { }; } + if (id.startsWith('?fhir_vs=ecl/')) { + // the ECL is URL-encoded in the value set URL; accept it already decoded as well + let ecl = id.substring(13); + try { + ecl = decodeURIComponent(ecl); + } catch (e) { + // not encoded (or not validly): use it as it stands + } + if (!ecl.trim()) { + return null; + } + return { + resourceType: 'ValueSet', + url, + status: 'active', + name: 'SNOMEDCTECL', + title: 'SNOMED CT ECL ' + ecl, + description: 'The SNOMED CT concepts selected by the expression constraint ' + ecl, + date: now, + compose: { + include: [{ + system: URI_SNOMED, + filter: [{ + property: 'constraint', + op: '=', + value: ecl, + }], + }], + }, + }; + } + if (id.startsWith('?fhir_vs=isa/')) { const conceptId = id.substring(13); let ref = this.snomedServices.concepts.findConcept(conceptId); @@ -4149,7 +4538,6 @@ class SnomedServicesFactory extends CodeSystemFactoryProvider { resourceType: 'ValueSet', url, status: 'active', - version: this.versionDate, name: 'SNOMEDCTConcept' + conceptId, title: 'SNOMED CT Concept ' + conceptId + ' and descendants', description: 'All Snomed CT concepts for ' + this.snomedServices.getDisplayName(ref.index), diff --git a/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-en_TEST1000003_20250909.txt b/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-en_TEST1000003_20250909.txt index 05657b64..60372ada 100644 --- a/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-en_TEST1000003_20250909.txt +++ b/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-en_TEST1000003_20250909.txt @@ -33,3 +33,27 @@ c279ce20-14c0-5cec-a278-86d4cbda7f71 20250909 1 31000003106 900000000000508004 1 b1593b38-25da-51dc-aa7b-c63ae2c0ffa6 20250909 1 31000003106 900000000000509007 1000391000003113 900000000000548007 fc097f04-417b-5dc1-af89-3e90e5fbd5ad 20250909 1 31000003106 900000000000508004 1000401000003111 900000000000548007 a634257f-0fbc-5fcb-a14f-cd850ec835b6 20250909 1 31000003106 900000000000509007 1000401000003111 900000000000548007 +c8bce317-7e7f-5f31-be08-04bfe3ddd187 20250909 1 31000003106 900000000000508004 1000611000003111 900000000000548007 +ec582eab-f244-59a2-98fa-791316a41afb 20250909 1 31000003106 900000000000509007 1000611000003111 900000000000548007 +ca468079-07fc-51d3-9a94-941bf8a1cf49 20250909 1 31000003106 900000000000508004 1000621000003118 900000000000548007 +ae91b711-bde6-57d0-9d25-390dbcb83317 20250909 1 31000003106 900000000000509007 1000621000003118 900000000000548007 +cb5caf2d-ab5f-562b-9757-fb1a5a61084a 20250909 1 31000003106 900000000000508004 1000731000003113 900000000000548007 +63a0156f-ecf5-5935-a51b-5849709a6228 20250909 1 31000003106 900000000000509007 1000731000003113 900000000000548007 +3bf06ea8-64ee-5939-a89d-7a24e86410cf 20250909 1 31000003106 900000000000508004 1000741000003117 900000000000548007 +b2f9b2b3-1970-5d3b-86f8-2e3aabab1dba 20250909 1 31000003106 900000000000509007 1000741000003117 900000000000548007 +72d1da10-f7e4-58ae-b92f-a561ea8a9bab 20250909 1 31000003106 900000000000508004 1000751000003119 900000000000548007 +8b8d2fa5-e2fb-5e80-8c3f-d85800086954 20250909 1 31000003106 900000000000509007 1000751000003119 900000000000548007 +6c96d388-6c6b-5ed8-a50b-08fd7a603440 20250909 1 31000003106 900000000000508004 1000761000003116 900000000000548007 +72971a87-4102-5bd7-855d-d700f700d0e1 20250909 1 31000003106 900000000000509007 1000761000003116 900000000000548007 +b6a1324d-8397-5c97-b542-00611e9e9693 20250909 1 31000003106 900000000000508004 1000771000003110 900000000000548007 +c5eba7cd-f0db-557f-9942-9ee8d7b327a4 20250909 1 31000003106 900000000000509007 1000771000003110 900000000000548007 +22038678-1172-5af6-8bfd-8bdadc4f9de8 20250909 1 31000003106 900000000000508004 1000781000003112 900000000000548007 +05ca728c-c180-5450-89b8-9bb1b787fd38 20250909 1 31000003106 900000000000509007 1000781000003112 900000000000548007 +729c0a19-1a81-57dc-9983-75448e9488af 20250909 1 31000003106 900000000000508004 1000911000003116 900000000000548007 +257fde96-9fee-5f18-a029-18376e60301e 20250909 1 31000003106 900000000000509007 1000911000003116 900000000000548007 +73a0538d-6d41-58f1-a6d6-1b793a65ae1e 20250909 1 31000003106 900000000000508004 1000921000003114 900000000000548007 +f9e48b6d-9d84-5fcb-8815-fdfb5bf860e2 20250909 1 31000003106 900000000000509007 1000921000003114 900000000000548007 +96e42e22-ca19-5864-895d-f59c2eb0133e 20250909 1 31000003106 900000000000508004 1000931000003111 900000000000548007 +df04f2d5-2354-59f8-a981-4668b02d6447 20250909 1 31000003106 900000000000509007 1000931000003111 900000000000548007 +29f6d967-8173-5015-9666-32f091a769c7 20250909 1 31000003106 900000000000508004 1000941000003115 900000000000548007 +4c37592e-8a67-5e74-a14c-c31f5a377a64 20250909 1 31000003106 900000000000509007 1000941000003115 900000000000548007 diff --git a/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-nl_TEST1000003_20250909.txt b/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-nl_TEST1000003_20250909.txt new file mode 100644 index 00000000..a08ef28b --- /dev/null +++ b/tx/data/snomed/Refset/Language/der2_cRefset_LanguageSnapshot-nl_TEST1000003_20250909.txt @@ -0,0 +1,7 @@ +id effectiveTime active moduleId refsetId referencedComponentId acceptabilityId +6448ecb9-b9ee-5e88-af35-1852ecbb8891 20250909 1 31000003106 1000601000003105 1000631000003116 900000000000548007 +443b79c1-acd3-5346-bc41-8a6e836eac99 20250909 1 31000003106 1000601000003105 1000641000003112 900000000000549004 +66317744-2f04-5d45-96ed-6229ac08f8c6 20250909 1 31000003106 1000601000003105 1000651000003110 900000000000548007 +342275b2-9f6d-5dcd-bc09-276dc67975bd 20250909 1 31000003106 1000601000003105 1000661000003113 900000000000548007 +24050739-89e9-5ed2-ae17-d91170701870 20250909 1 31000003106 1000601000003105 1000671000003119 900000000000548007 +39263b71-61e7-50d9-99aa-992d13c25812 20250909 1 31000003106 1000601000003105 1000681000003117 900000000000548007 diff --git a/tx/data/snomed/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_TEST1000003_20250909.txt b/tx/data/snomed/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_TEST1000003_20250909.txt new file mode 100644 index 00000000..d4e88570 --- /dev/null +++ b/tx/data/snomed/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_TEST1000003_20250909.txt @@ -0,0 +1,5 @@ +id effectiveTime active moduleId refsetId referencedComponentId mapGroup mapPriority mapRule mapAdvice mapTarget correlationId mapCategoryId +1db4d6ae-ba19-5213-bb42-c59ddbf4a77d 20250909 1 31000003106 1000901000003102 262797004 1 1 TRUE ALWAYS S36.115 S36.115 447561005 447637006 +aaade1af-c04d-5632-8edd-3b1863b5fabb 20250909 1 31000003106 1000901000003102 262797004 2 1 TRUE ALWAYS Y92.9 Y92.9 447561005 447637006 +488b3089-844e-578b-9675-5d10d4b3fb0b 20250909 1 31000003106 1000901000003102 262798009 1 1 TRUE ALWAYS S36.116 S36.116 447561005 447637006 +ebeeecf5-b825-5911-a532-0fa95b193aeb 20250909 0 31000003106 1000901000003102 405559000 1 1 TRUE ALWAYS S36.115 S36.115 447561005 447637006 diff --git a/tx/data/snomed/Refset/Map/der2_sRefset_SimpleMapSnapshot_TEST1000003_20250909.txt b/tx/data/snomed/Refset/Map/der2_sRefset_SimpleMapSnapshot_TEST1000003_20250909.txt new file mode 100644 index 00000000..0737f681 --- /dev/null +++ b/tx/data/snomed/Refset/Map/der2_sRefset_SimpleMapSnapshot_TEST1000003_20250909.txt @@ -0,0 +1,3 @@ +id effectiveTime active moduleId refsetId referencedComponentId mapTarget +2853dcf9-9b78-5a42-9f05-79fe77358515 20250909 1 31000003106 1000891000003103 262796008 S36.11 +c55dd22a-c8ef-5e82-ac03-f4ac54819489 20250909 1 31000003106 1000891000003103 16763008 K76.89 diff --git a/tx/data/snomed/Terminology/sct2_Concept_Snapshot_TEST1000003_20250909.txt b/tx/data/snomed/Terminology/sct2_Concept_Snapshot_TEST1000003_20250909.txt index 78a96b59..f9888df7 100644 --- a/tx/data/snomed/Terminology/sct2_Concept_Snapshot_TEST1000003_20250909.txt +++ b/tx/data/snomed/Terminology/sct2_Concept_Snapshot_TEST1000003_20250909.txt @@ -6,3 +6,9 @@ id effectiveTime active moduleId definitionStatusId 1000311000003101 20250909 1 31000003106 900000000000074008 1000351000003102 20250909 0 31000003106 900000000000074008 1000381000003107 20250909 0 31000003106 900000000000074008 +1000601000003105 20250909 1 31000003106 900000000000074008 +1000701000003104 20250909 1 31000003106 900000000000074008 +1000711000003102 20250909 1 31000003106 900000000000074008 +1000721000003107 20250909 1 31000003106 900000000000074008 +1000891000003103 20250909 1 31000003106 900000000000074008 +1000901000003102 20250909 1 31000003106 900000000000074008 diff --git a/tx/data/snomed/Terminology/sct2_Description_Snapshot-en_TEST1000003_20250909.txt b/tx/data/snomed/Terminology/sct2_Description_Snapshot-en_TEST1000003_20250909.txt index 1c11b947..cb062bd5 100644 --- a/tx/data/snomed/Terminology/sct2_Description_Snapshot-en_TEST1000003_20250909.txt +++ b/tx/data/snomed/Terminology/sct2_Description_Snapshot-en_TEST1000003_20250909.txt @@ -16,3 +16,15 @@ id effectiveTime active moduleId conceptId languageCode typeId term caseSignific 1000371000003114 20250909 1 31000003106 1000351000003102 en 900000000000013009 Hepatic dropsy 900000000000448009 1000391000003113 20250909 1 31000003106 1000381000003107 en 900000000000003001 Hepatic revenge syndrome (disorder) 900000000000448009 1000401000003111 20250909 1 31000003106 1000381000003107 en 900000000000013009 Hepatic revenge 900000000000448009 +1000611000003111 20250909 1 31000003106 1000601000003105 en 900000000000003001 Test edition Dutch language reference set (foundation metadata concept) 900000000000448009 +1000621000003118 20250909 1 31000003106 1000601000003105 en 900000000000013009 Test edition Dutch language reference set 900000000000448009 +1000731000003113 20250909 1 31000003106 1000701000003104 en 900000000000003001 Has test product nickname (attribute) 900000000000448009 +1000741000003117 20250909 1 31000003106 1000701000003104 en 900000000000013009 Has test product nickname 900000000000448009 +1000751000003119 20250909 1 31000003106 1000711000003102 en 900000000000003001 Is test product fish-derived (attribute) 900000000000448009 +1000761000003116 20250909 1 31000003106 1000711000003102 en 900000000000013009 Is test product fish-derived 900000000000448009 +1000771000003110 20250909 1 31000003106 1000721000003107 en 900000000000003001 Has test potency value (attribute) 900000000000448009 +1000781000003112 20250909 1 31000003106 1000721000003107 en 900000000000013009 Has test potency value 900000000000448009 +1000911000003116 20250909 1 31000003106 1000891000003103 en 900000000000003001 Test edition simple map reference set (foundation metadata concept) 900000000000448009 +1000921000003114 20250909 1 31000003106 1000891000003103 en 900000000000013009 Test edition simple map reference set 900000000000448009 +1000931000003111 20250909 1 31000003106 1000901000003102 en 900000000000003001 Test edition extended map reference set (foundation metadata concept) 900000000000448009 +1000941000003115 20250909 1 31000003106 1000901000003102 en 900000000000013009 Test edition extended map reference set 900000000000448009 diff --git a/tx/data/snomed/Terminology/sct2_Description_Snapshot-nl_TEST1000003_20250909.txt b/tx/data/snomed/Terminology/sct2_Description_Snapshot-nl_TEST1000003_20250909.txt new file mode 100644 index 00000000..cf4d1c24 --- /dev/null +++ b/tx/data/snomed/Terminology/sct2_Description_Snapshot-nl_TEST1000003_20250909.txt @@ -0,0 +1,7 @@ +id effectiveTime active moduleId conceptId languageCode typeId term caseSignificanceId +1000631000003116 20250909 1 31000003106 35566002 nl 900000000000013009 hematoom 900000000000448009 +1000641000003112 20250909 1 31000003106 35566002 nl 900000000000013009 bloeduitstorting 900000000000448009 +1000651000003110 20250909 1 31000003106 262796008 nl 900000000000013009 leverhematoom 900000000000448009 +1000661000003113 20250909 1 31000003106 262797004 nl 900000000000013009 subcapsulair leverhematoom 900000000000448009 +1000671000003119 20250909 1 31000003106 262798009 nl 900000000000013009 intrahepatisch hematoom 900000000000448009 +1000681000003117 20250909 1 31000003106 16763008 nl 900000000000013009 leverbloeding 900000000000448009 diff --git a/tx/data/snomed/Terminology/sct2_RelationshipConcreteValues_Snapshot_TEST1000003_20250909.txt b/tx/data/snomed/Terminology/sct2_RelationshipConcreteValues_Snapshot_TEST1000003_20250909.txt new file mode 100644 index 00000000..9bb0dc75 --- /dev/null +++ b/tx/data/snomed/Terminology/sct2_RelationshipConcreteValues_Snapshot_TEST1000003_20250909.txt @@ -0,0 +1,8 @@ +id effectiveTime active moduleId sourceId value relationshipGroup typeId characteristicTypeId modifierId +1000821000003129 20250909 1 31000003106 329238006 "Sunshine in a capsule" 0 1000701000003104 900000000000011006 900000000000451002 +1000831000003127 20250909 1 31000003106 329238006 true 0 1000711000003102 900000000000011006 900000000000451002 +1000841000003124 20250909 1 31000003106 329238006 #2.5 0 1000721000003107 900000000000011006 900000000000451002 +1000851000003122 20250909 1 31000003106 776168003 true 0 1000711000003102 900000000000011006 900000000000451002 +1000861000003120 20250909 1 31000003106 1204474000 "Barium milkshake" 0 1000701000003104 900000000000011006 900000000000451002 +1000871000003125 20250909 1 31000003106 1204474000 false 0 1000711000003102 900000000000011006 900000000000451002 +1000881000003128 20250909 1 31000003106 1204474000 #0.75 0 1000721000003107 900000000000011006 900000000000451002 diff --git a/tx/data/snomed/Terminology/sct2_Relationship_Snapshot_TEST1000003_20250909.txt b/tx/data/snomed/Terminology/sct2_Relationship_Snapshot_TEST1000003_20250909.txt index 832319f0..7444c544 100644 --- a/tx/data/snomed/Terminology/sct2_Relationship_Snapshot_TEST1000003_20250909.txt +++ b/tx/data/snomed/Terminology/sct2_Relationship_Snapshot_TEST1000003_20250909.txt @@ -15,3 +15,11 @@ id effectiveTime active moduleId sourceId destinationId relationshipGroup typeId 1000251000003121 20250909 1 31000003106 1000161000003101 10200004 2 405813007 900000000000011006 900000000000451002 1000301000003126 20250909 1 31000003106 1000261000003107 300338006 0 116680003 900000000000011006 900000000000451002 1000341000003128 20250909 1 31000003106 1000311000003101 404684003 0 116680003 900000000000011006 900000000000451002 +1000501000003123 20250909 0 31000003106 50325005 64033007 1 363698007 900000000000011006 900000000000451002 +1000511000003120 20250909 0 31000003106 50325005 51292008 0 116680003 900000000000011006 900000000000451002 +1000691000003127 20250909 1 31000003106 1000601000003105 900000000000506000 0 116680003 900000000000011006 900000000000451002 +1000791000003122 20250909 1 31000003106 1000701000003104 762706009 0 116680003 900000000000011006 900000000000451002 +1000801000003121 20250909 1 31000003106 1000711000003102 762706009 0 116680003 900000000000011006 900000000000451002 +1000811000003123 20250909 1 31000003106 1000721000003107 762706009 0 116680003 900000000000011006 900000000000451002 +1000951000003129 20250909 1 31000003106 1000891000003103 900000000000455006 0 116680003 900000000000011006 900000000000451002 +1000961000003126 20250909 1 31000003106 1000901000003102 900000000000455006 0 116680003 900000000000011006 900000000000451002 diff --git a/tx/data/snomed/Terminology/sct2_sRefset_OWLExpressionSnapshot_TEST1000003_20250909.txt b/tx/data/snomed/Terminology/sct2_sRefset_OWLExpressionSnapshot_TEST1000003_20250909.txt index 42ba2950..8d002fe9 100644 --- a/tx/data/snomed/Terminology/sct2_sRefset_OWLExpressionSnapshot_TEST1000003_20250909.txt +++ b/tx/data/snomed/Terminology/sct2_sRefset_OWLExpressionSnapshot_TEST1000003_20250909.txt @@ -4,3 +4,9 @@ ae008243-6b25-5d48-a6c4-2847146b49e2 20250909 1 31000003106 733073007 1000011000 3addd297-d1cd-5097-a7c3-0447fc635afb 20250909 1 31000003106 733073007 1000161000003101 EquivalentClasses(:1000161000003101 :438300009) d283eaac-c12d-5762-a6f9-67bc767190f2 20250909 1 31000003106 733073007 1000261000003107 SubClassOf(:1000261000003107 :300338006) 9ee21c50-0a30-5513-ade2-558b8bbb0c5b 20250909 1 31000003106 733073007 1000311000003101 SubClassOf(:1000311000003101 :404684003) +e89ae6b0-1b56-582c-aa41-250dda22d97b 20250909 1 31000003106 733073007 1000601000003105 SubClassOf(:1000601000003105 :900000000000506000) +7976ace9-ce9e-5b68-9539-d7bc5025d5b2 20250909 1 31000003106 733073007 1000701000003104 SubDataPropertyOf(:1000701000003104 :762706009) +1e4b9ff4-33b0-5df2-b60d-33a73c1bf786 20250909 1 31000003106 733073007 1000711000003102 SubDataPropertyOf(:1000711000003102 :762706009) +0f41f71e-4e8c-55c7-bb98-dd6b36dc4340 20250909 1 31000003106 733073007 1000721000003107 SubDataPropertyOf(:1000721000003107 :762706009) +c73360e6-5fa0-540a-af6b-035497450716 20250909 1 31000003106 733073007 1000891000003103 SubClassOf(:1000891000003103 :900000000000455006) +8887f167-9cdd-5969-941a-c95a0bd9d493 20250909 1 31000003106 733073007 1000901000003102 SubClassOf(:1000901000003102 :900000000000455006) diff --git a/tx/fixtures/test-cases.yml b/tx/fixtures/test-cases.yml index 9f1cf525..c048cfca 100644 --- a/tx/fixtures/test-cases.yml +++ b/tx/fixtures/test-cases.yml @@ -11,9 +11,9 @@ sources: - internal:hgvs - internal:urls - ucum:tx/data/ucum-essence.xml - - loinc!:Loinc_2.83.db + - loinc!:Loinc_2.83b.db - snomed!:Snomed_Intl_20260701.cache - - snomed:sct_test_20250909e.cache + - snomed:sct_test_20250909i.cache - omop:omop_v20250227.db - icd11:icd11-2026-01-b.db - npm:hl7.terminology=tho diff --git a/tx/importers/import-loinc.module.js b/tx/importers/import-loinc.module.js index c2eef0c5..68090ac9 100644 --- a/tx/importers/import-loinc.module.js +++ b/tx/importers/import-loinc.module.js @@ -596,6 +596,9 @@ class LoincDataMigrator { // Initialize tracking variables this.codeKey = 0; this.relKey = 0; + this.relSeen = new Set(); + this.axisParts = new Map(); // term key -> { COMPONENT: part key, ... } (see processLanguage) + this.partNamesSeen = new Set(); this.descKey = 0; this.propKey = 0; this.propValueKey = 0; @@ -826,7 +829,8 @@ class LoincDataMigrator { // Description types const descriptionTypes = [ [1, 'LONG_COMMON_NAME'], [2, 'SHORTNAME'], [3, 'ConsumerName'], - [4, 'RELATEDNAMES2'], [5, 'DisplayName'], [6, 'LinguisticVariantDisplayName'] + [4, 'RELATEDNAMES2'], [5, 'DisplayName'], [6, 'LinguisticVariantDisplayName'], + [7, 'PartName'] ]; descriptionTypes.forEach(([key, desc]) => { db.run('INSERT INTO DescriptionTypes (DescriptionTypeKey, Description) VALUES (?, ?)', [key, desc]); @@ -1002,6 +1006,7 @@ class LoincDataMigrator { const clsCode = this.partNames.get('CLASS.' + items[7]); if (clsCode && this.codes.has(clsCode)) { this.addRelationship(db, codeKey, this.codes.get(clsCode).key, this.rels.get('CLASS')); + this.#noteAxisPart(codeKey, 'CLASS', this.codes.get(clsCode).key); } // Add properties @@ -1269,6 +1274,9 @@ class LoincDataMigrator { this.rels.get(relType), this.statii.get(status) || 0 ); + if (LINGUISTIC_VARIANT_AXES.includes(relType)) { + this.#noteAxisPart(this.codes.get(sourceCode).key, relType, this.codes.get(targetCode).key); + } } processedCount++; @@ -1472,6 +1480,24 @@ class LoincDataMigrator { this.addDescription(db, codeKey, langKey, this.dTypes.get('SHORTNAME'), items[8]); this.addDescription(db, codeKey, langKey, this.dTypes.get('LinguisticVariantDisplayName'), items[11]); + // The axis columns (COMPONENT ... CLASS) translate the term's Parts. Each becomes a + // PartName designation on the Part the term links to for that axis - for some languages + // (ko-KR, most of fr-CA) these are the only translations LOINC has. Each distinct name once + const axes = this.axisParts.get(codeKey); + if (axes) { + LINGUISTIC_VARIANT_AXES.forEach((axis, i) => { + const value = items[i + 1] ? items[i + 1].trim() : ''; + const partKey = axes[axis]; + if (value && partKey) { + const id = partKey + '|' + langKey + '|' + value; + if (!this.partNamesSeen.has(id)) { + this.partNamesSeen.add(id); + this.addDescription(db, partKey, langKey, this.dTypes.get('PartName'), value); + } + } + }); + } + processedCount++; if (processedCount % 100 === 0) { @@ -1485,6 +1511,15 @@ class LoincDataMigrator { } } + #noteAxisPart(termKey, axis, partKey) { + let axes = this.axisParts.get(termKey); + if (!axes) { + axes = {}; + this.axisParts.set(termKey, axes); + } + axes[axis] = partKey; + } + addDescription(db, codeKey, languageKey, descriptionType, value) { if (!value) return; @@ -1508,6 +1543,14 @@ class LoincDataMigrator { addRelationship(db, sourceKey, targetKey, relationshipType, statusKey = 0) { if (!relationshipType) return; + // each relationship once: the source files repeat some (a part under the same parent on + // two paths in ComponentHierarchyBySystem, the same answer list linked more than once) + if (!this.relSeen) { + this.relSeen = new Set(); + } + const id = relationshipType + '|' + sourceKey + '|' + targetKey; + if (this.relSeen.has(id)) return; + this.relSeen.add(id); this.relKey++; db.run('INSERT INTO Relationships (RelationshipKey, RelationshipTypeKey, SourceKey, TargetKey, StatusKey) VALUES (?, ?, ?, ?, ?)', @@ -1537,6 +1580,10 @@ class LoincDataMigrator { } // Property name mappings from Pascal code +// The axis columns of a LinguisticVariant file, in file order (columns 1 to 7), named as the +// term's relationships to its Parts are +const LINGUISTIC_VARIANT_AXES = ['COMPONENT', 'PROPERTY', 'TIME_ASPCT', 'SYSTEM', 'SCALE_TYP', 'METHOD_TYP', 'CLASS']; + const KNOWN_PROPERTY_NAMES = [ 'AskAtOrderEntry', 'AssociatedObservations', 'CHANGE_REASON_PUBLIC', 'CHNG_TYPE', 'CLASS', 'CLASSTYPE', 'COMMON_ORDER_RANK', 'COMMON_TEST_RANK', 'COMPONENT', 'CONSUMER_NAME', 'DefinitionDescription', 'DisplayName', 'EXAMPLE_UCUM_UNITS', 'EXAMPLE_UNITS', 'EXMPL_ANSWERS', 'EXTERNAL_COPYRIGHT_LINK', 'EXTERNAL_COPYRIGHT_NOTICE', 'FORMULA', diff --git a/tx/importers/import-rxnorm.module.js b/tx/importers/import-rxnorm.module.js index 6f431e9b..ca160a16 100644 --- a/tx/importers/import-rxnorm.module.js +++ b/tx/importers/import-rxnorm.module.js @@ -428,6 +428,10 @@ class RxNormModule extends BaseTerminologyModule { 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rel ON RXNREL(REL)', 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela ON RXNREL(RELA)', 'CREATE INDEX IF NOT EXISTS X_RXNSTY_2 ON RXNSTY(TUI)', + 'CREATE INDEX IF NOT EXISTS idx_rxnsty_rxcui ON RXNSTY(RXCUI)', + 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rxcui2 ON RXNREL(RXCUI2, SAB)', + 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela_rxcui1 ON RXNREL(RELA, RXCUI1)', + 'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela_rxaui1 ON RXNREL(RELA, RXAUI1)', 'CREATE INDEX IF NOT EXISTS idx_rxnstems_stem_cui ON RXNSTEMS(stem, CUI)', 'CREATE INDEX IF NOT EXISTS idx_rxnstems_cui_stem ON RXNSTEMS(CUI, stem)' ]; @@ -707,70 +711,103 @@ class RxNormImporter { this.updateProgress(0, `Loading ${fileName}`); + // Rows go in as multi-row INSERTs, as many rows per statement as fit in SQLite's 999 + // parameters: one statement (and one callback) per row made the import slow, and slower + // still when the process is busy collecting a large heap - the test suite runs it in a + // jest process already holding GBs. OR IGNORE keeps the old per-row behaviour, where a row + // that broke a constraint was dropped and the rest of the file went in + const m = insertSQL.match(/^\s*INSERT\s+(?:OR\s+IGNORE\s+)?INTO([\s\S]*?VALUES\s*)(\([^)]*\))\s*$/i); + if (!m) { + throw new Error(`Unexpected insert statement for ${fileName}: ${insertSQL}`); + } + const head = 'INSERT OR IGNORE INTO' + m[1]; + const group = m[2]; + const cols = (group.match(/\?/g) || []).length; + const batchRows = Math.max(1, Math.floor(999 / cols)); + const sqlFor = (n) => head + new Array(n).fill(group).join(', '); + const fullSql = sqlFor(batchRows); + const rl = readline.createInterface({ input: fs.createReadStream(filePath), crlfDelay: Infinity }); return new Promise((resolve, reject) => { - db.serialize(() => { - db.run('BEGIN TRANSACTION'); - - const stmt = db.prepare(insertSQL); - let lineCount = 0; - let processedCount = 0; - let errorCount = 0; - - rl.on('line', (line) => { - lineCount++; - - const items = line.split('|'); - if (items.length === 0) return; - - try { - const values = extractValues(items); - stmt.run(values, (err) => { - if (err && err.code !== 'SQLITE_CONSTRAINT') { - errorCount++; - if (this.options.verbose && errorCount <= 10) { - console.warn(`Error processing line ${lineCount} in ${fileName}: ${err.message}`); - } - } - }); - processedCount++; + // serialized: BEGIN, every batch, and COMMIT run in the order they are queued + db.serialize(); + db.run('BEGIN TRANSACTION'); - if (processedCount % 1000 === 0) { - this.updateProgress(1000); - } - } catch (error) { + let lineCount = 0; + let processedCount = 0; + let errorCount = 0; + let pending = []; + let pendingRows = 0; + + const flush = () => { + if (pendingRows === 0) { + return; + } + const rows = pendingRows; + db.run(rows === batchRows ? fullSql : sqlFor(rows), pending, (err) => { + if (err) { errorCount++; if (this.options.verbose && errorCount <= 10) { - console.warn(`Error processing line ${lineCount} in ${fileName}: ${error.message}`); + console.warn(`Error loading ${rows} rows of ${fileName}: ${err.message}`); } } }); + pending = []; + pendingRows = 0; + }; - rl.on('close', () => { - stmt.finalize(); - db.run('COMMIT', (err) => { - if (err) reject(err); - else { - // Update progress for remaining records - const remaining = processedCount % 1000; - if (remaining > 0) { - this.updateProgress(remaining); - } + rl.on('line', (line) => { + lineCount++; - if (this.options.verbose) { - console.log(` Loaded ${processedCount} records from ${fileName}${errorCount > 0 ? ` (${errorCount} errors)` : ''}`); - } - resolve(); + const items = line.split('|'); + if (items.length === 0) return; + + try { + const values = extractValues(items); + for (let i = 0; i < cols; i++) { + pending.push(values[i] === undefined ? null : values[i]); + } + pendingRows++; + if (pendingRows === batchRows) { + flush(); + } + processedCount++; + + if (processedCount % 1000 === 0) { + this.updateProgress(1000); + } + } catch (error) { + errorCount++; + if (this.options.verbose && errorCount <= 10) { + console.warn(`Error processing line ${lineCount} in ${fileName}: ${error.message}`); + } + } + }); + + rl.on('close', () => { + flush(); + db.run('COMMIT', (err) => { + if (err) reject(err); + else { + // Update progress for remaining records + const remaining = processedCount % 1000; + if (remaining > 0) { + this.updateProgress(remaining); } - }); - }); - rl.on('error', reject); + if (this.options.verbose) { + console.log(` Loaded ${processedCount} records from ${fileName}${errorCount > 0 ? ` (${errorCount} errors)` : ''}`); + } + resolve(); + } + }); }); + + rl.on('error', reject); }); } diff --git a/tx/importers/import-sct.module.js b/tx/importers/import-sct.module.js index ada04b32..c8b51b1f 100644 --- a/tx/importers/import-sct.module.js +++ b/tx/importers/import-sct.module.js @@ -2392,13 +2392,10 @@ class SnomedImporter { lineNumber++; if (lineNumber === 1) { - // Parse headers + // Parse headers. Map reference sets (simple, complex, extended) are loaded like + // any other: ECL ^ and ?fhir_vs=refset/ can name any reference set, and the + // ICD-10 maps are among the most used. headers = line.split('\t').map(h => h.trim()); - - // Skip if this looks like a mapping reference set - if (line.toLowerCase().includes('map')) { - return; - } continue; } diff --git a/tx/library/implicit-valuesets.js b/tx/library/implicit-valuesets.js new file mode 100644 index 00000000..13bf5e75 --- /dev/null +++ b/tx/library/implicit-valuesets.js @@ -0,0 +1,66 @@ +/** + * Choosing which version of a code system an implicit value set URL is drawn from, when + * several versions are loaded (SNOMED CT editions, typically). A factory that can say what + * form an implicit value set URL takes implements implicitValueSetForm(url), returning + * {form: 'generic'|'edition'|'version'} or null: + * + * generic (http://snomed.info/sct?fhir_vs...) - the version the request asks for + * (system-version and the like, through systemVersionFor) where a loaded version + * matches it, else the default version: the one registered under the bare system + * edition (http://snomed.info/sct/?fhir_vs...) - the latest version of that + * edition the server has + * version (.../version/?fhir_vs...) - that version; only it matches + * + * @param {Iterable} factories - the registered code system factories (duplicates are ignored) + * @param {function(string): object} defaultFor - the default factory for a system url + * @param {string} url - the value set url + * @param {function(string): (string|null)} [systemVersionFor] - the version the request asks for, for a system + * @returns {object|null} the factory to build the value set, or null if none recognises the url + */ +function chooseImplicitValueSetFactory(factories, defaultFor, url, systemVersionFor = null) { + const matches = []; + const seen = new Set(); + for (const csp of factories) { + if (seen.has(csp) || typeof csp.implicitValueSetForm !== 'function') { + continue; + } + seen.add(csp); + const form = csp.implicitValueSetForm(url); + if (form) { + matches.push({ csp, form: form.form }); + } + } + if (matches.length === 0) { + return null; + } + if (matches.every(m => m.form === 'generic')) { + const requested = systemVersionFor ? systemVersionFor(matches[0].csp.system()) : null; + if (requested) { + // an edition (with or without its version) or a full version uri; the latest that matches + const asked = matches.filter(m => versionMatches(m.csp, requested)); + if (asked.length > 0) { + asked.sort((a, b) => (b.csp.version() || '').localeCompare(a.csp.version() || '')); + return asked[0].csp; + } + } + const preferred = defaultFor(matches[0].csp.system()); + return matches.some(m => m.csp === preferred) ? preferred : matches[0].csp; + } + // edition or version form: the latest version that matches. Version URIs of one edition + // differ only in their date, so they order as strings. + const specific = matches.filter(m => m.form !== 'generic'); + specific.sort((a, b) => (b.csp.version() || '').localeCompare(a.csp.version() || '')); + return specific[0].csp; +} + +// Whether a factory's version is the one asked for: the same version, or a version of the +// edition asked for (SNOMED CT: http://snomed.info/sct/ covers .../version/) +function versionMatches(csp, requested) { + if (typeof csp.isMyEdition === 'function') { + return csp.isMyEdition(requested); + } + const mine = csp.version() || ''; + return mine === requested || mine.startsWith(requested + '/version/'); +} + +module.exports = { chooseImplicitValueSetFactory }; diff --git a/tx/params.js b/tx/params.js index e5e8d2de..91bfe941 100644 --- a/tx/params.js +++ b/tx/params.js @@ -502,6 +502,19 @@ class TxParameters { return ''; } + // The code system version the request's version rules ask for - force-system-version, + // else system-version, else check-system-version - or null when they name none + requestedSystemVersion(systemURI) { + const rules = this.rulesForSystem(systemURI).filter(t => !t.vs); + for (const mode of ['override', 'default', 'check']) { + const rule = rules.find(t => t.mode === mode); + if (rule) { + return rule.version; + } + } + return null; + } + rulesForSystem(systemURI) { let result = []; for (let t of this.FVersionRules) { @@ -521,6 +534,18 @@ class TxParameters { } } + // The languages the request named - displayLanguage, else the HTTP Accept-Language - or + // null when it named none (workingLanguages() falls back to a default; this does not) + requestedLanguages() { + if (this.FHasDisplayLanguages && this.FDisplayLanguages) { + return this.FDisplayLanguages; + } + if (this.FHasHTTPLanguages && this.FHTTPLanguages) { + return this.FHTTPLanguages; + } + return null; + } + workingLanguages() { if (this.FDisplayLanguages) { return this.FDisplayLanguages; diff --git a/tx/provider.js b/tx/provider.js index c551072d..d5ebef2e 100644 --- a/tx/provider.js +++ b/tx/provider.js @@ -1,3 +1,4 @@ +const { chooseImplicitValueSetFactory } = require('./library/implicit-valuesets'); const { CodeSystem } = require("./library/codesystem"); const { assertNoActiveContent } = require('../library/narrative-check'); const {VersionUtilities} = require("../library/version-utilities"); @@ -254,7 +255,7 @@ class Provider { return null; } - async findValueSet(opContext, url, version) { + async findValueSet(opContext, url, version, systemVersionFor = null) { for (const vp of this.valueSetProviders) { if (opContext) opContext.deadCheck('findValueSet'); let vs = await vp.fetchValueSet(url, version); @@ -262,7 +263,7 @@ class Provider { return vs; } } - let vs1 = await this.findKnownValueSet(url, version); + let vs1 = await this.findKnownValueSet(url, version, systemVersionFor); return vs1; } @@ -325,7 +326,18 @@ class Provider { return result; } - async findKnownValueSet(url, version) { + async findKnownValueSet(url, version, systemVersionFor = null) { + // A code system with several versions loaded (SNOMED CT editions) says which form of + // implicit value set url is: the generic form comes from the version the request asks + // for (systemVersionFor, from system-version and the like) or else the default version + // (the one registered under the bare system), the edition form from the latest version + // of that edition, and the version form from that version alone - not from whichever + // version happens to be asked first. + const chosen = chooseImplicitValueSetFactory(this.codeSystemFactories.values(), system => this.codeSystemFactories.get(system), url, systemVersionFor); + if (chosen) { + const vs = await chosen.buildKnownValueSet(url, version); + return vs != null ? new ValueSet(vs) : null; + } for (let csp of this.codeSystemFactories.values()) { let vs = await csp.buildKnownValueSet(url, version); if (vs != null) { diff --git a/tx/sct/ecl.js b/tx/sct/ecl.js index f42bf0d4..7c6fb90c 100644 --- a/tx/sct/ecl.js +++ b/tx/sct/ecl.js @@ -701,7 +701,8 @@ class ECLParser { } /** - * Parse one "{{ ... }}" block. Returns { kind: 'M'|'D'|'C'|'+', filters } where + * Parse one "{{ ... }}" block (the D of a description filter is optional). Returns + * { kind: 'M'|'D'|'C'|'+', filters } where * filters is only populated for member filters; the other kinds keep their * raw token text so the evaluator can report them as unsupported. */ @@ -714,8 +715,13 @@ class ECLParser { } else if (this.match(ECLTokenType.SCTID) && /^[MDCmdc]$/.test(this.current.value)) { kind = this.current.value.toUpperCase(); this.advance(); + } else if (this.match(ECLTokenType.SCTID) && /^[A-Za-z_]/.test(this.current.value)) { + // ECL 2.2: descriptionFilterConstraint = "{{" ws [ "d" / "D" ] ws descriptionFilter ... + // - the D is optional, so a filter that starts with a field name (term, language, + // type, ...) is a description filter + kind = 'D'; } else { - this.error('Expected M, D, C or + after {{'); + this.error('Expected M, D, C, + or a description filter after {{'); } if (kind === 'C' || kind === 'D') { diff --git a/tx/sct/structures.js b/tx/sct/structures.js index ccc5e3e8..89bddf67 100644 --- a/tx/sct/structures.js +++ b/tx/sct/structures.js @@ -11,10 +11,20 @@ function adopt(buffer) { return fileBuffers.has(buffer.buffer) ? buffer : Buffer.from(buffer); } +// A structure made from a buffer is a loaded one, and is read-only: it may be sharing the +// cache file's memory (see adopt). Only a structure the importer creates empty and builds can +// be written to. Every method that changes a structure calls this first. +function checkWritable(structure) { + if (structure.readOnly) { + throw new Error(`${structure.constructor.name} is read-only: it was loaded from a cache file, and can only be changed while importing`); + } +} + class SnomedStrings { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.decoder = new TextDecoder('utf-8'); this.builder = null; this.currentOffset = 0; // ADD: Track offset directly @@ -39,11 +49,13 @@ class SnomedStrings { } startBuild() { + checkWritable(this); this.builder = []; this.currentOffset = 0; // RESET: offset when starting build } reopen() { + checkWritable(this); this.builder = [this.master]; this.currentOffset = this.master.length; // SET: offset to current master length } @@ -86,6 +98,7 @@ class SnomedStrings { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); this.currentOffset = 0; // RESET: offset when clearing } @@ -98,6 +111,7 @@ class SnomedStrings { class SnomedWords { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; } @@ -125,6 +139,7 @@ class SnomedWords { } startBuild() { + checkWritable(this); this.builder = []; } @@ -151,6 +166,7 @@ class SnomedWords { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); } @@ -162,6 +178,7 @@ class SnomedWords { class SnomedStems { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; } @@ -189,6 +206,7 @@ class SnomedStems { } startBuild() { + checkWritable(this); this.builder = []; } @@ -215,6 +233,7 @@ class SnomedStems { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); } @@ -226,6 +245,7 @@ class SnomedStems { class SnomedReferences { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; this.currentOffset = 0; // ADD: Track offset directly this.MAGIC_NO_CHILDREN = 0xFFFFFFFF; // Assuming this constant - adjust as needed @@ -274,6 +294,7 @@ class SnomedReferences { } startBuild() { + checkWritable(this); this.builder = []; this.currentOffset = 0; // RESET: offset when starting build } @@ -327,6 +348,7 @@ class SnomedReferences { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); this.currentOffset = 0; // RESET: offset when clearing } @@ -354,6 +376,7 @@ class SnomedDescriptions { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; this.currentOffset = 0; // ADD: Track offset directly } @@ -393,6 +416,7 @@ class SnomedDescriptions { } startBuild() { + checkWritable(this); this.builder = []; this.currentOffset = 0; // RESET: offset when starting build } @@ -428,7 +452,44 @@ class SnomedDescriptions { return currentOffset; } + /** + * The concepts that have an active description in the language (the RF2 language code + * byte) whose term passes test, for descriptions [start, end) by position, leaving out + * descriptions of type skipKind (a concept index, or -1 for none). Reads the records in + * place rather than decoding each one, since a scan covers the whole edition. Adds the + * concept indexes to out (a Set); returns how many descriptions there are. + */ + matchActiveTerms(start, end, lang, strings, test, out, skipKind = -1) { + const size = SnomedDescriptions.DESC_SIZE; + const count = Math.floor(this.master.length / size); + const last = Math.min(end, count); + for (let i = start; i < last; i++) { + const offset = i * size; + if (this.master.readUInt8(offset + 4) === 0 || this.master.readUInt8(offset + 31) !== lang + || this.master.readUInt32LE(offset + 21) === skipKind) { + continue; + } + if (test(strings.getEntry(this.master.readUInt32LE(offset)))) { + out.add(this.master.readUInt32LE(offset + 13)); + } + } + return count; + } + + /** The language code bytes that active descriptions use */ + activeLanguages() { + const size = SnomedDescriptions.DESC_SIZE; + const langs = new Set(); + for (let offset = 0; offset + size <= this.master.length; offset += size) { + if (this.master.readUInt8(offset + 4) !== 0) { + langs.add(this.master.readUInt8(offset + 31)); + } + } + return langs; + } + setRefsets(index, refsets, valueses) { + checkWritable(this); if (index >= this.master.length) { throw new Error('Wrong length index getting snomed Desc Details'); } @@ -452,6 +513,7 @@ class SnomedDescriptions { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); this.currentOffset = 0; // RESET: offset when clearing } @@ -464,6 +526,7 @@ class SnomedDescriptions { class SnomedDescriptionIndex { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; } @@ -499,6 +562,7 @@ class SnomedDescriptionIndex { } startBuild() { + checkWritable(this); this.builder = []; } @@ -528,6 +592,7 @@ class SnomedDescriptionIndex { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); } @@ -549,6 +614,7 @@ class SnomedConceptList { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; this.currentOffset = 0; // ADD: Track offset directly } @@ -680,6 +746,7 @@ class SnomedConceptList { } setNormalFormDuringBuild(index, value) { + checkWritable(this); // Special version that can be called during building phase // This bypasses the post-build access check for normal forms specifically if (index >= this.master.length) { @@ -720,67 +787,80 @@ class SnomedConceptList { // All the setter methods - these require doneBuild() to have been called setParents(index, active, inactive) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(active, index + 9); this.master.writeUInt32LE(inactive, index + 52); } setDescriptions(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 13); } setModuleId(index, moduleIndex) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(moduleIndex, index + 36); } setInbounds(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 17); } setOutbounds(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 21); } setAllDesc(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 25); } setDepth(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt8(value, index + 29); } setStems(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 30); } setDate(index, effectiveTime) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt16LE(effectiveTime, index + 34); } setStatus(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 40); } setRefsets(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 44); } setNormalForm(index, value) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt32LE(value, index + 48); } setFlag(index, flags) { + checkWritable(this); this.#checkPostBuildAccess(index); this.master.writeUInt8(flags, index + 8); } @@ -791,6 +871,7 @@ class SnomedConceptList { // Build methods startBuild() { + checkWritable(this); this.builder = []; this.currentOffset = 0; // RESET: offset when starting build } @@ -842,6 +923,7 @@ class SnomedConceptList { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); this.currentOffset = 0; // RESET: offset when clearing } @@ -858,6 +940,7 @@ class SnomedRelationshipList { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; this.currentOffset = 0; // ADD: Track offset directly } @@ -882,12 +965,21 @@ class SnomedRelationshipList { }; } + // Whether the relationship row at index is active, without decoding the whole row + isActive(index) { + if (index >= this.master.length) { + throw new Error('Wrong length index getting snomed relationship Details'); + } + return this.master.readUInt8(index + 26) !== 0; + } + count() { return Math.floor(this.master.length / SnomedRelationshipList.RELATIONSHIP_SIZE); } // Build methods startBuild() { + checkWritable(this); this.builder = []; this.currentOffset = 0; // RESET: offset when starting build } @@ -935,6 +1027,7 @@ class SnomedRelationshipList { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); this.currentOffset = 0; // RESET: offset when clearing } @@ -949,6 +1042,7 @@ class SnomedReferenceSetMembers { constructor(buffer = null) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; this.currentOffset = 0; // ADD: Track offset directly } @@ -1018,6 +1112,7 @@ class SnomedReferenceSetMembers { } startBuild() { + checkWritable(this); this.builder = []; this.currentOffset = 0; // RESET: offset when starting build } @@ -1112,6 +1207,7 @@ class SnomedReferenceSetMembers { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); this.currentOffset = 0; // RESET: offset when clearing } @@ -1127,6 +1223,7 @@ class SnomedReferenceSetIndex { constructor(buffer = null, hasLangs = true) { this.master = adopt(buffer); + this.readOnly = !!buffer; this.builder = null; this.hasLangs = hasLangs; this.recordSize = hasLangs ? SnomedReferenceSetIndex.REFSET_SIZE_LANG : SnomedReferenceSetIndex.REFSET_SIZE_NOLANG; @@ -1197,6 +1294,7 @@ class SnomedReferenceSetIndex { // Build methods startBuild() { + checkWritable(this); this.builder = []; } @@ -1233,6 +1331,7 @@ class SnomedReferenceSetIndex { } clear() { + checkWritable(this); this.master = Buffer.alloc(0); } diff --git a/tx/tx.fhir.org.yml b/tx/tx.fhir.org.yml index c8473be0..2eef815b 100644 --- a/tx/tx.fhir.org.yml +++ b/tx/tx.fhir.org.yml @@ -20,8 +20,8 @@ sources: - internal:hgvs - ucum:tx/data/ucum-essence.xml - loinc:loinc-2.77-a.db - - loinc!:Loinc_2.83.db - - rxnorm:rxnorm_03022026.db + - loinc!:Loinc_2.83b.db + - rxnorm:rxnorm_20261005.cache - ndc:ndc-20211101.db - unii:unii_20240622.db - snomed:sct_intl_20250201.cache @@ -32,7 +32,7 @@ sources: - snomed:sct_us_20240301.cache - snomed:sct_us_20250901b.cache - snomed:Snomed_loinc_20261021.cache - - snomed:sct_test_20250909e.cache + - snomed:sct_test_20250909i.cache - cpt:CodeSystem-cpt.db|cpt-2023-fragment-0.1.db - omop:omop_v20250227.db - npm:hl7.terminology=tho diff --git a/tx/workers/cache-control.js b/tx/workers/cache-control.js index 4d9d58b0..62c277c2 100644 --- a/tx/workers/cache-control.js +++ b/tx/workers/cache-control.js @@ -97,7 +97,8 @@ class CacheControlWorker extends TerminologyWorker { /** * mode=start: mint a server-issued cache-id, create the (per-endpoint) cache, - * front-load any supplied resources into it, and return the id in the body. + * front-load any supplied resources into it, and return the id in the body + * (and, for proxy request logs, in the `${CACHE_ID_HEADER}` response header). * * The id is returned in the body rather than a header to keep it readable by * browser clients without CORS expose-header configuration; subsequent calls @@ -138,6 +139,12 @@ class CacheControlWorker extends TerminologyWorker { const cacheId = cache.newCacheId(); cache.set(cacheId, resources, sealed); + // Also echoed as a response header, so that a request log at the proxy (nginx + // $sent_http_x_cache_id) can tie each issued id to the start request that got + // it - which is what replaying a captured request log needs, because a replay + // target issues different ids. The body stays the authoritative carrier (see above). + res.setHeader(CACHE_ID_HEADER, cacheId); + return res.status(200).json({ resourceType: 'Parameters', parameter: [ diff --git a/tx/workers/expand.js b/tx/workers/expand.js index 606de1c9..fdd6a7f9 100644 --- a/tx/workers/expand.js +++ b/tx/workers/expand.js @@ -201,6 +201,7 @@ class ValueSetExpander { params; doingVersion = true; excludedSystems = new Set(); + excludedProviders = []; // {system, cs} - see checkForExclusionVersionSpecialCase excluded = new Set(); hasExclusions = false; requiredSupplements = new Set(); @@ -271,11 +272,21 @@ class ValueSetExpander { return true; } for (const s of this.params.designations) { - const [l, r] = s.split('|'); - if (cd.use != null && cd.use.system === l && cd.use.code === r) { + // a token: system|code, or a bare code (or |code), which matches a language or a use + // code from any system. designation=nl-NL is the same as urn:ietf:bcp:47|nl-NL + const bar = s.indexOf('|'); + const system = bar < 0 ? '' : s.substring(0, bar); + const code = bar < 0 ? s : s.substring(bar + 1); + if (!system) { + if ((cd.language != null && cd.language.code === code) || (cd.use != null && cd.use.code === code)) { + return true; + } + continue; + } + if (cd.use != null && cd.use.system === system && cd.use.code === code) { return true; } - if (cd.language != null && l === 'urn:ietf:bcp:47' && r === cd.language.code) { + if (cd.language != null && system === 'urn:ietf:bcp:47' && code === cd.language.code) { return true; } } @@ -300,7 +311,7 @@ class ValueSetExpander { } } - includeCode(cs, parent, system, version, code, isAbstract, isInactive, deprecated, status, displays, definition, itemWeight, expansion, imports, csExtList, vsExtList, csProps, expProps, excludeInactive, srcURL) { + async includeCode(cs, parent, system, version, code, isAbstract, isInactive, deprecated, status, displays, definition, itemWeight, expansion, imports, csExtList, vsExtList, csProps, expProps, excludeInactive, srcURL) { let result = null; this.worker.deadCheck('processCode'); @@ -310,7 +321,7 @@ class ValueSetExpander { if (isInactive && excludeInactive) { return null; } - if (this.isExcluded(system, version, code)) { + if (await this.isExcluded(system, version, code)) { return null; } @@ -612,7 +623,7 @@ class ValueSetExpander { let count = 0; await this.worker.checkAndYield('importValueSetItem'); const s = this.keyC(c); - if (this.passesImports(imports, c.system, c.code, offset) && !this.map.has(s) && !this.isExcluded(c.system, c.version, c.code)) { + if (this.passesImports(imports, c.system, c.code, offset) && !this.map.has(s) && !(await this.isExcluded(c.system, c.version, c.code))) { count++; if (imported) { imported.push(c); @@ -815,6 +826,14 @@ class ValueSetExpander { Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", "The code System '" + cs.system() + "' has a grammar and so has infinite members"); } notClosed.value = true; + } else if (cs.openCodeSpaceReason()) { + // all the codes the code system defines are listed, but not the ones users define in its + // reserved code spaces: $validate-code accepts those as members, so the expansion isn't closed + if (!Extensions.has(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed")) { + Extensions.addBoolean(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", true); + Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", cs.openCodeSpaceReason()); + } + notClosed.value = true; } if (this.count === 0 && filters.length === 0 && valueSets.length === 0) { @@ -930,7 +949,7 @@ class ValueSetExpander { if (cset.filter && cset.filter.length > 0) { this.worker.opContext.log('prepare filters'); const fcl = cset.filter; - const prep = await cs.getPrepContext(true); + const prep = await cs.getPrepContext(true, this.params.requestedLanguages()); if (!filter.isNull) { await cs.searchFilter(prep, filter, true); @@ -1086,15 +1105,11 @@ class ValueSetExpander { // excluding a whole system - we don't list the codes in this case this.excludedSystems.add(cset.system + (this.doingVersion && cset.version ? '|' + cset.version : '')); } else { - const iter = await cs.iteratorAll(); - if (iter) { - let c = await cs.nextContext(iter); - while (c) { - await this.worker.checkAndYield('processCodes#3aa'); - this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url); - c = await cs.nextContext(iter); - } - } + // include one version, exclude another whole version: a code is + // excluded if it exists in the excluded version. We don't list + // the excluded version's codes (most providers can't - e.g. + // SNOMED CT, LOINC); we ask it about each included code instead + this.excludedProviders.push({ system: cs.system(), cs }); } } else { if (cs.isNotClosed(filter)) { @@ -1110,15 +1125,15 @@ class ValueSetExpander { } } - const iter = await cs.iteratorAll(); - if (iter) { - let c = await cs.nextContext(iter); - while (c) { - await this.worker.checkAndYield('processCodes#3a'); - if (await this.passesFilters(cs, c, prep, filters, 0)) { - this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url); - } - c = await cs.nextContext(iter); + // the codes of this system that are in all the value sets. Walk the + // first value set's codes, not the code system (most providers + // can't list theirs - e.g. SNOMED CT, LOINC); excludeCode checks + // the other value sets + const system = cs.system(); + for (const entry of valueSets[0].codeMap.values()) { + await this.worker.checkAndYield('processCodes#3a'); + if (entry.system === system && await this.#knownTo(cs, entry.code)) { + this.excludeCode(cs, system, cs.version(), entry.code, expansion, valueSets, vsSrc.url); } } } @@ -1146,7 +1161,7 @@ class ValueSetExpander { if (cset.filter) { this.worker.opContext.log('prep filters'); - const prep = await cs.getPrepContext(true); + const prep = await cs.getPrepContext(true, this.params.requestedLanguages()); if (!filter.isNull) { await cs.searchFilter(prep, filter, true); } @@ -1591,6 +1606,10 @@ class ValueSetExpander { } } + // declare only the properties the returned codes use: codes outside the page (or a count=0 + // expansion) declared theirs as they were added, and a declaration nothing uses is noise + this.#pruneUnusedPropertyDeclarations(exp); + if (result.expansion.contains && result.expansion.contains.length > 0) { let hasChildren = false; for (let c of result.expansion.contains) { @@ -1729,11 +1748,31 @@ class ValueSetExpander { } + #pruneUnusedPropertyDeclarations(exp) { + if (!exp.property || exp.property.length === 0) { + return; + } + const used = new Set(); + const walk = (list) => { + for (const c of list || []) { + for (const p of c.property || []) { + used.add(p.code); + } + walk(c.contains); + } + }; + walk(exp.contains); + exp.property = exp.property.filter(pd => used.has(pd.code)); + if (exp.property.length === 0) { + delete exp.property; + } + } + hasParam(exp, name, value) { return (exp.parameter || []).find((ex => ex.name == name && getValuePrimitive(ex) == value)); } - isExcluded(system, version, code) { + async isExcluded(system, version, code) { if (this.excludedSystems.has(system)) { return true; } @@ -1742,7 +1781,27 @@ class ValueSetExpander { return true; } key = (this.doingVersion && !this.params.versionsMatch ? system+'|'+version : system)+'#'+code; - return this.excluded.has(key); + if (this.excluded.has(key)) { + return true; + } + for (const ep of this.excludedProviders) { + if (ep.system === system && await this.#knownTo(ep.cs, code)) { + return true; + } + } + return false; + } + + /** + * @returns {Promise} whether the code exists in that provider's code system (version) + */ + async #knownTo(cs, code) { + try { + const loc = await cs.locate(code); + return !!(loc && loc.context); + } catch (e) { + return false; + } } keyS(system, version, code) { @@ -2110,6 +2169,9 @@ class ExpandWorker extends TerminologyWorker { let txp = new TxParameters(this.opContext.i18n.languageDefinitions, this.opContext.i18n, false); txp.readParams(params); + // set now, not only in performExpansion: finding the value set by url can depend on them + // (an implicit value set is drawn from the code system version the request asks for) + this.params = txp; // If no valueSet yet, try to find by url if (!valueSet) { @@ -2174,6 +2236,9 @@ class ExpandWorker extends TerminologyWorker { let txp = new TxParameters(this.opContext.i18n.languageDefinitions, this.opContext.i18n, false); txp.readParams(params); + // set now, not only in performExpansion: finding the value set by url can depend on them + // (an implicit value set is drawn from the code system version the request asks for) + this.params = txp; // Perform the expansion return await this.doExpand(valueSet, txp, logExtraOutput); @@ -2222,6 +2287,9 @@ class ExpandWorker extends TerminologyWorker { let txp = new TxParameters(this.opContext.i18n.languageDefinitions, this.opContext.i18n, false); txp.readParams(params); + // set now, not only in performExpansion: finding the value set by url can depend on them + // (an implicit value set is drawn from the code system version the request asks for) + this.params = txp; // Perform the expansion const result = await this.doExpand(valueSet, txp, logExtraOutput); diff --git a/tx/workers/validate.js b/tx/workers/validate.js index 8fc01f0b..0779cf54 100644 --- a/tx/workers/validate.js +++ b/tx/workers/validate.js @@ -1795,7 +1795,7 @@ class ValueSetChecker { if (cset.filter) { let cfl = cset.filter; - let prep = await cs.getPrepContext(false); + let prep = await cs.getPrepContext(false, this.params.requestedLanguages()); for (let fc of cfl) { this.worker.deadCheck('checkConceptSet#2'); if (!fc.value) { @@ -2374,7 +2374,12 @@ class ValidateWorker extends TerminologyWorker { return fromAdditional; } - let vs = await this.provider.findValueSet(this.opContext, url, version); + // an implicit value set in the generic form is drawn from the code system version the + // request asks for (system-version and the like) + const systemVersionFor = txParams && txParams.requestedSystemVersion + ? (system) => txParams.requestedSystemVersion(system) + : null; + let vs = await this.provider.findValueSet(this.opContext, url, version, systemVersionFor); this.seeSourceVS(vs, url); if (vs == null) { throw new Issue('error', 'not-found', null, 'Unable_to_resolve_value_Set_', this.i18n.translate('Unable_to_resolve_value_Set_', params.HTTPLanguages, [url+(version ? "|"+version : "")]), 'not-found', 422); diff --git a/tx/workers/worker.js b/tx/workers/worker.js index b52e9056..dd3d27b0 100644 --- a/tx/workers/worker.js +++ b/tx/workers/worker.js @@ -420,7 +420,12 @@ class TerminologyWorker { // Then try the provider if (this.provider && this.provider.findValueSet) { - const vs = await this.provider.findValueSet(this.opContext, effectiveUrl, effectiveVersion); + // an implicit value set in the generic form (http://snomed.info/sct?fhir_vs...) is drawn + // from the code system version the request asks for, if it asks for one + const systemVersionFor = this.params && this.params.requestedSystemVersion + ? (system) => this.params.requestedSystemVersion(system) + : null; + const vs = await this.provider.findValueSet(this.opContext, effectiveUrl, effectiveVersion, systemVersionFor); if (vs) { return vs; }