From 319f353bfae006af6bfc14d2ae773a04c5084577 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 11:01:12 -0500 Subject: [PATCH 1/2] Extract shared guest-link library handling Module and flow compilers share the guest-link library contract. Co-Authored-By: Codex --- src/compiler/guestLinkLibraries.js | 47 ++++++++++++++++++++++++++++++ src/flow/flowCompiler.js | 42 +------------------------- 2 files changed, 48 insertions(+), 41 deletions(-) create mode 100644 src/compiler/guestLinkLibraries.js diff --git a/src/compiler/guestLinkLibraries.js b/src/compiler/guestLinkLibraries.js new file mode 100644 index 0000000..8889cd5 --- /dev/null +++ b/src/compiler/guestLinkLibraries.js @@ -0,0 +1,47 @@ +import path from "node:path"; +import { readFile } from "node:fs/promises"; +import { bytesToHex } from "../utils/encoding.js"; +import { sha256Bytes } from "../utils/crypto.js"; + +export async function readGuestLinkLibraries(guestLink, pluginId) { + const listed = guestLink.metadata?.libraries; + if (listed === undefined || listed === null) return []; + if (!Array.isArray(listed)) { + throw new Error(`Plugin "${pluginId}": guest-link metadata.libraries must be an array.`); + } + const libraries = []; + for (const library of listed) { + const file = String(library?.file ?? ""); + const resolved = path.resolve(guestLink.guestLinkDir, file); + if (!file || !resolved.startsWith(guestLink.guestLinkDir + path.sep)) { + throw new Error(`Plugin "${pluginId}": guest-link library ${JSON.stringify(library?.name)} has no file inside its guest-link dir.`); + } + if (!/^sdm_lib_[A-Za-z0-9_]+_$/.test(String(library?.namespace ?? ""))) { + throw new Error(`Plugin "${pluginId}": guest-link library ${JSON.stringify(library?.name)} has no sdm_lib_ namespace.`); + } + const objectBytes = new Uint8Array(await readFile(resolved)); + const sha256 = bytesToHex(await sha256Bytes(objectBytes)); + if (sha256 !== library.sha256) { + throw new Error( + `Plugin "${pluginId}": guest-link library ${file} hashes to ${sha256}, but its metadata records ${library.sha256}.`, + ); + } + libraries.push({ ...library, objectBytes }); + } + return libraries; +} + +// Each shared library namespace once, in first-use order. A namespace is the +// library's name, version and a digest of its source, defines, flags and +// toolchain kind, so two modules carrying the same namespace carry the same +// library and the flow links one copy of it. +export function collectFlowLibraryObjects(linkedDependencies) { + const byNamespace = new Map(); + for (const dependency of linkedDependencies) { + for (const library of dependency.guestLink?.libraries ?? []) { + if (!byNamespace.has(library.namespace)) byNamespace.set(library.namespace, library); + } + } + return Array.from(byNamespace.values()); +} + diff --git a/src/flow/flowCompiler.js b/src/flow/flowCompiler.js index 6a1c8e5..b6bd89d 100644 --- a/src/flow/flowCompiler.js +++ b/src/flow/flowCompiler.js @@ -1,3 +1,4 @@ +import { readGuestLinkLibraries, collectFlowLibraryObjects } from "../compiler/guestLinkLibraries.js"; /** * SDN flow compiler (loop C.3c) — `space-data-module flow check|compile`. * @@ -427,47 +428,6 @@ async function loadDependencyFromDirectory(dir) { return entry; } -async function readGuestLinkLibraries(guestLink, pluginId) { - const listed = guestLink.metadata?.libraries; - if (listed === undefined || listed === null) return []; - if (!Array.isArray(listed)) { - throw new Error(`Plugin "${pluginId}": guest-link metadata.libraries must be an array.`); - } - const libraries = []; - for (const library of listed) { - const file = String(library?.file ?? ""); - const resolved = path.resolve(guestLink.guestLinkDir, file); - if (!file || !resolved.startsWith(guestLink.guestLinkDir + path.sep)) { - throw new Error(`Plugin "${pluginId}": guest-link library ${JSON.stringify(library?.name)} has no file inside its guest-link dir.`); - } - if (!/^sdm_lib_[A-Za-z0-9_]+_$/.test(String(library?.namespace ?? ""))) { - throw new Error(`Plugin "${pluginId}": guest-link library ${JSON.stringify(library?.name)} has no sdm_lib_ namespace.`); - } - const objectBytes = new Uint8Array(await readFile(resolved)); - const sha256 = bytesToHex(await sha256Bytes(objectBytes)); - if (sha256 !== library.sha256) { - throw new Error( - `Plugin "${pluginId}": guest-link library ${file} hashes to ${sha256}, but its metadata records ${library.sha256}.`, - ); - } - libraries.push({ ...library, objectBytes }); - } - return libraries; -} - -// Each shared library namespace once, in first-use order. A namespace is the -// library's name, version and a digest of its source, defines, flags and -// toolchain kind, so two modules carrying the same namespace carry the same -// library and the flow links one copy of it. -function collectFlowLibraryObjects(linkedDependencies) { - const byNamespace = new Map(); - for (const dependency of linkedDependencies) { - for (const library of dependency.guestLink?.libraries ?? []) { - if (!byNamespace.has(library.namespace)) byNamespace.set(library.namespace, library); - } - } - return Array.from(byNamespace.values()); -} async function scanForDependencyDirs(root, found) { let entries; From dcb5c0923e4b571a43868729dc1928b83f18c8d1 Mon Sep 17 00:00:00 2001 From: TJKoury Date: Mon, 16 Nov 2026 11:09:11 -0500 Subject: [PATCH 2/2] Support prebuilt guest-link objects and archives in module compilation Verify hashes and kinds through shared guest-link handling. Preserve link order and producer symbols, carry libraries into guest-link output, and reject conflicting flow inputs. Add CLI input, documentation, real compile coverage including PEPPER, and version 0.10.2. Co-Authored-By: Codex --- CHANGELOG.md | 6 + README.md | 42 ++++++ bin/space-data-module.js | 17 ++- package-lock.json | 4 +- package.json | 2 +- src/compiler/compileModule.js | 44 ++++-- src/compiler/guestLinkFiles.js | 16 +-- src/compiler/guestLinkLibraries.js | 115 +++++++++++++--- src/flow/flowCompiler.js | 75 +++++----- src/index.d.ts | 15 +- test/prebuilt-libraries.test.js | 214 +++++++++++++++++++++++++++++ 11 files changed, 467 insertions(+), 83 deletions(-) create mode 100644 test/prebuilt-libraries.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 6193d96..0939018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.10.2 + +- MODULE compilation accepts hash-verified prebuilt WebAssembly objects and static archives through `prebuiltLibraries`. Input order and producer symbols are preserved; input bytes participate in the link cache. +- Guest-link output carries prebuilt libraries into FLOW compilation. Identical names and hashes link once; conflicting content for a name or namespace is refused. +- The compile CLI accepts guest-link metadata with `--prebuilt-libraries`. Artifact guards retain their checks and include library names in failures. + ## 0.9.16 ### hd-wallet-wasm 2.1.0 -> 2.2.2: off both published advisories diff --git a/README.md b/README.md index b99f767..c7d0eed 100644 --- a/README.md +++ b/README.md @@ -556,6 +556,48 @@ const bundle = await createSingleFileBundle({ const parsed = await parseSingleFileBundle(bundle.wasmBytes); ``` +`prebuiltLibraries` links relocatable WebAssembly objects and static `ar` archives after +all module and source-library objects, in the supplied order: + +```js +import { readFile } from "node:fs/promises"; +import { writeGuestLinkFiles } from "space-data-module-sdk/compiler"; + +const compilation = await compileModuleFromSource({ + manifest, + sourceCode, + language: "c++", + threadModel: "wasi-sequential", + prebuiltLibraries: [{ + name: "vector_math", + version: "1.0.0", + language: "c", // also "c++" or "fortran" + namespace: "sdm_lib_vector_math_", + kind: "archive", // or "wasm-object" + sha256: releaseMetadata.sha256, // the producer's pinned SHA-256 + objectBytes: new Uint8Array(await readFile("./libvector_math.a")), + }], +}); +await writeGuestLinkFiles(compilation, "./dist/guest-link"); +``` + +The compiler verifies each hash before use. It preserves the producer's symbols; +`namespace` identifies a library and does not rename prebuilt code. Producers supply +compatible symbols and a toolchain compatible with the selected thread model. +Archives retain lazy member extraction. Exact input bytes participate in the link +cache key. Linked artifacts pass the same thread, import, SIMD, and sequential +checks; errors include the participating library names. + +Guest-link output includes each library's bytes, kind, and SHA-256. Flow compilation +links identical names and hashes once, in first-use order. Conflicting hashes for a +name or namespace are refused. Existing metadata without `kind` is accepted based +on the file's contents. Source libraries remain available through `libraries`. + +The CLI accepts `--prebuilt-libraries ./dist/guest-link/metadata.json`. Its +`libraries` entries name files relative to that metadata file and carry the same +name, version, language, namespace, kind, and SHA-256 fields. +`--thread-model wasi-sequential` selects the sequential WASI compile profile. + Each subsystem is also available as a subpath export: ```js diff --git a/bin/space-data-module.js b/bin/space-data-module.js index b919c35..4fb24e6 100755 --- a/bin/space-data-module.js +++ b/bin/space-data-module.js @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { readGuestLinkLibraries } from "../src/compiler/guestLinkLibraries.js"; import path from "node:path"; import process from "node:process"; @@ -93,6 +94,12 @@ function parseArgs(argv) { case "--wasm": options.wasmPath = path.resolve(requireValue(argv, ++index, value)); break; + case "--thread-model": + options.threadModel = requireValue(argv, ++index, value); + break; + case "--prebuilt-libraries": + options.prebuiltLibrariesPath = path.resolve(requireValue(argv, ++index, value)); + break; case "--source": options.sourcePath = path.resolve(requireValue(argv, ++index, value)); break; @@ -234,7 +241,7 @@ function printUsage() { console.log(`Usage: space-data-module check --repo-root . space-data-module check --manifest ./manifest.json --wasm ./dist/module.wasm - space-data-module compile --manifest ./manifest.json --source ./src/module.c --out ./dist/module.wasm + space-data-module compile --manifest ./manifest.json --source ./src/module.c --out ./dist/module.wasm [--prebuilt-libraries ./dist/guest-link/metadata.json] [--thread-model wasi-sequential] space-data-module init --family propagator --name my-propagator space-data-module init --family propagator --name my-propagator --out ./modules/my-propagator [--plugin-id com.orbpro.my.propagator] [--force] [--json] space-data-module parity --wasm ./dist/isomorphic/module.wasm --fixture ./fixtures/parity/basic.json @@ -628,8 +635,16 @@ async function runCompile(argv) { } const manifest = await loadManifestFromFile(options.manifestPath); const sourceCode = await readFile(options.sourcePath, "utf8"); + const prebuiltLibraries = options.prebuiltLibrariesPath + ? await readGuestLinkLibraries({ + guestLinkDir: path.dirname(options.prebuiltLibrariesPath), + metadata: JSON.parse(await readFile(options.prebuiltLibrariesPath, "utf8")), + }, manifest.pluginId) + : undefined; const result = await compileModuleFromSource({ manifest, + prebuiltLibraries, + threadModel: options.threadModel, sourceCode, language: options.language, outputPath: options.outputPath, diff --git a/package-lock.json b/package-lock.json index 30cfae4..397b92c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "space-data-module-sdk", - "version": "0.10.1", + "version": "0.10.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "space-data-module-sdk", - "version": "0.10.1", + "version": "0.10.2", "license": "Apache-2.0", "dependencies": { "flatbuffers": "^25.9.23", diff --git a/package.json b/package.json index df5bb4e..932c05d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "space-data-module-sdk", - "version": "0.10.1", + "version": "0.10.2", "license": "Apache-2.0", "description": "Module SDK for building, validating, signing, and deploying WebAssembly modules on the Space Data Network.", "type": "module", diff --git a/src/compiler/compileModule.js b/src/compiler/compileModule.js index 0980e36..e7b5e9e 100644 --- a/src/compiler/compileModule.js +++ b/src/compiler/compileModule.js @@ -1,3 +1,4 @@ +import { normalizePrebuiltLibraries, collectGuestLinkLibraries, guestLinkLibraryEntries, withLibraryContext } from "./guestLinkLibraries.js"; import os from "node:os"; import path from "node:path"; import { execFile } from "node:child_process"; @@ -642,7 +643,17 @@ function createGuestLinkBundleEntries(guestLink) { if (!guestLink?.objectBytes || guestLink.objectBytes.length === 0) { return []; } + const libraries = guestLinkLibraryEntries(guestLink.libraries); return [ + ...libraries.map(({ metadata, objectBytes }) => ({ + entryId: metadata.file, + role: "auxiliary", + sectionName: SDS_GUEST_LINK_SECTION_NAME, + payloadEncoding: "raw-bytes", + mediaType: metadata.kind === "archive" ? "application/octet-stream" : "application/wasm", + payload: objectBytes, + description: `Guest-link library ${metadata.name}.`, + })), { entryId: SDS_GUEST_LINK_OBJECT_ENTRY_ID, role: "auxiliary", @@ -659,7 +670,8 @@ function createGuestLinkBundleEntries(guestLink) { payloadEncoding: "json-utf8", mediaType: SDS_GUEST_LINK_MEDIA_TYPE, payload: { - version: 1, + version: libraries.length ? 2 : 1, + ...(libraries.length ? { libraries: libraries.map(({ metadata }) => metadata) } : {}), format: "wasm-object", symbolPrefix: guestLink.symbolPrefix, methodSymbols: guestLink.methodSymbols, @@ -1068,6 +1080,10 @@ async function compileWithToolchain(kind, options = {}) { outputPath ?? path.join(tempDir, "module.wasm"), ); const libraries = normalizeLibraries(compileOptions.libraries); + const prebuilt = compileOptions.prebuiltLibraries.map((library, index) => ({ + library, namespace: library.namespace, kind: library.kind, objectBytes: library.objectBytes, + object: `prebuilt-${index}.${library.kind === "archive" ? "a" : "o"}`, + })); const profile = await resolveToolchainProfile(kind, { language, sourceCompilerCommand, @@ -1110,7 +1126,7 @@ async function compileWithToolchain(kind, options = {}) { argv: profile.compileArgv(step.role, at(step.source), at(step.object), step.extra ?? [], step.library?.language), })); const linkOutput = "module.wasm"; - const linkObjects = compileSteps.map((step) => step.object); + const linkObjects = [...compileSteps.map((step) => step.object), ...prebuilt.map((library) => library.object)]; const linkArgv = profile.linkArgv(linkObjects.map(at), at(linkOutput), linkArgs); const cacheReport = { enabled: Boolean(cache), dir: cache?.root ?? null, hits: [], misses: [], stepMs: {} }; @@ -1144,10 +1160,8 @@ async function compileWithToolchain(kind, options = {}) { const objects = new Map(); let linked = null; const prepareLinkInputs = () => { - if (libraries.length === 0) { - return { inputs: new Map(objects), libraries: [] }; - } - const inputs = new Map(objects); + const inputs = new Map([...objects, ...prebuilt.map((library) => [library.object, library.objectBytes])]); + if (libraries.length === 0) return { inputs, libraries: prebuilt }; const references = new Map(); const namespaced = libraries.map((library) => { const step = compileSteps.find((entry) => entry.library === library); @@ -1183,7 +1197,11 @@ async function compileWithToolchain(kind, options = {}) { { undefinedOnly: true }, ), ); - return { inputs, libraries: namespaced }; + const all = [...namespaced, ...prebuilt]; + collectGuestLinkLibraries(all.map(({ library, namespace, objectBytes }) => ({ + ...library, namespace, sha256: sha256Hex(objectBytes), + }))); + return { inputs, libraries: all }; }; if (cache) { @@ -1280,11 +1298,12 @@ async function compileWithToolchain(kind, options = {}) { objectBytes: guestLink.objectBytes, ...(linked.libraries.length > 0 ? { - libraries: linked.libraries.map(({ library, namespace, objectBytes }) => ({ + libraries: linked.libraries.map(({ library, namespace, kind, objectBytes }) => ({ name: library.name, version: library.version, language: library.language, namespace, + ...(kind ? { kind } : {}), sha256: sha256Hex(objectBytes), objectBytes, })), @@ -1317,6 +1336,7 @@ export async function compileModuleFromSource(options = {}) { throw error; } + const prebuiltLibraries = await normalizePrebuiltLibraries(options.prebuiltLibraries); const compiler = selectCompiler(options.language); const invokeSurfaces = resolveInvokeSurfaces(manifest); const includeCommandMain = invokeSurfaces.includes(InvokeSurface.COMMAND); @@ -1356,6 +1376,7 @@ export async function compileModuleFromSource(options = {}) { const simd128 = resolveSimd128(options.simd128, threadModel); const compileOptions = { ...options, + prebuiltLibraries, noEntry: includeCommandMain !== true, threadModel, simd128, @@ -1465,6 +1486,11 @@ export async function compileModuleFromSource(options = {}) { ...standardsOptions, }); + if (prebuiltLibraries.length && !report.ok) { + const error = new Error(`Linked artifact validation failed: ${JSON.stringify(report.errors)}`); + error.report = report; + throw error; + } handedOff = keepTempDir; return { compiler: useWasiThreads @@ -1490,6 +1516,8 @@ export async function compileModuleFromSource(options = {}) { simd128: simd128Report, report, }; + } catch (error) { + throw withLibraryContext(error, prebuiltLibraries); } finally { if (!handedOff) { await rm(tempDir, { recursive: true, force: true }); diff --git a/src/compiler/guestLinkFiles.js b/src/compiler/guestLinkFiles.js index 889928b..05789aa 100644 --- a/src/compiler/guestLinkFiles.js +++ b/src/compiler/guestLinkFiles.js @@ -17,6 +17,8 @@ import path from "node:path"; import { mkdir, rm, writeFile } from "node:fs/promises"; +import { guestLinkLibraryEntries } from "./guestLinkLibraries.js"; + import { canonicalBytes } from "../auth/canonicalize.js"; function concatNewline(bytes) { @@ -42,18 +44,10 @@ export async function writeGuestLinkFiles(compilation, dir, metadata = {}) { const librariesDir = path.join(dir, "libraries"); await rm(librariesDir, { recursive: true, force: true }); const libraryEntries = []; - for (const library of libraries) { - const file = `libraries/${library.name}-${library.version}-${library.sha256.slice(0, 12)}.o`; + for (const { metadata, objectBytes } of guestLinkLibraryEntries(libraries)) { await mkdir(librariesDir, { recursive: true }); - await writeFile(path.join(dir, file), library.objectBytes); - libraryEntries.push({ - name: library.name, - version: library.version, - language: library.language, - namespace: library.namespace, - sha256: library.sha256, - file, - }); + await writeFile(path.join(dir, metadata.file), objectBytes); + libraryEntries.push(metadata); } const written = { version: libraryEntries.length > 0 ? 2 : 1, diff --git a/src/compiler/guestLinkLibraries.js b/src/compiler/guestLinkLibraries.js index 8889cd5..5c86f13 100644 --- a/src/compiler/guestLinkLibraries.js +++ b/src/compiler/guestLinkLibraries.js @@ -2,10 +2,78 @@ import path from "node:path"; import { readFile } from "node:fs/promises"; import { bytesToHex } from "../utils/encoding.js"; import { sha256Bytes } from "../utils/crypto.js"; +import { parseWasmModuleSections } from "../bundle/wasm.js"; + +const NAME = /^[A-Za-z][A-Za-z0-9._-]*$/; +const VERSION = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const NAMESPACE = /^sdm_lib_[A-Za-z0-9_]+_$/; + +/** The same byte, hash and metadata contract applies to MODULE and FLOW inputs. */ +export async function verifyGuestLinkLibrary(library, { requireKind = false } = {}) { + const label = `Guest-link library "${library?.name ?? "unnamed"}"`; + if (!NAME.test(library?.name ?? "") || !VERSION.test(library?.version ?? "") || + !NAMESPACE.test(library?.namespace ?? "") || + !["c", "c++", "fortran"].includes(library?.language)) { + throw new Error(`${label} has invalid name, version, namespace or language.`); + } + if (!(library.objectBytes instanceof Uint8Array) || !library.objectBytes.length) { + throw new Error(`${label} needs nonempty objectBytes.`); + } + const objectBytes = new Uint8Array(library.objectBytes); + const sha256 = bytesToHex(await sha256Bytes(objectBytes)); + if (sha256 !== library.sha256) { + throw new Error(`${label} SHA-256 mismatch: expected ${library.sha256}, received ${sha256}.`); + } + const archive = new TextDecoder().decode(objectBytes.subarray(0, 8)) === "!\n"; + const kind = library.kind ?? (requireKind ? null : archive ? "archive" : "wasm-object"); + if (!["wasm-object", "archive"].includes(kind) || archive !== (kind === "archive")) { + throw new Error(`${label} kind must match its bytes (wasm-object or archive).`); + } + if (kind === "wasm-object") { + try { + const parsed = parseWasmModuleSections(objectBytes); + if (!parsed.sections.some(section => section.id === 0 && section.name === "linking")) { + throw new Error("missing relocatable linking section"); + } + } catch (error) { + throw new Error(`${label} is not a relocatable wasm object: ${error.message}`, { cause: error }); + } + } + return { ...library, kind, sha256, objectBytes }; +} + +// Same names identify the same bytes. Namespaces also cannot alias different bytes. +// First-use order preserves archive dependency ordering. +export function collectGuestLinkLibraries(libraries) { + const byName = new Map(); + const byNamespace = new Map(); + const unique = []; + for (const library of libraries) { + for (const prior of [byName.get(library.name), byNamespace.get(library.namespace)]) { + if (prior && prior.sha256 !== library.sha256) { + throw new Error(`Conflicting guest-link library "${library.name}": ${prior.sha256} and ${library.sha256} (namespace ${library.namespace}).`); + } + } + if (!byName.has(library.name) && !byNamespace.has(library.namespace)) { + unique.push(library); + } + byName.set(library.name, library); + byNamespace.set(library.namespace, library); + } + return unique; +} + +export async function normalizePrebuiltLibraries(libraries) { + if (libraries == null) return []; + if (!Array.isArray(libraries)) throw new Error("prebuiltLibraries must be an array."); + const verified = []; + for (const library of libraries) verified.push(await verifyGuestLinkLibrary(library, { requireKind: true })); + return collectGuestLinkLibraries(verified); +} export async function readGuestLinkLibraries(guestLink, pluginId) { const listed = guestLink.metadata?.libraries; - if (listed === undefined || listed === null) return []; + if (listed == null) return []; if (!Array.isArray(listed)) { throw new Error(`Plugin "${pluginId}": guest-link metadata.libraries must be an array.`); } @@ -16,32 +84,39 @@ export async function readGuestLinkLibraries(guestLink, pluginId) { if (!file || !resolved.startsWith(guestLink.guestLinkDir + path.sep)) { throw new Error(`Plugin "${pluginId}": guest-link library ${JSON.stringify(library?.name)} has no file inside its guest-link dir.`); } - if (!/^sdm_lib_[A-Za-z0-9_]+_$/.test(String(library?.namespace ?? ""))) { - throw new Error(`Plugin "${pluginId}": guest-link library ${JSON.stringify(library?.name)} has no sdm_lib_ namespace.`); - } - const objectBytes = new Uint8Array(await readFile(resolved)); - const sha256 = bytesToHex(await sha256Bytes(objectBytes)); - if (sha256 !== library.sha256) { - throw new Error( - `Plugin "${pluginId}": guest-link library ${file} hashes to ${sha256}, but its metadata records ${library.sha256}.`, - ); + try { + libraries.push(await verifyGuestLinkLibrary({ ...library, objectBytes: new Uint8Array(await readFile(resolved)) })); + } catch (error) { + throw new Error(`Plugin "${pluginId}" library "${library?.name}": ${error.message}`, { cause: error }); } - libraries.push({ ...library, objectBytes }); } - return libraries; + return collectGuestLinkLibraries(libraries); } -// Each shared library namespace once, in first-use order. A namespace is the -// library's name, version and a digest of its source, defines, flags and -// toolchain kind, so two modules carrying the same namespace carry the same -// library and the flow links one copy of it. -export function collectFlowLibraryObjects(linkedDependencies) { - const byNamespace = new Map(); +export async function collectFlowLibraryObjects(linkedDependencies) { + const libraries = []; for (const dependency of linkedDependencies) { for (const library of dependency.guestLink?.libraries ?? []) { - if (!byNamespace.has(library.namespace)) byNamespace.set(library.namespace, library); + libraries.push(await verifyGuestLinkLibrary(library)); } } - return Array.from(byNamespace.values()); + return collectGuestLinkLibraries(libraries); } +export function guestLinkLibraryEntries(libraries = []) { + return libraries.map(({ objectBytes, name, version, language, namespace, sha256, kind }) => ({ + objectBytes, + metadata: { + name, version, language, namespace, sha256, + ...(kind ? { kind } : {}), + file: `libraries/${name}-${version}-${sha256.slice(0, 12)}.${kind === "archive" ? "a" : "o"}`, + }, + })); +} + +export function withLibraryContext(error, libraries) { + if (!libraries?.length) return error; + const wrapped = new Error(`${error.message}\nGuest-link libraries: ${libraries.map(library => JSON.stringify(library.name)).join(", ")}.`, { cause: error }); + if (error.report) wrapped.report = error.report; + return wrapped; +} diff --git a/src/flow/flowCompiler.js b/src/flow/flowCompiler.js index b6bd89d..062f332 100644 --- a/src/flow/flowCompiler.js +++ b/src/flow/flowCompiler.js @@ -1,4 +1,4 @@ -import { readGuestLinkLibraries, collectFlowLibraryObjects } from "../compiler/guestLinkLibraries.js"; +import { readGuestLinkLibraries, collectFlowLibraryObjects, withLibraryContext } from "../compiler/guestLinkLibraries.js"; /** * SDN flow compiler (loop C.3c) — `space-data-module flow check|compile`. * @@ -3196,40 +3196,47 @@ export async function compileFlowProgram({ check.threadModel !== FLOW_THREAD_MODEL.SINGLE_THREAD ? linkFlowArtifactWithWasiThreads : linkFlowArtifactWithEmception; - let wasmBytes = await linkFlowArtifactCached(linkFlowArtifact, check.threadModel, { - runtimeSource, - invokeHeader: generateInvokeSupportHeader(), - generatedInc, - manifestSource, - programSource, - dependencyObjects: [ - ...linkedDependencies.map((dependency) => dependency.guestLink.objectBytes), - ...collectFlowLibraryObjects(linkedDependencies).map((library) => library.objectBytes), - ], - engineLinked: Boolean(check.engineLinkage), - sequential: check.threadModel === FLOW_THREAD_MODEL.WASI_SEQUENTIAL, - }); - wasmBytes = appendWasmCustomSection( - wasmBytes, - SDS_MANIFEST_SECTION_NAME, - flowPlgBytes, - ); - if (check.threadModel === FLOW_THREAD_MODEL.WASI_THREADS) { - assertPthreadArtifact(wasmBytes, { source: sourceName }); - } else if (check.threadModel === FLOW_THREAD_MODEL.WASI_SEQUENTIAL) { - assertSequentialArtifact(wasmBytes, { source: sourceName }); - } + const flowLibraries = await collectFlowLibraryObjects(linkedDependencies); + let wasmBytes; + let report; + try { + wasmBytes = await linkFlowArtifactCached(linkFlowArtifact, check.threadModel, { + runtimeSource, + invokeHeader: generateInvokeSupportHeader(), + generatedInc, + manifestSource, + programSource, + dependencyObjects: [ + ...linkedDependencies.map((dependency) => dependency.guestLink.objectBytes), + ...flowLibraries.map((library) => library.objectBytes), + ], + engineLinked: Boolean(check.engineLinkage), + sequential: check.threadModel === FLOW_THREAD_MODEL.WASI_SEQUENTIAL, + }); + wasmBytes = appendWasmCustomSection( + wasmBytes, + SDS_MANIFEST_SECTION_NAME, + flowPlgBytes, + ); + if (check.threadModel === FLOW_THREAD_MODEL.WASI_THREADS) { + assertPthreadArtifact(wasmBytes, { source: sourceName }); + } else if (check.threadModel === FLOW_THREAD_MODEL.WASI_SEQUENTIAL) { + assertSequentialArtifact(wasmBytes, { source: sourceName }); + } - const report = await validateArtifactWithStandards({ - manifest, - wasmBytes, - sourceName, - ...standardsOptions, - }); - if (!report.ok) { - const error = new Error("Flow artifact compliance validation failed."); - error.report = report; - throw error; + report = await validateArtifactWithStandards({ + manifest, + wasmBytes, + sourceName, + ...standardsOptions, + }); + if (!report.ok) { + const error = new Error("Flow artifact compliance validation failed."); + error.report = report; + throw error; + } + } catch (error) { + throw withLibraryContext(error, flowLibraries); } const dependencyRecords = []; diff --git a/src/index.d.ts b/src/index.d.ts index 89a02b7..5022b8d 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -1059,7 +1059,7 @@ export interface GuestLinkArtifact { /** Present (true) only for a module that uses WebAssembly SIMD, declared with `simd128: true` or not. */ simd128?: true; objectBytes: Uint8Array; - /** Present when the compile linked `libraries`. */ + /** Present when the compile linked source or prebuilt libraries. */ libraries?: GuestLinkLibrary[]; } @@ -1165,6 +1165,8 @@ export function compileModuleFromSource(options: { * declares their functions; the library sources define them. */ libraries?: ModuleLibrarySource[]; + /** Hash-verified inputs in link order, with producer-owned symbols. */ + prebuiltLibraries?: (GuestLinkLibrary & { kind: "wasm-object" | "archive" })[]; }): Promise; export interface ModuleLibrarySource { @@ -1180,15 +1182,16 @@ export interface ModuleLibrarySource { } /** - * A shared library object. Its strong global symbols are renamed into - * `namespace` (sdm_lib____, the digest covering its - * source, defines, flags and toolchain kind); the module object's references - * follow. A flow links each namespace once. + * A guest-link library. Source libraries have SDK-namespaced symbols; + * prebuilt libraries preserve the producer's symbols. A flow links identical + * names and hashes once and refuses conflicting bytes for a name or namespace. */ export interface GuestLinkLibrary { name: string; version: string; - language: "c" | "c++"; + language: "c" | "c++" | "fortran"; + /** Omitted in legacy source-library metadata (wasm-object). */ + kind?: "wasm-object" | "archive"; namespace: string; sha256: string; objectBytes: Uint8Array; diff --git a/test/prebuilt-libraries.test.js b/test/prebuilt-libraries.test.js new file mode 100644 index 0000000..cf16da8 --- /dev/null +++ b/test/prebuilt-libraries.test.js @@ -0,0 +1,214 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, writeFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { compileModuleFromSource, protectModuleArtifact, resolveWasiThreadsToolchain, writeGuestLinkFiles } from "../src/compiler/index.js"; +import { compileFlowProgram, resolveFlowDependencies } from "../src/flow/flowCompiler.js"; +import { createFlowRuntimeHost } from "../src/flow/flowRuntimeHost.js"; +import { createBrowserModuleHarness } from "../src/host/browserModuleHarness.js"; + +import { parseSingleFileBundle } from "../src/bundle/index.js"; +import { fileURLToPath } from "node:url"; + +const exec = promisify(execFile); +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +const port = portId => ({ + portId, required: true, minStreams: 1, maxStreams: 1, + acceptedTypeSets: [{ setId: portId, allowedTypes: [{ acceptsAnyFlatbuffer: true }], + wildcardJustification: { kind: "foreign-wire-format", detail: "A four-byte little-endian integer for an arithmetic integration test.", mediaType: "application/vnd.sdn.test-uint32" } }], +}); +const manifestFor = pluginId => ({ + pluginId, name: pluginId, version: "1.0.0", pluginFamily: "foundation", + capabilities: [], externalInterfaces: [], invokeSurfaces: ["direct"], + runtimeTargets: ["browser", "wasmedge"], schemasUsed: [], abiVersion: 1, + sequentialJustification: { kind: "pure-transform", detail: "Integer arithmetic has no concurrency or shared state." }, + methods: [ + { methodId: "answer", displayName: "Answer", inputPorts: [], outputPorts: [], maxBatch: 1, drainPolicy: "single-shot" }, + { methodId: "tick", displayName: "Tick", inputPorts: [port("in")], outputPorts: [port("out")], maxBatch: 1, drainPolicy: "single-shot" }, + ], +}); +const sourceCode = ` +#include "space_data_module_invoke.h" +#include +#include +extern "C" int plain_value(void); +extern "C" int answer(void) { return plain_value(); } +extern "C" int tick(void) { + const plugin_input_frame_t *frame = plugin_get_input_frame(0); + if (!frame || frame->payload_length != 4) return 400; + uint32_t value; + memcpy(&value, frame->payload, 4); + value += plain_value(); + plugin_push_output("out", 0, 0, (const uint8_t *)&value, 4); + return 0; +} +`; +const optionsFor = pluginId => ({ manifest: manifestFor(pluginId), sourceCode, language: "c++", threadModel: "wasi-sequential", catalog: [] }); + +async function temporary(t) { + const dir = await mkdtemp(path.join(os.tmpdir(), "sdk-prebuilt-e2e-")); + t.after(() => rm(dir, { recursive: true, force: true })); + return dir; +} +async function runAnswer(t, result, expected) { + assert.equal(result.report.ok, true, JSON.stringify(result.report)); + const host = await createBrowserModuleHarness({ wasmSource: result.wasmBytes, surface: "direct" }); + try { assert.equal(host.instance.exports.answer(), expected); } + finally { host.destroy(); } +} +async function buildLibraries(dir, value) { + const toolchain = resolveWasiThreadsToolchain(); + const compile = async (name, source) => { + await writeFile(path.join(dir, `${name}.c`), source); + await exec(toolchain.clang, [...toolchain.toolchainArgs, "-O2", "-c", path.join(dir, `${name}.c`), "-o", path.join(dir, `${name}.o`)]); + }; + await compile("used", `int archive_value(void) { return ${value}; }`); + // An unresolved reference in the unused archive member proves lazy extraction. + await compile("unused", "extern int must_not_link(void); int unused(void) { return must_not_link(); }"); + await compile("plain", "extern int archive_value(void); int plain_value(void) { return archive_value() + 2; }"); + const { stdout } = await exec(toolchain.clang, ["--print-prog-name=llvm-ar"]); + const archive = path.join(dir, "arithmetic.a"); + await exec(stdout.trim(), ["rcs", archive, path.join(dir, "used.o"), path.join(dir, "unused.o")]); + const library = async (name, file, kind) => { + const objectBytes = new Uint8Array(await readFile(file)); + return { name, version: "1.0.0", language: "c", namespace: `sdm_lib_${name}_`, kind, sha256: hash(objectBytes), objectBytes }; + }; + return [await library("plain", path.join(dir, "plain.o"), "wasm-object"), await library("arithmetic", archive, "archive")]; +} +async function persist(dir, manifest, result) { + await mkdir(dir, { recursive: true }); + await writeFile(path.join(dir, "plugin-manifest.json"), JSON.stringify(manifest)); + const metadata = await writeGuestLinkFiles(result, path.join(dir, "dist", "guest-link")); + for (const library of metadata.libraries) { + assert.equal(hash(await readFile(path.join(dir, "dist", "guest-link", library.file))), library.sha256); + } + assert.equal(metadata.libraries[1].kind, "archive"); + assert.match(metadata.libraries[1].file, /\.a$/); +} + +test("prebuilt objects and archives compile, invalidate cache, compose once, and reject conflicts and forbidden imports", async t => { + const dir = await temporary(t); + const libraries = await buildLibraries(dir, 40); + const opts = { ...optionsFor("test.prebuilt.first"), compileCache: { dir: path.join(dir, "cache") }, prebuiltLibraries: libraries }; + const first = await compileModuleFromSource(opts); + await runAnswer(t, first, 42); + assert.deepEqual(first.guestLink.libraries.map(l => l.objectBytes), libraries.map(l => l.objectBytes)); + const cached = await compileModuleFromSource(opts); + assert.ok(cached.compileCache.hits.includes("link")); + await runAnswer(t, cached, 42); + await assert.rejects(compileModuleFromSource({ ...opts, prebuiltLibraries: [{ ...libraries[0], sha256: "0".repeat(64) }] }), /plain.*SHA-256 mismatch/); + await assert.rejects(compileModuleFromSource({ ...opts, prebuiltLibraries: [{ ...libraries[0], kind: "archive" }] }), /plain.*kind/); + const changed = await buildLibraries(dir, 41); + const changedResult = await compileModuleFromSource({ ...opts, prebuiltLibraries: changed }); + assert.ok(changedResult.compileCache.misses.includes("link")); + assert.ok(changedResult.compileCache.hits.includes("module")); + await runAnswer(t, changedResult, 43); + + const secondOptions = optionsFor("test.prebuilt.second"); + const second = await compileModuleFromSource({ ...secondOptions, prebuiltLibraries: libraries }); + const packages = path.join(dir, "packages"); + await persist(path.join(packages, "first"), opts.manifest, first); + await persist(path.join(packages, "second"), secondOptions.manifest, second); + // Protection creates a throwaway in-memory test identity; no key is persisted. + const protectedArtifact = await protectModuleArtifact({ manifest: opts.manifest, wasmBytes: first.wasmBytes, guestLink: first.guestLink, singleFileBundle: true }); + const bundle = await parseSingleFileBundle(protectedArtifact.singleFileBundle.wasmBytes); + const bundledMetadata = bundle.entries.find(entry => entry.entryId === "guest-link-metadata").decodedPayload; + assert.equal(bundledMetadata.version, 2); + for (const library of bundledMetadata.libraries) { + assert.equal(hash(bundle.entries.find(entry => entry.entryId === library.file).payloadBytes), library.sha256); + } + const cliSource = path.join(dir, "module.cpp"); + const cliOutput = path.join(dir, "cli.wasm"); + await writeFile(cliSource, sourceCode); + await exec(process.execPath, [fileURLToPath(new URL("../bin/space-data-module.js", import.meta.url)), "compile", + "--manifest", path.join(packages, "first", "plugin-manifest.json"), "--source", cliSource, + "--language", "c++", "--thread-model", "wasi-sequential", "--out", cliOutput, + "--prebuilt-libraries", path.join(packages, "first", "dist", "guest-link", "metadata.json")]); + await runAnswer(t, { report: { ok: true }, wasmBytes: await readFile(cliOutput) }, 42); + const dependencies = await resolveFlowDependencies({ depsPath: packages }); + const flow = { + programId: "test.prebuilt.flow", name: "Prebuilt arithmetic", version: "1.0.0", + nodes: [ + { nodeId: "first", pluginId: opts.manifest.pluginId, methodId: "tick", kind: "transform" }, + { nodeId: "second", pluginId: secondOptions.manifest.pluginId, methodId: "tick", kind: "transform" }, + { nodeId: "sink", pluginId: "test.sink", methodId: "collect", kind: "sink" }, + ], + edges: [ + { fromNodeId: "first", fromPortId: "out", toNodeId: "second", toPortId: "in", opaque: true }, + { fromNodeId: "second", fromPortId: "out", toNodeId: "sink", toPortId: "result", opaque: true }, + ], + triggers: [{ triggerId: "manual", kind: "manual" }], + triggerBindings: [{ triggerId: "manual", targetNodeId: "first", targetPortId: "in" }], + requiredPlugins: [opts.manifest.pluginId, secondOptions.manifest.pluginId], + }; + const result = await compileFlowProgram({ flow, dependencies, catalog: [] }); + assert.equal(result.report.ok, true, JSON.stringify(result.report)); + const host = await createFlowRuntimeHost({ wasmSource: result.wasmBytes }); + host.enqueueTriggerFrame(0, { portId: "in", bytes: new Uint8Array(4) }); + const frames = []; + await host.drain({ "test.sink:collect": result => { frames.push(...result.frames); return { statusCode: 0 }; } }, { maxIterations: 20 }); + assert.equal(frames.length, 1); + assert.equal(new DataView(frames[0].bytes.buffer, frames[0].bytes.byteOffset).getUint32(0, true), 84); + assert.equal(host.getNodeState(0).invocationCount, 1n); + assert.equal(host.getNodeState(1).invocationCount, 1n); + + // Same name, different bytes, even under a different namespace, is refused. + const changedSecond = await compileModuleFromSource({ ...secondOptions, prebuiltLibraries: changed.map(l => ({ ...l, namespace: l.namespace + "other_" })) }); + await persist(path.join(packages, "second"), secondOptions.manifest, changedSecond); + await assert.rejects(compileFlowProgram({ flow, dependencies: await resolveFlowDependencies({ depsPath: packages }), catalog: [] }), /Conflicting guest-link library "arithmetic"/); + await writeFile(path.join(packages, "first", "dist", "guest-link", "libraries", `plain-1.0.0-${libraries[0].sha256.slice(0, 12)}.o`), changed[1].objectBytes); + await assert.rejects(resolveFlowDependencies({ depsPath: packages }), /plain.*SHA-256 mismatch/); + + const toolchain = resolveWasiThreadsToolchain(); + await writeFile(path.join(dir, "forbidden.c"), '__attribute__((import_module("env"), import_name("__pthread_create_js"))) extern int forbidden(void); int plain_value(void) { return forbidden(); }'); + await exec(toolchain.clang, [...toolchain.toolchainArgs, "-c", path.join(dir, "forbidden.c"), "-o", path.join(dir, "forbidden.o")]); + const objectBytes = new Uint8Array(await readFile(path.join(dir, "forbidden.o"))); + await assert.rejects(compileModuleFromSource({ ...opts, prebuiltLibraries: [{ ...libraries[0], name: "forbidden_threads", objectBytes, sha256: hash(objectBytes) }] }), error => { + assert.match(error.message, /forbidden_threads/); + assert.match(error.message, /imports Emscripten.*thread hooks/); + return true; + }); +}); + +const PEPPER_URL = "https://github.com/TJKoury/pepper/releases/download/wasm-v0.1.0/pepper-wasm-0.1.0.tgz"; +const PEPPER_SHA256 = "1c9cfaedef4dc3769f9b1fbeee6088f451f004d65f27eb8142264f807df89c51"; +test("PEPPER release libraries link and run a kernel-free vector norm", async t => { + const dir = await temporary(t); + let tarball; + if (process.env.SDN_PEPPER_TARBALL) tarball = await readFile(process.env.SDN_PEPPER_TARBALL); + else { + try { + const response = await fetch(PEPPER_URL); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + tarball = new Uint8Array(await response.arrayBuffer()); + } catch (error) { + t.skip(`PEPPER release download unavailable: ${error.message}`); + return; + } + } + assert.equal(hash(tarball), PEPPER_SHA256); + const archivePath = path.join(dir, "pepper.tgz"); + await writeFile(archivePath, tarball); + await exec("tar", ["-xzf", archivePath, "-C", dir]); + const guestDir = path.join(dir, "package", "dist", "guest-link"); + const metadata = JSON.parse(await readFile(path.join(guestDir, "metadata.json"))); + const prebuiltLibraries = []; + for (const library of metadata.libraries) { + const objectBytes = new Uint8Array(await readFile(path.join(guestDir, library.file))); + const kind = Buffer.from(objectBytes.subarray(0, 8)).toString() === "!\n" ? "archive" : "wasm-object"; + prebuiltLibraries.push({ ...library, kind, objectBytes }); + } + const header = await readFile(path.join(dir, "package", "dist", "include", "pepper_wasm.h"), "utf8"); + const manifest = manifestFor("test.prebuilt.pepper"); + manifest.methods = [manifest.methods[0]]; + const result = await compileModuleFromSource({ + manifest, language: "c++", threadModel: "wasi-sequential", catalog: [], prebuiltLibraries, + sourceCode: `${header}\nextern "C" int answer(void) { const double v[3] = {3, 4, 0}; double result = 0; if (pepper_vnorm(v, &result) != 0) return -1; return (int)result; }`, + }); + await runAnswer(t, result, 5); +});