Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## 0.10.3

- MODULE compilation accepts hash-verified prebuilt WebAssembly objects and static archives through `prebuiltLibraries`. Input order and producer symbols are preserved; input bytes participate in the link cache.
- Guest-link output carries prebuilt libraries into FLOW compilation. Identical names and hashes link once; conflicting content for a name or namespace is refused.
- The compile CLI accepts guest-link metadata with `--prebuilt-libraries`. Artifact guards retain their checks and include library names in failures.

## 0.10.2

### The browser host instantiates the owning instance before it warms the thread pool
Expand Down
42 changes: 42 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,48 @@ const bundle = await createSingleFileBundle({
const parsed = await parseSingleFileBundle(bundle.wasmBytes);
```

`prebuiltLibraries` links relocatable WebAssembly objects and static `ar` archives after
all module and source-library objects, in the supplied order:

```js
import { readFile } from "node:fs/promises";
import { writeGuestLinkFiles } from "space-data-module-sdk/compiler";

const compilation = await compileModuleFromSource({
manifest,
sourceCode,
language: "c++",
threadModel: "wasi-sequential",
prebuiltLibraries: [{
name: "vector_math",
version: "1.0.0",
language: "c", // also "c++" or "fortran"
namespace: "sdm_lib_vector_math_",
kind: "archive", // or "wasm-object"
sha256: releaseMetadata.sha256, // the producer's pinned SHA-256
objectBytes: new Uint8Array(await readFile("./libvector_math.a")),
}],
});
await writeGuestLinkFiles(compilation, "./dist/guest-link");
```

The compiler verifies each hash before use. It preserves the producer's symbols;
`namespace` identifies a library and does not rename prebuilt code. Producers supply
compatible symbols and a toolchain compatible with the selected thread model.
Archives retain lazy member extraction. Exact input bytes participate in the link
cache key. Linked artifacts pass the same thread, import, SIMD, and sequential
checks; errors include the participating library names.

Guest-link output includes each library's bytes, kind, and SHA-256. Flow compilation
links identical names and hashes once, in first-use order. Conflicting hashes for a
name or namespace are refused. Existing metadata without `kind` is accepted based
on the file's contents. Source libraries remain available through `libraries`.

The CLI accepts `--prebuilt-libraries ./dist/guest-link/metadata.json`. Its
`libraries` entries name files relative to that metadata file and carry the same
name, version, language, namespace, kind, and SHA-256 fields.
`--thread-model wasi-sequential` selects the sequential WASI compile profile.

Each subsystem is also available as a subpath export:

```js
Expand Down
17 changes: 16 additions & 1 deletion bin/space-data-module.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/usr/bin/env node
import { readGuestLinkLibraries } from "../src/compiler/guestLinkLibraries.js";

import path from "node:path";
import process from "node:process";
Expand Down Expand Up @@ -93,6 +94,12 @@ function parseArgs(argv) {
case "--wasm":
options.wasmPath = path.resolve(requireValue(argv, ++index, value));
break;
case "--thread-model":
options.threadModel = requireValue(argv, ++index, value);
break;
case "--prebuilt-libraries":
options.prebuiltLibrariesPath = path.resolve(requireValue(argv, ++index, value));
break;
case "--source":
options.sourcePath = path.resolve(requireValue(argv, ++index, value));
break;
Expand Down Expand Up @@ -234,7 +241,7 @@ function printUsage() {
console.log(`Usage:
space-data-module check --repo-root .
space-data-module check --manifest ./manifest.json --wasm ./dist/module.wasm
space-data-module compile --manifest ./manifest.json --source ./src/module.c --out ./dist/module.wasm
space-data-module compile --manifest ./manifest.json --source ./src/module.c --out ./dist/module.wasm [--prebuilt-libraries ./dist/guest-link/metadata.json] [--thread-model wasi-sequential]
space-data-module init --family propagator --name my-propagator
space-data-module init --family propagator --name my-propagator --out ./modules/my-propagator [--plugin-id com.orbpro.my.propagator] [--force] [--json]
space-data-module parity --wasm ./dist/isomorphic/module.wasm --fixture ./fixtures/parity/basic.json
Expand Down Expand Up @@ -628,8 +635,16 @@ async function runCompile(argv) {
}
const manifest = await loadManifestFromFile(options.manifestPath);
const sourceCode = await readFile(options.sourcePath, "utf8");
const prebuiltLibraries = options.prebuiltLibrariesPath
? await readGuestLinkLibraries({
guestLinkDir: path.dirname(options.prebuiltLibrariesPath),
metadata: JSON.parse(await readFile(options.prebuiltLibrariesPath, "utf8")),
}, manifest.pluginId)
: undefined;
const result = await compileModuleFromSource({
manifest,
prebuiltLibraries,
threadModel: options.threadModel,
sourceCode,
language: options.language,
outputPath: options.outputPath,
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "space-data-module-sdk",
"version": "0.10.2",
"version": "0.10.3",
"license": "Apache-2.0",
"description": "Module SDK for building, validating, signing, and deploying WebAssembly modules on the Space Data Network.",
"type": "module",
Expand Down
44 changes: 36 additions & 8 deletions src/compiler/compileModule.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { normalizePrebuiltLibraries, collectGuestLinkLibraries, guestLinkLibraryEntries, withLibraryContext } from "./guestLinkLibraries.js";
import os from "node:os";
import path from "node:path";
import { execFile } from "node:child_process";
Expand Down Expand Up @@ -642,7 +643,17 @@ function createGuestLinkBundleEntries(guestLink) {
if (!guestLink?.objectBytes || guestLink.objectBytes.length === 0) {
return [];
}
const libraries = guestLinkLibraryEntries(guestLink.libraries);
return [
...libraries.map(({ metadata, objectBytes }) => ({
entryId: metadata.file,
role: "auxiliary",
sectionName: SDS_GUEST_LINK_SECTION_NAME,
payloadEncoding: "raw-bytes",
mediaType: metadata.kind === "archive" ? "application/octet-stream" : "application/wasm",
payload: objectBytes,
description: `Guest-link library ${metadata.name}.`,
})),
{
entryId: SDS_GUEST_LINK_OBJECT_ENTRY_ID,
role: "auxiliary",
Expand All @@ -659,7 +670,8 @@ function createGuestLinkBundleEntries(guestLink) {
payloadEncoding: "json-utf8",
mediaType: SDS_GUEST_LINK_MEDIA_TYPE,
payload: {
version: 1,
version: libraries.length ? 2 : 1,
...(libraries.length ? { libraries: libraries.map(({ metadata }) => metadata) } : {}),
format: "wasm-object",
symbolPrefix: guestLink.symbolPrefix,
methodSymbols: guestLink.methodSymbols,
Expand Down Expand Up @@ -1068,6 +1080,10 @@ async function compileWithToolchain(kind, options = {}) {
outputPath ?? path.join(tempDir, "module.wasm"),
);
const libraries = normalizeLibraries(compileOptions.libraries);
const prebuilt = compileOptions.prebuiltLibraries.map((library, index) => ({
library, namespace: library.namespace, kind: library.kind, objectBytes: library.objectBytes,
object: `prebuilt-${index}.${library.kind === "archive" ? "a" : "o"}`,
}));
const profile = await resolveToolchainProfile(kind, {
language,
sourceCompilerCommand,
Expand Down Expand Up @@ -1110,7 +1126,7 @@ async function compileWithToolchain(kind, options = {}) {
argv: profile.compileArgv(step.role, at(step.source), at(step.object), step.extra ?? [], step.library?.language),
}));
const linkOutput = "module.wasm";
const linkObjects = compileSteps.map((step) => step.object);
const linkObjects = [...compileSteps.map((step) => step.object), ...prebuilt.map((library) => library.object)];
const linkArgv = profile.linkArgv(linkObjects.map(at), at(linkOutput), linkArgs);

const cacheReport = { enabled: Boolean(cache), dir: cache?.root ?? null, hits: [], misses: [], stepMs: {} };
Expand Down Expand Up @@ -1144,10 +1160,8 @@ async function compileWithToolchain(kind, options = {}) {
const objects = new Map();
let linked = null;
const prepareLinkInputs = () => {
if (libraries.length === 0) {
return { inputs: new Map(objects), libraries: [] };
}
const inputs = new Map(objects);
const inputs = new Map([...objects, ...prebuilt.map((library) => [library.object, library.objectBytes])]);
if (libraries.length === 0) return { inputs, libraries: prebuilt };
const references = new Map();
const namespaced = libraries.map((library) => {
const step = compileSteps.find((entry) => entry.library === library);
Expand Down Expand Up @@ -1183,7 +1197,11 @@ async function compileWithToolchain(kind, options = {}) {
{ undefinedOnly: true },
),
);
return { inputs, libraries: namespaced };
const all = [...namespaced, ...prebuilt];
collectGuestLinkLibraries(all.map(({ library, namespace, objectBytes }) => ({
...library, namespace, sha256: sha256Hex(objectBytes),
})));
return { inputs, libraries: all };
};

if (cache) {
Expand Down Expand Up @@ -1280,11 +1298,12 @@ async function compileWithToolchain(kind, options = {}) {
objectBytes: guestLink.objectBytes,
...(linked.libraries.length > 0
? {
libraries: linked.libraries.map(({ library, namespace, objectBytes }) => ({
libraries: linked.libraries.map(({ library, namespace, kind, objectBytes }) => ({
name: library.name,
version: library.version,
language: library.language,
namespace,
...(kind ? { kind } : {}),
sha256: sha256Hex(objectBytes),
objectBytes,
})),
Expand Down Expand Up @@ -1317,6 +1336,7 @@ export async function compileModuleFromSource(options = {}) {
throw error;
}

const prebuiltLibraries = await normalizePrebuiltLibraries(options.prebuiltLibraries);
const compiler = selectCompiler(options.language);
const invokeSurfaces = resolveInvokeSurfaces(manifest);
const includeCommandMain = invokeSurfaces.includes(InvokeSurface.COMMAND);
Expand Down Expand Up @@ -1356,6 +1376,7 @@ export async function compileModuleFromSource(options = {}) {
const simd128 = resolveSimd128(options.simd128, threadModel);
const compileOptions = {
...options,
prebuiltLibraries,
noEntry: includeCommandMain !== true,
threadModel,
simd128,
Expand Down Expand Up @@ -1465,6 +1486,11 @@ export async function compileModuleFromSource(options = {}) {
...standardsOptions,
});

if (prebuiltLibraries.length && !report.ok) {
const error = new Error(`Linked artifact validation failed: ${JSON.stringify(report.errors)}`);
error.report = report;
throw error;
}
handedOff = keepTempDir;
return {
compiler: useWasiThreads
Expand All @@ -1490,6 +1516,8 @@ export async function compileModuleFromSource(options = {}) {
simd128: simd128Report,
report,
};
} catch (error) {
throw withLibraryContext(error, prebuiltLibraries);
} finally {
if (!handedOff) {
await rm(tempDir, { recursive: true, force: true });
Expand Down
16 changes: 5 additions & 11 deletions src/compiler/guestLinkFiles.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import path from "node:path";
import { mkdir, rm, writeFile } from "node:fs/promises";

import { guestLinkLibraryEntries } from "./guestLinkLibraries.js";

import { canonicalBytes } from "../auth/canonicalize.js";

function concatNewline(bytes) {
Expand All @@ -42,18 +44,10 @@ export async function writeGuestLinkFiles(compilation, dir, metadata = {}) {
const librariesDir = path.join(dir, "libraries");
await rm(librariesDir, { recursive: true, force: true });
const libraryEntries = [];
for (const library of libraries) {
const file = `libraries/${library.name}-${library.version}-${library.sha256.slice(0, 12)}.o`;
for (const { metadata, objectBytes } of guestLinkLibraryEntries(libraries)) {
await mkdir(librariesDir, { recursive: true });
await writeFile(path.join(dir, file), library.objectBytes);
libraryEntries.push({
name: library.name,
version: library.version,
language: library.language,
namespace: library.namespace,
sha256: library.sha256,
file,
});
await writeFile(path.join(dir, metadata.file), objectBytes);
libraryEntries.push(metadata);
}
const written = {
version: libraryEntries.length > 0 ? 2 : 1,
Expand Down
Loading
Loading