diff --git a/.github/scripts/fixtures/pd-edge-0.2.0/Cargo.toml b/.github/scripts/fixtures/pd-edge-0.2.0/Cargo.toml new file mode 100644 index 0000000..1291c24 --- /dev/null +++ b/.github/scripts/fixtures/pd-edge-0.2.0/Cargo.toml @@ -0,0 +1,105 @@ +[workspace] +members = [ + ".", + "pd-edge-abi", + "pd-edge-host-function", +] +resolver = "2" + +[workspace.package] +edition = "2024" +version = "0.1.0" + +[package] +name = "pd-edge" +version.workspace = true +edition.workspace = true +description = "RustScript edge runtime and proxy host integration" +license = "MIT" +homepage = "https://rustscript.org/" +repository = "https://github.com/rustscript-lang/pd-edge" +build = "build.rs" +autobins = false + +[lib] +name = "edge" + +[features] +# Keep HTTP/3 opt-in in normal builds; release.yml enables it explicitly with --all-features. +# Do not enable `vm/edge-abi`: frozen pd-vm's optional edge-abi depends on +# crates.io pd-edge-abi 0.1.1 (ABI24). HTTP compiles and runs against the +# in-tree ABI25 catalog, exact compiler overlay, and runtime descriptors. +default = ["console", "http", "tls", "websocket"] +console = ["http", "tls", "websocket", "edge_abi/console"] +native-jit = ["vm/cranelift-jit"] +http = ["edge_abi/http"] +http2 = ["http", "edge_abi/http2", "dep:h2", "dep:tokio-rustls", "dep:webpki-roots", "dep:rustls-pemfile"] +http3 = ["http", "dep:quinn", "dep:h3", "dep:h3-quinn", "dep:webpki-roots", "dep:rustls-pemfile", "dep:rcgen"] +mqtt = ["tls", "edge_abi/mqtt"] +tls = ["http", "edge_abi/tls", "dep:tokio-rustls", "dep:rcgen", "dep:webpki-roots", "dep:rustls-pemfile"] +websocket = ["http", "edge_abi/websocket", "dep:tokio-tungstenite"] +webrtc = ["http", "edge_abi/webrtc", "dep:webrtc"] + +[[bin]] +name = "pd-edge-http-proxy" +path = "src/bin/pd-edge-http-proxy.rs" + +[[bin]] +name = "pd-edge-transport-proxy" +path = "src/bin/pd-edge-transport-proxy.rs" + +[[bin]] +name = "pd-edge-console" +path = "src/bin/pd-edge-console.rs" +required-features = ["console"] + +[[bin]] +name = "pd-edge-sample-echo-server" +path = "src/bin/pd-edge-sample-echo-server.rs" + +[dependencies] +axum = "0.8" +bytes = "1" +tokio = { version = "1", features = ["full"] } +http-body-util = "0.1" +hyper = { version = "1", features = ["http1", "http2", "server"] } +hyper-rustls = { version = "0.27", features = ["webpki-roots", "http2"] } +hyper-util = { version = "0.1", features = ["tokio", "server", "server-auto", "client", "client-legacy", "http1", "http2"] } +arc-swap = "1" +socket2 = "0.5.10" +h2 = { version = "0.4", optional = true } +h3 = { version = "0.0.8", optional = true } +h3-quinn = { version = "0.0.10", optional = true } +futures-util = "0.3" +pd-edge-host-function = { path = "./pd-edge-host-function", version = "0.1.0" } +linkme = "0.3" +vm = { package = "pd-vm", git = "https://github.com/rustscript-lang/rustscript.git", rev = "b1d6cffede77f49410bf63525f30b9a46b02dc01", version = "0.1.0", default-features = false, features = ["runtime"] } +edge_abi = { package = "pd-edge-abi", path = "./pd-edge-abi", version = "0.1.0", default-features = false } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "ansi"] } +owo-colors = "4" +supports-color = "3" +base64 = "0.22" +rand = "0.8" +uuid = { version = "1", features = ["v4"] } +url = "2" +tower = { version = "0.5", features = ["util"] } +parking_lot = "0.12" +tokio-tungstenite = { version = "0.28", features = ["rustls-tls-webpki-roots"], optional = true } +tokio-rustls = { version = "0.26", optional = true } +quinn = { version = "0.11.9", default-features = false, features = ["runtime-tokio", "rustls-aws-lc-rs"], optional = true } +rcgen = { version = "0.12", optional = true } +webrtc = { version = "0.17", optional = true } +rustls = "0.23" +webpki-roots = { version = "1", optional = true } +rustls-pemfile = { version = "2", optional = true } +httparse = "1" + +[dev-dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "http2"] } + +[target.'cfg(windows)'.dependencies] +mimalloc = "0.1" +windows = { version = "0.58", features = ["Win32_Foundation", "Win32_System_Console"] } diff --git a/.github/scripts/fixtures/pd-edge-0.2.0/pd-edge-abi/Cargo.toml b/.github/scripts/fixtures/pd-edge-0.2.0/pd-edge-abi/Cargo.toml new file mode 100644 index 0000000..9ecd4a2 --- /dev/null +++ b/.github/scripts/fixtures/pd-edge-0.2.0/pd-edge-abi/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "pd-edge-abi" +version.workspace = true +edition.workspace = true +description = "ABI definitions for RustScript edge host functions" +license = "MIT" +homepage = "https://rustscript.org/" +repository = "https://github.com/rustscript-lang/pd-edge" +build = "build.rs" + +[lib] +name = "edge_abi" + +[features] +default = ["console", "http", "tls", "websocket"] +console = [] +http = [] +http2 = ["http"] +mqtt = [] +tls = [] +websocket = ["http"] +webrtc = ["http"] + +[dependencies] +pd-host-function = { git = "https://github.com/rustscript-lang/rustscript.git", rev = "b1d6cffede77f49410bf63525f30b9a46b02dc01", version = "0.1.0" } + +[build-dependencies] +syn = { version = "2", features = ["full"] } diff --git a/.github/scripts/fixtures/pd-edge-0.2.0/pd-edge-host-function/Cargo.toml b/.github/scripts/fixtures/pd-edge-0.2.0/pd-edge-host-function/Cargo.toml new file mode 100644 index 0000000..2e92ef0 --- /dev/null +++ b/.github/scripts/fixtures/pd-edge-0.2.0/pd-edge-host-function/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "pd-edge-host-function" +version.workspace = true +edition.workspace = true +description = "Proc macro support for RustScript edge host functions" +license = "MIT" +homepage = "https://rustscript.org/" +repository = "https://github.com/rustscript-lang/pd-edge" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full"] } diff --git a/.github/scripts/prepare_publish.py b/.github/scripts/prepare_publish.py new file mode 100644 index 0000000..5ec352b --- /dev/null +++ b/.github/scripts/prepare_publish.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python3 +"""Materialise standalone crates.io-ready publish trees for a workspace release. + +Why this exists +--------------- +`cargo publish -p ` resolves the *whole* workspace before packaging, so a +path dependency that has been rewritten into a registry requirement inside the +workspace deadlocks the release: publishing `pd-edge-abi` fails because +`crates.io` does not have `pd-edge-abi 0.2.0` yet, which is exactly what the +member `pd-edge` (already rewritten to `pd-edge-abi = "0.2.0"`) demands. + +So instead of mutating the checkout in place, every released package is copied +into its own standalone tree whose manifest only contains dependencies that +crates.io can resolve: + +* `path` / `git` / `rev` / `branch` / `tag` selectors are removed and replaced + by the version of the crate being released (or an explicit registry version + for external crates such as `pd-vm` / `pd-host-function`); +* `dev-dependencies` are dropped (a consumer never builds them, and they must + not block a release); +* `[workspace]` / `[workspace.package]` inheritance is materialised so the tree + is its own workspace root; +* the publish order is derived from the actual normal + build dependency graph, + so dependencies are always published first. + +Usage +----- + python3 .github/scripts/prepare_publish.py \ + --workspace-root . \ + --out "$RUNNER_TEMP/publish-crates" \ + --version 0.2.0 \ + --package-order pd-edge-host-function pd-edge-abi pd-edge \ + --registry-dep pd-vm=0.22.2 \ + --registry-dep pd-host-function=0.22.2 +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +SOURCE_KEYS = ("path", "git", "rev", "branch", "tag") +DEPENDENCY_SUFFIXES = ("dependencies", "build-dependencies") +DEV_SUFFIXES = ("dev-dependencies",) +IGNORED_DIR_NAMES = { + ".git", + ".github", + ".ci-tmp", + "target", + "dist", + "node_modules", +} + +_HEADER_RE = re.compile(r"^\[(?P\[[^\[\]]+\]|[^\[\]]+)\]$") + + +class PublishError(RuntimeError): + """Raised when a manifest cannot be made crates.io-resolvable.""" + + +@dataclass +class StagedPackage: + name: str + version: str + manifest: Path + dependencies: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# tiny TOML value helpers (only the shapes used by inline dependency tables) + + +def _toml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False) + + +def emit_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return repr(value) + if isinstance(value, str): + return _toml_string(value) + if isinstance(value, list): + return "[" + ", ".join(emit_value(item) for item in value) + "]" + if isinstance(value, dict): + return emit_inline_table(value) + raise PublishError(f"unsupported TOML value in a dependency table: {value!r}") + + +def emit_inline_table(table: dict[str, Any]) -> str: + if not table: + return "{}" + inner = ", ".join(f"{key} = {emit_value(value)}" for key, value in table.items()) + return "{ " + inner + " }" + + +# --------------------------------------------------------------------------- +# manifest text surgery + + +def section_kind(header_name: str) -> str: + name = header_name.strip() + if name == "workspace" or name.startswith("workspace."): + return "workspace" + if name == "package": + return "package" + for suffix in DEV_SUFFIXES: + if name == suffix or name.endswith("." + suffix): + return "dev-dependencies" + for suffix in DEPENDENCY_SUFFIXES: + if name == suffix or name.endswith("." + suffix): + return "dependencies" + return "other" + + +def split_sections(lines: list[str]) -> list[tuple[str, str | None, list[str]]]: + """Split a manifest into (kind, header-line, body-lines) triples.""" + sections: list[tuple[str, str | None, list[str]]] = [("other", None, [])] + for line in lines: + match = _HEADER_RE.match(line.strip()) + if match: + sections.append((section_kind(match.group("name")), line, [])) + else: + sections[-1][2].append(line) + return sections + + +def _balanced(text: str) -> bool: + depth = 0 + in_string = False + escaped = False + for char in text: + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char == "[": + depth += 1 + elif char == "]": + depth -= 1 + return depth <= 0 + + +def rewrite_dependency(alias: str, spec: Any, versions: dict[str, str]) -> Any: + """Return a crates.io-resolvable dependency spec (or a plain version string).""" + if not isinstance(spec, dict): + name = alias + override = versions.get(name) + if override is None: + return spec + return override + + if spec.get("workspace") is True: + raise PublishError( + f"dependency {alias!r} uses `workspace = true`; declare an explicit " + "version/path so the publish rewrite can resolve it" + ) + + name = spec.get("package") or alias + has_source = any(key in spec for key in SOURCE_KEYS) + target = versions.get(name) + + if has_source and target is None: + selectors = ", ".join(f"{key} = {spec[key]!r}" for key in SOURCE_KEYS if key in spec) + raise PublishError( + f"dependency {alias!r} ({name}) is not on crates.io and has no known " + f"registry version ({selectors}); pass --registry-dep {name}=" + ) + + if not has_source and target is None: + return spec + + rewritten: dict[str, Any] = {} + if "package" in spec: + rewritten["package"] = spec["package"] + rewritten["version"] = target + for key, value in spec.items(): + if key in ("package", "version") or key in SOURCE_KEYS: + continue + rewritten[key] = value + return rewritten + + +def rewrite_dependency_section( + lines: list[str], versions: dict[str, str] +) -> tuple[list[str], list[str]]: + """Rewrite every dependency entry in a section; return (lines, dep names).""" + out: list[str] = [] + names: list[str] = [] + index = 0 + while index < len(lines): + line = lines[index] + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + out.append(line) + index += 1 + continue + + chunk = [line] + raw = line.split("=", 1)[1] + while not _balanced(raw) and index + len(chunk) < len(lines): + chunk.append(lines[index + len(chunk)]) + raw += "\n" + chunk[-1] + index += len(chunk) + + alias = line.split("=", 1)[0].strip() + try: + parsed = tomllib.loads(f"dep = {raw.strip()}") + except tomllib.TOMLDecodeError: + # Not an inline table we can reason about; keep verbatim. + out.extend(chunk) + continue + + spec = parsed["dep"] + name = spec.get("package") or alias if isinstance(spec, dict) else alias + names.append(name) + rewritten = rewrite_dependency(alias, spec, versions) + if isinstance(rewritten, str): + if rewritten == spec: + out.extend(chunk) + else: + out.append(f'{alias} = "{rewritten}"') + else: + out.append(f"{alias} = {emit_inline_table(rewritten)}") + return out, names + + +def materialise_package_section( + lines: list[str], inherited: dict[str, Any], name: str, version: str +) -> list[str]: + out: list[str] = [] + for line in lines: + stripped = line.strip() + match = re.match(r"^([A-Za-z0-9_-]+)\.workspace\s*=\s*true$", stripped) + if match: + key = match.group(1) + if key == "version": + # The release version always wins over the inherited one. + out.append(f'version = "{version}"') + continue + if key not in inherited: + raise PublishError( + f"[package] {key}.workspace = true has no [workspace.package].{key}" + ) + out.append(f"{key} = {emit_value(inherited[key])}") + continue + if re.match(r"^version\s*=", stripped): + out.append(f'version = "{version}"') + continue + out.append(line) + if not any(re.match(r"^version\s*=", line.strip()) for line in out): + raise PublishError(f"[package] {name}: no version found after rewrite") + return out + + +def prepare_manifest( + text: str, name: str, version: str, versions: dict[str, str], inherited: dict[str, Any] +) -> tuple[str, list[str]]: + lines = text.splitlines() + dependencies: list[str] = [] + out: list[str] = [] + has_workspace_table = False + for kind, header, body in split_sections(lines): + if kind == "workspace": + continue + if kind == "dev-dependencies": + continue + if kind == "package": + out.append(header or "") + out.extend(materialise_package_section(body, inherited, name, version)) + continue + if kind == "dependencies": + rewritten, names = rewrite_dependency_section(body, versions) + out.append(header or "") + out.extend(rewritten) + dependencies.extend(names) + continue + if header is not None: + out.append(header) + out.extend(body) + + # Make the staged tree its own workspace root so Cargo never tries to join + # the parent workspace (the tree lives inside the checkout). + if not has_workspace_table: + while out and not out[-1].strip(): + out.pop() + out.extend(["", "[workspace]"]) + + return "\n".join(out).rstrip("\n") + "\n", dependencies + + +# --------------------------------------------------------------------------- +# workspace discovery / staging + + +def _read_manifest(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + return tomllib.load(handle) + + +def workspace_packages(root: Path) -> tuple[dict[str, Path], dict[str, Any]]: + manifest = _read_manifest(root / "Cargo.toml") + workspace = manifest.get("workspace") + if workspace is None: + raise PublishError(f"{root / 'Cargo.toml'} has no [workspace] table") + inherited = workspace.get("package", {}) or {} + packages: dict[str, Path] = {} + for member in workspace.get("members", []): + if any(char in member for char in "*?["): + for candidate in sorted(root.glob(member)): + if (candidate / "Cargo.toml").is_file(): + data = _read_manifest(candidate / "Cargo.toml") + packages[data["package"]["name"]] = candidate / "Cargo.toml" + continue + candidate = (root / member).resolve() + data = _read_manifest(candidate / "Cargo.toml") + packages[data["package"]["name"]] = candidate / "Cargo.toml" + return packages, inherited + + +def _copy_ignore(nested: set[Path]): + def ignore(directory: str, names: list[str]) -> set[str]: + skipped: set[str] = set() + current = Path(directory) + for name in names: + if name in IGNORED_DIR_NAMES: + skipped.add(name) + continue + if name == "Cargo.lock": + skipped.add(name) + continue + candidate = (current / name).resolve() + if candidate in nested: + skipped.add(name) + return skipped + + return ignore + + +def stage_package( + name: str, + manifest_path: Path, + version: str, + versions: dict[str, str], + inherited: dict[str, Any], + nested: set[Path], + out_dir: Path, +) -> StagedPackage: + package_dir = manifest_path.parent + destination = out_dir / name + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(package_dir, destination, ignore=_copy_ignore(nested)) + + text = manifest_path.read_text() + rewritten, dependencies = prepare_manifest(text, name, version, versions, inherited) + manifest = destination / "Cargo.toml" + manifest.write_text(rewritten) + + parsed = _read_manifest(manifest) + package = parsed.get("package", {}) + if package.get("name") != name: + raise PublishError(f"staged {manifest} is not package {name}") + if package.get("version") != version: + raise PublishError( + f"staged {manifest} version is {package.get('version')!r}, expected {version!r}" + ) + for section in ("dev-dependencies",): + if section in parsed: + raise PublishError(f"staged {manifest} still contains [{section}]") + for target in (parsed.get("target") or {}).values(): + if isinstance(target, dict) and "dev-dependencies" in target: + raise PublishError(f"staged {manifest} still contains target dev-dependencies") + if "workspace" not in parsed: + raise PublishError(f"staged {manifest} is not its own workspace root") + for dependency_name, spec in _iter_dependency_specs(parsed): + if isinstance(spec, dict) and any(key in spec for key in SOURCE_KEYS): + raise PublishError( + f"staged {manifest}: dependency {dependency_name!r} still has a " + "path/git selector" + ) + return StagedPackage(name, version, manifest, dependencies) + + +def _iter_dependency_specs(manifest: dict[str, Any]) -> Iterable[tuple[str, Any]]: + tables: list[dict[str, Any]] = [] + for key in ("dependencies", "build-dependencies"): + value = manifest.get(key) + if isinstance(value, dict): + tables.append(value) + for target in (manifest.get("target") or {}).values(): + if not isinstance(target, dict): + continue + for key in ("dependencies", "build-dependencies"): + value = target.get(key) + if isinstance(value, dict): + tables.append(value) + for table in tables: + for dependency_name, spec in table.items(): + yield dependency_name, spec + + +def publish_order(packages: dict[str, StagedPackage]) -> list[StagedPackage]: + """Topologically order packages so dependencies are published first.""" + order: list[StagedPackage] = [] + visiting: set[str] = set() + done: set[str] = set() + + def visit(name: str) -> None: + if name in done: + return + if name in visiting: + raise PublishError(f"dependency cycle among crates.io packages at {name}") + visiting.add(name) + for dependency in packages[name].dependencies: + if dependency in packages: + visit(dependency) + visiting.discard(name) + done.add(name) + order.append(packages[name]) + + for name in packages: + visit(name) + return order + + +def prepare_publish( + workspace_root: Path, + out_dir: Path, + version: str, + package_order: list[str] | None = None, + registry_deps: dict[str, str] | None = None, +) -> list[StagedPackage]: + workspace_root = workspace_root.resolve() + out_dir = out_dir.resolve() + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.+-]+)?", version or ""): + raise PublishError(f"invalid publish version {version!r}") + manifests, inherited = workspace_packages(workspace_root) + if not manifests: + raise PublishError(f"no workspace packages found in {workspace_root}") + + if package_order: + expected = set(package_order) + actual = set(manifests) + if expected != actual: + raise PublishError( + "PACKAGE_ORDER does not match the workspace packages: " + f"missing={sorted(actual - expected)} unknown={sorted(expected - actual)}" + ) + + versions = dict(registry_deps or {}) + for name in manifests: + versions[name] = version + + nested = {path.parent.resolve() for path in manifests.values()} + out_dir.mkdir(parents=True, exist_ok=True) + staged: dict[str, StagedPackage] = {} + for name, manifest_path in manifests.items(): + staged[name] = stage_package( + name, manifest_path, version, versions, inherited, nested, out_dir + ) + + ordered = publish_order(staged) + if package_order: + allowed = {name: index for index, name in enumerate(package_order)} + declared = [package.name for package in ordered] + if declared != sorted(declared, key=allowed.__getitem__): + print( + "prepare_publish: PACKAGE_ORDER is not dependency safe; using " + f"computed order {declared}", + file=sys.stderr, + ) + return ordered + + +def write_plan(ordered: list[StagedPackage], version: str, path: Path) -> None: + plan = { + "version": version, + "packages": [ + { + "name": package.name, + "version": package.version, + "manifest": str(package.manifest), + "dependencies": package.dependencies, + } + for package in ordered + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(plan, indent=2) + "\n") + + +def parse_named_versions(values: list[str] | None, flag: str) -> dict[str, str]: + parsed: dict[str, str] = {} + for value in values or []: + if "=" not in value: + raise PublishError(f"{flag} expects NAME=VERSION, got {value!r}") + name, version = value.split("=", 1) + if not name.strip() or not version.strip(): + raise PublishError(f"{flag} expects NAME=VERSION, got {value!r}") + parsed[name.strip()] = version.strip() + return parsed + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace-root", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--package-order", nargs="*", default=None) + parser.add_argument("--registry-dep", action="append", default=None, + metavar="NAME=VERSION") + parser.add_argument("--summary", type=Path, default=None) + args = parser.parse_args(argv) + + try: + ordered = prepare_publish( + args.workspace_root, + args.out, + args.version, + args.package_order, + parse_named_versions(args.registry_dep, "--registry-dep"), + ) + except PublishError as error: + print(f"prepare_publish: {error}", file=sys.stderr) + return 1 + + summary = args.summary or (args.out / "publish-plan.json") + write_plan(ordered, args.version, summary) + print(f"prepare_publish: staged {len(ordered)} package(s) for {args.version}") + for package in ordered: + print(f" {package.name} {package.version} -> {package.manifest}") + print(f"prepare_publish: plan written to {summary}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/publish_crates.py b/.github/scripts/publish_crates.py new file mode 100644 index 0000000..1df5e29 --- /dev/null +++ b/.github/scripts/publish_crates.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Publish the staged crates from prepare_publish.py in dependency order. + +Design notes +------------ +* Every package is published from its own standalone tree + (``cargo publish --manifest-path /Cargo.toml``), so Cargo never has to + resolve a sibling that is not on crates.io yet. +* Versions are checked against the *sparse index* (``index.crates.io``) and not + only the crates.io API: the API answers before Cargo's index can see the new + version, and publishing a dependent in that window fails with + ``candidate versions found which didn't match``. +* A dependency-resolution failure is retried only while the dependency is still + missing from the index; the last Cargo output is printed verbatim on failure. +* ``--dry-run`` prints the plan and executes nothing (used by the tests). +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +INDEX_ROOT = "https://index.crates.io" +API_ROOT = "https://crates.io/api/v1/crates" + + +@dataclass +class Package: + name: str + version: str + manifest: Path + dependencies: list[str] + + +def index_path(name: str) -> str: + lowered = name.lower() + if len(lowered) == 1: + return f"1/{lowered}" + if len(lowered) == 2: + return f"2/{lowered}" + if len(lowered) == 3: + return f"3/{lowered[0]}/{lowered}" + return f"{lowered[:2]}/{lowered[2:4]}/{lowered}" + + +def _fetch(url: str, timeout: float = 20.0) -> bytes | None: + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return response.read() + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + + +def index_has(name: str, version: str) -> bool: + payload = _fetch(f"{INDEX_ROOT}/{index_path(name)}") + if payload is None: + return False + for line in payload.decode("utf-8", "replace").splitlines(): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("vers") == version and not entry.get("yanked", False): + return True + return False + + +def api_has(name: str, version: str) -> bool: + return _fetch(f"{API_ROOT}/{name}/{version}") is not None + + +def wait_for_index(name: str, version: str, timeout: float, interval: float) -> bool: + deadline = time.monotonic() + timeout + while True: + try: + if index_has(name, version): + return True + except (urllib.error.URLError, TimeoutError, OSError) as error: + print(f" index lookup for {name} {version} failed: {error}") + if time.monotonic() >= deadline: + return False + time.sleep(interval) + + +def load_plan(path: Path) -> list[Package]: + plan = json.loads(path.read_text()) + # The plan is already in dependency order; keep it verbatim. + return [ + Package( + entry["name"], + entry["version"], + Path(entry["manifest"]), + list(entry.get("dependencies", [])), + ) + for entry in plan["packages"] + ] + + +def ensure_dependencies_indexed(packages: list[Package], + timeout: float, interval: float) -> None: + names = {package.name: package.version for package in packages} + for package in packages: + for dependency in package.dependencies: + if dependency not in names: + continue + expected = names[dependency] + if not wait_for_index(dependency, expected, timeout, interval): + raise SystemExit( + f"{dependency} {expected} is not visible in the crates.io index; " + f"refusing to publish {package.name}" + ) + print(f" dependency {dependency} {expected} is indexed") + + +def publish(package: Package, token: str | None, retries: int, wait: float, + timeout: float, interval: float) -> None: + if index_has(package.name, package.version): + print(f"{package.name} {package.version} already published; skipping") + return + + command = [ + "cargo", "publish", + "--manifest-path", str(package.manifest), + "--no-verify", + "--allow-dirty", + ] + environment = dict(os.environ) + if token: + # `cargo publish --token` is deprecated; the env var is the supported path. + environment["CARGO_REGISTRY_TOKEN"] = token + + for attempt in range(1, retries + 1): + print(f"publishing {package.name} {package.version}, attempt {attempt}") + proc = subprocess.run( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=environment, + ) + print(proc.stdout.rstrip()) + if proc.returncode == 0 or wait_for_index(package.name, package.version, + timeout, interval): + print(f"{package.name} {package.version} published") + return + if attempt == retries: + raise SystemExit( + f"{package.name} {package.version} failed to publish after " + f"{retries} attempt(s) (cargo exit {proc.returncode})" + ) + time.sleep(wait) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", required=True, type=Path) + parser.add_argument("--token", default=None, + help="registry token; defaults to $CARGO_REGISTRY_TOKEN") + parser.add_argument("--retries", type=int, default=12) + parser.add_argument("--retry-delay", type=float, default=20.0) + parser.add_argument("--index-timeout", type=float, default=300.0) + parser.add_argument("--index-interval", type=float, default=5.0) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + + packages = load_plan(args.plan) + if not packages: + print("publish_crates: plan contains no packages", file=sys.stderr) + return 1 + + print("publish_crates: order " + " -> ".join( + f"{package.name} {package.version}" for package in packages)) + if args.dry_run: + for package in packages: + print(f" cargo publish --manifest-path {package.manifest}") + return 0 + + token = args.token or os.environ.get("CARGO_REGISTRY_TOKEN") + if not token: + print("CARGO_REGISTRY_TOKEN is required", file=sys.stderr) + return 1 + + for package in packages: + ensure_dependencies_indexed(packages, args.index_timeout, args.index_interval) + publish(package, token, args.retries, args.retry_delay, + args.index_timeout, args.index_interval) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 8ef23dd..df60a67 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -33,30 +33,23 @@ jobs: PACKAGE_ORDER: pd-edge-host-function pd-edge-abi pd-edge PD_VM_VERSION: ${{ inputs.pd_vm_version || '0.22.2' }} PD_HOST_FUNCTION_VERSION: ${{ inputs.pd_host_function_version || '0.22.2' }} - WORKSPACE_VERSION_OVERRIDES: pd-edge=${{ inputs.version }} rustscript=${{ inputs.pd_vm_version || '0.22.2' }} + PUBLISH_STAGE: ${{ runner.temp }}/publish-crates steps: - name: Checkout pd-edge uses: actions/checkout@v4 with: path: pd-edge - - name: Checkout rustscript dependency - uses: actions/checkout@v4 - with: - repository: rustscript-lang/rustscript - path: rustscript - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - name: Publish crates + # crates.io publishes each package from its own staged tree, so the + # rustscript checkout is not needed: pd-vm / pd-host-function git + # dependencies are rewritten to registry versions by the prepare step. + - name: Prepare publish trees working-directory: pd-edge env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - if [[ -z "${CARGO_REGISTRY_TOKEN:-}" ]]; then - echo "CARGO_REGISTRY_TOKEN is required" >&2 - exit 1 - fi version="${INPUT_VERSION:-}" if [[ -z "$version" ]]; then version="${GITHUB_REF_NAME#v}" @@ -65,80 +58,22 @@ jobs: echo "version is required" >&2 exit 1 fi - export PUBLISH_VERSION="$version" - python3 - <<'PY' - import os, re - from pathlib import Path - version = os.environ['PUBLISH_VERSION'] - dep_versions = { - 'pd-edge-host-function': version, - 'pd-edge-abi': version, - 'pd-edge': version, - 'pd-vm': os.environ.get('PD_VM_VERSION') or version, - 'pd-host-function': os.environ.get('PD_HOST_FUNCTION_VERSION') or version, - } - workspace_versions = {} - for item in os.environ.get('WORKSPACE_VERSION_OVERRIDES', '').split(): - if '=' in item: - name, value = item.split('=', 1) - if value: - workspace_versions[name] = value - dep_re = re.compile(r'^(\s*([A-Za-z0-9_-]+)\s*=\s*\{)(.*)(\}\s*)$') - for path in Path('..').rglob('Cargo.toml'): - if '.git' in path.parts or 'target' in path.parts: - continue - manifest_version = workspace_versions.get(path.parent.name, version) - lines = path.read_text().splitlines() - out = [] - section = '' - for line in lines: - stripped = line.strip() - if stripped.startswith('[') and stripped.endswith(']'): - section = stripped - if section in ('[workspace.package]', '[package]') and stripped.startswith('version = '): - line = re.sub(r'version\s*=\s*"[^"]*"', f'version = "{manifest_version}"', line) - match = dep_re.match(line) - if match and 'path' in match.group(3): - prefix, key, body, suffix = match.groups() - package_match = re.search(r'package\s*=\s*"([^"]+)"', body) - dep_name = package_match.group(1) if package_match else key - dep_version = dep_versions.get(dep_name) - if dep_version: - if re.search(r'version\s*=\s*"[^"]*"', body): - body = re.sub(r'version\s*=\s*"[^"]*"', f'version = "{dep_version}"', body) - else: - body = body.rstrip() - if body and not body.endswith(','): - body += ',' - body += f' version = "{dep_version}"' - line = prefix + body + suffix - out.append(line) - path.write_text('\n'.join(out) + '\n') - PY - python3 - <<'PY' - import urllib.error, urllib.request, os, subprocess, time, sys - version = os.environ['PUBLISH_VERSION'] - packages = os.environ['PACKAGE_ORDER'].split() - token = os.environ['CARGO_REGISTRY_TOKEN'] - def exists(crate, version): - try: - urllib.request.urlopen(f'https://crates.io/api/v1/crates/{crate}/{version}', timeout=20).read() - return True - except urllib.error.HTTPError as exc: - if exc.code == 404: - return False - raise - for package in packages: - if exists(package, version): - print(f'{package} {version} already published; skipping') - continue - for attempt in range(1, 19): - print(f'publishing {package} {version}, attempt {attempt}') - proc = subprocess.run(['cargo', 'publish', '-p', package, '--no-verify', '--allow-dirty', '--token', token], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(proc.stdout) - if proc.returncode == 0 or exists(package, version): - break - if attempt == 18: - sys.exit(proc.returncode) - time.sleep(20) - PY + rm -rf "$PUBLISH_STAGE" + python3 .github/scripts/prepare_publish.py \ + --workspace-root . \ + --out "$PUBLISH_STAGE" \ + --version "$version" \ + --package-order $PACKAGE_ORDER \ + --registry-dep "pd-vm=${PD_VM_VERSION}" \ + --registry-dep "pd-host-function=${PD_HOST_FUNCTION_VERSION}" + - name: Publish crates + working-directory: pd-edge + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: | + set -euo pipefail + if [[ -z "${CARGO_REGISTRY_TOKEN:-}" ]]; then + echo "CARGO_REGISTRY_TOKEN is required" >&2 + exit 1 + fi + python3 .github/scripts/publish_crates.py --plan "$PUBLISH_STAGE/publish-plan.json" diff --git a/.github/workflows/test_publish_crates.py b/.github/workflows/test_publish_crates.py new file mode 100644 index 0000000..7d6f974 --- /dev/null +++ b/.github/workflows/test_publish_crates.py @@ -0,0 +1,221 @@ +"""Regression tests for the crates.io publish pipeline. + +The 0.2.0 tag run failed 18 times with:: + + error: failed to select a version for the requirement `pd-edge-abi = "^0.2.0"` + candidate versions found which didn't match: 0.1.1 + required by package `pd-edge v0.2.0` + +because the old workflow rewrote path dependencies into registry requirements +*inside* the workspace: `cargo publish -p pd-edge-abi` then had to resolve the +workspace member `pd-edge`, which already demanded the not-yet-published +`pd-edge-abi 0.2.0`. + +These tests run the replacement prepare/publish helpers against verbatim copies +of the 0.2.0 release manifests (``.github/scripts/fixtures/pd-edge-0.2.0``). +Nothing is published and no network is used. +""" + +import contextlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import tomllib +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/publish-crates.yml" +SCRIPTS = ROOT / ".github/scripts" +FIXTURE = SCRIPTS / "fixtures/pd-edge-0.2.0" + +sys.path.insert(0, str(SCRIPTS)) + +import prepare_publish # noqa: E402 +import publish_crates # noqa: E402 + + +RELEASE = "0.2.0" +REGISTRY_DEPS = {"pd-vm": "0.22.2", "pd-host-function": "0.22.2"} +ORDER = ["pd-edge-host-function", "pd-edge-abi", "pd-edge"] + + +def run_prepare(workspace: Path, out_dir: Path, package_order=None, registry_deps=None): + return prepare_publish.prepare_publish( + workspace, + out_dir, + RELEASE, + package_order if package_order is not None else ORDER, + registry_deps if registry_deps is not None else REGISTRY_DEPS, + ) + + +def load_staged(out_dir: Path, name: str) -> dict: + return tomllib.loads((out_dir / name / "Cargo.toml").read_text()) + + +class PreparePublishTests(unittest.TestCase): + def test_staged_tree_is_resolvable_without_unpublished_siblings(self): + with tempfile.TemporaryDirectory() as temporary: + out_dir = Path(temporary) / "stage" + ordered = run_prepare(FIXTURE, out_dir) + self.assertEqual([package.name for package in ordered], ORDER) + + abi = load_staged(out_dir, "pd-edge-abi") + edge = load_staged(out_dir, "pd-edge") + + # The root cause of the 0.2.0 failure: the tree that publishes + # pd-edge-abi must not require anything that is published later. + abi_deps = set(abi.get("dependencies", {})) | set(abi.get("build-dependencies", {})) + self.assertEqual(abi_deps & set(ORDER), set(), + "pd-edge-abi must not depend on a crate released after it") + self.assertEqual(abi["dependencies"]["pd-host-function"], {"version": "0.22.2"}) + + # pd-edge keeps the crates.io-visible names/features but no selectors. + self.assertEqual( + edge["dependencies"]["edge_abi"], + {"package": "pd-edge-abi", "version": RELEASE, "default-features": False}, + ) + self.assertEqual( + edge["dependencies"]["vm"], + {"package": "pd-vm", "version": "0.22.2", "default-features": False, + "features": ["runtime"]}, + ) + self.assertEqual(edge["dependencies"]["pd-edge-host-function"], {"version": RELEASE}) + self.assertEqual(edge["dependencies"]["tokio"]["features"], ["full"]) + + # dev-dependencies never reach crates.io and must not gate a release. + self.assertNotIn("dev-dependencies", edge) + self.assertNotIn("reqwest", json.dumps(edge["dependencies"])) + + def test_versions_are_materialised_per_package(self): + with tempfile.TemporaryDirectory() as temporary: + out_dir = Path(temporary) / "stage" + run_prepare(FIXTURE, out_dir) + for name in ORDER: + manifest = load_staged(out_dir, name) + self.assertEqual(manifest["package"]["version"], RELEASE) + self.assertEqual(manifest["package"]["name"], name) + # Standalone workspace root: no stale workspace inheritance. + self.assertIn("workspace", manifest) + self.assertNotIn("package", manifest["workspace"]) + self.assertEqual(load_staged(out_dir, "pd-edge-abi")["package"]["edition"], "2024") + + def test_no_path_or_git_selectors_survive(self): + with tempfile.TemporaryDirectory() as temporary: + out_dir = Path(temporary) / "stage" + run_prepare(FIXTURE, out_dir) + for name in ORDER: + manifest = load_staged(out_dir, name) + for dependency, spec in prepare_publish._iter_dependency_specs(manifest): + if isinstance(spec, dict): + self.assertFalse( + set(prepare_publish.SOURCE_KEYS) & set(spec), + f"{name}: {dependency} kept a path/git selector: {spec}", + ) + self.assertTrue(spec.get("version"), f"{name}: {dependency} has no version") + + def test_unknown_git_dependency_fails_loudly(self): + # Without a registry version the rewrite must stop instead of silently + # shipping an unresolvable dependency to crates.io. + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(prepare_publish.PublishError): + run_prepare(FIXTURE, Path(temporary) / "stage", registry_deps={}) + + def test_package_order_is_dependency_sorted(self): + with tempfile.TemporaryDirectory() as temporary: + out_dir = Path(temporary) / "stage" + ordered = run_prepare(FIXTURE, out_dir, package_order=list(reversed(ORDER))) + self.assertEqual([package.name for package in ordered], ORDER) + + def test_package_order_must_match_workspace(self): + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(prepare_publish.PublishError): + run_prepare(FIXTURE, Path(temporary) / "stage", + package_order=["pd-edge-abi", "pd-edge"]) + + def test_live_manifests_prepare_cleanly(self): + # Guards against manifest drift: the checked-out 0.2.0 tree must stage + # with every source selector removed and the release version applied. + with tempfile.TemporaryDirectory() as temporary: + out_dir = Path(temporary) / "stage" + ordered = run_prepare(ROOT, out_dir) + self.assertEqual([package.name for package in ordered], ORDER) + for name in ORDER: + manifest = load_staged(out_dir, name) + self.assertEqual(manifest["package"]["version"], RELEASE) + for dependency, spec in prepare_publish._iter_dependency_specs(manifest): + if isinstance(spec, dict): + self.assertFalse(set(prepare_publish.SOURCE_KEYS) & set(spec), + f"{name}: {dependency} kept a path/git selector") + + +class PublishPlanTests(unittest.TestCase): + def test_cli_writes_plan_and_dry_run_keeps_order(self): + with tempfile.TemporaryDirectory() as temporary: + out_dir = Path(temporary) / "stage" + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = prepare_publish.main([ + "--workspace-root", str(FIXTURE), + "--out", str(out_dir), + "--version", RELEASE, + "--package-order", *ORDER, + "--registry-dep", "pd-vm=0.22.2", + "--registry-dep", "pd-host-function=0.22.2", + ]) + self.assertEqual(exit_code, 0, stdout.getvalue()) + + plan_path = out_dir / "publish-plan.json" + plan = json.loads(plan_path.read_text()) + self.assertEqual([entry["name"] for entry in plan["packages"]], ORDER) + self.assertEqual(sorted(plan["packages"][1]["dependencies"]), + ["pd-host-function", "syn"]) + + dry_run = subprocess.run( + [sys.executable, str(SCRIPTS / "publish_crates.py"), + "--plan", str(plan_path), "--dry-run"], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, + ) + self.assertEqual(dry_run.returncode, 0) + lines = [line for line in dry_run.stdout.splitlines() if "cargo publish" in line] + self.assertEqual(len(lines), 3) + self.assertIn("pd-edge-host-function", lines[0]) + self.assertIn("pd-edge-abi", lines[1]) + self.assertIn("/pd-edge/Cargo.toml", lines[2]) + self.assertIn("--manifest-path", lines[2]) + + def test_index_paths_match_the_sparse_registry_layout(self): + self.assertEqual(publish_crates.index_path("pd-edge-abi"), "pd/-e/pd-edge-abi") + self.assertEqual(publish_crates.index_path("pd-vm"), "pd/-v/pd-vm") + self.assertEqual(publish_crates.index_path("ab"), "2/ab") + + +class WorkflowWiringTests(unittest.TestCase): + def test_workflow_uses_the_staged_publishers(self): + workflow = yaml.safe_load(WORKFLOW.read_text()) + steps = workflow["jobs"]["publish"]["steps"] + scripts = "\n".join(step.get("run", "") for step in steps) + self.assertIn(".github/scripts/prepare_publish.py", scripts) + self.assertIn(".github/scripts/publish_crates.py", scripts) + # The in-place rewrite that caused the deadlock must be gone. + self.assertNotIn("rglob('Cargo.toml')", scripts) + self.assertNotIn("cargo publish -p", scripts) + self.assertIn("--manifest-path", (SCRIPTS / "publish_crates.py").read_text()) + + def test_tag_trigger_still_covers_versions(self): + workflow = yaml.safe_load(WORKFLOW.read_text()) + # PyYAML maps the bare `on:` key to True. + triggers = workflow.get("on") or workflow.get(True) + self.assertIn("v*", triggers["push"]["tags"]) + self.assertIn("[0-9]*", triggers["push"]["tags"]) + + +if __name__ == "__main__": + unittest.main()