diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index a7f315d4..cc9953e4 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -33,7 +33,7 @@ jobs: permissions: contents: read env: - PACKAGE_ORDER: pd-host-function pd-vm pd-vm-nostd rustscript + PACKAGE_ORDER: pd-host-schema pd-host-function pd-vm pd-vm-nostd rustscript steps: - name: Checkout rustscript uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 diff --git a/scripts/publish_crates.py b/scripts/publish_crates.py index 562421aa..102d0338 100644 --- a/scripts/publish_crates.py +++ b/scripts/publish_crates.py @@ -10,20 +10,39 @@ import urllib.request from pathlib import Path -PACKAGE_ORDER = ("pd-host-function", "pd-vm", "pd-vm-nostd", "rustscript") +# Every publishable workspace crate in dependency order. A crate must appear +# before every crate that declares a normal or build dependency on it. +PACKAGE_ORDER = ("pd-host-schema", "pd-host-function", "pd-vm", "pd-vm-nostd", "rustscript") + +# The proc-macro crate tracks its own compatibility line instead of the release +# version, because published crates depend on the macro through `^0.22`. +HOST_PACKAGE = "pd-host-function" + +SKIPPED_MANIFEST_DIRECTORIES = frozenset({".git", ".ci-tmp", "node_modules", "target"}) + +SECTION_NAME_RE = re.compile(r"^\[([^\]]+)\]$") +DEPENDENCY_ENTRY_RE = re.compile(r"^(\s*([A-Za-z0-9_-]+)\s*=\s*\{)(.*)(\}\s*)$") +VERSION_KEY_RE = re.compile(r'version\s*=\s*"([^"]*)"') +PACKAGE_KEY_RE = re.compile(r'package\s*=\s*"([^"]+)"') def package_versions(release_version: str, host_version: str) -> dict[str, str]: - return { - "pd-host-function": host_version, - "pd-vm": release_version, - "pd-vm-nostd": release_version, - "rustscript": release_version, - } + """Version each publishable workspace crate ships with in this release. + + This map is the single source of truth for both `publish_plan` and the path + dependency rewrite, so adding a workspace crate to `PACKAGE_ORDER` cannot + leave a stale requirement behind in the publish tree. + """ + versions = {package: release_version for package in PACKAGE_ORDER} + versions[HOST_PACKAGE] = host_version + return versions def publish_plan(release_version: str, host_version: str) -> list[tuple[str, str]]: versions = package_versions(release_version, host_version) + missing = set(PACKAGE_ORDER) - set(versions) + if missing: + raise SystemExit(f"publish order entries without a version: {sorted(missing)}") return [(package, versions[package]) for package in PACKAGE_ORDER] @@ -60,13 +79,22 @@ def refresh_compatible_registry_lock( package: str, target_version: str, ) -> None: + """Move stale registry entries of `package` onto `target_version`. + + Cargo can only resolve the publish tree when every path dependency already + points at its publish version; otherwise `cargo update` aborts with a + version selection error before it reaches the registry entry. + """ lock_path = root / "Cargo.lock" - while stale_versions := compatible_registry_lock_versions( - lock_path, - package, - target_version, - ): - subprocess.run( + for _attempt in range(16): + stale_versions = compatible_registry_lock_versions( + lock_path, + package, + target_version, + ) + if not stale_versions: + return + process = subprocess.run( [ "cargo", "update", @@ -76,37 +104,81 @@ def refresh_compatible_registry_lock( target_version, ], cwd=root, - check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, ) + print(process.stdout, flush=True) + if process.returncode != 0: + raise SystemExit( + f"failed to update {package} {stale_versions[0]} to " + f"{target_version} in the publish lock; every path dependency in " + "the publish tree must already require its publish version" + ) + raise SystemExit(f"publish lock did not converge on {package} {target_version}") def yank_command(package: str, version: str) -> list[str]: return ["cargo", "yank", "--vers", version, package] +def dependency_section_kind(section: str) -> str | None: + """Classify a manifest section as a normal, build or dev dependency table.""" + tail = section.rsplit(".", 1)[-1].strip() + if tail == "dependencies": + return "normal" + if tail == "build-dependencies": + return "build" + if tail == "dev-dependencies": + return "dev" + return None + + +def dependency_name(key: str, body: str) -> str: + package_match = PACKAGE_KEY_RE.search(body) + return package_match.group(1) if package_match else key + + +def _with_version(body: str, version: str) -> str: + if VERSION_KEY_RE.search(body): + return VERSION_KEY_RE.sub(f'version = "{version}"', body, count=1) + body = body.rstrip() + if body and not body.endswith(","): + body += "," + return f'{body} version = "{version}" ' + + +def _without_version(body: str) -> str: + """Drop the version requirement so cargo strips the dep when packaging. + + A versioned path dev-dependency is kept in the published manifest and must + exist on crates.io, which blocks publishing crates that come earlier in the + order (pd-host-function dev-depends on pd-vm but is published first). + """ + inner = body.strip() + inner = re.sub(r'(?:^|,)\s*version\s*=\s*"[^"]*"', "", inner) + inner = re.sub(r",\s*,", ",", inner).strip().strip(",").strip() + return f" {inner} " if inner else " " + + def _rewrite_dependency_versions(text: str, versions: dict[str, str]) -> str: - dependency_re = re.compile(r"^(\s*([A-Za-z0-9_-]+)\s*=\s*\{)(.*)(\}\s*)$") output = [] + section = "" for line in text.splitlines(): - match = dependency_re.match(line) + stripped = line.strip() + header = SECTION_NAME_RE.match(stripped) + if header: + section = header.group(1) + kind = dependency_section_kind(section) + match = DEPENDENCY_ENTRY_RE.match(line) if kind else None if match and "path" in match.group(3): prefix, key, body, suffix = match.groups() - package_match = re.search(r'package\s*=\s*"([^"]+)"', body) - dependency = package_match.group(1) if package_match else key - version = versions.get(dependency) + version = versions.get(dependency_name(key, body)) if version: - if re.search(r'version\s*=\s*"[^"]*"', body): - body = re.sub( - r'version\s*=\s*"[^"]*"', - f'version = "{version}"', - body, - ) + if kind == "dev": + line = prefix + _without_version(body) + suffix else: - body = body.rstrip() - if body and not body.endswith(","): - body += "," - body += f' version = "{version}" ' - line = prefix + body + suffix + line = prefix + _with_version(body, version) + suffix output.append(line) return "\n".join(output) + "\n" @@ -114,49 +186,93 @@ def _rewrite_dependency_versions(text: str, versions: dict[str, str]) -> str: def _rewrite_manifest( path: Path, release_version: str, - host_version: str, - dependency_versions: dict[str, str], + versions: dict[str, str], ) -> None: text = path.read_text() manifest = tomllib.loads(text) - package_name = manifest.get("package", {}).get("name") + package = manifest.get("package", {}) + package_name = package.get("name") + package_version = versions.get(package_name) output = [] section = "" for line in text.splitlines(): stripped = line.strip() - if stripped.startswith("[") and stripped.endswith("]"): - section = stripped - if section == "[workspace.package]" and stripped.startswith("version = "): - line = re.sub( - r'version\s*=\s*"[^"]*"', - f'version = "{release_version}"', - line, - ) - elif section == "[package]" and package_name == "pd-host-function": - if stripped == "version.workspace = true": - indent = line[: len(line) - len(line.lstrip())] - line = f'{indent}version = "{host_version}"' - elif stripped.startswith("version = "): - line = re.sub( - r'version\s*=\s*"[^"]*"', - f'version = "{host_version}"', - line, - ) + header = SECTION_NAME_RE.match(stripped) + if header: + section = header.group(1) + if section == "workspace.package" and stripped.startswith("version = "): + line = VERSION_KEY_RE.sub(f'version = "{release_version}"', line, count=1) + elif section == "package" and package_version and stripped.startswith( + ("version = ", "version.workspace") + ): + indent = line[: len(line) - len(line.lstrip())] + line = f'{indent}version = "{package_version}"' output.append(line) path.write_text( - _rewrite_dependency_versions("\n".join(output) + "\n", dependency_versions) + _rewrite_dependency_versions("\n".join(output) + "\n", versions) ) +def manifest_paths(root: Path) -> list[Path]: + return [ + path + for path in sorted(root.rglob("Cargo.toml")) + if not SKIPPED_MANIFEST_DIRECTORIES.intersection(path.relative_to(root).parts) + ] + + +def stale_path_dependencies(root: Path, versions: dict[str, str]) -> list[str]: + """Report path dependencies that do not match the publish version. + + Cargo aborts the publish tree with `failed to select a version` when a path + dependency keeps an old requirement (`pd-host-schema = "^0.1.0"` while the + crate itself is `0.31.2`), so this is checked before publishing. + """ + problems = [] + for path in manifest_paths(root): + section = "" + for number, line in enumerate(path.read_text().splitlines(), start=1): + header = SECTION_NAME_RE.match(line.strip()) + if header: + section = header.group(1) + kind = dependency_section_kind(section) + match = DEPENDENCY_ENTRY_RE.match(line) if kind else None + if not match or "path" not in match.group(3): + continue + dependency = dependency_name(match.group(2), match.group(3)) + expected = versions.get(dependency) + if expected is None: + continue + requirement = VERSION_KEY_RE.search(match.group(3)) + if kind == "dev": + if requirement: + problems.append( + f"{path}:{number}: dev-dependency {dependency} keeps " + f"version {requirement.group(1)!r}, which must be dropped" + ) + elif requirement is None: + problems.append( + f"{path}:{number}: {dependency} has no version requirement, " + f"expected {expected!r}" + ) + elif requirement.group(1) != expected: + problems.append( + f"{path}:{number}: {dependency} requires " + f"{requirement.group(1)!r}, expected {expected!r}" + ) + return problems + + def rewrite_manifests(root: Path, release_version: str, host_version: str) -> None: - dependency_versions = { - "pd-host-function": host_version, - "pd-vm": release_version, - } - for path in root.rglob("Cargo.toml"): - if ".git" in path.parts or "target" in path.parts: - continue - _rewrite_manifest(path, release_version, host_version, dependency_versions) + versions = package_versions(release_version, host_version) + for path in manifest_paths(root): + _rewrite_manifest(path, release_version, versions) + problems = stale_path_dependencies(root, versions) + if problems: + raise SystemExit( + "publish tree still has path dependencies that cannot resolve:\n" + + "\n".join(problems) + ) def crate_exists(package: str, version: str) -> bool: @@ -209,7 +325,7 @@ def main() -> None: root = args.root.resolve() rewrite_manifests(root, args.version, args.host_version) - refresh_compatible_registry_lock(root, "pd-host-function", args.host_version) + refresh_compatible_registry_lock(root, HOST_PACKAGE, args.host_version) plan = publish_plan(args.version, args.host_version) print(json.dumps({"publish_plan": plan})) if args.prepare_only: @@ -219,7 +335,7 @@ def main() -> None: if args.yank_host_version: subprocess.run( - yank_command("pd-host-function", args.yank_host_version), + yank_command(HOST_PACKAGE, args.yank_host_version), cwd=root, check=True, ) diff --git a/scripts/test_publish_crates.py b/scripts/test_publish_crates.py index bb69f138..f7544280 100644 --- a/scripts/test_publish_crates.py +++ b/scripts/test_publish_crates.py @@ -2,6 +2,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock from scripts import publish_crates @@ -55,6 +56,7 @@ def test_package_versions_keep_host_macro_on_compatible_line(self) -> None: self.assertEqual( publish_crates.package_versions("0.23.1", "0.22.7"), { + "pd-host-schema": "0.23.1", "pd-host-function": "0.22.7", "pd-vm": "0.23.1", "pd-vm-nostd": "0.23.1", @@ -91,8 +93,175 @@ def test_rewrite_manifests_applies_release_and_dependency_versions(self) -> None self.assertIn('version = "0.22.7"', host_manifest) self.assertNotIn("version.workspace = true", host_manifest) self.assertIn('pd-host-function = { path = "./pd-host-function", version = "0.22.7" }', root_manifest) - self.assertIn('vm = { package = "pd-vm", path = "..", version = "0.23.1" }', nostd_manifest) + # A versioned path dev-dependency stays in the published manifest and + # then needs an already published version, so it is left path-only. + self.assertIn('vm = { package = "pd-vm", path = ".." }', nostd_manifest) self.assertIn('version = "0.23.1"', alias_manifest) + self.assertEqual( + publish_crates.stale_path_dependencies(root, publish_crates.package_versions("0.23.1", "0.22.7")), + [], + ) + + def _write_publish_tree_shaped_like_0_31_2(self, root: Path) -> None: + (root / "pd-host-function").mkdir() + (root / "crates" / "pd-host-schema").mkdir(parents=True) + (root / "Cargo.toml").write_text( + """[workspace] +members = [".", "pd-host-function", "crates/pd-host-schema"] + +[workspace.package] +version = "0.1.0" + +[package] +name = "pd-vm" +version.workspace = true + +[dependencies] +pd-host-function = { path = "./pd-host-function", version = "0.1.0" } + +[build-dependencies] +pd-host-schema = { path = "./crates/pd-host-schema", version = "0.1.0" } + +[dev-dependencies] +pd-host-schema = { path = "./crates/pd-host-schema", version = "0.1.0" } + +[target.'cfg(not(target_family = "wasm"))'.dev-dependencies] +tokio = { version = "1", features = ["macros"] } +""" + ) + (root / "pd-host-function" / "Cargo.toml").write_text( + """[package] +name = "pd-host-function" +version.workspace = true + +[dependencies] +pd-host-schema = { path = "../crates/pd-host-schema", version = "0.1.0" } +proc-macro2 = "1" + +[dev-dependencies] +vm = { package = "pd-vm", path = ".." } +""" + ) + (root / "crates" / "pd-host-schema" / "Cargo.toml").write_text( + """[package] +name = "pd-host-schema" +version.workspace = true + +[dependencies] +syn = { version = "2", features = ["full"] } +""" + ) + + def test_publish_tree_rewrites_every_workspace_path_dependency(self) -> None: + # 0.31.2 failed with `failed to select a version for the requirement + # pd-host-schema = "^0.1.0"` because the workspace version bump moved + # pd-host-schema to 0.31.2 while its path dependents kept `^0.1.0`. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write_publish_tree_shaped_like_0_31_2(root) + + publish_crates.rewrite_manifests(root, "0.31.2", "0.22.7") + + root_manifest = (root / "Cargo.toml").read_text() + host_manifest = (root / "pd-host-function" / "Cargo.toml").read_text() + schema_manifest = (root / "crates" / "pd-host-schema" / "Cargo.toml").read_text() + self.assertIn('version = "0.31.2"', root_manifest) + self.assertIn( + 'pd-host-function = { path = "./pd-host-function", version = "0.22.7" }', + root_manifest, + ) + self.assertIn( + 'pd-host-schema = { path = "./crates/pd-host-schema", version = "0.31.2" }', + root_manifest, + ) + self.assertIn( + 'pd-host-schema = { path = "../crates/pd-host-schema", version = "0.31.2" }', + host_manifest, + ) + self.assertIn('version = "0.22.7"', host_manifest) + self.assertIn('version = "0.31.2"', schema_manifest) + self.assertNotIn('"0.1.0"', root_manifest + host_manifest) + self.assertEqual( + publish_crates.stale_path_dependencies( + root, publish_crates.package_versions("0.31.2", "0.22.7") + ), + [], + ) + + def test_dev_dependency_path_requirement_is_dropped(self) -> None: + # A versioned path dev-dependency must resolve on crates.io at package + # time, which blocks crates published before their dev-dependency. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write_publish_tree_shaped_like_0_31_2(root) + + publish_crates.rewrite_manifests(root, "0.31.2", "0.22.7") + + root_manifest = (root / "Cargo.toml").read_text() + dev_section = root_manifest.split("[dev-dependencies]", 1)[1].split("[", 1)[0] + self.assertIn( + 'pd-host-schema = { path = "./crates/pd-host-schema" }', + dev_section, + ) + self.assertNotIn("version", dev_section) + + def test_stale_path_dependencies_reports_unrewritten_requirement(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write_publish_tree_shaped_like_0_31_2(root) + # Simulate the 0.31.2 publish tree: workspace version bumped, path + # requirement left behind. + manifest = root / "Cargo.toml" + manifest.write_text( + manifest.read_text().replace( + 'version = "0.1.0"\n\n[package]', 'version = "0.31.2"\n\n[package]' + ) + ) + + problems = publish_crates.stale_path_dependencies( + root, publish_crates.package_versions("0.31.2", "0.22.7") + ) + + self.assertEqual(len(problems), 4) + self.assertTrue( + all("pd-host-schema" in problem or "pd-host-function" in problem for problem in problems) + ) + self.assertTrue( + any("'0.1.0', expected '0.31.2'" in problem for problem in problems) + ) + + def test_refresh_compatible_registry_lock_leaves_current_lock_alone(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "Cargo.lock").write_text( + """version = 4 + +[[package]] +name = "pd-host-function" +version = "0.22.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "current" +""" + ) + with mock.patch.object( + publish_crates.subprocess, "run", side_effect=AssertionError("cargo must not run") + ): + publish_crates.refresh_compatible_registry_lock( + root, "pd-host-function", "0.22.7" + ) + + def test_workflow_package_order_matches_publish_plan(self) -> None: + workflow = ( + Path(__file__).parents[1] / ".github" / "workflows" / "publish-crates.yml" + ).read_text() + order_line = next( + line for line in workflow.splitlines() if "PACKAGE_ORDER:" in line + ) + + self.assertEqual( + tuple(order_line.split("PACKAGE_ORDER:", 1)[1].split()), + publish_crates.PACKAGE_ORDER, + ) def test_yank_command_targets_requested_host_version(self) -> None: self.assertEqual( @@ -104,6 +273,7 @@ def test_publish_plan_uses_dependency_order_and_per_package_versions(self) -> No self.assertEqual( publish_crates.publish_plan("0.23.1", "0.22.7"), [ + ("pd-host-schema", "0.23.1"), ("pd-host-function", "0.22.7"), ("pd-vm", "0.23.1"), ("pd-vm-nostd", "0.23.1"), diff --git a/scripts/test_verify_publish_order.py b/scripts/test_verify_publish_order.py index dcb77a06..c3d475bf 100644 --- a/scripts/test_verify_publish_order.py +++ b/scripts/test_verify_publish_order.py @@ -23,12 +23,13 @@ def workspace(*packages: tuple[str, dict[str, tuple[str, ...]]]) -> dict[str, di return result -ORDER = ["pd-host-function", "pd-vm", "pd-vm-nostd", "rustscript"] +ORDER = ["pd-host-schema", "pd-host-function", "pd-vm", "pd-vm-nostd", "rustscript"] class VerifyPublishOrderTests(unittest.TestCase): def test_dev_dependency_does_not_constrain_publish_order(self) -> None: packages = workspace( + ("pd-host-schema", {}), ("pd-host-function", {"normal": ("pd-host-schema",), "dev": ("pd-vm",)}), ("pd-vm", {}), ("pd-vm-nostd", {"normal": ("pd-vm",)}), @@ -40,6 +41,28 @@ def test_dev_dependency_does_not_constrain_publish_order(self) -> None: [], ) + def test_build_dependency_missing_from_order_is_rejected(self) -> None: + # 0.31.2 published straight into `failed to select a version for the + # requirement pd-host-schema = "^0.1.0"` because the order omitted the + # build dependency that the publish tree had to rewrite. + packages = workspace( + ("pd-host-schema", {}), + ("pd-host-function", {"normal": ("pd-host-schema",)}), + ("pd-vm", {"build": ("pd-host-schema",)}), + ) + + errors = verify_publish_order.publish_order_errors( + packages, + ["pd-host-function", "pd-vm"], + ) + + self.assertEqual(len(errors), 2) + self.assertIn( + "pd-vm has a build dependency on workspace package pd-host-schema, " + "so pd-host-schema must be published first", + errors, + ) + def test_normal_dependency_must_be_published_first(self) -> None: packages = workspace( ("pd-host-function", {"normal": ("pd-vm",)}), diff --git a/scripts/verify_publish_order.py b/scripts/verify_publish_order.py index 36a0e5b8..24a5d6e0 100644 --- a/scripts/verify_publish_order.py +++ b/scripts/verify_publish_order.py @@ -49,6 +49,12 @@ def publish_order_errors(packages: dict[str, dict], order: list[str]) -> list[st continue dependency_name = dependency["name"] if dependency_name not in positions: + if dependency_name in packages: + errors.append( + f"{package_name} has a {kind} dependency on workspace " + f"package {dependency_name}, so {dependency_name} must be " + "published first" + ) continue if positions[dependency_name] > positions[package_name]: errors.append(