diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index 57c3a6cd42..120816a295 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -53,15 +53,32 @@ In `babylon`, workspace packages depend on each other by plain semver range way: - vlt rewrites most of those ranges (257 of 323) to `workspace:*` in - `package.json` on its first install. That rewrite is part of its timed run - wherever the manifests start out unmigrated, which is `clean`; in the other - variations the untimed first warmup migrates them. vlt 1.8.0 leaves the 66 - exact and peer specs on the published `@babylonjs/*` names as they are, and - in about half of cold installs it resolves four of them (`gui`, `loaders`, - `materials`, `serializers`) to registry copies instead of the workspace, - downloading 4 extra tarballs ([vltpkg/vltpkg#2154](https://github.com/vltpkg/vltpkg/issues/2154)). - So vlt's `babylon` `clean` results can differ between runs until that is - fixed. + `package.json` on its first install. A vlt user's repo stays migrated after + that first install, because the rewritten manifests get committed. So in the + warm variations an untimed setup step migrates the manifests before vlt's + first install of a set (`vlt install --lockfile-only`, which rewrites the + manifests without installing anything; the manifests are kept and the + lockfile it writes is discarded, or the one that was there is put back), and + `clean` times the migration as part of the first install + (`migrate_workspace_protocol` in `suite/scripts/clean-helpers.sh`). The + migration does not make vlt's first warm run on `babylon` the same as the + later ones: on a first install without a lockfile, vlt 1.8.0 writes an + optional dependency (`canvas`, 17 packages with its dependencies) to the + lockfile but doesn't install it; the next install, from that lockfile, does + ([vltpkg/vltpkg#2155](https://github.com/vltpkg/vltpkg/issues/2155)). So in + the no-lockfile variations (`clean`, `cache`, `node_modules` and their + combinations without a lockfile) vlt installs about 17 fewer packages than + its own lockfile and the other tools, 1,601 vs 1,618, which flatters vlt + slightly until that is fixed; in the lockfile variations the first install + from the lockfile downloads those 17. The suite doesn't work around it, + because forcing `canvas` in would make vlt behave unlike vlt. Separately, + vlt leaves the 66 exact and peer specs on the published `@babylonjs/*` + names as they are, and in about half of cold installs it resolves four of + them (`gui`, `loaders`, `materials`, `serializers`) to registry copies + instead of the workspace, downloading 4 extra tarballs + ([vltpkg/vltpkg#2154](https://github.com/vltpkg/vltpkg/issues/2154)). So + vlt's `babylon` results in installs without a lockfile can differ between + runs until that is fixed, migrated manifests or not. - pnpm links them (the fixture's `pnpm-workspace.yaml` sets `linkWorkspacePackages: true`), but first asks the registry for each one: about 240 lookups per cold install, most answered 404 for private packages. diff --git a/suite/scripts/clean-helpers.sh b/suite/scripts/clean-helpers.sh index f572589c90..591c77be30 100644 --- a/suite/scripts/clean-helpers.sh +++ b/suite/scripts/clean-helpers.sh @@ -312,6 +312,46 @@ clean_workspace_protocol() { fi } +# The inverse of clean_workspace_protocol: migrate the workspace ranges the +# way vlt's first install does, untimed, before vlt's first install of a set. +# A vlt user's repo stays migrated after that first install (the rewritten +# manifests get committed), so in the warm variations vlt starts from +# already-migrated manifests. `clean` is the exception: there the migration is +# part of what a first install costs, so clean.sh sets BENCH_SETUP_VLT empty. +# +# `vlt install --lockfile-only` rewrites the manifests exactly as a full +# install does (on `babylon`, 80 package.json files, plus 36 unrelated `bugs` +# field rewrites, a vlt quirk), with no node_modules and no tarballs. The +# lockfile it writes is not what the variation wants: discard it, or put back +# the one that was there byte for byte. Runs once per set: every later +# prepare finds `workspace:` already present and does nothing. On a fixture +# without workspaces (no packages/) there is nothing to migrate. +# +# If `--lockfile-only` fails (registry or token trouble), warn and carry on: +# vlt's first install then migrates the manifests as part of its timed run, +# as it did before this step existed. +migrate_workspace_protocol() { + if ! command -v vlt &> /dev/null || [ ! -d packages ]; then + return 0 + fi + if grep -rqs --include=package.json --exclude-dir=node_modules '"workspace:' package.json packages; then + return 0 + fi + echo "Migrating workspace ranges to workspace:* (vlt install --lockfile-only)..." + local had_lockfile="" + if [ -f vlt-lock.json ]; then + had_lockfile=1 + cp vlt-lock.json .bench-vlt-lock.json + fi + vlt install --lockfile-only --view=silent >/dev/null 2>&1 || + echo "Warning: vlt install --lockfile-only failed; vlt's first install will migrate instead" + if [ -n "$had_lockfile" ]; then + mv .bench-vlt-lock.json vlt-lock.json + else + rm -f vlt-lock.json + fi +} + clean_all() { clean_node_modules clean_lockfiles @@ -352,6 +392,7 @@ show_help() { echo " clean_build_output" echo " clean_git" echo " clean_all" + echo " migrate_workspace_protocol" echo " wait_for_vlt_background" echo "" echo "Usage: $0 [function_name1] [function_name2] ..." @@ -427,6 +468,9 @@ else clean_workspace_protocol) clean_workspace_protocol ;; + migrate_workspace_protocol) + migrate_workspace_protocol + ;; clean_git) clean_git ;; diff --git a/suite/scripts/clean-helpers.test.js b/suite/scripts/clean-helpers.test.js index 300533a914..3c98bce178 100644 --- a/suite/scripts/clean-helpers.test.js +++ b/suite/scripts/clean-helpers.test.js @@ -487,3 +487,88 @@ test("package-count.sh tells zpm from berry by the packageManager pin", () => { } } }); + +// migrate_workspace_protocol, with a fake vlt that does what +// `vlt install --lockfile-only` does to a workspace: rewrites a sibling range +// to workspace:* and writes vlt-lock.json. +const migrateWorkspaceProtocol = (t, { packages = true, migrated = false, lockfile } = {}) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "migrate-ws-test-")); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const binDir = path.join(dir, "bin"); + const project = path.join(dir, "project"); + const vltLog = path.join(dir, "vlt-calls.txt"); + fs.mkdirSync(binDir); + fs.mkdirSync(project); + fs.writeFileSync( + path.join(binDir, "vlt"), + `#!/bin/sh +echo "$*" >> "${vltLog}" +sed -i 's/"\\^1\\.0\\.0"/"workspace:*"/' packages/b/package.json +echo '{"lockfileVersion":1,"fake":true}' > vlt-lock.json +`, + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(project, "package.json"), '{"name":"x","workspaces":["packages/*"]}'); + if (packages) { + fs.mkdirSync(path.join(project, "packages", "a"), { recursive: true }); + fs.mkdirSync(path.join(project, "packages", "b"), { recursive: true }); + fs.writeFileSync(path.join(project, "packages", "a", "package.json"), '{"name":"a","version":"1.0.0"}'); + fs.writeFileSync( + path.join(project, "packages", "b", "package.json"), + `{"name":"b","dependencies":{"a":${migrated ? '"workspace:*"' : '"^1.0.0"'}}}`, + ); + // A workspace's node_modules can hold manifests that use workspace: + // (dependencies installed from a pnpm-style project); they don't count. + fs.mkdirSync(path.join(project, "packages", "a", "node_modules", "dep"), { recursive: true }); + fs.writeFileSync( + path.join(project, "packages", "a", "node_modules", "dep", "package.json"), + '{"name":"dep","dependencies":{"peer":"workspace:^"}}', + ); + } + if (lockfile !== undefined) { + fs.writeFileSync(path.join(project, "vlt-lock.json"), lockfile); + } + const result = spawnSync("bash", [path.join(__dirname, "clean-helpers.sh"), "migrate_workspace_protocol"], { + cwd: project, + encoding: "utf8", + env: { ...process.env, PATH: `${binDir}${path.delimiter}${process.env.PATH}` }, + }); + assert.equal(result.status, 0, result.stderr); + const read = (file) => (fs.existsSync(path.join(project, file)) ? fs.readFileSync(path.join(project, file), "utf8") : null); + return { + vltCalls: fs.existsSync(vltLog) ? fs.readFileSync(vltLog, "utf8").trim().split("\n") : [], + manifest: read("packages/b/package.json"), + lockfile: read("vlt-lock.json"), + tempLockfile: read(".bench-vlt-lock.json"), + }; +}; + +test("migrate_workspace_protocol: migrates the manifests and discards the lockfile", (t) => { + const out = migrateWorkspaceProtocol(t); + assert.deepEqual(out.vltCalls, ["install --lockfile-only --view=silent"]); + assert.equal(out.manifest, '{"name":"b","dependencies":{"a":"workspace:*"}}'); + assert.equal(out.lockfile, null, "the lockfile --lockfile-only wrote was kept"); + assert.equal(out.tempLockfile, null); +}); + +test("migrate_workspace_protocol: keeps a pre-existing lockfile byte for byte", (t) => { + const lockfile = '{"lockfileVersion":1,"nodes":{"":["x"]}}\n'; + const out = migrateWorkspaceProtocol(t, { lockfile }); + assert.deepEqual(out.vltCalls, ["install --lockfile-only --view=silent"]); + assert.equal(out.manifest, '{"name":"b","dependencies":{"a":"workspace:*"}}'); + assert.equal(out.lockfile, lockfile); + assert.equal(out.tempLockfile, null); +}); + +test("migrate_workspace_protocol: does nothing without packages/", (t) => { + const out = migrateWorkspaceProtocol(t, { packages: false, lockfile: "orig" }); + assert.deepEqual(out.vltCalls, []); + assert.equal(out.lockfile, "orig"); +}); + +test("migrate_workspace_protocol: does nothing once workspace: is present", (t) => { + const out = migrateWorkspaceProtocol(t, { migrated: true }); + assert.deepEqual(out.vltCalls, []); + assert.equal(out.manifest, '{"name":"b","dependencies":{"a":"workspace:*"}}'); + assert.equal(out.lockfile, null); +}); diff --git a/suite/scripts/variations/clean.sh b/suite/scripts/variations/clean.sh index 5de81f54ac..4e941a21d4 100644 --- a/suite/scripts/variations/clean.sh +++ b/suite/scripts/variations/clean.sh @@ -4,6 +4,12 @@ set -Eeuxo pipefail # Load common variables source "$1/variations/common.sh" +# Every run starts from the manifests as committed, so on `babylon` vlt's +# workspace:* migration is part of what a first install costs: no untimed +# migration here (common.sh sets it for the warm variations). Set before +# collect_process_count reads it, so the process-count pass matches. +BENCH_SETUP_VLT="" + # Prepare command base for each run BENCH_PREPARE_BASE="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index ca13a0a07c..025e1f9da2 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -90,7 +90,12 @@ BENCH_SETUP_BERRY="touch yarn.lock; echo \"$BENCH_COMMAND_YARN_MODERN_CONFIG\" > BENCH_SETUP_ZPM="touch yarn.lock; echo \"$BENCH_COMMAND_YARN_MODERN_CONFIG\" > .yarnrc.yml; { echo '[zpm prepare]'; echo 'cwd:'; pwd; echo 'package.json:'; ls -la package.json || true; echo 'yarn path:'; echo $BENCH_ZPM_BIN; echo 'yarn version:'; $BENCH_ZPM_BIN -v || true; echo 'canary version:'; echo \"$BENCH_ZPM_VERSION\"; echo 'packageManager (before):'; npm pkg get packageManager || true; echo 'set packageManager=yarn@'"$BENCH_ZPM_VERSION"':' ; npm pkg set packageManager=\"yarn@$BENCH_ZPM_VERSION\" || true; echo 'packageManager (after):'; npm pkg get packageManager || true; } >> $BENCH_OUTPUT_FOLDER/zpm-prepare.log 2>&1" BENCH_SETUP_PNPM="npm pkg delete packageManager >/dev/null 2>&1 || true" -BENCH_SETUP_VLT="" +# Migrate babylon's workspace ranges to `workspace:*` before vlt's first +# install of a set, untimed, the way a vlt user's repo stays after its first +# install (see migrate_workspace_protocol). Appended to vlt's --prepare only, +# never to hyperfine's shared --setup, which runs for every tool. clean.sh +# sets this empty: there the migration is part of a first install's cost. +BENCH_SETUP_VLT="bash $BENCH_SCRIPTS/clean-helpers.sh migrate_workspace_protocol" BENCH_SETUP_BUN="" BENCH_SETUP_DENO="" BENCH_SETUP_AUBE="npm pkg delete packageManager >/dev/null 2>&1 || true"