diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index c35ed00bfb..1e0addf113 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -36,16 +36,28 @@ Only the first row publishes. Between runs the suite resets state with counts are taken from a real installed tree rather than inferred from a lockfile. -vlt unpacks fetched tarballs into its cache in a detached process -(`vlt-cache-unzip`) that keeps running after the `vlt` command exits — about -19 seconds after a cold `babylon` install. The timed run ends when `vlt` -exits, which is when a user gets their prompt back. So that this deferred work -can't overlap the next timed run (of any manager) or write into a cache that -is being cleaned, every `--prepare`, `--setup` and `--cleanup` first waits for -it to finish, and every `--prepare` waits again right before the timed command +Each tool is measured in its **default configuration**, the experience a user +gets out of the box. For vlt that is its own registry: every fixture's +`vlt.json` points vlt at `https://registry.vlt.io/vlt-benchmarks/npm/`, and the +workflow authenticates to it (`VLT_REGISTRY`, `VLT_TOKEN`), while the other +tools use `registry.npmjs.org`. Some of vlt's features, such as server-side +resolution, only exist on its registry, so this is the comparison we intend. +vlt can also be pointed at `registry.npmjs.org`; these numbers don't measure +that. The `registry-*` variations are the ones that hold the client constant +and compare registries. + +vlt finishes some work in detached processes that keep running after the +`vlt` command exits: unpacking fetched tarballs into its cache (about 19 +seconds after a cold `babylon` install), revalidating cached manifests, +deleting replaced `node_modules` entries, and update checks +(`suite/scripts/vlt-background.sh`). The timed run ends when `vlt` exits, +which is when a user gets their prompt back. So that this deferred work can't +overlap the next timed run (of any manager) or write into a cache that is +being cleaned, every `--prepare`, `--setup` and `--cleanup` first waits for it +to finish, and every `--prepare` waits again right before the timed command (`suite/scripts/hyperfine-wait.sh`, `wait_for_vlt_background` in `clean-helpers.sh`). The waits are untimed; in real use, the next install -rarely starts that soon. +rarely starts that soon. No other tool was seen leaving processes behind. ## Normalization: why per-package time matters diff --git a/suite/scripts/add-remove-state.sh b/suite/scripts/add-remove-state.sh index ac31363c70..41914c0d07 100644 --- a/suite/scripts/add-remove-state.sh +++ b/suite/scripts/add-remove-state.sh @@ -70,20 +70,21 @@ cache_snapshot="$state_dir/cache" # - It renames replaced store entries to node_modules/.vlt/.VLT.DELETE.* # and deletes them in the background. rsync racing that deletion fails # with exit 24 ("some files vanished"). -# - It unpacks fetched tarballs into its cache (vlt_background_running in -# clean-helpers.sh). A snapshot or restore taken while that is still -# writing captures, or is undone by, a half-written cache. +# - Its other detached children (vlt-background.sh) keep writing to its +# cache, e.g. unpacking tarballs. A snapshot or restore taken while that +# is still writing captures, or is undone by, a half-written cache. # Wait for both here (untimed), never copy the trash, and treat a # vanished-file warning as success. Two clear checks in a row are required. # Unlike wait_for_vlt_background in clean-helpers.sh, this returns non-zero if # the work is still running after 300s, and the caller then fails this PM # rather than snapshot or restore a cache still being written. +source "$SCRIPTS_DIR/vlt-background.sh" + wait_for_pm_background() { local i clear=0 for i in $(seq 1 3000); do if ! compgen -G "node_modules/.vlt/.VLT.DELETE.*" >/dev/null && - ! pgrep -u "$(id -u)" -x vlt-cache-unzip >/dev/null && - ! pgrep -u "$(id -u)" -f 'cache-unzip-src-unzip[.]js' >/dev/null; then + ! vlt_background_running; then clear=$((clear + 1)) if [ "$clear" -ge 2 ]; then return 0 diff --git a/suite/scripts/clean-helpers.sh b/suite/scripts/clean-helpers.sh index f051f2ae4c..55e8cdfd3d 100644 --- a/suite/scripts/clean-helpers.sh +++ b/suite/scripts/clean-helpers.sh @@ -26,12 +26,19 @@ clean_berry_cache() { if command -v corepack &> /dev/null; then corepack yarn@latest cache clean --all >/dev/null || true fi + # `cache clean --all` leaves berry's metadata cache (packuments) in its + # global folder, so cold runs skipped every packument download. The global + # folder is ~/.yarn/berry, or under $XDG_DATA_HOME when that is set. + safe_remove "$HOME/.yarn/berry" + safe_remove "${XDG_DATA_HOME:-$HOME/.local/share}/yarn/berry" } +# zpm keeps its cache and metadata in ~/.yarn/zpm (with or without XDG_*). +# `yarn cache clean` doesn't reach it: without a packageManager field the +# Yarn Switch `yarn` runs Yarn 1, and with one zpm reports "No entries to +# clear". The Yarn Switch binaries (~/.yarn/switch) are the tool, not cache. clean_zpm_cache() { - if command -v yarn &> /dev/null; then - yarn cache clean >/dev/null || true - fi + safe_remove "$HOME/.yarn/zpm" } # Function to safely clean pnpm cache @@ -56,17 +63,12 @@ clean_vlt_cache() { fi } -# Whether vlt's background cache unpacking is running. vlt spawns it as a -# detached `node .../cache-unzip-src-unzip.js` process as it exits, and the -# process then renames itself `vlt-cache-unzip`, so match either form. The -# [.] keeps a shell whose arguments contain this pattern from matching itself. -vlt_background_running() { - pgrep -u "$(id -u)" -x vlt-cache-unzip >/dev/null || - pgrep -u "$(id -u)" -f 'cache-unzip-src-unzip[.]js' >/dev/null -} +# vlt_background_running: whether any of vlt's detached children is running. +source "$(dirname "${BASH_SOURCE[0]}")/vlt-background.sh" -# Wait for vlt's background cache unpacking to finish. It outlives the vlt -# command (about 19s after a cold babylon install), so without this it +# Wait for vlt's detached children to finish (cache unpacking, manifest +# revalidation, rollback-remove, ...). Unpacking alone runs about 19s after a +# cold babylon install, so without this they # overlaps the next timed run and writes into a cache that is being cleaned. # Waiting is untimed: in real use nobody starts the next install that soon. # Requires two clear checks in a row; returns non-zero after $1 seconds @@ -105,8 +107,13 @@ clean_bun_cache() { } # Function to safely clean nx cache +# Outside an Nx workspace these commands fail ("not part of an Nx workspace") +# and nx also runs `npm view nx@latest`, writing nx's packument into the npm +# cache that was just cleaned. nx's own cache lives in .nx, which +# clean_build_files removes. clean_nx_cache() { - if command -v nx &> /dev/null; then + if command -v nx &> /dev/null && + { [ -f nx.json ] || grep -q '"nx"' package.json 2>/dev/null; }; then nx daemon --stop >/dev/null || true nx clear-cache >/dev/null || true nx reset >/dev/null || true diff --git a/suite/scripts/clean-helpers.test.js b/suite/scripts/clean-helpers.test.js index 9d95468da5..b1cf574258 100644 --- a/suite/scripts/clean-helpers.test.js +++ b/suite/scripts/clean-helpers.test.js @@ -87,17 +87,28 @@ safe_remove() { printf '%s\\n' "$1" >> "$REMOVE_LOG"; } }); } -// A stand-in for vlt's background unzip: a copy of `sleep` named -// vlt-cache-unzip, so pgrep -x sees the same process name. Started from a -// throwaway sh, like vlt's detached child, so it is reaped when it exits -// instead of lingering as a zombie of this (blocked) test process. -const startFakeUnzip = (t, seconds) => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "vlt-unzip-test-")); - const bin = path.join(dir, "vlt-cache-unzip"); - fs.copyFileSync(execFileSync("sh", ["-c", "command -v sleep"], { encoding: "utf8" }).trim(), bin); - fs.chmodSync(bin, 0o755); +// A stand-in for one of vlt's detached children: `sleep` copied to a file +// with the given name, so pgrep sees that process name (truncated to 15 +// characters, as Linux does), or a node script at a vlt-style path when +// `script` is set. Started from a throwaway sh, like vlt's detached children, +// so it is reaped when it exits instead of lingering as a zombie of this +// (blocked) test process. +const startFake = (t, seconds, { name = "vlt-cache-unzip", script } = {}) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "vlt-child-test-")); + let argv; + if (script) { + const file = path.join(dir, "node_modules", "vlt", script); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `setTimeout(() => {}, ${seconds * 1000});\n`); + argv = [process.execPath, file]; + } else { + const bin = path.join(dir, name); + fs.copyFileSync(execFileSync("sh", ["-c", "command -v sleep"], { encoding: "utf8" }).trim(), bin); + fs.chmodSync(bin, 0o755); + argv = [bin, String(seconds)]; + } const pid = Number( - execFileSync("sh", ["-c", `"$0" "$1" >/dev/null 2>&1 & echo $!`, bin, String(seconds)], { + execFileSync("sh", ["-c", `"$@" >/dev/null 2>&1 & echo $!`, "sh", ...argv], { encoding: "utf8", }), ); @@ -108,6 +119,7 @@ const startFakeUnzip = (t, seconds) => { fs.rmSync(dir, { recursive: true, force: true }); }); }; +const startFakeUnzip = (t, seconds) => startFake(t, seconds); const waitForVltBackground = (limit) => spawnSync( @@ -193,3 +205,134 @@ test("hyperfine-wait.sh: wraps prepare, setup and cleanup", () => { fs.rmSync(tempDir, { recursive: true, force: true }); } }); + +const vltBackgroundRunning = () => + spawnSync("bash", ["-c", `source "$VB"; vlt_background_running`], { + env: { ...process.env, VB: path.join(__dirname, "vlt-background.sh") }, + }).status; + +test("vlt_background_running: false when nothing is running", () => { + assert.equal(vltBackgroundRunning(), 1); +}); + +for (const [label, opts] of [ + ["a renamed child, by its truncated name (vlt-cache-revalidate)", { name: "vlt-cache-revalidate" }], + ["a child that keeps its command line (rollback-remove)", { script: "rollback-remove-src-remove.js" }], + ["a child before it renames itself", { script: "registry-client-src-revalidate.js" }], +]) { + test(`vlt_background_running: matches ${label}`, (t) => { + startFake(t, 30, opts); + // Give sh a moment to exec the child. + execFileSync("sleep", ["0.3"]); + assert.equal(vltBackgroundRunning(), 0); + }); +} + +// Record what a cleaner removes, without touching anything real. +const removedBy = (command, env = {}) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "cleaner-test-")); + try { + const removeLog = path.join(tempDir, "removed.txt"); + fs.writeFileSync(removeLog, ""); + const result = spawnSync( + "bash", + ["-c", `source "$HELPERS" >/dev/null; safe_remove() { printf '%s\\n' "$1" >> "$REMOVE_LOG"; }; ${command}`], + { + encoding: "utf8", + cwd: tempDir, + env: { + ...process.env, + PATH: `${path.join(tempDir, "bin")}${path.delimiter}${process.env.PATH}`, + HOME: "/home/runner", + HELPERS: path.join(__dirname, "clean-helpers.sh"), + REMOVE_LOG: removeLog, + ...env, + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + return fs.readFileSync(removeLog, "utf8").trim().split("\n").filter(Boolean); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}; + +test("clean_zpm_cache removes ~/.yarn/zpm", () => { + assert.deepEqual(removedBy("clean_zpm_cache"), ["/home/runner/.yarn/zpm"]); +}); + +test("clean_berry_cache removes berry's global folder, with and without XDG_DATA_HOME", () => { + assert.deepEqual(removedBy("clean_berry_cache", { XDG_DATA_HOME: "" }).slice(-2), [ + "/home/runner/.yarn/berry", + "/home/runner/.local/share/yarn/berry", + ]); + assert.deepEqual(removedBy("clean_berry_cache", { XDG_DATA_HOME: "/xdg/data" }).slice(-2), [ + "/home/runner/.yarn/berry", + "/xdg/data/yarn/berry", + ]); +}); + +test("clean_nx_cache does nothing outside an Nx workspace", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nx-clean-test-")); + try { + const binDir = path.join(tempDir, "bin"); + const log = path.join(tempDir, "nx-calls.txt"); + fs.mkdirSync(binDir); + fs.writeFileSync(path.join(binDir, "nx"), `#!/bin/sh\necho "$*" >> "${log}"\n`, { mode: 0o755 }); + const run = (cwd) => + spawnSync("bash", ["-c", `source "$HELPERS" >/dev/null; clean_nx_cache`], { + cwd, + env: { + ...process.env, + PATH: `${binDir}${path.delimiter}${process.env.PATH}`, + HELPERS: path.join(__dirname, "clean-helpers.sh"), + }, + }).status; + const plain = path.join(tempDir, "plain"); + fs.mkdirSync(plain); + fs.writeFileSync(path.join(plain, "package.json"), '{"name":"x"}'); + assert.equal(run(plain), 0); + assert.equal(fs.existsSync(log), false, "nx ran outside a workspace"); + const ws = path.join(tempDir, "ws"); + fs.mkdirSync(ws); + fs.writeFileSync(path.join(ws, "package.json"), '{"name":"x","nx":{}}'); + assert.equal(run(ws), 0); + assert.deepEqual(fs.readFileSync(log, "utf8").trim().split("\n"), [ + "daemon --stop", + "clear-cache", + "reset", + ]); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test("package-count.sh files a Yarn 1 lockfile under yarn even with a .yarnrc.yml", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "count-test-")); + try { + const project = path.join(tempDir, "project"); + const out = path.join(tempDir, "out"); + fs.mkdirSync(path.join(project, "node_modules", "a"), { recursive: true }); + fs.writeFileSync( + path.join(project, "node_modules", "a", "package.json"), + '{"name":"a","version":"1.0.0"}', + ); + fs.writeFileSync( + path.join(project, "package.json"), + '{"name":"x","dependencies":{"a":"1.0.0"}}', + ); + fs.writeFileSync(path.join(project, ".yarnrc.yml"), "nodeLinker: node-modules\n"); + fs.writeFileSync( + path.join(project, "yarn.lock"), + "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile v1\n", + ); + const result = spawnSync("bash", [path.join(__dirname, "package-count.sh"), out], { + cwd: project, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(fs.readdirSync(out), ["yarn-count.txt"]); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); diff --git a/suite/scripts/hyperfine-wait.sh b/suite/scripts/hyperfine-wait.sh index ecb141e2c7..0315e9c207 100644 --- a/suite/scripts/hyperfine-wait.sh +++ b/suite/scripts/hyperfine-wait.sh @@ -1,6 +1,6 @@ # Sourced by variations/common.sh and registry/common.sh. # -# Wraps hyperfine so vlt's background cache unpacking (see +# Wraps hyperfine so vlt's detached background work (see vlt-background.sh and # wait_for_vlt_background in clean-helpers.sh) can't spill into the next run. # Every --setup and --cleanup waits for it first, and every --prepare waits # both first (before it cleans caches) and last, right before the timed diff --git a/suite/scripts/package-count.sh b/suite/scripts/package-count.sh index 99573b2fa3..4db0e0280f 100644 --- a/suite/scripts/package-count.sh +++ b/suite/scripts/package-count.sh @@ -22,7 +22,12 @@ infer_package_manager() { elif [[ -f "pnpm-lock.yaml" ]]; then echo "pnpm" elif [[ -f "yarn.lock" ]]; then - if [[ -f ".yarnrc.yml" ]] && command -v yarn >/dev/null 2>&1; then + # Yarn 1 writes a v1 lockfile header; berry and zpm write __metadata. + # Check it first: next and babylon commit a .yarnrc.yml, which used to + # file Yarn 1's counts under berry. + if grep -q "^# yarn lockfile v1" yarn.lock 2>/dev/null; then + echo "yarn" + elif [[ -f ".yarnrc.yml" ]] && command -v yarn >/dev/null 2>&1; then yarn_version=$(yarn -v 2>/dev/null || echo "") if [[ "$yarn_version" =~ ^6\. ]]; then echo "zpm" diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index 2e4ec0bf78..cab358ef69 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -38,11 +38,15 @@ SETUP_INSTALL_BERRY="" # corepack auto-downloads on first invocation # zpm is installed via the Yarn Switch installer (same as setup.sh) SETUP_INSTALL_ZPM="curl -sS https://repo.yarnpkg.com/install | bash >/dev/null 2>&1" SETUP_INSTALL_PNPM="" # corepack auto-downloads on first invocation -SETUP_INSTALL_VLT="npm install -g vlt@latest >/dev/null 2>&1" -SETUP_INSTALL_BUN="npm install -g --allow-scripts=bun bun@latest >/dev/null 2>&1" -SETUP_INSTALL_DENO="npm install -g --allow-scripts=deno deno@latest >/dev/null 2>&1" -SETUP_INSTALL_AUBE="npm install -g --allow-scripts=@endevco/aube @endevco/aube@latest >/dev/null 2>&1" -SETUP_INSTALL_UPM="npm install -g upm@latest >/dev/null 2>&1" +# The tools installed with npm get their own npm cache, deleted with the tool +# below, like npm's. Otherwise the timed install is served from ~/.npm, which +# the prepare's own install of the same tool just filled. +TOOL_SETUP_CACHE="/tmp/bench-tool-setup-cache" +SETUP_INSTALL_VLT="npm install -g --cache $TOOL_SETUP_CACHE vlt@latest >/dev/null 2>&1" +SETUP_INSTALL_BUN="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=bun bun@latest >/dev/null 2>&1" +SETUP_INSTALL_DENO="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=deno deno@latest >/dev/null 2>&1" +SETUP_INSTALL_AUBE="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=@endevco/aube @endevco/aube@latest >/dev/null 2>&1" +SETUP_INSTALL_UPM="npm install -g --cache $TOOL_SETUP_CACHE upm@latest >/dev/null 2>&1" # --- PM uninstall commands (run at end of prepare to force re-install) --- SETUP_UNINSTALL_NPM="rm -rf $NPM_SETUP_PREFIX $NPM_SETUP_CACHE" @@ -50,11 +54,11 @@ SETUP_UNINSTALL_YARN="" # corepack managed; cache purged below SETUP_UNINSTALL_BERRY="" SETUP_UNINSTALL_ZPM="rm -rf \$HOME/.yarn/switch >/dev/null 2>&1 || true" SETUP_UNINSTALL_PNPM="" -SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true" -SETUP_UNINSTALL_BUN="npm uninstall -g bun >/dev/null 2>&1 || true" -SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true" -SETUP_UNINSTALL_AUBE="npm uninstall -g @endevco/aube >/dev/null 2>&1 || true" -SETUP_UNINSTALL_UPM="npm uninstall -g upm >/dev/null 2>&1 || true" +SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" +SETUP_UNINSTALL_BUN="npm uninstall -g bun >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" +SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" +SETUP_UNINSTALL_AUBE="npm uninstall -g @endevco/aube >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" +SETUP_UNINSTALL_UPM="npm uninstall -g upm >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" # Purge the isolated corepack home so corepack-managed PMs must re-download. COREPACK_PURGE="rm -rf $COREPACK_HOME" diff --git a/suite/scripts/vlt-background.sh b/suite/scripts/vlt-background.sh new file mode 100644 index 0000000000..2b173f9f4c --- /dev/null +++ b/suite/scripts/vlt-background.sh @@ -0,0 +1,24 @@ +# Sourced by clean-helpers.sh and add-remove-state.sh. +# +# vlt finishes work in detached child processes that outlive the vlt command. +# vlt 1.8.0 has five, each a `node .../vlt/-src-.js` script: +# +# cache-unzip-src-unzip.js unpacks fetched tarballs into the +# cache (renames itself vlt-cache-unzip) +# registry-client-src-revalidate.js revalidates cached manifests +# (vlt-cache-revalidate) +# rollback-remove-src-remove.js deletes node_modules/.vlt/.VLT.DELETE.* +# (keeps its command line) +# security-archive-src-update-expired.js (vlt-security-archive-update) +# cli-sdk-src-update-check.js (vlt-update-check) +# +# Match the renamed ones by process name (Linux truncates it to 15 +# characters, e.g. "vlt-cache-reval", so match the vlt- prefix), and every one +# by its original command line, which covers rollback-remove and the moment +# before a child renames itself. Matching the name, not the command line, for +# the first check means a shell whose arguments mention these names can never +# match itself; the [.] does the same for the second. +vlt_background_running() { + pgrep -u "$(id -u)" '^vlt-' >/dev/null || + pgrep -u "$(id -u)" -f '/vlt/[a-z-]+-src-[a-z-]+[.]js' >/dev/null +}