From 15da8ab71da1fd2704d20fa8021f0071834c263a Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Tue, 17 Nov 2026 23:56:23 +0000 Subject: [PATCH 1/3] fix: run pnpm, yarn 1 and yarn 4 the way their maintainers recommend pnpm, yarn classic and yarn berry ran through `corepack pnpm@latest`, `corepack yarn@1` and `corepack yarn@latest` in every timed command. That started a Node.js process (Corepack) in front of each one, and the @latest forms made a registry request on every call: about half a second for each pnpm command. - pnpm: `npm install -g --allow-scripts=pnpm pnpm@latest` (the native binary; npm 12 otherwise blocks the postinstall that installs it), called as `pnpm`. setup.sh fails if `pnpm` on PATH is not that ELF binary. - yarn 1: `npm install --global yarn@1`, into its own prefix. - berry: `npm install -g corepack` into its own prefix (the corepack package ships yarn/pnpm shims that would clash), and each berry run pins the version setup.sh resolved in packageManager and runs the `yarn` shim. - zpm: unchanged, but called by full path like the other two yarns. - ci+setup: the setup part of the timed command is now each tool's recommended install into a fresh prefix with an empty npm cache. - Cleaners call the direct binaries; pnpm's also follows XDG_*. - COREPACK_ENABLE_STRICT / COREPACK_ENABLE_AUTO_PIN are no longer needed. Closes #207 Co-Authored-By: Claude Opus 5.5 --- README.md | 16 ++++- docs/METHODOLOGY.md | 34 +++++++++++ suite/scripts/clean-helpers.sh | 37 +++++++----- suite/scripts/package-count.sh | 11 ++-- suite/scripts/pm-bins.sh | 28 +++++++++ suite/scripts/setup.sh | 53 +++++++++++------ suite/scripts/variations/add-remove.sh | 16 ++--- suite/scripts/variations/build-cache.sh | 16 ++--- suite/scripts/variations/build.sh | 8 +-- suite/scripts/variations/ci+setup.sh | 58 +++++++++++-------- suite/scripts/variations/ci.sh | 16 ++--- suite/scripts/variations/common.sh | 23 +++++--- .../variations/lockfile+node_modules.sh | 5 +- suite/scripts/variations/lockfile.sh | 5 +- suite/scripts/variations/node_modules.sh | 5 +- suite/scripts/variations/registry-lockfile.sh | 8 +-- suite/scripts/variations/run.sh | 8 +-- 17 files changed, 230 insertions(+), 117 deletions(-) create mode 100644 suite/scripts/pm-bins.sh diff --git a/README.md b/README.md index 49e427a5bf9..42524fd1364 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,21 @@ This suite also tests the performance of basic script execution (ex. `npm run fo - 1. Setup: - 1.1 Install `jq` using your OS package manager - 1.2 Install `hyperfine`, version >= 1.19.0 is required - - 1.3 Install package managers and corepack: + - 1.3 Install the package managers the way `suite/scripts/setup.sh` does + (see "How each tool is installed and run" in + [`docs/METHODOLOGY.md`](docs/METHODOLOGY.md)). Yarn 1, Yarn 4 and Yarn 6 + are all called `yarn`, so yarn 1 and Corepack go into their own prefixes + under `~/.bench-pms` (`suite/scripts/pm-bins.sh`): ```bash - npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest + npm install -g npm@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest + # --allow-scripts lets pnpm's postinstall install its native binary + npm install -g --allow-scripts=pnpm pnpm@latest + npm install -g --prefix ~/.bench-pms/yarn yarn@1 + npm install -g --prefix ~/.bench-pms/corepack corepack@latest + # the Yarn 4 version every berry benchmark pins in packageManager + ~/.bench-pms/corepack/bin/corepack yarn@latest -v > ~/.bench-pms/berry-version + # Yarn 6 (zpm), through Yarn Switch + curl -sS https://repo.yarnpkg.com/install | bash ``` - 1.4 No directory setup is needed. `./suite/bench` creates its output tree at `suite/results/`, which is the only results tree there is. diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index 1e0addf113e..37ed84bbb26 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -59,6 +59,40 @@ to finish, and every `--prepare` waits again right before the timed command `clean-helpers.sh`). The waits are untimed; in real use, the next install rarely starts that soon. No other tool was seen leaving processes behind. +## How each tool is installed and run + +Each package manager is installed the way its maintainers recommend +(`suite/scripts/setup.sh`), and every timed command runs that tool directly, +the way a user would, with no extra launcher in front of it. + +| Tool | Installed with | Timed command runs | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| npm, vlt, bun, deno | `npm install -g @latest` | `` on `PATH` | +| pnpm | `npm install -g --allow-scripts=pnpm pnpm@latest` ([pnpm.io](https://pnpm.io/installation)) | `pnpm` on `PATH`, pnpm's native binary | +| yarn (1) | `npm install --global yarn@1` ([classic.yarnpkg.com](https://classic.yarnpkg.com/en/docs/install)) | that `yarn`, by full path | +| berry (Yarn 4) | `npm install -g corepack` ([yarnpkg.com](https://yarnpkg.com/getting-started/install)) | Corepack's `yarn` shim, by full path, with `packageManager: yarn@` pinned | +| zpm (Yarn 6) | Yarn Switch (`curl -sS https://repo.yarnpkg.com/install \| bash`, [v6.yarnpkg.com](https://v6.yarnpkg.com/getting-started)) | Yarn Switch's `yarn`, by full path, with `packageManager: yarn@` pinned | + +Yarn 1, Yarn 4 and Yarn 6 all install a command called `yarn`, so yarn 1 and +Corepack get their own npm prefixes (`suite/scripts/pm-bins.sh`) and all three +are called by full path. npm 12 blocks install scripts by default; without +`--allow-scripts=pnpm`, pnpm's postinstall doesn't put its native binary in +place and `pnpm` runs through a JavaScript launcher instead. The berry version +is resolved once per job (`yarn@latest`) and pinned in `package.json` for each +berry run, as `yarn set version` would; `clean-helpers.sh` removes the pin +before the next tool runs. In `ci+setup`, the "setup" part of each timed +command is the same install (into a fresh prefix with an empty npm cache, so +it downloads every time); for berry that is installing Corepack plus +Corepack's download of Yarn. + +Until [#207](https://github.com/vltpkg/benchmarks/issues/207), pnpm, yarn and +berry ran through `corepack pnpm@latest`, `corepack yarn@1` and +`corepack yarn@latest`. That put a Node.js process (Corepack) in front of every +timed command, and for the `@latest` forms a registry lookup too: about half a +second per pnpm command, less for yarn. pnpm's (and to a lesser degree yarn's) +results drop at that point because that overhead is gone, not because pnpm +got faster. + ## Normalization: why per-package time matters Package managers disagree about what a dependency graph contains. Comparing diff --git a/suite/scripts/clean-helpers.sh b/suite/scripts/clean-helpers.sh index 55e8cdfd3d2..29dab6e2da2 100644 --- a/suite/scripts/clean-helpers.sh +++ b/suite/scripts/clean-helpers.sh @@ -1,6 +1,9 @@ # Exit on error set -Eeuxo pipefail +# Where yarn classic, berry and zpm live (BENCH_YARN_BIN, BENCH_BERRY_BIN, ...) +source "$(dirname "${BASH_SOURCE[0]}")/pm-bins.sh" + # Function to safely remove files/directories safe_remove() { if [ -e "$1" ]; then @@ -17,8 +20,8 @@ clean_npm_cache() { # Function to safely clean yarn cache clean_yarn_cache() { - if command -v corepack &> /dev/null; then - corepack yarn@1 cache clean --all --silent || true + if [ -x "$BENCH_YARN_BIN" ]; then + "$BENCH_YARN_BIN" cache clean --silent || true fi } @@ -43,16 +46,16 @@ clean_zpm_cache() { # Function to safely clean pnpm cache clean_pnpm_cache() { - if command -v corepack &> /dev/null; then - corepack pnpm cache delete '*' --silent || true + if command -v pnpm &> /dev/null; then + pnpm cache delete '*' --silent || true # Remove the version-specific store directory reported by this pnpm - safe_remove "$(corepack pnpm store path 2>/dev/null | xargs)" + safe_remove "$(pnpm store path 2>/dev/null | xargs)" fi # Also wipe the entire metadata cache and store parent to catch any # leftover version directories (v3, v9, v10, v11 …) that a single # `pnpm store path` wouldn't return. - safe_remove "$HOME/.cache/pnpm" - safe_remove "$HOME/.local/share/pnpm/store" + safe_remove "${XDG_CACHE_HOME:-$HOME/.cache}/pnpm" + safe_remove "${XDG_DATA_HOME:-$HOME/.local/share}/pnpm/store" } @@ -190,12 +193,15 @@ clean_lockfiles() { clean_package_manager_field() { echo "Removing packageManager / devEngines.packageManager from package.json..." if command -v vlt &> /dev/null; then + # The berry and zpm prepares pin packageManager to their own version; + # yarn classic and pnpm refuse to run in a project pinned to another + # package manager. vlt pkg rm packageManager - # Recent corepack/yarn auto-pin a devEngines.packageManager entry (e.g. - # yarn) into package.json. npm and pnpm enforce devEngines and refuse to - # run any command when it doesn't match (EBADDEVENGINES), which breaks - # every benchmark that isn't that package manager. Strip it so each PM can - # run. vlt is used because it doesn't enforce devEngines, whereas npm/pnpm + # Some package managers can write a devEngines.packageManager entry into + # package.json. npm and pnpm enforce devEngines and refuse to run any + # command when it doesn't match (EBADDEVENGINES), which breaks every + # benchmark that isn't that package manager. Strip it so each PM can run. + # vlt is used because it doesn't enforce devEngines, whereas npm/pnpm # would themselves fail on the field we're trying to remove. vlt pkg rm devEngines.packageManager fi @@ -288,10 +294,9 @@ clean_all() { clean_package_manager_files clean_workspace_protocol clean_all_cache - # Strip packageManager / devEngines.packageManager AFTER the cache cleans: - # the corepack-based cache cleans in clean_all_cache can re-pin a - # devEngines.packageManager entry into package.json, so this must run last - # to guarantee a neutral manifest before the next benchmark. + # Strip packageManager / devEngines.packageManager AFTER the cache cleans, + # so nothing they write survives: this must run last to guarantee a + # neutral manifest before the next benchmark. clean_package_manager_field clean_build_files clean_build_output diff --git a/suite/scripts/package-count.sh b/suite/scripts/package-count.sh index 4db0e0280f9..2052fe7d9e0 100644 --- a/suite/scripts/package-count.sh +++ b/suite/scripts/package-count.sh @@ -22,18 +22,15 @@ infer_package_manager() { elif [[ -f "pnpm-lock.yaml" ]]; then echo "pnpm" elif [[ -f "yarn.lock" ]]; then + # The zpm and berry setups pin their own version in packageManager. + package_manager=$(node -p 'require("./package.json").packageManager || ""' 2>/dev/null || echo "") # Yarn 1 writes a v1 lockfile header; berry and zpm write __metadata. # Check it first: next and babylon commit a .yarnrc.yml, which used to # file Yarn 1's counts under berry. if grep -q "^# yarn lockfile v1" yarn.lock 2>/dev/null; then echo "yarn" - elif [[ -f ".yarnrc.yml" ]] && command -v yarn >/dev/null 2>&1; then - yarn_version=$(yarn -v 2>/dev/null || echo "") - if [[ "$yarn_version" =~ ^6\. ]]; then - echo "zpm" - else - echo "berry" - fi + elif [[ "$package_manager" =~ ^yarn@6\. ]]; then + echo "zpm" elif [[ -d ".yarn" ]] || [[ -f ".yarnrc.yml" ]]; then echo "berry" else diff --git a/suite/scripts/pm-bins.sh b/suite/scripts/pm-bins.sh new file mode 100644 index 00000000000..42f27fc0493 --- /dev/null +++ b/suite/scripts/pm-bins.sh @@ -0,0 +1,28 @@ +# Where setup.sh installs yarn classic and yarn berry, and the binaries the +# variations call. Sourced by setup.sh, variations/common.sh and +# clean-helpers.sh. +# +# Yarn 1 (`npm install --global yarn@1`), Yarn 4 (Corepack's `yarn` shim) and +# Yarn 6 (Yarn Switch) all install a `yarn` command. Each one lives in its own +# directory, none of them on PATH except Yarn Switch, and every benchmark calls +# them by full path so it's never ambiguous which one runs. pnpm has no clash +# and is called as plain `pnpm` (installed with `npm install -g` by setup.sh). +BENCH_PM_HOME="${BENCH_PM_HOME:-$HOME/.bench-pms}" + +# Yarn 1, installed with npm into its own prefix. +BENCH_YARN_PREFIX="$BENCH_PM_HOME/yarn" +BENCH_YARN_BIN="$BENCH_YARN_PREFIX/bin/yarn" + +# Yarn 4, through Corepack installed with npm into its own prefix: the corepack +# package ships `yarn` and `pnpm` shims of its own, which would collide with +# the npm-installed pnpm in the global prefix. Each berry benchmark pins the +# version setup.sh resolved in package.json#packageManager, like a project +# set up with `yarn set version`, and runs the `yarn` shim. +BENCH_COREPACK_PREFIX="$BENCH_PM_HOME/corepack" +BENCH_COREPACK_BIN="$BENCH_COREPACK_PREFIX/bin/corepack" +BENCH_BERRY_BIN="$BENCH_COREPACK_PREFIX/bin/yarn" +BENCH_BERRY_VERSION_FILE="$BENCH_PM_HOME/berry-version" +BENCH_BERRY_VERSION="${BENCH_BERRY_VERSION:-$(cat "$BENCH_BERRY_VERSION_FILE" 2>/dev/null || true)}" + +# Yarn 6 (zpm), installed by the Yarn Switch installer. +BENCH_ZPM_BIN="$HOME/.yarn/switch/bin/yarn" diff --git a/suite/scripts/setup.sh b/suite/scripts/setup.sh index 925cbc049f1..a28d36f57dc 100644 --- a/suite/scripts/setup.sh +++ b/suite/scripts/setup.sh @@ -4,18 +4,10 @@ set -e # Environment Configuration -export COREPACK_ENABLE_STRICT=0 -export COREPACK_ENABLE_AUTO_PIN=0 export YARN_ENABLE_IMMUTABLE_INSTALLS=false -# Persist corepack env vars for subsequent CI steps (e.g., benchmark runs). -# Without these, corepack 0.34+ auto-pins packageManager and refuses to run -# a different PM than what packageManager specifies — breaking benchmarks -# that cycle through multiple package managers on the same fixture. -if [ -n "${GITHUB_ENV:-}" ]; then - echo "COREPACK_ENABLE_STRICT=0" >> "$GITHUB_ENV" - echo "COREPACK_ENABLE_AUTO_PIN=0" >> "$GITHUB_ENV" -fi +# Install locations for yarn classic and yarn berry (BENCH_YARN_PREFIX, ...) +source "$(dirname "${BASH_SOURCE[0]}")/pm-bins.sh" # Check Node version REQUIRED_NODE_VERSION="26" @@ -45,7 +37,28 @@ echo "hyperfine: $HYPERFINE_VERSION" # Install Node.js package managers and tools echo "Installing package managers and tools..." -npm install -g --allow-scripts=bun,deno,nx npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest +npm install -g --allow-scripts=bun,deno,nx npm@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest + +# Install pnpm with npm, one of the methods in https://pnpm.io/installation. +# npm 12 blocks install scripts by default, but pnpm's postinstall is what puts +# its native binary in place of the JavaScript launcher, so allow it. +npm install -g --allow-scripts=pnpm pnpm@latest +PNPM_BIN="$(command -v pnpm)" +if [ "$PNPM_BIN" != "$(npm prefix -g)/bin/pnpm" ] || [ "$(head -c 4 "$(readlink -f "$PNPM_BIN")")" != $'\x7fELF' ]; then + echo "Error: expected pnpm on PATH to be the native binary installed by npm, got $PNPM_BIN ($(file -L "$PNPM_BIN" 2>/dev/null || true))" + exit 1 +fi + +# Install Yarn 1 the way https://classic.yarnpkg.com/en/docs/install +# recommends (npm install --global yarn@1), into its own prefix: zpm's Yarn +# Switch and berry's Corepack shim are also called `yarn` (see pm-bins.sh). +# npm 12 skips yarn's preinstall script, which only removes a Corepack `yarn` +# symlink from the prefix being installed into; there is none in this one. +npm install -g --prefix "$BENCH_YARN_PREFIX" yarn@1 + +# Install Corepack for Yarn 4, as https://yarnpkg.com/getting-started/install +# recommends (npm install -g corepack), into its own prefix (see pm-bins.sh). +npm install -g --prefix "$BENCH_COREPACK_PREFIX" corepack@latest # Install Vite+ (vp) via npm (available as the `vite-plus` package) npm install -g vite-plus@latest @@ -75,9 +88,12 @@ if ! command -v aube &>/dev/null; then fi fi -# Configure Package Managers -echo "Configuring package managers..." -corepack enable yarn pnpm +# Resolve the berry version once, so every berry benchmark pins the same one +# in package.json#packageManager (and Corepack downloads it now, not in a +# timed run). +BERRY_VERSION="$("$BENCH_COREPACK_BIN" yarn@latest -v)" +mkdir -p "$BENCH_PM_HOME" +echo "$BERRY_VERSION" > "$BENCH_BERRY_VERSION_FILE" # Install Yarn v6 Canary (zpm) curl -sS https://repo.yarnpkg.com/install | bash @@ -92,10 +108,9 @@ mkdir -p ./results/ echo "Logging package manager versions..." NPM_VERSION="$(npm -v)" VLT_VERSION="$(vlt -v)" -YARN_VERSION="$(corepack yarn@1 -v)" -BERRY_VERSION="$(corepack yarn@latest -v)" +YARN_VERSION="$("$BENCH_YARN_BIN" -v)" ZPM_VERSION="${BENCH_ZPM_VERSION:-$(curl -fsSL https://repo.yarnpkg.com/channels/default/canary)}" -PNPM_VERSION="$(corepack pnpm@latest -v)" +PNPM_VERSION="$(pnpm -v)" BUN_VERSION="$(bun -v)" DENO_VERSION="$(npm view deno@latest version)" NX_VERSION="$(npm view nx@latest version)" @@ -120,6 +135,10 @@ echo "vp: $VP_VERSION" echo "aube: $AUBE_VERSION" echo "upm: $UPM_VERSION" echo "node: $NODE_VERSION" +echo "pnpm binary: $PNPM_BIN" +echo "yarn binary: $BENCH_YARN_BIN" +echo "berry binary: $BENCH_BERRY_BIN (packageManager: yarn@$BERRY_VERSION)" +echo "zpm binary: $BENCH_ZPM_BIN" # Save versions to JSON file echo "{ diff --git a/suite/scripts/variations/add-remove.sh b/suite/scripts/variations/add-remove.sh index eaa10610406..f364775393e 100644 --- a/suite/scripts/variations/add-remove.sh +++ b/suite/scripts/variations/add-remove.sh @@ -97,10 +97,10 @@ if [ "$BENCH_FIXTURE" = "large" ]; then AR_NPM_FLAGS="$AR_NPM_FLAGS --legacy-peer-deps" fi BENCH_ADD_NPM="npm install $AR_SPEC $AR_NPM_FLAGS" -BENCH_ADD_YARN="corepack yarn@1 add $AR_SPEC$AR_WS_YARN --ignore-scripts --silent" -BENCH_ADD_BERRY="corepack yarn@latest add $AR_SPEC" -BENCH_ADD_ZPM="yarn add $AR_SPEC" -BENCH_ADD_PNPM="corepack pnpm@latest add $AR_SPEC$AR_WS_PNPM --ignore-scripts --silent" +BENCH_ADD_YARN="$BENCH_YARN_BIN add $AR_SPEC$AR_WS_YARN --ignore-scripts --silent" +BENCH_ADD_BERRY="$BENCH_BERRY_BIN add $AR_SPEC" +BENCH_ADD_ZPM="$BENCH_ZPM_BIN add $AR_SPEC" +BENCH_ADD_PNPM="pnpm add $AR_SPEC$AR_WS_PNPM --ignore-scripts --silent" BENCH_ADD_VLT="vlt install $AR_SPEC --view=silent" BENCH_ADD_BUN="bun add $AR_SPEC --ignore-scripts --silent" BENCH_ADD_DENO="deno add --quiet npm:$AR_SPEC" @@ -108,10 +108,10 @@ BENCH_ADD_AUBE="aube add $AR_SPEC$AR_WS_PNPM --ignore-scripts --silent" BENCH_ADD_UPM="upm add $AR_SPEC --silent" BENCH_REMOVE_NPM="npm uninstall $AR_NAME $AR_NPM_FLAGS" -BENCH_REMOVE_YARN="corepack yarn@1 remove $AR_NAME$AR_WS_YARN --ignore-scripts --silent" -BENCH_REMOVE_BERRY="corepack yarn@latest remove $AR_NAME" -BENCH_REMOVE_ZPM="yarn remove $AR_NAME" -BENCH_REMOVE_PNPM="corepack pnpm@latest remove $AR_NAME$AR_WS_PNPM --ignore-scripts --silent" +BENCH_REMOVE_YARN="$BENCH_YARN_BIN remove $AR_NAME$AR_WS_YARN --ignore-scripts --silent" +BENCH_REMOVE_BERRY="$BENCH_BERRY_BIN remove $AR_NAME" +BENCH_REMOVE_ZPM="$BENCH_ZPM_BIN remove $AR_NAME" +BENCH_REMOVE_PNPM="pnpm remove $AR_NAME$AR_WS_PNPM --ignore-scripts --silent" BENCH_REMOVE_VLT="vlt uninstall $AR_NAME --view=silent" BENCH_REMOVE_BUN="bun remove $AR_NAME --ignore-scripts --silent" BENCH_REMOVE_DENO="deno remove --quiet npm:$AR_NAME" diff --git a/suite/scripts/variations/build-cache.sh b/suite/scripts/variations/build-cache.sh index 3eda632721b..244f47452d4 100644 --- a/suite/scripts/variations/build-cache.sh +++ b/suite/scripts/variations/build-cache.sh @@ -34,10 +34,10 @@ BENCH_INSTALL_PREPARE_UPM="$(prepend_setup "$BENCH_INSTALL_COMMAND_UPM" "$BENCH_ # Warm build commands (run once in prepare to populate the .next/ cache). BENCH_WARM_BUILD_NPM="timeout $BENCH_TIMEOUT npm run build > $BENCH_OUTPUT_FOLDER/npm-warm-build-output.log 2>&1" -BENCH_WARM_BUILD_YARN="timeout $BENCH_TIMEOUT corepack yarn@1 run build > $BENCH_OUTPUT_FOLDER/yarn-warm-build-output.log 2>&1" -BENCH_WARM_BUILD_BERRY="timeout $BENCH_TIMEOUT corepack yarn@latest run build > $BENCH_OUTPUT_FOLDER/berry-warm-build-output.log 2>&1" -BENCH_WARM_BUILD_ZPM="timeout $BENCH_TIMEOUT yarn run build > $BENCH_OUTPUT_FOLDER/zpm-warm-build-output.log 2>&1" -BENCH_WARM_BUILD_PNPM="timeout $BENCH_TIMEOUT corepack pnpm@latest run build > $BENCH_OUTPUT_FOLDER/pnpm-warm-build-output.log 2>&1" +BENCH_WARM_BUILD_YARN="timeout $BENCH_TIMEOUT $BENCH_YARN_BIN run build > $BENCH_OUTPUT_FOLDER/yarn-warm-build-output.log 2>&1" +BENCH_WARM_BUILD_BERRY="timeout $BENCH_TIMEOUT $BENCH_BERRY_BIN run build > $BENCH_OUTPUT_FOLDER/berry-warm-build-output.log 2>&1" +BENCH_WARM_BUILD_ZPM="timeout $BENCH_TIMEOUT $BENCH_ZPM_BIN run build > $BENCH_OUTPUT_FOLDER/zpm-warm-build-output.log 2>&1" +BENCH_WARM_BUILD_PNPM="timeout $BENCH_TIMEOUT pnpm run build > $BENCH_OUTPUT_FOLDER/pnpm-warm-build-output.log 2>&1" BENCH_WARM_BUILD_VLT="timeout $BENCH_TIMEOUT vlt run build --view=human > $BENCH_OUTPUT_FOLDER/vlt-warm-build-output.log 2>&1" BENCH_WARM_BUILD_BUN="timeout $BENCH_TIMEOUT bun run build > $BENCH_OUTPUT_FOLDER/bun-warm-build-output.log 2>&1" BENCH_WARM_BUILD_DENO="timeout $BENCH_TIMEOUT deno run build > $BENCH_OUTPUT_FOLDER/deno-warm-build-output.log 2>&1" @@ -49,10 +49,10 @@ BENCH_WARM_BUILD_NODE="timeout $BENCH_TIMEOUT node --run build > $BENCH_OUTPUT_F # Timed build commands (the actual benchmark — rebuild with .next/ cache). BENCH_COMMAND_NPM="timeout $BENCH_TIMEOUT npm run build > $BENCH_OUTPUT_FOLDER/npm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT corepack yarn@1 run build > $BENCH_OUTPUT_FOLDER/yarn-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT corepack yarn@latest run build > $BENCH_OUTPUT_FOLDER/berry-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT yarn run build > $BENCH_OUTPUT_FOLDER/zpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT corepack pnpm@latest run build > $BENCH_OUTPUT_FOLDER/pnpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT $BENCH_YARN_BIN run build > $BENCH_OUTPUT_FOLDER/yarn-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT $BENCH_BERRY_BIN run build > $BENCH_OUTPUT_FOLDER/berry-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT $BENCH_ZPM_BIN run build > $BENCH_OUTPUT_FOLDER/zpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT pnpm run build > $BENCH_OUTPUT_FOLDER/pnpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_VLT="timeout $BENCH_TIMEOUT vlt run build --view=human > $BENCH_OUTPUT_FOLDER/vlt-build-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_BUN="timeout $BENCH_TIMEOUT bun run build > $BENCH_OUTPUT_FOLDER/bun-build-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_DENO="timeout $BENCH_TIMEOUT deno run build > $BENCH_OUTPUT_FOLDER/deno-build-output-\${HYPERFINE_ITERATION}.log 2>&1" diff --git a/suite/scripts/variations/build.sh b/suite/scripts/variations/build.sh index 595ade5a89b..c0876b78e35 100644 --- a/suite/scripts/variations/build.sh +++ b/suite/scripts/variations/build.sh @@ -35,10 +35,10 @@ BENCH_INSTALL_PREPARE_UPM="$(prepend_setup "$BENCH_INSTALL_COMMAND_UPM" "$BENCH_ # Build commands — run ` run build` and log output. # Each prepare also removes .next/ so every iteration is a cold build. BENCH_COMMAND_NPM="timeout $BENCH_TIMEOUT npm run build > $BENCH_OUTPUT_FOLDER/npm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT corepack yarn@1 run build > $BENCH_OUTPUT_FOLDER/yarn-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT corepack yarn@latest run build > $BENCH_OUTPUT_FOLDER/berry-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT yarn run build > $BENCH_OUTPUT_FOLDER/zpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT corepack pnpm@latest run build > $BENCH_OUTPUT_FOLDER/pnpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT $BENCH_YARN_BIN run build > $BENCH_OUTPUT_FOLDER/yarn-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT $BENCH_BERRY_BIN run build > $BENCH_OUTPUT_FOLDER/berry-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT $BENCH_ZPM_BIN run build > $BENCH_OUTPUT_FOLDER/zpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT pnpm run build > $BENCH_OUTPUT_FOLDER/pnpm-build-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_VLT="timeout $BENCH_TIMEOUT vlt run build --view=human > $BENCH_OUTPUT_FOLDER/vlt-build-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_BUN="timeout $BENCH_TIMEOUT bun run build > $BENCH_OUTPUT_FOLDER/bun-build-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_DENO="timeout $BENCH_TIMEOUT deno run build > $BENCH_OUTPUT_FOLDER/deno-build-output-\${HYPERFINE_ITERATION}.log 2>&1" diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index cab358ef69c..d4091315643 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -28,16 +28,28 @@ export COREPACK_HOME="/tmp/bench-corepack-home" # --- PM install commands --- # These mirror what scripts/setup.sh does for each PM. -# npm: install into a dedicated prefix so the bootstrap npm and its cache are -# separate from the dependency-level npm cache used by `npm ci`. +# npm, yarn classic, pnpm and berry's corepack: install into a dedicated prefix +# with a dedicated npm cache, both removed by the uninstall step, so every +# timed install downloads the package manager again. This also keeps the +# bootstrap npm and the copies installed by setup.sh untouched, and the npm +# cache used by `npm ci` separate. NPM_SETUP_PREFIX="/tmp/bench-npm" NPM_SETUP_CACHE="/tmp/bench-npm-cache" +YARN_SETUP_PREFIX="/tmp/bench-yarn" +YARN_SETUP_CACHE="/tmp/bench-yarn-npm-cache" +PNPM_SETUP_PREFIX="/tmp/bench-pnpm" +PNPM_SETUP_CACHE="/tmp/bench-pnpm-npm-cache" +COREPACK_SETUP_PREFIX="/tmp/bench-corepack" +COREPACK_SETUP_CACHE="/tmp/bench-corepack-npm-cache" SETUP_INSTALL_NPM="npm install -g npm@latest --prefix $NPM_SETUP_PREFIX --cache $NPM_SETUP_CACHE >/dev/null 2>&1" -SETUP_INSTALL_YARN="" # corepack auto-downloads on first invocation -SETUP_INSTALL_BERRY="" # corepack auto-downloads on first invocation +SETUP_INSTALL_YARN="npm install -g yarn@1 --prefix $YARN_SETUP_PREFIX --cache $YARN_SETUP_CACHE >/dev/null 2>&1" +# berry: install corepack; its yarn shim then downloads the pinned yarn on +# first use (COREPACK_HOME is purged too). +SETUP_INSTALL_BERRY="npm install -g corepack@latest --prefix $COREPACK_SETUP_PREFIX --cache $COREPACK_SETUP_CACHE >/dev/null 2>&1" # zpm is installed via the Yarn Switch installer (same as setup.sh) SETUP_INSTALL_ZPM="curl -sS https://repo.yarnpkg.com/install | bash >/dev/null 2>&1" -SETUP_INSTALL_PNPM="" # corepack auto-downloads on first invocation +# --allow-scripts=pnpm lets pnpm's postinstall put its native binary in place. +SETUP_INSTALL_PNPM="npm install -g --allow-scripts=pnpm pnpm@latest --prefix $PNPM_SETUP_PREFIX --cache $PNPM_SETUP_CACHE >/dev/null 2>&1" # The tools installed with npm get their own npm cache, deleted with the tool # below, like npm's. Otherwise the timed install is served from ~/.npm, which # the prepare's own install of the same tool just filled. @@ -50,17 +62,17 @@ SETUP_INSTALL_UPM="npm install -g --cache $TOOL_SETUP_CACHE upm@latest >/dev/nul # --- PM uninstall commands (run at end of prepare to force re-install) --- SETUP_UNINSTALL_NPM="rm -rf $NPM_SETUP_PREFIX $NPM_SETUP_CACHE" -SETUP_UNINSTALL_YARN="" # corepack managed; cache purged below -SETUP_UNINSTALL_BERRY="" +SETUP_UNINSTALL_YARN="rm -rf $YARN_SETUP_PREFIX $YARN_SETUP_CACHE" +SETUP_UNINSTALL_BERRY="rm -rf $COREPACK_SETUP_PREFIX $COREPACK_SETUP_CACHE" SETUP_UNINSTALL_ZPM="rm -rf \$HOME/.yarn/switch >/dev/null 2>&1 || true" -SETUP_UNINSTALL_PNPM="" +SETUP_UNINSTALL_PNPM="rm -rf $PNPM_SETUP_PREFIX $PNPM_SETUP_CACHE" SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_BUN="npm uninstall -g bun >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_AUBE="npm uninstall -g @endevco/aube >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_UPM="npm uninstall -g upm >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" -# Purge the isolated corepack home so corepack-managed PMs must re-download. +# Purge the isolated corepack home so berry must be downloaded again. COREPACK_PURGE="rm -rf $COREPACK_HOME" # --- CI commands (same as ci variation) --- @@ -68,10 +80,10 @@ BENCH_CI_NPM="$NPM_SETUP_PREFIX/bin/npm ci --no-audit --no-fund --ignore-scripts if [ "$BENCH_FIXTURE" = "large" ]; then BENCH_CI_NPM="$BENCH_CI_NPM --legacy-peer-deps" fi -BENCH_CI_YARN="corepack yarn@1 install --frozen-lockfile --ignore-scripts --silent" -BENCH_CI_BERRY="corepack yarn@latest install --immutable" -BENCH_CI_ZPM="yarn install --immutable --silent" -BENCH_CI_PNPM="corepack pnpm@latest install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_YARN="$YARN_SETUP_PREFIX/bin/yarn install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_BERRY="$COREPACK_SETUP_PREFIX/bin/yarn install --immutable" +BENCH_CI_ZPM="$BENCH_ZPM_BIN install --immutable --silent" +BENCH_CI_PNPM="$PNPM_SETUP_PREFIX/bin/pnpm install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_VLT="vlt ci --view=silent" BENCH_CI_BUN="bun install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_DENO="deno install --frozen --quiet" @@ -93,10 +105,10 @@ setup_and_ci() { } BENCH_COMBINED_NPM="$(setup_and_ci "$SETUP_INSTALL_NPM" "$BENCH_CI_NPM")" -BENCH_COMBINED_YARN="$(setup_and_ci "" "$BENCH_CI_YARN")" # corepack download is part of the yarn invocation -BENCH_COMBINED_BERRY="$(setup_and_ci "" "$BENCH_CI_BERRY")" +BENCH_COMBINED_YARN="$(setup_and_ci "$SETUP_INSTALL_YARN" "$BENCH_CI_YARN")" +BENCH_COMBINED_BERRY="$(setup_and_ci "$SETUP_INSTALL_BERRY" "$BENCH_CI_BERRY")" # + corepack's download of yarn BENCH_COMBINED_ZPM="$(setup_and_ci "$SETUP_INSTALL_ZPM" "$BENCH_CI_ZPM")" -BENCH_COMBINED_PNPM="$(setup_and_ci "" "$BENCH_CI_PNPM")" +BENCH_COMBINED_PNPM="$(setup_and_ci "$SETUP_INSTALL_PNPM" "$BENCH_CI_PNPM")" BENCH_COMBINED_VLT="$(setup_and_ci "$SETUP_INSTALL_VLT" "$BENCH_CI_VLT")" BENCH_COMBINED_BUN="$(setup_and_ci "$SETUP_INSTALL_BUN" "$BENCH_CI_BUN")" BENCH_COMBINED_DENO="$(setup_and_ci "$SETUP_INSTALL_DENO" "$BENCH_CI_DENO")" @@ -144,7 +156,7 @@ ci_setup_prepare() { local result="sleep 1; $BENCH_CLEAN_PRE" - # Install PM binary if needed (for non-corepack PMs) + # Install PM binary if [ -n "$pm_install" ]; then result="$result; $pm_install" fi @@ -181,10 +193,10 @@ if [ "$BENCH_FIXTURE" = "large" ]; then fi BENCH_PREPARE_NPM="$(ci_setup_prepare "$SETUP_INSTALL_NPM" "$BENCH_SETUP_NPM" "$_INSTALL_NPM" "$SETUP_UNINSTALL_NPM" "")" -BENCH_PREPARE_YARN="$(ci_setup_prepare "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "corepack yarn@1 install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "$COREPACK_PURGE")" -BENCH_PREPARE_BERRY="$(ci_setup_prepare "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "corepack yarn@latest install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" -BENCH_PREPARE_ZPM="$(ci_setup_prepare "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "yarn install --silent" "$SETUP_UNINSTALL_ZPM" "")" -BENCH_PREPARE_PNPM="$(ci_setup_prepare "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "corepack pnpm@latest install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "$COREPACK_PURGE")" +BENCH_PREPARE_YARN="$(ci_setup_prepare "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "$YARN_SETUP_PREFIX/bin/yarn install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "")" +BENCH_PREPARE_BERRY="$(ci_setup_prepare "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "$COREPACK_SETUP_PREFIX/bin/yarn install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" +BENCH_PREPARE_ZPM="$(ci_setup_prepare "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "$BENCH_ZPM_BIN install --silent" "$SETUP_UNINSTALL_ZPM" "")" +BENCH_PREPARE_PNPM="$(ci_setup_prepare "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "$PNPM_SETUP_PREFIX/bin/pnpm install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "")" BENCH_PREPARE_VLT="$(ci_setup_prepare "$SETUP_INSTALL_VLT" "$BENCH_SETUP_VLT" "vlt install --view=silent" "$SETUP_UNINSTALL_VLT" "")" BENCH_PREPARE_BUN="$(ci_setup_prepare "$SETUP_INSTALL_BUN" "$BENCH_SETUP_BUN" "bun install --ignore-scripts --silent" "$SETUP_UNINSTALL_BUN" "")" BENCH_PREPARE_DENO="$(ci_setup_prepare "$SETUP_INSTALL_DENO" "$BENCH_SETUP_DENO" "deno install --quiet" "$SETUP_UNINSTALL_DENO" "")" @@ -226,8 +238,8 @@ collect_package_count # Unlike other variations that use a single BENCH_PREPARE_BASE for all PMs, # ci+setup needs per-PM prepare commands because each PM's prepare includes # PM-specific uninstall/cache-purge steps. Using a single base (e.g. npm's -# prepare) would leave the wrong lockfile and skip corepack purges for -# non-npm PMs, producing inaccurate process counts. +# prepare) would leave the wrong lockfile and skip the uninstall/purge steps +# for non-npm PMs, producing inaccurate process counts. collect_process_count_per_pm() { if ! command -v strace &>/dev/null; then echo "Warning: strace not available, skipping process count collection" diff --git a/suite/scripts/variations/ci.sh b/suite/scripts/variations/ci.sh index ff1f7ddba17..07b5dfbc62e 100644 --- a/suite/scripts/variations/ci.sh +++ b/suite/scripts/variations/ci.sh @@ -19,10 +19,10 @@ BENCH_CI_NPM="npm ci --no-audit --no-fund --ignore-scripts --silent" if [ "$BENCH_FIXTURE" = "large" ]; then BENCH_CI_NPM="$BENCH_CI_NPM --legacy-peer-deps" fi -BENCH_CI_YARN="corepack yarn@1 install --frozen-lockfile --ignore-scripts --silent" -BENCH_CI_BERRY="corepack yarn@latest install --immutable" -BENCH_CI_ZPM="yarn install --immutable --silent" -BENCH_CI_PNPM="corepack pnpm@latest install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_YARN="$BENCH_YARN_BIN install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_BERRY="$BENCH_BERRY_BIN install --immutable" +BENCH_CI_ZPM="$BENCH_ZPM_BIN install --immutable --silent" +BENCH_CI_PNPM="pnpm install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_VLT="vlt ci --view=silent" BENCH_CI_BUN="bun install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_DENO="deno install --frozen --quiet" @@ -85,10 +85,10 @@ if [ "$BENCH_FIXTURE" = "large" ]; then fi BENCH_PREPARE_NPM="$(ci_prepare "$BENCH_SETUP_NPM" "$_INSTALL_NPM")" -BENCH_PREPARE_YARN="$(ci_prepare "$BENCH_SETUP_YARN" "corepack yarn@1 install --ignore-scripts --silent")" -BENCH_PREPARE_BERRY="$(ci_prepare "$BENCH_SETUP_BERRY" "corepack yarn@latest install")" -BENCH_PREPARE_ZPM="$(ci_prepare "$BENCH_SETUP_ZPM" "yarn install --silent")" -BENCH_PREPARE_PNPM="$(ci_prepare "$BENCH_SETUP_PNPM" "corepack pnpm@latest install --ignore-scripts --silent")" +BENCH_PREPARE_YARN="$(ci_prepare "$BENCH_SETUP_YARN" "$BENCH_YARN_BIN install --ignore-scripts --silent")" +BENCH_PREPARE_BERRY="$(ci_prepare "$BENCH_SETUP_BERRY" "$BENCH_BERRY_BIN install")" +BENCH_PREPARE_ZPM="$(ci_prepare "$BENCH_SETUP_ZPM" "$BENCH_ZPM_BIN install --silent")" +BENCH_PREPARE_PNPM="$(ci_prepare "$BENCH_SETUP_PNPM" "pnpm install --ignore-scripts --silent")" BENCH_PREPARE_VLT="$(ci_prepare "$BENCH_SETUP_VLT" "vlt install --view=silent")" BENCH_PREPARE_BUN="$(ci_prepare "$BENCH_SETUP_BUN" "bun install --ignore-scripts --silent")" BENCH_PREPARE_DENO="$(ci_prepare "$BENCH_SETUP_DENO" "deno install --quiet")" diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index 36577df63ac..bbfa7b0e190 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -56,6 +56,12 @@ for pm in npm yarn berry zpm pnpm vlt bun deno aube upm nx turbo vp node; do fi done BENCH_OUTPUT_FOLDER="$BENCH_RESULTS/$BENCH_FIXTURE/$BENCH_VARIATION" +# Where yarn classic, berry and zpm live (BENCH_YARN_BIN, BENCH_BERRY_BIN, ...) +source "$BENCH_SCRIPTS/pm-bins.sh" +if [ -n "$BENCH_INCLUDE_BERRY" ] && [ -z "$BENCH_BERRY_VERSION" ]; then + echo "Error: no berry version in $BENCH_BERRY_VERSION_FILE; run scripts/setup.sh first" + exit 1 +fi BENCH_COMMAND_YARN_MODERN_CONFIG=$(cat < .yarnrc.yml" -BENCH_SETUP_ZPM="touch yarn.lock; echo \"$BENCH_COMMAND_YARN_MODERN_CONFIG\" > .yarnrc.yml; { echo '[zpm prepare]'; echo 'cwd:'; pwd; echo 'package.json:'; ls -la package.json || true; echo 'yarn path:'; command -v yarn || true; echo 'yarn version:'; yarn -v || true; echo 'canary version:'; echo \"$BENCH_ZPM_VERSION\"; echo 'packageManager (before):'; npm pkg get packageManager || true; echo 'set packageManager=yarn@'"$BENCH_ZPM_VERSION"':' ; npm pkg set packageManager=\"yarn@$BENCH_ZPM_VERSION\" || true; echo 'packageManager (after):'; npm pkg get packageManager || true; } >> $BENCH_OUTPUT_FOLDER/zpm-prepare.log 2>&1" +# The packageManager pin is how a Yarn 4 project selects its version through +# Corepack (what `yarn set version` writes). +BENCH_SETUP_BERRY="touch yarn.lock; echo \"$BENCH_COMMAND_YARN_MODERN_CONFIG\" > .yarnrc.yml; npm pkg set packageManager=yarn@$BENCH_BERRY_VERSION" +BENCH_SETUP_ZPM="touch yarn.lock; echo \"$BENCH_COMMAND_YARN_MODERN_CONFIG\" > .yarnrc.yml; { echo '[zpm prepare]'; echo 'cwd:'; pwd; echo 'package.json:'; ls -la package.json || true; echo 'yarn path:'; echo $BENCH_ZPM_BIN; echo 'yarn version:'; $BENCH_ZPM_BIN -v || true; echo 'canary version:'; echo \"$BENCH_ZPM_VERSION\"; echo 'packageManager (before):'; npm pkg get packageManager || true; echo 'set packageManager=yarn@'"$BENCH_ZPM_VERSION"':' ; npm pkg set packageManager=\"yarn@$BENCH_ZPM_VERSION\" || true; echo 'packageManager (after):'; npm pkg get packageManager || true; } >> $BENCH_OUTPUT_FOLDER/zpm-prepare.log 2>&1" BENCH_SETUP_PNPM="npm pkg delete packageManager >/dev/null 2>&1 || true" BENCH_SETUP_VLT="node $BENCH_SCRIPTS/add-workspace-protocol.js . >> $BENCH_OUTPUT_FOLDER/vlt-prepare.log 2>&1" @@ -107,10 +116,10 @@ BENCH_INSTALL_NPM="npm install --no-audit --no-fund --ignore-scripts --silent" if [ "$BENCH_FIXTURE" = "large" ]; then BENCH_INSTALL_NPM="$BENCH_INSTALL_NPM --legacy-peer-deps" fi -BENCH_INSTALL_YARN="corepack yarn@1 install --ignore-scripts --silent" -BENCH_INSTALL_BERRY="corepack yarn@latest install" -BENCH_INSTALL_ZPM="yarn install --silent" -BENCH_INSTALL_PNPM="corepack pnpm@latest install --ignore-scripts --silent" +BENCH_INSTALL_YARN="$BENCH_YARN_BIN install --ignore-scripts --silent" +BENCH_INSTALL_BERRY="$BENCH_BERRY_BIN install" +BENCH_INSTALL_ZPM="$BENCH_ZPM_BIN install --silent" +BENCH_INSTALL_PNPM="pnpm install --ignore-scripts --silent" # vlt uses the npm registry alias configured by each fixture's vlt.json. BENCH_INSTALL_VLT="vlt install --view=silent" BENCH_INSTALL_BUN="bun install --ignore-scripts --silent" diff --git a/suite/scripts/variations/lockfile+node_modules.sh b/suite/scripts/variations/lockfile+node_modules.sh index 8a5cb3c5095..c0dbe4d0992 100644 --- a/suite/scripts/variations/lockfile+node_modules.sh +++ b/suite/scripts/variations/lockfile+node_modules.sh @@ -5,9 +5,8 @@ set -Eeuxo pipefail source "$1/variations/common.sh" # Prepare command base for each run. -# clean_package_manager_field must run AFTER clean_all_cache: the corepack -# yarn cache cleans re-pin devEngines.packageManager (yarn) into package.json, -# which makes npm and pnpm refuse to run entirely (see clean_all). +# clean_package_manager_field runs last, so no packageManager pin from the +# previous run (berry, zpm) is left for the next PM (see clean_all). BENCH_PREPARE_BASE="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all_cache clean_package_manager_files clean_package_manager_field" # Run the benchmark suite diff --git a/suite/scripts/variations/lockfile.sh b/suite/scripts/variations/lockfile.sh index b6013c15713..527b64b0fd2 100644 --- a/suite/scripts/variations/lockfile.sh +++ b/suite/scripts/variations/lockfile.sh @@ -5,9 +5,8 @@ set -Eeuxo pipefail source "$1/variations/common.sh" # Prepare command base for each run. -# clean_package_manager_field must run AFTER clean_all_cache: the corepack -# yarn cache cleans re-pin devEngines.packageManager (yarn) into package.json, -# which makes npm and pnpm refuse to run entirely (see clean_all). +# clean_package_manager_field runs last, so no packageManager pin from the +# previous run (berry, zpm) is left for the next PM (see clean_all). BENCH_PREPARE_BASE="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all_cache clean_node_modules clean_package_manager_files clean_package_manager_field" # Run the benchmark suite diff --git a/suite/scripts/variations/node_modules.sh b/suite/scripts/variations/node_modules.sh index e545d3f7e7b..42096d7b484 100644 --- a/suite/scripts/variations/node_modules.sh +++ b/suite/scripts/variations/node_modules.sh @@ -5,9 +5,8 @@ set -Eeuxo pipefail source "$1/variations/common.sh" # Prepare command base for each run. -# clean_package_manager_field must run AFTER clean_all_cache: the corepack -# yarn cache cleans re-pin devEngines.packageManager (yarn) into package.json, -# which makes npm and pnpm refuse to run entirely (see clean_all). +# clean_package_manager_field runs last, so no packageManager pin from the +# previous run (berry, zpm) is left for the next PM (see clean_all). BENCH_PREPARE_BASE="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all_cache clean_lockfiles clean_package_manager_files clean_package_manager_field" # Run the benchmark suite diff --git a/suite/scripts/variations/registry-lockfile.sh b/suite/scripts/variations/registry-lockfile.sh index 7bbd1a31489..f72a0ed41f0 100644 --- a/suite/scripts/variations/registry-lockfile.sh +++ b/suite/scripts/variations/registry-lockfile.sh @@ -5,10 +5,10 @@ set -Eeuxo pipefail source "$1/registry/common.sh" # Prepare command base for each run: clean cache, node_modules, pm files, but keep lockfile. -# clean_package_manager_field runs AFTER clean_all_cache: the corepack-based yarn -# cache cleans re-pin a devEngines.packageManager entry into package.json, which -# makes the subsequent `npm config set` prepare step fail with EBADDEVENGINES. -# (It only touches package.json, never the lockfile, so the lockfile is preserved.) +# clean_package_manager_field runs last, so package.json is neutral for the +# `npm config set` prepare step (npm fails with EBADDEVENGINES on a mismatched +# devEngines.packageManager). It only touches package.json, never the +# lockfile, so the lockfile is preserved. BENCH_PREPARE_BASE="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all_cache clean_package_manager_field clean_node_modules clean_package_manager_files clean_npmrc" # Run the benchmark suite diff --git a/suite/scripts/variations/run.sh b/suite/scripts/variations/run.sh index 484e0f6215d..b615ff55afc 100644 --- a/suite/scripts/variations/run.sh +++ b/suite/scripts/variations/run.sh @@ -36,10 +36,10 @@ BENCH_INSTALL_PREPARE_VP="$(prepend_setup "$BENCH_INSTALL_COMMAND_VP" "$BENCH_SE # Run defines its own command scripts to actually run a test # script in the fixture directory instead of installing packages. BENCH_COMMAND_NPM="timeout $BENCH_TIMEOUT npm run test > $BENCH_OUTPUT_FOLDER/npm-run-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT corepack yarn@1 run test > $BENCH_OUTPUT_FOLDER/yarn-run-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT corepack yarn@latest run test > $BENCH_OUTPUT_FOLDER/berry-run-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT yarn run test > $BENCH_OUTPUT_FOLDER/zpm-run-output-\${HYPERFINE_ITERATION}.log 2>&1" -BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT corepack pnpm@latest run test > $BENCH_OUTPUT_FOLDER/pnpm-run-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT $BENCH_YARN_BIN run test > $BENCH_OUTPUT_FOLDER/yarn-run-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT $BENCH_BERRY_BIN run test > $BENCH_OUTPUT_FOLDER/berry-run-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT $BENCH_ZPM_BIN run test > $BENCH_OUTPUT_FOLDER/zpm-run-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT pnpm run test > $BENCH_OUTPUT_FOLDER/pnpm-run-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_VLT="timeout $BENCH_TIMEOUT vlt run test --view=human > $BENCH_OUTPUT_FOLDER/vlt-run-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_BUN="timeout $BENCH_TIMEOUT bun run test > $BENCH_OUTPUT_FOLDER/bun-run-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_DENO="timeout $BENCH_TIMEOUT deno run test > $BENCH_OUTPUT_FOLDER/deno-run-output-\${HYPERFINE_ITERATION}.log 2>&1" From 27021a4777945028318b8256da4a6fb7eddfd8b6 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 01:23:44 +0000 Subject: [PATCH 2/3] keep yarn 1's cache clean flags as they were Co-Authored-By: Claude Opus 5.5 --- suite/scripts/clean-helpers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suite/scripts/clean-helpers.sh b/suite/scripts/clean-helpers.sh index 29dab6e2da2..d5255c9cdbe 100644 --- a/suite/scripts/clean-helpers.sh +++ b/suite/scripts/clean-helpers.sh @@ -21,7 +21,7 @@ clean_npm_cache() { # Function to safely clean yarn cache clean_yarn_cache() { if [ -x "$BENCH_YARN_BIN" ]; then - "$BENCH_YARN_BIN" cache clean --silent || true + "$BENCH_YARN_BIN" cache clean --all --silent || true fi } From 3f94699869b2276391eca4a5d217e5b0954bc718 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 02:45:18 +0000 Subject: [PATCH 3/3] fix(clean): remove yarn 1's cache directly, since it won't run under a berry/zpm pin clean_all empties caches before it strips packageManager, and yarn 1 refuses to run in a project pinned to another yarn, so `yarn cache clean` silently did nothing after a berry or zpm run. Also add tests for that and for package-count's packageManager check, and note in the README that vlt must be on PATH and that setup.sh wants the npm-installed pnpm first. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 ++++++ suite/scripts/clean-helpers.sh | 4 +++ suite/scripts/clean-helpers.test.js | 41 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/README.md b/README.md index 42524fd1364..198e973b6ac 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ This suite also tests the performance of basic script execution (ex. `npm run fo [`docs/METHODOLOGY.md`](docs/METHODOLOGY.md)). Yarn 1, Yarn 4 and Yarn 6 are all called `yarn`, so yarn 1 and Corepack go into their own prefixes under `~/.bench-pms` (`suite/scripts/pm-bins.sh`): + ```bash npm install -g npm@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest # --allow-scripts lets pnpm's postinstall install its native binary @@ -169,6 +170,13 @@ This suite also tests the performance of basic script execution (ex. `npm run fo # Yarn 6 (zpm), through Yarn Switch curl -sS https://repo.yarnpkg.com/install | bash ``` + + `vlt` must be on `PATH` even when you don't benchmark it: the cleanup + uses it to remove the `packageManager` pins the berry and zpm runs set, + and yarn 1 and pnpm refuse to run while one is left. `setup.sh` also + requires the npm-installed pnpm to be the first `pnpm` on `PATH`, so it + fails if, for example, pnpm from the standalone installer comes first. + - 1.4 No directory setup is needed. `./suite/bench` creates its output tree at `suite/results/`, which is the only results tree there is. diff --git a/suite/scripts/clean-helpers.sh b/suite/scripts/clean-helpers.sh index d5255c9cdbe..9a60eeeb6fe 100644 --- a/suite/scripts/clean-helpers.sh +++ b/suite/scripts/clean-helpers.sh @@ -23,6 +23,10 @@ clean_yarn_cache() { if [ -x "$BENCH_YARN_BIN" ]; then "$BENCH_YARN_BIN" cache clean --all --silent || true fi + # Yarn 1 refuses to run in a project whose packageManager names another + # yarn (a berry or zpm pin from the previous run), so also remove its cache + # directly: ~/.cache/yarn, or under $XDG_CACHE_HOME when that is set. + safe_remove "${XDG_CACHE_HOME:-$HOME/.cache}/yarn" } clean_berry_cache() { diff --git a/suite/scripts/clean-helpers.test.js b/suite/scripts/clean-helpers.test.js index b1cf5742582..1d8fc9dfbb5 100644 --- a/suite/scripts/clean-helpers.test.js +++ b/suite/scripts/clean-helpers.test.js @@ -257,6 +257,15 @@ const removedBy = (command, env = {}) => { } }; +test("clean_yarn_cache removes Yarn 1's cache, with and without XDG_CACHE_HOME", () => { + assert.deepEqual(removedBy("clean_yarn_cache", { XDG_CACHE_HOME: "" }).slice(-1), [ + "/home/runner/.cache/yarn", + ]); + assert.deepEqual(removedBy("clean_yarn_cache", { XDG_CACHE_HOME: "/xdg/cache" }).slice(-1), [ + "/xdg/cache/yarn", + ]); +}); + test("clean_zpm_cache removes ~/.yarn/zpm", () => { assert.deepEqual(removedBy("clean_zpm_cache"), ["/home/runner/.yarn/zpm"]); }); @@ -336,3 +345,35 @@ test("package-count.sh files a Yarn 1 lockfile under yarn even with a .yarnrc.ym fs.rmSync(tempDir, { recursive: true, force: true }); } }); + +test("package-count.sh tells zpm from berry by the packageManager pin", () => { + for (const [packageManager, expected] of [ + ["yarn@6.0.0-rc.20", "zpm-count.txt"], + ["yarn@4.18.1", "berry-count.txt"], + ]) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "count-test-")); + try { + const project = path.join(tempDir, "project"); + const out = path.join(tempDir, "out"); + fs.mkdirSync(path.join(project, "node_modules", "a"), { recursive: true }); + fs.writeFileSync( + path.join(project, "node_modules", "a", "package.json"), + '{"name":"a","version":"1.0.0"}', + ); + fs.writeFileSync( + path.join(project, "package.json"), + JSON.stringify({ name: "x", dependencies: { a: "1.0.0" }, packageManager }), + ); + fs.writeFileSync(path.join(project, ".yarnrc.yml"), "nodeLinker: node-modules\n"); + fs.writeFileSync(path.join(project, "yarn.lock"), "__metadata:\n version: 8\n"); + const result = spawnSync("bash", [path.join(__dirname, "package-count.sh"), out], { + cwd: project, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(fs.readdirSync(out), [expected]); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + } +});