From a5490a3f00b4031672698bb099ab7d39c3b2f7fd Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 03:49:56 +0000 Subject: [PATCH 1/3] fix(process-count): lockfile variations count each tool from its own lockfile The four lockfile variations (lockfile, cache+lockfile, lockfile+node_modules, cache+lockfile+node_modules) counted every tool with one shared prepare that didn't clean lockfiles. Each counted install found the lockfiles earlier tools left and not its own, and upm, which refuses more than one lockfile (ELOCK), failed and had no count. set_lockfile_prepares (common.sh) gives each tool the prepare its timed runs get: clean_all, one install that writes its lockfile, then the variation's prepare. The variations count with collect_process_count_per_pm, as ci does. Fixes #214 Co-Authored-By: Claude Opus 5.5 --- docs/METHODOLOGY.md | 4 +++- .../variations/cache+lockfile+node_modules.sh | 5 ++++- suite/scripts/variations/cache+lockfile.sh | 5 ++++- suite/scripts/variations/common.sh | 18 ++++++++++++++++++ .../variations/lockfile+node_modules.sh | 5 ++++- suite/scripts/variations/lockfile.sh | 5 ++++- 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index dc69913b0b..dbc3b836f2 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -156,7 +156,9 @@ once. Earlier counts (issue #209) counted every `execve` line, including the wrapper, failed lookups and the interpreter, and `ci` counted most tools -against npm's lockfile. Don't compare them with later counts. +against npm's lockfile. Until issue #214, the four lockfile variations counted +each tool against the lockfiles earlier tools left, and upm, which refuses more +than one lockfile, had no count. Don't compare them with later counts. ## Variations diff --git a/suite/scripts/variations/cache+lockfile+node_modules.sh b/suite/scripts/variations/cache+lockfile+node_modules.sh index b6724138dc..348e947022 100644 --- a/suite/scripts/variations/cache+lockfile+node_modules.sh +++ b/suite/scripts/variations/cache+lockfile+node_modules.sh @@ -40,4 +40,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_lockfile_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/cache+lockfile.sh b/suite/scripts/variations/cache+lockfile.sh index a7222dd91f..33f36ed8ec 100644 --- a/suite/scripts/variations/cache+lockfile.sh +++ b/suite/scripts/variations/cache+lockfile.sh @@ -40,4 +40,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_lockfile_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index def5ad2a75..72693dc5f5 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -296,3 +296,21 @@ collect_process_count_per_pm() { node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all } + +# For variations that start from the tool's own lockfile (lockfile, +# cache+lockfile, lockfile+node_modules, cache+lockfile+node_modules): sets +# each BENCH_PREPARE_ to what the timed runs do before a measured run. +# That is clean_all (hyperfine --setup), one install that writes the tool's +# lockfile (a warmup run), then the variation's prepare again. So the counted +# install finds its own lockfile and no other tool's. BENCH_PREPARE_BASE must +# be set before calling this. +set_lockfile_prepares() { + local pm P setup_var install_var prepare + for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do + P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') + setup_var="BENCH_SETUP_$P" + install_var="BENCH_INSTALL_$P" + prepare="$(append_setup "$BENCH_PREPARE_BASE" "${!setup_var}")" + printf -v "BENCH_PREPARE_$P" '%s' "bash $BENCH_SCRIPTS/clean-helpers.sh clean_all; $prepare; ${!install_var} >/dev/null 2>&1 || true; bash $BENCH_SCRIPTS/clean-helpers.sh wait_for_vlt_background; $prepare" + done +} diff --git a/suite/scripts/variations/lockfile+node_modules.sh b/suite/scripts/variations/lockfile+node_modules.sh index c0dbe4d099..91fe63ce2c 100644 --- a/suite/scripts/variations/lockfile+node_modules.sh +++ b/suite/scripts/variations/lockfile+node_modules.sh @@ -42,4 +42,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_lockfile_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/lockfile.sh b/suite/scripts/variations/lockfile.sh index 527b64b0fd..94752f7b1e 100644 --- a/suite/scripts/variations/lockfile.sh +++ b/suite/scripts/variations/lockfile.sh @@ -42,4 +42,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_lockfile_prepares +collect_process_count_per_pm From 278cec42d57b59fd6624ff0ea93fc067afdbabe2 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 04:34:24 +0000 Subject: [PATCH 2/3] fix(process-count): node_modules and cache variations count each tool from its own state node_modules, cache and cache+node_modules had the same problem as the lockfile variations: one shared prepare, so each counted install started from the node_modules and cache earlier tools left, and the cache variations never had the tool's own cache warm. Rename set_lockfile_prepares to set_per_pm_prepares and use it in these three too. clean keeps the shared pass: its prepare resets everything. Co-Authored-By: Claude Opus 5.5 --- docs/METHODOLOGY.md | 8 ++++--- .../variations/cache+lockfile+node_modules.sh | 2 +- suite/scripts/variations/cache+lockfile.sh | 2 +- .../scripts/variations/cache+node_modules.sh | 5 +++- suite/scripts/variations/cache.sh | 4 +++- suite/scripts/variations/common.sh | 23 ++++++++++--------- .../variations/lockfile+node_modules.sh | 2 +- suite/scripts/variations/lockfile.sh | 2 +- suite/scripts/variations/node_modules.sh | 4 +++- 9 files changed, 31 insertions(+), 21 deletions(-) diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index dbc3b836f2..93b2709f2a 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -156,9 +156,11 @@ once. Earlier counts (issue #209) counted every `execve` line, including the wrapper, failed lookups and the interpreter, and `ci` counted most tools -against npm's lockfile. Until issue #214, the four lockfile variations counted -each tool against the lockfiles earlier tools left, and upm, which refuses more -than one lockfile, had no count. Don't compare them with later counts. +against npm's lockfile. Until issue #214, the lockfile, `node_modules` and +cache variations counted each tool from what earlier tools left, not its own +lockfile, `node_modules` or cache, and upm, which refuses more than one +lockfile, had no count in the lockfile variations. Don't compare them with +later counts. ## Variations diff --git a/suite/scripts/variations/cache+lockfile+node_modules.sh b/suite/scripts/variations/cache+lockfile+node_modules.sh index 348e947022..38d6bbdb37 100644 --- a/suite/scripts/variations/cache+lockfile+node_modules.sh +++ b/suite/scripts/variations/cache+lockfile+node_modules.sh @@ -42,5 +42,5 @@ hyperfine --ignore-failure \ collect_package_count # Each tool starts from its own lockfile, so each gets its own prepare. -set_lockfile_prepares +set_per_pm_prepares collect_process_count_per_pm diff --git a/suite/scripts/variations/cache+lockfile.sh b/suite/scripts/variations/cache+lockfile.sh index 33f36ed8ec..49c408882c 100644 --- a/suite/scripts/variations/cache+lockfile.sh +++ b/suite/scripts/variations/cache+lockfile.sh @@ -42,5 +42,5 @@ hyperfine --ignore-failure \ collect_package_count # Each tool starts from its own lockfile, so each gets its own prepare. -set_lockfile_prepares +set_per_pm_prepares collect_process_count_per_pm diff --git a/suite/scripts/variations/cache+node_modules.sh b/suite/scripts/variations/cache+node_modules.sh index 48dee01e2c..7addb6bd9d 100644 --- a/suite/scripts/variations/cache+node_modules.sh +++ b/suite/scripts/variations/cache+node_modules.sh @@ -40,4 +40,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count +# Each tool starts from its own cache and node_modules, so each gets its own +# prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/cache.sh b/suite/scripts/variations/cache.sh index b2fc9f855f..789722e42a 100644 --- a/suite/scripts/variations/cache.sh +++ b/suite/scripts/variations/cache.sh @@ -40,4 +40,6 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count +# Each tool starts from its own cache, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index 72693dc5f5..1b56c88c4d 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -194,9 +194,11 @@ collect_package_count() { } # Function to collect spawned process counts via strace. -# Runs a single strace'd install for each included package manager under the -# same state as the benchmark variation. BENCH_PREPARE_BASE must be set by the -# calling variation script before invoking this function. +# Runs a single strace'd install for each included package manager, one after +# another, with one shared prepare. Only right when that prepare resets +# everything (clean); otherwise a tool starts from what the previous one left, +# so use set_per_pm_prepares and collect_process_count_per_pm. BENCH_PREPARE_BASE +# must be set by the calling variation script before invoking this function. collect_process_count() { if ! command -v strace &>/dev/null; then echo "Warning: strace not available, skipping process count collection" @@ -297,14 +299,13 @@ collect_process_count_per_pm() { bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all } -# For variations that start from the tool's own lockfile (lockfile, -# cache+lockfile, lockfile+node_modules, cache+lockfile+node_modules): sets -# each BENCH_PREPARE_ to what the timed runs do before a measured run. -# That is clean_all (hyperfine --setup), one install that writes the tool's -# lockfile (a warmup run), then the variation's prepare again. So the counted -# install finds its own lockfile and no other tool's. BENCH_PREPARE_BASE must -# be set before calling this. -set_lockfile_prepares() { +# For variations whose starting state is what the tool itself left behind: +# its own lockfile, node_modules or cache. Sets each BENCH_PREPARE_ to what +# the timed runs do before a measured run: clean_all (hyperfine --setup), one +# install that leaves that state (a warmup run), then the variation's prepare +# again. So the counted install starts from its own state and nothing another +# tool left. BENCH_PREPARE_BASE must be set before calling this. +set_per_pm_prepares() { local pm P setup_var install_var prepare for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') diff --git a/suite/scripts/variations/lockfile+node_modules.sh b/suite/scripts/variations/lockfile+node_modules.sh index 91fe63ce2c..1fdebe623b 100644 --- a/suite/scripts/variations/lockfile+node_modules.sh +++ b/suite/scripts/variations/lockfile+node_modules.sh @@ -44,5 +44,5 @@ hyperfine --ignore-failure \ collect_package_count # Each tool starts from its own lockfile, so each gets its own prepare. -set_lockfile_prepares +set_per_pm_prepares collect_process_count_per_pm diff --git a/suite/scripts/variations/lockfile.sh b/suite/scripts/variations/lockfile.sh index 94752f7b1e..847f2a6e64 100644 --- a/suite/scripts/variations/lockfile.sh +++ b/suite/scripts/variations/lockfile.sh @@ -44,5 +44,5 @@ hyperfine --ignore-failure \ collect_package_count # Each tool starts from its own lockfile, so each gets its own prepare. -set_lockfile_prepares +set_per_pm_prepares collect_process_count_per_pm diff --git a/suite/scripts/variations/node_modules.sh b/suite/scripts/variations/node_modules.sh index 42096d7b48..cdaa84328e 100644 --- a/suite/scripts/variations/node_modules.sh +++ b/suite/scripts/variations/node_modules.sh @@ -42,4 +42,6 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count +# Each tool starts from its own node_modules, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm From 04a4ee31b8b2382a35faece3f29563f6343bf3fa Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 05:46:30 +0000 Subject: [PATCH 3/3] fix(process-count): time out the warmup install, skip the count when it fails From review on #218: - The per-tool warmup install in set_per_pm_prepares runs under `timeout $BENCH_TIMEOUT`, like every other install, so a stuck registry can't hang the job. ci_prepare (ci.sh) and ci_setup_prepare (ci+setup.sh) get the same timeout; their warmups stay `|| true` because hyperfine runs them too. - A failed or timed-out warmup exits the prepare with its status, and process-count.sh now runs the prepare in a subshell and skips the count when it fails. Before, the count was recorded from the wrong state (e.g. a clean install counted as the lockfile count). - Comments: process-count.sh names all its callers; collect_process_count_per_pm lists the variations that use it. - METHODOLOGY: a failed setup install records nothing. Co-Authored-By: Claude Opus 5.5 --- docs/METHODOLOGY.md | 4 ++-- suite/scripts/process-count.sh | 19 ++++++++++++++----- suite/scripts/variations/ci+setup.sh | 2 +- suite/scripts/variations/ci.sh | 2 +- suite/scripts/variations/common.sh | 11 ++++++++--- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index 93b2709f2a..7476f61c65 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -151,8 +151,8 @@ once. background cache unpacking only when there is something to unpack: **2** in `clean`, **1** in `ci` on a warm cache. Its update-check and revalidate processes may add to that, depending on timing. -- If the command exits non-zero or runs past `BENCH_TIMEOUT`, nothing is - recorded for that tool. +- If the command, or the install that sets up its starting state, exits + non-zero or runs past `BENCH_TIMEOUT`, nothing is recorded for that tool. Earlier counts (issue #209) counted every `execve` line, including the wrapper, failed lookups and the interpreter, and `ci` counted most tools diff --git a/suite/scripts/process-count.sh b/suite/scripts/process-count.sh index 26c588116a..f6cefa8b52 100644 --- a/suite/scripts/process-count.sh +++ b/suite/scripts/process-count.sh @@ -8,12 +8,14 @@ # processes it started (see count-processes.js for what counts). The count is # appended to /-process-count.txt (one number per # line, matching the pattern used by package-count.sh). Nothing is recorded if -# the install fails: a failed command's processes aren't the ones the timed -# runs measure. +# the prepare or the install fails: a failed prepare leaves the wrong starting +# state, and a failed command's processes aren't the ones the timed runs +# measure. # # This script is meant to run OUTSIDE the timed benchmark (strace adds -# overhead). It is invoked by `collect_process_count` in common.sh after -# the hyperfine run completes. +# overhead). It is invoked after the hyperfine run completes by +# `collect_process_count` and `collect_process_count_per_pm` in common.sh, and +# by add-remove.sh. set -Eeuo pipefail @@ -50,11 +52,18 @@ fi # for vlt's background work before and after it, so it can't overlap the # counted install (see hyperfine-wait.sh). WAIT_CMD="bash $(dirname "$0")/clean-helpers.sh wait_for_vlt_background" +# The prepare runs in a subshell, so one that exits (set_per_pm_prepares does +# when its warmup install fails) ends only the prepare. if [ -n "$PREPARE_CMD" ]; then echo "[process-count] Running prepare for $PM_NAME..." $WAIT_CMD - eval "$PREPARE_CMD" || true + PREPARE_EXIT=0 + (eval "$PREPARE_CMD") || PREPARE_EXIT=$? $WAIT_CMD + if [ "$PREPARE_EXIT" -ne 0 ]; then + echo "Warning: prepare exited with $PREPARE_EXIT for $PM_NAME, skipping process count" + exit 0 + fi fi # Run the install command under strace. The timeout wraps strace, not the diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index f2c76aa239..b5dd824549 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -167,7 +167,7 @@ ci_setup_prepare() { fi # Generate lockfile via regular install - result="$result; $install_cmd >/dev/null 2>&1 || true" + result="$result; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 || true" # Clean node_modules (ci starts fresh) result="$result; $BENCH_CLEAN_POST" diff --git a/suite/scripts/variations/ci.sh b/suite/scripts/variations/ci.sh index cb310ce520..aa236f090d 100644 --- a/suite/scripts/variations/ci.sh +++ b/suite/scripts/variations/ci.sh @@ -74,7 +74,7 @@ ci_prepare() { if [ -n "$setup_cmd" ]; then result="$result; $setup_cmd" fi - result="$result; $install_cmd >/dev/null 2>&1 || true; $BENCH_CLEAN_POST" + result="$result; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 || true; $BENCH_CLEAN_POST" echo "$result" } diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index 1b56c88c4d..26ed22914d 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -269,7 +269,8 @@ collect_process_count() { # Like collect_process_count, but each package manager gets its own prepare, # BENCH_PREPARE_, as in the timed runs. For variations whose prepare -# depends on the tool, e.g. `ci` needs the tool's own lockfile. +# depends on the tool: `ci` and `ci+setup`, and the lockfile, node_modules and +# cache variations (see set_per_pm_prepares). collect_process_count_per_pm() { if ! command -v strace &>/dev/null; then echo "Warning: strace not available, skipping process count collection" @@ -304,7 +305,11 @@ collect_process_count_per_pm() { # the timed runs do before a measured run: clean_all (hyperfine --setup), one # install that leaves that state (a warmup run), then the variation's prepare # again. So the counted install starts from its own state and nothing another -# tool left. BENCH_PREPARE_BASE must be set before calling this. +# tool left. If the warmup install fails or times out, the prepare exits with +# its status, so process-count.sh skips the count: without the warmup the tool +# wouldn't start from its own state. Only process-count.sh runs these prepares, +# in a subshell, never hyperfine. BENCH_PREPARE_BASE must be set before calling +# this. set_per_pm_prepares() { local pm P setup_var install_var prepare for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do @@ -312,6 +317,6 @@ set_per_pm_prepares() { setup_var="BENCH_SETUP_$P" install_var="BENCH_INSTALL_$P" prepare="$(append_setup "$BENCH_PREPARE_BASE" "${!setup_var}")" - printf -v "BENCH_PREPARE_$P" '%s' "bash $BENCH_SCRIPTS/clean-helpers.sh clean_all; $prepare; ${!install_var} >/dev/null 2>&1 || true; bash $BENCH_SCRIPTS/clean-helpers.sh wait_for_vlt_background; $prepare" + printf -v "BENCH_PREPARE_$P" '%s' "bash $BENCH_SCRIPTS/clean-helpers.sh clean_all; $prepare; timeout $BENCH_TIMEOUT ${!install_var} >/dev/null 2>&1 || exit \$?; bash $BENCH_SCRIPTS/clean-helpers.sh wait_for_vlt_background; $prepare" done }