diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index dc69913b0b..7476f61c65 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -151,12 +151,16 @@ once. background cache unpacking only when there is something to unpack: **2** in `clean`, **1** in `ci` on a warm cache. Its update-check and revalidate processes may add to that, depending on timing. -- If the command exits non-zero or runs past `BENCH_TIMEOUT`, nothing is - recorded for that tool. +- If the command, or the install that sets up its starting state, exits + non-zero or runs past `BENCH_TIMEOUT`, nothing is recorded for that tool. Earlier counts (issue #209) counted every `execve` line, including the wrapper, failed lookups and the interpreter, and `ci` counted most tools -against npm's lockfile. Don't compare them with later counts. +against npm's lockfile. Until issue #214, the lockfile, `node_modules` and +cache variations counted each tool from what earlier tools left, not its own +lockfile, `node_modules` or cache, and upm, which refuses more than one +lockfile, had no count in the lockfile variations. Don't compare them with +later counts. ## Variations diff --git a/suite/scripts/process-count.sh b/suite/scripts/process-count.sh index 26c588116a..f6cefa8b52 100644 --- a/suite/scripts/process-count.sh +++ b/suite/scripts/process-count.sh @@ -8,12 +8,14 @@ # processes it started (see count-processes.js for what counts). The count is # appended to /-process-count.txt (one number per # line, matching the pattern used by package-count.sh). Nothing is recorded if -# the install fails: a failed command's processes aren't the ones the timed -# runs measure. +# the prepare or the install fails: a failed prepare leaves the wrong starting +# state, and a failed command's processes aren't the ones the timed runs +# measure. # # This script is meant to run OUTSIDE the timed benchmark (strace adds -# overhead). It is invoked by `collect_process_count` in common.sh after -# the hyperfine run completes. +# overhead). It is invoked after the hyperfine run completes by +# `collect_process_count` and `collect_process_count_per_pm` in common.sh, and +# by add-remove.sh. set -Eeuo pipefail @@ -50,11 +52,18 @@ fi # for vlt's background work before and after it, so it can't overlap the # counted install (see hyperfine-wait.sh). WAIT_CMD="bash $(dirname "$0")/clean-helpers.sh wait_for_vlt_background" +# The prepare runs in a subshell, so one that exits (set_per_pm_prepares does +# when its warmup install fails) ends only the prepare. if [ -n "$PREPARE_CMD" ]; then echo "[process-count] Running prepare for $PM_NAME..." $WAIT_CMD - eval "$PREPARE_CMD" || true + PREPARE_EXIT=0 + (eval "$PREPARE_CMD") || PREPARE_EXIT=$? $WAIT_CMD + if [ "$PREPARE_EXIT" -ne 0 ]; then + echo "Warning: prepare exited with $PREPARE_EXIT for $PM_NAME, skipping process count" + exit 0 + fi fi # Run the install command under strace. The timeout wraps strace, not the diff --git a/suite/scripts/variations/cache+lockfile+node_modules.sh b/suite/scripts/variations/cache+lockfile+node_modules.sh index b6724138dc..38d6bbdb37 100644 --- a/suite/scripts/variations/cache+lockfile+node_modules.sh +++ b/suite/scripts/variations/cache+lockfile+node_modules.sh @@ -40,4 +40,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/cache+lockfile.sh b/suite/scripts/variations/cache+lockfile.sh index a7222dd91f..49c408882c 100644 --- a/suite/scripts/variations/cache+lockfile.sh +++ b/suite/scripts/variations/cache+lockfile.sh @@ -40,4 +40,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/cache+node_modules.sh b/suite/scripts/variations/cache+node_modules.sh index 48dee01e2c..7addb6bd9d 100644 --- a/suite/scripts/variations/cache+node_modules.sh +++ b/suite/scripts/variations/cache+node_modules.sh @@ -40,4 +40,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count +# Each tool starts from its own cache and node_modules, so each gets its own +# prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/cache.sh b/suite/scripts/variations/cache.sh index b2fc9f855f..789722e42a 100644 --- a/suite/scripts/variations/cache.sh +++ b/suite/scripts/variations/cache.sh @@ -40,4 +40,6 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count +# Each tool starts from its own cache, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index f2c76aa239..b5dd824549 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -167,7 +167,7 @@ ci_setup_prepare() { fi # Generate lockfile via regular install - result="$result; $install_cmd >/dev/null 2>&1 || true" + result="$result; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 || true" # Clean node_modules (ci starts fresh) result="$result; $BENCH_CLEAN_POST" diff --git a/suite/scripts/variations/ci.sh b/suite/scripts/variations/ci.sh index cb310ce520..aa236f090d 100644 --- a/suite/scripts/variations/ci.sh +++ b/suite/scripts/variations/ci.sh @@ -74,7 +74,7 @@ ci_prepare() { if [ -n "$setup_cmd" ]; then result="$result; $setup_cmd" fi - result="$result; $install_cmd >/dev/null 2>&1 || true; $BENCH_CLEAN_POST" + result="$result; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 || true; $BENCH_CLEAN_POST" echo "$result" } diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index def5ad2a75..26ed22914d 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -194,9 +194,11 @@ collect_package_count() { } # Function to collect spawned process counts via strace. -# Runs a single strace'd install for each included package manager under the -# same state as the benchmark variation. BENCH_PREPARE_BASE must be set by the -# calling variation script before invoking this function. +# Runs a single strace'd install for each included package manager, one after +# another, with one shared prepare. Only right when that prepare resets +# everything (clean); otherwise a tool starts from what the previous one left, +# so use set_per_pm_prepares and collect_process_count_per_pm. BENCH_PREPARE_BASE +# must be set by the calling variation script before invoking this function. collect_process_count() { if ! command -v strace &>/dev/null; then echo "Warning: strace not available, skipping process count collection" @@ -267,7 +269,8 @@ collect_process_count() { # Like collect_process_count, but each package manager gets its own prepare, # BENCH_PREPARE_, as in the timed runs. For variations whose prepare -# depends on the tool, e.g. `ci` needs the tool's own lockfile. +# depends on the tool: `ci` and `ci+setup`, and the lockfile, node_modules and +# cache variations (see set_per_pm_prepares). collect_process_count_per_pm() { if ! command -v strace &>/dev/null; then echo "Warning: strace not available, skipping process count collection" @@ -296,3 +299,24 @@ collect_process_count_per_pm() { node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all } + +# For variations whose starting state is what the tool itself left behind: +# its own lockfile, node_modules or cache. Sets each BENCH_PREPARE_ to what +# the timed runs do before a measured run: clean_all (hyperfine --setup), one +# install that leaves that state (a warmup run), then the variation's prepare +# again. So the counted install starts from its own state and nothing another +# tool left. If the warmup install fails or times out, the prepare exits with +# its status, so process-count.sh skips the count: without the warmup the tool +# wouldn't start from its own state. Only process-count.sh runs these prepares, +# in a subshell, never hyperfine. BENCH_PREPARE_BASE must be set before calling +# this. +set_per_pm_prepares() { + local pm P setup_var install_var prepare + for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do + P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') + setup_var="BENCH_SETUP_$P" + install_var="BENCH_INSTALL_$P" + prepare="$(append_setup "$BENCH_PREPARE_BASE" "${!setup_var}")" + printf -v "BENCH_PREPARE_$P" '%s' "bash $BENCH_SCRIPTS/clean-helpers.sh clean_all; $prepare; timeout $BENCH_TIMEOUT ${!install_var} >/dev/null 2>&1 || exit \$?; bash $BENCH_SCRIPTS/clean-helpers.sh wait_for_vlt_background; $prepare" + done +} diff --git a/suite/scripts/variations/lockfile+node_modules.sh b/suite/scripts/variations/lockfile+node_modules.sh index c0dbe4d099..1fdebe623b 100644 --- a/suite/scripts/variations/lockfile+node_modules.sh +++ b/suite/scripts/variations/lockfile+node_modules.sh @@ -42,4 +42,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/lockfile.sh b/suite/scripts/variations/lockfile.sh index 527b64b0fd..847f2a6e64 100644 --- a/suite/scripts/variations/lockfile.sh +++ b/suite/scripts/variations/lockfile.sh @@ -42,4 +42,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count + +# Each tool starts from its own lockfile, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm diff --git a/suite/scripts/variations/node_modules.sh b/suite/scripts/variations/node_modules.sh index 42096d7b48..cdaa84328e 100644 --- a/suite/scripts/variations/node_modules.sh +++ b/suite/scripts/variations/node_modules.sh @@ -42,4 +42,6 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count -collect_process_count +# Each tool starts from its own node_modules, so each gets its own prepare. +set_per_pm_prepares +collect_process_count_per_pm