diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index 1e0addf113..0447a3d151 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -73,6 +73,33 @@ variation, and the site reports both total time and time per package: B is faster on the clock and ten times slower per unit of work. Both numbers are published; neither alone is the answer. +## Process counts + +After the timed runs, each tool's command runs once more, untimed, under +`strace -f -e trace=execve,execveat`, from the same starting state as its timed +runs (`suite/scripts/process-count.sh`). The count is the number of processes +that ran a program: each process with at least one successful `execve` counts +once. + +- Failed `execve` calls don't count. They are lookups along `PATH`, so they + depend on the machine, not the tool. +- An interpreter doesn't count on its own. A `#!/usr/bin/env node` script runs + `env` and then `node` in the same process, so npm with no child processes + counts **1**. +- The `bash -c` the harness wraps the command in doesn't count. +- A launcher counts. yarn, berry and pnpm run through `corepack`, so pnpm's + **3** is `corepack`, `pnpm-native` and `node --version`. +- Detached processes count, so a count can depend on state. vlt counts its + background cache unpacking only when there is something to unpack: **2** in + `clean`, **1** in `ci` on a warm cache. Its update-check and revalidate + processes may add to that, depending on timing. +- If the command exits non-zero or runs past `BENCH_TIMEOUT`, nothing is + recorded for that tool. + +Earlier counts (issue #209) counted every `execve` line, including the +wrapper, failed lookups and the interpreter, and `ci` counted most tools +against npm's lockfile. Don't compare them with later counts. + ## Variations | Variation | Starting state / what it measures | diff --git a/suite/scripts/count-processes.js b/suite/scripts/count-processes.js new file mode 100644 index 0000000000..4ceb284986 --- /dev/null +++ b/suite/scripts/count-processes.js @@ -0,0 +1,92 @@ +// Counts the processes a command started, from an +// `strace -f -e trace=execve,execveat` log. +// +// A process is counted once if it ran a program, i.e. made at least one +// successful execve or execveat. Failed calls aren't processes: they are +// lookups along PATH (e.g. `/usr/bin/env node` trying each PATH entry). A +// process that execs more than once, like a `#!/usr/bin/env node` script becoming `env` and +// then `node`, is still one process, so a JS tool's interpreter isn't counted +// on its own. +// +// The harness runs every counted command through `bash -c`, so the traced +// process starts as a shell. Leading shell execs of that first process are +// not counted: when bash execs the tool in place, the tool's process is +// counted; when the command is a compound one and the shell stays, only the +// processes it started are counted. +// +// Usage: node count-processes.js +// Prints the count to stdout. + +const fs = require('fs'); +const path = require('path'); + +const SHELLS = new Set(['bash', 'sh', 'dash']); + +module.exports = { countProcesses }; + +if (require.main === module) { + const log = process.argv[2]; + if (!log) throw new Error('strace log is required'); + console.log(countProcesses(fs.readFileSync(log, 'utf8'))); +} + +// Returns the number of processes in the strace log that ran a program, not +// counting the wrapper shell. +function countProcesses(log) { + const execs = successfulExecs(log); + const [root] = execs.keys(); + if (root !== undefined) { + const programs = execs.get(root); + while (programs.length && SHELLS.has(path.basename(programs[0]))) { + programs.shift(); + } + } + return [...execs.values()].filter((programs) => programs.length).length; +} + +// Map of pid -> programs it successfully exec'd, in order. Every traced pid +// appears, in the order it was first seen, so the first key is the traced +// command itself. +function successfulExecs(log) { + const execs = new Map(); + // calls split by another thread's syscall: + // 12 execve("/bin/x", [...] + // 12 <... execve resumed>) = 0 + const unfinished = new Map(); + + for (const line of log.split('\n')) { + const match = /^(\d+)\s+(.*)$/.exec(line); + if (!match) continue; + const [, pid, rest] = match; + if (!execs.has(pid)) execs.set(pid, []); + + let program; + let result; + if (/^execve(at)?\(/.test(rest)) { + program = execPath(rest); + if (rest.endsWith('')) { + unfinished.set(pid, program); + continue; + } + result = rest; + } else if (/^<\.\.\. execve(at)? resumed>/.test(rest)) { + program = unfinished.get(pid); + unfinished.delete(pid); + result = rest; + } else { + continue; + } + + if (program !== undefined && /\)\s+=\s+0$/.test(result)) { + execs.get(pid).push(program); + } + } + + return execs; +} + +// The program an execve or execveat line runs: the first string argument, +// i.e. execve's path or execveat's path after the directory fd. +function execPath(call) { + return /^execve(?:at)?\((?:[^,"]*, )?"((?:[^"\\]|\\.)*)"/.exec(call)?.[1]; +} diff --git a/suite/scripts/count-processes.test.js b/suite/scripts/count-processes.test.js new file mode 100644 index 0000000000..ff109a724a --- /dev/null +++ b/suite/scripts/count-processes.test.js @@ -0,0 +1,89 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const { countProcesses } = require("./count-processes.js"); + +test("counts the tool once, not the wrapper, PATH lookups or its interpreter", () => { + // npm ci on next, from the 2026-11-17 audit: npm spawned nothing + const log = [ + `393 execve("/usr/bin/bash", ["bash", "-c", "npm ci"], 0x7ffe /* 30 vars */) = 0`, + `393 execve("/opt/node/bin/npm", ["npm", "ci"], 0x55f6 /* 30 vars */) = 0`, + `393 execve("/home/runner/.yarn/switch/bin/node", ["node", "/opt/node/bin/npm", "ci"], 0x7ffc /* 30 vars */) = -1 ENOENT (No such file or directory)`, + `393 execve("/opt/node/bin/node", ["node", "/opt/node/bin/npm", "ci"], 0x7ffc /* 30 vars */) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 1); +}); + +test("counts child processes that ran a program", () => { + const log = [ + `2407 execve("/usr/bin/bash", ["bash", "-c", "corepack pnpm@latest install"], 0x7ffc /* 30 vars */) = 0`, + `2407 execve("/opt/node/bin/corepack", ["corepack", "pnpm@latest", "install"], 0x5572 /* 30 vars */) = 0`, + `2407 execve("/opt/node/bin/node", ["node", "/opt/node/bin/corepack", "pnpm@latest", "install"], 0x7ffc /* 30 vars */) = 0`, + `2419 execve("/home/runner/.cache/pnpm-native", ["pnpm-native", "install"], 0x34aa /* 32 vars */) = 0`, + `2499 execve("/home/runner/.yarn/switch/bin/node", ["node", "--version"], 0x7ffd /* 32 vars */) = -1 ENOENT (No such file or directory)`, + `2499 execve("/opt/node/bin/node", ["node", "--version"], 0x7ffd /* 32 vars */) = 0`, + `2465 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=2499, si_uid=1003, si_status=0} ---`, + `2407 +++ exited with 0 +++`, + ].join("\n"); + // corepack, pnpm-native, node --version; 2465 is a thread that ran nothing + assert.equal(countProcesses(log), 3); +}); + +test("doesn't count a process whose execs all failed", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/node/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execve("/usr/local/bin/missing", ["missing"], 0x1 /* 1 vars */) = -1 ENOENT (No such file or directory)`, + `2 execve("/usr/bin/missing", ["missing"], 0x1 /* 1 vars */) = -1 ENOENT (No such file or directory)`, + ].join("\n"); + assert.equal(countProcesses(log), 1); +}); + +test("counts only the children of a wrapper shell that stays", () => { + // ci+setup: bash -c 'bash -c "npm install -g ... && npm ci"' + const log = [ + `10 execve("/usr/bin/bash", ["bash", "-c", "bash -c '...'"], 0x1 /* 1 vars */) = 0`, + `10 execve("/usr/bin/bash", ["bash", "-c", "npm install -g npm && npm ci"], 0x1 /* 1 vars */) = 0`, + `11 execve("/opt/node/bin/npm", ["npm", "install", "-g", "npm"], 0x1 /* 1 vars */) = 0`, + `11 execve("/opt/node/bin/node", ["node", "/opt/node/bin/npm"], 0x1 /* 1 vars */) = 0`, + `12 execve("/tmp/bench-npm/bin/npm", ["npm", "ci"], 0x1 /* 1 vars */) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 2); +}); + +test("counts shells the tool starts", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execve("/bin/sh", ["sh", "-c", "node install.js"], 0x1 /* 1 vars */) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 2); +}); + +test("follows execve calls split across lines", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execve("/opt/bin/missing", ["missing"], 0x1 /* 1 vars */ `, + `3 execve("/opt/bin/child", ["child"], 0x1 /* 1 vars */ `, + `2 <... execve resumed>) = -1 ENOENT (No such file or directory)`, + `3 <... execve resumed>) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 2); +}); + +test("counts execveat like execve", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execveat(AT_FDCWD, "/opt/bin/missing", ["missing"], 0x1 /* 1 vars */, 0) = -1 ENOENT (No such file or directory)`, + `3 execveat(3, "", ["child"], 0x1 /* 1 vars */, AT_EMPTY_PATH) = 0`, + `4 execveat(AT_FDCWD, "/opt/bin/split", ["split"], 0x1 /* 1 vars */, 0 `, + `4 <... execveat resumed>) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 3); +}); + +test("counts nothing for an empty log", () => { + assert.equal(countProcesses(""), 0); +}); diff --git a/suite/scripts/process-count.sh b/suite/scripts/process-count.sh index 43b4aaaaf5..26c588116a 100644 --- a/suite/scripts/process-count.sh +++ b/suite/scripts/process-count.sh @@ -4,10 +4,12 @@ # Usage: # bash process-count.sh [prepare-command] # -# Runs a single install wrapped in `strace -f -e trace=execve` and counts -# the number of execve() calls. The raw count is appended to -# /-process-count.txt (one number per line, matching -# the pattern used by package-count.sh). +# Runs a single install wrapped in `strace -f -e trace=execve,execveat` and counts the +# processes it started (see count-processes.js for what counts). The count is +# appended to /-process-count.txt (one number per +# line, matching the pattern used by package-count.sh). Nothing is recorded if +# the install fails: a failed command's processes aren't the ones the timed +# runs measure. # # This script is meant to run OUTSIDE the timed benchmark (strace adds # overhead). It is invoked by `collect_process_count` in common.sh after @@ -44,25 +46,40 @@ if ! command -v strace &>/dev/null; then exit 0 fi -# Run optional prepare step (e.g. clean + setup) +# Run optional prepare step (e.g. clean + setup). Like the timed runs, wait +# for vlt's background work before and after it, so it can't overlap the +# counted install (see hyperfine-wait.sh). +WAIT_CMD="bash $(dirname "$0")/clean-helpers.sh wait_for_vlt_background" if [ -n "$PREPARE_CMD" ]; then echo "[process-count] Running prepare for $PM_NAME..." + $WAIT_CMD eval "$PREPARE_CMD" || true + $WAIT_CMD fi -# Run the install command under strace +# Run the install command under strace. The timeout wraps strace, not the +# command: `strace -f` waits for every traced process, detached ones included, +# so a stuck background child would otherwise hang the counting pass. strace +# ignores SIGTERM while it has tracees, hence -k. A timed-out run exits 124 or +# 137 and is skipped like any other failure. echo "[process-count] Running strace'd install for $PM_NAME..." -strace -f -e trace=execve -o "$STRACE_LOG" \ - bash -c "$INSTALL_CMD" > /dev/null 2>&1 || true +EXIT_CODE=0 +timeout -k 10 "${BENCH_TIMEOUT:-300}" \ + strace -f -e trace=execve,execveat -o "$STRACE_LOG" \ + bash -c "$INSTALL_CMD" > /dev/null 2>&1 || EXIT_CODE=$? -# Count execve calls (each line with 'execve(' is one spawned process) -if [ -f "$STRACE_LOG" ]; then - COUNT=$(grep -c 'execve(' "$STRACE_LOG" 2>/dev/null || echo "0") - echo "$COUNT" >> "$PROCESS_COUNT_FILE" - echo "[process-count] $PM_NAME spawned $COUNT processes" - - # Remove strace log to save space (can be large) - rm -f "$STRACE_LOG" -else +if [ "$EXIT_CODE" -ne 0 ]; then + # strace passes the command's exit status through, but exits non-zero on its + # own if it can't trace (e.g. no ptrace permission), so name both. + echo "Warning: strace'd install exited with $EXIT_CODE for $PM_NAME, skipping process count" +elif [ ! -f "$STRACE_LOG" ]; then echo "Warning: strace log not found for $PM_NAME" +elif ! COUNT=$(node "$(dirname "$0")/count-processes.js" "$STRACE_LOG"); then + echo "Warning: could not count processes from the strace log for $PM_NAME, skipping process count" +else + echo "$COUNT" >> "$PROCESS_COUNT_FILE" + echo "[process-count] $PM_NAME started $COUNT processes" fi + +# Remove strace log to save space (can be large) +rm -f "$STRACE_LOG" diff --git a/suite/scripts/variations/add-remove.sh b/suite/scripts/variations/add-remove.sh index eaa1061040..e64d4a996a 100644 --- a/suite/scripts/variations/add-remove.sh +++ b/suite/scripts/variations/add-remove.sh @@ -193,9 +193,10 @@ if command -v strace &>/dev/null; then bash "$BENCH_SCRIPTS/process-count.sh" "$BENCH_OUTPUT_FOLDER" "$pm" "${!op_var}" fi done + # Collect first, so a failing clean can't lose the counts + node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" eval "$BENCH_AR_DESTROY" bash "$BENCH_SCRIPTS/clean-helpers.sh" clean_all - node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" else echo "Warning: strace not available, skipping process count collection" fi diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index cab358ef69..e57da164bc 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -223,67 +223,6 @@ hyperfine --ignore-failure \ collect_package_count # --- Process count collection --- -# Unlike other variations that use a single BENCH_PREPARE_BASE for all PMs, -# ci+setup needs per-PM prepare commands because each PM's prepare includes -# PM-specific uninstall/cache-purge steps. Using a single base (e.g. npm's -# prepare) would leave the wrong lockfile and skip corepack purges for -# non-npm PMs, producing inaccurate process counts. -collect_process_count_per_pm() { - if ! command -v strace &>/dev/null; then - echo "Warning: strace not available, skipping process count collection" - return 0 - fi - - echo "=== Collecting spawned process counts (per-PM prepare) ===" - - local -A PM_PREPARE=( - [npm]="$BENCH_PREPARE_NPM" - [yarn]="$BENCH_PREPARE_YARN" - [berry]="$BENCH_PREPARE_BERRY" - [zpm]="$BENCH_PREPARE_ZPM" - [pnpm]="$BENCH_PREPARE_PNPM" - [vlt]="$BENCH_PREPARE_VLT" - [bun]="$BENCH_PREPARE_BUN" - [deno]="$BENCH_PREPARE_DENO" - [aube]="$BENCH_PREPARE_AUBE" - [upm]="$BENCH_PREPARE_UPM" - ) - local -A PM_INSTALL=( - [npm]="$BENCH_INSTALL_NPM" - [yarn]="$BENCH_INSTALL_YARN" - [berry]="$BENCH_INSTALL_BERRY" - [zpm]="$BENCH_INSTALL_ZPM" - [pnpm]="$BENCH_INSTALL_PNPM" - [vlt]="$BENCH_INSTALL_VLT" - [bun]="$BENCH_INSTALL_BUN" - [deno]="$BENCH_INSTALL_DENO" - [aube]="$BENCH_INSTALL_AUBE" - [upm]="$BENCH_INSTALL_UPM" - ) - local -A PM_INCLUDE=( - [npm]="$BENCH_INCLUDE_NPM" - [yarn]="$BENCH_INCLUDE_YARN" - [berry]="$BENCH_INCLUDE_BERRY" - [zpm]="$BENCH_INCLUDE_ZPM" - [pnpm]="$BENCH_INCLUDE_PNPM" - [vlt]="$BENCH_INCLUDE_VLT" - [bun]="$BENCH_INCLUDE_BUN" - [deno]="$BENCH_INCLUDE_DENO" - [aube]="$BENCH_INCLUDE_AUBE" - [upm]="$BENCH_INCLUDE_UPM" - ) - - for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do - if [ -n "${PM_INCLUDE[$pm]:-}" ]; then - bash "$BENCH_SCRIPTS/process-count.sh" \ - "$BENCH_OUTPUT_FOLDER" \ - "$pm" \ - "${PM_INSTALL[$pm]}" \ - "${PM_PREPARE[$pm]}" - fi - done - - node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" -} - +# Each PM's prepare includes PM-specific uninstall/cache-purge steps and its +# own lockfile, so each gets its own prepare. collect_process_count_per_pm diff --git a/suite/scripts/variations/ci.sh b/suite/scripts/variations/ci.sh index ff1f7ddba1..307b366ef7 100644 --- a/suite/scripts/variations/ci.sh +++ b/suite/scripts/variations/ci.sh @@ -126,7 +126,5 @@ hyperfine --ignore-failure \ collect_package_count -# For process counting, use a representative prepare base. -# collect_process_count needs BENCH_PREPARE_BASE to set up state before strace. -BENCH_PREPARE_BASE="$BENCH_PREPARE_NPM" -collect_process_count +# Each tool's ci command needs its own lockfile, so each gets its own prepare. +collect_process_count_per_pm diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index 36577df63a..ad06789743 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -38,6 +38,8 @@ BENCH_RUNS="${BENCH_RUNS:=5}" # If a single install exceeds this, it is killed. hyperfine --ignore-failure # lets the suite continue; the timed-out run records as a failure. BENCH_TIMEOUT="${BENCH_TIMEOUT:=300}" +# process-count.sh runs as its own script and times its counted command too +export BENCH_TIMEOUT for pm in npm yarn berry zpm pnpm vlt bun deno aube upm nx turbo vp node; do CHOICE=$(echo "$pm" | tr '[:lower:]' '[:upper:]') if echo "$BENCH_INCLUDE" | grep -qw "$pm"; then @@ -248,5 +250,40 @@ collect_process_count() { fi done + # Collect first, so a failing clean can't lose the counts. Then don't leave + # the last tool's node_modules, lockfile and cache behind. node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" + bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all +} + +# Like collect_process_count, but each package manager gets its own prepare, +# BENCH_PREPARE_, as in the timed runs. For variations whose prepare +# depends on the tool, e.g. `ci` needs the tool's own lockfile. +collect_process_count_per_pm() { + if ! command -v strace &>/dev/null; then + echo "Warning: strace not available, skipping process count collection" + return 0 + fi + + echo "=== Collecting spawned process counts (per-PM prepare) ===" + + local pm P include_var install_var prepare_var + for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do + P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') + include_var="BENCH_INCLUDE_$P" + install_var="BENCH_INSTALL_$P" + prepare_var="BENCH_PREPARE_$P" + if [ -n "${!include_var:-}" ]; then + bash "$BENCH_SCRIPTS/process-count.sh" \ + "$BENCH_OUTPUT_FOLDER" \ + "$pm" \ + "${!install_var}" \ + "${!prepare_var}" + fi + done + + # Collect first, so a failing clean can't lose the counts. Then don't leave + # the last tool's node_modules, lockfile and cache behind. + node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" + bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all }