From 4e4c9cd76b7fcff947dcf894e8c2ad3e5b5bb324 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 01:10:50 +0000 Subject: [PATCH 1/2] fix(process-count): count processes, not exec attempts; ci counts each tool's own frozen install Process counts counted every `execve(` line: the harness's `bash -c` wrapper, failed PATH lookups and the interpreter exec of JS tools. npm on next published 4 while spawning no children. Count processes that ran a program instead (distinct pids with a successful execve), not counting the wrapper shell (count-processes.js, with tests). - process-count.sh records nothing when the counted command fails. - ci.sh used npm's prepare for every tool, so berry, zpm, pnpm, vlt and deno counted a failing frozen install. It now uses each tool's own prepare, via collect_process_count_per_pm, moved to common.sh from ci+setup.sh. - The counting pass waits for vlt's background work around each prepare, like the timed runs, and cleans up (clean_all) when it's done. - METHODOLOGY documents what a process count is. Published counts change: on next ci, npm 4 -> 1, pnpm 5 -> 3. Fixes #209 Co-Authored-By: Claude Opus 5.5 --- docs/METHODOLOGY.md | 20 +++++++ suite/scripts/count-processes.js | 86 +++++++++++++++++++++++++++ suite/scripts/count-processes.test.js | 77 ++++++++++++++++++++++++ suite/scripts/process-count.sh | 35 +++++++---- suite/scripts/variations/ci+setup.sh | 65 +------------------- suite/scripts/variations/ci.sh | 6 +- suite/scripts/variations/common.sh | 33 ++++++++++ 7 files changed, 242 insertions(+), 80 deletions(-) create mode 100644 suite/scripts/count-processes.js create mode 100644 suite/scripts/count-processes.test.js diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index 1e0addf113e..c564ef33ed1 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -73,6 +73,26 @@ variation, and the site reports both total time and time per package: B is faster on the clock and ten times slower per unit of work. Both numbers are published; neither alone is the answer. +## Process counts + +After the timed runs, each tool's command runs once more, untimed, under +`strace -f -e trace=execve`, from the same starting state as its timed runs +(`suite/scripts/process-count.sh`). The count is the number of processes that +ran a program: each process with at least one successful `execve` counts once. + +- Failed `execve` calls don't count. They are lookups along `PATH`, so they + depend on the machine, not the tool. +- An interpreter doesn't count on its own. A `#!/usr/bin/env node` script runs + `env` and then `node` in the same process, so npm with no child processes + counts **1**. +- The `bash -c` the harness wraps the command in doesn't count. +- Detached processes do count, e.g. vlt's background cache work. +- If the command exits non-zero, nothing is recorded for that tool. + +Earlier counts (issue #209) counted every `execve` line, including the +wrapper, failed lookups and the interpreter, and `ci` counted most tools +against npm's lockfile. Don't compare them with later counts. + ## Variations | Variation | Starting state / what it measures | diff --git a/suite/scripts/count-processes.js b/suite/scripts/count-processes.js new file mode 100644 index 00000000000..829ced76a0a --- /dev/null +++ b/suite/scripts/count-processes.js @@ -0,0 +1,86 @@ +// Counts the processes a command started, from an `strace -f -e trace=execve` +// log. +// +// A process is counted once if it ran a program, i.e. made at least one +// successful execve. Failed execve calls aren't processes: they are lookups +// along PATH (e.g. `/usr/bin/env node` trying each PATH entry). A process that +// execs more than once, like a `#!/usr/bin/env node` script becoming `env` and +// then `node`, is still one process, so a JS tool's interpreter isn't counted +// on its own. +// +// The harness runs every counted command through `bash -c`, so the traced +// process starts as a shell. Leading shell execs of that first process are +// not counted: when bash execs the tool in place, the tool's process is +// counted; when the command is a compound one and the shell stays, only the +// processes it started are counted. +// +// Usage: node count-processes.js +// Prints the count to stdout. + +const fs = require('fs'); +const path = require('path'); + +const SHELLS = new Set(['bash', 'sh', 'dash']); + +module.exports = { countProcesses }; + +if (require.main === module) { + const log = process.argv[2]; + if (!log) throw new Error('strace log is required'); + console.log(countProcesses(fs.readFileSync(log, 'utf8'))); +} + +// Returns the number of processes in the strace log that ran a program, not +// counting the wrapper shell. +function countProcesses(log) { + const execs = successfulExecs(log); + const [root] = execs.keys(); + if (root !== undefined) { + const programs = execs.get(root); + while (programs.length && SHELLS.has(path.basename(programs[0]))) { + programs.shift(); + } + } + return [...execs.values()].filter((programs) => programs.length).length; +} + +// Map of pid -> programs it successfully exec'd, in order. Every traced pid +// appears, in the order it was first seen, so the first key is the traced +// command itself. +function successfulExecs(log) { + const execs = new Map(); + // execve calls split by another thread's syscall: + // 12 execve("/bin/x", [...] + // 12 <... execve resumed>) = 0 + const unfinished = new Map(); + + for (const line of log.split('\n')) { + const match = /^(\d+)\s+(.*)$/.exec(line); + if (!match) continue; + const [, pid, rest] = match; + if (!execs.has(pid)) execs.set(pid, []); + + let program; + let result; + if (rest.startsWith('execve(')) { + program = /^execve\("((?:[^"\\]|\\.)*)"/.exec(rest)?.[1]; + if (rest.endsWith('')) { + unfinished.set(pid, program); + continue; + } + result = rest; + } else if (rest.startsWith('<... execve resumed>')) { + program = unfinished.get(pid); + unfinished.delete(pid); + result = rest; + } else { + continue; + } + + if (program !== undefined && /\)\s+=\s+0$/.test(result)) { + execs.get(pid).push(program); + } + } + + return execs; +} diff --git a/suite/scripts/count-processes.test.js b/suite/scripts/count-processes.test.js new file mode 100644 index 00000000000..b15f844decf --- /dev/null +++ b/suite/scripts/count-processes.test.js @@ -0,0 +1,77 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const { countProcesses } = require("./count-processes.js"); + +test("counts the tool once, not the wrapper, PATH lookups or its interpreter", () => { + // npm ci on next, from the 2026-11-17 audit: npm spawned nothing + const log = [ + `393 execve("/usr/bin/bash", ["bash", "-c", "npm ci"], 0x7ffe /* 30 vars */) = 0`, + `393 execve("/opt/node/bin/npm", ["npm", "ci"], 0x55f6 /* 30 vars */) = 0`, + `393 execve("/home/runner/.yarn/switch/bin/node", ["node", "/opt/node/bin/npm", "ci"], 0x7ffc /* 30 vars */) = -1 ENOENT (No such file or directory)`, + `393 execve("/opt/node/bin/node", ["node", "/opt/node/bin/npm", "ci"], 0x7ffc /* 30 vars */) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 1); +}); + +test("counts child processes that ran a program", () => { + const log = [ + `2407 execve("/usr/bin/bash", ["bash", "-c", "corepack pnpm@latest install"], 0x7ffc /* 30 vars */) = 0`, + `2407 execve("/opt/node/bin/corepack", ["corepack", "pnpm@latest", "install"], 0x5572 /* 30 vars */) = 0`, + `2407 execve("/opt/node/bin/node", ["node", "/opt/node/bin/corepack", "pnpm@latest", "install"], 0x7ffc /* 30 vars */) = 0`, + `2419 execve("/home/runner/.cache/pnpm-native", ["pnpm-native", "install"], 0x34aa /* 32 vars */) = 0`, + `2499 execve("/home/runner/.yarn/switch/bin/node", ["node", "--version"], 0x7ffd /* 32 vars */) = -1 ENOENT (No such file or directory)`, + `2499 execve("/opt/node/bin/node", ["node", "--version"], 0x7ffd /* 32 vars */) = 0`, + `2465 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=2499, si_uid=1003, si_status=0} ---`, + `2407 +++ exited with 0 +++`, + ].join("\n"); + // corepack, pnpm-native, node --version; 2465 is a thread that ran nothing + assert.equal(countProcesses(log), 3); +}); + +test("doesn't count a process whose execs all failed", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/node/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execve("/usr/local/bin/missing", ["missing"], 0x1 /* 1 vars */) = -1 ENOENT (No such file or directory)`, + `2 execve("/usr/bin/missing", ["missing"], 0x1 /* 1 vars */) = -1 ENOENT (No such file or directory)`, + ].join("\n"); + assert.equal(countProcesses(log), 1); +}); + +test("counts only the children of a wrapper shell that stays", () => { + // ci+setup: bash -c 'bash -c "npm install -g ... && npm ci"' + const log = [ + `10 execve("/usr/bin/bash", ["bash", "-c", "bash -c '...'"], 0x1 /* 1 vars */) = 0`, + `10 execve("/usr/bin/bash", ["bash", "-c", "npm install -g npm && npm ci"], 0x1 /* 1 vars */) = 0`, + `11 execve("/opt/node/bin/npm", ["npm", "install", "-g", "npm"], 0x1 /* 1 vars */) = 0`, + `11 execve("/opt/node/bin/node", ["node", "/opt/node/bin/npm"], 0x1 /* 1 vars */) = 0`, + `12 execve("/tmp/bench-npm/bin/npm", ["npm", "ci"], 0x1 /* 1 vars */) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 2); +}); + +test("counts shells the tool starts", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execve("/bin/sh", ["sh", "-c", "node install.js"], 0x1 /* 1 vars */) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 2); +}); + +test("follows execve calls split across lines", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execve("/opt/bin/missing", ["missing"], 0x1 /* 1 vars */ `, + `3 execve("/opt/bin/child", ["child"], 0x1 /* 1 vars */ `, + `2 <... execve resumed>) = -1 ENOENT (No such file or directory)`, + `3 <... execve resumed>) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 2); +}); + +test("counts nothing for an empty log", () => { + assert.equal(countProcesses(""), 0); +}); diff --git a/suite/scripts/process-count.sh b/suite/scripts/process-count.sh index 43b4aaaaf56..22f90269eb7 100644 --- a/suite/scripts/process-count.sh +++ b/suite/scripts/process-count.sh @@ -4,10 +4,12 @@ # Usage: # bash process-count.sh [prepare-command] # -# Runs a single install wrapped in `strace -f -e trace=execve` and counts -# the number of execve() calls. The raw count is appended to -# /-process-count.txt (one number per line, matching -# the pattern used by package-count.sh). +# Runs a single install wrapped in `strace -f -e trace=execve` and counts the +# processes it started (see count-processes.js for what counts). The count is +# appended to /-process-count.txt (one number per +# line, matching the pattern used by package-count.sh). Nothing is recorded if +# the install fails: a failed command's processes aren't the ones the timed +# runs measure. # # This script is meant to run OUTSIDE the timed benchmark (strace adds # overhead). It is invoked by `collect_process_count` in common.sh after @@ -44,25 +46,32 @@ if ! command -v strace &>/dev/null; then exit 0 fi -# Run optional prepare step (e.g. clean + setup) +# Run optional prepare step (e.g. clean + setup). Like the timed runs, wait +# for vlt's background work before and after it, so it can't overlap the +# counted install (see hyperfine-wait.sh). +WAIT_CMD="bash $(dirname "$0")/clean-helpers.sh wait_for_vlt_background" if [ -n "$PREPARE_CMD" ]; then echo "[process-count] Running prepare for $PM_NAME..." + $WAIT_CMD eval "$PREPARE_CMD" || true + $WAIT_CMD fi # Run the install command under strace echo "[process-count] Running strace'd install for $PM_NAME..." +EXIT_CODE=0 strace -f -e trace=execve -o "$STRACE_LOG" \ - bash -c "$INSTALL_CMD" > /dev/null 2>&1 || true + bash -c "$INSTALL_CMD" > /dev/null 2>&1 || EXIT_CODE=$? -# Count execve calls (each line with 'execve(' is one spawned process) -if [ -f "$STRACE_LOG" ]; then - COUNT=$(grep -c 'execve(' "$STRACE_LOG" 2>/dev/null || echo "0") +if [ "$EXIT_CODE" -ne 0 ]; then + echo "Warning: install exited with $EXIT_CODE for $PM_NAME, skipping process count" +elif [ -f "$STRACE_LOG" ]; then + COUNT=$(node "$(dirname "$0")/count-processes.js" "$STRACE_LOG") echo "$COUNT" >> "$PROCESS_COUNT_FILE" - echo "[process-count] $PM_NAME spawned $COUNT processes" - - # Remove strace log to save space (can be large) - rm -f "$STRACE_LOG" + echo "[process-count] $PM_NAME started $COUNT processes" else echo "Warning: strace log not found for $PM_NAME" fi + +# Remove strace log to save space (can be large) +rm -f "$STRACE_LOG" diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index cab358ef69c..e57da164bcc 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -223,67 +223,6 @@ hyperfine --ignore-failure \ collect_package_count # --- Process count collection --- -# Unlike other variations that use a single BENCH_PREPARE_BASE for all PMs, -# ci+setup needs per-PM prepare commands because each PM's prepare includes -# PM-specific uninstall/cache-purge steps. Using a single base (e.g. npm's -# prepare) would leave the wrong lockfile and skip corepack purges for -# non-npm PMs, producing inaccurate process counts. -collect_process_count_per_pm() { - if ! command -v strace &>/dev/null; then - echo "Warning: strace not available, skipping process count collection" - return 0 - fi - - echo "=== Collecting spawned process counts (per-PM prepare) ===" - - local -A PM_PREPARE=( - [npm]="$BENCH_PREPARE_NPM" - [yarn]="$BENCH_PREPARE_YARN" - [berry]="$BENCH_PREPARE_BERRY" - [zpm]="$BENCH_PREPARE_ZPM" - [pnpm]="$BENCH_PREPARE_PNPM" - [vlt]="$BENCH_PREPARE_VLT" - [bun]="$BENCH_PREPARE_BUN" - [deno]="$BENCH_PREPARE_DENO" - [aube]="$BENCH_PREPARE_AUBE" - [upm]="$BENCH_PREPARE_UPM" - ) - local -A PM_INSTALL=( - [npm]="$BENCH_INSTALL_NPM" - [yarn]="$BENCH_INSTALL_YARN" - [berry]="$BENCH_INSTALL_BERRY" - [zpm]="$BENCH_INSTALL_ZPM" - [pnpm]="$BENCH_INSTALL_PNPM" - [vlt]="$BENCH_INSTALL_VLT" - [bun]="$BENCH_INSTALL_BUN" - [deno]="$BENCH_INSTALL_DENO" - [aube]="$BENCH_INSTALL_AUBE" - [upm]="$BENCH_INSTALL_UPM" - ) - local -A PM_INCLUDE=( - [npm]="$BENCH_INCLUDE_NPM" - [yarn]="$BENCH_INCLUDE_YARN" - [berry]="$BENCH_INCLUDE_BERRY" - [zpm]="$BENCH_INCLUDE_ZPM" - [pnpm]="$BENCH_INCLUDE_PNPM" - [vlt]="$BENCH_INCLUDE_VLT" - [bun]="$BENCH_INCLUDE_BUN" - [deno]="$BENCH_INCLUDE_DENO" - [aube]="$BENCH_INCLUDE_AUBE" - [upm]="$BENCH_INCLUDE_UPM" - ) - - for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do - if [ -n "${PM_INCLUDE[$pm]:-}" ]; then - bash "$BENCH_SCRIPTS/process-count.sh" \ - "$BENCH_OUTPUT_FOLDER" \ - "$pm" \ - "${PM_INSTALL[$pm]}" \ - "${PM_PREPARE[$pm]}" - fi - done - - node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" -} - +# Each PM's prepare includes PM-specific uninstall/cache-purge steps and its +# own lockfile, so each gets its own prepare. collect_process_count_per_pm diff --git a/suite/scripts/variations/ci.sh b/suite/scripts/variations/ci.sh index ff1f7ddba17..307b366ef70 100644 --- a/suite/scripts/variations/ci.sh +++ b/suite/scripts/variations/ci.sh @@ -126,7 +126,5 @@ hyperfine --ignore-failure \ collect_package_count -# For process counting, use a representative prepare base. -# collect_process_count needs BENCH_PREPARE_BASE to set up state before strace. -BENCH_PREPARE_BASE="$BENCH_PREPARE_NPM" -collect_process_count +# Each tool's ci command needs its own lockfile, so each gets its own prepare. +collect_process_count_per_pm diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index 36577df63ac..00418c49d43 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -248,5 +248,38 @@ collect_process_count() { fi done + # Don't leave the last tool's node_modules, lockfile and cache behind + bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all + node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" +} + +# Like collect_process_count, but each package manager gets its own prepare, +# BENCH_PREPARE_, as in the timed runs. For variations whose prepare +# depends on the tool, e.g. `ci` needs the tool's own lockfile. +collect_process_count_per_pm() { + if ! command -v strace &>/dev/null; then + echo "Warning: strace not available, skipping process count collection" + return 0 + fi + + echo "=== Collecting spawned process counts (per-PM prepare) ===" + + local pm P include_var install_var prepare_var + for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do + P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') + include_var="BENCH_INCLUDE_$P" + install_var="BENCH_INSTALL_$P" + prepare_var="BENCH_PREPARE_$P" + if [ -n "${!include_var:-}" ]; then + bash "$BENCH_SCRIPTS/process-count.sh" \ + "$BENCH_OUTPUT_FOLDER" \ + "$pm" \ + "${!install_var}" \ + "${!prepare_var}" + fi + done + + # Don't leave the last tool's node_modules, lockfile and cache behind + bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" } From e8295d9cd52ff7d516fd5a72cdabada7cfa520ac Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 02:44:53 +0000 Subject: [PATCH 2/2] fix(process-count): time out the counted command, collect before cleaning From review on #212: - Wrap strace in `timeout -k 10 $BENCH_TIMEOUT`. `strace -f` waits for detached children, so a timeout inside it can't stop a stuck one, and strace ignores SIGTERM while tracing, hence -k. A timed-out count is skipped like any other failure. common.sh exports BENCH_TIMEOUT. - Collect process-count.json before clean_all (common.sh, add-remove.sh), so a failing clean can't lose the counts. - Trace execveat too. - Skip the count with a warning if the parser fails, instead of failing the variation. - Name strace in the failure warning, since strace exits non-zero when it can't trace. - METHODOLOGY: launchers like corepack count; vlt's count depends on state. Co-Authored-By: Claude Opus 5.5 --- docs/METHODOLOGY.md | 17 ++++++++++++----- suite/scripts/count-processes.js | 24 +++++++++++++++--------- suite/scripts/count-processes.test.js | 12 ++++++++++++ suite/scripts/process-count.sh | 24 ++++++++++++++++-------- suite/scripts/variations/add-remove.sh | 3 ++- suite/scripts/variations/common.sh | 12 ++++++++---- 6 files changed, 65 insertions(+), 27 deletions(-) diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index c564ef33ed1..0447a3d1517 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -76,9 +76,10 @@ are published; neither alone is the answer. ## Process counts After the timed runs, each tool's command runs once more, untimed, under -`strace -f -e trace=execve`, from the same starting state as its timed runs -(`suite/scripts/process-count.sh`). The count is the number of processes that -ran a program: each process with at least one successful `execve` counts once. +`strace -f -e trace=execve,execveat`, from the same starting state as its timed +runs (`suite/scripts/process-count.sh`). The count is the number of processes +that ran a program: each process with at least one successful `execve` counts +once. - Failed `execve` calls don't count. They are lookups along `PATH`, so they depend on the machine, not the tool. @@ -86,8 +87,14 @@ ran a program: each process with at least one successful `execve` counts once. `env` and then `node` in the same process, so npm with no child processes counts **1**. - The `bash -c` the harness wraps the command in doesn't count. -- Detached processes do count, e.g. vlt's background cache work. -- If the command exits non-zero, nothing is recorded for that tool. +- A launcher counts. yarn, berry and pnpm run through `corepack`, so pnpm's + **3** is `corepack`, `pnpm-native` and `node --version`. +- Detached processes count, so a count can depend on state. vlt counts its + background cache unpacking only when there is something to unpack: **2** in + `clean`, **1** in `ci` on a warm cache. Its update-check and revalidate + processes may add to that, depending on timing. +- If the command exits non-zero or runs past `BENCH_TIMEOUT`, nothing is + recorded for that tool. Earlier counts (issue #209) counted every `execve` line, including the wrapper, failed lookups and the interpreter, and `ci` counted most tools diff --git a/suite/scripts/count-processes.js b/suite/scripts/count-processes.js index 829ced76a0a..4ceb284986f 100644 --- a/suite/scripts/count-processes.js +++ b/suite/scripts/count-processes.js @@ -1,10 +1,10 @@ -// Counts the processes a command started, from an `strace -f -e trace=execve` -// log. +// Counts the processes a command started, from an +// `strace -f -e trace=execve,execveat` log. // // A process is counted once if it ran a program, i.e. made at least one -// successful execve. Failed execve calls aren't processes: they are lookups -// along PATH (e.g. `/usr/bin/env node` trying each PATH entry). A process that -// execs more than once, like a `#!/usr/bin/env node` script becoming `env` and +// successful execve or execveat. Failed calls aren't processes: they are +// lookups along PATH (e.g. `/usr/bin/env node` trying each PATH entry). A +// process that execs more than once, like a `#!/usr/bin/env node` script becoming `env` and // then `node`, is still one process, so a JS tool's interpreter isn't counted // on its own. // @@ -49,7 +49,7 @@ function countProcesses(log) { // command itself. function successfulExecs(log) { const execs = new Map(); - // execve calls split by another thread's syscall: + // calls split by another thread's syscall: // 12 execve("/bin/x", [...] // 12 <... execve resumed>) = 0 const unfinished = new Map(); @@ -62,14 +62,14 @@ function successfulExecs(log) { let program; let result; - if (rest.startsWith('execve(')) { - program = /^execve\("((?:[^"\\]|\\.)*)"/.exec(rest)?.[1]; + if (/^execve(at)?\(/.test(rest)) { + program = execPath(rest); if (rest.endsWith('')) { unfinished.set(pid, program); continue; } result = rest; - } else if (rest.startsWith('<... execve resumed>')) { + } else if (/^<\.\.\. execve(at)? resumed>/.test(rest)) { program = unfinished.get(pid); unfinished.delete(pid); result = rest; @@ -84,3 +84,9 @@ function successfulExecs(log) { return execs; } + +// The program an execve or execveat line runs: the first string argument, +// i.e. execve's path or execveat's path after the directory fd. +function execPath(call) { + return /^execve(?:at)?\((?:[^,"]*, )?"((?:[^"\\]|\\.)*)"/.exec(call)?.[1]; +} diff --git a/suite/scripts/count-processes.test.js b/suite/scripts/count-processes.test.js index b15f844decf..ff109a724ae 100644 --- a/suite/scripts/count-processes.test.js +++ b/suite/scripts/count-processes.test.js @@ -72,6 +72,18 @@ test("follows execve calls split across lines", () => { assert.equal(countProcesses(log), 2); }); +test("counts execveat like execve", () => { + const log = [ + `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, + `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, + `2 execveat(AT_FDCWD, "/opt/bin/missing", ["missing"], 0x1 /* 1 vars */, 0) = -1 ENOENT (No such file or directory)`, + `3 execveat(3, "", ["child"], 0x1 /* 1 vars */, AT_EMPTY_PATH) = 0`, + `4 execveat(AT_FDCWD, "/opt/bin/split", ["split"], 0x1 /* 1 vars */, 0 `, + `4 <... execveat resumed>) = 0`, + ].join("\n"); + assert.equal(countProcesses(log), 3); +}); + test("counts nothing for an empty log", () => { assert.equal(countProcesses(""), 0); }); diff --git a/suite/scripts/process-count.sh b/suite/scripts/process-count.sh index 22f90269eb7..26c588116a1 100644 --- a/suite/scripts/process-count.sh +++ b/suite/scripts/process-count.sh @@ -4,7 +4,7 @@ # Usage: # bash process-count.sh [prepare-command] # -# Runs a single install wrapped in `strace -f -e trace=execve` and counts the +# Runs a single install wrapped in `strace -f -e trace=execve,execveat` and counts the # processes it started (see count-processes.js for what counts). The count is # appended to /-process-count.txt (one number per # line, matching the pattern used by package-count.sh). Nothing is recorded if @@ -57,20 +57,28 @@ if [ -n "$PREPARE_CMD" ]; then $WAIT_CMD fi -# Run the install command under strace +# Run the install command under strace. The timeout wraps strace, not the +# command: `strace -f` waits for every traced process, detached ones included, +# so a stuck background child would otherwise hang the counting pass. strace +# ignores SIGTERM while it has tracees, hence -k. A timed-out run exits 124 or +# 137 and is skipped like any other failure. echo "[process-count] Running strace'd install for $PM_NAME..." EXIT_CODE=0 -strace -f -e trace=execve -o "$STRACE_LOG" \ +timeout -k 10 "${BENCH_TIMEOUT:-300}" \ + strace -f -e trace=execve,execveat -o "$STRACE_LOG" \ bash -c "$INSTALL_CMD" > /dev/null 2>&1 || EXIT_CODE=$? if [ "$EXIT_CODE" -ne 0 ]; then - echo "Warning: install exited with $EXIT_CODE for $PM_NAME, skipping process count" -elif [ -f "$STRACE_LOG" ]; then - COUNT=$(node "$(dirname "$0")/count-processes.js" "$STRACE_LOG") + # strace passes the command's exit status through, but exits non-zero on its + # own if it can't trace (e.g. no ptrace permission), so name both. + echo "Warning: strace'd install exited with $EXIT_CODE for $PM_NAME, skipping process count" +elif [ ! -f "$STRACE_LOG" ]; then + echo "Warning: strace log not found for $PM_NAME" +elif ! COUNT=$(node "$(dirname "$0")/count-processes.js" "$STRACE_LOG"); then + echo "Warning: could not count processes from the strace log for $PM_NAME, skipping process count" +else echo "$COUNT" >> "$PROCESS_COUNT_FILE" echo "[process-count] $PM_NAME started $COUNT processes" -else - echo "Warning: strace log not found for $PM_NAME" fi # Remove strace log to save space (can be large) diff --git a/suite/scripts/variations/add-remove.sh b/suite/scripts/variations/add-remove.sh index eaa10610406..e64d4a996ab 100644 --- a/suite/scripts/variations/add-remove.sh +++ b/suite/scripts/variations/add-remove.sh @@ -193,9 +193,10 @@ if command -v strace &>/dev/null; then bash "$BENCH_SCRIPTS/process-count.sh" "$BENCH_OUTPUT_FOLDER" "$pm" "${!op_var}" fi done + # Collect first, so a failing clean can't lose the counts + node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" eval "$BENCH_AR_DESTROY" bash "$BENCH_SCRIPTS/clean-helpers.sh" clean_all - node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" else echo "Warning: strace not available, skipping process count collection" fi diff --git a/suite/scripts/variations/common.sh b/suite/scripts/variations/common.sh index 00418c49d43..ad06789743a 100644 --- a/suite/scripts/variations/common.sh +++ b/suite/scripts/variations/common.sh @@ -38,6 +38,8 @@ BENCH_RUNS="${BENCH_RUNS:=5}" # If a single install exceeds this, it is killed. hyperfine --ignore-failure # lets the suite continue; the timed-out run records as a failure. BENCH_TIMEOUT="${BENCH_TIMEOUT:=300}" +# process-count.sh runs as its own script and times its counted command too +export BENCH_TIMEOUT for pm in npm yarn berry zpm pnpm vlt bun deno aube upm nx turbo vp node; do CHOICE=$(echo "$pm" | tr '[:lower:]' '[:upper:]') if echo "$BENCH_INCLUDE" | grep -qw "$pm"; then @@ -248,9 +250,10 @@ collect_process_count() { fi done - # Don't leave the last tool's node_modules, lockfile and cache behind - bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all + # Collect first, so a failing clean can't lose the counts. Then don't leave + # the last tool's node_modules, lockfile and cache behind. node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" + bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all } # Like collect_process_count, but each package manager gets its own prepare, @@ -279,7 +282,8 @@ collect_process_count_per_pm() { fi done - # Don't leave the last tool's node_modules, lockfile and cache behind - bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all + # Collect first, so a failing clean can't lose the counts. Then don't leave + # the last tool's node_modules, lockfile and cache behind. node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" + bash "$BENCH_SCRIPTS/clean-helpers.sh" wait_for_vlt_background clean_all }