Repository navigation
fix(process-count): count processes, not exec attempts; ci counts each tool's own frozen install #212
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix(process-count): count processes, not exec attempts; ci counts each tool's own frozen install #212
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| // Counts the processes a command started, from an | ||
| // `strace -f -e trace=execve,execveat` log. | ||
| // | ||
| // A process is counted once if it ran a program, i.e. made at least one | ||
| // successful execve or execveat. Failed calls aren't processes: they are | ||
| // lookups along PATH (e.g. `/usr/bin/env node` trying each PATH entry). A | ||
| // process that execs more than once, like a `#!/usr/bin/env node` script becoming `env` and | ||
| // then `node`, is still one process, so a JS tool's interpreter isn't counted | ||
| // on its own. | ||
| // | ||
| // The harness runs every counted command through `bash -c`, so the traced | ||
| // process starts as a shell. Leading shell execs of that first process are | ||
| // not counted: when bash execs the tool in place, the tool's process is | ||
| // counted; when the command is a compound one and the shell stays, only the | ||
| // processes it started are counted. | ||
| // | ||
| // Usage: node count-processes.js <strace-log> | ||
| // Prints the count to stdout. | ||
|
|
||
| const fs = require('fs'); | ||
| const path = require('path'); | ||
|
|
||
| const SHELLS = new Set(['bash', 'sh', 'dash']); | ||
|
|
||
| module.exports = { countProcesses }; | ||
|
|
||
| if (require.main === module) { | ||
| const log = process.argv[2]; | ||
| if (!log) throw new Error('strace log is required'); | ||
| console.log(countProcesses(fs.readFileSync(log, 'utf8'))); | ||
| } | ||
|
|
||
| // Returns the number of processes in the strace log that ran a program, not | ||
| // counting the wrapper shell. | ||
| function countProcesses(log) { | ||
| const execs = successfulExecs(log); | ||
| const [root] = execs.keys(); | ||
| if (root !== undefined) { | ||
| const programs = execs.get(root); | ||
| while (programs.length && SHELLS.has(path.basename(programs[0]))) { | ||
| programs.shift(); | ||
| } | ||
| } | ||
| return [...execs.values()].filter((programs) => programs.length).length; | ||
| } | ||
|
|
||
| // Map of pid -> programs it successfully exec'd, in order. Every traced pid | ||
| // appears, in the order it was first seen, so the first key is the traced | ||
| // command itself. | ||
| function successfulExecs(log) { | ||
| const execs = new Map(); | ||
| // calls split by another thread's syscall: | ||
| // 12 execve("/bin/x", [...] <unfinished ...> | ||
| // 12 <... execve resumed>) = 0 | ||
| const unfinished = new Map(); | ||
|
|
||
| for (const line of log.split('\n')) { | ||
| const match = /^(\d+)\s+(.*)$/.exec(line); | ||
| if (!match) continue; | ||
| const [, pid, rest] = match; | ||
| if (!execs.has(pid)) execs.set(pid, []); | ||
|
|
||
| let program; | ||
| let result; | ||
| if (/^execve(at)?\(/.test(rest)) { | ||
| program = execPath(rest); | ||
| if (rest.endsWith('<unfinished ...>')) { | ||
| unfinished.set(pid, program); | ||
| continue; | ||
| } | ||
| result = rest; | ||
| } else if (/^<\.\.\. execve(at)? resumed>/.test(rest)) { | ||
| program = unfinished.get(pid); | ||
| unfinished.delete(pid); | ||
| result = rest; | ||
| } else { | ||
| continue; | ||
| } | ||
|
|
||
| if (program !== undefined && /\)\s+=\s+0$/.test(result)) { | ||
| execs.get(pid).push(program); | ||
| } | ||
| } | ||
|
|
||
| return execs; | ||
| } | ||
|
|
||
| // The program an execve or execveat line runs: the first string argument, | ||
| // i.e. execve's path or execveat's path after the directory fd. | ||
| function execPath(call) { | ||
| return /^execve(?:at)?\((?:[^,"]*, )?"((?:[^"\\]|\\.)*)"/.exec(call)?.[1]; | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| const assert = require("node:assert/strict"); | ||
| const test = require("node:test"); | ||
|
|
||
| const { countProcesses } = require("./count-processes.js"); | ||
|
|
||
| test("counts the tool once, not the wrapper, PATH lookups or its interpreter", () => { | ||
| // npm ci on next, from the 2026-11-17 audit: npm spawned nothing | ||
| const log = [ | ||
| `393 execve("/usr/bin/bash", ["bash", "-c", "npm ci"], 0x7ffe /* 30 vars */) = 0`, | ||
| `393 execve("/opt/node/bin/npm", ["npm", "ci"], 0x55f6 /* 30 vars */) = 0`, | ||
| `393 execve("/home/runner/.yarn/switch/bin/node", ["node", "/opt/node/bin/npm", "ci"], 0x7ffc /* 30 vars */) = -1 ENOENT (No such file or directory)`, | ||
| `393 execve("/opt/node/bin/node", ["node", "/opt/node/bin/npm", "ci"], 0x7ffc /* 30 vars */) = 0`, | ||
| ].join("\n"); | ||
| assert.equal(countProcesses(log), 1); | ||
| }); | ||
|
|
||
| test("counts child processes that ran a program", () => { | ||
| const log = [ | ||
| `2407 execve("/usr/bin/bash", ["bash", "-c", "corepack pnpm@latest install"], 0x7ffc /* 30 vars */) = 0`, | ||
| `2407 execve("/opt/node/bin/corepack", ["corepack", "pnpm@latest", "install"], 0x5572 /* 30 vars */) = 0`, | ||
| `2407 execve("/opt/node/bin/node", ["node", "/opt/node/bin/corepack", "pnpm@latest", "install"], 0x7ffc /* 30 vars */) = 0`, | ||
| `2419 execve("/home/runner/.cache/pnpm-native", ["pnpm-native", "install"], 0x34aa /* 32 vars */) = 0`, | ||
| `2499 execve("/home/runner/.yarn/switch/bin/node", ["node", "--version"], 0x7ffd /* 32 vars */) = -1 ENOENT (No such file or directory)`, | ||
| `2499 execve("/opt/node/bin/node", ["node", "--version"], 0x7ffd /* 32 vars */) = 0`, | ||
| `2465 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=2499, si_uid=1003, si_status=0} ---`, | ||
| `2407 +++ exited with 0 +++`, | ||
| ].join("\n"); | ||
| // corepack, pnpm-native, node --version; 2465 is a thread that ran nothing | ||
| assert.equal(countProcesses(log), 3); | ||
| }); | ||
|
|
||
| test("doesn't count a process whose execs all failed", () => { | ||
| const log = [ | ||
| `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, | ||
| `1 execve("/opt/node/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, | ||
| `2 execve("/usr/local/bin/missing", ["missing"], 0x1 /* 1 vars */) = -1 ENOENT (No such file or directory)`, | ||
| `2 execve("/usr/bin/missing", ["missing"], 0x1 /* 1 vars */) = -1 ENOENT (No such file or directory)`, | ||
| ].join("\n"); | ||
| assert.equal(countProcesses(log), 1); | ||
| }); | ||
|
|
||
| test("counts only the children of a wrapper shell that stays", () => { | ||
| // ci+setup: bash -c 'bash -c "npm install -g ... && npm ci"' | ||
| const log = [ | ||
| `10 execve("/usr/bin/bash", ["bash", "-c", "bash -c '...'"], 0x1 /* 1 vars */) = 0`, | ||
| `10 execve("/usr/bin/bash", ["bash", "-c", "npm install -g npm && npm ci"], 0x1 /* 1 vars */) = 0`, | ||
| `11 execve("/opt/node/bin/npm", ["npm", "install", "-g", "npm"], 0x1 /* 1 vars */) = 0`, | ||
| `11 execve("/opt/node/bin/node", ["node", "/opt/node/bin/npm"], 0x1 /* 1 vars */) = 0`, | ||
| `12 execve("/tmp/bench-npm/bin/npm", ["npm", "ci"], 0x1 /* 1 vars */) = 0`, | ||
| ].join("\n"); | ||
| assert.equal(countProcesses(log), 2); | ||
| }); | ||
|
|
||
| test("counts shells the tool starts", () => { | ||
| const log = [ | ||
| `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, | ||
| `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, | ||
| `2 execve("/bin/sh", ["sh", "-c", "node install.js"], 0x1 /* 1 vars */) = 0`, | ||
| ].join("\n"); | ||
| assert.equal(countProcesses(log), 2); | ||
| }); | ||
|
|
||
| test("follows execve calls split across lines", () => { | ||
| const log = [ | ||
| `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, | ||
| `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, | ||
| `2 execve("/opt/bin/missing", ["missing"], 0x1 /* 1 vars */ <unfinished ...>`, | ||
| `3 execve("/opt/bin/child", ["child"], 0x1 /* 1 vars */ <unfinished ...>`, | ||
| `2 <... execve resumed>) = -1 ENOENT (No such file or directory)`, | ||
| `3 <... execve resumed>) = 0`, | ||
| ].join("\n"); | ||
| assert.equal(countProcesses(log), 2); | ||
| }); | ||
|
|
||
| test("counts execveat like execve", () => { | ||
| const log = [ | ||
| `1 execve("/usr/bin/bash", ["bash", "-c", "x"], 0x1 /* 1 vars */) = 0`, | ||
| `1 execve("/opt/bin/x", ["x"], 0x1 /* 1 vars */) = 0`, | ||
| `2 execveat(AT_FDCWD, "/opt/bin/missing", ["missing"], 0x1 /* 1 vars */, 0) = -1 ENOENT (No such file or directory)`, | ||
| `3 execveat(3, "", ["child"], 0x1 /* 1 vars */, AT_EMPTY_PATH) = 0`, | ||
| `4 execveat(AT_FDCWD, "/opt/bin/split", ["split"], 0x1 /* 1 vars */, 0 <unfinished ...>`, | ||
| `4 <... execveat resumed>) = 0`, | ||
| ].join("\n"); | ||
| assert.equal(countProcesses(log), 3); | ||
| }); | ||
|
|
||
| test("counts nothing for an empty log", () => { | ||
| assert.equal(countProcesses(""), 0); | ||
| }); |
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -4,10 +4,12 @@ | |||||||||
| # Usage: | ||||||||||
| # bash process-count.sh <output-folder> <pm-name> <install-command> [prepare-command] | ||||||||||
| # | ||||||||||
| # Runs a single install wrapped in `strace -f -e trace=execve` and counts | ||||||||||
| # the number of execve() calls. The raw count is appended to | ||||||||||
| # <output-folder>/<pm-name>-process-count.txt (one number per line, matching | ||||||||||
| # the pattern used by package-count.sh). | ||||||||||
| # Runs a single install wrapped in `strace -f -e trace=execve,execveat` and counts the | ||||||||||
| # processes it started (see count-processes.js for what counts). The count is | ||||||||||
| # appended to <output-folder>/<pm-name>-process-count.txt (one number per | ||||||||||
| # line, matching the pattern used by package-count.sh). Nothing is recorded if | ||||||||||
| # the install fails: a failed command's processes aren't the ones the timed | ||||||||||
| # runs measure. | ||||||||||
| # | ||||||||||
| # This script is meant to run OUTSIDE the timed benchmark (strace adds | ||||||||||
| # overhead). It is invoked by `collect_process_count` in common.sh after | ||||||||||
|
|
@@ -44,25 +46,40 @@ if ! command -v strace &>/dev/null; then | |||||||||
| exit 0 | ||||||||||
| fi | ||||||||||
|
|
||||||||||
| # Run optional prepare step (e.g. clean + setup) | ||||||||||
| # Run optional prepare step (e.g. clean + setup). Like the timed runs, wait | ||||||||||
| # for vlt's background work before and after it, so it can't overlap the | ||||||||||
| # counted install (see hyperfine-wait.sh). | ||||||||||
| WAIT_CMD="bash $(dirname "$0")/clean-helpers.sh wait_for_vlt_background" | ||||||||||
| if [ -n "$PREPARE_CMD" ]; then | ||||||||||
| echo "[process-count] Running prepare for $PM_NAME..." | ||||||||||
| $WAIT_CMD | ||||||||||
| eval "$PREPARE_CMD" || true | ||||||||||
| $WAIT_CMD | ||||||||||
| fi | ||||||||||
|
|
||||||||||
| # Run the install command under strace | ||||||||||
| # Run the install command under strace. The timeout wraps strace, not the | ||||||||||
| # command: `strace -f` waits for every traced process, detached ones included, | ||||||||||
| # so a stuck background child would otherwise hang the counting pass. strace | ||||||||||
| # ignores SIGTERM while it has tracees, hence -k. A timed-out run exits 124 or | ||||||||||
| # 137 and is skipped like any other failure. | ||||||||||
| echo "[process-count] Running strace'd install for $PM_NAME..." | ||||||||||
| strace -f -e trace=execve -o "$STRACE_LOG" \ | ||||||||||
| bash -c "$INSTALL_CMD" > /dev/null 2>&1 || true | ||||||||||
| EXIT_CODE=0 | ||||||||||
| timeout -k 10 "${BENCH_TIMEOUT:-300}" \ | ||||||||||
| strace -f -e trace=execve,execveat -o "$STRACE_LOG" \ | ||||||||||
|
Comment on lines
+67
to
+68
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Terminate traced descendants when the timeout kills strace. When Adding Suggested fix
Committable suggestion
|
||||||||||
| bash -c "$INSTALL_CMD" > /dev/null 2>&1 || EXIT_CODE=$? | ||||||||||
|
|
||||||||||
| # Count execve calls (each line with 'execve(' is one spawned process) | ||||||||||
| if [ -f "$STRACE_LOG" ]; then | ||||||||||
| COUNT=$(grep -c 'execve(' "$STRACE_LOG" 2>/dev/null || echo "0") | ||||||||||
| echo "$COUNT" >> "$PROCESS_COUNT_FILE" | ||||||||||
| echo "[process-count] $PM_NAME spawned $COUNT processes" | ||||||||||
|
|
||||||||||
| # Remove strace log to save space (can be large) | ||||||||||
| rm -f "$STRACE_LOG" | ||||||||||
| else | ||||||||||
| if [ "$EXIT_CODE" -ne 0 ]; then | ||||||||||
| # strace passes the command's exit status through, but exits non-zero on its | ||||||||||
| # own if it can't trace (e.g. no ptrace permission), so name both. | ||||||||||
| echo "Warning: strace'd install exited with $EXIT_CODE for $PM_NAME, skipping process count" | ||||||||||
| elif [ ! -f "$STRACE_LOG" ]; then | ||||||||||
| echo "Warning: strace log not found for $PM_NAME" | ||||||||||
| elif ! COUNT=$(node "$(dirname "$0")/count-processes.js" "$STRACE_LOG"); then | ||||||||||
| echo "Warning: could not count processes from the strace log for $PM_NAME, skipping process count" | ||||||||||
| else | ||||||||||
| echo "$COUNT" >> "$PROCESS_COUNT_FILE" | ||||||||||
| echo "[process-count] $PM_NAME started $COUNT processes" | ||||||||||
| fi | ||||||||||
|
|
||||||||||
| # Remove strace log to save space (can be large) | ||||||||||
| rm -f "$STRACE_LOG" | ||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Preserve pending execs when strace changes the PID.
When a non-leader thread executes a program, strace can start the call under PID 3 and resume it under leader PID 2. It emits a
superseded by execverecord between those lines. This is a documented trace sequence. (lists.strace.io)The lookup on Line 73 then returns
undefined. If the child has no earlier successful exec, the parser omits that process and publishes an undercount.Handle the superseded record by transferring the pending call to the leader PID. Add a fixture that expects root plus child to count as 2. Preserve ordinary same-PID pairing.