diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index c5695449ef..edb6323c9e 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -124,7 +124,11 @@ berry run, as `yarn set version` would; `clean-helpers.sh` removes the pin before the next tool runs. In `ci+setup`, the "setup" part of each timed command is the same install (into a fresh prefix with an empty npm cache, so it downloads every time); for berry that is installing Corepack plus -Corepack's download of Yarn. +Corepack's download of Yarn. The lockfile and the dependency cache are made +once per tool, untimed, by a regular install on its first prepare; before +every timed run the prepare then only removes `node_modules`, the tool's +binary and its download cache, so each run starts from the same state: tool +absent, lockfile present, cache warm, no `node_modules`. Until [#207](https://github.com/vltpkg/benchmarks/issues/207), pnpm, yarn and berry ran through `corepack pnpm@latest`, `corepack yarn@1` and @@ -199,6 +203,7 @@ missing. Don't compare them with later counts. | `lockfile+node_modules` | lockfile and `node_modules` | | `cache+lockfile+node_modules` | everything warm — best case | | `ci` | the tool's CI-oriented install (`npm ci` and equivalents) | +| `ci+setup` | `ci`, plus installing the tool itself first | | `add-singleton` | add one uncached zero-dependency package to an installed project | | `remove-singleton` | remove that package from an installed project | | `add-overlap` | add one uncached package whose deps overlap the existing graph | diff --git a/suite/scripts/ci-setup-prepare.test.js b/suite/scripts/ci-setup-prepare.test.js new file mode 100644 index 0000000000..61f54a4516 --- /dev/null +++ b/suite/scripts/ci-setup-prepare.test.js @@ -0,0 +1,180 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const test = require("node:test"); + +// Runs variations/ci+setup.sh with every tool, hyperfine and strace stubbed +// (each stub logs its arguments; `bun install` writes a lockfile and a +// node_modules). The hyperfine stub records its argv, so the test can read +// each tool's --prepare and run one by hand. Nothing touches the real home, +// /tmp or the caches: HOME, BENCH_SETUP_TMP and the fixture are temporary +// directories and XDG_* / npm_config_* are not passed on. +const PMS = ["npm", "yarn", "berry", "zpm", "pnpm", "vlt", "bun", "deno", "aube", "upm"]; +const STUBS = [...PMS, "nx", "vp", "corepack", "curl", "strace"]; + +function setup(t, { include, hyperfineExit = 0 }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-setup-test-")); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const bin = path.join(dir, "bin"); + const home = path.join(dir, "home"); + const tmp = path.join(dir, "tmp"); + const fixture = path.join(dir, "fixtures", "next"); + const results = path.join(dir, "results"); + const commandLog = path.join(dir, "commands.txt"); + const argvFile = path.join(dir, "hyperfine-argv.json"); + for (const d of [bin, home, tmp, fixture]) fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync(path.join(fixture, "package.json"), '{"name":"next","version":"1.0.0"}\n'); + fs.writeFileSync(commandLog, ""); + for (const name of STUBS) { + fs.writeFileSync( + path.join(bin, name), + `#!/bin/sh +printf '%s %s\\n' "$(basename "$0")" "$*" >> "$COMMAND_LOG" +if [ "$(basename "$0")" = bun ] && [ "$1" = install ]; then + [ "$BUN_FAIL" = 1 ] && exit 1 + printf 'lockfile for %s\\n' "$(pwd)" > bun.lock + mkdir -p node_modules/a && printf '{"name":"a","version":"1.0.0"}' > node_modules/a/package.json +fi +exit 0 +`, + { mode: 0o755 }, + ); + } + fs.writeFileSync( + path.join(bin, "hyperfine"), + `#!/usr/bin/env node +require("node:fs").writeFileSync(process.env.HYPERFINE_ARGV, JSON.stringify(process.argv.slice(2))); +process.exit(${hyperfineExit}); +`, + { mode: 0o755 }, + ); + const env = { + PATH: `${bin}${path.delimiter}${process.env.PATH}`, + HOME: home, + LANG: "C.UTF-8", + BENCH_SETUP_TMP: tmp, + BENCH_BERRY_VERSION: "4.9.2", + BENCH_ZPM_VERSION: "6.0.0-rc.20", + BENCH_INCLUDE: include, + BENCH_WARMUP: "1", + BENCH_RUNS: "1", + COMMAND_LOG: commandLog, + HYPERFINE_ARGV: argvFile, + }; + const scripts = path.resolve(__dirname); + const run = () => + spawnSync("bash", [path.join(scripts, "variations", "ci+setup.sh"), scripts, path.join(dir, "results"), "next", "ci+setup"], { + cwd: fixture, + encoding: "utf8", + env, + }); + const commands = () => fs.readFileSync(commandLog, "utf8").trim().split("\n").filter(Boolean); + const argv = () => JSON.parse(fs.readFileSync(argvFile, "utf8")); + // the --prepare that precedes --command-name= + const prepareOf = (pm) => { + const a = argv(); + const i = a.indexOf(`--command-name=${pm}`); + assert.ok(i > 0, `no command named ${pm}`); + assert.match(a[i - 1], /^--prepare=/); + return a[i - 1].slice("--prepare=".length); + }; + const marker = (pm) => path.join(results, "next", "ci+setup", `.ci-setup-ready-${pm}`); + return { dir, fixture, results, env, run, commands, argv, prepareOf, marker, commandLog }; +} + +test("ci+setup: every tool's prepare installs the lockfile once, and uninstalls the PM every time", (t) => { + const s = setup(t, { include: PMS.join(","), hyperfineExit: 1 }); + const r = s.run(); + assert.notEqual(r.status, 0, "the hyperfine stub fails, so the script stops there"); + for (const pm of PMS) { + const prepare = s.prepareOf(pm); + const guard = `if [ ! -e ${s.marker(pm)} ]; then `; + const i = prepare.indexOf(guard); + assert.ok(i >= 0, `${pm}: prepare has no once-per-tool guard: ${prepare}`); + const once = prepare.slice(i + guard.length, prepare.indexOf("; fi", i)); + const after = prepare.slice(prepare.indexOf("; fi", i) + 4); + // once: clean the previous tool's state, install the PM, write the lockfile + assert.match(once, /clean_node_modules clean_lockfiles clean_package_manager_files clean_package_manager_field/, pm); + assert.match(once, / install /, `${pm}: no install in the once part`); + assert.match(once, new RegExp(`&& touch ${s.marker(pm).replace(/[+.]/g, "\\$&")} \\|\\| true`), pm); + // every run: clean node_modules, uninstall the PM, nothing that touches the lockfile + assert.match(after, /clean_node_modules clean_package_manager_files/, pm); + assert.doesNotMatch(after, /clean_lockfiles|clean_package_manager_field| install /, `${pm}: ${after}`); + assert.match(after, /rm -rf|uninstall -g/, `${pm}: no uninstall after the guard`); + } + // berry: corepack's home is purged every run, after the guard + assert.match(s.prepareOf("berry").split("; fi")[1], /rm -rf .*bench-corepack-home/); + // --conclude keeps the lockfile and the packageManager pin for the next run + const conclude = s.argv().find((a) => a.startsWith("--conclude=")); + assert.match(conclude, /package-count\.sh/); + assert.match(conclude, /clean_node_modules/); + assert.doesNotMatch(conclude, /clean_lockfiles|clean_package_manager_field/); + // --cleanup still resets everything between tools + assert.match(s.argv().find((a) => a.startsWith("--cleanup=")), /clean_all/); +}); + +test("ci+setup: the second prepare of a tool keeps the lockfile and only uninstalls the PM", (t) => { + const s = setup(t, { include: "bun" }); + const r = s.run(); + assert.equal(r.status, 0, r.stderr); + const prepare = s.prepareOf("bun"); + const runPrepare = (extra = {}) => { + const p = spawnSync("sh", ["-c", prepare], { cwd: s.fixture, encoding: "utf8", env: { ...s.env, ...extra } }); + assert.equal(p.status, 0, p.stderr); + }; + const count = (re) => s.commands().filter((c) => re.test(c)).length; + const lockfile = () => fs.existsSync(path.join(s.fixture, "bun.lock")) && fs.readFileSync(path.join(s.fixture, "bun.lock"), "utf8"); + + // the script's own process-count pass already ran bun's prepare once (see below) + fs.writeFileSync(s.commandLog, ""); + fs.rmSync(s.marker("bun"), { force: true }); + + // a failed lockfile install leaves no marker, so the next prepare retries + runPrepare({ BUN_FAIL: "1" }); + assert.equal(fs.existsSync(s.marker("bun")), false); + assert.equal(lockfile(), false); + assert.equal(count(/^npm install -g .*bun@latest$/), 1); + assert.equal(count(/^npm uninstall -g bun$/), 1); + + // first successful prepare: PM installed, lockfile written, then PM removed + runPrepare(); + assert.equal(fs.existsSync(s.marker("bun")), true); + assert.equal(lockfile(), `lockfile for ${s.fixture}\n`); + assert.equal(fs.existsSync(path.join(s.fixture, "node_modules")), false); + assert.equal(count(/^npm install -g .*bun@latest$/), 2); + assert.equal(count(/^bun install --ignore-scripts --silent$/), 2); + assert.equal(count(/^npm uninstall -g bun$/), 2); + + // what a timed run leaves behind + fs.mkdirSync(path.join(s.fixture, "node_modules/b"), { recursive: true }); + + // second prepare: no install, lockfile untouched, node_modules and the PM gone + runPrepare(); + assert.equal(lockfile(), `lockfile for ${s.fixture}\n`); + assert.equal(fs.existsSync(path.join(s.fixture, "node_modules")), false); + assert.equal(count(/^npm install -g .*bun@latest$/), 2); + assert.equal(count(/^bun install --ignore-scripts --silent$/), 2); + assert.equal(count(/^npm uninstall -g bun$/), 3); +}); + +test("ci+setup: the process-count pass drops the marker so its prepare writes the lockfile again", (t) => { + const s = setup(t, { include: "bun" }); + const r = s.run(); + assert.equal(r.status, 0, r.stderr); + // hyperfine is a stub, so the only prepare that ran is process-count.sh's: + // a full first prepare, strace'd install, then clean_all + const commands = s.commands(); + assert.deepEqual( + commands.filter((c) => /^(npm (un)?install|bun install)/.test(c)), + [ + "npm install -g --cache " + path.join(s.dir, "tmp", "bench-tool-setup-cache") + " --allow-scripts=bun bun@latest", + "bun install --ignore-scripts --silent", + "npm uninstall -g bun", + ], + ); + assert.ok(commands.some((c) => c.startsWith("strace ")), "the counted install ran under strace"); + assert.equal(fs.existsSync(s.marker("bun")), true); + assert.equal(fs.existsSync(path.join(s.fixture, "bun.lock")), false, "clean_all ran last"); +}); diff --git a/suite/scripts/variations/ci+setup.sh b/suite/scripts/variations/ci+setup.sh index b5dd824549..cbea9a9603 100644 --- a/suite/scripts/variations/ci+setup.sh +++ b/suite/scripts/variations/ci+setup.sh @@ -10,20 +10,30 @@ source "$1/variations/common.sh" # the tool from scratch and running a CI install. # # Prepare step (not measured): +# Once per tool, on its first prepare (see ci_setup_prepare): # 1. Clean node_modules, lockfiles, PM files # 2. Install the PM binary (needed to generate the lockfile) # 3. Run a regular install to generate the lockfile + populate cache +# Before every timed run, this first one included: # 4. Clean node_modules (ci starts from scratch) # 5. Uninstall the PM binary (so the measured step re-installs it) +# The lockfile and the dependency cache stay from one run to the next: only +# the timed command's own work (the PM download and install, node_modules) +# is undone. hyperfine's --conclude must leave them too. hyperfine 1.19 has +# no per-command --setup, hence the marker file. # # Measured command: # 1. Install the PM binary # 2. Run the PM's ci/frozen-lockfile command +# Where the PM prefixes, their npm caches and the corepack home live. Tests +# point it somewhere private. +BENCH_SETUP_TMP="${BENCH_SETUP_TMP:-/tmp}" + # Use an isolated corepack home so purging it between runs doesn't affect # other benchmarks, and so the path is deterministic (no reliance on # `corepack cache path` which is not a documented subcommand). -export COREPACK_HOME="/tmp/bench-corepack-home" +export COREPACK_HOME="$BENCH_SETUP_TMP/bench-corepack-home" # --- PM install commands --- # These mirror what scripts/setup.sh does for each PM. @@ -33,14 +43,14 @@ export COREPACK_HOME="/tmp/bench-corepack-home" # timed install downloads the package manager again. This also keeps the # bootstrap npm and the copies installed by setup.sh untouched, and the npm # cache used by `npm ci` separate. -NPM_SETUP_PREFIX="/tmp/bench-npm" -NPM_SETUP_CACHE="/tmp/bench-npm-cache" -YARN_SETUP_PREFIX="/tmp/bench-yarn" -YARN_SETUP_CACHE="/tmp/bench-yarn-npm-cache" -PNPM_SETUP_PREFIX="/tmp/bench-pnpm" -PNPM_SETUP_CACHE="/tmp/bench-pnpm-npm-cache" -COREPACK_SETUP_PREFIX="/tmp/bench-corepack" -COREPACK_SETUP_CACHE="/tmp/bench-corepack-npm-cache" +NPM_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-npm" +NPM_SETUP_CACHE="$BENCH_SETUP_TMP/bench-npm-cache" +YARN_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-yarn" +YARN_SETUP_CACHE="$BENCH_SETUP_TMP/bench-yarn-npm-cache" +PNPM_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-pnpm" +PNPM_SETUP_CACHE="$BENCH_SETUP_TMP/bench-pnpm-npm-cache" +COREPACK_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-corepack" +COREPACK_SETUP_CACHE="$BENCH_SETUP_TMP/bench-corepack-npm-cache" SETUP_INSTALL_NPM="npm install -g npm@latest --prefix $NPM_SETUP_PREFIX --cache $NPM_SETUP_CACHE >/dev/null 2>&1" SETUP_INSTALL_YARN="npm install -g yarn@1 --prefix $YARN_SETUP_PREFIX --cache $YARN_SETUP_CACHE >/dev/null 2>&1" # berry: install corepack; its yarn shim then downloads the pinned yarn on @@ -53,7 +63,7 @@ SETUP_INSTALL_PNPM="npm install -g --allow-scripts=pnpm pnpm@latest --prefix $PN # The tools installed with npm get their own npm cache, deleted with the tool # below, like npm's. Otherwise the timed install is served from ~/.npm, which # the prepare's own install of the same tool just filled. -TOOL_SETUP_CACHE="/tmp/bench-tool-setup-cache" +TOOL_SETUP_CACHE="$BENCH_SETUP_TMP/bench-tool-setup-cache" SETUP_INSTALL_VLT="npm install -g --cache $TOOL_SETUP_CACHE vlt@latest >/dev/null 2>&1" SETUP_INSTALL_BUN="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=bun bun@latest >/dev/null 2>&1" SETUP_INSTALL_DENO="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=deno deno@latest >/dev/null 2>&1" @@ -142,32 +152,54 @@ BENCH_INSTALL_UPM="bash -c '$BENCH_COMBINED_UPM'" # --- Prepare commands --- # Like the ci variation, prepare generates a lockfile. Additionally, it # uninstalls the PM binary so the measured command includes re-installing it. +# The lockfile install runs once per tool, not before every timed run: it +# cost an extra full install and two deletions of node_modules per run +# (each about 5 minutes for zpm's 13 GB on babylon). --conclude keeps the +# lockfile, the packageManager pin and the PM config it wrote (the state a +# CI job checks out), so every later prepare only undoes the timed command. BENCH_CLEAN_PRE="bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_lockfiles clean_package_manager_files clean_package_manager_field" BENCH_CLEAN_POST="bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_package_manager_files" -# Helper: build per-PM prepare that installs PM, generates lockfile, cleans -# node_modules, then uninstalls the PM binary. +# The marker that says a tool's lockfile and cache are in place. Lives in the +# results folder, which common.sh empties at the start of every run. +ci_setup_marker() { + echo "$BENCH_OUTPUT_FOLDER/.ci-setup-ready-$1" +} + +# Helper: build per-PM prepare. Its first run (no marker yet) cleans what the +# previous tool left, installs the PM, runs the PM-specific setup and a +# regular install that writes the lockfile and fills the cache. Every run, +# that first one included, then cleans node_modules and uninstalls the PM +# binary. A failed lockfile install leaves no marker, so the next prepare +# tries again, as it did when every prepare ran the install. ci_setup_prepare() { - local pm_install="$1" # command to install the PM binary - local setup_cmd="$2" # PM-specific setup (e.g. .yarnrc.yml) - local install_cmd="$3" # regular install to generate lockfile - local pm_uninstall="$4" # command to uninstall the PM binary - local corepack_purge="$5" # optional: purge corepack cache + local pm="$1" # tool name (marker) + local pm_install="$2" # command to install the PM binary + local setup_cmd="$3" # PM-specific setup (e.g. .yarnrc.yml) + local install_cmd="$4" # regular install to generate lockfile + local pm_uninstall="$5" # command to uninstall the PM binary + local corepack_purge="$6" # optional: purge corepack cache - local result="sleep 1; $BENCH_CLEAN_PRE" + local marker + marker="$(ci_setup_marker "$pm")" + + # Once per tool: lockfile + cache + local once="$BENCH_CLEAN_PRE" # Install PM binary if [ -n "$pm_install" ]; then - result="$result; $pm_install" + once="$once; $pm_install" fi # PM-specific setup if [ -n "$setup_cmd" ]; then - result="$result; $setup_cmd" + once="$once; $setup_cmd" fi # Generate lockfile via regular install - result="$result; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 || true" + once="$once; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 && touch $marker || true" + + local result="sleep 1; if [ ! -e $marker ]; then $once; fi" # Clean node_modules (ci starts fresh) result="$result; $BENCH_CLEAN_POST" @@ -192,16 +224,16 @@ if [ "$BENCH_FIXTURE" = "large" ]; then _INSTALL_NPM="$_INSTALL_NPM --legacy-peer-deps" fi -BENCH_PREPARE_NPM="$(ci_setup_prepare "$SETUP_INSTALL_NPM" "$BENCH_SETUP_NPM" "$_INSTALL_NPM" "$SETUP_UNINSTALL_NPM" "")" -BENCH_PREPARE_YARN="$(ci_setup_prepare "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "$YARN_SETUP_PREFIX/bin/yarn install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "")" -BENCH_PREPARE_BERRY="$(ci_setup_prepare "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "$COREPACK_SETUP_PREFIX/bin/yarn install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" -BENCH_PREPARE_ZPM="$(ci_setup_prepare "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "$BENCH_ZPM_BIN install --silent" "$SETUP_UNINSTALL_ZPM" "")" -BENCH_PREPARE_PNPM="$(ci_setup_prepare "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "$PNPM_SETUP_PREFIX/bin/pnpm install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "")" -BENCH_PREPARE_VLT="$(ci_setup_prepare "$SETUP_INSTALL_VLT" "$BENCH_SETUP_VLT" "vlt install --view=silent" "$SETUP_UNINSTALL_VLT" "")" -BENCH_PREPARE_BUN="$(ci_setup_prepare "$SETUP_INSTALL_BUN" "$BENCH_SETUP_BUN" "bun install --ignore-scripts --silent" "$SETUP_UNINSTALL_BUN" "")" -BENCH_PREPARE_DENO="$(ci_setup_prepare "$SETUP_INSTALL_DENO" "$BENCH_SETUP_DENO" "deno install --quiet" "$SETUP_UNINSTALL_DENO" "")" -BENCH_PREPARE_AUBE="$(ci_setup_prepare "$SETUP_INSTALL_AUBE" "$BENCH_SETUP_AUBE" "aube install --ignore-scripts --silent" "$SETUP_UNINSTALL_AUBE" "")" -BENCH_PREPARE_UPM="$(ci_setup_prepare "$SETUP_INSTALL_UPM" "$BENCH_SETUP_UPM" "upm install --silent" "$SETUP_UNINSTALL_UPM" "")" +BENCH_PREPARE_NPM="$(ci_setup_prepare npm "$SETUP_INSTALL_NPM" "$BENCH_SETUP_NPM" "$_INSTALL_NPM" "$SETUP_UNINSTALL_NPM" "")" +BENCH_PREPARE_YARN="$(ci_setup_prepare yarn "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "$YARN_SETUP_PREFIX/bin/yarn install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "")" +BENCH_PREPARE_BERRY="$(ci_setup_prepare berry "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "$COREPACK_SETUP_PREFIX/bin/yarn install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" +BENCH_PREPARE_ZPM="$(ci_setup_prepare zpm "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "$BENCH_ZPM_BIN install --silent" "$SETUP_UNINSTALL_ZPM" "")" +BENCH_PREPARE_PNPM="$(ci_setup_prepare pnpm "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "$PNPM_SETUP_PREFIX/bin/pnpm install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "")" +BENCH_PREPARE_VLT="$(ci_setup_prepare vlt "$SETUP_INSTALL_VLT" "$BENCH_SETUP_VLT" "vlt install --view=silent" "$SETUP_UNINSTALL_VLT" "")" +BENCH_PREPARE_BUN="$(ci_setup_prepare bun "$SETUP_INSTALL_BUN" "$BENCH_SETUP_BUN" "bun install --ignore-scripts --silent" "$SETUP_UNINSTALL_BUN" "")" +BENCH_PREPARE_DENO="$(ci_setup_prepare deno "$SETUP_INSTALL_DENO" "$BENCH_SETUP_DENO" "deno install --quiet" "$SETUP_UNINSTALL_DENO" "")" +BENCH_PREPARE_AUBE="$(ci_setup_prepare aube "$SETUP_INSTALL_AUBE" "$BENCH_SETUP_AUBE" "aube install --ignore-scripts --silent" "$SETUP_UNINSTALL_AUBE" "")" +BENCH_PREPARE_UPM="$(ci_setup_prepare upm "$SETUP_INSTALL_UPM" "$BENCH_SETUP_UPM" "upm install --silent" "$SETUP_UNINSTALL_UPM" "")" # Run the benchmark suite hyperfine --ignore-failure \ @@ -209,7 +241,7 @@ hyperfine --ignore-failure \ --export-json="$BENCH_OUTPUT_FOLDER/benchmarks.json" \ --warmup="$BENCH_WARMUP" \ --runs="$BENCH_RUNS" \ - --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_lockfiles clean_package_manager_files clean_package_manager_field clean_build_files" \ + --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_package_manager_files clean_build_files" \ --cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \ ${BENCH_INCLUDE_NPM:+--prepare="$BENCH_PREPARE_NPM"} \ ${BENCH_INCLUDE_NPM:+--command-name="npm" "$BENCH_COMMAND_NPM"} \ @@ -236,5 +268,12 @@ collect_package_count # --- Process count collection --- # Each PM's prepare includes PM-specific uninstall/cache-purge steps and its -# own lockfile, so each gets its own prepare. +# own lockfile, so each gets its own prepare. hyperfine's --cleanup removed +# every lockfile and cache, so drop the markers: each counted install then +# starts from what its first prepare makes, the state the timed runs saw. +for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do + P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') + prepare_var="BENCH_PREPARE_$P" + printf -v "$prepare_var" '%s' "rm -f $(ci_setup_marker "$pm"); ${!prepare_var}" +done collect_process_count_per_pm