diff --git a/suite/scripts/clean-helpers.sh b/suite/scripts/clean-helpers.sh index 978676e19c..b16e182e8c 100644 --- a/suite/scripts/clean-helpers.sh +++ b/suite/scripts/clean-helpers.sh @@ -134,6 +134,39 @@ clean_deno_cache() { fi } +# Deno keeps compiled code for the CommonJS files it runs in +# $DENO_DIR/v8_code_cache_v2 (a SQLite database, with -wal/-shm sidecars). +# When a build writes a file with the same path and size as the previous +# build but different content (`next build` emits `.next/server/chunks/961.js` +# in two such variants), Deno runs the old compiled code against the new file +# and webpack's runtime throws `TypeError: e[o] is not a function`. A failed +# run doesn't refresh the entry, so it keeps failing until the variant flips +# back. Only the entries for the regenerated files can go stale, so remove just +# those for the current project's .next/ (deno-code-cache.js) and keep the rest: +# the entries for next's own files in node_modules are what a user's build runs +# with. If the database isn't shaped as expected, remove the whole code cache +# instead; the npm cache and the rest of $DENO_DIR always stay. (#215) +# +# `deno info --json` reports the directory wherever it is (DENO_DIR, XDG_CACHE_HOME +# or the platform default); fall back to the Linux default if it can't be asked. +clean_deno_code_cache() { + local deno_dir="" entry + if command -v deno &> /dev/null; then + deno_dir="$(deno info --json 2>/dev/null | sed -n 's/^ *"denoDir": *"\(.*\)",\{0,1\}$/\1/p' || true)" + fi + if [ -z "$deno_dir" ]; then + deno_dir="${DENO_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/deno}" + fi + [ -e "$deno_dir/v8_code_cache_v2" ] || return 0 + if node "$(dirname "${BASH_SOURCE[0]}")/deno-code-cache.js" "$deno_dir/v8_code_cache_v2" "$PWD"; then + return 0 + fi + echo "Warning: couldn't prune Deno's code cache for $PWD/.next, removing all of it" + for entry in "$deno_dir"/v8_code_cache_v2*; do + safe_remove "$entry" + done +} + # Function to safely clean vp cache clean_vp_cache() { if command -v vp &> /dev/null; then @@ -322,6 +355,7 @@ show_help() { echo " clean_bun_cache" echo " clean_nx_cache" echo " clean_deno_cache" + echo " clean_deno_code_cache" echo " clean_aube_cache" echo " clean_aube_metadata_cache" echo " clean_upm_cache" @@ -374,6 +408,9 @@ else clean_deno_cache) clean_deno_cache ;; + clean_deno_code_cache) + clean_deno_code_cache + ;; clean_aube_cache) clean_aube_cache ;; diff --git a/suite/scripts/clean-helpers.test.js b/suite/scripts/clean-helpers.test.js index 1d8fc9dfbb..7400f9e369 100644 --- a/suite/scripts/clean-helpers.test.js +++ b/suite/scripts/clean-helpers.test.js @@ -2,6 +2,7 @@ const assert = require("node:assert/strict"); const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); +const { pathToFileURL } = require("node:url"); const { execFileSync, spawnSync } = require("node:child_process"); const test = require("node:test"); @@ -281,6 +282,129 @@ test("clean_berry_cache removes berry's global folder, with and without XDG_DATA ]); }); +// Run clean_deno_code_cache from `cwd` against a fake deno dir (reported by a +// fake `deno info --json`, or found through env when `denoOnPath` is false) +// and record what it removes and what is left in the code cache database. +const { DatabaseSync } = require("node:sqlite"); +const CODECACHE_SCHEMA = + "CREATE TABLE codecache (specifier TEXT NOT NULL, type INTEGER NOT NULL, source_hash INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY (specifier, type))"; +const runDenoCodeCacheClean = ({ denoDir, denoOnPath = true, schema, rows = [], env = {} }) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "deno-clean-test-")); + try { + const binDir = path.join(tempDir, "bin"); + const cwd = path.join(tempDir, "my project"); + const removeLog = path.join(tempDir, "removed.txt"); + fs.mkdirSync(binDir); + fs.mkdirSync(cwd); + fs.writeFileSync(removeLog, ""); + if (denoOnPath) { + fs.writeFileSync( + path.join(binDir, "deno"), + `#!/bin/sh\n[ "$1 $2" = "info --json" ] || exit 1\nprintf '{\\n "version": 1,\\n "denoDir": "%s",\\n "npmCache": "%s/npm"\\n}\\n' "$FAKE_DENO_DIR" "$FAKE_DENO_DIR"\n`, + { mode: 0o755 }, + ); + } + const dbPath = path.join(denoDir, "v8_code_cache_v2"); + if (schema) { + // Like Deno's: a WAL-mode database, with the npm cache next to it. + fs.mkdirSync(path.join(denoDir, "npm"), { recursive: true }); + const db = new DatabaseSync(dbPath); + db.exec("PRAGMA journal_mode = WAL"); + db.exec(schema); + for (const [specifier, type] of rows.map((r) => r.replace(/^/, pathToFileURL(cwd).href).split(" "))) { + db.prepare("INSERT INTO codecache VALUES (?, ?, ?, ?)").run(specifier, Number(type), 1, Buffer.from("code")); + } + db.close(); + } + const result = spawnSync( + "bash", + ["-c", `source "$HELPERS" >/dev/null; safe_remove() { printf '%s\\n' "$1" >> "$REMOVE_LOG"; }; clean_deno_code_cache`], + { + encoding: "utf8", + cwd, + env: { + // Only the fake bin dir, this node and sed, so a real deno is never asked. + PATH: [binDir, path.dirname(process.execPath), "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(tempDir, "home"), + HELPERS: path.join(__dirname, "clean-helpers.sh"), + REMOVE_LOG: removeLog, + FAKE_DENO_DIR: denoDir, + ...env, + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + let left = null; + if (fs.existsSync(dbPath) && schema === CODECACHE_SCHEMA) { + const db = new DatabaseSync(dbPath, { readOnly: true }); + left = db + .prepare("SELECT specifier, type FROM codecache ORDER BY specifier, type") + .all() + .map((r) => `${r.specifier.replace(pathToFileURL(cwd).href, "")} ${r.type}`) + .sort(); + db.close(); + } + return { + removed: fs.readFileSync(removeLog, "utf8").trim().split("\n").filter(Boolean), + left, + stdout: result.stdout.replaceAll(cwd, ""), + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + fs.rmSync(denoDir, { recursive: true, force: true }); + } +}; +const tempDenoDir = () => path.join(os.tmpdir(), `deno dir ${process.pid} ${Math.random().toString(36).slice(2)}`); + +test("clean_deno_code_cache removes only the entries under the project's .next/", () => { + const { removed, left, stdout } = runDenoCodeCacheClean({ + denoDir: tempDenoDir(), + schema: CODECACHE_SCHEMA, + rows: [ + "/.next/server/chunks/961.js 0", + "/.next/server/chunks/961.js 1", + "/.next/server/webpack-runtime.js 1", + "/node_modules/next/dist/build/index.js 1", + "/.next-other/x.js 1", + "file:///other%20project/.next/server/a.js 1", + ], + }); + assert.deepEqual(removed, [], "fell back to removing the whole cache"); + assert.deepEqual(left, [ + "/.next-other/x.js 1", + "/node_modules/next/dist/build/index.js 1", + "file:///other%20project/.next/server/a.js 1", + ]); + assert.match(stdout, /removed 3 of 6 entries under \/\.next, kept 3/); +}); + +test("clean_deno_code_cache removes the whole code cache when the schema is not the expected one", () => { + const denoDir = tempDenoDir(); + const { removed, stdout } = runDenoCodeCacheClean({ + denoDir, + schema: "CREATE TABLE codecache (key TEXT PRIMARY KEY, data BLOB NOT NULL)", + }); + // Closing the test database checkpointed its -wal and -shm away; the glob + // covers them when Deno leaves them behind. + assert.deepEqual(removed, [path.join(denoDir, "v8_code_cache_v2")]); + assert.match(stdout, /Warning: couldn't prune Deno's code cache/); +}); + +test("clean_deno_code_cache does nothing without a code cache", () => { + assert.deepEqual(runDenoCodeCacheClean({ denoDir: tempDenoDir() }).removed, []); +}); + +test("clean_deno_code_cache finds the cache through DENO_DIR or XDG_CACHE_HOME when deno isn't on PATH", () => { + const badSchema = "CREATE TABLE codecache (key TEXT PRIMARY KEY, data BLOB NOT NULL)"; + for (const [denoDir, env] of [ + [tempDenoDir(), (d) => ({ DENO_DIR: d })], + [path.join(tempDenoDir(), "deno"), (d) => ({ XDG_CACHE_HOME: path.dirname(d) })], + ]) { + const { removed } = runDenoCodeCacheClean({ denoDir, denoOnPath: false, schema: badSchema, env: env(denoDir) }); + assert.deepEqual(removed, [path.join(denoDir, "v8_code_cache_v2")]); + } +}); + test("clean_nx_cache does nothing outside an Nx workspace", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nx-clean-test-")); try { diff --git a/suite/scripts/deno-code-cache.js b/suite/scripts/deno-code-cache.js new file mode 100644 index 0000000000..ce0e94b8cc --- /dev/null +++ b/suite/scripts/deno-code-cache.js @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// Remove the entries for one project's `.next/` output from Deno's compiled-code +// cache, and keep everything else (next's own files in node_modules never go +// stale). Called by clean_deno_code_cache in clean-helpers.sh: +// +// node deno-code-cache.js +// +// The cache is a SQLite database in WAL mode with one table, +// codecache (specifier TEXT, type INTEGER, source_hash INTEGER, data BLOB, +// PRIMARY KEY (specifier, type)) +// where specifier is the module's file:// URL. Prints what it removed. Exits 1, +// with the reason on stderr, when the database isn't shaped like that, so the +// caller can fall back to removing the whole cache. +const fs = require("node:fs"); +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); +const { DatabaseSync } = require("node:sqlite"); + +const EXPECTED_COLUMNS = ["specifier", "type", "source_hash", "data"]; + +const main = (dbPath, projectDir) => { + if (!dbPath || !projectDir) { + throw new Error("usage: deno-code-cache.js "); + } + if (!fs.existsSync(dbPath)) { + return; + } + const db = new DatabaseSync(dbPath); + try { + const columns = db + .prepare("SELECT name FROM pragma_table_info('codecache')") + .all() + .map((row) => row.name); + if (columns.join() !== EXPECTED_COLUMNS.join()) { + throw new Error(`unexpected codecache columns: ${JSON.stringify(columns)}`); + } + const total = db.prepare("SELECT count(*) AS n FROM codecache").get().n; + let removed = 0; + const remove = db.prepare("DELETE FROM codecache WHERE specifier LIKE ? ESCAPE '\\'"); + for (const prefix of nextOutputUrls(projectDir)) { + removed += remove.run(`${prefix.replace(/[\\%_]/g, "\\$&")}%`).changes; + } + console.log( + `deno code cache: removed ${removed} of ${total} entries under ${path.join(projectDir, ".next")}, kept ${total - removed}`, + ); + } finally { + db.close(); + } +}; + +// file:// URL prefixes of the project's .next/ directory: as given and with +// symlinks resolved, since Deno records the path it loaded the file from. +const nextOutputUrls = (projectDir) => { + const dirs = new Set([path.resolve(projectDir)]); + try { + dirs.add(fs.realpathSync(projectDir)); + } catch {} + return [...dirs].map((dir) => `${pathToFileURL(path.join(dir, ".next")).href}/`); +}; + +try { + main(process.argv[2], process.argv[3]); +} catch (err) { + console.error(`deno-code-cache.js: ${err.message}`); + process.exit(1); +} diff --git a/suite/scripts/variations/build-cache.sh b/suite/scripts/variations/build-cache.sh index 244f47452d..a684145cde 100644 --- a/suite/scripts/variations/build-cache.sh +++ b/suite/scripts/variations/build-cache.sh @@ -65,6 +65,12 @@ BENCH_COMMAND_NODE="timeout $BENCH_TIMEOUT node --run build > $BENCH_OUTPUT_FOLD # Run the benchmark build-cache benchmarks. # Prepare: install deps + run a warm build (populates .next/ cache). # Timed command: rebuild with .next/ cache present. +# deno's prepare drops the .next/ entries from Deno's compiled-code cache twice: +# before the warm build, so it can't fail on a stale entry from the previous +# iteration and leave a half-written .next/, and again right before the timed +# rebuild, in case the rebuild emits the other variant of a server chunk than +# its warm build did (see clean_deno_code_cache). The entries for next's own +# code stay, so the rebuild runs with the warm compile cache a user's would. hyperfine --ignore-failure \ --time-unit=millisecond \ --export-json="$BENCH_OUTPUT_FOLDER/benchmarks.json" \ @@ -85,7 +91,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_VLT:+--command-name="vlt" "$BENCH_COMMAND_VLT"} \ ${BENCH_INCLUDE_BUN:+--prepare="$BENCH_INSTALL_PREPARE_BUN || true; bash $BENCH_SCRIPTS/clean-helpers.sh clean_build_output; $BENCH_WARM_BUILD_BUN || true"} \ ${BENCH_INCLUDE_BUN:+--command-name="bun" "$BENCH_COMMAND_BUN"} \ - ${BENCH_INCLUDE_DENO:+--prepare="$BENCH_INSTALL_PREPARE_DENO || true; bash $BENCH_SCRIPTS/clean-helpers.sh clean_build_output; $BENCH_WARM_BUILD_DENO || true"} \ + ${BENCH_INCLUDE_DENO:+--prepare="$BENCH_INSTALL_PREPARE_DENO || true; bash $BENCH_SCRIPTS/clean-helpers.sh clean_deno_code_cache clean_build_output; $BENCH_WARM_BUILD_DENO || true; bash $BENCH_SCRIPTS/clean-helpers.sh clean_deno_code_cache"} \ ${BENCH_INCLUDE_DENO:+--command-name="deno" "$BENCH_COMMAND_DENO"} \ ${BENCH_INCLUDE_AUBE:+--prepare="$BENCH_INSTALL_PREPARE_AUBE || true; bash $BENCH_SCRIPTS/clean-helpers.sh clean_build_output; $BENCH_WARM_BUILD_AUBE || true"} \ ${BENCH_INCLUDE_AUBE:+--command-name="aube" "$BENCH_COMMAND_AUBE"} \ diff --git a/suite/scripts/variations/build.sh b/suite/scripts/variations/build.sh index c0876b78e3..5cd0fe114e 100644 --- a/suite/scripts/variations/build.sh +++ b/suite/scripts/variations/build.sh @@ -52,6 +52,10 @@ BENCH_COMMAND_NODE="timeout $BENCH_TIMEOUT node --run build > $BENCH_OUTPUT_FOLD # The install step is defined as a "prepare" option so that time spent # on the install is not accounted for in the benchmark results. # Before each timed run we also remove .next/ so every run is a cold build. +# deno's prepare also drops the entries for .next/ from Deno's compiled-code +# cache: left in place, they make the build fail whenever `next build` emits the +# other variant of a server chunk (same path and size, different content; see +# clean_deno_code_cache). The entries for next's own code stay. hyperfine --ignore-failure \ --time-unit=millisecond \ --export-json="$BENCH_OUTPUT_FOLDER/benchmarks.json" \ @@ -72,7 +76,7 @@ hyperfine --ignore-failure \ ${BENCH_INCLUDE_VLT:+--command-name="vlt" "$BENCH_COMMAND_VLT"} \ ${BENCH_INCLUDE_BUN:+--prepare="$BENCH_INSTALL_PREPARE_BUN; bash $BENCH_SCRIPTS/clean-helpers.sh clean_build_output"} \ ${BENCH_INCLUDE_BUN:+--command-name="bun" "$BENCH_COMMAND_BUN"} \ - ${BENCH_INCLUDE_DENO:+--prepare="$BENCH_INSTALL_PREPARE_DENO; bash $BENCH_SCRIPTS/clean-helpers.sh clean_build_output"} \ + ${BENCH_INCLUDE_DENO:+--prepare="$BENCH_INSTALL_PREPARE_DENO; bash $BENCH_SCRIPTS/clean-helpers.sh clean_deno_code_cache clean_build_output"} \ ${BENCH_INCLUDE_DENO:+--command-name="deno" "$BENCH_COMMAND_DENO"} \ ${BENCH_INCLUDE_AUBE:+--prepare="$BENCH_INSTALL_PREPARE_AUBE || true; bash $BENCH_SCRIPTS/clean-helpers.sh clean_build_output"} \ ${BENCH_INCLUDE_AUBE:+--command-name="aube" "$BENCH_COMMAND_AUBE"} \