diff --git a/packages/runtime/test/lifecycle-inspection.test.ts b/packages/runtime/test/lifecycle-inspection.test.ts index de98e98ad..d35608069 100644 --- a/packages/runtime/test/lifecycle-inspection.test.ts +++ b/packages/runtime/test/lifecycle-inspection.test.ts @@ -57,8 +57,8 @@ const startRun = (input: Parameters[0]): ReturnType { +function fakeInspectionEnv(project: string, lifetime: "test" | "process" = "test"): Record { const binDir = path.join(path.dirname(project), `${path.basename(project)}-fake-bin`); fs.mkdirSync(binDir, { recursive: true }); - registerTemporaryPath(binDir); + registerTemporaryPath(binDir, lifetime); const files = fakeInspectionFiles(project); const smithers = path.join(binDir, "smithers"); fs.writeFileSync( @@ -332,18 +332,22 @@ function fakeInspectionEnv(project: string): Record } /** A project with the fake runner installed and no run; `diagnoseProject` never reads one. */ -function projectWithFakeRunner(): { project: string; env: Record } { - const project = tempProject(); +function projectWithFakeRunner(lifetime: "test" | "process" = "test"): { + project: string; + env: Record; +} { + const project = tempProject(lifetime); initProject({ projectRoot: project, force: true }); writeSmallTopology(project); - return { project, env: fakeInspectionEnv(project) }; + return { project, env: fakeInspectionEnv(project, lifetime) }; } async function launchedProject( fixtures: FakeInspectionFixtures, - runId = "inspect-run" + runId = "inspect-run", + lifetime: "test" | "process" = "test" ): Promise<{ project: string; env: Record; runRoot: string }> { - const { project, env } = projectWithFakeRunner(); + const { project, env } = projectWithFakeRunner(lifetime); writeInspectionFixtures(project, fixtures); const run = await startRun({ projectRoot: project, @@ -386,7 +390,7 @@ let sharedInspectedRun: async function inspectedRun(fixtures: FakeInspectionFixtures): ReturnType { // A failed launch stays cached, so every later caller reports it as the shared launch failing, not // as its own regression. - sharedInspectedRun ??= launchedProject({}).then( + sharedInspectedRun ??= launchedProject({}, "inspect-run", "process").then( (launched) => ({ ...launched, launchedDocuments: readRunDocuments(launched.runRoot) }), (error: unknown) => { throw new Error("the shared inspection run failed to launch, so this test did not run", { cause: error }); @@ -402,6 +406,18 @@ async function inspectedRun(fixtures: FakeInspectionFixtures): ReturnType { + const { project } = await inspectedRun({}); + assert.equal(fs.existsSync(project), true); +}); + +test("a shared inspection project remains available in the next case", async () => { + const { project, env } = await inspectedRun({}); + assert.equal(fs.existsSync(project), true); + assert.ok(env.SMITHERS_BIN); + assert.equal(fs.existsSync(env.SMITHERS_BIN), true); +}); + function smithersLog(project: string): string { return fs.readFileSync(fakeInspectionFiles(project).commandLog, "utf8"); } diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index de9b302f2..08215e1be 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -176,7 +176,7 @@ const SMITHERS_TEST_ENVIRONMENT_ALLOWLIST = [ "SMITHERS_FAKE_SNAPSHOT_BYTES_LOG" ] as const; const OPENROUTER_TEST_STDERR_PENDING_LIMIT = 64 * 1024; -process.env.ULTRAFUZZ_PROVIDER_HOME_ROOT = temporaryRoot("ufz-provider-homes-"); +process.env.ULTRAFUZZ_PROVIDER_HOME_ROOT = temporaryRoot("ufz-provider-homes-", "process"); function tempProject(): string { return temporaryRoot("ufz-runtime-"); diff --git a/packages/runtime/test/smithers-report-retry.integration.test.ts b/packages/runtime/test/smithers-report-retry.integration.test.ts index 2e93a7b4c..f8f8a6a69 100644 --- a/packages/runtime/test/smithers-report-retry.integration.test.ts +++ b/packages/runtime/test/smithers-report-retry.integration.test.ts @@ -145,7 +145,7 @@ export default smithers(() => ( // detached engine's PATH makes any bare `smithers` call fail this test, even // though `pnpm test` puts node_modules/.bin on PATH and a host may have its own // `smithers`. -const poisonBin = temporaryRoot("ultrafuzz-poison-smithers-"); +const poisonBin = temporaryRoot("ultrafuzz-poison-smithers-", "process"); const poisonSmithers = "#!/bin/sh\necho 'bare smithers resolved from PATH' >&2\nexit 97\n"; fs.writeFileSync(path.join(poisonBin, "smithers"), poisonSmithers, { mode: 0o755 }); const productionPath = `${poisonBin}${path.delimiter}${process.env.PATH ?? ""}`; diff --git a/packages/runtime/test/temporary-root.test.ts b/packages/runtime/test/temporary-root.test.ts index 6abebdb1f..c75ffb7ed 100644 --- a/packages/runtime/test/temporary-root.test.ts +++ b/packages/runtime/test/temporary-root.test.ts @@ -2,15 +2,20 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; +import nodeTest from "node:test"; import { test } from "./runtime-test-shard.js"; import { registerTemporaryPath, temporaryRoot } from "./temporary-root.js"; +const directNodeTest = process.versions.bun === undefined ? nodeTest : nodeTest.skip; + let sealedRoot: string; let adjacentRoot: string; let sharedRoot: string; let outsideHardlink: string; let teardownRoot: string; +let directRoot: string; +let parentRoot: string; test("test-scoped scratch includes sealed snapshots and registered adjacent paths", () => { sealedRoot = temporaryRoot("ufz-test-cleanup-"); @@ -50,6 +55,38 @@ test("scratch is reclaimed after the test's own after hook", () => { assert.equal(fs.existsSync(teardownRoot), false); }); +directNodeTest("direct node:test fixtures remain available to their teardown hooks", (context) => { + directRoot = temporaryRoot("ufz-direct-cleanup-"); + fs.chmodSync(directRoot, 0o500); + context.after(() => assert.equal(fs.existsSync(directRoot), true)); +}); + +directNodeTest("direct node:test fixtures are reclaimed before the next case", () => { + assert.equal(fs.existsSync(directRoot), false); +}); + +directNodeTest("nested tests reclaim only their own scratch", async (context) => { + parentRoot = temporaryRoot("ufz-parent-cleanup-"); + let childRoot: string | undefined; + context.after(() => assert.equal(fs.existsSync(parentRoot), true)); + await context.test("nested child", (childContext) => { + const root = temporaryRoot("ufz-child-cleanup-"); + childRoot = root; + assert.equal(fs.existsSync(parentRoot), true); + childContext.after(() => { + assert.equal(fs.existsSync(root), true); + assert.equal(fs.existsSync(parentRoot), true); + }); + }); + assert.ok(childRoot); + assert.equal(fs.existsSync(childRoot), false); + assert.equal(fs.existsSync(parentRoot), true); +}); + +directNodeTest("the parent fixture is reclaimed after its nested tests", () => { + assert.equal(fs.existsSync(parentRoot), false); +}); + test("scratch is reclaimed after a failed test body", () => { const fixture = temporaryRoot("ufz-failed-test-"); const marker = path.join(fixture, "created-root.txt"); @@ -80,3 +117,116 @@ test("scratch is reclaimed after a failed test body", () => { assert.equal(fs.readFileSync(verified, "utf8"), "yes", `${child.stdout}${child.stderr}`); assert.equal(fs.existsSync(fs.readFileSync(marker, "utf8")), false); }); + +directNodeTest("a failed direct node:test case releases scratch before the next case", () => { + const fixture = temporaryRoot("ufz-failed-direct-"); + const marker = path.join(fixture, "created-root.txt"); + const verified = path.join(fixture, "verified.txt"); + const childTest = path.join(fixture, "failure.test.mjs"); + fs.writeFileSync( + childTest, + [ + 'import assert from "node:assert/strict";', + 'import fs from "node:fs";', + 'import test from "node:test";', + `import { temporaryRoot } from ${JSON.stringify(new URL("./temporary-root.js", import.meta.url).href)};`, + 'test("deliberate direct failure", () => {', + ` fs.writeFileSync(${JSON.stringify(marker)}, temporaryRoot("ufz-failing-direct-body-"));`, + ' throw new Error("deliberate direct failure sentinel");', + "});", + 'test("cleanup follows failure", () => {', + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(marker)}, "utf8")), false);`, + ` fs.writeFileSync(${JSON.stringify(verified)}, "yes");`, + "});" + ].join("\n") + ); + const child = spawnSync(process.execPath, [childTest], { + encoding: "utf8" + }); + assert.equal(child.status, 1, `${child.stdout}${child.stderr}`); + assert.match(`${child.stdout}${child.stderr}`, /deliberate direct failure sentinel/u); + assert.equal(fs.readFileSync(verified, "utf8"), "yes", `${child.stdout}${child.stderr}`); +}); + +directNodeTest("a failed direct teardown releases scratch before the next case", () => { + const fixture = temporaryRoot("ufz-failed-teardown-"); + const marker = path.join(fixture, "root.txt"); + const afterMarker = path.join(fixture, "after-root.txt"); + const verified = path.join(fixture, "verified.txt"); + const childTest = path.join(fixture, "failure.test.mjs"); + fs.writeFileSync( + childTest, + [ + 'import assert from "node:assert/strict";', + 'import fs from "node:fs";', + 'import test from "node:test";', + `import { temporaryRoot } from ${JSON.stringify(new URL("./temporary-root.js", import.meta.url).href)};`, + 'test("teardown fails", (context) => {', + ` fs.writeFileSync(${JSON.stringify(marker)}, temporaryRoot("ufz-teardown-body-"));`, + " context.after(() => {", + ` fs.writeFileSync(${JSON.stringify(afterMarker)}, temporaryRoot("ufz-teardown-after-"));`, + ' throw new Error("teardown sentinel");', + " });", + "});", + 'test("following case", () => {', + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(marker)}, "utf8")), false);`, + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(afterMarker)}, "utf8")), false);`, + ` fs.writeFileSync(${JSON.stringify(verified)}, "yes");`, + "});" + ].join("\n") + ); + const child = spawnSync(process.execPath, [childTest], { encoding: "utf8" }); + assert.equal(child.status, 1, `${child.stdout}${child.stderr}`); + assert.match(`${child.stdout}${child.stderr}`, /teardown sentinel/u); + assert.equal(fs.readFileSync(verified, "utf8"), "yes", `${child.stdout}${child.stderr}`); +}); + +directNodeTest("a failed nested teardown releases only the child scope before parent teardown", () => { + const fixture = temporaryRoot("ufz-failed-nested-teardown-"); + const childMarker = path.join(fixture, "child-root.txt"); + const childAfterMarker = path.join(fixture, "child-after-root.txt"); + const parentMarker = path.join(fixture, "parent-root.txt"); + const resumedMarker = path.join(fixture, "resumed-parent-root.txt"); + const parentAfter = path.join(fixture, "parent-after.txt"); + const verified = path.join(fixture, "verified.txt"); + const childTest = path.join(fixture, "failure.test.mjs"); + fs.writeFileSync( + childTest, + [ + 'import assert from "node:assert/strict";', + 'import fs from "node:fs";', + 'import test from "node:test";', + `import { temporaryRoot } from ${JSON.stringify(new URL("./temporary-root.js", import.meta.url).href)};`, + 'test("parent", async (context) => {', + ' const parent = temporaryRoot("ufz-parent-teardown-");', + ` fs.writeFileSync(${JSON.stringify(parentMarker)}, parent);`, + " context.after(() => {", + " assert.equal(fs.existsSync(parent), true);", + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(childMarker)}, "utf8")), false);`, + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(childAfterMarker)}, "utf8")), false);`, + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(resumedMarker)}, "utf8")), true);`, + ` fs.writeFileSync(${JSON.stringify(parentAfter)}, "yes");`, + " });", + ' await context.test("child", (childContext) => {', + ` fs.writeFileSync(${JSON.stringify(childMarker)}, temporaryRoot("ufz-child-teardown-"));`, + " childContext.after(async () => {", + " await Promise.resolve();", + ` fs.writeFileSync(${JSON.stringify(childAfterMarker)}, temporaryRoot("ufz-child-after-"));`, + ' throw new Error("nested teardown sentinel");', + " });", + " });", + ` fs.writeFileSync(${JSON.stringify(resumedMarker)}, temporaryRoot("ufz-resumed-parent-"));`, + "});", + 'test("following case", () => {', + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(parentMarker)}, "utf8")), false);`, + ` assert.equal(fs.existsSync(fs.readFileSync(${JSON.stringify(resumedMarker)}, "utf8")), false);`, + ` assert.equal(fs.readFileSync(${JSON.stringify(parentAfter)}, "utf8"), "yes");`, + ` fs.writeFileSync(${JSON.stringify(verified)}, "yes");`, + "});" + ].join("\n") + ); + const child = spawnSync(process.execPath, [childTest], { encoding: "utf8" }); + assert.equal(child.status, 1, `${child.stdout}${child.stderr}`); + assert.match(`${child.stdout}${child.stderr}`, /nested teardown sentinel/u); + assert.equal(fs.readFileSync(verified, "utf8"), "yes", `${child.stdout}${child.stderr}`); +}); diff --git a/packages/runtime/test/temporary-root.ts b/packages/runtime/test/temporary-root.ts index b6980aae9..3bea4e000 100644 --- a/packages/runtime/test/temporary-root.ts +++ b/packages/runtime/test/temporary-root.ts @@ -2,7 +2,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import type { TestContext } from "node:test"; +import { afterEach, beforeEach, type TestContext } from "node:test"; /** * Create a temporary directory for a test and remove it after that test. @@ -24,8 +24,77 @@ import type { TestContext } from "node:test"; */ const registered = new Set(); const testRoots = new AsyncLocalStorage(); +type DirectTestScope = { roots: string[]; pending: boolean }; +const directTestScopes: DirectTestScope[] = []; +const directTestContextScopes = new WeakMap(); let exitHookInstalled = false; +function drainPendingDirectTestScopes(): void { + while (directTestScopes.at(-1)?.pending) { + const scope = directTestScopes.pop(); + if (scope !== undefined) removeRoots(scope.roots); + } +} + +function currentDirectTestRoots(): string[] | undefined { + // A child with a failed teardown remains pending while its parent body may + // resume. New parent fixtures belong to the nearest still-active ancestor. + for (let index = directTestScopes.length - 1; index >= 0; index -= 1) { + const scope = directTestScopes[index]; + if (scope !== undefined && !scope.pending) return scope.roots; + } + return directTestScopes.at(-1)?.roots; +} + +function scopeDirectAfterHooks(context: TestContext, scope: DirectTestScope): void { + const registerAfter = context.after.bind(context); + context.after = (callback, options) => { + if (callback === undefined) return registerAfter(undefined, options); + if (callback.length >= 2) { + return registerAfter( + (hookContext, done) => testRoots.run(scope.roots, () => callback(hookContext, done)), + options + ); + } + return registerAfter((hookContext) => testRoots.run(scope.roots, () => callback(hookContext, () => {})), options); + }; +} + +// Supporting files run under Node's test runner, which isolates each file. +// Bun caches this helper across files, so hooks registered by one import would +// not cover direct tests in later files. Bun's runtime lane uses the scoped +// wrapper below instead. Node's beforeEach async context does not reach the +// test body, so track serial direct tests until their after hooks finish. +if (process.versions.bun === undefined) { + beforeEach((context) => { + // An earlier teardown hook can throw before our late context.after runs. + drainPendingDirectTestScopes(); + const scope = { roots: [], pending: false }; + scopeDirectAfterHooks(context as TestContext, scope); + directTestContextScopes.set(context, scope); + directTestScopes.push(scope); + }); + + afterEach((context) => { + // A nested child can leave its pending scope above the active parent. + drainPendingDirectTestScopes(); + const scope = directTestContextScopes.get(context); + if (scope === undefined) return; + if (directTestScopes.at(-1) !== scope) { + throw new Error("Runtime test scratch scopes overlap; run these tests without test concurrency"); + } + if (!("after" in context)) throw new Error("Runtime test scratch cleanup requires a test context"); + scope.pending = true; + context.after(() => { + if (directTestScopes.at(-1) !== scope) { + throw new Error("Runtime test scratch scopes overlap; run these tests without test concurrency"); + } + directTestScopes.pop(); + removeRoots(scope.roots); + }); + }); +} + /** Attach roots created by a test body to its after hook, including on assertion failure. */ export function withTemporaryRootScope(context: TestContext, body: () => void | Promise): void | Promise { const roots: string[] = []; @@ -80,7 +149,7 @@ export function privateHomeEnv(): { HOME: string } { */ export function registerTemporaryPath(directory: string, lifetime: "test" | "process" = "test"): string { registered.add(directory); - if (lifetime === "test") testRoots.getStore()?.push(directory); + if (lifetime === "test") (testRoots.getStore() ?? currentDirectTestRoots())?.push(directory); installExitHook(); return directory; }