diff --git a/packages/core/src/__test-utils__/vitest-teardown.ts b/packages/core/src/__test-utils__/vitest-teardown.ts index 55ba855b83..fe174d4d28 100644 --- a/packages/core/src/__test-utils__/vitest-teardown.ts +++ b/packages/core/src/__test-utils__/vitest-teardown.ts @@ -10,6 +10,45 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +let workerRootRmSync = rmSync; +let workerRootSleepMsSync = sleepMsSync; + +export function __setWorkerRootRmSyncForTests(nextRmSync: typeof rmSync): void { + workerRootRmSync = typeof nextRmSync === "function" ? nextRmSync : rmSync; +} + +export function __setWorkerRootSleepMsSyncForTests(nextSleep: (ms: number) => void): void { + workerRootSleepMsSync = typeof nextSleep === "function" ? nextSleep : sleepMsSync; +} + +function sleepMsSync(ms: number): void { + if (ms <= 0) return; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function isEnoent(error: unknown): boolean { + return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT"); +} + +export function removeWorkerRootWithRetry(workerRoot: string, retries = 3, delayMs = 75): void { + let lastError: unknown = null; + for (let attempt = 1; attempt <= retries; attempt++) { + try { + workerRootRmSync(workerRoot, { recursive: true, force: true }); + return; + } catch (error) { + if (isEnoent(error)) return; + lastError = error; + if (attempt < retries) { + workerRootSleepMsSync(delayMs); + } + } + } + + const message = lastError instanceof Error ? lastError.message : String(lastError); + console.warn(`[vitest-teardown] failed to remove worker root ${workerRoot} after ${retries} attempts: ${message}`); +} + export default function setup(): () => Promise { // Use a fresh root for each Vitest invocation. A static shared root makes the // setup-time redirect sweep proportional to stale directories left by every @@ -23,12 +62,9 @@ export default function setup(): () => Promise { } catch { // Ignore — cleanup below is best-effort and uses an absolute path. } - try { - rmSync(workerRoot, { recursive: true, force: true }); - } catch { - // Ignore — interrupted or still-active workers may leave a per-run root - // behind, but future runs no longer sweep it because every invocation gets - // a fresh root. - } + // FN-6360: macOS can report transient EBUSY/ENOTEMPTY while SQLite WALs or + // redirected temp dirs are still closing. Retry boundedly so a brief busy-fd + // race does not leak the per-invocation fusion-test-workers-* root. + removeWorkerRootWithRetry(workerRoot); }; } diff --git a/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts b/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts new file mode 100644 index 0000000000..d03fd35a12 --- /dev/null +++ b/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts @@ -0,0 +1,88 @@ +import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import setup, { + __setWorkerRootRmSyncForTests, + __setWorkerRootSleepMsSyncForTests, +} from "../__test-utils__/vitest-teardown"; + +const createdPaths: string[] = []; +const originalWorkerRoot = process.env.FUSION_TEST_WORKER_ROOT; + +function remember(path: string): string { + createdPaths.push(path); + return path; +} + +function makeWorkerChild(root: string, label: string): void { + const workerDir = join(root, `w-${process.pid}-${label}`); + mkdirSync(workerDir, { recursive: true }); + writeFileSync(join(workerDir, "file.txt"), "worker temp payload"); +} + +function restoreWorkerRootEnv(): void { + if (originalWorkerRoot === undefined) { + delete process.env.FUSION_TEST_WORKER_ROOT; + } else { + process.env.FUSION_TEST_WORKER_ROOT = originalWorkerRoot; + } +} + +afterEach(() => { + __setWorkerRootRmSyncForTests(rmSync); + __setWorkerRootSleepMsSyncForTests(() => {}); + restoreWorkerRootEnv(); + for (const path of createdPaths.splice(0).reverse()) { + rmSync(path, { recursive: true, force: true }); + } +}); + +describe("vitest global teardown worker-root cleanup", () => { + it("removes the per-invocation worker root on the clean path", async () => { + const teardown = setup(); + const workerRoot = remember(process.env.FUSION_TEST_WORKER_ROOT!); + makeWorkerChild(workerRoot, "clean"); + + await teardown(); + + expect(existsSync(workerRoot)).toBe(false); + }); + + it("retries an EBUSY worker-root removal and removes the root", async () => { + const teardown = setup(); + const workerRoot = remember(process.env.FUSION_TEST_WORKER_ROOT!); + makeWorkerChild(workerRoot, "busy"); + let attempts = 0; + const sleeps: number[] = []; + + __setWorkerRootRmSyncForTests((path, options) => { + attempts++; + if (attempts === 1) { + const error = new Error("resource busy") as NodeJS.ErrnoException; + error.code = "EBUSY"; + throw error; + } + rmSync(path, options); + }); + __setWorkerRootSleepMsSyncForTests((ms) => { + sleeps.push(ms); + }); + + await teardown(); + + expect(attempts).toBe(2); + expect(sleeps).toEqual([75]); + expect(existsSync(workerRoot)).toBe(false); + }); + + it("tolerates ENOENT when the worker root is already gone", async () => { + const teardown = setup(); + const workerRoot = remember(process.env.FUSION_TEST_WORKER_ROOT!); + makeWorkerChild(workerRoot, "enoent"); + rmSync(workerRoot, { recursive: true, force: true }); + + await teardown(); + + expect(existsSync(workerRoot)).toBe(false); + }); +}); diff --git a/scripts/__tests__/test-changed.test.mjs b/scripts/__tests__/test-changed.test.mjs index 1aa9ff284f..ea2909afc5 100644 --- a/scripts/__tests__/test-changed.test.mjs +++ b/scripts/__tests__/test-changed.test.mjs @@ -29,6 +29,7 @@ import { __setCleanupRmSyncForTests, emitModeDecision, pruneFusionTestHomes, + pruneFusionTestWorkers, buildForwardDependencyMap, collectTransitiveDependencies, computeOwnHash, @@ -951,6 +952,23 @@ test("pruneFusionTestHomes: bounded — removes at most maxEntries per call", () } }); +test("pruneFusionTestWorkers: bounded — removes at most maxEntries per call", () => { + const created = []; + try { + for (let i = 0; i < 5; i++) { + const dir = path.join(tmpdir(), `fusion-test-workers-prune-budget-${process.pid}-${i}`); + mkdirSync(dir, { recursive: true }); + created.push(dir); + } + // Cap at 2 → at least 3 of ours survive this call. + pruneFusionTestWorkers(2); + const survivors = created.filter((dir) => existsSync(dir)); + assert.ok(survivors.length >= 3, `expected >=3 survivors with cap=2, got ${survivors.length}`); + } finally { + for (const dir of created) rmSync(dir, { recursive: true, force: true }); + } +}); + // --------------------------------------------------------------------------- // U4: real-git-fixture integration (dirty working tree + transitive deps). // @@ -1357,3 +1375,18 @@ test("pruneFusionTestHomes: only targets the fusion-test-home-root- prefix", () rmSync(foreign, { recursive: true, force: true }); } }); + +test("pruneFusionTestWorkers: only targets the fusion-test-workers- prefix", () => { + const ours = path.join(tmpdir(), `fusion-test-workers-prune-prefix-${process.pid}`); + const foreign = path.join(tmpdir(), `not-ours-workers-prune-prefix-${process.pid}`); + mkdirSync(ours, { recursive: true }); + mkdirSync(foreign, { recursive: true }); + try { + pruneFusionTestWorkers(); + assert.equal(existsSync(ours), false, "orphaned worker root should be pruned"); + assert.equal(existsSync(foreign), true, "foreign dir must be left untouched"); + } finally { + rmSync(ours, { recursive: true, force: true }); + rmSync(foreign, { recursive: true, force: true }); + } +}); diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index 59a07aa1ed..f9e1c8a773 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -198,6 +198,37 @@ export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES) { } } +export function pruneFusionTestWorkers(maxEntries = PRUNE_MAX_ENTRIES) { + let tmpEntries = []; + try { + tmpEntries = readdirSync(tmpdir(), { withFileTypes: true }); + } catch { + return; + } + + let removed = 0; + for (const entry of tmpEntries) { + if (removed >= maxEntries) break; + if (!entry.isDirectory() || !entry.name.startsWith("fusion-test-workers-")) continue; + const rawPath = path.join(tmpdir(), entry.name); + try { + realpathSync(rawPath); + } catch { + // Keep raw path fallback. + } + try { + // FN-6360: if a Vitest invocation is SIGKILL'd, globalTeardown never runs. + // This capped, single-level prefix prune mirrors pruneFusionTestHomes so + // orphaned worker roots are swept before check-test-isolation runs. + rmSync(rawPath, { recursive: true, force: true }); + removed++; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[test-changed] failed to prune leftover ${rawPath}: ${message}`); + } + } +} + function runMaybeIsolated(command, commandArgs, options = {}) { const enabled = shouldRunIsolationGuard(); const env = options.env ?? process.env; @@ -210,6 +241,7 @@ function runMaybeIsolated(command, commandArgs, options = {}) { onBeforeAfterCheck(); } pruneFusionTestHomes(); + pruneFusionTestWorkers(); if (enabled) runIsolationCheck(false, env); } }