diff --git a/packages/core/src/__test-utils__/vitest-setup.ts b/packages/core/src/__test-utils__/vitest-setup.ts index 6aa4bb6c29..04ca6ccc6c 100644 --- a/packages/core/src/__test-utils__/vitest-setup.ts +++ b/packages/core/src/__test-utils__/vitest-setup.ts @@ -14,6 +14,7 @@ import { afterEach, expect } from "vitest"; import { createRequire, syncBuiltinESMExports } from "node:module"; +import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; @@ -74,6 +75,8 @@ function installWarningFilter(): void { installWarningFilter(); const TEST_HOME_PREFIX = "fn-test-home-"; +const WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner"; +const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN"; const DEFAULT_TEST_SUBPROCESS_TIMEOUT_MS = Math.max( 1_000, Number.parseInt(process.env.FUSION_TEST_SUBPROCESS_TIMEOUT_MS ?? "30000", 10) || 30_000, @@ -170,14 +173,43 @@ if (!process.env.FUSION_MASTER_KEY_DISABLE_KEYCHAIN) { // bounded one-level sweep of WORKER_ROOT, and a static root can accumulate enough // stale worker/home dirs after interrupted runs to make every mkdtempSync call // take seconds. -const WORKER_ROOT = (() => { +function ensureTestRunToken(): string { + const existing = process.env[FUSION_TEST_RUN_TOKEN_ENV]; + if (existing && existing.trim().length > 0) return existing; + const token = randomUUID(); + process.env[FUSION_TEST_RUN_TOKEN_ENV] = token; + return token; +} + +function writeWorkerRootOwnerMarker(root: string): void { + try { + writeFileSync( + join(root, WORKER_ROOT_OWNER_FILE), + `${process.pid}\nrunToken=${ensureTestRunToken()}\n`, + ); + } catch { + // Best effort only. The marker helps the pnpm-test runner distinguish a + // live same-run root from stale pid reuse; local exit cleanup still owns + // self-minted fallback roots by absolute path. + } +} + +const { root: WORKER_ROOT, selfMinted: SELF_MINTED_WORKER_ROOT } = (() => { const fromEnv = process.env.FUSION_TEST_WORKER_ROOT; - const root = fromEnv && fromEnv.trim().length > 0 - ? resolve(fromEnv) - : realpathSync(mkdtempSync(join(tmpdir(), "fusion-test-workers-"))); + const selfMinted = !(fromEnv && fromEnv.trim().length > 0); + const root = selfMinted + ? realpathSync(mkdtempSync(join(tmpdir(), "fusion-test-workers-"))) + : resolve(fromEnv); try { mkdirSync(root, { recursive: true }); } catch { /* ignore */ } process.env.FUSION_TEST_WORKER_ROOT = root; - return root; + ensureTestRunToken(); + if (selfMinted) { + // FN-6396/FN-6360 recurrence: without globalSetup there is no teardown + // owner for this fallback root. Mark it and remove the root itself on exit + // so an empty fusion-test-workers-* shell cannot trip check-test-isolation. + writeWorkerRootOwnerMarker(root); + } + return { root, selfMinted }; })(); const REAL_TMPDIR = (() => { @@ -1036,6 +1068,32 @@ afterEach(async () => { } }); +function sleepMsSync(ms: number): void { + if (ms <= 0) return; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function removeSelfMintedWorkerRootWithRetry( + workerRoot = WORKER_ROOT, + selfMinted = SELF_MINTED_WORKER_ROOT, + delayMs = 25, +): void { + if (!selfMinted) return; + for (let attempt = 1; attempt <= 3; attempt++) { + try { + rmSync(workerRoot, { recursive: true, force: true }); + return; + } catch { + if (attempt < 3) sleepMsSync(delayMs); + } + } +} + +export const __fusionWorkerRootCleanupTestHooks = { + removeSelfMintedWorkerRootWithRetry, + writeWorkerRootOwnerMarker, +}; + process.on("exit", () => { for (const [proc] of trackedSubprocesses) { try { @@ -1045,11 +1103,14 @@ process.on("exit", () => { } cleanupTrackedSubprocess(proc); } - if (!workerTempDir) return; - try { - originalChdir(tmpdir()); - rmSync(workerTempDir, { recursive: true, force: true }); - } catch { - // Ignore — globalTeardown sweeps WORKER_ROOT anyway. + if (workerTempDir) { + try { + originalChdir(tmpdir()); + rmSync(workerTempDir, { recursive: true, force: true }); + } catch { + // Ignore — globalTeardown sweeps env-owned WORKER_ROOT; self-minted roots + // get their own bounded best-effort removal below. + } } + removeSelfMintedWorkerRootWithRetry(); }); diff --git a/packages/core/src/__test-utils__/vitest-teardown.ts b/packages/core/src/__test-utils__/vitest-teardown.ts index 20f52e3482..e9f4ca38f3 100644 --- a/packages/core/src/__test-utils__/vitest-teardown.ts +++ b/packages/core/src/__test-utils__/vitest-teardown.ts @@ -11,6 +11,7 @@ import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; export const WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner"; +const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN"; let workerRootRmSync = rmSync; let workerRootSleepMsSync = sleepMsSync; @@ -57,10 +58,13 @@ export default function setup(): () => Promise { // prior interrupted run. const workerRoot = resolve(mkdtempSync(join(tmpdir(), "fusion-test-workers-"))); try { - writeFileSync(join(workerRoot, WORKER_ROOT_OWNER_FILE), `${process.pid}\n`); + const runToken = process.env[FUSION_TEST_RUN_TOKEN_ENV]; + const tokenLine = runToken && runToken.trim().length > 0 ? `runToken=${runToken}\n` : ""; + writeFileSync(join(workerRoot, WORKER_ROOT_OWNER_FILE), `${process.pid}\n${tokenLine}`); } catch { // Best effort only. The marker protects active roots from external orphan - // pruning; teardown still owns this root by absolute path. + // pruning; FN-6396 adds the runner token so stale pid reuse cannot keep an + // orphaned root alive. Teardown still owns this root by absolute path. } process.env.FUSION_TEST_WORKER_ROOT = workerRoot; diff --git a/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts b/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts index d03fd35a12..1234e4e3aa 100644 --- a/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts +++ b/packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts @@ -1,6 +1,8 @@ -import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { __fusionWorkerRootCleanupTestHooks } from "../__test-utils__/vitest-setup"; import setup, { __setWorkerRootRmSyncForTests, __setWorkerRootSleepMsSyncForTests, @@ -85,4 +87,19 @@ describe("vitest global teardown worker-root cleanup", () => { expect(existsSync(workerRoot)).toBe(false); }); + + it("removes a self-minted fallback worker root during exit cleanup", () => { + const workerRoot = remember(mkdtempSync(join(tmpdir(), "fusion-test-workers-self-minted-"))); + const workerDir = join(workerRoot, `w-${process.pid}-fallback`); + const redirDir = join(workerRoot, `redir-${process.pid}`); + mkdirSync(workerDir, { recursive: true }); + mkdirSync(redirDir, { recursive: true }); + writeFileSync(join(workerDir, "payload.txt"), "worker temp payload"); + writeFileSync(join(redirDir, "payload.txt"), "redirect temp payload"); + __fusionWorkerRootCleanupTestHooks.writeWorkerRootOwnerMarker(workerRoot); + + __fusionWorkerRootCleanupTestHooks.removeSelfMintedWorkerRootWithRetry(workerRoot, true, 0); + + expect(existsSync(workerRoot)).toBe(false); + }); }); diff --git a/scripts/__tests__/test-changed.test.mjs b/scripts/__tests__/test-changed.test.mjs index 8cfeb977a1..90df9ccd36 100644 --- a/scripts/__tests__/test-changed.test.mjs +++ b/scripts/__tests__/test-changed.test.mjs @@ -27,6 +27,7 @@ import { cleanupIsolatedHomePath, knownIsolatedHomeBasenames, __setCleanupRmSyncForTests, + __setProcessAliveForTests, emitModeDecision, pruneFusionTestHomes, pruneFusionTestWorkers, @@ -35,7 +36,7 @@ import { computeOwnHash, } from "../test-changed.mjs"; -import { mkdirSync, writeFileSync, mkdtempSync, rmSync, existsSync } from "node:fs"; +import { mkdirSync, writeFileSync, mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { spawnSync } from "node:child_process"; @@ -1058,6 +1059,77 @@ test("pruneFusionTestWorkers: skips markerless roots with live redirect sinks", } }); +function setOldMtime(pathValue) { + const old = new Date(Date.now() - 60_000); + utimesSync(pathValue, old, old); +} + +function withAlivePid(pid, fn) { + __setProcessAliveForTests((candidate) => candidate === pid); + try { + fn(); + } finally { + __setProcessAliveForTests(null); + } +} + +test("pruneFusionTestWorkers: prunes owner-marker roots when pid liveness is stale", () => { + const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-stale-owner-${process.pid}-`)); + const recycledPid = 424_242; + try { + writeFileSync(path.join(root, ".fusion-test-worker-root-owner"), `${recycledPid}\nrunToken=prior-run\n`); + withAlivePid(recycledPid, () => pruneFusionTestWorkers(1024)); + assert.equal(existsSync(root), false, "stale pid reuse must not preserve an orphaned worker root"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pruneFusionTestWorkers: preserves same-run owner-marker roots with live pids", () => { + const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-current-owner-${process.pid}-`)); + const ownerPid = 515_151; + try { + writeFileSync( + path.join(root, ".fusion-test-worker-root-owner"), + `${ownerPid}\nrunToken=${process.env.FUSION_TEST_RUN_TOKEN}\n`, + ); + withAlivePid(ownerPid, () => pruneFusionTestWorkers(1024)); + assert.equal(existsSync(root), true, "current-run live worker root must not be pruned"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pruneFusionTestWorkers: prunes old markerless redir roots when pid liveness is stale", () => { + const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-stale-redir-${process.pid}-`)); + const recycledPid = 626_262; + try { + const redir = path.join(root, `redir-${recycledPid}`); + mkdirSync(redir, { recursive: true }); + writeFileSync(path.join(redir, "payload.txt"), "stale\n"); + setOldMtime(redir); + setOldMtime(root); + withAlivePid(recycledPid, () => pruneFusionTestWorkers(1024)); + assert.equal(existsSync(root), false, "old markerless redir root must be pruned despite pid reuse"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pruneFusionTestWorkers: removes SIGKILL-style orphan roots and leaves foreign prefixes alone", () => { + const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-sigkill-orphan-${process.pid}-`)); + const foreign = mkdtempSync(path.join(tmpdir(), `not-fusion-test-workers-${process.pid}-`)); + try { + mkdirSync(path.join(root, `w-${process.pid}-orphan`), { recursive: true }); + pruneFusionTestWorkers(1024); + assert.equal(existsSync(root), false, "orphaned worker root should be pruned"); + assert.equal(existsSync(foreign), true, "foreign prefixes must not be touched"); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(foreign, { recursive: true, force: true }); + } +}); + test("pruneFusionTestWorkers: reclaims non-empty root after transient ENOTEMPTY", () => { const root = createNonEmptyPruneRoot("fusion-test-workers-", "transient"); withTransientPruneFailure(root, pruneFusionTestWorkers); diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index 32f8df27d9..53ebb2e2c2 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -1,10 +1,10 @@ #!/usr/bin/env node -import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync } from "node:fs"; +import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync, statSync } from "node:fs"; import path from "node:path"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { cpus, tmpdir } from "node:os"; import { createRequire } from "node:module"; import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs"; @@ -174,13 +174,32 @@ const PRUNE_REMOVE_RETRIES = 3; const PRUNE_REMOVE_DELAY_MS = 75; const PRUNE_DIAGNOSTIC_CHILD_LIMIT = 8; const FUSION_WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner"; +const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN"; +const LEGACY_MARKERLESS_ACTIVE_ROOT_MAX_AGE_MS = 30_000; + +function ensureFusionTestRunToken(env = process.env) { + const existing = env[FUSION_TEST_RUN_TOKEN_ENV]; + if (typeof existing === "string" && existing.trim().length > 0) return existing; + const token = randomUUID(); + env[FUSION_TEST_RUN_TOKEN_ENV] = token; + return token; +} + +ensureFusionTestRunToken(); function isEnoentError(err) { return Boolean(err && typeof err === "object" && "code" in err && err.code === "ENOENT"); } +let processAliveForTests = null; + +export function __setProcessAliveForTests(nextProcessAlive) { + processAliveForTests = typeof nextProcessAlive === "function" ? nextProcessAlive : null; +} + function isProcessAlive(pid) { if (!Number.isInteger(pid) || pid <= 0) return false; + if (processAliveForTests) return Boolean(processAliveForTests(pid)); try { process.kill(pid, 0); return true; @@ -189,28 +208,61 @@ function isProcessAlive(pid) { } } -function readWorkerRootOwnerPid(rootPath) { +function readWorkerRootOwnerInfo(rootPath) { try { const raw = readFileSync(path.join(rootPath, FUSION_WORKER_ROOT_OWNER_FILE), "utf8").trim(); - const pid = Number.parseInt(raw, 10); - return Number.isInteger(pid) && pid > 0 ? pid : null; + const lines = raw.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + const pid = Number.parseInt(lines[0] ?? "", 10); + if (!Number.isInteger(pid) || pid <= 0) return null; + const info = { pid, runToken: null }; + for (const line of lines.slice(1)) { + const match = /^runToken=(.+)$/.exec(line); + if (match) info.runToken = match[1]; + } + return info; } catch { return null; } } -function isActiveFusionWorkerRoot(rootPath) { - const ownerPid = readWorkerRootOwnerPid(rootPath); - if (ownerPid !== null && isProcessAlive(ownerPid)) return true; +function hasCurrentRunToken(ownerInfo) { + const currentToken = process.env[FUSION_TEST_RUN_TOKEN_ENV]; + return Boolean(ownerInfo?.runToken && currentToken && ownerInfo.runToken === currentToken); +} - // Backward-compatible guard for worker roots created before the owner marker - // landed, or marker writes that failed: an alive redir- child means a - // Vitest worker still owns temp workspaces beneath this root. +function isFreshLegacyMarkerlessRoot(rootPath) { + try { + return Date.now() - statSync(rootPath).mtimeMs <= LEGACY_MARKERLESS_ACTIVE_ROOT_MAX_AGE_MS; + } catch { + return false; + } +} + +function isActiveFusionWorkerRoot(rootPath) { + const ownerInfo = readWorkerRootOwnerInfo(rootPath); + if (ownerInfo !== null && isProcessAlive(ownerInfo.pid)) { + if (ownerInfo.pid === process.pid || hasCurrentRunToken(ownerInfo)) return true; + // FN-6396/FN-6360 recurrence: bare pid liveness is not enough evidence. + // macOS can recycle a dead Vitest owner's pid to an unrelated process, so + // the pnpm-test prune must require the same-run token before preserving the + // root. Otherwise stale fusion-test-workers-* shells survive to the after + // check-test-isolation pass and fail the merge gate. + } + + // Backward-compatible guard for markerless roots. New roots are marked by + // globalSetup or by vitest-setup's self-minted fallback path; old markerless + // redir roots are only considered active while very fresh, preventing stale + // redir- pid reuse from keeping orphans alive forever. try { for (const child of readdirSync(rootPath, { withFileTypes: true })) { if (!child.isDirectory()) continue; const match = /^redir-(\d+)$/.exec(child.name); - if (match && isProcessAlive(Number.parseInt(match[1], 10))) return true; + if (!match) continue; + const redirPid = Number.parseInt(match[1], 10); + if (!isProcessAlive(redirPid)) continue; + if (redirPid === process.pid || (ownerInfo && hasCurrentRunToken(ownerInfo)) || isFreshLegacyMarkerlessRoot(rootPath)) { + return true; + } } } catch { // If we cannot inspect it, fall through to normal best-effort pruning. @@ -290,7 +342,7 @@ export function pruneFusionTestWorkers(maxEntries = PRUNE_MAX_ENTRIES, retryOpti function runMaybeIsolated(command, commandArgs, options = {}) { const enabled = shouldRunIsolationGuard(); - const env = options.env ?? process.env; + const env = { ...(options.env ?? process.env), [FUSION_TEST_RUN_TOKEN_ENV]: ensureFusionTestRunToken(options.env ?? process.env) }; const { onBeforeAfterCheck, ...spawnOptions } = options; if (enabled) runIsolationCheck(true, env, /* fastBefore */ true); try {