root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FN-6396: harden test worker temp cleanup

Browse Source 
Prevent stale Fusion test worker roots from leaking across merge-gate runs.

- Add per-run tokens to worker-root owner markers and pruning checks.
- Remove self-minted fallback worker roots during Vitest exit cleanup.
- Cover stale pid reuse, markerless redir roots, and SIGKILL-style orphan pruning with regression tests.

Files changed:
 packages/core/src/__test-utils__/vitest-setup.ts   | 83 +++++++++++++++++++---
 .../core/src/__test-utils__/vitest-teardown.ts     |  8 ++-
 .../vitest-teardown-worker-root-cleanup.test.ts    | 19 ++++-
 scripts/__tests__/test-changed.test.mjs            | 74 ++++++++++++++++++-
 scripts/test-changed.mjs                           | 76 ++++++++++++++++----
 5 files changed, 233 insertions(+), 27 deletions(-)

Fusion-Task-Id: FN-6396

Fusion-Task-Lineage: 711d966d-c70e-4cd7-81cd-accd18f17202
This commit is contained in:
gsxdsm
2026-06-13 15:17:24 -07:00
parent 7fa8c4e5f4
commit 80fbcdd5a3
5 changed files with 234 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
packages/core/src/__test-utils__/vitest-setup.ts
View File

@@ -14,6 +14,7 @@
import { afterEach, expect } from "vitest";
import { createRequire, syncBuiltinESMExports } from "node:module";
import { randomUUID } from "node:crypto";
import { tmpdir } from "node:os";
import { basename, dirname, join, resolve } from "node:path";
import { promisify } from "node:util";
@@ -74,6 +75,8 @@ function installWarningFilter(): void {
installWarningFilter();
const TEST_HOME_PREFIX = "fn-test-home-";
const WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner";
const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN";
const DEFAULT_TEST_SUBPROCESS_TIMEOUT_MS = Math.max(
1_000,
Number.parseInt(process.env.FUSION_TEST_SUBPROCESS_TIMEOUT_MS ?? "30000", 10) || 30_000,
@@ -170,14 +173,43 @@ if (!process.env.FUSION_MASTER_KEY_DISABLE_KEYCHAIN) {
// bounded one-level sweep of WORKER_ROOT, and a static root can accumulate enough
// stale worker/home dirs after interrupted runs to make every mkdtempSync call
// take seconds.
const WORKER_ROOT = (() => {
function ensureTestRunToken(): string {
const existing = process.env[FUSION_TEST_RUN_TOKEN_ENV];
if (existing && existing.trim().length > 0) return existing;
const token = randomUUID();
process.env[FUSION_TEST_RUN_TOKEN_ENV] = token;
return token;
}
function writeWorkerRootOwnerMarker(root: string): void {
try {
writeFileSync(
join(root, WORKER_ROOT_OWNER_FILE),
`${process.pid}\nrunToken=${ensureTestRunToken()}\n`,
);
} catch {
// Best effort only. The marker helps the pnpm-test runner distinguish a
// live same-run root from stale pid reuse; local exit cleanup still owns
// self-minted fallback roots by absolute path.
}
}
const { root: WORKER_ROOT, selfMinted: SELF_MINTED_WORKER_ROOT } = (() => {
const fromEnv = process.env.FUSION_TEST_WORKER_ROOT;
const root = fromEnv && fromEnv.trim().length > 0
? resolve(fromEnv)
: realpathSync(mkdtempSync(join(tmpdir(), "fusion-test-workers-")));
const selfMinted = !(fromEnv && fromEnv.trim().length > 0);
const root = selfMinted
? realpathSync(mkdtempSync(join(tmpdir(), "fusion-test-workers-")))
: resolve(fromEnv);
try { mkdirSync(root, { recursive: true }); } catch { /* ignore */ }
process.env.FUSION_TEST_WORKER_ROOT = root;
return root;
ensureTestRunToken();
if (selfMinted) {
// FN-6396/FN-6360 recurrence: without globalSetup there is no teardown
// owner for this fallback root. Mark it and remove the root itself on exit
// so an empty fusion-test-workers-* shell cannot trip check-test-isolation.
writeWorkerRootOwnerMarker(root);
}
return { root, selfMinted };
})();
const REAL_TMPDIR = (() => {
@@ -1036,6 +1068,32 @@ afterEach(async () => {
}
});
function sleepMsSync(ms: number): void {
if (ms <= 0) return;
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}
function removeSelfMintedWorkerRootWithRetry(
workerRoot = WORKER_ROOT,
selfMinted = SELF_MINTED_WORKER_ROOT,
delayMs = 25,
): void {
if (!selfMinted) return;
for (let attempt = 1; attempt <= 3; attempt++) {
try {
rmSync(workerRoot, { recursive: true, force: true });
return;
} catch {
if (attempt < 3) sleepMsSync(delayMs);
}
}
}
export const __fusionWorkerRootCleanupTestHooks = {
removeSelfMintedWorkerRootWithRetry,
writeWorkerRootOwnerMarker,
};
process.on("exit", () => {
for (const [proc] of trackedSubprocesses) {
try {
@@ -1045,11 +1103,14 @@ process.on("exit", () => {
}
cleanupTrackedSubprocess(proc);
}
if (!workerTempDir) return;
try {
originalChdir(tmpdir());
rmSync(workerTempDir, { recursive: true, force: true });
} catch {
// Ignore — globalTeardown sweeps WORKER_ROOT anyway.
if (workerTempDir) {
try {
originalChdir(tmpdir());
rmSync(workerTempDir, { recursive: true, force: true });
} catch {
// Ignore — globalTeardown sweeps env-owned WORKER_ROOT; self-minted roots
// get their own bounded best-effort removal below.
}
}
removeSelfMintedWorkerRootWithRetry();
});

8
packages/core/src/__test-utils__/vitest-teardown.ts
View File

@@ -11,6 +11,7 @@ import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
export const WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner";
const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN";
let workerRootRmSync = rmSync;
let workerRootSleepMsSync = sleepMsSync;
@@ -57,10 +58,13 @@ export default function setup(): () => Promise<void> {
// prior interrupted run.
const workerRoot = resolve(mkdtempSync(join(tmpdir(), "fusion-test-workers-")));
try {
writeFileSync(join(workerRoot, WORKER_ROOT_OWNER_FILE), `${process.pid}\n`);
const runToken = process.env[FUSION_TEST_RUN_TOKEN_ENV];
const tokenLine = runToken && runToken.trim().length > 0 ? `runToken=${runToken}\n` : "";
writeFileSync(join(workerRoot, WORKER_ROOT_OWNER_FILE), `${process.pid}\n${tokenLine}`);
} catch {
// Best effort only. The marker protects active roots from external orphan
// pruning; teardown still owns this root by absolute path.
// pruning; FN-6396 adds the runner token so stale pid reuse cannot keep an
// orphaned root alive. Teardown still owns this root by absolute path.
}
process.env.FUSION_TEST_WORKER_ROOT = workerRoot;

19
packages/core/src/__tests__/vitest-teardown-worker-root-cleanup.test.ts
View File

@@ -1,6 +1,8 @@
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { __fusionWorkerRootCleanupTestHooks } from "../__test-utils__/vitest-setup";
import setup, {
__setWorkerRootRmSyncForTests,
__setWorkerRootSleepMsSyncForTests,
@@ -85,4 +87,19 @@ describe("vitest global teardown worker-root cleanup", () => {
expect(existsSync(workerRoot)).toBe(false);
});
it("removes a self-minted fallback worker root during exit cleanup", () => {
const workerRoot = remember(mkdtempSync(join(tmpdir(), "fusion-test-workers-self-minted-")));
const workerDir = join(workerRoot, `w-${process.pid}-fallback`);
const redirDir = join(workerRoot, `redir-${process.pid}`);
mkdirSync(workerDir, { recursive: true });
mkdirSync(redirDir, { recursive: true });
writeFileSync(join(workerDir, "payload.txt"), "worker temp payload");
writeFileSync(join(redirDir, "payload.txt"), "redirect temp payload");
__fusionWorkerRootCleanupTestHooks.writeWorkerRootOwnerMarker(workerRoot);
__fusionWorkerRootCleanupTestHooks.removeSelfMintedWorkerRootWithRetry(workerRoot, true, 0);
expect(existsSync(workerRoot)).toBe(false);
});
});

74
scripts/__tests__/test-changed.test.mjs
View File

@@ -27,6 +27,7 @@ import {
cleanupIsolatedHomePath,
knownIsolatedHomeBasenames,
__setCleanupRmSyncForTests,
__setProcessAliveForTests,
emitModeDecision,
pruneFusionTestHomes,
pruneFusionTestWorkers,
@@ -35,7 +36,7 @@ import {
computeOwnHash,
} from "../test-changed.mjs";
import { mkdirSync, writeFileSync, mkdtempSync, rmSync, existsSync } from "node:fs";
import { mkdirSync, writeFileSync, mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
@@ -1058,6 +1059,77 @@ test("pruneFusionTestWorkers: skips markerless roots with live redirect sinks",
}
});
function setOldMtime(pathValue) {
const old = new Date(Date.now() - 60_000);
utimesSync(pathValue, old, old);
}
function withAlivePid(pid, fn) {
__setProcessAliveForTests((candidate) => candidate === pid);
try {
fn();
} finally {
__setProcessAliveForTests(null);
}
}
test("pruneFusionTestWorkers: prunes owner-marker roots when pid liveness is stale", () => {
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-stale-owner-${process.pid}-`));
const recycledPid = 424_242;
try {
writeFileSync(path.join(root, ".fusion-test-worker-root-owner"), `${recycledPid}\nrunToken=prior-run\n`);
withAlivePid(recycledPid, () => pruneFusionTestWorkers(1024));
assert.equal(existsSync(root), false, "stale pid reuse must not preserve an orphaned worker root");
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("pruneFusionTestWorkers: preserves same-run owner-marker roots with live pids", () => {
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-current-owner-${process.pid}-`));
const ownerPid = 515_151;
try {
writeFileSync(
path.join(root, ".fusion-test-worker-root-owner"),
`${ownerPid}\nrunToken=${process.env.FUSION_TEST_RUN_TOKEN}\n`,
);
withAlivePid(ownerPid, () => pruneFusionTestWorkers(1024));
assert.equal(existsSync(root), true, "current-run live worker root must not be pruned");
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("pruneFusionTestWorkers: prunes old markerless redir roots when pid liveness is stale", () => {
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-stale-redir-${process.pid}-`));
const recycledPid = 626_262;
try {
const redir = path.join(root, `redir-${recycledPid}`);
mkdirSync(redir, { recursive: true });
writeFileSync(path.join(redir, "payload.txt"), "stale\n");
setOldMtime(redir);
setOldMtime(root);
withAlivePid(recycledPid, () => pruneFusionTestWorkers(1024));
assert.equal(existsSync(root), false, "old markerless redir root must be pruned despite pid reuse");
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("pruneFusionTestWorkers: removes SIGKILL-style orphan roots and leaves foreign prefixes alone", () => {
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-sigkill-orphan-${process.pid}-`));
const foreign = mkdtempSync(path.join(tmpdir(), `not-fusion-test-workers-${process.pid}-`));
try {
mkdirSync(path.join(root, `w-${process.pid}-orphan`), { recursive: true });
pruneFusionTestWorkers(1024);
assert.equal(existsSync(root), false, "orphaned worker root should be pruned");
assert.equal(existsSync(foreign), true, "foreign prefixes must not be touched");
} finally {
rmSync(root, { recursive: true, force: true });
rmSync(foreign, { recursive: true, force: true });
}
});
test("pruneFusionTestWorkers: reclaims non-empty root after transient ENOTEMPTY", () => {
const root = createNonEmptyPruneRoot("fusion-test-workers-", "transient");
withTransientPruneFailure(root, pruneFusionTestWorkers);

78
scripts/test-changed.mjs
View File

@@ -1,10 +1,10 @@
#!/usr/bin/env node
import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync } from "node:fs";
import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync, statSync } from "node:fs";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import { cpus, tmpdir } from "node:os";
import { createRequire } from "node:module";
import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs";
@@ -174,13 +174,32 @@ const PRUNE_REMOVE_RETRIES = 3;
const PRUNE_REMOVE_DELAY_MS = 75;
const PRUNE_DIAGNOSTIC_CHILD_LIMIT = 8;
const FUSION_WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner";
const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN";
const LEGACY_MARKERLESS_ACTIVE_ROOT_MAX_AGE_MS = 30_000;
function ensureFusionTestRunToken(env = process.env) {
const existing = env[FUSION_TEST_RUN_TOKEN_ENV];
if (typeof existing === "string" && existing.trim().length > 0) return existing;
const token = randomUUID();
env[FUSION_TEST_RUN_TOKEN_ENV] = token;
return token;
}
ensureFusionTestRunToken();
function isEnoentError(err) {
return Boolean(err && typeof err === "object" && "code" in err && err.code === "ENOENT");
}
let processAliveForTests = null;
export function __setProcessAliveForTests(nextProcessAlive) {
processAliveForTests = typeof nextProcessAlive === "function" ? nextProcessAlive : null;
}
function isProcessAlive(pid) {
if (!Number.isInteger(pid) || pid <= 0) return false;
if (processAliveForTests) return Boolean(processAliveForTests(pid));
try {
process.kill(pid, 0);
return true;
@@ -189,28 +208,61 @@ function isProcessAlive(pid) {
}
}
function readWorkerRootOwnerPid(rootPath) {
function readWorkerRootOwnerInfo(rootPath) {
try {
const raw = readFileSync(path.join(rootPath, FUSION_WORKER_ROOT_OWNER_FILE), "utf8").trim();
const pid = Number.parseInt(raw, 10);
return Number.isInteger(pid) && pid > 0 ? pid : null;
const lines = raw.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
const pid = Number.parseInt(lines[0] ?? "", 10);
if (!Number.isInteger(pid) || pid <= 0) return null;
const info = { pid, runToken: null };
for (const line of lines.slice(1)) {
const match = /^runToken=(.+)$/.exec(line);
if (match) info.runToken = match[1];
}
return info;
} catch {
return null;
}
}
function isActiveFusionWorkerRoot(rootPath) {
const ownerPid = readWorkerRootOwnerPid(rootPath);
if (ownerPid !== null && isProcessAlive(ownerPid)) return true;
function hasCurrentRunToken(ownerInfo) {
const currentToken = process.env[FUSION_TEST_RUN_TOKEN_ENV];
return Boolean(ownerInfo?.runToken && currentToken && ownerInfo.runToken === currentToken);
}
// Backward-compatible guard for worker roots created before the owner marker
// landed, or marker writes that failed: an alive redir-<pid> child means a
// Vitest worker still owns temp workspaces beneath this root.
function isFreshLegacyMarkerlessRoot(rootPath) {
try {
return Date.now() - statSync(rootPath).mtimeMs <= LEGACY_MARKERLESS_ACTIVE_ROOT_MAX_AGE_MS;
} catch {
return false;
}
}
function isActiveFusionWorkerRoot(rootPath) {
const ownerInfo = readWorkerRootOwnerInfo(rootPath);
if (ownerInfo !== null && isProcessAlive(ownerInfo.pid)) {
if (ownerInfo.pid === process.pid || hasCurrentRunToken(ownerInfo)) return true;
// FN-6396/FN-6360 recurrence: bare pid liveness is not enough evidence.
// macOS can recycle a dead Vitest owner's pid to an unrelated process, so
// the pnpm-test prune must require the same-run token before preserving the
// root. Otherwise stale fusion-test-workers-* shells survive to the after
// check-test-isolation pass and fail the merge gate.
}
// Backward-compatible guard for markerless roots. New roots are marked by
// globalSetup or by vitest-setup's self-minted fallback path; old markerless
// redir roots are only considered active while very fresh, preventing stale
// redir-<pid> pid reuse from keeping orphans alive forever.
try {
for (const child of readdirSync(rootPath, { withFileTypes: true })) {
if (!child.isDirectory()) continue;
const match = /^redir-(\d+)$/.exec(child.name);
if (match && isProcessAlive(Number.parseInt(match[1], 10))) return true;
if (!match) continue;
const redirPid = Number.parseInt(match[1], 10);
if (!isProcessAlive(redirPid)) continue;
if (redirPid === process.pid || (ownerInfo && hasCurrentRunToken(ownerInfo)) || isFreshLegacyMarkerlessRoot(rootPath)) {
return true;
}
}
} catch {
// If we cannot inspect it, fall through to normal best-effort pruning.
@@ -290,7 +342,7 @@ export function pruneFusionTestWorkers(maxEntries = PRUNE_MAX_ENTRIES, retryOpti
function runMaybeIsolated(command, commandArgs, options = {}) {
const enabled = shouldRunIsolationGuard();
const env = options.env ?? process.env;
const env = { ...(options.env ?? process.env), [FUSION_TEST_RUN_TOKEN_ENV]: ensureFusionTestRunToken(options.env ?? process.env) };
const { onBeforeAfterCheck, ...spawnOptions } = options;
if (enabled) runIsolationCheck(true, env, /* fastBefore */ true);
try {