FN-5918: reap dashboard vitest workers on wrapper exit

Prevent dashboard test wrappers from leaving orphaned Vitest subprocesses after interruption or timeout.

- run dashboard vitest wrappers in a detached process group and forward shutdown signals to the whole group
- add an exit-time cleanup path and spawn override seam for process-lifecycle handling without launching real vitest
- cover SIGINT/SIGTERM orphan reaping and keep the dashboard test config guard aligned with the new script test

Files changed:
 .../scripts/__tests__/run-vitest-with-heap.test.ts | 172 +++++++++++++++++++++
 .../dashboard/scripts/run-vitest-with-heap.mjs     |  89 ++++++++++-
 .../__tests__/dashboard-test-config-guard.test.ts  |   1 +
 packages/dashboard/vitest.config.ts                |   1 +
 4 files changed, 256 insertions(+), 7 deletions(-)

Fusion-Task-Id: FN-5918

Fusion-Task-Lineage: 5c695b3b-14c5-4749-ad9c-eb9f33d5ecd5
This commit is contained in:
gsxdsm
2026-06-02 19:51:12 -07:00
parent dbab9e1fd9
commit ed80f98216
4 changed files with 256 additions and 7 deletions

View File

@@ -0,0 +1,172 @@
// @vitest-environment node
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { spawn, type ChildProcess } from "node:child_process";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
const __dirname = dirname(fileURLToPath(import.meta.url));
const dashboardRoot = join(__dirname, "..", "..");
const wrapperPath = join(dashboardRoot, "scripts", "run-vitest-with-heap.mjs");
const activeWrappers = new Set<ChildProcess>();
const tempDirs = new Set<string>();
const trackedGroupLeaders = new Set<number>();
const trackedPids = new Set<number>();
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch (error) {
if (error instanceof Error && "code" in error && error.code === "ESRCH") {
return false;
}
throw error;
}
}
async function waitFor(condition: () => boolean, timeoutMs = 5_000, intervalMs = 50): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) {
return;
}
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
throw new Error(`Condition not met within ${timeoutMs}ms`);
}
function registerPid(pid: number) {
trackedPids.add(pid);
}
function registerGroupLeader(pid: number) {
trackedGroupLeaders.add(pid);
registerPid(pid);
}
function createStubProcessTree() {
const tempDir = mkdtempSync(join(tmpdir(), "fusion-run-vitest-"));
tempDirs.add(tempDir);
const pidFile = join(tempDir, "pids.json");
const grandchildPath = join(tempDir, "grandchild.mjs");
const childPath = join(tempDir, "child.mjs");
writeFileSync(
grandchildPath,
['setInterval(() => {}, 1_000);'].join("\n"),
);
writeFileSync(
childPath,
[
'import { writeFileSync } from "node:fs";',
'import { spawn } from "node:child_process";',
'',
'const pidFile = process.argv[2];',
'const grandchildPath = process.argv[3];',
'const grandchild = spawn(process.execPath, [grandchildPath], { stdio: "ignore" });',
'writeFileSync(pidFile, JSON.stringify({ childPid: process.pid, grandchildPid: grandchild.pid }));',
'setInterval(() => {}, 1_000);',
].join("\n"),
);
return { pidFile, childPath, grandchildPath, tempDir };
}
async function spawnWrapperTree(signal: NodeJS.Signals) {
const { pidFile, childPath, grandchildPath } = createStubProcessTree();
const wrapper = spawn(
process.execPath,
[wrapperPath, "--heap=6144", "run", "--project", "dashboard-api-quality"],
{
cwd: dashboardRoot,
stdio: "pipe",
env: {
...process.env,
FUSION_RUN_VITEST_SPAWN_OVERRIDE: JSON.stringify({
command: process.execPath,
args: [childPath, pidFile, grandchildPath],
}),
},
},
);
activeWrappers.add(wrapper);
let pids: { childPid: number; grandchildPid: number } | null = null;
await waitFor(() => {
try {
pids = JSON.parse(readFileSync(pidFile, "utf8")) as { childPid: number; grandchildPid: number };
return Boolean(
pids &&
Number.isInteger(pids.childPid) &&
Number.isInteger(pids.grandchildPid) &&
isProcessAlive(pids.childPid) &&
isProcessAlive(pids.grandchildPid),
);
} catch {
return false;
}
});
registerGroupLeader(pids!.childPid);
registerPid(pids!.grandchildPid);
wrapper.kill(signal);
await new Promise<void>((resolve, reject) => {
wrapper.once("error", reject);
wrapper.once("close", () => resolve());
});
activeWrappers.delete(wrapper);
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
}
afterEach(async () => {
for (const wrapper of activeWrappers) {
wrapper.kill("SIGKILL");
}
activeWrappers.clear();
for (const leaderPid of trackedGroupLeaders) {
try {
process.kill(-leaderPid, "SIGKILL");
} catch (error) {
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") {
throw error;
}
}
}
trackedGroupLeaders.clear();
for (const pid of trackedPids) {
try {
process.kill(pid, "SIGKILL");
} catch (error) {
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") {
throw error;
}
}
}
trackedPids.clear();
for (const tempDir of tempDirs) {
rmSync(tempDir, { recursive: true, force: true });
}
tempDirs.clear();
});
describe("run-vitest-with-heap", () => {
it("reaps the spawned process group on SIGTERM", async () => {
await spawnWrapperTree("SIGTERM");
});
it("reaps the spawned process group on SIGINT", async () => {
await spawnWrapperTree("SIGINT");
});
});

View File

@@ -2,6 +2,7 @@
/* global clearInterval, console, process, setInterval */
import { spawn } from "node:child_process";
const rawArgs = process.argv.slice(2);
const heapArg = rawArgs.find((arg) => arg.startsWith("--heap="));
const heapMb = heapArg?.slice("--heap=".length) || "6144";
@@ -15,7 +16,34 @@ if (vitestArgs.length === 0) {
const nodeOptions = [`--max-old-space-size=${heapMb}`, process.env.NODE_OPTIONS || ""]
.join(" ")
.trim();
const child = spawn("pnpm", ["exec", "vitest", ...vitestArgs], {
function resolveSpawnCommand() {
const override = process.env.FUSION_RUN_VITEST_SPAWN_OVERRIDE;
if (!override) {
return { command: "pnpm", args: ["exec", "vitest", ...vitestArgs] };
}
const parsedOverride = JSON.parse(override);
if (
!parsedOverride ||
typeof parsedOverride.command !== "string" ||
parsedOverride.command.length === 0 ||
!Array.isArray(parsedOverride.args) ||
parsedOverride.args.some((arg) => typeof arg !== "string")
) {
throw new Error(
"FUSION_RUN_VITEST_SPAWN_OVERRIDE must be valid JSON with string command and string[] args",
);
}
// Test seam for process-lifecycle coverage without launching real vitest.
return { command: parsedOverride.command, args: parsedOverride.args };
}
const { command, args } = resolveSpawnCommand();
// process-supervisor-allowlist: foreground wrapper signals the entire vitest process group on death/timeout; not a background daemon
const child = spawn(command, args, {
detached: true,
stdio: "inherit",
env: { ...process.env, NODE_OPTIONS: nodeOptions },
});
@@ -24,15 +52,62 @@ const heartbeat = setInterval(() => {
console.log(`[dashboard-vitest] still running: ${vitestArgs.join(" ")}`);
}, 5_000);
const forwardSignal = (signal) => {
child.kill(signal);
};
function clearHeartbeat() {
clearInterval(heartbeat);
}
process.on("SIGINT", () => forwardSignal("SIGINT"));
process.on("SIGTERM", () => forwardSignal("SIGTERM"));
function forwardSignal(signal) {
clearHeartbeat();
try {
process.kill(-child.pid, signal);
return;
} catch (error) {
if (!(error instanceof Error) || !("code" in error)) {
throw error;
}
if (error.code !== "ESRCH" && error.code !== "EPERM") {
throw error;
}
}
try {
child.kill(signal);
} catch (error) {
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") {
throw error;
}
}
}
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
process.on(signal, () => forwardSignal(signal));
}
process.on("exit", () => {
clearHeartbeat();
try {
process.kill(-child.pid, "SIGTERM");
} catch (error) {
if (
!(error instanceof Error) ||
!("code" in error) ||
(error.code !== "ESRCH" && error.code !== "EPERM")
) {
throw error;
}
}
});
child.on("error", (error) => {
clearHeartbeat();
console.error(error);
process.exit(1);
});
child.on("close", (code, signal) => {
clearInterval(heartbeat);
clearHeartbeat();
if (signal) {
process.kill(process.pid, signal);
return;

View File

@@ -73,5 +73,6 @@ describe("dashboard test config guard", () => {
}
expect(vitestConfig).toContain('"app/__tests__/spinner-animation.css.test.ts"');
expect(vitestConfig).toContain('"scripts/__tests__/run-vitest-with-heap.test.ts"');
});
});

View File

@@ -213,6 +213,7 @@ const qualityApiTests = [
"src/__tests__/{api-error,auth-middleware,auth-middleware-integration,chat-attachment-routes,chat-routes,file-service,github,github-webhooks,initialize,planning-flow-diagnostics-guardrail,pr-routes-auto-merge,pr-routes.contract,project-routes,project-store-resolver,register-git-github.pr-options-preflight-metadata,remote-access-routes,remote-auth,routes-agent-budget,routes-agent-keys,routes-agent-permissions,routes-agent-ratings,routes-agent-runs,routes-agent-soul-memory,routes-agents,routes-automation,routes-branch-groups,routes-git,routes-github,routes-merge-advance-push-origin,routes-nodes,routes-nodes-sync-contract,routes-planning,routes-secrets-sync,routes-settings,routes-task-commit-associations,routes-tasks,routes-tasks-deterministic-dedup,routes-tasks-duplicate-check,routes-tasks-explicit-duplicate-marker,server,server-static-assets,server-webhook,server.events,setup-routes,sse,sse-buffer,test-isolation-guard,update-check-route,websocket,recover-branch-binding-route}.test.ts",
"src/__tests__/dashboard-test-config-guard.test.ts",
"src/routes/__tests__/{custom-provider-routes,custom-providers,register-docker-node-routes,register-diagnostics-routes,stash-recovery-routes}.test.ts",
"scripts/__tests__/run-vitest-with-heap.test.ts",
];
export default defineConfig({