FN-5918: reap dashboard vitest workers on wrapper exit
Prevent dashboard test wrappers from leaving orphaned Vitest subprocesses after interruption or timeout. - run dashboard vitest wrappers in a detached process group and forward shutdown signals to the whole group - add an exit-time cleanup path and spawn override seam for process-lifecycle handling without launching real vitest - cover SIGINT/SIGTERM orphan reaping and keep the dashboard test config guard aligned with the new script test Files changed: .../scripts/__tests__/run-vitest-with-heap.test.ts | 172 +++++++++++++++++++++ .../dashboard/scripts/run-vitest-with-heap.mjs | 89 ++++++++++- .../__tests__/dashboard-test-config-guard.test.ts | 1 + packages/dashboard/vitest.config.ts | 1 + 4 files changed, 256 insertions(+), 7 deletions(-) Fusion-Task-Id: FN-5918 Fusion-Task-Lineage: 5c695b3b-14c5-4749-ad9c-eb9f33d5ecd5
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const dashboardRoot = join(__dirname, "..", "..");
|
||||
const wrapperPath = join(dashboardRoot, "scripts", "run-vitest-with-heap.mjs");
|
||||
|
||||
const activeWrappers = new Set<ChildProcess>();
|
||||
const tempDirs = new Set<string>();
|
||||
const trackedGroupLeaders = new Set<number>();
|
||||
const trackedPids = new Set<number>();
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && "code" in error && error.code === "ESRCH") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function waitFor(condition: () => boolean, timeoutMs = 5_000, intervalMs = 50): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (condition()) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
|
||||
throw new Error(`Condition not met within ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
function registerPid(pid: number) {
|
||||
trackedPids.add(pid);
|
||||
}
|
||||
|
||||
function registerGroupLeader(pid: number) {
|
||||
trackedGroupLeaders.add(pid);
|
||||
registerPid(pid);
|
||||
}
|
||||
|
||||
function createStubProcessTree() {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "fusion-run-vitest-"));
|
||||
tempDirs.add(tempDir);
|
||||
|
||||
const pidFile = join(tempDir, "pids.json");
|
||||
const grandchildPath = join(tempDir, "grandchild.mjs");
|
||||
const childPath = join(tempDir, "child.mjs");
|
||||
|
||||
writeFileSync(
|
||||
grandchildPath,
|
||||
['setInterval(() => {}, 1_000);'].join("\n"),
|
||||
);
|
||||
|
||||
writeFileSync(
|
||||
childPath,
|
||||
[
|
||||
'import { writeFileSync } from "node:fs";',
|
||||
'import { spawn } from "node:child_process";',
|
||||
'',
|
||||
'const pidFile = process.argv[2];',
|
||||
'const grandchildPath = process.argv[3];',
|
||||
'const grandchild = spawn(process.execPath, [grandchildPath], { stdio: "ignore" });',
|
||||
'writeFileSync(pidFile, JSON.stringify({ childPid: process.pid, grandchildPid: grandchild.pid }));',
|
||||
'setInterval(() => {}, 1_000);',
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
return { pidFile, childPath, grandchildPath, tempDir };
|
||||
}
|
||||
|
||||
async function spawnWrapperTree(signal: NodeJS.Signals) {
|
||||
const { pidFile, childPath, grandchildPath } = createStubProcessTree();
|
||||
const wrapper = spawn(
|
||||
process.execPath,
|
||||
[wrapperPath, "--heap=6144", "run", "--project", "dashboard-api-quality"],
|
||||
{
|
||||
cwd: dashboardRoot,
|
||||
stdio: "pipe",
|
||||
env: {
|
||||
...process.env,
|
||||
FUSION_RUN_VITEST_SPAWN_OVERRIDE: JSON.stringify({
|
||||
command: process.execPath,
|
||||
args: [childPath, pidFile, grandchildPath],
|
||||
}),
|
||||
},
|
||||
},
|
||||
);
|
||||
activeWrappers.add(wrapper);
|
||||
|
||||
let pids: { childPid: number; grandchildPid: number } | null = null;
|
||||
await waitFor(() => {
|
||||
try {
|
||||
pids = JSON.parse(readFileSync(pidFile, "utf8")) as { childPid: number; grandchildPid: number };
|
||||
return Boolean(
|
||||
pids &&
|
||||
Number.isInteger(pids.childPid) &&
|
||||
Number.isInteger(pids.grandchildPid) &&
|
||||
isProcessAlive(pids.childPid) &&
|
||||
isProcessAlive(pids.grandchildPid),
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
registerGroupLeader(pids!.childPid);
|
||||
registerPid(pids!.grandchildPid);
|
||||
|
||||
wrapper.kill(signal);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
wrapper.once("error", reject);
|
||||
wrapper.once("close", () => resolve());
|
||||
});
|
||||
activeWrappers.delete(wrapper);
|
||||
|
||||
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const wrapper of activeWrappers) {
|
||||
wrapper.kill("SIGKILL");
|
||||
}
|
||||
activeWrappers.clear();
|
||||
|
||||
for (const leaderPid of trackedGroupLeaders) {
|
||||
try {
|
||||
process.kill(-leaderPid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
trackedGroupLeaders.clear();
|
||||
|
||||
for (const pid of trackedPids) {
|
||||
try {
|
||||
process.kill(pid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
trackedPids.clear();
|
||||
|
||||
for (const tempDir of tempDirs) {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
tempDirs.clear();
|
||||
});
|
||||
|
||||
describe("run-vitest-with-heap", () => {
|
||||
it("reaps the spawned process group on SIGTERM", async () => {
|
||||
await spawnWrapperTree("SIGTERM");
|
||||
});
|
||||
|
||||
it("reaps the spawned process group on SIGINT", async () => {
|
||||
await spawnWrapperTree("SIGINT");
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@
|
||||
/* global clearInterval, console, process, setInterval */
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const rawArgs = process.argv.slice(2);
|
||||
const heapArg = rawArgs.find((arg) => arg.startsWith("--heap="));
|
||||
const heapMb = heapArg?.slice("--heap=".length) || "6144";
|
||||
@@ -15,7 +16,34 @@ if (vitestArgs.length === 0) {
|
||||
const nodeOptions = [`--max-old-space-size=${heapMb}`, process.env.NODE_OPTIONS || ""]
|
||||
.join(" ")
|
||||
.trim();
|
||||
const child = spawn("pnpm", ["exec", "vitest", ...vitestArgs], {
|
||||
|
||||
function resolveSpawnCommand() {
|
||||
const override = process.env.FUSION_RUN_VITEST_SPAWN_OVERRIDE;
|
||||
if (!override) {
|
||||
return { command: "pnpm", args: ["exec", "vitest", ...vitestArgs] };
|
||||
}
|
||||
|
||||
const parsedOverride = JSON.parse(override);
|
||||
if (
|
||||
!parsedOverride ||
|
||||
typeof parsedOverride.command !== "string" ||
|
||||
parsedOverride.command.length === 0 ||
|
||||
!Array.isArray(parsedOverride.args) ||
|
||||
parsedOverride.args.some((arg) => typeof arg !== "string")
|
||||
) {
|
||||
throw new Error(
|
||||
"FUSION_RUN_VITEST_SPAWN_OVERRIDE must be valid JSON with string command and string[] args",
|
||||
);
|
||||
}
|
||||
|
||||
// Test seam for process-lifecycle coverage without launching real vitest.
|
||||
return { command: parsedOverride.command, args: parsedOverride.args };
|
||||
}
|
||||
|
||||
const { command, args } = resolveSpawnCommand();
|
||||
// process-supervisor-allowlist: foreground wrapper signals the entire vitest process group on death/timeout; not a background daemon
|
||||
const child = spawn(command, args, {
|
||||
detached: true,
|
||||
stdio: "inherit",
|
||||
env: { ...process.env, NODE_OPTIONS: nodeOptions },
|
||||
});
|
||||
@@ -24,15 +52,62 @@ const heartbeat = setInterval(() => {
|
||||
console.log(`[dashboard-vitest] still running: ${vitestArgs.join(" ")}`);
|
||||
}, 5_000);
|
||||
|
||||
const forwardSignal = (signal) => {
|
||||
child.kill(signal);
|
||||
};
|
||||
function clearHeartbeat() {
|
||||
clearInterval(heartbeat);
|
||||
}
|
||||
|
||||
process.on("SIGINT", () => forwardSignal("SIGINT"));
|
||||
process.on("SIGTERM", () => forwardSignal("SIGTERM"));
|
||||
function forwardSignal(signal) {
|
||||
clearHeartbeat();
|
||||
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !("code" in error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.code !== "ESRCH" && error.code !== "EPERM") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
child.kill(signal);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
||||
process.on(signal, () => forwardSignal(signal));
|
||||
}
|
||||
|
||||
process.on("exit", () => {
|
||||
clearHeartbeat();
|
||||
try {
|
||||
process.kill(-child.pid, "SIGTERM");
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof Error) ||
|
||||
!("code" in error) ||
|
||||
(error.code !== "ESRCH" && error.code !== "EPERM")
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
child.on("error", (error) => {
|
||||
clearHeartbeat();
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
child.on("close", (code, signal) => {
|
||||
clearInterval(heartbeat);
|
||||
clearHeartbeat();
|
||||
if (signal) {
|
||||
process.kill(process.pid, signal);
|
||||
return;
|
||||
|
||||
@@ -73,5 +73,6 @@ describe("dashboard test config guard", () => {
|
||||
}
|
||||
|
||||
expect(vitestConfig).toContain('"app/__tests__/spinner-animation.css.test.ts"');
|
||||
expect(vitestConfig).toContain('"scripts/__tests__/run-vitest-with-heap.test.ts"');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -213,6 +213,7 @@ const qualityApiTests = [
|
||||
"src/__tests__/{api-error,auth-middleware,auth-middleware-integration,chat-attachment-routes,chat-routes,file-service,github,github-webhooks,initialize,planning-flow-diagnostics-guardrail,pr-routes-auto-merge,pr-routes.contract,project-routes,project-store-resolver,register-git-github.pr-options-preflight-metadata,remote-access-routes,remote-auth,routes-agent-budget,routes-agent-keys,routes-agent-permissions,routes-agent-ratings,routes-agent-runs,routes-agent-soul-memory,routes-agents,routes-automation,routes-branch-groups,routes-git,routes-github,routes-merge-advance-push-origin,routes-nodes,routes-nodes-sync-contract,routes-planning,routes-secrets-sync,routes-settings,routes-task-commit-associations,routes-tasks,routes-tasks-deterministic-dedup,routes-tasks-duplicate-check,routes-tasks-explicit-duplicate-marker,server,server-static-assets,server-webhook,server.events,setup-routes,sse,sse-buffer,test-isolation-guard,update-check-route,websocket,recover-branch-binding-route}.test.ts",
|
||||
"src/__tests__/dashboard-test-config-guard.test.ts",
|
||||
"src/routes/__tests__/{custom-provider-routes,custom-providers,register-docker-node-routes,register-diagnostics-routes,stash-recovery-routes}.test.ts",
|
||||
"scripts/__tests__/run-vitest-with-heap.test.ts",
|
||||
];
|
||||
|
||||
export default defineConfig({
|
||||
|
||||
Reference in New Issue
Block a user