FN-6184: log forwarded vitest wrapper signals
Improve dashboard vitest wrapper diagnostics so transient SIGTERM handling is easier to trace. - track the last signal and reason forwarded from the heap wrapper to the vitest process group - log when the wrapper receives SIGINT or SIGTERM and when timeout shutdown escalates - extend wrapper tests to assert process-group leadership and emitted diagnostic stderr Files changed: .../scripts/__tests__/run-vitest-with-heap.test.ts | 25 +++++++++++++++++++--- .../dashboard/scripts/run-vitest-with-heap.mjs | 19 ++++++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) Fusion-Task-Id: FN-6184 Fusion-Task-Lineage: b276d88e-a22d-4fdf-a533-c4b0aa4e199d
This commit is contained in:
@@ -49,6 +49,10 @@ function registerGroupLeader(pid: number) {
|
||||
registerPid(pid);
|
||||
}
|
||||
|
||||
function expectProcessGroupLeader(pid: number) {
|
||||
expect(() => process.kill(-pid, 0)).not.toThrow();
|
||||
}
|
||||
|
||||
function createStubProcessTree() {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "fusion-run-vitest-"));
|
||||
tempDirs.add(tempDir);
|
||||
@@ -97,6 +101,10 @@ async function spawnWrapperTree(signal: NodeJS.Signals) {
|
||||
},
|
||||
);
|
||||
activeWrappers.add(wrapper);
|
||||
let stderr = "";
|
||||
wrapper.stderr?.on("data", (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
|
||||
let pids: { childPid: number; grandchildPid: number } | null = null;
|
||||
await waitFor(() => {
|
||||
@@ -116,6 +124,7 @@ async function spawnWrapperTree(signal: NodeJS.Signals) {
|
||||
|
||||
registerGroupLeader(pids!.childPid);
|
||||
registerPid(pids!.grandchildPid);
|
||||
expectProcessGroupLeader(pids!.childPid);
|
||||
|
||||
wrapper.kill(signal);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -125,6 +134,7 @@ async function spawnWrapperTree(signal: NodeJS.Signals) {
|
||||
activeWrappers.delete(wrapper);
|
||||
|
||||
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
|
||||
return { stderr };
|
||||
}
|
||||
|
||||
async function spawnWrapperTreeUntilTimeout() {
|
||||
@@ -147,6 +157,10 @@ async function spawnWrapperTreeUntilTimeout() {
|
||||
},
|
||||
);
|
||||
activeWrappers.add(wrapper);
|
||||
let stderr = "";
|
||||
wrapper.stderr?.on("data", (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
|
||||
let pids: { childPid: number; grandchildPid: number } | null = null;
|
||||
await waitFor(() => {
|
||||
@@ -166,6 +180,7 @@ async function spawnWrapperTreeUntilTimeout() {
|
||||
|
||||
registerGroupLeader(pids!.childPid);
|
||||
registerPid(pids!.grandchildPid);
|
||||
expectProcessGroupLeader(pids!.childPid);
|
||||
|
||||
const exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
wrapper.once("error", reject);
|
||||
@@ -175,6 +190,7 @@ async function spawnWrapperTreeUntilTimeout() {
|
||||
|
||||
expect(exitCode).toBe(124);
|
||||
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
|
||||
return { stderr };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -213,14 +229,17 @@ afterEach(async () => {
|
||||
|
||||
describe("run-vitest-with-heap", () => {
|
||||
it("reaps the spawned process group on SIGTERM", async () => {
|
||||
await spawnWrapperTree("SIGTERM");
|
||||
const { stderr } = await spawnWrapperTree("SIGTERM");
|
||||
expect(stderr).toContain("[dashboard-vitest] received SIGTERM; forwarding to vitest process group");
|
||||
});
|
||||
|
||||
it("reaps the spawned process group on SIGINT", async () => {
|
||||
await spawnWrapperTree("SIGINT");
|
||||
const { stderr } = await spawnWrapperTree("SIGINT");
|
||||
expect(stderr).toContain("[dashboard-vitest] received SIGINT; forwarding to vitest process group");
|
||||
});
|
||||
|
||||
it("times out and reaps the spawned process group", async () => {
|
||||
await spawnWrapperTreeUntilTimeout();
|
||||
const { stderr } = await spawnWrapperTreeUntilTimeout();
|
||||
expect(stderr).toContain("[dashboard-vitest] timeout after 100ms");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -55,13 +55,15 @@ const heartbeat = setInterval(() => {
|
||||
}, 5_000);
|
||||
let timeoutExitCode = null;
|
||||
let forceKillTimer = null;
|
||||
let lastForwardedSignal = null;
|
||||
let lastForwardReason = null;
|
||||
const timeout = Number.isFinite(timeoutMs) && timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
timeoutExitCode = 124;
|
||||
console.error(`[dashboard-vitest] timeout after ${timeoutMs}ms: ${vitestArgs.join(" ")}`);
|
||||
forwardSignal("SIGTERM");
|
||||
forwardSignal("SIGTERM", "timeout");
|
||||
forceKillTimer = setTimeout(() => {
|
||||
forwardSignal("SIGKILL");
|
||||
forwardSignal("SIGKILL", "timeout-grace-expired");
|
||||
}, Math.max(1, forceKillGraceMs));
|
||||
forceKillTimer.unref();
|
||||
}, timeoutMs)
|
||||
@@ -78,8 +80,10 @@ function clearTimers() {
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
}
|
||||
|
||||
function forwardSignal(signal) {
|
||||
function forwardSignal(signal, reason = "external-signal") {
|
||||
clearHeartbeat();
|
||||
lastForwardedSignal = signal;
|
||||
lastForwardReason = reason;
|
||||
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
@@ -104,7 +108,10 @@ function forwardSignal(signal) {
|
||||
}
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
||||
process.on(signal, () => forwardSignal(signal));
|
||||
process.on(signal, () => {
|
||||
console.error(`[dashboard-vitest] received ${signal}; forwarding to vitest process group: ${vitestArgs.join(" ")}`);
|
||||
forwardSignal(signal, "wrapper-received-signal");
|
||||
});
|
||||
}
|
||||
|
||||
process.on("exit", () => {
|
||||
@@ -134,6 +141,10 @@ child.on("close", (code, signal) => {
|
||||
process.exit(timeoutExitCode);
|
||||
}
|
||||
if (signal) {
|
||||
const forwardedContext = lastForwardedSignal
|
||||
? ` after forwarding ${lastForwardedSignal} (${lastForwardReason ?? "unknown-reason"})`
|
||||
: " without a wrapper-forwarded signal";
|
||||
console.error(`[dashboard-vitest] child exited via ${signal}${forwardedContext}: ${vitestArgs.join(" ")}`);
|
||||
process.kill(process.pid, signal);
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user