Files
fusion/scripts/__tests__/run-vitest-watchdog.test.mjs
gsxdsm 2cff1864c9 fix: bound @fusion/core affected lane + tighten changed watchdog under engine kill
Three changes to make `pnpm test` reliably minimal and fail gracefully:

- @fusion/core is now a memory-envelope/wide-fan-out package (was unguarded).
  It's the hub nearly everything imports (~354 test files), so a core source
  edit made `vitest --changed` expand to ~the whole core suite and blow past the
  engine's 15-min verification kill -> SIGKILL + task restart. Adding it to
  SCOPED_AFFECTED_MEMORY_ENVELOPES applies the wide-fan-out guard (run only
  directly-changed core tests, else delegate) and the bounded env. core is NOT
  gate-covered, so delegation warns loudly rather than false-greens.

- Lower CLASS_BUDGET_BANDS.changed ceiling 20min -> 13min so the script watchdog
  fails a runaway local lane itself (exit 124, no restart) BEFORE the engine's
  15-min kill restarts the whole task. A tightening, not a timeout-widening.
  Guard test pins ceiling < 900_000ms.

- Raise scoped-affected worker fan-out 1 -> 4 (operator decision). Was 1 only
  for OOM safety (FN-6854/FN-6874); the fan-out guard now bounds the set so the
  hundreds-of-files OOM driver no longer reaches these workers. Heap stays
  6144MB/worker (~4x6GB on the lane) — revisit if a RAM-constrained CI runner
  OOMs. Trades FN-5048 worker-knob guidance for throughput, scoped to the
  bounded affected lanes only.

Tests: test-changed 117/117, watchdog 15/15, eslint clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-25 21:42:22 -07:00

249 lines
7.7 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { test } from "node:test";
import assert from "node:assert/strict";
import { EventEmitter } from "node:events";
import {
CLASS_BUDGET_BANDS,
DEFAULT_BUDGET_MULTIPLIER,
TIMEOUT_EXIT_CODE,
deriveBudgetMs,
summarizeActiveHandles,
captureHangDiagnostics,
runWithWatchdog,
} from "../lib/run-vitest-watchdog.mjs";
function makeFakeChild() {
const child = new EventEmitter();
child.pid = 999999;
child.kill = () => {};
return child;
}
// A spawn stub that returns a controllable fake child.
function fakeSpawn(child) {
return () => child;
}
test("deriveBudgetMs: no fresh timing falls back to the per-class ceiling", () => {
assert.equal(deriveBudgetMs({ klass: "shard" }), CLASS_BUDGET_BANDS.shard.ceiling);
assert.equal(
deriveBudgetMs({ klass: "changed", expectedDurationMs: 1000, timingsFresh: false }),
CLASS_BUDGET_BANDS.changed.ceiling,
);
// Zero / negative expected duration is treated as unusable → ceiling.
assert.equal(
deriveBudgetMs({ klass: "shard", expectedDurationMs: 0, timingsFresh: true }),
CLASS_BUDGET_BANDS.shard.ceiling,
);
});
test("deriveBudgetMs: fresh timing tightens within the band", () => {
// expected×multiplier between floor and ceiling → use the tightened value.
// 300s × 3.5 = 1050s, which sits between the shard floor (15min) and
// ceiling (30min) so the tightened value is used un-clamped.
const expected = 300_000; // 300s
const derived = deriveBudgetMs({ klass: "shard", expectedDurationMs: expected, timingsFresh: true });
assert.equal(derived, Math.round(expected * DEFAULT_BUDGET_MULTIPLIER));
assert.ok(derived >= CLASS_BUDGET_BANDS.shard.floor);
assert.ok(derived <= CLASS_BUDGET_BANDS.shard.ceiling);
});
test("deriveBudgetMs: clamps to floor and ceiling", () => {
// Tiny expected → clamps up to floor.
assert.equal(
deriveBudgetMs({ klass: "shard", expectedDurationMs: 1, timingsFresh: true }),
CLASS_BUDGET_BANDS.shard.floor,
);
// Huge expected → clamps down to ceiling.
assert.equal(
deriveBudgetMs({ klass: "shard", expectedDurationMs: 10 ** 9, timingsFresh: true }),
CLASS_BUDGET_BANDS.shard.ceiling,
);
});
test("deriveBudgetMs: shard floor pins heavy slices above the false-kill window", () => {
// FNXC:TestInfrastructure 2026-06-20-21:52:
// Regression guard for the 5min -> 15min shard-floor raise. A value whose
// expected×multiplier lands in the *old* un-clamped window (300s..900s) must
// now clamp UP to the 15min floor. 150s × 3.5 = 525s, which was returned
// verbatim under the old 5min floor but is below the new one. Pinning the
// concrete floor value here means an accidental revert to 5min fails loudly
// instead of silently re-tightening the engine/core slices into SIGKILLs.
assert.equal(CLASS_BUDGET_BANDS.shard.floor, 15 * 60_000);
assert.equal(
deriveBudgetMs({ klass: "shard", expectedDurationMs: 150_000, timingsFresh: true }),
CLASS_BUDGET_BANDS.shard.floor,
);
});
test("deriveBudgetMs: unknown class falls back to the changed band", () => {
assert.equal(deriveBudgetMs({ klass: "nonexistent" }), CLASS_BUDGET_BANDS.changed.ceiling);
});
test("summarizeActiveHandles: returns a bounded string", () => {
const summary = summarizeActiveHandles({ limit: 3 });
assert.equal(typeof summary, "string");
assert.ok(summary.length > 0);
});
test("captureHangDiagnostics: names the invocation, elapsed, and budget", () => {
const msg = captureHangDiagnostics({
label: "shard 1/4",
command: "pnpm",
args: ["test"],
budgetMs: 1000,
startedAt: 0,
lastHeartbeatAt: 500,
now: 1500,
});
assert.match(msg, /HANG: shard 1\/4/);
assert.match(msg, /elapsed 1500ms/);
assert.match(msg, /budget 1000ms/);
assert.match(msg, /last heartbeat: 1000ms ago/);
});
test("runWithWatchdog: clean exit propagates code 0, no kill", async () => {
const child = makeFakeChild();
const killed = [];
const p = runWithWatchdog({
command: "fake",
args: [],
budgetMs: 10_000,
label: "clean",
log: () => {},
spawn: fakeSpawn(child),
killGroup: (sig) => killed.push(sig),
});
child.emit("close", 0, null);
const result = await p;
assert.equal(result.code, 0);
assert.equal(result.timedOut, false);
assert.equal(result.signal, null);
assert.deepEqual(killed, []);
});
test("runWithWatchdog: non-zero exit code is propagated unchanged", async () => {
const child = makeFakeChild();
const p = runWithWatchdog({
command: "fake",
args: [],
budgetMs: 10_000,
label: "fails",
log: () => {},
spawn: fakeSpawn(child),
killGroup: () => {},
});
child.emit("close", 7, null);
const result = await p;
assert.equal(result.code, 7);
assert.equal(result.timedOut, false);
});
test("runWithWatchdog: timeout fires SIGTERM then SIGKILL and returns 124", async () => {
const child = makeFakeChild();
const killed = [];
let diagnosticsLogged = "";
const p = runWithWatchdog({
command: "pnpm",
args: ["exec", "vitest"],
budgetMs: 30, // fire fast
graceMs: 20,
heartbeatMs: 1000,
label: "hanger",
log: (m) => {
diagnosticsLogged += m + "\n";
},
spawn: fakeSpawn(child),
killGroup: (sig) => {
killed.push(sig);
// Emulate the group dying only after SIGKILL.
if (sig === "SIGKILL") setTimeout(() => child.emit("close", null, "SIGKILL"), 1);
},
});
const result = await p;
assert.equal(result.timedOut, true);
assert.equal(result.code, TIMEOUT_EXIT_CODE);
assert.deepEqual(killed, ["SIGTERM", "SIGKILL"]);
assert.match(diagnosticsLogged, /HANG: hanger/);
});
test("runWithWatchdog: child error rejects", async () => {
const child = makeFakeChild();
const p = runWithWatchdog({
command: "fake",
args: [],
budgetMs: 10_000,
label: "errors",
log: () => {},
spawn: fakeSpawn(child),
killGroup: () => {},
});
child.emit("error", new Error("spawn failed"));
await assert.rejects(p, /spawn failed/);
});
test("runWithWatchdog: removes its process listeners after settling", async () => {
const beforeTerm = process.listenerCount("SIGTERM");
const beforeExit = process.listenerCount("exit");
const child = makeFakeChild();
const p = runWithWatchdog({
command: "fake",
args: [],
budgetMs: 10_000,
label: "cleanup",
log: () => {},
spawn: fakeSpawn(child),
killGroup: () => {},
});
child.emit("close", 0, null);
await p;
assert.equal(process.listenerCount("SIGTERM"), beforeTerm);
assert.equal(process.listenerCount("exit"), beforeExit);
});
test("runWithWatchdog: forwarded signal escalates to SIGKILL after grace", async () => {
const child = makeFakeChild();
const killed = [];
const p = runWithWatchdog({
command: "pnpm",
args: [],
budgetMs: 10_000,
graceMs: 15,
heartbeatMs: 1000,
label: "cancel",
log: () => {},
spawn: fakeSpawn(child),
killGroup: (sig) => {
killed.push(sig);
// The child ignores SIGHUP; only SIGKILL takes it down.
if (sig === "SIGKILL") child.emit("close", null, "SIGKILL");
},
});
// Simulate external cancellation (Ctrl-C / CI cancel) reaching the wrapper.
process.emit("SIGHUP");
await new Promise((resolve) => setTimeout(resolve, 50));
await p;
assert.deepEqual(killed, ["SIGHUP", "SIGKILL"]);
});
test("runWithWatchdog: passes cwd through to spawn when provided", async () => {
let capturedOpts = null;
const child = makeFakeChild();
const p = runWithWatchdog({
command: "pnpm",
args: ["test"],
cwd: "/tmp/repo-root",
budgetMs: 10_000,
label: "cwd",
log: () => {},
spawn: (_cmd, _args, opts) => {
capturedOpts = opts;
return child;
},
killGroup: () => {},
});
child.emit("close", 0, null);
await p;
assert.equal(capturedOpts.cwd, "/tmp/repo-root");
});