fix: bound @fusion/core affected lane + tighten changed watchdog under engine kill
Three changes to make `pnpm test` reliably minimal and fail gracefully: - @fusion/core is now a memory-envelope/wide-fan-out package (was unguarded). It's the hub nearly everything imports (~354 test files), so a core source edit made `vitest --changed` expand to ~the whole core suite and blow past the engine's 15-min verification kill -> SIGKILL + task restart. Adding it to SCOPED_AFFECTED_MEMORY_ENVELOPES applies the wide-fan-out guard (run only directly-changed core tests, else delegate) and the bounded env. core is NOT gate-covered, so delegation warns loudly rather than false-greens. - Lower CLASS_BUDGET_BANDS.changed ceiling 20min -> 13min so the script watchdog fails a runaway local lane itself (exit 124, no restart) BEFORE the engine's 15-min kill restarts the whole task. A tightening, not a timeout-widening. Guard test pins ceiling < 900_000ms. - Raise scoped-affected worker fan-out 1 -> 4 (operator decision). Was 1 only for OOM safety (FN-6854/FN-6874); the fan-out guard now bounds the set so the hundreds-of-files OOM driver no longer reaches these workers. Heap stays 6144MB/worker (~4x6GB on the lane) — revisit if a RAM-constrained CI runner OOMs. Trades FN-5048 worker-knob guidance for throughput, scoped to the bounded affected lanes only. Tests: test-changed 117/117, watchdog 15/15, eslint clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
@@ -37,6 +37,24 @@ test("deriveBudgetMs: no fresh timing falls back to the per-class ceiling", () =
|
||||
);
|
||||
});
|
||||
|
||||
test("the `changed` ceiling must sit below the engine's 15-min verification kill", () => {
|
||||
// FNXC:TestInfrastructure 2026-06-26-13:05: a stale/missing timings snapshot
|
||||
// makes deriveBudgetMs return the `changed` ceiling. That ceiling MUST stay
|
||||
// under VERIFICATION_TIMEOUT_WORKSPACE_MS (900_000 = 15min in
|
||||
// packages/engine/src/verification-utils.ts) so the script watchdog fails a
|
||||
// runaway local affected lane itself (exit 124, no task restart) instead of
|
||||
// letting the engine SIGKILL `pnpm test` and restart the whole task. If this
|
||||
// assertion ever trips, lower CLASS_BUDGET_BANDS.changed.ceiling — do not
|
||||
// raise the engine kill.
|
||||
const ENGINE_VERIFICATION_KILL_MS = 900_000;
|
||||
const ceiling = deriveBudgetMs({ klass: "changed", expectedDurationMs: 1000, timingsFresh: false });
|
||||
assert.equal(ceiling, CLASS_BUDGET_BANDS.changed.ceiling);
|
||||
assert.ok(
|
||||
CLASS_BUDGET_BANDS.changed.ceiling < ENGINE_VERIFICATION_KILL_MS,
|
||||
`changed ceiling ${CLASS_BUDGET_BANDS.changed.ceiling}ms must be < engine kill ${ENGINE_VERIFICATION_KILL_MS}ms`,
|
||||
);
|
||||
});
|
||||
|
||||
test("deriveBudgetMs: fresh timing tightens within the band", () => {
|
||||
// expected×multiplier between floor and ceiling → use the tightened value.
|
||||
// 300s × 3.5 = 1050s, which sits between the shard floor (15min) and
|
||||
|
||||
@@ -47,6 +47,11 @@ import {
|
||||
changedSourceFilesAffectingPackage,
|
||||
existingChangedTestFilesInPackage,
|
||||
GATE_COVERED_MEMORY_ENVELOPE_PACKAGES,
|
||||
SCOPED_AFFECTED_MEMORY_ENVELOPES,
|
||||
CORE_SCOPED_AFFECTED_PACKAGE,
|
||||
CORE_SCOPED_AFFECTED_HEAP_MB,
|
||||
CORE_SCOPED_AFFECTED_WORKERS,
|
||||
createScopedAffectedMemoryEnvelopeEnv,
|
||||
} from "../test-changed.mjs";
|
||||
|
||||
import { deriveBudgetMs } from "../lib/run-vitest-watchdog.mjs";
|
||||
@@ -289,7 +294,7 @@ function assertScopedAffectedEnv(env, { heapMb, workers }) {
|
||||
assert.equal(env.HOME, "/tmp/fusion-home");
|
||||
}
|
||||
|
||||
test("partitionScopedAffectedPackages: isolates dashboard and engine into separate envelope groups", () => {
|
||||
test("partitionScopedAffectedPackages: isolates core, dashboard, and engine into separate envelope groups", () => {
|
||||
assert.deepEqual(summarizeScopedAffectedGroups([DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [
|
||||
{
|
||||
packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE],
|
||||
@@ -298,19 +303,30 @@ test("partitionScopedAffectedPackages: isolates dashboard and engine into separa
|
||||
},
|
||||
]);
|
||||
|
||||
assert.deepEqual(summarizeScopedAffectedGroups(["@fusion/core", DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [
|
||||
{ packages: ["@fusion/core"], engineMemoryEnvelope: false, memoryEnvelopePackage: null },
|
||||
// FNXC:TestInfrastructure 2026-06-26-12:40: @fusion/core is now its own
|
||||
// memory-envelope group (no longer a regular package), so the wide-fan-out
|
||||
// guard and bounded heap/worker env apply. Group order follows
|
||||
// SCOPED_AFFECTED_MEMORY_ENVELOPES key order: engine, dashboard, core.
|
||||
assert.deepEqual(summarizeScopedAffectedGroups([CORE_SCOPED_AFFECTED_PACKAGE, DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [
|
||||
{
|
||||
packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE],
|
||||
engineMemoryEnvelope: false,
|
||||
memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE,
|
||||
},
|
||||
{
|
||||
packages: [CORE_SCOPED_AFFECTED_PACKAGE],
|
||||
engineMemoryEnvelope: false,
|
||||
memoryEnvelopePackage: CORE_SCOPED_AFFECTED_PACKAGE,
|
||||
},
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
summarizeScopedAffectedGroups(["@fusion/core", DASHBOARD_SCOPED_AFFECTED_PACKAGE, ENGINE_SCOPED_AFFECTED_PACKAGE]),
|
||||
summarizeScopedAffectedGroups([
|
||||
CORE_SCOPED_AFFECTED_PACKAGE,
|
||||
DASHBOARD_SCOPED_AFFECTED_PACKAGE,
|
||||
ENGINE_SCOPED_AFFECTED_PACKAGE,
|
||||
]),
|
||||
[
|
||||
{ packages: ["@fusion/core"], engineMemoryEnvelope: false, memoryEnvelopePackage: null },
|
||||
{
|
||||
packages: [ENGINE_SCOPED_AFFECTED_PACKAGE],
|
||||
engineMemoryEnvelope: true,
|
||||
@@ -321,14 +337,47 @@ test("partitionScopedAffectedPackages: isolates dashboard and engine into separa
|
||||
engineMemoryEnvelope: false,
|
||||
memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE,
|
||||
},
|
||||
{
|
||||
packages: [CORE_SCOPED_AFFECTED_PACKAGE],
|
||||
engineMemoryEnvelope: false,
|
||||
memoryEnvelopePackage: CORE_SCOPED_AFFECTED_PACKAGE,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
assert.deepEqual(summarizeScopedAffectedGroups(["@fusion/core", "@runfusion/fusion"]), [
|
||||
{ packages: ["@fusion/core", "@runfusion/fusion"], engineMemoryEnvelope: false, memoryEnvelopePackage: null },
|
||||
// A genuinely regular package stays in the shared regular group; core splits out.
|
||||
assert.deepEqual(summarizeScopedAffectedGroups([CORE_SCOPED_AFFECTED_PACKAGE, "@runfusion/fusion"]), [
|
||||
{ packages: ["@runfusion/fusion"], engineMemoryEnvelope: false, memoryEnvelopePackage: null },
|
||||
{
|
||||
packages: [CORE_SCOPED_AFFECTED_PACKAGE],
|
||||
engineMemoryEnvelope: false,
|
||||
memoryEnvelopePackage: CORE_SCOPED_AFFECTED_PACKAGE,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("@fusion/core is a wide-fan-out memory-envelope package but is NOT gate-covered", () => {
|
||||
// It must be bounded (guard applies) ...
|
||||
assert.ok(
|
||||
Object.keys(SCOPED_AFFECTED_MEMORY_ENVELOPES).includes(CORE_SCOPED_AFFECTED_PACKAGE),
|
||||
"core must be a memory-envelope package so the wide-fan-out guard runs only directly-changed core tests",
|
||||
);
|
||||
// ... yet must NOT claim gate coverage (the merge gate runs no core suite),
|
||||
// so a delegated core lane warns loudly instead of reporting a false green.
|
||||
assert.ok(
|
||||
!GATE_COVERED_MEMORY_ENVELOPE_PACKAGES.has(CORE_SCOPED_AFFECTED_PACKAGE),
|
||||
"core is not covered by the merge gate; delegation must warn, not reassure",
|
||||
);
|
||||
});
|
||||
|
||||
test("core scoped-affected env applies the bounded heap and single-worker envelope", () => {
|
||||
const env = createScopedAffectedMemoryEnvelopeEnv(CORE_SCOPED_AFFECTED_PACKAGE, {
|
||||
NODE_OPTIONS: "--trace-warnings",
|
||||
HOME: "/tmp/fusion-home",
|
||||
});
|
||||
assertScopedAffectedEnv(env, { heapMb: CORE_SCOPED_AFFECTED_HEAP_MB, workers: CORE_SCOPED_AFFECTED_WORKERS });
|
||||
});
|
||||
|
||||
test("createDashboardScopedAffectedEnv: caps heap, preserves env, lowers workers, and leaves watchdog finite", () => {
|
||||
const env = createDashboardScopedAffectedEnv({
|
||||
NODE_OPTIONS: "--trace-warnings",
|
||||
|
||||
@@ -52,8 +52,21 @@ export const CLASS_BUDGET_BANDS = {
|
||||
the two values are not coupled and may diverge.
|
||||
*/
|
||||
shard: { floor: 15 * MINUTE, ceiling: 30 * MINUTE },
|
||||
/*
|
||||
FNXC:TestInfrastructure 2026-06-26-12:40:
|
||||
The `changed` ceiling MUST sit below the engine's per-task verification kill
|
||||
(`VERIFICATION_TIMEOUT_WORKSPACE_MS = 900_000` = 15min, verification-utils.ts).
|
||||
This band is the bound for a local changed-file affected-lane invocation
|
||||
(`pnpm test` in changed mode). When the timings snapshot is stale (the common
|
||||
case), deriveBudgetMs returns this ceiling. At the old 20min ceiling the script
|
||||
watchdog NEVER fired before the engine's 15min kill, so a runaway lane was
|
||||
SIGKILLed by the engine and the whole task RESTARTED (stacked 15-min timeouts)
|
||||
instead of failing the lane cleanly here (exit 124, no restart). 13min leaves
|
||||
margin under the 15min kill so the script fails the lane itself first. Lowering
|
||||
a ceiling is a tightening, not a timeout-widening appeasement.
|
||||
*/
|
||||
// One local changed-file package invocation.
|
||||
changed: { floor: 2 * MINUTE, ceiling: 20 * MINUTE },
|
||||
changed: { floor: 2 * MINUTE, ceiling: 13 * MINUTE },
|
||||
// One dashboard quality lane (heap-managed). Matches the historical 15min.
|
||||
"dashboard-lane": { floor: 15 * MINUTE, ceiling: 30 * MINUTE },
|
||||
};
|
||||
|
||||
@@ -1317,13 +1317,45 @@ export function packageHasVitestConfig(pkgDir, projectRoot = rootDir) {
|
||||
return VITEST_CONFIG_BASENAMES.some((name) => existsSync(path.join(projectRoot, pkgDir, name)));
|
||||
}
|
||||
|
||||
/*
|
||||
FNXC:TestInfrastructure 2026-06-26-13:05:
|
||||
Scoped-affected worker fan-out was raised 1 -> 4 (operator decision). It was 1
|
||||
purely for OOM safety (FN-6854/FN-6874: heavy affected lanes OS-OOM-SIGKILLed
|
||||
even at concurrency=1). Two things make 4 acceptable now: (1) the wide-fan-out
|
||||
guard below bounds each heavy lane to a few directly-changed test files, so the
|
||||
hundreds-of-files set that drove the OOM no longer reaches these workers; (2) the
|
||||
heap cap stays 6144MB PER WORKER, so this lane can now use up to ~4x6GB ≈ 24GB —
|
||||
fine on the 256GB host, but if a RAM-constrained CI runner OOM-SIGKILLs a heavy
|
||||
lane again, lower this back toward 1 (or drop the per-worker heap) rather than
|
||||
widening timeouts. This intentionally trades the FN-5048 "don't raise worker
|
||||
knobs" guidance for throughput, scoped to the bounded affected lanes only.
|
||||
*/
|
||||
export const ENGINE_SCOPED_AFFECTED_PACKAGE = "@fusion/engine";
|
||||
export const ENGINE_SCOPED_AFFECTED_HEAP_MB = "6144";
|
||||
export const ENGINE_SCOPED_AFFECTED_WORKERS = "1";
|
||||
export const ENGINE_SCOPED_AFFECTED_WORKERS = "4";
|
||||
export const DASHBOARD_SCOPED_AFFECTED_PACKAGE = "@fusion/dashboard";
|
||||
export const DASHBOARD_SCOPED_AFFECTED_HEAP_MB = "6144";
|
||||
export const DASHBOARD_SCOPED_AFFECTED_WORKERS = "1";
|
||||
export const DASHBOARD_SCOPED_AFFECTED_WORKERS = "4";
|
||||
export const CORE_SCOPED_AFFECTED_PACKAGE = "@fusion/core";
|
||||
export const CORE_SCOPED_AFFECTED_HEAP_MB = "6144";
|
||||
export const CORE_SCOPED_AFFECTED_WORKERS = "4";
|
||||
|
||||
/*
|
||||
FNXC:TestInfrastructure 2026-06-26-12:40:
|
||||
`@fusion/core` is a memory-envelope/wide-fan-out package too — it was the
|
||||
remaining `pnpm test` timeout path after engine/dashboard were bounded. core is
|
||||
the hub nearly every package imports and has ~354 test files (db.test 21s,
|
||||
mission-store 16s, ...). A non-test core SOURCE edit (e.g. store.ts/db.ts) makes
|
||||
`vitest --changed` expand to ~the whole core suite at this real-git +
|
||||
sqlite-heavy lane and blow past the engine's 15-min verification kill, which then
|
||||
SIGKILLs `pnpm test` and RESTARTS the task — stacked 15-min timeouts. Listing
|
||||
core here makes `partitionScopedAffectedPackages` treat it as its own
|
||||
memory-envelope group so the wide-fan-out guard (run only directly-changed core
|
||||
test files, else delegate) and the bounded heap/worker env both apply. core is
|
||||
intentionally NOT in GATE_COVERED_MEMORY_ENVELOPE_PACKAGES (the gate runs no core
|
||||
suite), so a delegated core lane emits the loud "not covered by gate; run
|
||||
`pnpm test:full`" warning rather than a silent false-green.
|
||||
*/
|
||||
export const SCOPED_AFFECTED_MEMORY_ENVELOPES = Object.freeze({
|
||||
[ENGINE_SCOPED_AFFECTED_PACKAGE]: Object.freeze({
|
||||
packageName: ENGINE_SCOPED_AFFECTED_PACKAGE,
|
||||
@@ -1335,6 +1367,11 @@ export const SCOPED_AFFECTED_MEMORY_ENVELOPES = Object.freeze({
|
||||
heapMb: DASHBOARD_SCOPED_AFFECTED_HEAP_MB,
|
||||
workers: DASHBOARD_SCOPED_AFFECTED_WORKERS,
|
||||
}),
|
||||
[CORE_SCOPED_AFFECTED_PACKAGE]: Object.freeze({
|
||||
packageName: CORE_SCOPED_AFFECTED_PACKAGE,
|
||||
heapMb: CORE_SCOPED_AFFECTED_HEAP_MB,
|
||||
workers: CORE_SCOPED_AFFECTED_WORKERS,
|
||||
}),
|
||||
});
|
||||
|
||||
/*
|
||||
@@ -1359,7 +1396,7 @@ export function createScopedAffectedMemoryEnvelopeEnv(packageName, env = process
|
||||
if (!envelope) return env;
|
||||
/*
|
||||
FNXC:TestInfrastructure 2026-06-21-11:24:
|
||||
The engine affected lane can select hundreds of real-git-heavy files when `vitest --changed` sees a widely imported boundary. Run that scoped lane in its own memory envelope: cap Node old-space like the dashboard heap runner and lower Vitest worker fan-out to one process so the lane returns a real pass/fail verdict instead of an OS OOM SIGKILL. Keep watchdog timing outside this env so hangs still fail through `runWithWatchdog`.
|
||||
The engine affected lane can select hundreds of real-git-heavy files when `vitest --changed` sees a widely imported boundary. Run that scoped lane in its own memory envelope: cap Node old-space like the dashboard heap runner and bound Vitest worker fan-out (see SCOPED_AFFECTED_WORKERS) so the lane returns a real pass/fail verdict instead of an OS OOM SIGKILL. Keep watchdog timing outside this env so hangs still fail through `runWithWatchdog`.
|
||||
|
||||
FNXC:TestInfrastructure 2026-06-21-16:28:
|
||||
FN-6874 showed the dashboard changed-mode affected lane can OOM/SIGKILL even with `FUSION_TEST_CONCURRENCY=1 FUSION_TEST_WORKSPACE_CONCURRENCY=1`, so worker fan-out alone is not the failure mode. Give each heavy scoped package its own bounded heap envelope while preserving caller env and keeping the finite changed-class watchdog outside this env so hangs still fail instead of being masked.
|
||||
|
||||
Reference in New Issue
Block a user