From 2cff1864c96c44418cbd29d4d08e2a10f7b5cd36 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Thu, 25 Jun 2026 21:42:22 -0700 Subject: [PATCH] fix: bound @fusion/core affected lane + tighten changed watchdog under engine kill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes to make `pnpm test` reliably minimal and fail gracefully: - @fusion/core is now a memory-envelope/wide-fan-out package (was unguarded). It's the hub nearly everything imports (~354 test files), so a core source edit made `vitest --changed` expand to ~the whole core suite and blow past the engine's 15-min verification kill -> SIGKILL + task restart. Adding it to SCOPED_AFFECTED_MEMORY_ENVELOPES applies the wide-fan-out guard (run only directly-changed core tests, else delegate) and the bounded env. core is NOT gate-covered, so delegation warns loudly rather than false-greens. - Lower CLASS_BUDGET_BANDS.changed ceiling 20min -> 13min so the script watchdog fails a runaway local lane itself (exit 124, no restart) BEFORE the engine's 15-min kill restarts the whole task. A tightening, not a timeout-widening. Guard test pins ceiling < 900_000ms. - Raise scoped-affected worker fan-out 1 -> 4 (operator decision). Was 1 only for OOM safety (FN-6854/FN-6874); the fan-out guard now bounds the set so the hundreds-of-files OOM driver no longer reaches these workers. Heap stays 6144MB/worker (~4x6GB on the lane) — revisit if a RAM-constrained CI runner OOMs. Trades FN-5048 worker-knob guidance for throughput, scoped to the bounded affected lanes only. Tests: test-changed 117/117, watchdog 15/15, eslint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/run-vitest-watchdog.test.mjs | 18 ++++++ scripts/__tests__/test-changed.test.mjs | 63 ++++++++++++++++--- scripts/lib/run-vitest-watchdog.mjs | 15 ++++- scripts/test-changed.mjs | 43 ++++++++++++- 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/scripts/__tests__/run-vitest-watchdog.test.mjs b/scripts/__tests__/run-vitest-watchdog.test.mjs index 191dbf0ec2..65ba64f730 100644 --- a/scripts/__tests__/run-vitest-watchdog.test.mjs +++ b/scripts/__tests__/run-vitest-watchdog.test.mjs @@ -37,6 +37,24 @@ test("deriveBudgetMs: no fresh timing falls back to the per-class ceiling", () = ); }); +test("the `changed` ceiling must sit below the engine's 15-min verification kill", () => { + // FNXC:TestInfrastructure 2026-06-26-13:05: a stale/missing timings snapshot + // makes deriveBudgetMs return the `changed` ceiling. That ceiling MUST stay + // under VERIFICATION_TIMEOUT_WORKSPACE_MS (900_000 = 15min in + // packages/engine/src/verification-utils.ts) so the script watchdog fails a + // runaway local affected lane itself (exit 124, no task restart) instead of + // letting the engine SIGKILL `pnpm test` and restart the whole task. If this + // assertion ever trips, lower CLASS_BUDGET_BANDS.changed.ceiling — do not + // raise the engine kill. + const ENGINE_VERIFICATION_KILL_MS = 900_000; + const ceiling = deriveBudgetMs({ klass: "changed", expectedDurationMs: 1000, timingsFresh: false }); + assert.equal(ceiling, CLASS_BUDGET_BANDS.changed.ceiling); + assert.ok( + CLASS_BUDGET_BANDS.changed.ceiling < ENGINE_VERIFICATION_KILL_MS, + `changed ceiling ${CLASS_BUDGET_BANDS.changed.ceiling}ms must be < engine kill ${ENGINE_VERIFICATION_KILL_MS}ms`, + ); +}); + test("deriveBudgetMs: fresh timing tightens within the band", () => { // expected×multiplier between floor and ceiling → use the tightened value. // 300s × 3.5 = 1050s, which sits between the shard floor (15min) and diff --git a/scripts/__tests__/test-changed.test.mjs b/scripts/__tests__/test-changed.test.mjs index 62e85c4c0f..b0da039c2d 100644 --- a/scripts/__tests__/test-changed.test.mjs +++ b/scripts/__tests__/test-changed.test.mjs @@ -47,6 +47,11 @@ import { changedSourceFilesAffectingPackage, existingChangedTestFilesInPackage, GATE_COVERED_MEMORY_ENVELOPE_PACKAGES, + SCOPED_AFFECTED_MEMORY_ENVELOPES, + CORE_SCOPED_AFFECTED_PACKAGE, + CORE_SCOPED_AFFECTED_HEAP_MB, + CORE_SCOPED_AFFECTED_WORKERS, + createScopedAffectedMemoryEnvelopeEnv, } from "../test-changed.mjs"; import { deriveBudgetMs } from "../lib/run-vitest-watchdog.mjs"; @@ -289,7 +294,7 @@ function assertScopedAffectedEnv(env, { heapMb, workers }) { assert.equal(env.HOME, "/tmp/fusion-home"); } -test("partitionScopedAffectedPackages: isolates dashboard and engine into separate envelope groups", () => { +test("partitionScopedAffectedPackages: isolates core, dashboard, and engine into separate envelope groups", () => { assert.deepEqual(summarizeScopedAffectedGroups([DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [ { packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE], @@ -298,19 +303,30 @@ test("partitionScopedAffectedPackages: isolates dashboard and engine into separa }, ]); - assert.deepEqual(summarizeScopedAffectedGroups(["@fusion/core", DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [ - { packages: ["@fusion/core"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, + // FNXC:TestInfrastructure 2026-06-26-12:40: @fusion/core is now its own + // memory-envelope group (no longer a regular package), so the wide-fan-out + // guard and bounded heap/worker env apply. Group order follows + // SCOPED_AFFECTED_MEMORY_ENVELOPES key order: engine, dashboard, core. + assert.deepEqual(summarizeScopedAffectedGroups([CORE_SCOPED_AFFECTED_PACKAGE, DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [ { packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE], engineMemoryEnvelope: false, memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE, }, + { + packages: [CORE_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: false, + memoryEnvelopePackage: CORE_SCOPED_AFFECTED_PACKAGE, + }, ]); assert.deepEqual( - summarizeScopedAffectedGroups(["@fusion/core", DASHBOARD_SCOPED_AFFECTED_PACKAGE, ENGINE_SCOPED_AFFECTED_PACKAGE]), + summarizeScopedAffectedGroups([ + CORE_SCOPED_AFFECTED_PACKAGE, + DASHBOARD_SCOPED_AFFECTED_PACKAGE, + ENGINE_SCOPED_AFFECTED_PACKAGE, + ]), [ - { packages: ["@fusion/core"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, { packages: [ENGINE_SCOPED_AFFECTED_PACKAGE], engineMemoryEnvelope: true, @@ -321,14 +337,47 @@ test("partitionScopedAffectedPackages: isolates dashboard and engine into separa engineMemoryEnvelope: false, memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE, }, + { + packages: [CORE_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: false, + memoryEnvelopePackage: CORE_SCOPED_AFFECTED_PACKAGE, + }, ], ); - assert.deepEqual(summarizeScopedAffectedGroups(["@fusion/core", "@runfusion/fusion"]), [ - { packages: ["@fusion/core", "@runfusion/fusion"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, + // A genuinely regular package stays in the shared regular group; core splits out. + assert.deepEqual(summarizeScopedAffectedGroups([CORE_SCOPED_AFFECTED_PACKAGE, "@runfusion/fusion"]), [ + { packages: ["@runfusion/fusion"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, + { + packages: [CORE_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: false, + memoryEnvelopePackage: CORE_SCOPED_AFFECTED_PACKAGE, + }, ]); }); +test("@fusion/core is a wide-fan-out memory-envelope package but is NOT gate-covered", () => { + // It must be bounded (guard applies) ... + assert.ok( + Object.keys(SCOPED_AFFECTED_MEMORY_ENVELOPES).includes(CORE_SCOPED_AFFECTED_PACKAGE), + "core must be a memory-envelope package so the wide-fan-out guard runs only directly-changed core tests", + ); + // ... yet must NOT claim gate coverage (the merge gate runs no core suite), + // so a delegated core lane warns loudly instead of reporting a false green. + assert.ok( + !GATE_COVERED_MEMORY_ENVELOPE_PACKAGES.has(CORE_SCOPED_AFFECTED_PACKAGE), + "core is not covered by the merge gate; delegation must warn, not reassure", + ); +}); + +test("core scoped-affected env applies the bounded heap and single-worker envelope", () => { + const env = createScopedAffectedMemoryEnvelopeEnv(CORE_SCOPED_AFFECTED_PACKAGE, { + NODE_OPTIONS: "--trace-warnings", + HOME: "/tmp/fusion-home", + }); + assertScopedAffectedEnv(env, { heapMb: CORE_SCOPED_AFFECTED_HEAP_MB, workers: CORE_SCOPED_AFFECTED_WORKERS }); +}); + test("createDashboardScopedAffectedEnv: caps heap, preserves env, lowers workers, and leaves watchdog finite", () => { const env = createDashboardScopedAffectedEnv({ NODE_OPTIONS: "--trace-warnings", diff --git a/scripts/lib/run-vitest-watchdog.mjs b/scripts/lib/run-vitest-watchdog.mjs index 1294836a0d..9ee9c90ed6 100644 --- a/scripts/lib/run-vitest-watchdog.mjs +++ b/scripts/lib/run-vitest-watchdog.mjs @@ -52,8 +52,21 @@ export const CLASS_BUDGET_BANDS = { the two values are not coupled and may diverge. */ shard: { floor: 15 * MINUTE, ceiling: 30 * MINUTE }, + /* + FNXC:TestInfrastructure 2026-06-26-12:40: + The `changed` ceiling MUST sit below the engine's per-task verification kill + (`VERIFICATION_TIMEOUT_WORKSPACE_MS = 900_000` = 15min, verification-utils.ts). + This band is the bound for a local changed-file affected-lane invocation + (`pnpm test` in changed mode). When the timings snapshot is stale (the common + case), deriveBudgetMs returns this ceiling. At the old 20min ceiling the script + watchdog NEVER fired before the engine's 15min kill, so a runaway lane was + SIGKILLed by the engine and the whole task RESTARTED (stacked 15-min timeouts) + instead of failing the lane cleanly here (exit 124, no restart). 13min leaves + margin under the 15min kill so the script fails the lane itself first. Lowering + a ceiling is a tightening, not a timeout-widening appeasement. + */ // One local changed-file package invocation. - changed: { floor: 2 * MINUTE, ceiling: 20 * MINUTE }, + changed: { floor: 2 * MINUTE, ceiling: 13 * MINUTE }, // One dashboard quality lane (heap-managed). Matches the historical 15min. "dashboard-lane": { floor: 15 * MINUTE, ceiling: 30 * MINUTE }, }; diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index 027dd83ec6..b7df291da1 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -1317,13 +1317,45 @@ export function packageHasVitestConfig(pkgDir, projectRoot = rootDir) { return VITEST_CONFIG_BASENAMES.some((name) => existsSync(path.join(projectRoot, pkgDir, name))); } +/* +FNXC:TestInfrastructure 2026-06-26-13:05: +Scoped-affected worker fan-out was raised 1 -> 4 (operator decision). It was 1 +purely for OOM safety (FN-6854/FN-6874: heavy affected lanes OS-OOM-SIGKILLed +even at concurrency=1). Two things make 4 acceptable now: (1) the wide-fan-out +guard below bounds each heavy lane to a few directly-changed test files, so the +hundreds-of-files set that drove the OOM no longer reaches these workers; (2) the +heap cap stays 6144MB PER WORKER, so this lane can now use up to ~4x6GB ≈ 24GB — +fine on the 256GB host, but if a RAM-constrained CI runner OOM-SIGKILLs a heavy +lane again, lower this back toward 1 (or drop the per-worker heap) rather than +widening timeouts. This intentionally trades the FN-5048 "don't raise worker +knobs" guidance for throughput, scoped to the bounded affected lanes only. +*/ export const ENGINE_SCOPED_AFFECTED_PACKAGE = "@fusion/engine"; export const ENGINE_SCOPED_AFFECTED_HEAP_MB = "6144"; -export const ENGINE_SCOPED_AFFECTED_WORKERS = "1"; +export const ENGINE_SCOPED_AFFECTED_WORKERS = "4"; export const DASHBOARD_SCOPED_AFFECTED_PACKAGE = "@fusion/dashboard"; export const DASHBOARD_SCOPED_AFFECTED_HEAP_MB = "6144"; -export const DASHBOARD_SCOPED_AFFECTED_WORKERS = "1"; +export const DASHBOARD_SCOPED_AFFECTED_WORKERS = "4"; +export const CORE_SCOPED_AFFECTED_PACKAGE = "@fusion/core"; +export const CORE_SCOPED_AFFECTED_HEAP_MB = "6144"; +export const CORE_SCOPED_AFFECTED_WORKERS = "4"; +/* +FNXC:TestInfrastructure 2026-06-26-12:40: +`@fusion/core` is a memory-envelope/wide-fan-out package too — it was the +remaining `pnpm test` timeout path after engine/dashboard were bounded. core is +the hub nearly every package imports and has ~354 test files (db.test 21s, +mission-store 16s, ...). A non-test core SOURCE edit (e.g. store.ts/db.ts) makes +`vitest --changed` expand to ~the whole core suite at this real-git + +sqlite-heavy lane and blow past the engine's 15-min verification kill, which then +SIGKILLs `pnpm test` and RESTARTS the task — stacked 15-min timeouts. Listing +core here makes `partitionScopedAffectedPackages` treat it as its own +memory-envelope group so the wide-fan-out guard (run only directly-changed core +test files, else delegate) and the bounded heap/worker env both apply. core is +intentionally NOT in GATE_COVERED_MEMORY_ENVELOPE_PACKAGES (the gate runs no core +suite), so a delegated core lane emits the loud "not covered by gate; run +`pnpm test:full`" warning rather than a silent false-green. +*/ export const SCOPED_AFFECTED_MEMORY_ENVELOPES = Object.freeze({ [ENGINE_SCOPED_AFFECTED_PACKAGE]: Object.freeze({ packageName: ENGINE_SCOPED_AFFECTED_PACKAGE, @@ -1335,6 +1367,11 @@ export const SCOPED_AFFECTED_MEMORY_ENVELOPES = Object.freeze({ heapMb: DASHBOARD_SCOPED_AFFECTED_HEAP_MB, workers: DASHBOARD_SCOPED_AFFECTED_WORKERS, }), + [CORE_SCOPED_AFFECTED_PACKAGE]: Object.freeze({ + packageName: CORE_SCOPED_AFFECTED_PACKAGE, + heapMb: CORE_SCOPED_AFFECTED_HEAP_MB, + workers: CORE_SCOPED_AFFECTED_WORKERS, + }), }); /* @@ -1359,7 +1396,7 @@ export function createScopedAffectedMemoryEnvelopeEnv(packageName, env = process if (!envelope) return env; /* FNXC:TestInfrastructure 2026-06-21-11:24: - The engine affected lane can select hundreds of real-git-heavy files when `vitest --changed` sees a widely imported boundary. Run that scoped lane in its own memory envelope: cap Node old-space like the dashboard heap runner and lower Vitest worker fan-out to one process so the lane returns a real pass/fail verdict instead of an OS OOM SIGKILL. Keep watchdog timing outside this env so hangs still fail through `runWithWatchdog`. + The engine affected lane can select hundreds of real-git-heavy files when `vitest --changed` sees a widely imported boundary. Run that scoped lane in its own memory envelope: cap Node old-space like the dashboard heap runner and bound Vitest worker fan-out (see SCOPED_AFFECTED_WORKERS) so the lane returns a real pass/fail verdict instead of an OS OOM SIGKILL. Keep watchdog timing outside this env so hangs still fail through `runWithWatchdog`. FNXC:TestInfrastructure 2026-06-21-16:28: FN-6874 showed the dashboard changed-mode affected lane can OOM/SIGKILL even with `FUSION_TEST_CONCURRENCY=1 FUSION_TEST_WORKSPACE_CONCURRENCY=1`, so worker fan-out alone is not the failure mode. Give each heavy scoped package its own bounded heap envelope while preserving caller env and keeping the finite changed-class watchdog outside this env so hangs still fail instead of being masked.