From 74e056ed9963586fab0010acef0a8017dca90012 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 20 Jun 2026 21:48:14 -0700 Subject: [PATCH] fix(ci): raise shard watchdog floor to 15min to stop false-kills The Full Suite (non-blocking) workflow has been red for 30+ runs on main. Diagnosis: the @fusion/engine [1/2], [2/2] and @fusion/core [2/2] shard slices were SIGKILLed at their watchdog budgets (405s/405s/338s), not because they hang but because those budgets are too tight for current wall-clock. Local baselines (this machine, all pass, exit 0): - engine [2/2]: 145s wall / 309 files - core [2/2]: 283s wall / 172 files (old budget was only 338s!) deriveBudgetMs tightens the budget to expected*3.5 whenever the committed scripts/test-timings.json is <30d old. The snapshot (2026-06-03) undercounts the import- and real-git-subprocess overhead of these heavy slices, so the 'fresh' snapshot produced a too-tight, false-kill budget on slower CI runners -- the exact failure mode the floor/ceiling band exists to prevent. Fix (plan KTD-2): raise the shard band floor 5min -> 15min so the heaviest slices can't be tightened into a false-kill, while a true hang is still bounded far under the job's 60min ceiling. Mirrors the dashboard-lane heavy-lane floor. Follow-up: refresh scripts/test-timings.json from a default-branch CI run. --- scripts/__tests__/run-vitest-watchdog.test.mjs | 4 +++- scripts/lib/run-vitest-watchdog.mjs | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/__tests__/run-vitest-watchdog.test.mjs b/scripts/__tests__/run-vitest-watchdog.test.mjs index fa4c8411a6..3d9f4f791b 100644 --- a/scripts/__tests__/run-vitest-watchdog.test.mjs +++ b/scripts/__tests__/run-vitest-watchdog.test.mjs @@ -39,7 +39,9 @@ test("deriveBudgetMs: no fresh timing falls back to the per-class ceiling", () = test("deriveBudgetMs: fresh timing tightens within the band", () => { // expected×multiplier between floor and ceiling → use the tightened value. - const expected = 200_000; // 200s + // 300s × 3.5 = 1050s, which sits between the shard floor (15min) and + // ceiling (30min) so the tightened value is used un-clamped. + const expected = 300_000; // 300s const derived = deriveBudgetMs({ klass: "shard", expectedDurationMs: expected, timingsFresh: true }); assert.equal(derived, Math.round(expected * DEFAULT_BUDGET_MULTIPLIER)); assert.ok(derived >= CLASS_BUDGET_BANDS.shard.floor); diff --git a/scripts/lib/run-vitest-watchdog.mjs b/scripts/lib/run-vitest-watchdog.mjs index b047a70b07..362ca0623a 100644 --- a/scripts/lib/run-vitest-watchdog.mjs +++ b/scripts/lib/run-vitest-watchdog.mjs @@ -36,7 +36,17 @@ const MINUTE = 60_000; */ export const CLASS_BUDGET_BANDS = { // One CI shard command (may fan out across several packages via --filter). - shard: { floor: 5 * MINUTE, ceiling: 30 * MINUTE }, + // Floor is 15min, not 5min: the heaviest shard slices (@fusion/engine and + // @fusion/core, each split [1/2]+[2/2]) are import- and real-git-subprocess + // heavy, and the committed scripts/test-timings.json undercounts that + // overhead (most files bucket to the 100ms floor; the snapshot total is far + // below current wall-clock). A "fresh" but undercounting snapshot was + // tightening these slices to ~340-405s and SIGKILLing healthy runs on slower + // CI runners — the exact false-kill the band is meant to prevent (see the + // deriveBudgetMs note above and KTD-2). 15min sits comfortably above the + // observed CI need while still bounding a true hang far under the job's + // 60min ceiling. Mirrors the dashboard-lane heavy-lane floor. + shard: { floor: 15 * MINUTE, ceiling: 30 * MINUTE }, // One local changed-file package invocation. changed: { floor: 2 * MINUTE, ceiling: 20 * MINUTE }, // One dashboard quality lane (heap-managed). Matches the historical 15min.