From b480699e06cc187b6d320a9d05994f3fa0455d6a Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Thu, 18 Jun 2026 09:16:32 -0700 Subject: [PATCH] FN-6642: rescue dashboard flaky test coverage Rescue the dashboard quarantine candidates by restoring stable shared mocks and refreshing quarantine evidence. - Remove QuickEntryBox and chat-routes from the dashboard quarantine config and ledger. - Add iterable chat task document tool mocks across dashboard session test helpers. - Refresh the test velocity baseline and history with zero quarantined tests. Files changed: docs/test-velocity-baseline.md | 20 ++-- .../src/__tests__/session-error-recovery.test.ts | 2 + .../session-persistence-roundtrip.test.ts | 2 + .../src/__tests__/session-reconnect.test.ts | 2 + .../src/__tests__/session-resume-history.test.ts | 2 + packages/dashboard/src/test/mockCoreEngine.ts | 6 ++ packages/dashboard/vitest.config.ts | 13 ++- scripts/lib/test-quarantine.json | 10 -- scripts/test-velocity-history.json | 111 +++++++++++++++++++++ 9 files changed, 144 insertions(+), 24 deletions(-) Fusion-Task-Id: FN-6642 Fusion-Task-Lineage: 76e51d76-ce9a-4e68-9aa0-ad83cdf351a9 --- docs/test-velocity-baseline.md | 20 ++-- .../__tests__/session-error-recovery.test.ts | 2 + .../session-persistence-roundtrip.test.ts | 2 + .../src/__tests__/session-reconnect.test.ts | 2 + .../__tests__/session-resume-history.test.ts | 2 + packages/dashboard/src/test/mockCoreEngine.ts | 6 + packages/dashboard/vitest.config.ts | 13 +- scripts/lib/test-quarantine.json | 10 -- scripts/test-velocity-history.json | 111 ++++++++++++++++++ 9 files changed, 144 insertions(+), 24 deletions(-) diff --git a/docs/test-velocity-baseline.md b/docs/test-velocity-baseline.md index 687d2a6dcd..7bc1070f1b 100644 --- a/docs/test-velocity-baseline.md +++ b/docs/test-velocity-baseline.md @@ -5,7 +5,7 @@ ## Latest baseline - Cycle: **2026-W25** -- Captured at: **2026-06-18T03:04:28.794Z** +- Captured at: **2026-06-18T16:12:01.248Z** - Timing snapshot: `scripts/test-timings.json` captured at **2026-06-03T23:45:49.672Z** - Quarantine ledger: `scripts/lib/test-quarantine.json` @@ -13,10 +13,10 @@ | Metric | Current | Delta vs previous | |---|---:|---:| -| Merge gate wall-time (`pnpm test:gate`) | 6.2s | -2.3s | -| Boot smoke wall-time (`pnpm smoke:boot`) | 18.2s | n/a | -| Changed-only test wall-time (`pnpm test`) | 7.7s | -30.7s | -| Quarantine / flake count | 2 | 0 | +| Merge gate wall-time (`pnpm test:gate`) | 5.4s | -779ms | +| Boot smoke wall-time (`pnpm smoke:boot`) | 18.1s | -123ms | +| Changed-only test wall-time (`pnpm test`) | 7.2s | -500ms | +| Quarantine / flake count | 0 | -2 | | Deletion-due quarantines | 0 | n/a | ## Measurement failures @@ -52,7 +52,7 @@ | Age bucket | Count | |---|---:| -| 0-6 days | 2 | +| 0-6 days | 0 | | 7-13 days | 0 | | deletion due (>=14 days) | 0 | | unknown/future | 0 | @@ -67,16 +67,16 @@ | Row | Captured at | Gate | Boot smoke | `pnpm test` | Quarantine count | |---|---|---:|---:|---:|---:| -| Previous | 2026-06-18T02:53:34.158Z | 8.5s | unavailable | 38.4s | 2 | -| Latest | 2026-06-18T03:04:28.794Z | 6.2s | 18.2s | 7.7s | 2 | -| Delta | — | -2.3s | n/a | -30.7s | 0 | +| Previous | 2026-06-18T03:04:28.794Z | 6.2s | 18.2s | 7.7s | 2 | +| Latest | 2026-06-18T16:12:01.248Z | 5.4s | 18.1s | 7.2s | 0 | +| Delta | — | -779ms | -123ms | -500ms | -2 | _Future weekly rows append to `scripts/test-velocity-history.json`; compare the latest row against the previous row before posting to #leads._ ## Post to #leads ```text -FN-6612 weekly test velocity: gate 6.2s (-2.3s), boot smoke 18.2s (n/a), pnpm test 7.7s (-30.7s), quarantine ledger 2 (0). Slowest file: packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts at 13.9s. Deletion-due quarantines: 0. +FN-6612 weekly test velocity: gate 5.4s (-779ms), boot smoke 18.1s (-123ms), pnpm test 7.2s (-500ms), quarantine ledger 0 (-2). Slowest file: packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts at 13.9s. Deletion-due quarantines: 0. ``` ## How to refresh diff --git a/packages/dashboard/src/__tests__/session-error-recovery.test.ts b/packages/dashboard/src/__tests__/session-error-recovery.test.ts index 28628a0f91..839c4fa9bd 100644 --- a/packages/dashboard/src/__tests__/session-error-recovery.test.ts +++ b/packages/dashboard/src/__tests__/session-error-recovery.test.ts @@ -51,6 +51,8 @@ vi.mock("@fusion/engine", () => ({ listCliAdapterDescriptors: () => [], // FNXC:DashboardSessionTests 2026-06-14-09:06: planning.ts spreads createWorkflowAuthoringTools into agent customTools; this focused engine mock must export it to keep AI-session tests aligned with production planning setup. createWorkflowAuthoringTools: vi.fn(() => []), + // FNXC:DashboardSessionTests 2026-06-18-09:12: planning.ts also spreads chat task document tools during dashboard API backfill runs; focused engine mocks must return an iterable list so rescued chat-routes coverage does not destabilize planning-session tests. + createChatTaskDocumentTools: vi.fn(() => []), // FNXC:DashboardSessionTests 2026-06-17-19:33: planning and mission-interview sessions now request skills through the shared helper; focused engine mocks must return the shaped helper result so lifecycle tests do not crash before createFnAgent is captured. buildSessionSkillContextSync: vi.fn(() => ({ skillSelectionContext: undefined, diff --git a/packages/dashboard/src/__tests__/session-persistence-roundtrip.test.ts b/packages/dashboard/src/__tests__/session-persistence-roundtrip.test.ts index a1a12ee6b0..e0103d758f 100644 --- a/packages/dashboard/src/__tests__/session-persistence-roundtrip.test.ts +++ b/packages/dashboard/src/__tests__/session-persistence-roundtrip.test.ts @@ -41,6 +41,8 @@ vi.mock("@fusion/engine", () => ({ listCliAdapterDescriptors: () => [], // FNXC:DashboardSessionTests 2026-06-14-09:06: planning.ts spreads createWorkflowAuthoringTools into agent customTools; this focused engine mock must export it to keep AI-session tests aligned with production planning setup. createWorkflowAuthoringTools: vi.fn(() => []), + // FNXC:DashboardSessionTests 2026-06-18-09:12: planning.ts also spreads chat task document tools during dashboard API backfill runs; focused engine mocks must return an iterable list so rescued chat-routes coverage does not destabilize planning-session tests. + createChatTaskDocumentTools: vi.fn(() => []), // FNXC:DashboardSessionTests 2026-06-17-19:33: planning and mission-interview sessions now request skills through the shared helper; focused engine mocks must return the shaped helper result so lifecycle tests do not crash before createFnAgent is captured. buildSessionSkillContextSync: vi.fn(() => ({ skillSelectionContext: undefined, diff --git a/packages/dashboard/src/__tests__/session-reconnect.test.ts b/packages/dashboard/src/__tests__/session-reconnect.test.ts index db5bb5dceb..578b73431b 100644 --- a/packages/dashboard/src/__tests__/session-reconnect.test.ts +++ b/packages/dashboard/src/__tests__/session-reconnect.test.ts @@ -44,6 +44,8 @@ vi.mock("@fusion/engine", () => ({ listCliAdapterDescriptors: () => [], // FNXC:DashboardSessionTests 2026-06-14-09:06: planning.ts spreads createWorkflowAuthoringTools into agent customTools; this focused engine mock must export it to keep AI-session tests aligned with production planning setup. createWorkflowAuthoringTools: vi.fn(() => []), + // FNXC:DashboardSessionTests 2026-06-18-09:12: planning.ts also spreads chat task document tools during dashboard API backfill runs; focused engine mocks must return an iterable list so rescued chat-routes coverage does not destabilize planning-session tests. + createChatTaskDocumentTools: vi.fn(() => []), // FNXC:DashboardSessionTests 2026-06-17-19:33: planning and mission-interview sessions now request skills through the shared helper; focused engine mocks must return the shaped helper result so lifecycle tests do not crash before createFnAgent is captured. buildSessionSkillContextSync: vi.fn(() => ({ skillSelectionContext: undefined, diff --git a/packages/dashboard/src/__tests__/session-resume-history.test.ts b/packages/dashboard/src/__tests__/session-resume-history.test.ts index 7c4c4b9a36..cc6e0d9c8a 100644 --- a/packages/dashboard/src/__tests__/session-resume-history.test.ts +++ b/packages/dashboard/src/__tests__/session-resume-history.test.ts @@ -40,6 +40,8 @@ const { mockCreateFnAgent } = vi.hoisted(() => ({ vi.mock("@fusion/engine", () => ({ listCliAdapterDescriptors: () => [], createWorkflowAuthoringTools: vi.fn(() => []), + // FNXC:DashboardSessionTests 2026-06-18-09:12: planning.ts also spreads chat task document tools during dashboard API backfill runs; focused engine mocks must return an iterable list so rescued chat-routes coverage does not destabilize planning-session tests. + createChatTaskDocumentTools: vi.fn(() => []), // FNXC:DashboardSessionTests 2026-06-17-19:33: planning and mission-interview sessions now request skills through the shared helper; focused engine mocks must return the shaped helper result so lifecycle tests do not crash before createFnAgent is captured. buildSessionSkillContextSync: vi.fn(() => ({ skillSelectionContext: undefined, diff --git a/packages/dashboard/src/test/mockCoreEngine.ts b/packages/dashboard/src/test/mockCoreEngine.ts index fba654edf7..2a52ca4c20 100644 --- a/packages/dashboard/src/test/mockCoreEngine.ts +++ b/packages/dashboard/src/test/mockCoreEngine.ts @@ -59,6 +59,12 @@ export function createEngineMock(overrides: AnyModule = {}): AnyModule { // Returns an iterable tool list; dashboard code spreads its result // (`...createWorkflowAuthoringTools(...)`), so it must not be undefined. createWorkflowAuthoringTools: vi.fn(() => []), + /* + FNXC:DashboardRouteTests 2026-06-18-09:07: + Planning and chat route files can share worker-level @fusion/engine mocks during broad dashboard API quality runs. + Keep chat task document tools iterable by default so rescuing chat-routes from quarantine does not poison planning route imports with a fallback vi.fn() result. + */ + createChatTaskDocumentTools: vi.fn(() => []), ...overrides, }); } diff --git a/packages/dashboard/vitest.config.ts b/packages/dashboard/vitest.config.ts index fe1b65b9fe..f3f8c7ec73 100644 --- a/packages/dashboard/vitest.config.ts +++ b/packages/dashboard/vitest.config.ts @@ -257,11 +257,16 @@ Keep the ledger entry and exclude removed together; git history remains the arch FNXC:DashboardTestQuarantine 2026-06-18-06:12: FN-6633 workspace verification observed unrelated QuickEntryBox focus and chat-routes SSE lifecycle flakes after the targeted chat prompt regression suite passed. Quarantine the files under the deletion ratchet so this prompt-only chat guidance change does not appease flaky timing/focus behavior. + +FNXC:DashboardTestQuarantine 2026-06-18-09:02: +FN-6642 rescued QuickEntryBox after the single-file and full app-backfill lanes passed with the exclude removed. +Keep QuickEntryBox out of this list so focus-restoration coverage remains active instead of deleting useful user-facing behavior coverage. + +FNXC:DashboardTestQuarantine 2026-06-18-09:07: +FN-6642 rescued chat-routes by fixing the shared engine mock to return an iterable chat-task-document tool list during broad API lanes. +Keep chat-routes out of this list so SSE lifecycle coverage remains active and the ledger/config stay in lockstep. */ -const quarantinedDashboardTests: string[] = [ - "app/components/__tests__/QuickEntryBox.test.tsx", - "src/__tests__/chat-routes.test.ts", -]; +const quarantinedDashboardTests: string[] = []; const qualityApiTests = [ // Critical HTTP/server behavior: auth, task/project/settings mutation, diff --git a/scripts/lib/test-quarantine.json b/scripts/lib/test-quarantine.json index c729235e58..d8744f3523 100644 --- a/scripts/lib/test-quarantine.json +++ b/scripts/lib/test-quarantine.json @@ -1,15 +1,5 @@ { "$comment": "Flaky-test quarantine ledger (deletion ratchet — see AGENTS.md 'Flaky tests: quarantine on sight' and docs/testing.md 'Quarantine ledger and the deletion ratchet'). A test observed failing without a corresponding real bug is quarantined ON SIGHT: add an entry here AND a matching one-line `exclude` entry in that package's vitest config, in the same commit. Every entry needs a non-empty `reason` (link the failing run) and a `quarantinedAt` date — the entry expires 14 days later, at which point the test file is DELETED unless someone rescues it with evidence it catches real regressions plus a root-cause fix (never appeasement). There is deliberately no loader module and no automation around this file: it is a dated record, the vitest config exclude is the mechanism, and the sweep is policy executed by whoever touches the suite.", "entries": [ - { - "file": "packages/dashboard/app/components/__tests__/QuickEntryBox.test.tsx", - "reason": "FN-6633 workspace pnpm test observed focus-restoration assertion flake after this task's targeted chat-manager coverage passed; unrelated dashboard jsdom focus race, quarantined under deletion ratchet instead of appeasement.", - "quarantinedAt": "2026-06-18" - }, - { - "file": "packages/dashboard/src/__tests__/chat-routes.test.ts", - "reason": "FN-6633 workspace pnpm test observed SSE lifecycle test timeout after this task's targeted chat-manager coverage passed; unrelated dashboard route timing flake, quarantined under deletion ratchet instead of broadening this prompt-only change.", - "quarantinedAt": "2026-06-18" - } ] } diff --git a/scripts/test-velocity-history.json b/scripts/test-velocity-history.json index a358d63ab8..2cb78eaa72 100644 --- a/scripts/test-velocity-history.json +++ b/scripts/test-velocity-history.json @@ -226,6 +226,117 @@ ], "measurementFailures": [], "timingSnapshotCapturedAt": "2026-06-03T23:45:49.672Z" + }, + { + "capturedAt": "2026-06-18T16:12:01.248Z", + "gateMs": 5398, + "bootSmokeMs": 18097, + "testMs": 7240, + "quarantineCount": 0, + "slowestTop20": [ + { + "file": "packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts", + "ms": 13900, + "package": "@fusion/engine" + }, + { + "file": "packages/core/src/__tests__/agent-store.test.ts", + "ms": 11600, + "package": "@fusion/core" + }, + { + "file": "packages/dashboard/src/__tests__/routes-agents.test.ts", + "ms": 11200, + "package": "@fusion/dashboard" + }, + { + "file": "packages/core/src/__tests__/mission-store.test.ts", + "ms": 10700, + "package": "@fusion/core" + }, + { + "file": "packages/core/src/__tests__/db.test.ts", + "ms": 10100, + "package": "@fusion/core" + }, + { + "file": "packages/dashboard/src/__tests__/routes-git.test.ts", + "ms": 9400, + "package": "@fusion/dashboard" + }, + { + "file": "packages/engine/src/__tests__/reliability-interactions/branch-group-automerge-precedence.test.ts", + "ms": 9000, + "package": "@fusion/engine" + }, + { + "file": "packages/engine/src/__tests__/merger-ai.test.ts", + "ms": 8700, + "package": "@fusion/engine" + }, + { + "file": "packages/engine/src/__tests__/reliability-interactions/branch-group-merge-routing.test.ts", + "ms": 8400, + "package": "@fusion/engine" + }, + { + "file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion-gate.test.ts", + "ms": 8400, + "package": "@fusion/engine" + }, + { + "file": "packages/core/src/__tests__/task-documents.test.ts", + "ms": 8300, + "package": "@fusion/core" + }, + { + "file": "packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts", + "ms": 7800, + "package": "@fusion/engine" + }, + { + "file": "packages/cli/src/__tests__/extension.test.ts", + "ms": 7000, + "package": "@runfusion/fusion" + }, + { + "file": "packages/core/src/__tests__/run-audit.test.ts", + "ms": 6900, + "package": "@fusion/core" + }, + { + "file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion.test.ts", + "ms": 6100, + "package": "@fusion/engine" + }, + { + "file": "packages/dashboard/src/__tests__/routes-planning.test.ts", + "ms": 5600, + "package": "@fusion/dashboard" + }, + { + "file": "packages/core/src/__tests__/store-merge-queue.test.ts", + "ms": 5200, + "package": "@fusion/core" + }, + { + "file": "packages/dashboard/app/components/__tests__/FileEditor.test.tsx", + "ms": 5100, + "package": "@fusion/dashboard" + }, + { + "file": "packages/engine/src/__tests__/reliability-interactions/integration-worktree-state.test.ts", + "ms": 4900, + "package": "@fusion/engine" + }, + { + "file": "packages/engine/src/__tests__/self-healing-already-merged.real-git.test.ts", + "ms": 4900, + "package": "@fusion/engine" + } + ], + "measurementFailures": [], + "timingSnapshotCapturedAt": "2026-06-03T23:45:49.672Z" } ] }