From 648d3598e045480c66d12f64f45f98160bcef42c Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 10 Jun 2026 22:18:52 -0700 Subject: [PATCH] FN-6206: quarantine flaky engine tests Move undocumented flaky engine tests into the formal quarantine path. - add quarantine ledger entries for flaky engine test files observed under concurrent/full-suite load - exclude the quarantined engine test files from vitest projects so they no longer run in gate and reliability pools - remove the undocumented it.skip markers by quarantining at the file/config level instead Files changed: .../__tests__/merger-file-scope-invariant.test.ts | 4 ++-- .../src/__tests__/project-engine-manager.test.ts | 2 +- packages/engine/vitest.config.ts | 10 ++++++++-- scripts/lib/test-quarantine.json | 23 +++++++++++++++++++++- 4 files changed, 33 insertions(+), 6 deletions(-) Fusion-Task-Id: FN-6206 Fusion-Task-Lineage: 8d88725e-1e92-4a06-aaa6-be34287e613a --- .../merger-file-scope-invariant.test.ts | 4 ++-- .../__tests__/project-engine-manager.test.ts | 2 +- packages/engine/vitest.config.ts | 10 ++++++-- scripts/lib/test-quarantine.json | 23 ++++++++++++++++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/__tests__/merger-file-scope-invariant.test.ts b/packages/engine/src/__tests__/merger-file-scope-invariant.test.ts index 7694b9efb3..04e1d882fb 100644 --- a/packages/engine/src/__tests__/merger-file-scope-invariant.test.ts +++ b/packages/engine/src/__tests__/merger-file-scope-invariant.test.ts @@ -157,7 +157,7 @@ describe("assertSquashOverlapsFileScope", () => { // (which reports staged files unrelated to the test scope and trips the // FileScopeViolationError). The same logic is covered by the existing // real-git fixture tests in reliability-interactions/workflow-and-file-scope. - it.skip("accepts declared scope as a single changeset file when staged matches exactly", async () => { + it("accepts declared scope as a single changeset file when staged matches exactly", async () => { const store = createInvariantStore([".changeset/fn-4767-pr-flow.md"]); mockStagedFiles([".changeset/fn-4767-pr-flow.md"]); @@ -170,7 +170,7 @@ describe("assertSquashOverlapsFileScope", () => { }); // Skipped: same flake mode as the test above. - it.skip("accepts declared scope as a changeset glob when staged file matches", async () => { + it("accepts declared scope as a changeset glob when staged file matches", async () => { const store = createInvariantStore([".changeset/*.md"]); mockStagedFiles([".changeset/fn-4767-pr-flow.md"]); diff --git a/packages/engine/src/__tests__/project-engine-manager.test.ts b/packages/engine/src/__tests__/project-engine-manager.test.ts index 96291b4b08..ac07296761 100644 --- a/packages/engine/src/__tests__/project-engine-manager.test.ts +++ b/packages/engine/src/__tests__/project-engine-manager.test.ts @@ -550,7 +550,7 @@ describe("ProjectEngineManager", () => { // Flake under full reliability-suite load: 30s timeout, but passes in ~46ms // standalone. Setinterval-driven reconciliation appears to race with vitest // fake-timer contention when other reliability-pool files are co-resident. - it.skip("retries failed project starts on subsequent reconciliation ticks", async () => { + it("retries failed project starts on subsequent reconciliation ticks", async () => { // Track how many times start() is called to fail only the FIRST set let startCallCount = 0; const manager = new ProjectEngineManager(centralCore); diff --git a/packages/engine/vitest.config.ts b/packages/engine/vitest.config.ts index 81afd0a195..d0a883290b 100644 --- a/packages/engine/vitest.config.ts +++ b/packages/engine/vitest.config.ts @@ -68,7 +68,6 @@ export default defineConfig({ "src/__tests__/merger-post-merge.test.ts", "src/__tests__/merger-conflict-resolution.test.ts", "src/__tests__/merger-diff-scope.test.ts", - "src/__tests__/merger-file-scope-invariant.test.ts", "src/__tests__/merger-landed-files-capture.test.ts", "src/__tests__/branch-attribution.test.ts", "src/__tests__/executor-core.test.ts", @@ -86,7 +85,11 @@ export default defineConfig({ "src/__tests__/workflow-node-handlers.test.ts", "src/__tests__/workflow-policy-ownership-map.test.ts", ], - exclude: ["node_modules/**", "dist/**"], + exclude: [ + "node_modules/**", + "dist/**", + "src/__tests__/merger-file-scope-invariant.test.ts", + ], }, }, { @@ -102,6 +105,9 @@ export default defineConfig({ "src/**/*.slow.test.ts", "node_modules/**", "dist/**", + "src/__tests__/merger-file-scope-invariant.test.ts", + "src/__tests__/project-engine-manager.test.ts", + "src/__tests__/merger-ai-cleanup.test.ts", ], }, }, diff --git a/scripts/lib/test-quarantine.json b/scripts/lib/test-quarantine.json index 39eac9c428..7750ff7a8e 100644 --- a/scripts/lib/test-quarantine.json +++ b/scripts/lib/test-quarantine.json @@ -1,4 +1,25 @@ { "$comment": "Flaky-test quarantine ledger (deletion ratchet — see AGENTS.md 'Flaky tests: quarantine on sight' and docs/testing.md 'Quarantine ledger and the deletion ratchet'). A test observed failing without a corresponding real bug is quarantined ON SIGHT: add an entry here AND a matching one-line `exclude` entry in that package's vitest config, in the same commit. Every entry needs a non-empty `reason` (link the failing run) and a `quarantinedAt` date — the entry expires 14 days later, at which point the test file is DELETED unless someone rescues it with evidence it catches real regressions plus a root-cause fix (never appeasement). There is deliberately no loader module and no automation around this file: it is a dated record, the vitest config exclude is the mechanism, and the sweep is policy executed by whoever touches the suite.", - "entries": [] + "entries": [ + { + "file": "packages/engine/src/__tests__/project-engine-manager.test.ts", + "reason": "Flake: setInterval-driven reconciliation races with vitest fake-timer contention under full reliability-suite load. Test passes standalone (~46ms) but times out (30s) when reliability-pool files are co-resident. FN-6206.", + "quarantinedAt": "2026-06-10" + }, + { + "file": "packages/engine/src/__tests__/merger-file-scope-invariant.test.ts", + "reason": "Flake: vi.mock('node:child_process') occasionally doesn't take under workspace-concurrent runs, letting real git binary leak and report staged files unrelated to test scope (trips FileScopeViolationError). Same logic covered by real-git fixture tests in reliability-interactions/workflow-and-file-scope. FN-6206.", + "quarantinedAt": "2026-06-10" + }, + { + "file": "packages/engine/src/__tests__/merger-file-scope-invariant.test.ts", + "reason": "Flake: same mock-contention mode as the sibling changeset-file test above (vi.mock('node:child_process') not taking under concurrent load). FN-6206.", + "quarantinedAt": "2026-06-10" + }, + { + "file": "packages/engine/src/__tests__/merger-ai-cleanup.test.ts", + "reason": "Flake observed during FN-6206 verification: `pruneExistingAiMergeWorktrees skips active-session paths` failed in full `pnpm --filter @fusion/engine test` runs while the file passed standalone, indicating suite-order/concurrency sensitivity. Follow-up FN-6207.", + "quarantinedAt": "2026-06-10" + } + ] }