diff --git a/docs/test-velocity-baseline.md b/docs/test-velocity-baseline.md index df1cd75756..70a0200b7d 100644 --- a/docs/test-velocity-baseline.md +++ b/docs/test-velocity-baseline.md @@ -16,7 +16,7 @@ | Merge gate wall-time (`pnpm test:gate`) | 7.7s | +778ms | | Boot smoke wall-time (`pnpm smoke:boot`) | 17.4s | -557ms | | Changed-only test wall-time (`pnpm test`) | 9.3s | -4.4s | -| Quarantine / flake count | 0 | 0 | +| Quarantine / flake count | 1 | +1 | | Deletion-due quarantines | 0 | n/a | ## Measurement failures @@ -56,7 +56,7 @@ | Age bucket | Count | |---|---:| -| 0-6 days | 0 | +| 0-6 days | 1 | | 7-13 days | 0 | | deletion due (>=14 days) | 0 | | unknown/future | 0 | @@ -72,15 +72,15 @@ | Row | Captured at | Gate | Boot smoke | `pnpm test` | Quarantine count | |---|---|---:|---:|---:|---:| | Previous | 2026-06-27T05:43:17.293Z | 6.9s | 18.0s | 13.7s | 0 | -| Latest | 2026-07-02T08:47:17.721Z | 7.7s | 17.4s | 9.3s | 0 | -| Delta | — | +778ms | -557ms | -4.4s | 0 | +| Latest | 2026-07-02T08:47:17.721Z | 7.7s | 17.4s | 9.3s | 1 | +| Delta | — | +778ms | -557ms | -4.4s | +1 | _Future weekly rows append to `scripts/test-velocity-history.json`; compare the latest row against the previous row before posting to #leads._ ## Post to #leads ```text -FN-6612 weekly test velocity: gate 7.7s (+778ms), boot smoke 17.4s (-557ms), pnpm test 9.3s (-4.4s), quarantine ledger 0 (0). Slowest file: packages/dashboard/src/__tests__/insights-routes.test.ts at 26.5s. Deletion-due quarantines: 0. +FN-6612 weekly test velocity: gate 7.7s (+778ms), boot smoke 17.4s (-557ms), pnpm test 9.3s (-4.4s), quarantine ledger 1 (+1). Slowest file: packages/dashboard/src/__tests__/insights-routes.test.ts at 26.5s. Deletion-due quarantines: 0. ``` ## How to refresh diff --git a/packages/cli/src/__tests__/extension-dist-barrel.test.ts b/packages/cli/src/__tests__/extension-dist-barrel.test.ts new file mode 100644 index 0000000000..7a3f496197 --- /dev/null +++ b/packages/cli/src/__tests__/extension-dist-barrel.test.ts @@ -0,0 +1,230 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { setTimeout as delay } from "node:timers/promises"; + +/* +FNXC:CliTests 2026-07-04-13:50: +FN-7530 split this single test out of extension.test.ts. The whole-file exclude that FN-7447 applied to +extension.test.ts to quarantine this one dist-barrel recompilation case was collaterally dropping ~68 otherwise-stable +tests. Isolating it here lets extension.test.ts return to the default lane while this file (and only this file) carries +its own quarantine entry/deletion clock in lockstep with scripts/lib/test-quarantine.json. See that ledger entry and +packages/cli/vitest.config.ts for the current in/out-of-lane status and root-cause note. +*/ + +vi.mock("@fusion/core/gh-cli", () => ({ + isGhAvailable: vi.fn(() => true), + isGhAuthenticated: vi.fn(() => true), + runGhJsonAsync: vi.fn(), + getGhErrorMessage: vi.fn((error: unknown) => (error instanceof Error ? error.message : String(error))), +})); + +vi.mock("../commands/task.js", () => ({ + runTaskPlan: vi.fn(), +})); + +import kbExtension, { closeCachedStores } from "../extension.js"; +import { TaskStore, MAX_TASK_LIST_TEXT_CHARS } from "@fusion/core"; +import { hasBuiltCoreDistBarrel } from "@fusion/test-utils"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ── Mock ExtensionAPI that captures registrations (mirrors extension.test.ts) ── + +interface RegisteredTool { + name: string; + label: string; + description: string; + execute: ( + toolCallId: string, + params: any, + signal: AbortSignal | undefined, + onUpdate: ((update: any) => void) | undefined, + ctx: any, + ) => Promise; +} + +function createMockAPI() { + const tools = new Map(); + const commands = new Map(); + const events = new Map(); + + const api = { + registerTool(def: any) { + tools.set(def.name, def); + }, + registerCommand(name: string, def: any) { + commands.set(name, def); + }, + registerShortcut: vi.fn(), + registerFlag: vi.fn(), + on(event: string, handler: Function) { + events.set(event, handler); + }, + tools, + commands, + events, + }; + + return api as any; +} + +function makeCtx(cwd: string) { + return { cwd } as any; +} + +async function removeDirWithRetries(path: string) { + /* + FNXC:CliTests 2026-06-19-11:23: + FN-6734 showed fixture removal can race SQLite/WAL close on loaded CLI workers; retry cleanup long enough for handles to drain instead of masking test bodies with larger timeouts or worker limits. + */ + const maxAttempts = 12; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + await rm(path, { recursive: true, force: true }); + return; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOTEMPTY" && code !== "EBUSY") { + throw error; + } + + if (attempt === maxAttempts) { + throw error; + } + + await delay(50 * attempt); + } + } +} + +describe("fn pi extension (dist-barrel recompilation slice)", () => { + let tmpDir: string; + let api: ReturnType; + let openStores: TaskStore[] = []; + + function createStore(): TaskStore { + const store = new TaskStore(tmpDir); + openStores.push(store); + return store; + } + + beforeEach(async () => { + tmpDir = await mkdtemp(join(tmpdir(), "kb-ext-distbarrel-")); + await mkdir(join(tmpDir, ".fusion"), { recursive: true }); + api = createMockAPI(); + kbExtension(api); + }); + + afterEach(async () => { + for (const store of openStores.splice(0)) { + try { + await store.close(); + } catch { + // Best effort: close all real stores before removing fixture roots. + } + } + await closeCachedStores(); + await removeDirWithRetries(tmpDir); + }); + + /* + FNXC:TaskListOutput 2026-06-17-02:37: + FN-6535 reproduces the heartbeat failure at the actual CLI tool surface while forcing @fusion/core to resolve through the built dist barrel. The normal CLI suite aliases @fusion/core to source, so this targeted mock is the regression guard for stale exports.import dist artifacts. + + FNXC:CoreTests 2026-06-18-01:35: + FN-6627 aligns the skip gate with every built @fusion/core dist artifact this runtime-dist mock loads, so a partial stale dist skips cleanly while a complete dist still exercises the heartbeat fn_task_list surface. + + FNXC:CliTests 2026-06-19-11:17: + FN-6734 keeps this guard in the default 5s lane by preserving the runtime-dist truncation invariant with fewer fixture writes instead of appeasing timeouts or reducing workers. + + FNXC:CliTests 2026-06-19-13:16: + The full CLI affected lane runs this file beside many module-mocking suites; verify the built barrel is importable in the executing worker before installing the mock, then skip like the partial-dist gate if a concurrent lane observes stale dist artifacts. + + FNXC:CliTests 2026-07-04-13:50: + FN-7530 moved this case out of extension.test.ts unchanged (same assertions, same dist-resolution invariant, same skip gate). The sibling source-@fusion/core test "bounds large column-filtered listings as a single plain-text block" in extension.test.ts covers the identical truncation invariant against source; this test's only marginal coverage is that the built dist barrel resolves/executes identically, which is why it stays a dedicated, narrowly-scoped file rather than being deleted. + */ + it.skipIf(!hasBuiltCoreDistBarrel(resolve(__dirname, "../../../core/dist")))( + "executes with @fusion/core resolved through the built dist barrel", + async () => { + const distCoreIndex = resolve(__dirname, "../../../core/dist/index.js"); + + const store = createStore(); + await store.init(); + try { + const first = await store.createTask({ + title: `Runtime-dist todo task 001 ${"x".repeat(300)}`, + description: "Runtime-dist todo task 001", + column: "todo", + }); + for (let i = 2; i <= 20; i += 1) { + await store.createTask({ + title: `Runtime-dist todo task ${String(i).padStart(3, "0")} ${"x".repeat(300)}`, + description: `Runtime-dist todo task ${String(i).padStart(3, "0")}`, + column: "todo", + dependencies: [first.id], + }); + } + } finally { + await store.close(); + } + + vi.resetModules(); + const distCoreUrl = pathToFileURL(distCoreIndex).href; + let distCoreModule: typeof import("@fusion/core"); + try { + distCoreModule = await vi.importActual(distCoreUrl); + } catch (error) { + const code = error instanceof Error && "code" in error ? (error as Error & { code?: string }).code : undefined; + if (code === "ERR_MODULE_NOT_FOUND") { + return; + } + throw error; + } + vi.doMock("@fusion/core", () => distCoreModule); + try { + const { default: runtimeCoreExtension } = await import("../extension.js?fn6535-runtime-core-dist"); + const runtimeApi = createMockAPI(); + runtimeCoreExtension(runtimeApi); + const listTool = runtimeApi.tools.get("fn_task_list")!; + + const broadResult = await listTool.execute( + "list-runtime-dist-broad", + { limit: 20 }, + undefined, + undefined, + makeCtx(tmpDir), + ); + const broadText = broadResult.content[0].text; + expect(broadResult.content).toHaveLength(1); + expect(broadResult.content[0].type).toBe("text"); + expect(broadText.length).toBeLessThanOrEqual(MAX_TASK_LIST_TEXT_CHARS); + expect(broadText).toContain("Todo (20):"); + expect(broadText).toContain("truncated to fit; narrow with column/limit"); + + const todoResult = await listTool.execute( + "list-runtime-dist-todo", + { column: "todo", limit: 20 }, + undefined, + undefined, + makeCtx(tmpDir), + ); + const todoText = todoResult.content[0].text; + expect(todoResult.content).toHaveLength(1); + expect(todoResult.content[0].type).toBe("text"); + expect(todoText.length).toBeLessThanOrEqual(MAX_TASK_LIST_TEXT_CHARS); + expect(todoText).toContain("Todo (20):"); + expect(todoText).toContain("FN-001"); + expect(todoText).toContain("[deps: FN-001]"); + expect(todoText).toContain("truncated to fit; narrow with column/limit"); + expect(todoResult.details.count).toBe(20); + } finally { + vi.doUnmock("@fusion/core"); + vi.resetModules(); + } + }, + ); +}); diff --git a/packages/cli/src/__tests__/extension.test.ts b/packages/cli/src/__tests__/extension.test.ts index 2745855777..d10b7fce03 100644 --- a/packages/cli/src/__tests__/extension.test.ts +++ b/packages/cli/src/__tests__/extension.test.ts @@ -1,8 +1,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; +import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { fileURLToPath } from "node:url"; import { setTimeout as delay } from "node:timers/promises"; /* @@ -26,7 +26,6 @@ import kbExtension, { closeCachedStores, resolveTaskListFormatter } from "../ext import { TaskStore, AgentStore, MANUAL_RETRY_RESET_COUNTER_KEYS, RESEARCH_RUN_STATUSES, MAX_TASK_LIST_TEXT_CHARS, formatTaskListText, COLUMN_LABELS } from "@fusion/core"; import type { WorkflowIr } from "@fusion/core"; import { isGhAvailable, isGhAuthenticated, runGhJsonAsync } from "@fusion/core/gh-cli"; -import { hasBuiltCoreDistBarrel } from "@fusion/test-utils"; import { runTaskPlan } from "../commands/task.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -3024,6 +3023,10 @@ describe("fn pi extension (runnable structured-output regression slice)", () => expect(result.details.count).toBe(15); }); + /* + FNXC:CliTests 2026-07-04-13:50: + FN-7530 split the sibling "executes with @fusion/core resolved through the built dist barrel" case (formerly directly below this test) into packages/cli/src/__tests__/extension-dist-barrel.test.ts. That test's own in-test dist-barrel recompilation (vi.resetModules + vi.importActual of the built @fusion/core dist barrel) is CPU-bound and timeout-prone under 4-shard CI contention (FN-6483/FN-6705/FN-6795/FN-6839/FN-7447 same signature); isolating it kept the ~68 stable tests in this file on the default lane while only the isolated file carries its own quarantine entry. This test covers the identical truncation invariant against the source-aliased @fusion/core. + */ it("bounds large column-filtered listings as a single plain-text block", async () => { const store = createStore(); await store.init(); @@ -3068,100 +3071,6 @@ describe("fn pi extension (runnable structured-output regression slice)", () => expect(result.details.count).toBe(20); }); - /** - * FNXC:TaskListOutput 2026-06-17-02:37: - * FN-6535 reproduces the heartbeat failure at the actual CLI tool surface while forcing @fusion/core to resolve through the built dist barrel. The normal CLI suite aliases @fusion/core to source, so this targeted mock is the regression guard for stale exports.import dist artifacts. - * - * FNXC:CoreTests 2026-06-18-01:35: - * FN-6627 aligns the skip gate with every built @fusion/core dist artifact this runtime-dist mock loads, so a partial stale dist skips cleanly while a complete dist still exercises the heartbeat fn_task_list surface. - * - * FNXC:CliTests 2026-06-19-11:17: - * FN-6734 keeps this guard in the default 5s lane by preserving the runtime-dist truncation invariant with fewer fixture writes instead of appeasing timeouts or reducing workers. - * - * FNXC:CliTests 2026-06-19-13:16: - * The full CLI affected lane runs this file beside many module-mocking suites; verify the built barrel is importable in the executing worker before installing the mock, then skip like the partial-dist gate if a concurrent lane observes stale dist artifacts. - */ - it.skipIf(!hasBuiltCoreDistBarrel(resolve(__dirname, "../../../core/dist")))( - "executes with @fusion/core resolved through the built dist barrel", - async () => { - const distCoreIndex = resolve(__dirname, "../../../core/dist/index.js"); - - const store = createStore(); - await store.init(); - try { - const first = await store.createTask({ - title: `Runtime-dist todo task 001 ${"x".repeat(300)}`, - description: "Runtime-dist todo task 001", - column: "todo", - }); - for (let i = 2; i <= 20; i += 1) { - await store.createTask({ - title: `Runtime-dist todo task ${String(i).padStart(3, "0")} ${"x".repeat(300)}`, - description: `Runtime-dist todo task ${String(i).padStart(3, "0")}`, - column: "todo", - dependencies: [first.id], - }); - } - } finally { - await store.close(); - } - - vi.resetModules(); - const distCoreUrl = pathToFileURL(distCoreIndex).href; - let distCoreModule: typeof import("@fusion/core"); - try { - distCoreModule = await vi.importActual(distCoreUrl); - } catch (error) { - const code = error instanceof Error && "code" in error ? (error as Error & { code?: string }).code : undefined; - if (code === "ERR_MODULE_NOT_FOUND") { - return; - } - throw error; - } - vi.doMock("@fusion/core", () => distCoreModule); - try { - const { default: runtimeCoreExtension } = await import("../extension.js?fn6535-runtime-core-dist"); - const runtimeApi = createMockAPI(); - runtimeCoreExtension(runtimeApi); - const listTool = runtimeApi.tools.get("fn_task_list")!; - - const broadResult = await listTool.execute( - "list-runtime-dist-broad", - { limit: 20 }, - undefined, - undefined, - makeCtx(tmpDir), - ); - const broadText = broadResult.content[0].text; - expect(broadResult.content).toHaveLength(1); - expect(broadResult.content[0].type).toBe("text"); - expect(broadText.length).toBeLessThanOrEqual(MAX_TASK_LIST_TEXT_CHARS); - expect(broadText).toContain("Todo (20):"); - expect(broadText).toContain("truncated to fit; narrow with column/limit"); - - const todoResult = await listTool.execute( - "list-runtime-dist-todo", - { column: "todo", limit: 20 }, - undefined, - undefined, - makeCtx(tmpDir), - ); - const todoText = todoResult.content[0].text; - expect(todoResult.content).toHaveLength(1); - expect(todoResult.content[0].type).toBe("text"); - expect(todoText.length).toBeLessThanOrEqual(MAX_TASK_LIST_TEXT_CHARS); - expect(todoText).toContain("Todo (20):"); - expect(todoText).toContain("FN-001"); - expect(todoText).toContain("[deps: FN-001]"); - expect(todoText).toContain("truncated to fit; narrow with column/limit"); - expect(todoResult.details.count).toBe(20); - } finally { - vi.doUnmock("@fusion/core"); - vi.resetModules(); - } - }, - ); - it("degrades to bounded text when formatter exports are unavailable", () => { const boardLinesWithoutParams = [ "Planning (2):", diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 76b7cbc878..2201e62314 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -45,8 +45,11 @@ const quarantinedCliTests: string[] = [ FNXC:CliTests 2026-07-04-10:40: FN-7447 re-quarantines extension.test.ts after its built-dist-barrel fn_task_list test (line ~3084) timed out at 5000ms in full-suite shard 4/4 (run 28697507894) while passing locally at ~1.2s and in 3 of the 4 surrounding CI runs. The root-cause invariant is the loaded-lane signature: in-test dist-barrel recompilation (vi.resetModules + vi.importActual of the full @fusion/core dist barrel + a fresh dynamic import of extension.js) inside the default 5s timeout is CPU-bound and degrades non-linearly under 4-shard CI contention. This is the same signature rescued in FN-6483/FN-6705/FN-6795/FN-6839; widening the timeout is forbidden by the flaky-test rule and removing the recompilation removes the test's only purpose, so the file is excluded per the deletion ratchet rather than re-attempting a fifth fixture rescue. Mirrors scripts/lib/test-quarantine.json; collateral is the ~68 otherwise-stable tests in this file, recoverable via rescue before the 2026-07-18 deletion deadline. + + FNXC:CliTests 2026-07-04-13:50: + FN-7530 resolved the FN-7447 entry: RESCUE-by-split, not delete. The single dist-barrel recompilation test (unchanged assertions) moved to packages/cli/src/__tests__/extension-dist-barrel.test.ts; extension.test.ts is back in the default lane and its ~68 stable tests run again. The isolated file still stays quarantined here under its OWN fresh entry, because the root cause is loaded-lane CPU contention during vi.resetModules()/vi.importActual(dist barrel)/dynamic import() -- a property of that operation under 4-shard CI, not of file layout -- so splitting the file does not by itself make it safe to re-admit, and with only one test in the file there is no second call site to amortize a module-top-level rescue against. No testTimeout widening, retries, or worker/concurrency changes were made. Mirrors scripts/lib/test-quarantine.json; this isolated file's own 14-day deletion clock is due 2026-07-18. */ - "src/__tests__/extension.test.ts", + "src/__tests__/extension-dist-barrel.test.ts", ]; export default defineConfig({ diff --git a/scripts/lib/test-quarantine.json b/scripts/lib/test-quarantine.json index b86cb1b478..7423081654 100644 --- a/scripts/lib/test-quarantine.json +++ b/scripts/lib/test-quarantine.json @@ -2,8 +2,8 @@ "$comment": "Flaky-test quarantine ledger (deletion ratchet — see AGENTS.md 'Flaky tests: quarantine on sight' and docs/testing.md 'Quarantine ledger and the deletion ratchet'). A test observed failing without a corresponding real bug is quarantined ON SIGHT: add an entry here AND a matching one-line `exclude` entry in that package's vitest config, in the same commit. Every entry needs a non-empty `reason` (link the failing run) and a `quarantinedAt` date — the entry expires 14 days later, at which point the test file is DELETED unless someone rescues it with evidence it catches real regressions plus a root-cause fix (never appeasement). There is deliberately no loader module and no automation around this file: it is a dated record, the vitest config exclude is the mechanism, and the sweep is policy executed by whoever touches the suite.", "entries": [ { - "file": "packages/cli/src/__tests__/extension.test.ts", - "reason": "Loaded-lane CI timeout: the dist-barrel fn_task_list test (extension.test.ts line ~3084) timed out at 5000ms in the full-suite shard 4/4 while passing locally in ~1.2s and in 3 of the 4 surrounding CI runs. Root-cause invariant: the test does in-test module recompilation (vi.resetModules + vi.importActual of the full @fusion/core dist barrel + a fresh dynamic import of extension.js) inside the default 5s test timeout; that work is CPU-bound and degrades non-linearly under 4-shard CI contention. This is the same loaded-lane signature rescued in FN-6483/FN-6705/FN-6795/FN-6839; widening the timeout is forbidden by the flaky-test rule and removing the recompilation removes the test's only purpose. The sibling source-@fusion/core test 'bounds large column-filtered listings' covers the identical truncation invariant, so the dist-barrel slice's marginal coverage is dist-resolution, which has been stable. Collateral: file-granular exclude also drops ~68 otherwise-stable tests in this file until a rescue. Failing run: https://github.com/Runfusion/Fusion/actions/runs/28697507894 (Test shard 4/4).", + "file": "packages/cli/src/__tests__/extension-dist-barrel.test.ts", + "reason": "FN-7530 RESCUE-by-split of the prior extension.test.ts entry: the dist-barrel fn_task_list test timed out at 5000ms in the full-suite shard 4/4 (run https://github.com/Runfusion/Fusion/actions/runs/28697507894) while passing locally in ~1.2s and in 3 of the 4 surrounding CI runs. Root-cause invariant: the test does in-test module recompilation (vi.resetModules + vi.importActual of the full @fusion/core dist barrel + a fresh dynamic import of extension.js) inside the default 5s test timeout; that work is CPU-bound and degrades non-linearly under 4-shard CI contention (same loaded-lane signature as FN-6483/FN-6705/FN-6795/FN-6839). Widening the timeout is forbidden by the flaky-test rule and removing the recompilation removes the test's only purpose. The sibling source-@fusion/core test 'bounds large column-filtered listings' (extension.test.ts) covers the identical truncation invariant, so this test's marginal coverage is dist-resolution only, which has been stable -- it is isolated rather than deleted. Isolating this single test into its own file let the ~68 otherwise-stable tests in extension.test.ts return to the default lane immediately; only this narrowly-scoped file remains quarantined under its own fresh clock.", "quarantinedAt": "2026-07-04" } ]