diff --git a/docs/testing.md b/docs/testing.md index 64725841cf..a21d082712 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -227,6 +227,16 @@ Flaky tests are quarantined ON SIGHT and deleted on a 2-week clock. This is writ **The clock:** an entry expires 14 days after `quarantinedAt`. Whoever touches the suite and finds an expired entry deletes the test file, its ledger entry, and its config exclude (git history is the archive). `scripts/check-test-inventory.mjs --diff` stays deliberately unwired in CI because it would fail on exactly these deletions. +### Quarantine deadline visibility check + +Run `pnpm check:quarantine-ledger` to print a soonest-deadline-first summary of `scripts/lib/test-quarantine.json`. The command uses the same 14-day deletion clock (`quarantinedAt + 14d`) as the velocity baseline and reports each entry as expired, near-deadline, healthy, or unknown when `quarantinedAt` is missing/invalid. It is a visibility aid only: default mode exits 0 even when entries are near or expired, preserving the deliberately-unwired policy and leaving rescue-or-delete decisions to maintainers. + +Flags: + +- `--warn-within=` changes the near-deadline window from the default 5 days. +- `--json` emits the computed rows plus summary counts for machine consumption. +- `--strict` exits 1 when any entry is expired or near-deadline, for opt-in local or project-specific gates only. Do not wire this into `pretest`, `test:gate`, or other default blocking lanes without an explicit policy change. + **Rescue** (before the clock runs out) requires both: evidence the test catches real regressions, and a root-cause fix for the flake. Stabilization passes — widened timeouts, retries, loosened assertions — are appeasement, not rescue, and are banned (for agents especially). ### Vitest timeout-appeasement guard diff --git a/package.json b/package.json index 909f14cadc..7b16df5df9 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "pretest:full": "node scripts/check-no-nohup.mjs && node scripts/check-no-kill-4040.mjs && node scripts/check-no-test-timeout-appeasement.mjs && node scripts/check-changeset-format.mjs", "check:line-count": "node scripts/check-file-line-count.mjs", "check:changesets": "node scripts/check-changeset-format.mjs", + "check:quarantine-ledger": "node scripts/check-quarantine-ledger.mjs", "test:gate": "node scripts/check-no-nohup.mjs && node scripts/check-no-kill-4040.mjs && node scripts/check-no-test-timeout-appeasement.mjs && node scripts/check-changeset-format.mjs && pnpm --filter @fusion/engine test:core && pnpm --filter @runfusion/fusion test:ci-shape", "smoke:boot": "node scripts/boot-smoke.mjs", "local": "node scripts/start-local.mjs", diff --git a/scripts/__tests__/check-quarantine-ledger.test.mjs b/scripts/__tests__/check-quarantine-ledger.test.mjs new file mode 100644 index 0000000000..cf9b8a38ac --- /dev/null +++ b/scripts/__tests__/check-quarantine-ledger.test.mjs @@ -0,0 +1,159 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + computeDeadlines, + main, + readLedger, + renderReport, +} from "../check-quarantine-ledger.mjs"; + +function captureStream() { + let text = ""; + return { + stream: { write(chunk) { text += chunk; } }, + get text() { return text; }, + }; +} + +function tempRoot() { + return mkdtempSync(path.join(tmpdir(), "fusion-quarantine-ledger-")); +} + +function writeLedger(rootDir, ledger) { + const ledgerPath = path.join(rootDir, "scripts/lib/test-quarantine.json"); + mkdirSync(path.dirname(ledgerPath), { recursive: true }); + writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, "utf8"); + return ledgerPath; +} + +const fixedNow = new Date("2026-07-12T12:00:00.000Z"); + +const fixtureLedger = { + entries: [ + { + file: "healthy.test.ts", + reason: "fresh quarantine", + quarantinedAt: "2026-07-12", + }, + { + file: "near.test.ts", + reason: "approaching deletion deadline", + quarantinedAt: "2026-07-04", + }, + { + file: "expired.test.ts", + reason: "past deletion deadline", + quarantinedAt: "2026-06-27", + }, + { + file: "unknown.test.ts", + reason: "missing quarantine date", + }, + ], +}; + +test("computeDeadlines buckets healthy, near, expired, and unknown entries", () => { + const rows = computeDeadlines(fixtureLedger, { now: fixedNow, warnWithinDays: 6 }); + const byFile = Object.fromEntries(rows.map((row) => [row.file, row])); + + assert.equal(byFile["healthy.test.ts"].status, "healthy"); + assert.equal(byFile["healthy.test.ts"].daysRemaining, 14); + assert.equal(byFile["healthy.test.ts"].deadline, "2026-07-26"); + + assert.equal(byFile["near.test.ts"].status, "near"); + assert.equal(byFile["near.test.ts"].daysRemaining, 6); + + assert.equal(byFile["expired.test.ts"].status, "expired"); + assert.ok(byFile["expired.test.ts"].daysRemaining <= 0); + + assert.equal(byFile["unknown.test.ts"].status, "unknown"); + assert.equal(byFile["unknown.test.ts"].daysRemaining, null); + assert.equal(byFile["unknown.test.ts"].deadline, null); +}); + +test("computeDeadlines sorts soonest deadline first with unknown entries last", () => { + const rows = computeDeadlines(fixtureLedger, { now: fixedNow, warnWithinDays: 6 }); + + assert.deepEqual(rows.map((row) => row.file), [ + "expired.test.ts", + "near.test.ts", + "healthy.test.ts", + "unknown.test.ts", + ]); +}); + +test("renderReport handles an empty ledger without throwing", () => { + const rows = computeDeadlines({ entries: [] }, { now: fixedNow }); + const report = renderReport(rows); + + assert.deepEqual(rows, []); + assert.match(report, /Ledger is empty; nothing quarantined\./); + assert.match(report, /Summary: total=0 expired=0 near=0 healthy=0 unknown=0/); +}); + +test("readLedger tolerates a missing ledger and rejects non-array entries", () => { + const rootDir = tempRoot(); + try { + assert.deepEqual(readLedger(path.join(rootDir, "missing.json")), { entries: [] }); + const ledgerPath = writeLedger(rootDir, { entries: {} }); + assert.throws( + () => readLedger(ledgerPath), + /quarantine ledger .* must have an "entries" array/, + ); + } finally { + rmSync(rootDir, { recursive: true, force: true }); + } +}); + +test("main is report-only by default but --strict fails on near or expired entries", () => { + const rootDir = tempRoot(); + try { + const ledgerPath = writeLedger(rootDir, fixtureLedger); + const stdout = captureStream(); + const stderr = captureStream(); + + assert.equal(main([], { rootDir, ledgerPath, stdout: stdout.stream, stderr: stderr.stream, now: fixedNow }), 0); + assert.match(stdout.text, /expired=1 near=0 healthy=2 unknown=1/); + assert.equal(stderr.text, ""); + + const strictStdout = captureStream(); + assert.equal(main(["--strict", "--warn-within=6"], { rootDir, ledgerPath, stdout: strictStdout.stream, stderr: stderr.stream, now: fixedNow }), 1); + + const healthyLedgerPath = writeLedger(rootDir, { entries: [{ file: "healthy.test.ts", quarantinedAt: "2026-07-12" }] }); + const healthyStdout = captureStream(); + assert.equal(main(["--strict"], { rootDir, ledgerPath: healthyLedgerPath, stdout: healthyStdout.stream, stderr: stderr.stream, now: fixedNow }), 0); + } finally { + rmSync(rootDir, { recursive: true, force: true }); + } +}); + +test("--json output parses and includes per-entry status and days remaining", () => { + const rootDir = tempRoot(); + try { + const ledgerPath = writeLedger(rootDir, fixtureLedger); + const stdout = captureStream(); + const stderr = captureStream(); + + assert.equal(main(["--json", "--warn-within=6"], { rootDir, ledgerPath, stdout: stdout.stream, stderr: stderr.stream, now: fixedNow }), 0); + + const parsed = JSON.parse(stdout.text); + assert.equal(parsed.summary.expired, 1); + assert.equal(parsed.summary.near, 1); + assert.deepEqual( + parsed.rows.map((row) => ({ file: row.file, status: row.status, daysRemaining: row.daysRemaining })), + [ + { file: "expired.test.ts", status: "expired", daysRemaining: -1 }, + { file: "near.test.ts", status: "near", daysRemaining: 6 }, + { file: "healthy.test.ts", status: "healthy", daysRemaining: 14 }, + { file: "unknown.test.ts", status: "unknown", daysRemaining: null }, + ], + ); + assert.equal(stderr.text, ""); + } finally { + rmSync(rootDir, { recursive: true, force: true }); + } +}); diff --git a/scripts/check-quarantine-ledger.mjs b/scripts/check-quarantine-ledger.mjs new file mode 100644 index 0000000000..1aca8c281a --- /dev/null +++ b/scripts/check-quarantine-ledger.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +/* +FNXC:TestQuarantine 2026-07-12-00:00: +The flaky-test deletion ratchet had no visibility tool for entries approaching the `quarantinedAt + 14d` deletion deadline. +This report surfaces near-deadline quarantines so maintainers can make deliberate rescue-or-expire decisions while preserving the policy's report-only default; `--strict` is the opt-in enforcement path. +*/ + +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { DEFAULT_QUARANTINE_PATH, DELETION_CLOCK_DAYS } from "./test-velocity-baseline.mjs"; + +const MS_PER_DAY = 86_400_000; +const DEFAULT_WARN_WITHIN_DAYS = 5; +const REASON_MAX_LENGTH = 140; + +const currentFilePath = fileURLToPath(import.meta.url); +const repoRoot = path.resolve(path.dirname(currentFilePath), ".."); + +function toDate(value) { + if (!value) return null; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function ageDays(quarantinedAt, now) { + const quarantinedAtDate = toDate(quarantinedAt); + if (!quarantinedAtDate) return null; + return Math.floor((now.getTime() - quarantinedAtDate.getTime()) / MS_PER_DAY); +} + +function formatIsoDate(date) { + return date.toISOString().slice(0, 10); +} + +function normalizeWarnWithinDays(value) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`--warn-within must be a non-negative integer, got ${value}`); + } + return parsed; +} + +function truncateReason(reason) { + const normalized = String(reason ?? "").replace(/\s+/g, " ").trim(); + if (normalized.length <= REASON_MAX_LENGTH) return normalized; + return `${normalized.slice(0, REASON_MAX_LENGTH - 1)}…`; +} + +function summarizeRows(rows) { + return rows.reduce( + (summary, row) => { + summary[row.status] += 1; + summary.total += 1; + return summary; + }, + { total: 0, expired: 0, near: 0, healthy: 0, unknown: 0 }, + ); +} + +export function readLedger(ledgerPath) { + if (!existsSync(ledgerPath)) { + return { entries: [] }; + } + + const json = JSON.parse(readFileSync(ledgerPath, "utf8")); + if (json?.entries != null && !Array.isArray(json.entries)) { + throw new Error(`quarantine ledger ${ledgerPath} must have an "entries" array`); + } + return json ?? { entries: [] }; +} + +export function computeDeadlines(json, { now = new Date(), warnWithinDays = DEFAULT_WARN_WITHIN_DAYS } = {}) { + const entries = Array.isArray(json?.entries) ? json.entries : []; + const rows = entries.map((entry, index) => { + const quarantinedAtDate = toDate(entry?.quarantinedAt); + const age = ageDays(entry?.quarantinedAt, now); + const daysRemaining = age == null ? null : DELETION_CLOCK_DAYS - age; + const deadlineDate = quarantinedAtDate == null + ? null + : new Date(quarantinedAtDate.getTime() + DELETION_CLOCK_DAYS * MS_PER_DAY); + let status = "unknown"; + if (daysRemaining != null) { + if (daysRemaining <= 0) { + status = "expired"; + } else if (daysRemaining <= warnWithinDays) { + status = "near"; + } else { + status = "healthy"; + } + } + + return { + index, + file: entry?.file ?? "unknown", + reason: entry?.reason ?? "", + quarantinedAt: entry?.quarantinedAt ?? null, + ageDays: age, + daysRemaining, + deadline: deadlineDate == null ? null : formatIsoDate(deadlineDate), + status, + }; + }); + + return rows.sort((a, b) => { + if (a.deadline == null && b.deadline == null) return a.index - b.index; + if (a.deadline == null) return 1; + if (b.deadline == null) return -1; + return a.deadline.localeCompare(b.deadline) || a.file.localeCompare(b.file) || a.index - b.index; + }); +} + +export function renderReport(rows, { warnWithinDays = DEFAULT_WARN_WITHIN_DAYS } = {}) { + const summary = summarizeRows(rows); + const lines = [ + "Quarantine ledger deadline report", + `Deletion clock: quarantinedAt + ${DELETION_CLOCK_DAYS} days; near-deadline window: ${warnWithinDays} days`, + `Summary: total=${summary.total} expired=${summary.expired} near=${summary.near} healthy=${summary.healthy} unknown=${summary.unknown}`, + ]; + + if (rows.length === 0) { + lines.push("Ledger is empty; nothing quarantined."); + return `${lines.join("\n")}\n`; + } + + lines.push("Entries (soonest deadline first):"); + for (const row of rows) { + const timing = row.status === "expired" + ? `EXPIRED (${Math.abs(row.daysRemaining)} day${Math.abs(row.daysRemaining) === 1 ? "" : "s"} overdue)` + : row.daysRemaining == null + ? "deadline unknown" + : `${row.daysRemaining} day${row.daysRemaining === 1 ? "" : "s"} remaining`; + const deadline = row.deadline == null ? "unknown" : row.deadline; + const reason = truncateReason(row.reason) || "no reason recorded"; + lines.push(`- [${row.status}] ${row.file} — ${timing}; deadline=${deadline}; reason=${reason}`); + } + + return `${lines.join("\n")}\n`; +} + +function parseArgs(argv) { + const args = { + warnWithinDays: DEFAULT_WARN_WITHIN_DAYS, + json: false, + strict: false, + help: false, + }; + + for (const arg of argv) { + if (arg === "--json") { + args.json = true; + } else if (arg === "--strict") { + args.strict = true; + } else if (arg === "--help" || arg === "-h") { + args.help = true; + } else if (arg.startsWith("--warn-within=")) { + args.warnWithinDays = normalizeWarnWithinDays(arg.slice("--warn-within=".length)); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + return args; +} + +export function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr, now = new Date(), ledgerPath = path.join(rootDir, DEFAULT_QUARANTINE_PATH) } = {}) { + let args; + try { + args = parseArgs(argv); + } catch (error) { + stderr.write(`${error.message}\n`); + return 1; + } + + if (args.help) { + stdout.write("Usage: node scripts/check-quarantine-ledger.mjs [--warn-within=] [--json] [--strict]\n"); + return 0; + } + + let ledger; + try { + ledger = readLedger(ledgerPath); + } catch (error) { + stderr.write(`Failed to read quarantine ledger: ${error.message}\n`); + return 1; + } + + const rows = computeDeadlines(ledger, { now, warnWithinDays: args.warnWithinDays }); + const summary = summarizeRows(rows); + if (args.json) { + stdout.write(`${JSON.stringify({ summary, rows }, null, 2)}\n`); + } else { + stdout.write(renderReport(rows, { warnWithinDays: args.warnWithinDays })); + } + + return args.strict && (summary.expired > 0 || summary.near > 0) ? 1 : 0; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + process.exitCode = main(); +}