FN-7912: add quarantine-ledger deadline visibility check
Add a report-only script that surfaces flaky-test quarantine entries approaching their 14-day deletion clock, so maintainers can make deliberate rescue-or-expire decisions before entries silently expire. - Add scripts/check-quarantine-ledger.mjs: reads scripts/lib/test-quarantine.json, computes days-remaining against the existing 14-day deletion clock (shared DELETION_CLOCK_DAYS from scripts/test-velocity-baseline.mjs), and buckets each entry as expired/near/healthy/unknown - Support --warn-within=<days> (default 5) to tune the near-deadline window, --json for machine-readable output, and --strict as an opt-in local/CI gate (exits 1 on expired/near entries) while default mode stays exit-0 and non-blocking - Wire pnpm check:quarantine-ledger script in package.json - Add scripts/__tests__/check-quarantine-ledger.test.mjs covering deadline bucketing/sorting, empty/missing ledger handling, --strict behavior, and --json output shape - Document the new command and its flags in docs/testing.md under the quarantine ledger/deletion ratchet section Files changed: docs/testing.md | 10 + package.json | 1 + scripts/__tests__/check-quarantine-ledger.test.mjs | 159 ++++++++++++++++ scripts/check-quarantine-ledger.mjs | 202 +++++++++++++++++++++ 4 files changed, 372 insertions(+) Fusion-Task-Id: FN-7912 Fusion-Task-Lineage: c08e2e09-473a-4ad0-8c27-43cbc3355168 Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
@@ -227,6 +227,16 @@ Flaky tests are quarantined ON SIGHT and deleted on a 2-week clock. This is writ
|
||||
|
||||
**The clock:** an entry expires 14 days after `quarantinedAt`. Whoever touches the suite and finds an expired entry deletes the test file, its ledger entry, and its config exclude (git history is the archive). `scripts/check-test-inventory.mjs --diff` stays deliberately unwired in CI because it would fail on exactly these deletions.
|
||||
|
||||
### Quarantine deadline visibility check
|
||||
|
||||
Run `pnpm check:quarantine-ledger` to print a soonest-deadline-first summary of `scripts/lib/test-quarantine.json`. The command uses the same 14-day deletion clock (`quarantinedAt + 14d`) as the velocity baseline and reports each entry as expired, near-deadline, healthy, or unknown when `quarantinedAt` is missing/invalid. It is a visibility aid only: default mode exits 0 even when entries are near or expired, preserving the deliberately-unwired policy and leaving rescue-or-delete decisions to maintainers.
|
||||
|
||||
Flags:
|
||||
|
||||
- `--warn-within=<days>` changes the near-deadline window from the default 5 days.
|
||||
- `--json` emits the computed rows plus summary counts for machine consumption.
|
||||
- `--strict` exits 1 when any entry is expired or near-deadline, for opt-in local or project-specific gates only. Do not wire this into `pretest`, `test:gate`, or other default blocking lanes without an explicit policy change.
|
||||
|
||||
**Rescue** (before the clock runs out) requires both: evidence the test catches real regressions, and a root-cause fix for the flake. Stabilization passes — widened timeouts, retries, loosened assertions — are appeasement, not rescue, and are banned (for agents especially).
|
||||
|
||||
### Vitest timeout-appeasement guard
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"pretest:full": "node scripts/check-no-nohup.mjs && node scripts/check-no-kill-4040.mjs && node scripts/check-no-test-timeout-appeasement.mjs && node scripts/check-changeset-format.mjs",
|
||||
"check:line-count": "node scripts/check-file-line-count.mjs",
|
||||
"check:changesets": "node scripts/check-changeset-format.mjs",
|
||||
"check:quarantine-ledger": "node scripts/check-quarantine-ledger.mjs",
|
||||
"test:gate": "node scripts/check-no-nohup.mjs && node scripts/check-no-kill-4040.mjs && node scripts/check-no-test-timeout-appeasement.mjs && node scripts/check-changeset-format.mjs && pnpm --filter @fusion/engine test:core && pnpm --filter @runfusion/fusion test:ci-shape",
|
||||
"smoke:boot": "node scripts/boot-smoke.mjs",
|
||||
"local": "node scripts/start-local.mjs",
|
||||
|
||||
159
scripts/__tests__/check-quarantine-ledger.test.mjs
Normal file
159
scripts/__tests__/check-quarantine-ledger.test.mjs
Normal file
@@ -0,0 +1,159 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
computeDeadlines,
|
||||
main,
|
||||
readLedger,
|
||||
renderReport,
|
||||
} from "../check-quarantine-ledger.mjs";
|
||||
|
||||
function captureStream() {
|
||||
let text = "";
|
||||
return {
|
||||
stream: { write(chunk) { text += chunk; } },
|
||||
get text() { return text; },
|
||||
};
|
||||
}
|
||||
|
||||
function tempRoot() {
|
||||
return mkdtempSync(path.join(tmpdir(), "fusion-quarantine-ledger-"));
|
||||
}
|
||||
|
||||
function writeLedger(rootDir, ledger) {
|
||||
const ledgerPath = path.join(rootDir, "scripts/lib/test-quarantine.json");
|
||||
mkdirSync(path.dirname(ledgerPath), { recursive: true });
|
||||
writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, "utf8");
|
||||
return ledgerPath;
|
||||
}
|
||||
|
||||
const fixedNow = new Date("2026-07-12T12:00:00.000Z");
|
||||
|
||||
const fixtureLedger = {
|
||||
entries: [
|
||||
{
|
||||
file: "healthy.test.ts",
|
||||
reason: "fresh quarantine",
|
||||
quarantinedAt: "2026-07-12",
|
||||
},
|
||||
{
|
||||
file: "near.test.ts",
|
||||
reason: "approaching deletion deadline",
|
||||
quarantinedAt: "2026-07-04",
|
||||
},
|
||||
{
|
||||
file: "expired.test.ts",
|
||||
reason: "past deletion deadline",
|
||||
quarantinedAt: "2026-06-27",
|
||||
},
|
||||
{
|
||||
file: "unknown.test.ts",
|
||||
reason: "missing quarantine date",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
test("computeDeadlines buckets healthy, near, expired, and unknown entries", () => {
|
||||
const rows = computeDeadlines(fixtureLedger, { now: fixedNow, warnWithinDays: 6 });
|
||||
const byFile = Object.fromEntries(rows.map((row) => [row.file, row]));
|
||||
|
||||
assert.equal(byFile["healthy.test.ts"].status, "healthy");
|
||||
assert.equal(byFile["healthy.test.ts"].daysRemaining, 14);
|
||||
assert.equal(byFile["healthy.test.ts"].deadline, "2026-07-26");
|
||||
|
||||
assert.equal(byFile["near.test.ts"].status, "near");
|
||||
assert.equal(byFile["near.test.ts"].daysRemaining, 6);
|
||||
|
||||
assert.equal(byFile["expired.test.ts"].status, "expired");
|
||||
assert.ok(byFile["expired.test.ts"].daysRemaining <= 0);
|
||||
|
||||
assert.equal(byFile["unknown.test.ts"].status, "unknown");
|
||||
assert.equal(byFile["unknown.test.ts"].daysRemaining, null);
|
||||
assert.equal(byFile["unknown.test.ts"].deadline, null);
|
||||
});
|
||||
|
||||
test("computeDeadlines sorts soonest deadline first with unknown entries last", () => {
|
||||
const rows = computeDeadlines(fixtureLedger, { now: fixedNow, warnWithinDays: 6 });
|
||||
|
||||
assert.deepEqual(rows.map((row) => row.file), [
|
||||
"expired.test.ts",
|
||||
"near.test.ts",
|
||||
"healthy.test.ts",
|
||||
"unknown.test.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
test("renderReport handles an empty ledger without throwing", () => {
|
||||
const rows = computeDeadlines({ entries: [] }, { now: fixedNow });
|
||||
const report = renderReport(rows);
|
||||
|
||||
assert.deepEqual(rows, []);
|
||||
assert.match(report, /Ledger is empty; nothing quarantined\./);
|
||||
assert.match(report, /Summary: total=0 expired=0 near=0 healthy=0 unknown=0/);
|
||||
});
|
||||
|
||||
test("readLedger tolerates a missing ledger and rejects non-array entries", () => {
|
||||
const rootDir = tempRoot();
|
||||
try {
|
||||
assert.deepEqual(readLedger(path.join(rootDir, "missing.json")), { entries: [] });
|
||||
const ledgerPath = writeLedger(rootDir, { entries: {} });
|
||||
assert.throws(
|
||||
() => readLedger(ledgerPath),
|
||||
/quarantine ledger .* must have an "entries" array/,
|
||||
);
|
||||
} finally {
|
||||
rmSync(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("main is report-only by default but --strict fails on near or expired entries", () => {
|
||||
const rootDir = tempRoot();
|
||||
try {
|
||||
const ledgerPath = writeLedger(rootDir, fixtureLedger);
|
||||
const stdout = captureStream();
|
||||
const stderr = captureStream();
|
||||
|
||||
assert.equal(main([], { rootDir, ledgerPath, stdout: stdout.stream, stderr: stderr.stream, now: fixedNow }), 0);
|
||||
assert.match(stdout.text, /expired=1 near=0 healthy=2 unknown=1/);
|
||||
assert.equal(stderr.text, "");
|
||||
|
||||
const strictStdout = captureStream();
|
||||
assert.equal(main(["--strict", "--warn-within=6"], { rootDir, ledgerPath, stdout: strictStdout.stream, stderr: stderr.stream, now: fixedNow }), 1);
|
||||
|
||||
const healthyLedgerPath = writeLedger(rootDir, { entries: [{ file: "healthy.test.ts", quarantinedAt: "2026-07-12" }] });
|
||||
const healthyStdout = captureStream();
|
||||
assert.equal(main(["--strict"], { rootDir, ledgerPath: healthyLedgerPath, stdout: healthyStdout.stream, stderr: stderr.stream, now: fixedNow }), 0);
|
||||
} finally {
|
||||
rmSync(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("--json output parses and includes per-entry status and days remaining", () => {
|
||||
const rootDir = tempRoot();
|
||||
try {
|
||||
const ledgerPath = writeLedger(rootDir, fixtureLedger);
|
||||
const stdout = captureStream();
|
||||
const stderr = captureStream();
|
||||
|
||||
assert.equal(main(["--json", "--warn-within=6"], { rootDir, ledgerPath, stdout: stdout.stream, stderr: stderr.stream, now: fixedNow }), 0);
|
||||
|
||||
const parsed = JSON.parse(stdout.text);
|
||||
assert.equal(parsed.summary.expired, 1);
|
||||
assert.equal(parsed.summary.near, 1);
|
||||
assert.deepEqual(
|
||||
parsed.rows.map((row) => ({ file: row.file, status: row.status, daysRemaining: row.daysRemaining })),
|
||||
[
|
||||
{ file: "expired.test.ts", status: "expired", daysRemaining: -1 },
|
||||
{ file: "near.test.ts", status: "near", daysRemaining: 6 },
|
||||
{ file: "healthy.test.ts", status: "healthy", daysRemaining: 14 },
|
||||
{ file: "unknown.test.ts", status: "unknown", daysRemaining: null },
|
||||
],
|
||||
);
|
||||
assert.equal(stderr.text, "");
|
||||
} finally {
|
||||
rmSync(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
202
scripts/check-quarantine-ledger.mjs
Normal file
202
scripts/check-quarantine-ledger.mjs
Normal file
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
FNXC:TestQuarantine 2026-07-12-00:00:
|
||||
The flaky-test deletion ratchet had no visibility tool for entries approaching the `quarantinedAt + 14d` deletion deadline.
|
||||
This report surfaces near-deadline quarantines so maintainers can make deliberate rescue-or-expire decisions while preserving the policy's report-only default; `--strict` is the opt-in enforcement path.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEFAULT_QUARANTINE_PATH, DELETION_CLOCK_DAYS } from "./test-velocity-baseline.mjs";
|
||||
|
||||
const MS_PER_DAY = 86_400_000;
|
||||
const DEFAULT_WARN_WITHIN_DAYS = 5;
|
||||
const REASON_MAX_LENGTH = 140;
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(currentFilePath), "..");
|
||||
|
||||
function toDate(value) {
|
||||
if (!value) return null;
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
|
||||
function ageDays(quarantinedAt, now) {
|
||||
const quarantinedAtDate = toDate(quarantinedAt);
|
||||
if (!quarantinedAtDate) return null;
|
||||
return Math.floor((now.getTime() - quarantinedAtDate.getTime()) / MS_PER_DAY);
|
||||
}
|
||||
|
||||
function formatIsoDate(date) {
|
||||
return date.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function normalizeWarnWithinDays(value) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
throw new Error(`--warn-within must be a non-negative integer, got ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function truncateReason(reason) {
|
||||
const normalized = String(reason ?? "").replace(/\s+/g, " ").trim();
|
||||
if (normalized.length <= REASON_MAX_LENGTH) return normalized;
|
||||
return `${normalized.slice(0, REASON_MAX_LENGTH - 1)}…`;
|
||||
}
|
||||
|
||||
function summarizeRows(rows) {
|
||||
return rows.reduce(
|
||||
(summary, row) => {
|
||||
summary[row.status] += 1;
|
||||
summary.total += 1;
|
||||
return summary;
|
||||
},
|
||||
{ total: 0, expired: 0, near: 0, healthy: 0, unknown: 0 },
|
||||
);
|
||||
}
|
||||
|
||||
export function readLedger(ledgerPath) {
|
||||
if (!existsSync(ledgerPath)) {
|
||||
return { entries: [] };
|
||||
}
|
||||
|
||||
const json = JSON.parse(readFileSync(ledgerPath, "utf8"));
|
||||
if (json?.entries != null && !Array.isArray(json.entries)) {
|
||||
throw new Error(`quarantine ledger ${ledgerPath} must have an "entries" array`);
|
||||
}
|
||||
return json ?? { entries: [] };
|
||||
}
|
||||
|
||||
export function computeDeadlines(json, { now = new Date(), warnWithinDays = DEFAULT_WARN_WITHIN_DAYS } = {}) {
|
||||
const entries = Array.isArray(json?.entries) ? json.entries : [];
|
||||
const rows = entries.map((entry, index) => {
|
||||
const quarantinedAtDate = toDate(entry?.quarantinedAt);
|
||||
const age = ageDays(entry?.quarantinedAt, now);
|
||||
const daysRemaining = age == null ? null : DELETION_CLOCK_DAYS - age;
|
||||
const deadlineDate = quarantinedAtDate == null
|
||||
? null
|
||||
: new Date(quarantinedAtDate.getTime() + DELETION_CLOCK_DAYS * MS_PER_DAY);
|
||||
let status = "unknown";
|
||||
if (daysRemaining != null) {
|
||||
if (daysRemaining <= 0) {
|
||||
status = "expired";
|
||||
} else if (daysRemaining <= warnWithinDays) {
|
||||
status = "near";
|
||||
} else {
|
||||
status = "healthy";
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
index,
|
||||
file: entry?.file ?? "unknown",
|
||||
reason: entry?.reason ?? "",
|
||||
quarantinedAt: entry?.quarantinedAt ?? null,
|
||||
ageDays: age,
|
||||
daysRemaining,
|
||||
deadline: deadlineDate == null ? null : formatIsoDate(deadlineDate),
|
||||
status,
|
||||
};
|
||||
});
|
||||
|
||||
return rows.sort((a, b) => {
|
||||
if (a.deadline == null && b.deadline == null) return a.index - b.index;
|
||||
if (a.deadline == null) return 1;
|
||||
if (b.deadline == null) return -1;
|
||||
return a.deadline.localeCompare(b.deadline) || a.file.localeCompare(b.file) || a.index - b.index;
|
||||
});
|
||||
}
|
||||
|
||||
export function renderReport(rows, { warnWithinDays = DEFAULT_WARN_WITHIN_DAYS } = {}) {
|
||||
const summary = summarizeRows(rows);
|
||||
const lines = [
|
||||
"Quarantine ledger deadline report",
|
||||
`Deletion clock: quarantinedAt + ${DELETION_CLOCK_DAYS} days; near-deadline window: ${warnWithinDays} days`,
|
||||
`Summary: total=${summary.total} expired=${summary.expired} near=${summary.near} healthy=${summary.healthy} unknown=${summary.unknown}`,
|
||||
];
|
||||
|
||||
if (rows.length === 0) {
|
||||
lines.push("Ledger is empty; nothing quarantined.");
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
lines.push("Entries (soonest deadline first):");
|
||||
for (const row of rows) {
|
||||
const timing = row.status === "expired"
|
||||
? `EXPIRED (${Math.abs(row.daysRemaining)} day${Math.abs(row.daysRemaining) === 1 ? "" : "s"} overdue)`
|
||||
: row.daysRemaining == null
|
||||
? "deadline unknown"
|
||||
: `${row.daysRemaining} day${row.daysRemaining === 1 ? "" : "s"} remaining`;
|
||||
const deadline = row.deadline == null ? "unknown" : row.deadline;
|
||||
const reason = truncateReason(row.reason) || "no reason recorded";
|
||||
lines.push(`- [${row.status}] ${row.file} — ${timing}; deadline=${deadline}; reason=${reason}`);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
warnWithinDays: DEFAULT_WARN_WITHIN_DAYS,
|
||||
json: false,
|
||||
strict: false,
|
||||
help: false,
|
||||
};
|
||||
|
||||
for (const arg of argv) {
|
||||
if (arg === "--json") {
|
||||
args.json = true;
|
||||
} else if (arg === "--strict") {
|
||||
args.strict = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
args.help = true;
|
||||
} else if (arg.startsWith("--warn-within=")) {
|
||||
args.warnWithinDays = normalizeWarnWithinDays(arg.slice("--warn-within=".length));
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr, now = new Date(), ledgerPath = path.join(rootDir, DEFAULT_QUARANTINE_PATH) } = {}) {
|
||||
let args;
|
||||
try {
|
||||
args = parseArgs(argv);
|
||||
} catch (error) {
|
||||
stderr.write(`${error.message}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (args.help) {
|
||||
stdout.write("Usage: node scripts/check-quarantine-ledger.mjs [--warn-within=<days>] [--json] [--strict]\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
let ledger;
|
||||
try {
|
||||
ledger = readLedger(ledgerPath);
|
||||
} catch (error) {
|
||||
stderr.write(`Failed to read quarantine ledger: ${error.message}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const rows = computeDeadlines(ledger, { now, warnWithinDays: args.warnWithinDays });
|
||||
const summary = summarizeRows(rows);
|
||||
if (args.json) {
|
||||
stdout.write(`${JSON.stringify({ summary, rows }, null, 2)}\n`);
|
||||
} else {
|
||||
stdout.write(renderReport(rows, { warnWithinDays: args.warnWithinDays }));
|
||||
}
|
||||
|
||||
return args.strict && (summary.expired > 0 || summary.near > 0) ? 1 : 0;
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
process.exitCode = main();
|
||||
}
|
||||
Reference in New Issue
Block a user