Files
fusion/scripts/check-quarantine-ledger.mjs
gsxdsm 39812f4898 test: quarantine one suite-only flake, record another, fix the lockstep guard
Full engine suite at a97aa84a20: 3 failures out of 12,414. All three diagnosed:

- self-healing-pending-wedge-notification's marker-selection case fails ONLY in
  a full-suite run (expects 1 elapsed marker, sees 2) and passes deterministically
  alone. This is its SECOND sighting, so per AGENTS.md it is an on-sight
  quarantine with no further discretion: ledger entry + matching vitest exclude,
  same commit, 2026-09-06 deletion deadline.
- spec-drift-reconciler's exponential-backoff case shows the same shape on a
  FIRST sighting, so it is recorded in the observed register instead of evicting
  that file's other passing coverage. Both are timer-driven reconciler tests that
  only fail alongside other suites, pointing at cross-file fake-timer state.
- merge-orphan-durable-write-inventory drift was pure lineHint movement (19
  changed, zero newly unclassified entries) after product edits shifted lines.
  Regenerated.

Also fixes check-quarantine-ledger.mjs, which could not see the exclude I added:
its comment stripper treated the `/**` inside glob literals like "node_modules/**"
and "src/**/*.slow.test.ts" as a block-comment opener and deleted through to the
next "*/", swallowing whole array literals and every entry after them. It now
scans string-aware, so the lockstep check actually holds. Nothing was appeased:
no timeout widened, no retry added, no assertion relaxed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-23 15:43:00 -07:00

296 lines
12 KiB
JavaScript

#!/usr/bin/env node
/*
FNXC:TestQuarantine 2026-07-12-00:00:
The flaky-test deletion ratchet had no visibility tool for entries approaching the `quarantinedAt + 14d` deletion deadline.
This report surfaces near-deadline quarantines so maintainers can make deliberate rescue-or-expire decisions while preserving the policy's report-only default; `--strict` is the opt-in enforcement path.
FNXC:QuarantineLockstep 2026-08-17-11:10:
A ledger row and its Vitest exclusion are one quarantine decision. Scan only comment-free `exclude:` array literals so stale FNXC prose and dashboard include shards cannot turn a disposition check into a false failure; strict mode rejects either half pointing at a missing test file.
*/
import { existsSync, readFileSync, readdirSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { DEFAULT_QUARANTINE_PATH, DELETION_CLOCK_DAYS } from "./test-velocity-baseline.mjs";
const MS_PER_DAY = 86_400_000;
const DEFAULT_WARN_WITHIN_DAYS = 5;
const REASON_MAX_LENGTH = 140;
const currentFilePath = fileURLToPath(import.meta.url);
const repoRoot = path.resolve(path.dirname(currentFilePath), "..");
function toDate(value) {
if (!value) return null;
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? null : parsed;
}
function ageDays(quarantinedAt, now) {
const quarantinedAtDate = toDate(quarantinedAt);
if (!quarantinedAtDate) return null;
return Math.floor((now.getTime() - quarantinedAtDate.getTime()) / MS_PER_DAY);
}
function formatIsoDate(date) {
return date.toISOString().slice(0, 10);
}
function normalizeWarnWithinDays(value) {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 0) {
throw new Error(`--warn-within must be a non-negative integer, got ${value}`);
}
return parsed;
}
function truncateReason(reason) {
const normalized = String(reason ?? "").replace(/\s+/g, " ").trim();
if (normalized.length <= REASON_MAX_LENGTH) return normalized;
return `${normalized.slice(0, REASON_MAX_LENGTH - 1)}…`;
}
function summarizeRows(rows) {
return rows.reduce(
(summary, row) => {
summary[row.status] += 1;
summary.total += 1;
return summary;
},
{ total: 0, expired: 0, near: 0, healthy: 0, unknown: 0 },
);
}
/*
FNXC:QuarantineLockstep 2026-08-23-22:45:
STRING-AWARE. The previous regex stripper treated the `/**` inside a glob literal such as
"src/**\/*.slow.test.ts" or "node_modules/**" as the start of a block comment, so it deleted from
there to the next "*\/" — swallowing whole array literals and the entries after them. A concrete
quarantine exclude placed after any such glob was then invisible, and this guard reported
`missing-exclude` for a file that WAS excluded (observed 2026-08-23 quarantining
self-healing-pending-wedge-notification.test.ts). Scan character by character instead, tracking
string literals, so comment markers inside strings are left alone.
*/
function stripComments(source) {
let out = "";
let quote = null;
let escaped = false;
for (let index = 0; index < source.length; index += 1) {
const character = source[index];
if (quote) {
out += character;
if (escaped) escaped = false;
else if (character === "\\") escaped = true;
else if (character === quote) quote = null;
continue;
}
if (character === '"' || character === "'" || character === "`") {
quote = character;
out += character;
continue;
}
if (character === "/" && source[index + 1] === "*") {
const end = source.indexOf("*/", index + 2);
index = end === -1 ? source.length : end + 1;
continue;
}
if (character === "/" && source[index + 1] === "/") {
const end = source.indexOf("\n", index);
if (end === -1) break;
index = end - 1;
continue;
}
out += character;
}
return out;
}
function extractBalancedArray(source, openingBracket) {
let depth = 0;
let quote = null;
let escaped = false;
for (let index = openingBracket; index < source.length; index += 1) {
const character = source[index];
if (quote) {
if (escaped) {
escaped = false;
} else if (character === "\\") {
escaped = true;
} else if (character === quote) {
quote = null;
}
continue;
}
if (character === "\"" || character === "'") {
quote = character;
} else if (character === "[") {
depth += 1;
} else if (character === "]") {
depth -= 1;
if (depth === 0) return source.slice(openingBracket, index + 1);
}
}
return null;
}
function extractConcreteExcludes(source) {
const commentFree = stripComments(source);
const excludes = [];
const excludePattern = /\bexclude\s*:/g;
let match;
while ((match = excludePattern.exec(commentFree))) {
let index = match.index + match[0].length;
while (/\s/.test(commentFree[index] ?? "")) index += 1;
if (commentFree[index] !== "[") continue;
const array = extractBalancedArray(commentFree, index);
if (array == null) continue;
const strings = /"((?:\\.|[^"\\])*)"/g;
let stringMatch;
while ((stringMatch = strings.exec(array))) {
const value = JSON.parse(`"${stringMatch[1]}"`);
if (/\.test\.tsx?$/.test(value) && !/[*?{}]/.test(value)) excludes.push(value);
}
excludePattern.lastIndex = index + array.length;
}
return excludes;
}
function discoverPackageConfigs(rootDir) {
const packagesRoot = path.join(rootDir, "packages");
if (!existsSync(packagesRoot)) return [];
return readdirSync(packagesRoot, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => path.join("packages", entry.name, "vitest.config.ts"))
.filter((config) => existsSync(path.join(rootDir, config)));
}
function normalizeConfigPath(rootDir, config) {
return path.isAbsolute(config) ? config : path.join(rootDir, config);
}
/**
* Checks the two directions of the quarantine decision without evaluating Vitest configuration.
* `packageConfigs` accepts root-relative or absolute config paths to keep fixture tests narrow.
*/
export function findLockstepViolations({ rootDir, ledger, packageConfigs = discoverPackageConfigs(rootDir) }) {
const configExcludes = new Map();
for (const config of packageConfigs) {
const configPath = normalizeConfigPath(rootDir, config);
if (!existsSync(configPath)) continue;
const relativeConfig = path.relative(rootDir, configPath);
configExcludes.set(relativeConfig, extractConcreteExcludes(readFileSync(configPath, "utf8")));
}
const violations = [];
for (const entry of Array.isArray(ledger?.entries) ? ledger.entries : []) {
const file = String(entry?.file ?? "");
const filePath = path.join(rootDir, file);
if (!existsSync(filePath)) {
violations.push({ kind: "missing-file", file, detail: "ledger entry names no file on disk" });
continue;
}
const packageMatch = /^packages\/([^/]+)\/(.+)$/.exec(file);
const config = packageMatch ? path.join("packages", packageMatch[1], "vitest.config.ts") : null;
if (config == null || !configExcludes.has(config)) {
violations.push({ kind: "unmapped-entry", file, config: config ?? undefined, detail: "ledger file has no package Vitest config" });
continue;
}
const packageRelativeFile = packageMatch[2];
if (!configExcludes.get(config).includes(packageRelativeFile)) {
violations.push({ kind: "missing-exclude", file, config, detail: "ledger file is absent from the package exclude array" });
}
}
for (const [config, excludes] of configExcludes) {
for (const excludedFile of excludes) {
const packageRoot = path.dirname(config);
if (!existsSync(path.join(rootDir, packageRoot, excludedFile))) {
violations.push({ kind: "dangling-exclude", file: path.join(packageRoot, excludedFile), config, detail: "exclude array names no file on disk" });
}
}
}
return violations;
}
export function readLedger(ledgerPath) {
if (!existsSync(ledgerPath)) return { entries: [] };
const json = JSON.parse(readFileSync(ledgerPath, "utf8"));
if (json?.entries != null && !Array.isArray(json.entries)) {
throw new Error(`quarantine ledger ${ledgerPath} must have an "entries" array`);
}
return json ?? { entries: [] };
}
export function computeDeadlines(json, { now = new Date(), warnWithinDays = DEFAULT_WARN_WITHIN_DAYS } = {}) {
const entries = Array.isArray(json?.entries) ? json.entries : [];
const rows = entries.map((entry, index) => {
const quarantinedAtDate = toDate(entry?.quarantinedAt);
const age = ageDays(entry?.quarantinedAt, now);
const daysRemaining = age == null ? null : DELETION_CLOCK_DAYS - age;
const deadlineDate = quarantinedAtDate == null ? null : new Date(quarantinedAtDate.getTime() + DELETION_CLOCK_DAYS * MS_PER_DAY);
let status = "unknown";
if (daysRemaining != null) status = daysRemaining <= 0 ? "expired" : daysRemaining <= warnWithinDays ? "near" : "healthy";
return { index, file: entry?.file ?? "unknown", reason: entry?.reason ?? "", quarantinedAt: entry?.quarantinedAt ?? null, ageDays: age, daysRemaining, deadline: deadlineDate == null ? null : formatIsoDate(deadlineDate), status };
});
return rows.sort((a, b) => {
if (a.deadline == null && b.deadline == null) return a.index - b.index;
if (a.deadline == null) return 1;
if (b.deadline == null) return -1;
return a.deadline.localeCompare(b.deadline) || a.file.localeCompare(b.file) || a.index - b.index;
});
}
export function renderReport(rows, { warnWithinDays = DEFAULT_WARN_WITHIN_DAYS, violations = [] } = {}) {
const summary = summarizeRows(rows);
const lines = [
"Quarantine ledger deadline report",
`Deletion clock: quarantinedAt + ${DELETION_CLOCK_DAYS} days; near-deadline window: ${warnWithinDays} days`,
`Summary: total=${summary.total} expired=${summary.expired} near=${summary.near} healthy=${summary.healthy} unknown=${summary.unknown} lockstep=${violations.length}`,
];
if (rows.length === 0) lines.push("Ledger is empty; nothing quarantined.");
else {
lines.push("Entries (soonest deadline first):");
for (const row of rows) {
const timing = row.status === "expired" ? `EXPIRED (${Math.abs(row.daysRemaining)} day${Math.abs(row.daysRemaining) === 1 ? "" : "s"} overdue)` : row.daysRemaining == null ? "deadline unknown" : `${row.daysRemaining} day${row.daysRemaining === 1 ? "" : "s"} remaining`;
lines.push(`- [${row.status}] ${row.file} — ${timing}; deadline=${row.deadline ?? "unknown"}; reason=${truncateReason(row.reason) || "no reason recorded"}`);
}
}
if (violations.length > 0) {
lines.push("Lockstep violations:");
for (const violation of violations) lines.push(`- [${violation.kind}] ${violation.file}${violation.config ? ` (${violation.config})` : ""}: ${violation.detail}`);
}
return `${lines.join("\n")}\n`;
}
function parseArgs(argv) {
const args = { warnWithinDays: DEFAULT_WARN_WITHIN_DAYS, json: false, strict: false, help: false };
for (const arg of argv) {
if (arg === "--json") args.json = true;
else if (arg === "--strict") args.strict = true;
else if (arg === "--help" || arg === "-h") args.help = true;
else if (arg.startsWith("--warn-within=")) args.warnWithinDays = normalizeWarnWithinDays(arg.slice("--warn-within=".length));
else throw new Error(`Unknown argument: ${arg}`);
}
return args;
}
export function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr, now = new Date(), ledgerPath = path.join(rootDir, DEFAULT_QUARANTINE_PATH) } = {}) {
let args;
try { args = parseArgs(argv); } catch (error) { stderr.write(`${error.message}\n`); return 1; }
if (args.help) { stdout.write("Usage: node scripts/check-quarantine-ledger.mjs [--warn-within=<days>] [--json] [--strict]\n"); return 0; }
let ledger;
try { ledger = readLedger(ledgerPath); } catch (error) { stderr.write(`Failed to read quarantine ledger: ${error.message}\n`); return 1; }
const rows = computeDeadlines(ledger, { now, warnWithinDays: args.warnWithinDays });
const violations = findLockstepViolations({ rootDir, ledger });
const summary = { ...summarizeRows(rows), lockstep: violations.length };
if (args.json) stdout.write(`${JSON.stringify({ summary, rows, lockstep: violations }, null, 2)}\n`);
else stdout.write(renderReport(rows, { warnWithinDays: args.warnWithinDays, violations }));
return args.strict && (summary.expired > 0 || summary.near > 0 || violations.length > 0) ? 1 : 0;
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) process.exitCode = main();