diff --git a/docs/testing.md b/docs/testing.md index 05a52faeb9..5887eb2a4d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -125,11 +125,20 @@ the vocabulary its siblings use). That is why the four classes are reported sepa netted — a wrong classification stays visible instead of silently moving the bar. `--json` emits the machine-readable form. `--strict` compares per-file counts against -`scripts/lib/lifecycle-column-census-baseline.json` and fails when any file's column-guard count -**rises** — the ratchet shape. It is deliberately **not** wired into the merge gate: a -thousand-site backlog cannot be a blocking check the day it is first measured, and a guard nobody -can pass is a guard everyone disables. Owners tightening their own area should re-record the -baseline in the same PR that lowers it. +`scripts/lib/lifecycle-column-census-baseline.json`: + +- a **rise** fails hard — that is the ratchet's purpose, "no new guards"; +- a **drop** TIGHTENS the baseline automatically, reports what it lowered, and exits 0. + + +The tightened file must be **committed** — in CI the write is discarded with the runner, which is why the gate +goes green rather than silently passing a stale allowance. `--strict --exact` restores hard failure on a drop, +for the end state where the count is pinned and any divergence is a real event. `--strict --update-baseline` +re-records unconditionally and prints `ACCEPTED RISES`, which is the only way to record a rise deliberately. The regression suite is `packages/engine/src/__tests__/lifecycle-column-census.test.ts`. It pins each form the census must catch (all six ids, non-`column` locals, single quotes, negation, diff --git a/packages/engine/src/__tests__/lifecycle-column-census.test.ts b/packages/engine/src/__tests__/lifecycle-column-census.test.ts index f15a0e4686..1d7d403992 100644 --- a/packages/engine/src/__tests__/lifecycle-column-census.test.ts +++ b/packages/engine/src/__tests__/lifecycle-column-census.test.ts @@ -475,3 +475,126 @@ describe("the baseline can always be re-recorded", () => { expect(cliSource().split("writeFileSync(").length - 1).toBe(1); }); }); + +/* +FNXC:LifecycleColumnCensus 2026-08-01-02-45 (coordinator item 2 — the ratchet must FOLLOW THE COUNT DOWN): + +A DROP NOW TIGHTENS THE BASELINE INSTEAD OF FAILING. Failing hard was defensible in isolation — a stale +allowance is a hole, since those guards can return up to the old count while the check stays green. What it +missed is that the drop is almost never the author's to fix: eleven files dropped during one merge wave, none +of those PRs re-recorded, and none of their authors did anything wrong. + +Measured three times since CI began gating this: `columnRoles.ts` 0 -> 1, then `executor.ts` twice. A +permanently-red gate is a bigger hole than a stale allowance, because it gets ignored and then nothing is +guarded at all. The RISE check — the actual purpose — is untouched and still fails hard. + +Driven end to end through the real CLI with an isolated baseline (`FUSION_CENSUS_BASELINE`), because the exit +code and the file rewrite ARE the contract and no source-level assertion can prove them. All four transitions +were exercised by hand first: + drop, --strict exit 0, "TIGHTENED", baseline rewritten 9 -> 6 + drop, --strict --exact exit 1, baseline untouched + rise, --strict exit 1 + clean exit 0 +*/ +describe("the ratchet follows the count down", () => { + const repoRoot = new URL("../../../../", import.meta.url).pathname; + const cliPath = `${repoRoot}scripts/lifecycle-column-census.mjs`; + const realBaseline = `${repoRoot}scripts/lib/lifecycle-column-census-baseline.json`; + + async function run(mutate: (baseline: any) => string, args: string[], touchedPaths?: () => string) { + const { mkdtemp, writeFile, readFile } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { execFile } = await import("node:child_process"); + + const baseline = JSON.parse(await readFile(realBaseline, "utf8")); + const file = mutate(baseline); + const dir = await mkdtemp(join(tmpdir(), "fusion-census-tighten-")); + const path = join(dir, "baseline.json"); + await writeFile(path, `${JSON.stringify(baseline, null, 2)}\n`); + + const result = await new Promise<{ code: number; out: string }>((resolve) => { + execFile( + process.execPath, [cliPath, ...args], + { + cwd: repoRoot, + env: { + ...process.env, + FUSION_CENSUS_BASELINE_PATH: path, + /* Empty string = "this change touched nothing", which is the lenient path the other cases need. */ + FUSION_CENSUS_TOUCHED_PATHS: touchedPaths ? touchedPaths() : "", + }, + maxBuffer: 32 * 1024 * 1024, + }, + (error, stdout, stderr) => resolve({ code: (error as { code?: number } | null)?.code ?? 0, out: `${stdout}${stderr}` }), + ); + }); + const after = JSON.parse(await readFile(path, "utf8")); + return { + ...result, + file, + inflatedFrom: baseline.byFile[file] as number, + allowedAfter: after.byFile[file] as number, + }; + } + + /** Inflate one file's allowance, which is a DROP from the CLI's point of view. */ + const inflate = (baseline: any): string => { + const [file, count] = Object.entries(baseline.byFile as Record).find(([, c]) => c > 1) ?? []; + baseline.byFile[file as string] = (count as number) + 3; + return file as string; + }; + + it("TIGHTENS on a drop and exits 0, so somebody else's merge cannot redden the gate", async () => { + const run1 = await run(inflate, ["--strict"]); + + expect(run1.code).toBe(0); + expect(run1.out).toContain("TIGHTENED"); + /* + The WRITE is the point, so assert it directly against the inflated value rather than against itself — my + first version compared `allowedAfter` to `4 + allowedAfter`, which is true for every number and proved + nothing. Recording that here because it is the same vacuous-assertion trap this file keeps documenting, + and I walked into it while writing the case that guards against it. + */ + expect(run1.allowedAfter).toBe(run1.inflatedFrom - 3); + expect(run1.out).toContain("COMMIT IT"); + }, 30_000); + + it("FAILS when the change TOUCHES the file that dropped, so the allowance cannot stay open", async () => { + /* + FNXC:LifecycleColumnCensus 2026-07-30-12:10 (PR #2679 review — greptile P1): + The auto-tighten write is discarded with the CI runner, so the committed allowance stays stale and a + later change could regrow guards up to it while the gate is green. Regrowing means EDITING the file, + so a touched file must be re-recorded in the change that touched it. That is what makes the hole + unreachable rather than merely documented. + */ + let touchedFile = ""; + const run1 = await run((baseline) => { touchedFile = inflate(baseline); return touchedFile; }, ["--strict"], () => touchedFile); + + expect(run1.code).toBe(1); + expect(run1.out).toContain("TOUCHES files whose guard count dropped"); + // The baseline must be left ALONE on the failure path — a rewrite here would defeat the demand. + expect(run1.allowedAfter).toBe(run1.inflatedFrom); + }, 30_000); + + it("still FAILS on a drop under --exact, and leaves the baseline alone", async () => { + // The pinned end state: when the count is meant to be fixed, any divergence is a real event. + const run1 = await run(inflate, ["--strict", "--exact"]); + + expect(run1.code).toBe(1); + expect(run1.out).toContain("baseline is STALE"); + }, 30_000); + + it("still FAILS on a rise, which is the check's actual purpose", async () => { + const deflate = (baseline: any): string => { + const [file, count] = Object.entries(baseline.byFile as Record).find(([, c]) => c > 1) ?? []; + baseline.byFile[file as string] = (count as number) - 1; + return file as string; + }; + + const run1 = await run(deflate, ["--strict"]); + + expect(run1.code).toBe(1); + expect(run1.out).toContain("column-guard count ROSE"); + }, 30_000); +}); diff --git a/scripts/lifecycle-column-census.mjs b/scripts/lifecycle-column-census.mjs index 2a57cc8f6e..4e2c80a8be 100644 --- a/scripts/lifecycle-column-census.mjs +++ b/scripts/lifecycle-column-census.mjs @@ -88,6 +88,8 @@ const json = process.argv.includes("--json"); const strict = process.argv.includes("--strict"); const compare = process.argv.includes("--compare"); const updateBaseline = process.argv.includes("--update-baseline"); +/* `--exact` keeps hard failure on a DROP, for the end state where the count is pinned. */ +const exact = process.argv.includes("--exact"); if (json) { console.log(JSON.stringify({ scannedFiles: files.length, ...summary, byFile: summary.byFile }, null, 2)); @@ -362,7 +364,7 @@ The flag is an explicit operator action, so it re-records unconditionally and PR under `ACCEPTED RISES`. Silently swallowing a rise is the real danger; refusing to let anyone re-record is the same danger one step later, wearing a red check nobody trusts. */ -if (updateBaseline) { +function writeBaseline() { writeFileSync( BASELINE_PATH, `${JSON.stringify({ @@ -376,6 +378,10 @@ if (updateBaseline) { queryByFile: Object.fromEntries(summary.queryByFile), }, null, 2)}\n`, ); +} + +if (updateBaseline) { + writeBaseline(); if (regressions.length > 0) { console.log("\n ACCEPTED RISES (a merge or a conversion added guards here — convert them or they stay in the bar):"); for (const r of regressions) { @@ -410,16 +416,88 @@ it. The `!deliberateTracked && updateBaseline` condition went with it: the uncon legacy-shape migration too. */ if (stale.length > 0) { - console.error("\nlifecycle-column-census --strict: baseline is STALE — it allows more than the tree has\n"); - for (const s of stale) { - console.error(` ${s.file}: allows ${s.allowed}, tree has ${s.count}`); + /* + FNXC:LifecycleColumnCensus 2026-08-01-02-30 (coordinator item 2 — the ratchet must FOLLOW THE COUNT DOWN): + A DROP TIGHTENS THE BASELINE INSTEAD OF FAILING. The old behaviour failed hard, and the reasoning was sound + in isolation — a stale allowance is a hole, since those guards can return up to the old count while the + check stays green. What it missed is that the drop is almost never the author's to fix: eleven files dropped + during one merge wave, none of those PRs re-recorded, and none of their authors did anything wrong. Measured + three separate times since CI began gating this (`columnRoles.ts` 0->1, then `executor.ts` twice). + + A PERMANENTLY-RED GATE IS A BIGGER HOLE THAN A STALE ALLOWANCE, because it gets ignored and then nothing is + guarded at all. So the ceiling now follows the count down automatically and says so, while the RISE check — + the actual purpose, "no new guards" — still fails hard and untouched. + + THE RESIDUAL, named rather than glossed: in CI the write is discarded with the runner, so the committed + baseline stays stale until someone commits a tightened one. The exposure is bounded (regrowth only up to the + old count) and printed on every run, and it is strictly smaller than the exposure from a check people route + around. `--exact` keeps hard failure for the end state, when the count is meant to be pinned and any + divergence is a real event. + */ + /* + FNXC:LifecycleColumnCensus 2026-07-30-12:10 (PR #2679 review — greptile P1): + A TOUCHED FILE MUST BE RE-RECORDED; AN UNTOUCHED ONE IS AUTO-TIGHTENED. + + The residual named below is real: in CI the tightening write is discarded with the runner, so the + committed allowance stays stale and a later change can regrow guards up to it while the gate is + green. Naming that is not closing it. + + This closes it where the regrowth would have to happen. Regrowing a guard means EDITING the file, + so requiring an exact baseline only for files the change TOUCHES makes the hole unreachable — while + the case this PR exists for stays green, because those authors did not touch the files that dropped + (eleven files dropped in one merge wave; none of those authors did anything wrong). + + Falls back to the lenient path when no base ref resolves, so a detached or shallow checkout + degrades to the previous behaviour rather than failing closed on a git detail. + */ + let touched = new Set(); + /* + The touched set is overridable for the same reason BASELINE_PATH is: otherwise this branch can only + be tested against whatever the CURRENT branch happens to have changed, so the test's outcome would + depend on the diff of the PR running it. Production never sets it. + */ + if (process.env.FUSION_CENSUS_TOUCHED_PATHS !== undefined) { + touched = new Set(process.env.FUSION_CENSUS_TOUCHED_PATHS.split(",").map((f) => f.trim()).filter(Boolean)); + } else { + try { + const base = process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : "origin/main"; + touched = new Set( + execSync(`git diff --name-only ${base}...HEAD`, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }) + .split("\n").map((f) => f.trim()).filter(Boolean), + ); + } catch { + /* No usable base ref — leave `touched` empty so every entry takes the lenient path. */ + } } - console.error( - "\nA stale allowance is a hole: those guards can be reintroduced later and this check stays\n" + - "green. Re-record the baseline in the SAME PR that lowered the count:\n\n" + - " node scripts/lifecycle-column-census.mjs --strict --update-baseline\n", + + const staleTouched = stale.filter((entry) => touched.has(entry.file)); + if (staleTouched.length > 0) { + console.error( + "\nlifecycle-column-census --strict: this change TOUCHES files whose guard count dropped, so the\n" + + "baseline must be re-recorded in this change — otherwise the allowance stays open for regrowth.\n", + ); + for (const entry of staleTouched) { + console.error(` ${entry.file}: allows ${entry.allowed}, tree has ${entry.count}`); + } + console.error("\nRe-record it:\n\n node scripts/lifecycle-column-census.mjs --strict --update-baseline\n"); + process.exit(1); + } + + const lines = stale.map((entry) => ` ${entry.file}: allows ${entry.allowed}, tree has ${entry.count}`); + if (exact) { + console.error("\nlifecycle-column-census --strict --exact: baseline is STALE — it allows more than the tree has\n"); + for (const line of lines) console.error(line); + console.error("\nRe-record it:\n\n node scripts/lifecycle-column-census.mjs --strict --update-baseline\n"); + process.exit(1); + } + writeBaseline(); + console.log("\nlifecycle-column-census --strict: baseline TIGHTENED — the tree has fewer guards than it allowed\n"); + for (const line of lines) console.log(line); + console.log( + "\nThe baseline file has been rewritten downward. COMMIT IT so the allowance cannot be regrown into;\n" + + "in CI this write is discarded with the runner, which is why the gate is green and not silent.\n", ); - process.exit(1); + process.exit(0); } console.log("\nlifecycle-column-census --strict: every file matches its baseline exactly.");