FN-7079: Correct quarantine baseline reporting
Correct the velocity baseline to reflect an empty quarantine ledger and cover that report-only path. - Update the published test velocity baseline quarantine totals, buckets, trend row, and #leads summary to zero. - Add a regression test that regenerates the report from an empty live quarantine ledger without measuring lanes. - Assert deletion-due and bucket rows stay zero and stale nonzero quarantine text is removed. Files changed: docs/test-velocity-baseline.md | 10 ++-- scripts/__tests__/test-velocity-baseline.test.mjs | 72 ++++++++++++++++++++++- 2 files changed, 76 insertions(+), 6 deletions(-) Fusion-Task-Id: FN-7079 Fusion-Task-Lineage: d641adc3-f991-4d95-bcb2-7b20bbfeb0bb
This commit is contained in:
@@ -16,7 +16,7 @@
|
||||
| Merge gate wall-time (`pnpm test:gate`) | 7.5s | -9.3s |
|
||||
| Boot smoke wall-time (`pnpm smoke:boot`) | 18.2s | -4.7s |
|
||||
| Changed-only test wall-time (`pnpm test`) | 9.4s | -10.2s |
|
||||
| Quarantine / flake count | 3 | +3 |
|
||||
| Quarantine / flake count | 0 | 0 |
|
||||
| Deletion-due quarantines | 0 | n/a |
|
||||
|
||||
## Measurement failures
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
| Age bucket | Count |
|
||||
|---|---:|
|
||||
| 0-6 days | 3 |
|
||||
| 0-6 days | 0 |
|
||||
| 7-13 days | 0 |
|
||||
| deletion due (>=14 days) | 0 |
|
||||
| unknown/future | 0 |
|
||||
@@ -72,15 +72,15 @@
|
||||
| Row | Captured at | Gate | Boot smoke | `pnpm test` | Quarantine count |
|
||||
|---|---|---:|---:|---:|---:|
|
||||
| Previous | 2026-06-23T18:43:21.941Z | 16.8s | 22.8s | 19.6s | 0 |
|
||||
| Latest | 2026-06-25T05:47:54.804Z | 7.5s | 18.2s | 9.4s | 3 |
|
||||
| Delta | — | -9.3s | -4.7s | -10.2s | +3 |
|
||||
| Latest | 2026-06-25T05:47:54.804Z | 7.5s | 18.2s | 9.4s | 0 |
|
||||
| Delta | — | -9.3s | -4.7s | -10.2s | 0 |
|
||||
|
||||
_Future weekly rows append to `scripts/test-velocity-history.json`; compare the latest row against the previous row before posting to #leads._
|
||||
|
||||
## Post to #leads
|
||||
|
||||
```text
|
||||
FN-6612 weekly test velocity: gate 7.5s (-9.3s), boot smoke 18.2s (-4.7s), pnpm test 9.4s (-10.2s), quarantine ledger 3 (+3). Slowest file: packages/dashboard/app/components/__tests__/SettingsModal.test.tsx at 1m 01s. Deletion-due quarantines: 0.
|
||||
FN-6612 weekly test velocity: gate 7.5s (-9.3s), boot smoke 18.2s (-4.7s), pnpm test 9.4s (-10.2s), quarantine ledger 0 (0). Slowest file: packages/dashboard/app/components/__tests__/SettingsModal.test.tsx at 1m 01s. Deletion-due quarantines: 0.
|
||||
```
|
||||
|
||||
## How to refresh
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -40,6 +40,12 @@ function makeTimings(count = 25) {
|
||||
};
|
||||
}
|
||||
|
||||
function writeJson(rootDir, relativePath, value) {
|
||||
const fullPath = path.join(rootDir, relativePath);
|
||||
mkdirSync(path.dirname(fullPath), { recursive: true });
|
||||
writeFileSync(fullPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
describe("topSlowestFiles", () => {
|
||||
it("returns exactly 20 rows in descending duration order with package attribution and stable ties", () => {
|
||||
const rows = topSlowestFiles(makeTimings(), 20);
|
||||
@@ -207,6 +213,70 @@ describe("main", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("renders zero quarantine figures from an empty live ledger in report-only mode", async () => {
|
||||
const rootDir = tempRoot();
|
||||
try {
|
||||
writeJson(rootDir, "scripts/lib/test-quarantine.json", { entries: [] });
|
||||
writeJson(rootDir, "scripts/test-velocity-history.json", {
|
||||
entries: [
|
||||
{
|
||||
capturedAt: "2026-06-18T12:00:00.000Z",
|
||||
gateMs: 10_000,
|
||||
bootSmokeMs: 20_000,
|
||||
testMs: 30_000,
|
||||
quarantineCount: 0,
|
||||
slowestTop20: [{ file: "packages/a/src/__tests__/slow.test.ts", package: "@pkg/a", ms: 9_000 }],
|
||||
measurementFailures: [],
|
||||
},
|
||||
{
|
||||
capturedAt: "2026-06-25T12:00:00.000Z",
|
||||
gateMs: 7_000,
|
||||
bootSmokeMs: 18_000,
|
||||
testMs: 25_000,
|
||||
quarantineCount: 0,
|
||||
slowestTop20: [{ file: "packages/a/src/__tests__/slow.test.ts", package: "@pkg/a", ms: 9_000 }],
|
||||
measurementFailures: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
mkdirSync(path.join(rootDir, "docs"), { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(rootDir, "docs/test-velocity-baseline.md"),
|
||||
"| Quarantine / flake count | 3 | +3 |\n| 0-6 days | 3 |\nquarantine ledger 3 (+3)\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const exitCode = await main(["--write-report"], {
|
||||
rootDir,
|
||||
stdout: nullStream(),
|
||||
stderr: nullStream(),
|
||||
now: new Date("2026-06-26T12:00:00.000Z"),
|
||||
commandRunner: async (measurement) => {
|
||||
throw new Error(`unexpected command: ${measurement.label}`);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(exitCode, 0);
|
||||
const report = readFileSync(path.join(rootDir, "docs/test-velocity-baseline.md"), "utf8");
|
||||
assert.match(report, /\| Quarantine \/ flake count \| 0 \| 0 \|/);
|
||||
assert.match(report, /\| Deletion-due quarantines \| 0 \| n\/a \|/);
|
||||
assert.match(report, /\| 0-6 days \| 0 \|/);
|
||||
assert.match(report, /\| 7-13 days \| 0 \|/);
|
||||
assert.match(report, /\| deletion due \(>=14 days\) \| 0 \|/);
|
||||
assert.match(report, /\| unknown\/future \| 0 \|/);
|
||||
assert.match(report, /\| — \| — \| — \|/);
|
||||
assert.match(report, /\| Latest \| 2026-06-25T12:00:00\.000Z \| 7\.0s \| 18\.0s \| 25\.0s \| 0 \|/);
|
||||
assert.match(report, /\| Delta \| — \| -3\.0s \| -2\.0s \| -5\.0s \| 0 \|/);
|
||||
assert.match(report, /quarantine ledger 0 \(0\)/);
|
||||
assert.match(report, /Deletion-due quarantines: 0\./);
|
||||
assert.doesNotMatch(report, /\| Quarantine \/ flake count \| [1-9]/);
|
||||
assert.doesNotMatch(report, /\| (?:0-6 days|7-13 days|deletion due \(>=14 days\)|unknown\/future) \| [1-9]/);
|
||||
assert.doesNotMatch(report, /quarantine ledger [1-9]/);
|
||||
} finally {
|
||||
rmSync(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("honors --skip-build-preflight while still measuring lanes", async () => {
|
||||
const rootDir = tempRoot();
|
||||
const calls = [];
|
||||
|
||||
Reference in New Issue
Block a user