f150a176b4
Add a weekly baseline workflow for tracking test feedback-loop speed and flake pressure. - Add a stdlib-only script to record gate and pnpm test timings alongside slowest test files and quarantine counts. - Publish generated markdown and JSON baseline artifacts for #leads reporting. - Document the weekly refresh process and add package scripts plus coverage for the baseline helper. Files changed: docs/test-feedback-loop-baseline.md | 48 ++++++ docs/test-feedback-loop-baselines.json | 122 ++++++++++++++ docs/testing.md | 12 ++ package.json | 1 + scripts/__tests__/test-feedback-baseline.test.mjs | 90 ++++++++++ scripts/test-feedback-baseline.mjs | 192 ++++++++++++++++++++++ 6 files changed, 465 insertions(+) Fusion-Task-Id: FN-6612 Fusion-Task-Lineage: 72bdc070-50e3-4e6b-a07d-3b8aeb9c2e92
91 lines
3.5 KiB
JavaScript
91 lines
3.5 KiB
JavaScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { mkdirSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import {
|
|
collectFlakeSummary,
|
|
collectSlowestFiles,
|
|
createBaseline,
|
|
DEFAULT_BASELINES_PATH,
|
|
DEFAULT_MARKDOWN_PATH,
|
|
DEFAULT_QUARANTINE_PATH,
|
|
DEFAULT_TIMINGS_PATH,
|
|
main,
|
|
renderMarkdown,
|
|
} from "../test-feedback-baseline.mjs";
|
|
|
|
function writeJson(root, relativePath, value) {
|
|
const absolutePath = path.join(root, relativePath);
|
|
mkdirSync(path.dirname(absolutePath), { recursive: true });
|
|
writeFileSync(absolutePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
test("collectSlowestFiles ranks timing snapshot entries across packages", () => {
|
|
const rows = collectSlowestFiles({
|
|
packages: {
|
|
"@fusion/a": { files: { "a-fast.test.ts": 20, "a-slow.test.ts": 2000 } },
|
|
"@fusion/b": { files: { "b-medium.test.ts": 1000 } },
|
|
},
|
|
}, 2);
|
|
|
|
assert.deepEqual(rows, [
|
|
{ packageName: "@fusion/a", file: "a-slow.test.ts", durationMs: 2000 },
|
|
{ packageName: "@fusion/b", file: "b-medium.test.ts", durationMs: 1000 },
|
|
]);
|
|
});
|
|
|
|
test("collectFlakeSummary counts ledger entries and unique quarantined files", () => {
|
|
const summary = collectFlakeSummary({ entries: [
|
|
{ file: "one.test.ts" },
|
|
{ file: "one.test.ts" },
|
|
{ file: "two.test.ts" },
|
|
] });
|
|
|
|
assert.equal(summary.flakeCount, 3);
|
|
assert.equal(summary.uniqueQuarantinedFileCount, 2);
|
|
assert.deepEqual(summary.quarantinedFiles, ["one.test.ts", "two.test.ts"]);
|
|
});
|
|
|
|
test("renderMarkdown includes #leads summary, trend, and slowest files", () => {
|
|
const baseline = createBaseline({
|
|
now: new Date("2026-06-17T18:00:00.000Z"),
|
|
gateWallTimeMs: 12_300,
|
|
pnpmTestWallTimeMs: 45_600,
|
|
timings: { capturedAt: "2026-06-17T17:00:00.000Z", packages: { "@fusion/core": { files: { "packages/core/src/__tests__/agent-store.test.ts": 11_600 } } } },
|
|
quarantine: { entries: [{ file: "packages/core/src/__tests__/flake.test.ts" }] },
|
|
});
|
|
|
|
const markdown = renderMarkdown([baseline]);
|
|
|
|
assert.match(markdown, /Latest #leads summary/);
|
|
assert.match(markdown, /Gate suite wall-time: \*\*12\.3s\*\*/);
|
|
assert.match(markdown, /packages\/core\/src\/__tests__\/agent-store\.test\.ts/);
|
|
assert.match(markdown, /Quarantined tests remain on the 14-day rescue-or-delete clock/);
|
|
});
|
|
|
|
test("main records a baseline and writes the markdown publication artifact", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "fusion-test-feedback-baseline-"));
|
|
writeJson(root, DEFAULT_TIMINGS_PATH, {
|
|
capturedAt: "2026-06-17T17:00:00.000Z",
|
|
packages: { "@fusion/core": { files: { "slow.test.ts": 1500, "fast.test.ts": 100 } } },
|
|
});
|
|
writeJson(root, DEFAULT_QUARANTINE_PATH, { entries: [{ file: "slow.test.ts" }] });
|
|
|
|
const chunks = [];
|
|
const code = await main(["--record", "--gate-ms", "1000", "--test-ms", "2000", "--print-leads"], {
|
|
rootDir: root,
|
|
stdout: { write: (chunk) => chunks.push(String(chunk)) },
|
|
stderr: { write: () => {} },
|
|
});
|
|
|
|
assert.equal(code, 0);
|
|
assert.match(chunks.join(""), /gate 1\.0s, pnpm test 2\.0s/);
|
|
const store = JSON.parse(readFileSync(path.join(root, DEFAULT_BASELINES_PATH), "utf8"));
|
|
assert.equal(store.baselines.length, 1);
|
|
assert.equal(store.baselines[0].flakeCount, 1);
|
|
const markdown = readFileSync(path.join(root, DEFAULT_MARKDOWN_PATH), "utf8");
|
|
assert.match(markdown, /slow\.test\.ts/);
|
|
});
|