From 8f4897d3aa30f33e5f5ba623f789f24e8ef4ad8b Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 21 Jun 2026 22:29:13 -0700 Subject: [PATCH] FN-6905: add build preflight to velocity measurement Add a non-measured build setup step so velocity baselines do not misclassify missing dist artifacts as lane time. - Run a default `pnpm build` preflight before measured test velocity lanes, with an explicit skip flag for pre-built CI. - Add injectable command-runner coverage for preflight ordering, failure reporting, and skip behavior. - Document the preflight behavior in the generated baseline report and testing docs. Files changed: docs/test-velocity-baseline.md | 2 + docs/testing.md | 4 +- scripts/__tests__/test-velocity-baseline.test.mjs | 159 ++++++++++++++++++++++ scripts/test-velocity-baseline.mjs | 38 +++++- 4 files changed, 195 insertions(+), 8 deletions(-) Fusion-Task-Id: FN-6905 Fusion-Task-Lineage: 3016a542-4395-4d06-9f2e-1a91f4939e07 --- docs/test-velocity-baseline.md | 2 + docs/testing.md | 4 +- .../__tests__/test-velocity-baseline.test.mjs | 159 ++++++++++++++++++ scripts/test-velocity-baseline.mjs | 38 ++++- 4 files changed, 195 insertions(+), 8 deletions(-) diff --git a/docs/test-velocity-baseline.md b/docs/test-velocity-baseline.md index d93911cfcb..f4658da749 100644 --- a/docs/test-velocity-baseline.md +++ b/docs/test-velocity-baseline.md @@ -85,6 +85,8 @@ FN-6612 weekly test velocity: gate 8.3s (+2.9s), boot smoke unavailable (n/a), p pnpm test:velocity -- --measure --write-report ``` +In measure mode, the script runs a non-measured `pnpm build` preflight before timing `pnpm test:gate`, `pnpm smoke:boot`, or `pnpm test`. The preflight time is setup only and is excluded from lane metrics; if it fails, the Measurement failures section records `Build preflight (pnpm build)` as the reason. Use `--skip-build-preflight` only when the workspace is already built by CI. + Report-only regeneration is cheap and does not run any suite: ```bash diff --git a/docs/testing.md b/docs/testing.md index 30df812b3f..e5edafecf6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -297,7 +297,9 @@ FN-6612 tracks feedback-loop velocity as signal-per-second, not as a new blockin pnpm test:velocity -- --measure --write-report ``` -The script runs `pnpm test:gate`, `pnpm smoke:boot`, and `pnpm test` with bounded async process supervision, then appends the measured row to `scripts/test-velocity-history.json` and rewrites the postable artifact at `docs/test-velocity-baseline.md`. It reads the slowest 20 files from the committed `scripts/test-timings.json` snapshot and the flake/quarantine count plus 14-day deletion-clock buckets directly from `scripts/lib/test-quarantine.json`; do not run the full suite just to populate the slowest-file table. +In `--measure` mode, the script first runs a non-measured build preflight (`pnpm build`) so the built CLI and workspace dist artifacts exist before any lane is timed. The preflight duration is setup cost and is excluded from `pnpm test:gate`, `pnpm smoke:boot`, and `pnpm test` history fields; if the preflight fails, the report records `Build preflight (pnpm build)` in Measurement failures instead of fabricating lane times or letting boot smoke appear unavailable. Use `--skip-build-preflight` only in CI or another environment that has already built the workspace. + +After the preflight, the script runs `pnpm test:gate`, `pnpm smoke:boot`, and `pnpm test` with bounded async process supervision, then appends the measured row to `scripts/test-velocity-history.json` and rewrites the postable artifact at `docs/test-velocity-baseline.md`. It reads the slowest 20 files from the committed `scripts/test-timings.json` snapshot and the flake/quarantine count plus 14-day deletion-clock buckets directly from `scripts/lib/test-quarantine.json`; do not run the full suite just to populate the slowest-file table. Use cheap report-only regeneration when measurements already exist: diff --git a/scripts/__tests__/test-velocity-baseline.test.mjs b/scripts/__tests__/test-velocity-baseline.test.mjs index f7590b1672..8152f67cf4 100644 --- a/scripts/__tests__/test-velocity-baseline.test.mjs +++ b/scripts/__tests__/test-velocity-baseline.test.mjs @@ -1,12 +1,25 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { + main, + measureCommands, readQuarantineCount, renderReport, topSlowestFiles, } from "../test-velocity-baseline.mjs"; +function nullStream() { + return { write() {} }; +} + +function tempRoot() { + return mkdtempSync(path.join(tmpdir(), "fusion-test-velocity-")); +} + function makeTimings(count = 25) { const files = {}; for (let index = 0; index < count; index += 1) { @@ -72,6 +85,152 @@ describe("readQuarantineCount", () => { }); }); +describe("measureCommands", () => { + it("runs the build preflight before measured lanes and excludes setup time from lane ms", async () => { + const calls = []; + let built = false; + const result = await measureCommands({ + timeoutMs: 10_000, + cwd: "/repo", + stdout: nullStream(), + stderr: nullStream(), + commandRunner: async (measurement) => { + calls.push(measurement.command === "pnpm" ? `pnpm ${measurement.args.join(" ")}` : measurement.label); + if (measurement.args[0] === "build") { + built = true; + return { ms: 50_000, failure: null }; + } + if (measurement.args[0] === "smoke:boot") { + assert.equal(built, true, "boot smoke should only run after the build preflight creates CLI dist"); + return { ms: 406, failure: null }; + } + if (measurement.args[0] === "test") return { ms: 7_300, failure: null }; + return { ms: 12_000, failure: null }; + }, + }); + + assert.deepEqual(calls, ["pnpm build", "pnpm test:gate", "pnpm smoke:boot", "pnpm test"]); + assert.equal(result.bootSmokeMs, 406); + assert.equal(result.testMs, 7_300); + assert.deepEqual(result.measurementFailures, []); + }); + + it("records preflight failure instead of silently attributing missing build output to boot smoke", async () => { + const calls = []; + const result = await measureCommands({ + timeoutMs: 10_000, + cwd: "/repo", + stdout: nullStream(), + stderr: nullStream(), + commandRunner: async (measurement) => { + calls.push(`pnpm ${measurement.args.join(" ")}`); + return { + ms: null, + failure: { label: measurement.label, status: "exit 1 after 400ms" }, + }; + }, + }); + + assert.deepEqual(calls, ["pnpm build"]); + assert.equal(result.gateMs, undefined); + assert.equal(result.bootSmokeMs, undefined); + assert.equal(result.testMs, undefined); + assert.deepEqual(result.measurementFailures, [ + { label: "Build preflight (`pnpm build`)", status: "exit 1 after 400ms" }, + ]); + }); + + it("honors --skip-build-preflight-style opt out while still measuring lanes", async () => { + const calls = []; + const result = await measureCommands({ + timeoutMs: 10_000, + cwd: "/repo", + stdout: nullStream(), + stderr: nullStream(), + skipBuildPreflight: true, + commandRunner: async (measurement) => { + calls.push(`pnpm ${measurement.args.join(" ")}`); + return { ms: 100 + calls.length, failure: null }; + }, + }); + + assert.deepEqual(calls, ["pnpm test:gate", "pnpm smoke:boot", "pnpm test"]); + assert.equal(result.gateMs, 101); + assert.equal(result.bootSmokeMs, 102); + assert.equal(result.testMs, 103); + }); +}); + +describe("main", () => { + it("keeps report-only regeneration cheap by invoking neither preflight nor suites", async () => { + const rootDir = tempRoot(); + try { + const exitCode = await main([], { + rootDir, + stdout: nullStream(), + stderr: nullStream(), + now: new Date("2026-06-21T12:00:00.000Z"), + commandRunner: async (measurement) => { + throw new Error(`unexpected command: ${measurement.label}`); + }, + }); + + assert.equal(exitCode, 0); + const report = readFileSync(path.join(rootDir, "docs/test-velocity-baseline.md"), "utf8"); + assert.match(report, /Report-only regeneration is cheap and does not run any suite/); + } finally { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("honors --skip-build-preflight while still measuring lanes", async () => { + const rootDir = tempRoot(); + const calls = []; + try { + const exitCode = await main(["--measure", "--write-report", "--skip-build-preflight"], { + rootDir, + stdout: nullStream(), + stderr: nullStream(), + now: new Date("2026-06-21T12:00:00.000Z"), + commandRunner: async (measurement) => { + calls.push(`pnpm ${measurement.args.join(" ")}`); + return { ms: 1_000 * calls.length, failure: null }; + }, + }); + + assert.equal(exitCode, 0); + assert.deepEqual(calls, ["pnpm test:gate", "pnpm smoke:boot", "pnpm test"]); + const report = readFileSync(path.join(rootDir, "docs/test-velocity-baseline.md"), "utf8"); + assert.match(report, /Boot smoke wall-time \(`pnpm smoke:boot`\) \| 2\.0s/); + } finally { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("records preflight failure in the generated measurement failures section", async () => { + const rootDir = tempRoot(); + try { + const exitCode = await main(["--measure", "--write-report"], { + rootDir, + stdout: nullStream(), + stderr: nullStream(), + now: new Date("2026-06-21T12:00:00.000Z"), + commandRunner: async (measurement) => ({ + ms: null, + failure: { label: measurement.label, status: "exit 2 after 1.0s" }, + }), + }); + + assert.equal(exitCode, 0); + const report = readFileSync(path.join(rootDir, "docs/test-velocity-baseline.md"), "utf8"); + assert.match(report, /- Build preflight \(`pnpm build`\): exit 2 after 1\.0s/); + assert.doesNotMatch(report, /Boot smoke \(`pnpm smoke:boot`\):/); + } finally { + rmSync(rootDir, { recursive: true, force: true }); + } + }); +}); + describe("renderReport", () => { it("includes metrics, slowest rows, quarantine count, and previous-run deltas", () => { const report = renderReport({ diff --git a/scripts/test-velocity-baseline.mjs b/scripts/test-velocity-baseline.mjs index 0319a30ad9..20bb27bdfd 100755 --- a/scripts/test-velocity-baseline.mjs +++ b/scripts/test-velocity-baseline.mjs @@ -16,6 +16,13 @@ export const DEFAULT_REPORT_PATH = "docs/test-velocity-baseline.md"; export const DEFAULT_MEASURE_TIMEOUT_MS = 10 * 60 * 1000; export const DELETION_CLOCK_DAYS = 14; +const BUILD_PREFLIGHT_COMMAND = { + key: "buildPreflightMs", + label: "Build preflight (`pnpm build`)", + command: "pnpm", + args: ["build"], +}; + const MEASURE_COMMANDS = [ { key: "gateMs", label: "Merge gate (`pnpm test:gate`)", command: "pnpm", args: ["test:gate"] }, { key: "bootSmokeMs", label: "Boot smoke (`pnpm smoke:boot`)", command: "pnpm", args: ["smoke:boot"] }, @@ -172,7 +179,7 @@ export function renderReport({ gateMs, bootSmokeMs, testMs, slowest = [], quaran ? `| Previous | ${previous.capturedAt ?? "unknown"} | ${formatDuration(previous.gateMs)} | ${formatDuration(previous.bootSmokeMs)} | ${formatDuration(previous.testMs)} | ${previous.quarantineCount ?? "n/a"} |\n| Latest | ${latest.capturedAt} | ${formatDuration(latest.gateMs)} | ${formatDuration(latest.bootSmokeMs)} | ${formatDuration(latest.testMs)} | ${latest.quarantineCount} |\n| Delta | — | ${delta(latest, previous, "gateMs")} | ${delta(latest, previous, "bootSmokeMs")} | ${delta(latest, previous, "testMs")} | ${trendCell(latest.quarantineCount, previous.quarantineCount)} |` : `| Previous | _(seed baseline)_ | — | — | — | — |\n| Latest | ${latest.capturedAt} | ${formatDuration(latest.gateMs)} | ${formatDuration(latest.bootSmokeMs)} | ${formatDuration(latest.testMs)} | ${latest.quarantineCount} |\n| Delta | — | n/a | n/a | n/a | n/a |`; - return `# Test velocity baseline\n\n> Weekly FN-6612 signal-per-second baseline. Measure and report feedback-loop velocity; do **not** add slow tests or wire this report into blocking PR checks. The merge gate remains the existing thin Lint, Typecheck, Build, and Gate path.\n\n## Latest baseline\n\n- Cycle: **${cycle}**\n- Captured at: **${latest.capturedAt}**\n- Timing snapshot: \`${DEFAULT_TIMINGS_PATH}\`${timingSnapshotCapturedAt ? ` captured at **${timingSnapshotCapturedAt}**` : ""}\n- Quarantine ledger: \`${DEFAULT_QUARANTINE_PATH}\`\n\n## Metrics\n\n| Metric | Current | Delta vs previous |\n|---|---:|---:|\n${renderMetricRow("Merge gate wall-time (`pnpm test:gate`)", latest, previous, "gateMs")}\n${renderMetricRow("Boot smoke wall-time (`pnpm smoke:boot`)", latest, previous, "bootSmokeMs")}\n${renderMetricRow("Changed-only test wall-time (`pnpm test`)", latest, previous, "testMs")}\n| Quarantine / flake count | ${latest.quarantineCount} | ${trendCell(latest.quarantineCount, previous?.quarantineCount)} |\n| Deletion-due quarantines | ${quarantine?.deletionDueCount ?? 0} | n/a |\n\n## Measurement failures\n\n${failures}\n\n## Slowest 20 test files\n\n| Rank | File | Package | Duration |\n|---:|---|---|---:|\n${slowRows || "| — | — | — | — |"}\n\n## Quarantine age buckets\n\n| Age bucket | Count |\n|---|---:|\n| 0-6 days | ${quarantine?.byAgeBucket?.["0-6d"] ?? 0} |\n| 7-13 days | ${quarantine?.byAgeBucket?.["7-13d"] ?? 0} |\n| deletion due (>=14 days) | ${quarantine?.byAgeBucket?.deletionDue ?? 0} |\n| unknown/future | ${quarantine?.byAgeBucket?.unknown ?? 0} |\n\n### Deletion-due entries\n\n| File | Quarantined at | Age (days) |\n|---|---:|---:|\n${dueRows || "| — | — | — |"}\n\n## Before / after trend\n\n| Row | Captured at | Gate | Boot smoke | \`pnpm test\` | Quarantine count |\n|---|---|---:|---:|---:|---:|\n${previousRows}\n\n_Future weekly rows append to \`${DEFAULT_HISTORY_PATH}\`; compare the latest row against the previous row before posting to #leads._\n\n## Post to #leads\n\n\`\`\`text\nFN-6612 weekly test velocity: gate ${formatDuration(latest.gateMs)} (${delta(latest, previous, "gateMs")}), boot smoke ${formatDuration(latest.bootSmokeMs)} (${delta(latest, previous, "bootSmokeMs")}), pnpm test ${formatDuration(latest.testMs)} (${delta(latest, previous, "testMs")}), quarantine ledger ${latest.quarantineCount} (${trendCell(latest.quarantineCount, previous?.quarantineCount)}). Slowest file: ${slowest[0]?.file ?? "none"} at ${formatDuration(slowest[0]?.ms)}. Deletion-due quarantines: ${quarantine?.deletionDueCount ?? 0}.\n\`\`\`\n\n## How to refresh\n\n\`\`\`bash\npnpm test:velocity -- --measure --write-report\n\`\`\`\n\nReport-only regeneration is cheap and does not run any suite:\n\n\`\`\`bash\npnpm test:velocity\n\`\`\`\n`; + return `# Test velocity baseline\n\n> Weekly FN-6612 signal-per-second baseline. Measure and report feedback-loop velocity; do **not** add slow tests or wire this report into blocking PR checks. The merge gate remains the existing thin Lint, Typecheck, Build, and Gate path.\n\n## Latest baseline\n\n- Cycle: **${cycle}**\n- Captured at: **${latest.capturedAt}**\n- Timing snapshot: \`${DEFAULT_TIMINGS_PATH}\`${timingSnapshotCapturedAt ? ` captured at **${timingSnapshotCapturedAt}**` : ""}\n- Quarantine ledger: \`${DEFAULT_QUARANTINE_PATH}\`\n\n## Metrics\n\n| Metric | Current | Delta vs previous |\n|---|---:|---:|\n${renderMetricRow("Merge gate wall-time (`pnpm test:gate`)", latest, previous, "gateMs")}\n${renderMetricRow("Boot smoke wall-time (`pnpm smoke:boot`)", latest, previous, "bootSmokeMs")}\n${renderMetricRow("Changed-only test wall-time (`pnpm test`)", latest, previous, "testMs")}\n| Quarantine / flake count | ${latest.quarantineCount} | ${trendCell(latest.quarantineCount, previous?.quarantineCount)} |\n| Deletion-due quarantines | ${quarantine?.deletionDueCount ?? 0} | n/a |\n\n## Measurement failures\n\n${failures}\n\n## Slowest 20 test files\n\n| Rank | File | Package | Duration |\n|---:|---|---|---:|\n${slowRows || "| — | — | — | — |"}\n\n## Quarantine age buckets\n\n| Age bucket | Count |\n|---|---:|\n| 0-6 days | ${quarantine?.byAgeBucket?.["0-6d"] ?? 0} |\n| 7-13 days | ${quarantine?.byAgeBucket?.["7-13d"] ?? 0} |\n| deletion due (>=14 days) | ${quarantine?.byAgeBucket?.deletionDue ?? 0} |\n| unknown/future | ${quarantine?.byAgeBucket?.unknown ?? 0} |\n\n### Deletion-due entries\n\n| File | Quarantined at | Age (days) |\n|---|---:|---:|\n${dueRows || "| — | — | — |"}\n\n## Before / after trend\n\n| Row | Captured at | Gate | Boot smoke | \`pnpm test\` | Quarantine count |\n|---|---|---:|---:|---:|---:|\n${previousRows}\n\n_Future weekly rows append to \`${DEFAULT_HISTORY_PATH}\`; compare the latest row against the previous row before posting to #leads._\n\n## Post to #leads\n\n\`\`\`text\nFN-6612 weekly test velocity: gate ${formatDuration(latest.gateMs)} (${delta(latest, previous, "gateMs")}), boot smoke ${formatDuration(latest.bootSmokeMs)} (${delta(latest, previous, "bootSmokeMs")}), pnpm test ${formatDuration(latest.testMs)} (${delta(latest, previous, "testMs")}), quarantine ledger ${latest.quarantineCount} (${trendCell(latest.quarantineCount, previous?.quarantineCount)}). Slowest file: ${slowest[0]?.file ?? "none"} at ${formatDuration(slowest[0]?.ms)}. Deletion-due quarantines: ${quarantine?.deletionDueCount ?? 0}.\n\`\`\`\n\n## How to refresh\n\n\`\`\`bash\npnpm test:velocity -- --measure --write-report\n\`\`\`\n\nIn measure mode, the script runs a non-measured \`pnpm build\` preflight before timing \`pnpm test:gate\`, \`pnpm smoke:boot\`, or \`pnpm test\`. The preflight time is setup only and is excluded from lane metrics; if it fails, the Measurement failures section records \`Build preflight (pnpm build)\` as the reason. Use \`--skip-build-preflight\` only when the workspace is already built by CI.\n\nReport-only regeneration is cheap and does not run any suite:\n\n\`\`\`bash\npnpm test:velocity\n\`\`\`\n`; } function historyEntries(history) { @@ -195,13 +202,14 @@ function createEntry({ capturedAt = new Date().toISOString(), gateMs = null, boo } function parseArgs(argv) { - const args = { measure: false, writeReport: false, reportOnly: true, timeoutMs: DEFAULT_MEASURE_TIMEOUT_MS, help: false }; + const args = { measure: false, writeReport: false, reportOnly: true, timeoutMs: DEFAULT_MEASURE_TIMEOUT_MS, help: false, skipBuildPreflight: false }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--") continue; else if (arg === "--measure") args.measure = true; else if (arg === "--write-report") args.writeReport = true; else if (arg === "--report-only") args.reportOnly = true; + else if (arg === "--skip-build-preflight" || arg === "--no-build-preflight") args.skipBuildPreflight = true; else if (arg === "--timeout-ms") args.timeoutMs = Number(argv[++index]); else if (arg === "--help" || arg === "-h") args.help = true; else throw new Error(`Unknown argument: ${arg}`); @@ -248,11 +256,27 @@ async function timeCommand({ command, args, label, timeoutMs, cwd, stdout, stder }); } -async function measureCommands({ timeoutMs, cwd, stdout, stderr }) { +/* +FNXC:TestVelocityBaseline 2026-06-21-00:00: +FN-6905 needs seam-based orchestration tests for command ordering and lane timing without running real builds or suites. Keep production behavior on the default `timeCommand` path while allowing tests to inject a deterministic command runner. + +FNXC:TestVelocityBaseline 2026-06-21-00:07: +Clean-worktree velocity measurement must not let missing CLI dist make boot smoke look unavailable or push setup cost into `pnpm test`. Run `pnpm build` as non-measured setup before timed lanes, allow explicit opt-out for pre-built CI, and record preflight failure as the real measurement failure instead of fabricating lane timings. +*/ +export async function measureCommands({ timeoutMs, cwd, stdout, stderr, commandRunner = timeCommand, skipBuildPreflight = false }) { const results = {}; const failures = []; + + if (!skipBuildPreflight) { + const preflight = await commandRunner({ ...BUILD_PREFLIGHT_COMMAND, timeoutMs, cwd, stdout, stderr }); + if (preflight.failure) { + failures.push(preflight.failure); + return { ...results, measurementFailures: failures }; + } + } + for (const measurement of MEASURE_COMMANDS) { - const result = await timeCommand({ ...measurement, timeoutMs, cwd, stdout, stderr }); + const result = await commandRunner({ ...measurement, timeoutMs, cwd, stdout, stderr }); results[measurement.key] = result.ms; if (result.failure) failures.push(result.failure); } @@ -273,7 +297,7 @@ function renderFromEntry(entry, previous, quarantine) { }); } -export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr, now = new Date() } = {}) { +export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr, now = new Date(), commandRunner = timeCommand } = {}) { let args; try { args = parseArgs(argv); @@ -283,7 +307,7 @@ export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, s } if (args.help) { - stdout.write("Usage: node scripts/test-velocity-baseline.mjs [--measure] [--write-report] [--report-only] [--timeout-ms ]\n"); + stdout.write("Usage: node scripts/test-velocity-baseline.mjs [--measure] [--write-report] [--report-only] [--skip-build-preflight] [--timeout-ms ]\n"); return 0; } @@ -295,7 +319,7 @@ export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, s const slowest = topSlowestFiles(timings, 20); if (args.measure) { - const measured = await measureCommands({ timeoutMs: args.timeoutMs, cwd: rootDir, stdout, stderr }); + const measured = await measureCommands({ timeoutMs: args.timeoutMs, cwd: rootDir, stdout, stderr, commandRunner, skipBuildPreflight: args.skipBuildPreflight }); const entry = createEntry({ capturedAt: now.toISOString(), gateMs: measured.gateMs,