From 5a72ac94753a80b7c49b94c09fb75ac2982daadc Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 3 Jun 2026 19:25:40 -0700 Subject: [PATCH] fix(ci): honest shard weights + review autofixes - laneShardFraction: chained --shard invocations sum to full project weight (api backfill was half-weighted) - exclude *.slow.test.* from duration weighting (engine carried 75 phantom untimed files) - remove dead ENGINE_PACKAGE_NAME export - docs: --cold-start-probe, --inputs-dir (snapshot refresh now self-contained), --print-mode --- docs/testing.md | 11 ++++++- scripts/__tests__/ci-test-shard.test.mjs | 37 ++++++++++++++++++++++++ scripts/ci-test-shard.mjs | 35 ++++++++++++++++++---- 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 1b425c6f2d..3817e9a446 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -120,6 +120,9 @@ commensurably. Untimed packages are named in a logged warning. package duration is apportioned evenly across lanes (logged as `even-apportionment`). - **Inspect the plan without running it:** `node scripts/ci-test-shard.mjs --dry-run --total 4` (optionally `--shard N`) prints the planned `pnpm` commands and per-shard weight totals. +- **Measure per-process startup cost:** `node scripts/ci-test-shard.mjs --cold-start-probe ` + runs the package's cheapest test file in isolation and reports `wall − test time` overhead + (the signal behind the deferred vitest-4 upgrade gate). ### Snapshot staleness policy @@ -127,7 +130,9 @@ The snapshot carries `capturedAt`. If it is older than **30 days**, the planner prominent warning and proceeds (balance degrades gracefully toward the file-count status quo, never below it) — it does **not** fail the build. Refresh is **manual/scheduled from the default branch only**: each CI shard uploads per-shard JSON timing artifacts (U1), and -`node scripts/ci-test-shard.mjs --write-timings` merges them into the snapshot. A future +`node scripts/ci-test-shard.mjs --write-timings` merges them into the snapshot. Download the +shard artifacts into `.timings/` first (the default lookup directory), or pass +`--inputs-dir ` to point at wherever they were downloaded. A future scheduled job can gate on freshness via `node scripts/ci-test-shard.mjs --check-timings-staleness`, which exits non-zero when the snapshot is missing or older than the 30-day budget. @@ -154,6 +159,10 @@ packages affected by your branch diff (plus their reverse-dependents) and skips packages whose content hasn't changed since they last passed. A per-package pass-cache lives at `node_modules/.cache/fusion/test-cache.json`. +To see which mode a run would pick — and why — without running any tests: +`node scripts/test-changed.mjs --print-mode` prints the +`[test-changed] mode=… reason=… packages=…` decision line and exits. + ### What a cache entry's hash covers (dependency-aware invalidation) Each package's cache hash (`computePackageHash`) folds in, so any of these diff --git a/scripts/__tests__/ci-test-shard.test.mjs b/scripts/__tests__/ci-test-shard.test.mjs index 1602f9aa1d..734cb6b919 100644 --- a/scripts/__tests__/ci-test-shard.test.mjs +++ b/scripts/__tests__/ci-test-shard.test.mjs @@ -29,6 +29,7 @@ import { countPackageTestFiles, loadPlanningTimings, computePackageDurationWeight, + laneShardFraction, sumFileDurations, enumerateDashboardLanes, laneProjectNames, @@ -617,3 +618,39 @@ test("U6: --dry-run prints planned commands and per-shard weight for all 4 shard // Dashboard must NOT be virtual-sliced. assert.doesNotMatch(result.stdout, /--filter @fusion\/dashboard test --shard/); }); + +test("U6 fix: laneShardFraction sums chained --shard invocations, capped at 1", () => { + // Single half-shard lane (app backfill style): genuinely runs 1/4. + assert.equal(laneShardFraction("vitest run --project p --shard=1/4"), 0.25); + // Chained halves in one lane (api backfill style): runs the FULL project. + assert.equal( + laneShardFraction("run-heap --shard=1/2 && run-heap --shard=2/2"), + 1, + ); + // No shard flag: whole project. + assert.equal(laneShardFraction("vitest run --project p"), 1); + // Over-complete chains clamp at 1. + assert.equal( + laneShardFraction("a --shard=1/2 && b --shard=2/2 && c --shard=1/2"), + 1, + ); +}); + +test("U6 fix: computePackageDurationWeight excludes slow-tier files from weighting", (t) => { + const projectRoot = mkdtempSync(path.join(tmpdir(), "u6-slow-excl-")); + t.after(() => rmSync(projectRoot, { recursive: true, force: true })); + mkdirSync(path.join(projectRoot, "packages/eng/src/__tests__"), { recursive: true }); + writeFileSync(path.join(projectRoot, "packages/eng/src/__tests__/fast.test.ts"), ""); + writeFileSync(path.join(projectRoot, "packages/eng/src/__tests__/heavy.slow.test.ts"), ""); + const snapshotPath = writeSnapshot(projectRoot, new Date().toISOString(), { + "@x/eng": { files: { "packages/eng/src/__tests__/fast.test.ts": 400 } }, + }); + const timings = loadPlanningTimings({ snapshotPath }); + const weighted = computePackageDurationWeight({ name: "@x/eng", dir: "packages/eng" }, timings, { + projectRoot, + }); + // Only the fast file counts: 400ms timed, zero untimed fallback for the + // slow file (which the package `test` script never runs). + assert.equal(weighted.weight, 400); + assert.equal(weighted.partiallyUntimed, false); +}); diff --git a/scripts/ci-test-shard.mjs b/scripts/ci-test-shard.mjs index 94660a4b1e..729604fd6b 100644 --- a/scripts/ci-test-shard.mjs +++ b/scripts/ci-test-shard.mjs @@ -413,9 +413,6 @@ export const TIMINGS_STALENESS_DAYS = 30; /** Dashboard package name; its `test` chain is distributed lane-by-lane. */ export const DASHBOARD_PACKAGE_NAME = "@fusion/dashboard"; -/** Engine package name; kept on `vitest --shard` virtual slicing, by duration. */ -export const ENGINE_PACKAGE_NAME = "@fusion/engine"; - /** * Load the committed timing snapshot into a flat per-file duration map plus a * derived median per-file duration (used to scale the file-count fallback so @@ -525,7 +522,14 @@ export function sumFileDurations(files, fileDurations) { */ export function computePackageDurationWeight(pkg, timings, options = {}) { const projectRoot = options.projectRoot ?? process.cwd(); - const files = listPackageTestFiles(pkg.dir, { projectRoot }).map((f) => `${pkg.dir}/${f}`); + // Exclude `*.slow.test.*` from weighting: the slow tier never runs in a + // package's `test` script (it has its own CI gate), so counting those files + // — always untimed, hence median-fallback-weighted — inflates the package's + // shard weight with work the shard does not execute. + const files = listPackageTestFiles(pkg.dir, { + projectRoot, + extraExclude: (p) => /\.slow\.test\./.test(p), + }).map((f) => `${pkg.dir}/${f}`); const fallbackPerFile = timings.medianPerFileMs > 0 ? timings.medianPerFileMs : DURATION_BUCKET_MS; const { durationMs, timedCount, untimedCount } = sumFileDurations(files, timings.fileDurations); @@ -660,6 +664,26 @@ export function resolveDashboardProjectFiles(dashboardDir, options = {}) { * @param {{ projectRoot?: string }} [options] * @returns {{ units: Array<{ name: string, lane: string, runKind: "dashboard-lane", weight: number, splittable: false }>, lanes: string[], method: string, untimed: string[] }} */ +/** + * Fraction of a project a lane command actually runs, derived from its + * `--shard=i/n` flags. A lane may chain MULTIPLE --shard invocations of the + * same project (e.g. `--shard=1/2 && --shard=2/2` runs the full project): + * sum the fractions across all matches, capped at 1. A lane with a single + * `--shard=i/n` invocation genuinely runs 1/n. No --shard flag means the + * whole project. + * + * @param {string} command + * @returns {number} (0, 1] + */ +export function laneShardFraction(command) { + const shardMatches = [...command.matchAll(/--shard[=\s]+(\d+)\/(\d+)/g)]; + if (shardMatches.length === 0) return 1; + return Math.min( + 1, + shardMatches.reduce((sum, m) => sum + 1 / Number(m[2]), 0), + ); +} + export function buildDashboardLaneUnits(pkg, timings, options = {}) { const projectRoot = options.projectRoot ?? process.cwd(); const pkgJson = JSON.parse(readFileSync(path.join(projectRoot, pkg.dir, "package.json"), "utf8")); @@ -674,8 +698,7 @@ export function buildDashboardLaneUnits(pkg, timings, options = {}) { const units = lanes.map((lane) => { const command = scripts[lane] ?? ""; const projects = laneProjectNames(command); - const shardMatch = /--shard[=\s]+(\d+)\/(\d+)/.exec(command); - const shardFraction = shardMatch ? 1 / Number(shardMatch[2]) : 1; + const shardFraction = laneShardFraction(command); const files = new Set(); for (const project of projects) for (const f of projectFiles[project] ?? []) files.add(f); const { durationMs, timedCount, untimedCount } = sumFileDurations([...files], timings.fileDurations);