fix(ci): honest shard weights + review autofixes
- laneShardFraction: chained --shard invocations sum to full project weight (api backfill was half-weighted) - exclude *.slow.test.* from duration weighting (engine carried 75 phantom untimed files) - remove dead ENGINE_PACKAGE_NAME export - docs: --cold-start-probe, --inputs-dir (snapshot refresh now self-contained), --print-mode
This commit is contained in:
@@ -120,6 +120,9 @@ commensurably. Untimed packages are named in a logged warning.
|
||||
package duration is apportioned evenly across lanes (logged as `even-apportionment`).
|
||||
- **Inspect the plan without running it:** `node scripts/ci-test-shard.mjs --dry-run --total 4`
|
||||
(optionally `--shard N`) prints the planned `pnpm` commands and per-shard weight totals.
|
||||
- **Measure per-process startup cost:** `node scripts/ci-test-shard.mjs --cold-start-probe <package-name>`
|
||||
runs the package's cheapest test file in isolation and reports `wall − test time` overhead
|
||||
(the signal behind the deferred vitest-4 upgrade gate).
|
||||
|
||||
### Snapshot staleness policy
|
||||
|
||||
@@ -127,7 +130,9 @@ The snapshot carries `capturedAt`. If it is older than **30 days**, the planner
|
||||
prominent warning and proceeds (balance degrades gracefully toward the file-count status
|
||||
quo, never below it) — it does **not** fail the build. Refresh is **manual/scheduled from
|
||||
the default branch only**: each CI shard uploads per-shard JSON timing artifacts (U1), and
|
||||
`node scripts/ci-test-shard.mjs --write-timings` merges them into the snapshot. A future
|
||||
`node scripts/ci-test-shard.mjs --write-timings` merges them into the snapshot. Download the
|
||||
shard artifacts into `.timings/` first (the default lookup directory), or pass
|
||||
`--inputs-dir <path>` to point at wherever they were downloaded. A future
|
||||
scheduled job can gate on freshness via `node scripts/ci-test-shard.mjs --check-timings-staleness`,
|
||||
which exits non-zero when the snapshot is missing or older than the 30-day budget.
|
||||
|
||||
@@ -154,6 +159,10 @@ packages affected by your branch diff (plus their reverse-dependents) and skips
|
||||
packages whose content hasn't changed since they last passed. A per-package
|
||||
pass-cache lives at `node_modules/.cache/fusion/test-cache.json`.
|
||||
|
||||
To see which mode a run would pick — and why — without running any tests:
|
||||
`node scripts/test-changed.mjs --print-mode` prints the
|
||||
`[test-changed] mode=… reason=… packages=…` decision line and exits.
|
||||
|
||||
### What a cache entry's hash covers (dependency-aware invalidation)
|
||||
|
||||
Each package's cache hash (`computePackageHash`) folds in, so any of these
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
countPackageTestFiles,
|
||||
loadPlanningTimings,
|
||||
computePackageDurationWeight,
|
||||
laneShardFraction,
|
||||
sumFileDurations,
|
||||
enumerateDashboardLanes,
|
||||
laneProjectNames,
|
||||
@@ -617,3 +618,39 @@ test("U6: --dry-run prints planned commands and per-shard weight for all 4 shard
|
||||
// Dashboard must NOT be virtual-sliced.
|
||||
assert.doesNotMatch(result.stdout, /--filter @fusion\/dashboard test --shard/);
|
||||
});
|
||||
|
||||
test("U6 fix: laneShardFraction sums chained --shard invocations, capped at 1", () => {
|
||||
// Single half-shard lane (app backfill style): genuinely runs 1/4.
|
||||
assert.equal(laneShardFraction("vitest run --project p --shard=1/4"), 0.25);
|
||||
// Chained halves in one lane (api backfill style): runs the FULL project.
|
||||
assert.equal(
|
||||
laneShardFraction("run-heap --shard=1/2 && run-heap --shard=2/2"),
|
||||
1,
|
||||
);
|
||||
// No shard flag: whole project.
|
||||
assert.equal(laneShardFraction("vitest run --project p"), 1);
|
||||
// Over-complete chains clamp at 1.
|
||||
assert.equal(
|
||||
laneShardFraction("a --shard=1/2 && b --shard=2/2 && c --shard=1/2"),
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test("U6 fix: computePackageDurationWeight excludes slow-tier files from weighting", (t) => {
|
||||
const projectRoot = mkdtempSync(path.join(tmpdir(), "u6-slow-excl-"));
|
||||
t.after(() => rmSync(projectRoot, { recursive: true, force: true }));
|
||||
mkdirSync(path.join(projectRoot, "packages/eng/src/__tests__"), { recursive: true });
|
||||
writeFileSync(path.join(projectRoot, "packages/eng/src/__tests__/fast.test.ts"), "");
|
||||
writeFileSync(path.join(projectRoot, "packages/eng/src/__tests__/heavy.slow.test.ts"), "");
|
||||
const snapshotPath = writeSnapshot(projectRoot, new Date().toISOString(), {
|
||||
"@x/eng": { files: { "packages/eng/src/__tests__/fast.test.ts": 400 } },
|
||||
});
|
||||
const timings = loadPlanningTimings({ snapshotPath });
|
||||
const weighted = computePackageDurationWeight({ name: "@x/eng", dir: "packages/eng" }, timings, {
|
||||
projectRoot,
|
||||
});
|
||||
// Only the fast file counts: 400ms timed, zero untimed fallback for the
|
||||
// slow file (which the package `test` script never runs).
|
||||
assert.equal(weighted.weight, 400);
|
||||
assert.equal(weighted.partiallyUntimed, false);
|
||||
});
|
||||
|
||||
@@ -413,9 +413,6 @@ export const TIMINGS_STALENESS_DAYS = 30;
|
||||
/** Dashboard package name; its `test` chain is distributed lane-by-lane. */
|
||||
export const DASHBOARD_PACKAGE_NAME = "@fusion/dashboard";
|
||||
|
||||
/** Engine package name; kept on `vitest --shard` virtual slicing, by duration. */
|
||||
export const ENGINE_PACKAGE_NAME = "@fusion/engine";
|
||||
|
||||
/**
|
||||
* Load the committed timing snapshot into a flat per-file duration map plus a
|
||||
* derived median per-file duration (used to scale the file-count fallback so
|
||||
@@ -525,7 +522,14 @@ export function sumFileDurations(files, fileDurations) {
|
||||
*/
|
||||
export function computePackageDurationWeight(pkg, timings, options = {}) {
|
||||
const projectRoot = options.projectRoot ?? process.cwd();
|
||||
const files = listPackageTestFiles(pkg.dir, { projectRoot }).map((f) => `${pkg.dir}/${f}`);
|
||||
// Exclude `*.slow.test.*` from weighting: the slow tier never runs in a
|
||||
// package's `test` script (it has its own CI gate), so counting those files
|
||||
// — always untimed, hence median-fallback-weighted — inflates the package's
|
||||
// shard weight with work the shard does not execute.
|
||||
const files = listPackageTestFiles(pkg.dir, {
|
||||
projectRoot,
|
||||
extraExclude: (p) => /\.slow\.test\./.test(p),
|
||||
}).map((f) => `${pkg.dir}/${f}`);
|
||||
|
||||
const fallbackPerFile = timings.medianPerFileMs > 0 ? timings.medianPerFileMs : DURATION_BUCKET_MS;
|
||||
const { durationMs, timedCount, untimedCount } = sumFileDurations(files, timings.fileDurations);
|
||||
@@ -660,6 +664,26 @@ export function resolveDashboardProjectFiles(dashboardDir, options = {}) {
|
||||
* @param {{ projectRoot?: string }} [options]
|
||||
* @returns {{ units: Array<{ name: string, lane: string, runKind: "dashboard-lane", weight: number, splittable: false }>, lanes: string[], method: string, untimed: string[] }}
|
||||
*/
|
||||
/**
|
||||
* Fraction of a project a lane command actually runs, derived from its
|
||||
* `--shard=i/n` flags. A lane may chain MULTIPLE --shard invocations of the
|
||||
* same project (e.g. `--shard=1/2 && --shard=2/2` runs the full project):
|
||||
* sum the fractions across all matches, capped at 1. A lane with a single
|
||||
* `--shard=i/n` invocation genuinely runs 1/n. No --shard flag means the
|
||||
* whole project.
|
||||
*
|
||||
* @param {string} command
|
||||
* @returns {number} (0, 1]
|
||||
*/
|
||||
export function laneShardFraction(command) {
|
||||
const shardMatches = [...command.matchAll(/--shard[=\s]+(\d+)\/(\d+)/g)];
|
||||
if (shardMatches.length === 0) return 1;
|
||||
return Math.min(
|
||||
1,
|
||||
shardMatches.reduce((sum, m) => sum + 1 / Number(m[2]), 0),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildDashboardLaneUnits(pkg, timings, options = {}) {
|
||||
const projectRoot = options.projectRoot ?? process.cwd();
|
||||
const pkgJson = JSON.parse(readFileSync(path.join(projectRoot, pkg.dir, "package.json"), "utf8"));
|
||||
@@ -674,8 +698,7 @@ export function buildDashboardLaneUnits(pkg, timings, options = {}) {
|
||||
const units = lanes.map((lane) => {
|
||||
const command = scripts[lane] ?? "";
|
||||
const projects = laneProjectNames(command);
|
||||
const shardMatch = /--shard[=\s]+(\d+)\/(\d+)/.exec(command);
|
||||
const shardFraction = shardMatch ? 1 / Number(shardMatch[2]) : 1;
|
||||
const shardFraction = laneShardFraction(command);
|
||||
const files = new Set();
|
||||
for (const project of projects) for (const f of projectFiles[project] ?? []) files.add(f);
|
||||
const { durationMs, timedCount, untimedCount } = sumFileDurations([...files], timings.fileDurations);
|
||||
|
||||
Reference in New Issue
Block a user