diff --git a/docs/testing.md b/docs/testing.md index 2f602f8029..1b425c6f2d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -97,6 +97,40 @@ in CI via the `Engine slow tier` job in `pr-checks.yml`, which uses or config drift that silently empties the tier breaks CI instead of passing vacuously). The CI job uses `fetch-depth: 0` because these tests run real git operations. +## CI shard balancing (duration-weighted) + +`scripts/ci-test-shard.mjs` packs the 4 CI shards (`pnpm test:ci:shard --shard N --total 4`, +called from `pr-checks.yml`) by **measured duration**, not test-file count, using the +committed `scripts/test-timings.json` snapshot (U1/R4). A package's weight is the sum of +its files' recorded durations; files (or whole packages) absent from the snapshot fall +back to the snapshot's **median per-file duration** so untimed packages weigh +commensurably. Untimed packages are named in a logged warning. + +- **Engine** keeps `vitest --shard X/Y` virtual slicing (its `test` is a single vitest + invocation: `--project=engine-default --project=engine-reliability`); slices are now + weighted by duration. +- **Dashboard** is *not* `--shard`-sliced — its `test` script is a chain of many separate + vitest invocations, so a forwarded `--shard` cannot apply coherently. Instead each leaf + lane in the chain (enumerated programmatically from `packages/dashboard/package.json` by + expanding the `pnpm run ` graph under `test`) is a separately-weighted schedulable + unit; a shard runs `pnpm --filter @fusion/dashboard run ` for its assigned lanes. + Every lane is assigned to exactly one shard. **Lane weight** is the sum of durations of + the files the lane's `--project`s execute, derived from the vitest config project + `include`/`exclude` globs (imported via `tsx`); if the config cannot be imported the + package duration is apportioned evenly across lanes (logged as `even-apportionment`). +- **Inspect the plan without running it:** `node scripts/ci-test-shard.mjs --dry-run --total 4` + (optionally `--shard N`) prints the planned `pnpm` commands and per-shard weight totals. + +### Snapshot staleness policy + +The snapshot carries `capturedAt`. If it is older than **30 days**, the planner prints a +prominent warning and proceeds (balance degrades gracefully toward the file-count status +quo, never below it) — it does **not** fail the build. Refresh is **manual/scheduled from +the default branch only**: each CI shard uploads per-shard JSON timing artifacts (U1), and +`node scripts/ci-test-shard.mjs --write-timings` merges them into the snapshot. A future +scheduled job can gate on freshness via `node scripts/ci-test-shard.mjs --check-timings-staleness`, +which exits non-zero when the snapshot is missing or older than the 30-day budget. + ## Targeted commands ```bash diff --git a/plugins/fusion-plugin-openclaw-runtime/src/.index.reload-2.ts b/plugins/fusion-plugin-openclaw-runtime/src/.index.reload-2.ts new file mode 100644 index 0000000000..09745e7b06 --- /dev/null +++ b/plugins/fusion-plugin-openclaw-runtime/src/.index.reload-2.ts @@ -0,0 +1,95 @@ +/** + * OpenClaw Runtime Plugin + * + * Drives the local `openclaw` CLI as a subprocess (via + * `openclaw --no-color agent --local --json`). No daemon required. + */ + +import { definePlugin } from "@fusion/plugin-sdk"; +import { OpenClawRuntimeAdapter } from "./runtime-adapter.js"; +import { resolveCliConfig } from "./pi-module.js"; +import { probeOpenClawBinary } from "./probe.js"; +import type { + FusionPlugin, + PluginContext, + PluginRuntimeFactory, + PluginRuntimeManifestMetadata, +} from "@fusion/plugin-sdk"; + +const OPENCLAW_RUNTIME_ID = "openclaw"; +const OPENCLAW_RUNTIME_VERSION = "0.2.0"; + +const openclawRuntimeMetadata: PluginRuntimeManifestMetadata = { + runtimeId: OPENCLAW_RUNTIME_ID, + name: "OpenClaw Runtime", + description: "Drives the local `openclaw` CLI (openclaw/openclaw)", + version: OPENCLAW_RUNTIME_VERSION, +}; + +const openclawRuntimeFactory: PluginRuntimeFactory = async (ctx?: PluginContext) => { + return new OpenClawRuntimeAdapter(ctx?.settings as Record | undefined); +}; + +const plugin: FusionPlugin = definePlugin({ + manifest: { + id: "fusion-plugin-openclaw-runtime", + name: "OpenClaw Runtime Plugin", + version: OPENCLAW_RUNTIME_VERSION, + description: + "Drives the local `openclaw` CLI for Fusion agents — embedded `--local` mode by default; gateway optional.", + author: "Fusion Team", + homepage: "https://docs.openclaw.ai/", + runtime: openclawRuntimeMetadata, + }, + state: "installed", + hooks: { + onLoad: async (ctx: PluginContext) => { + const config = resolveCliConfig(ctx.settings); + const probe = await probeOpenClawBinary({ binaryPath: config.binaryPath }); + + ctx.logger.info( + probe.available + ? `OpenClaw Runtime Plugin loaded — binary=${config.binaryPath}${probe.version ? ` (${probe.version})` : ""}` + : `OpenClaw Runtime Plugin loaded but binary not detected: ${probe.reason ?? "unknown"}`, + ); + ctx.emitEvent("openclaw-runtime:loaded", { + runtimeId: OPENCLAW_RUNTIME_ID, + version: OPENCLAW_RUNTIME_VERSION, + binaryAvailable: probe.available, + binaryPath: probe.binaryPath ?? config.binaryPath, + }); + }, + onUnload: () => { + // No persistent state to clean up — each prompt spawns a fresh subprocess. + }, + }, + runtime: { + metadata: openclawRuntimeMetadata, + factory: openclawRuntimeFactory, + }, +}); + +export default plugin; + +// ── Public exports ──────────────────────────────────────────────────────────── + +export { openclawRuntimeMetadata, openclawRuntimeFactory, OPENCLAW_RUNTIME_ID }; +export { OpenClawRuntimeAdapter } from "./runtime-adapter.js"; +export { + resolveCliConfig, + buildOpenClawArgs, + createCliSession, + promptCli, + describeCliModel, + extractStderrError, + configureOpenClawMcpServer, +} from "./pi-module.js"; +export type { CliConfig, GatewaySession, OpenClawAgentJson } from "./types.js"; +export { + toolsToMcpToolDefs, + writeOpenClawMcpBridgeFiles, +} from "./mcp-config.js"; + +// Probe re-export for the dashboard's runtime-provider-probes façade. +export { probeOpenClawBinary } from "./probe.js"; +export type { OpenClawBinaryStatus } from "./probe.js"; diff --git a/plugins/fusion-plugin-openclaw-runtime/src/.index.reload-4.ts b/plugins/fusion-plugin-openclaw-runtime/src/.index.reload-4.ts new file mode 100644 index 0000000000..09745e7b06 --- /dev/null +++ b/plugins/fusion-plugin-openclaw-runtime/src/.index.reload-4.ts @@ -0,0 +1,95 @@ +/** + * OpenClaw Runtime Plugin + * + * Drives the local `openclaw` CLI as a subprocess (via + * `openclaw --no-color agent --local --json`). No daemon required. + */ + +import { definePlugin } from "@fusion/plugin-sdk"; +import { OpenClawRuntimeAdapter } from "./runtime-adapter.js"; +import { resolveCliConfig } from "./pi-module.js"; +import { probeOpenClawBinary } from "./probe.js"; +import type { + FusionPlugin, + PluginContext, + PluginRuntimeFactory, + PluginRuntimeManifestMetadata, +} from "@fusion/plugin-sdk"; + +const OPENCLAW_RUNTIME_ID = "openclaw"; +const OPENCLAW_RUNTIME_VERSION = "0.2.0"; + +const openclawRuntimeMetadata: PluginRuntimeManifestMetadata = { + runtimeId: OPENCLAW_RUNTIME_ID, + name: "OpenClaw Runtime", + description: "Drives the local `openclaw` CLI (openclaw/openclaw)", + version: OPENCLAW_RUNTIME_VERSION, +}; + +const openclawRuntimeFactory: PluginRuntimeFactory = async (ctx?: PluginContext) => { + return new OpenClawRuntimeAdapter(ctx?.settings as Record | undefined); +}; + +const plugin: FusionPlugin = definePlugin({ + manifest: { + id: "fusion-plugin-openclaw-runtime", + name: "OpenClaw Runtime Plugin", + version: OPENCLAW_RUNTIME_VERSION, + description: + "Drives the local `openclaw` CLI for Fusion agents — embedded `--local` mode by default; gateway optional.", + author: "Fusion Team", + homepage: "https://docs.openclaw.ai/", + runtime: openclawRuntimeMetadata, + }, + state: "installed", + hooks: { + onLoad: async (ctx: PluginContext) => { + const config = resolveCliConfig(ctx.settings); + const probe = await probeOpenClawBinary({ binaryPath: config.binaryPath }); + + ctx.logger.info( + probe.available + ? `OpenClaw Runtime Plugin loaded — binary=${config.binaryPath}${probe.version ? ` (${probe.version})` : ""}` + : `OpenClaw Runtime Plugin loaded but binary not detected: ${probe.reason ?? "unknown"}`, + ); + ctx.emitEvent("openclaw-runtime:loaded", { + runtimeId: OPENCLAW_RUNTIME_ID, + version: OPENCLAW_RUNTIME_VERSION, + binaryAvailable: probe.available, + binaryPath: probe.binaryPath ?? config.binaryPath, + }); + }, + onUnload: () => { + // No persistent state to clean up — each prompt spawns a fresh subprocess. + }, + }, + runtime: { + metadata: openclawRuntimeMetadata, + factory: openclawRuntimeFactory, + }, +}); + +export default plugin; + +// ── Public exports ──────────────────────────────────────────────────────────── + +export { openclawRuntimeMetadata, openclawRuntimeFactory, OPENCLAW_RUNTIME_ID }; +export { OpenClawRuntimeAdapter } from "./runtime-adapter.js"; +export { + resolveCliConfig, + buildOpenClawArgs, + createCliSession, + promptCli, + describeCliModel, + extractStderrError, + configureOpenClawMcpServer, +} from "./pi-module.js"; +export type { CliConfig, GatewaySession, OpenClawAgentJson } from "./types.js"; +export { + toolsToMcpToolDefs, + writeOpenClawMcpBridgeFiles, +} from "./mcp-config.js"; + +// Probe re-export for the dashboard's runtime-provider-probes façade. +export { probeOpenClawBinary } from "./probe.js"; +export type { OpenClawBinaryStatus } from "./probe.js"; diff --git a/scripts/__tests__/ci-test-shard.test.mjs b/scripts/__tests__/ci-test-shard.test.mjs index 3e2c9ae947..1602f9aa1d 100644 --- a/scripts/__tests__/ci-test-shard.test.mjs +++ b/scripts/__tests__/ci-test-shard.test.mjs @@ -3,14 +3,49 @@ import assert from "node:assert/strict"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +// A throwaway project root that contains only a stale timing snapshot, used to +// exercise the `--check-timings-staleness` exit-1 path without mutating the +// committed snapshot. The script resolves the snapshot relative to cwd. +const STALE_FIXTURE_ROOT = mkdtempSync(path.join(tmpdir(), "u6-stale-cli-")); +mkdirSync(path.join(STALE_FIXTURE_ROOT, "scripts"), { recursive: true }); +writeFileSync( + path.join(STALE_FIXTURE_ROOT, "scripts/test-timings.json"), + JSON.stringify({ + capturedAt: new Date(Date.now() - 90 * 86_400_000).toISOString(), + packages: { "@fusion/core": { files: { "packages/core/a.test.ts": 500 } } }, + }), +); +process.on("exit", () => rmSync(STALE_FIXTURE_ROOT, { recursive: true, force: true })); import { computeSplitPlan, planShardAssignments, selectShardPackages, countPackageTestFiles, + loadPlanningTimings, + computePackageDurationWeight, + sumFileDurations, + enumerateDashboardLanes, + laneProjectNames, + buildShardCommands, + TIMINGS_STALENESS_DAYS, } from "../ci-test-shard.mjs"; +function silentLogger() { + return { log() {}, warn() {}, error() {} }; +} + +function writeSnapshot(dir, capturedAt, packages) { + const file = path.join(dir, "test-timings.json"); + writeFileSync(file, JSON.stringify({ capturedAt, packages })); + return file; +} + test("computeSplitPlan: returns unsplit entries when package weights do not exceed split limit", () => { const packages = [ { name: "a", testFileCount: 2 }, @@ -378,3 +413,207 @@ test("countPackageTestFiles: returns 0 when no __tests__ matches exist", (t) => assert.equal(countPackageTestFiles("pkg", { projectRoot: tmpRoot }), 0); }); + +// --------------------------------------------------------------------------- +// U6 (R3, R4): duration-based weighting, staleness, dashboard lane distribution +// --------------------------------------------------------------------------- + +test("U6: duration weights produce balanced shards on skewed inputs where file-count would skew", () => { + // 10 "engine" real-git files at 10s each (100s) vs 50 "core" unit files at + // 0.4s each (20s). File-count weighting would call core the heavier package. + const durationPackages = [ + { name: "engine", weight: 100_000, splittable: true }, + { name: "core", weight: 20_000, splittable: true }, + { name: "cli", weight: 12_000, splittable: true }, + { name: "tail", weight: 8_000, splittable: true }, + ]; + const shards = planShardAssignments(durationPackages, 2); + const totals = shards.map((entries) => entries.reduce((sum, e) => sum + e.weight, 0)); + const spread = (Math.max(...totals) - Math.min(...totals)) / (totals.reduce((a, b) => a + b, 0) / 2); + assert.ok(spread <= 0.05, `duration spread ${(spread * 100).toFixed(1)}% should be <=5% (${totals.join("/")})`); + + // Motivation fixture: the SAME workload weighted by file count is badly + // skewed because the heavy engine has few files. + const fileCountPackages = [ + { name: "engine", testFileCount: 10 }, + { name: "core", testFileCount: 50 }, + { name: "cli", testFileCount: 30 }, + { name: "tail", testFileCount: 20 }, + ]; + // Disable splitting to expose the raw file-count balance signal. + const fcShards = planShardAssignments(fileCountPackages, 2, { threshold: Number.POSITIVE_INFINITY }); + const fcDurations = { engine: 100_000, core: 20_000, cli: 12_000, tail: 8_000 }; + const fcTotals = fcShards.map((entries) => entries.reduce((sum, e) => sum + fcDurations[e.name], 0)); + const fcSpread = + (Math.max(...fcTotals) - Math.min(...fcTotals)) / (fcTotals.reduce((a, b) => a + b, 0) / 2); + assert.ok( + fcSpread > 0.05, + `file-count weighting should mis-balance real durations (got ${(fcSpread * 100).toFixed(1)}%, ${fcTotals.join("/")})`, + ); +}); + +test("U6: loadPlanningTimings sums per-package durations and derives a median per-file fallback", (t) => { + const dir = mkdtempSync(path.join(tmpdir(), "u6-timings-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const snapshotPath = writeSnapshot(dir, new Date().toISOString(), { + "@fusion/core": { files: { "packages/core/a.test.ts": 200, "packages/core/b.test.ts": 600 } }, + "@fusion/engine": { files: { "packages/engine/x.test.ts": 1000 } }, + }); + const timings = loadPlanningTimings({ snapshotPath }); + assert.equal(timings.present, true); + assert.equal(timings.stale, false); + assert.equal(timings.fileDurations.get("packages/core/a.test.ts"), 200); + // median of [200, 600, 1000] = 600 + assert.equal(timings.medianPerFileMs, 600); +}); + +test("U6: sumFileDurations reports timed/untimed counts", () => { + const map = new Map([["a.test.ts", 300]]); + const result = sumFileDurations(["a.test.ts", "missing.test.ts"], map); + assert.equal(result.durationMs, 300); + assert.equal(result.timedCount, 1); + assert.equal(result.untimedCount, 1); +}); + +test("U6: untimed package falls back to median-scaled file-count weight with a warning", (t) => { + const projectRoot = mkdtempSync(path.join(tmpdir(), "u6-fallback-")); + t.after(() => rmSync(projectRoot, { recursive: true, force: true })); + // Build a fake package with 3 test files, none present in the snapshot. + mkdirSync(path.join(projectRoot, "packages/newpkg/src/__tests__"), { recursive: true }); + for (const f of ["one", "two", "three"]) { + writeFileSync(path.join(projectRoot, `packages/newpkg/src/__tests__/${f}.test.ts`), ""); + } + const snapshotPath = writeSnapshot(projectRoot, new Date().toISOString(), { + "@fusion/core": { files: { "packages/core/a.test.ts": 500, "packages/core/b.test.ts": 500 } }, + }); + const timings = loadPlanningTimings({ snapshotPath }); + const weighted = computePackageDurationWeight( + { name: "@fusion/newpkg", dir: "packages/newpkg" }, + timings, + { projectRoot }, + ); + assert.equal(weighted.fullyUntimed, true); + // 3 untimed files * median(500) = 1500 + assert.equal(weighted.weight, 1500); +}); + +test("U6: staleness — snapshot older than the budget is flagged stale (warning, not failure)", (t) => { + const dir = mkdtempSync(path.join(tmpdir(), "u6-stale-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const old = new Date(Date.now() - (TIMINGS_STALENESS_DAYS + 10) * 86_400_000).toISOString(); + const snapshotPath = writeSnapshot(dir, old, { p: { files: { "a.test.ts": 100 } } }); + const timings = loadPlanningTimings({ snapshotPath }); + assert.equal(timings.stale, true); + assert.ok(timings.ageDays > TIMINGS_STALENESS_DAYS); + + const fresh = new Date(Date.now() - 1 * 86_400_000).toISOString(); + const freshPath = writeSnapshot(dir, fresh, { p: { files: { "a.test.ts": 100 } } }); + assert.equal(loadPlanningTimings({ snapshotPath: freshPath }).stale, false); +}); + +test("U6: --check-timings-staleness exits non-zero on a stale snapshot", () => { + const result = spawnSync( + process.execPath, + [path.join(REPO_ROOT, "scripts/ci-test-shard.mjs"), "--check-timings-staleness"], + { cwd: STALE_FIXTURE_ROOT, encoding: "utf8" }, + ); + assert.equal(result.status, 1, result.stderr || result.stdout); + assert.match(result.stderr, /stale/i); +}); + +test("U6: enumerateDashboardLanes expands the test chain to leaf vitest lanes (no hardcoding)", () => { + const scripts = { + test: "pnpm run test:app && pnpm run test:api", + "test:app": "pnpm run test:app:foundation && pnpm run test:app:components", + "test:app:foundation": "vitest run --project dashboard-app-quality-foundation-api", + "test:app:components": "vitest run --project dashboard-app-quality-components-a", + "test:api": "vitest run --project dashboard-api-quality", + }; + const lanes = enumerateDashboardLanes(scripts, "test"); + assert.deepEqual(lanes, [ + "test:app:foundation", + "test:app:components", + "test:api", + ]); +}); + +test("U6: enumerateDashboardLanes reads lanes from a fixture package.json shape", () => { + const pkgJson = { + scripts: { + test: "pnpm run test:quality:app && pnpm run test:quality:api", + "test:quality:app": "pnpm run test:quality:app:a && pnpm run test:quality:app:b", + "test:quality:app:a": "vitest run --project x", + "test:quality:app:b": "vitest run --project y", + "test:quality:api": "vitest run --project z", + // unrelated script not reachable from `test` must not appear + "test:deep": "vitest run --project deep", + }, + }; + const lanes = enumerateDashboardLanes(pkgJson.scripts, "test"); + assert.deepEqual(lanes, ["test:quality:app:a", "test:quality:app:b", "test:quality:api"]); +}); + +test("U6: laneProjectNames extracts --project targets including = and space forms", () => { + assert.deepEqual(laneProjectNames("vitest run --project foo --project=bar baz"), ["foo", "bar"]); +}); + +test("U6: every dashboard lane is assigned to exactly one shard (union == enumerated list)", () => { + const lanes = ["lane-a", "lane-b", "lane-c", "lane-d", "lane-e"]; + const units = [ + { name: "@fusion/engine", weight: 50_000, splittable: true }, + { name: "@fusion/core", weight: 40_000, splittable: true }, + ...lanes.map((lane, i) => ({ + name: "@fusion/dashboard", + lane, + runKind: "dashboard-lane", + weight: 10_000 + i * 1000, + splittable: false, + })), + ]; + const shards = planShardAssignments(units, 4); + const occur = new Map(); + let dashboardShardSlices = 0; + for (const shard of shards) { + for (const entry of shard) { + if (entry.runKind === "dashboard-lane") occur.set(entry.lane, (occur.get(entry.lane) ?? 0) + 1); + if (entry.name === "@fusion/dashboard" && entry.shardCount) dashboardShardSlices += 1; + } + } + assert.equal(dashboardShardSlices, 0, "dashboard lane units must never be vitest --shard sliced"); + assert.deepEqual([...occur.keys()].sort(), [...lanes].sort()); + for (const lane of lanes) assert.equal(occur.get(lane), 1, `lane ${lane} should appear exactly once`); +}); + +test("U6: buildShardCommands emits per-lane `run `, plain `test`, and virtual `--shard`", () => { + const entries = [ + { name: "@fusion/core", weight: 1 }, + { name: "@fusion/engine", weight: 1, shardIndex: 1, shardCount: 2 }, + { name: "@fusion/dashboard", weight: 1, runKind: "dashboard-lane", lane: "test:quality:api" }, + ]; + const commands = buildShardCommands(entries); + const plain = commands.find((c) => c.kind === "plain"); + const virtual = commands.find((c) => c.kind === "virtual"); + const lane = commands.find((c) => c.kind === "dashboard-lane"); + assert.deepEqual(plain.args, ["--filter", "@fusion/core", "test"]); + assert.deepEqual(virtual.args, ["--filter", "@fusion/engine", "test", "--shard=1/2"]); + assert.deepEqual(lane.args, ["--filter", "@fusion/dashboard", "run", "test:quality:api"]); +}); + +test("U6: --dry-run prints planned commands and per-shard weight for all 4 shards", () => { + const result = spawnSync( + process.execPath, + [path.join(REPO_ROOT, "scripts/ci-test-shard.mjs"), "--dry-run", "--total", "4"], + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + assert.equal(result.status, 0, result.stderr); + for (let n = 1; n <= 4; n += 1) { + assert.match(result.stdout, new RegExp(`shard ${n}/4 — weight`)); + } + // Each dashboard lane appears exactly once across the printed plan. + const laneMatches = result.stdout.match(/--filter @fusion\/dashboard run [\w:-]+/g) ?? []; + const laneNames = laneMatches.map((m) => m.split("run ")[1]); + assert.equal(new Set(laneNames).size, laneNames.length, "no dashboard lane should be printed twice"); + assert.ok(laneNames.length >= 10, `expected the dashboard lane chain, saw ${laneNames.length}`); + // Dashboard must NOT be virtual-sliced. + assert.doesNotMatch(result.stdout, /--filter @fusion\/dashboard test --shard/); +}); diff --git a/scripts/ci-test-shard.mjs b/scripts/ci-test-shard.mjs index cd7e85299e..cd1f5be100 100644 --- a/scripts/ci-test-shard.mjs +++ b/scripts/ci-test-shard.mjs @@ -88,9 +88,24 @@ export function countPackageTestFiles(packageDir, { projectRoot = process.cwd() const DEFAULT_BALANCE_TOLERANCE = 0.05; +/** + * Resolve the schedulable weight of an input package descriptor. Duration-based + * weights (U6 / R3) are preferred via the explicit `weight` field; the legacy + * `testFileCount` field is the file-count fallback so existing callers and the + * untimed-package fallback path keep working unchanged. + * + * @param {{ weight?: number, testFileCount?: number }} pkg + * @returns {number} + */ +function inputWeightOf(pkg) { + if (typeof pkg.weight === "number" && Number.isFinite(pkg.weight)) return pkg.weight; + return pkg.testFileCount ?? 0; +} + function appendSplitEntries(result, pkg, total, perShardBudget) { - const sliceCount = Math.min(total, Math.max(2, Math.ceil(pkg.testFileCount / perShardBudget))); - const sliceWeight = Math.ceil(pkg.testFileCount / sliceCount); + const baseWeight = inputWeightOf(pkg); + const sliceCount = Math.min(total, Math.max(2, Math.ceil(baseWeight / perShardBudget))); + const sliceWeight = Math.ceil(baseWeight / sliceCount); for (let i = 1; i <= sliceCount; i += 1) { result.push({ name: pkg.name, @@ -180,21 +195,32 @@ function assignWeightedEntries(entries, total) { export function computeSplitPlan(packages, total, options = {}) { const threshold = options.threshold ?? 0.5; const balanceTolerance = options.balanceTolerance ?? DEFAULT_BALANCE_TOLERANCE; - const totalWeight = packages.reduce((sum, p) => sum + p.testFileCount, 0); + const totalWeight = packages.reduce((sum, p) => sum + inputWeightOf(p), 0); const perShardBudget = total > 0 ? totalWeight / total : 0; const splitLimit = perShardBudget * threshold; const maxAllowedProjected = perShardBudget * (1 + balanceTolerance); const result = []; for (const pkg of packages) { + const pkgWeight = inputWeightOf(pkg); + // Lane-distributed units (dashboard, U6) and any caller that opts out are + // never virtual-sliced via `vitest --shard`: their `test` script is a + // multi-invocation chain, so `--shard=X/Y` cannot be forwarded coherently. const shouldConsiderSplit = + pkg.splittable !== false && total > 1 && - pkg.testFileCount > 0 && + pkgWeight > 0 && perShardBudget > 0 && - pkg.testFileCount > splitLimit; + pkgWeight > splitLimit; if (!shouldConsiderSplit) { - result.push({ name: pkg.name, weight: pkg.testFileCount }); + result.push({ + name: pkg.name, + weight: pkgWeight, + ...(pkg.splittable === false ? { splittable: false } : {}), + ...(pkg.runKind ? { runKind: pkg.runKind } : {}), + ...(pkg.lane ? { lane: pkg.lane } : {}), + }); continue; } @@ -211,7 +237,8 @@ export function computeSplitPlan(packages, total, options = {}) { const forceSplitThreshold = splitLimit * threshold; let rebalanceResult = result.map((entry) => { - if (entry.shardCount) return entry; + // Lane-distributed / opt-out units (dashboard) are never `--shard`-sliced. + if (entry.shardCount || entry.splittable === false) return entry; const projectedBestCaseMax = perShardBudget + entry.weight; const shouldForceSplit = entry.weight > 0 && @@ -228,7 +255,10 @@ export function computeSplitPlan(packages, total, options = {}) { } const nextCandidate = rebalanceResult - .filter((entry) => !entry.shardCount && entry.weight > perShardBudget * balanceTolerance) + .filter( + (entry) => + !entry.shardCount && entry.splittable !== false && entry.weight > perShardBudget * balanceTolerance, + ) .sort((a, b) => b.weight - a.weight || a.name.localeCompare(b.name))[0]; if (!nextCandidate) { @@ -326,7 +356,12 @@ export function planShardAssignments(packages, total, options = {}) { shardIndex: entry.shardIndex, shardCount: entry.shardCount, weight: entry.weight, - } : { name: entry.name, weight: entry.weight }); + } : { + name: entry.name, + weight: entry.weight, + ...(entry.runKind ? { runKind: entry.runKind } : {}), + ...(entry.lane ? { lane: entry.lane } : {}), + }); shardWeights[targetIndex] += entry.weight; } @@ -354,7 +389,379 @@ export function listWorkspaceTestPackages({ projectRoot = process.cwd() } = {}) })); } +// --------------------------------------------------------------------------- +// Duration-based weighting and dashboard lane distribution (U6 / R3, R4) +// --------------------------------------------------------------------------- + +/** Snapshot older than this is reported as stale (warning, not a failure). */ +export const TIMINGS_STALENESS_DAYS = 30; + +/** Dashboard package name; its `test` chain is distributed lane-by-lane. */ +export const DASHBOARD_PACKAGE_NAME = "@fusion/dashboard"; + +/** Engine package name; kept on `vitest --shard` virtual slicing, by duration. */ +export const ENGINE_PACKAGE_NAME = "@fusion/engine"; + +/** + * Load the committed timing snapshot into a flat per-file duration map plus a + * derived median per-file duration (used to scale the file-count fallback so + * untimed packages are weighed commensurably with timed ones). + * + * @param {{ projectRoot?: string, snapshotPath?: string }} [options] + * @returns {{ + * present: boolean, + * capturedAt: string|null, + * fileDurations: Map, + * pkgDurations: Map, + * medianPerFileMs: number, + * ageDays: number|null, + * stale: boolean, + * }} + */ +export function loadPlanningTimings(options = {}) { + const projectRoot = options.projectRoot ?? process.cwd(); + const snapshotPath = options.snapshotPath ?? path.join(projectRoot, TIMINGS_SNAPSHOT_RELATIVE); + const snapshot = readTimingsSnapshot(snapshotPath); + + const fileDurations = new Map(); + const pkgDurations = new Map(); + const allDurations = []; + if (snapshot && snapshot.packages && typeof snapshot.packages === "object") { + for (const [pkgName, pkgEntry] of Object.entries(snapshot.packages)) { + const files = pkgEntry && typeof pkgEntry === "object" ? pkgEntry.files : null; + if (!files || typeof files !== "object") continue; + let pkgTotal = 0; + for (const [file, duration] of Object.entries(files)) { + const ms = Number(duration); + if (!Number.isFinite(ms) || ms <= 0) continue; + const normalized = file.split(path.sep).join("/"); + fileDurations.set(normalized, ms); + allDurations.push(ms); + pkgTotal += ms; + } + pkgDurations.set(pkgName, pkgTotal); + } + } + + let medianPerFileMs = 0; + if (allDurations.length > 0) { + const sorted = [...allDurations].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + medianPerFileMs = + sorted.length % 2 === 0 ? Math.round((sorted[mid - 1] + sorted[mid]) / 2) : sorted[mid]; + } + + let ageDays = null; + let stale = false; + if (snapshot && typeof snapshot.capturedAt === "string") { + const captured = new Date(snapshot.capturedAt).getTime(); + if (Number.isFinite(captured)) { + ageDays = (Date.now() - captured) / (1000 * 60 * 60 * 24); + stale = ageDays > TIMINGS_STALENESS_DAYS; + } + } + + return { + present: Boolean(snapshot), + capturedAt: snapshot?.capturedAt ?? null, + fileDurations, + pkgDurations, + medianPerFileMs, + ageDays, + stale, + }; +} + +/** + * Sum the snapshot durations for a set of repo-relative test files. Returns the + * matched duration total and the count of files with no timing data. + * + * @param {string[]} files repo-relative paths + * @param {Map} fileDurations + * @returns {{ durationMs: number, timedCount: number, untimedCount: number }} + */ +export function sumFileDurations(files, fileDurations) { + let durationMs = 0; + let timedCount = 0; + let untimedCount = 0; + for (const file of files) { + const normalized = file.split(path.sep).join("/"); + const ms = fileDurations.get(normalized); + if (typeof ms === "number" && ms > 0) { + durationMs += ms; + timedCount += 1; + } else { + untimedCount += 1; + } + } + return { durationMs, timedCount, untimedCount }; +} + +/** + * Compute a duration weight for a package from its files. Files present in the + * snapshot contribute their measured duration; files absent fall back to the + * snapshot's median per-file duration (R3 commensurable scaling). When the + * whole package is untimed, the entire weight is the fallback and the package + * name is collected for a logged warning. + * + * @param {{ name: string, dir: string }} pkg + * @param {ReturnType} timings + * @param {{ projectRoot?: string }} [options] + * @returns {{ name: string, dir: string, weight: number, fullyUntimed: boolean, partiallyUntimed: boolean }} + */ +export function computePackageDurationWeight(pkg, timings, options = {}) { + const projectRoot = options.projectRoot ?? process.cwd(); + const files = globSync("**/__tests__/**/*.test.{ts,tsx,mjs}", { + cwd: path.join(projectRoot, pkg.dir), + nodir: true, + exclude: (p) => p.startsWith("dist/") || p.includes("/dist/"), + }).map((f) => `${pkg.dir}/${f}`); + + const fallbackPerFile = timings.medianPerFileMs > 0 ? timings.medianPerFileMs : DURATION_BUCKET_MS; + const { durationMs, timedCount, untimedCount } = sumFileDurations(files, timings.fileDurations); + const weight = durationMs + untimedCount * fallbackPerFile; + + return { + name: pkg.name, + dir: pkg.dir, + weight, + fullyUntimed: timedCount === 0 && files.length > 0, + partiallyUntimed: timedCount > 0 && untimedCount > 0, + }; +} + +/** + * Recursively expand a package's `test` script into the leaf vitest lanes it + * runs. A leaf lane is a script whose command does NOT delegate to another + * `pnpm run `; the dashboard chain is `pnpm run a && pnpm run b ...`, so + * we follow each `pnpm run ` edge until reaching commands that invoke + * vitest. Lanes are enumerated from package.json — never hardcoded. + * + * @param {Record} scripts package.json `scripts` map + * @param {string} [entryScript] + * @returns {string[]} ordered, de-duplicated leaf lane script names + */ +export function enumerateDashboardLanes(scripts, entryScript = "test") { + const lanes = []; + const seen = new Set(); + const referencedRuns = (command) => { + const names = []; + const re = /pnpm\s+run\s+([\w:-]+)/g; + let match; + while ((match = re.exec(command)) !== null) names.push(match[1]); + return names; + }; + + const visit = (scriptName) => { + if (seen.has(scriptName)) return; + seen.add(scriptName); + const command = scripts?.[scriptName]; + if (typeof command !== "string") return; + const children = referencedRuns(command); + if (children.length === 0) { + // Leaf: a lane that actually invokes a test runner. + lanes.push(scriptName); + return; + } + for (const child of children) visit(child); + }; + + visit(entryScript); + return lanes; +} + +/** + * Extract the vitest `--project ` targets referenced by a lane command. + * + * @param {string} command + * @returns {string[]} + */ +export function laneProjectNames(command) { + const names = []; + const re = /--project[=\s]+([\w-]+)/g; + let match; + while ((match = re.exec(command)) !== null) names.push(match[1]); + return names; +} + +/** + * Resolve dashboard project name → repo-relative test files by importing the + * dashboard vitest config (via `tsx`, which resolves its extensionless TS + * imports) and globbing each project's `include`/`exclude`. This is the + * "derive from the vitest config project includes" path. On any failure + * (config not importable, tsx missing) it returns null so the caller falls back + * to even apportionment of the package duration across lanes. + * + * @param {string} dashboardDir repo-relative dashboard dir + * @param {{ projectRoot?: string }} [options] + * @returns {Record|null} projectName → repo-relative files + */ +export function resolveDashboardProjectFiles(dashboardDir, options = {}) { + const projectRoot = options.projectRoot ?? process.cwd(); + const dashboardAbs = path.join(projectRoot, dashboardDir); + const script = ` + import config from "./vitest.config.ts"; + import { globSync } from "node:fs"; + const projects = config?.test?.projects ?? []; + const out = {}; + for (const p of projects) { + const name = p?.test?.name; + if (!name) continue; + const include = Array.isArray(p.test.include) ? p.test.include : [p.test.include].filter(Boolean); + const exclude = Array.isArray(p.test.exclude) ? p.test.exclude : []; + const files = new Set(); + for (const g of include) for (const f of globSync(g, { cwd: process.cwd(), nodir: true })) files.add(f); + const excluded = new Set(); + for (const g of exclude) for (const f of globSync(g, { cwd: process.cwd(), nodir: true })) excluded.add(f); + out[name] = [...files].filter((f) => !excluded.has(f)); + } + process.stdout.write(JSON.stringify(out)); + `; + const tsxBin = path.join(projectRoot, "node_modules/.bin/tsx"); + const result = spawnSync( + tsxBin, + ["--eval", script], + { cwd: dashboardAbs, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ); + if (result.status !== 0 || !result.stdout) { + return null; + } + let parsed; + try { + parsed = JSON.parse(result.stdout); + } catch { + return null; + } + const out = {}; + for (const [name, files] of Object.entries(parsed)) { + out[name] = (Array.isArray(files) ? files : []).map((f) => `${dashboardDir}/${f}`.split(path.sep).join("/")); + } + return out; +} + +/** + * Build the dashboard lane schedulable units. Each enumerated leaf lane becomes + * one unit weighted by the durations of the files its `--project`s execute + * (a lane carrying `--shard=i/n` runs 1/n of those files). When the config + * cannot be imported, the package duration is apportioned evenly across lanes. + * + * @param {{ name: string, dir: string }} pkg + * @param {ReturnType} timings + * @param {{ projectRoot?: string }} [options] + * @returns {{ units: Array<{ name: string, lane: string, runKind: "dashboard-lane", weight: number, splittable: false }>, lanes: string[], method: string, untimed: string[] }} + */ +export function buildDashboardLaneUnits(pkg, timings, options = {}) { + const projectRoot = options.projectRoot ?? process.cwd(); + const pkgJson = JSON.parse(readFileSync(path.join(projectRoot, pkg.dir, "package.json"), "utf8")); + const scripts = pkgJson.scripts ?? {}; + const lanes = enumerateDashboardLanes(scripts, "test"); + + const projectFiles = resolveDashboardProjectFiles(pkg.dir, { projectRoot }); + const fallbackPerFile = timings.medianPerFileMs > 0 ? timings.medianPerFileMs : DURATION_BUCKET_MS; + const untimed = []; + + if (projectFiles) { + const units = lanes.map((lane) => { + const command = scripts[lane] ?? ""; + const projects = laneProjectNames(command); + const shardMatch = /--shard[=\s]+(\d+)\/(\d+)/.exec(command); + const shardFraction = shardMatch ? 1 / Number(shardMatch[2]) : 1; + const files = new Set(); + for (const project of projects) for (const f of projectFiles[project] ?? []) files.add(f); + const { durationMs, timedCount, untimedCount } = sumFileDurations([...files], timings.fileDurations); + const weight = (durationMs + untimedCount * fallbackPerFile) * shardFraction; + if (timedCount === 0 && files.size > 0) untimed.push(lane); + return { name: pkg.name, lane, runKind: "dashboard-lane", weight: Math.max(weight, fallbackPerFile), splittable: false }; + }); + return { units, lanes, method: "vitest-config-includes", untimed }; + } + + // Fallback: even apportionment of the package's measured duration. + const pkgWeight = computePackageDurationWeight(pkg, timings, { projectRoot }).weight; + const perLane = lanes.length > 0 ? pkgWeight / lanes.length : 0; + const units = lanes.map((lane) => ({ + name: pkg.name, + lane, + runKind: "dashboard-lane", + weight: Math.max(perLane, fallbackPerFile), + splittable: false, + })); + return { units, lanes, method: "even-apportionment", untimed: lanes }; +} + +/** + * Translate the raw workspace package list into duration-weighted schedulable + * units for the planner: the dashboard expands into per-lane units; every other + * package is a single duration-weighted unit (engine stays virtual-sliceable). + * Untimed packages fall back to median-scaled file-count weight with a warning. + * + * @param {{ projectRoot?: string, logger?: Console, timings?: ReturnType }} [options] + * @returns {{ units: Array<{ name: string, weight: number, runKind?: string, lane?: string, splittable?: boolean }>, dashboardLanes: string[], timings: ReturnType }} + */ +export function buildScheduleUnits(options = {}) { + const projectRoot = options.projectRoot ?? process.cwd(); + const logger = options.logger ?? console; + const timings = options.timings ?? loadPlanningTimings({ projectRoot }); + const packages = listWorkspaceTestPackages({ projectRoot }); + + if (!timings.present) { + logger.warn( + "[ci-test-shard] no timing snapshot found; falling back to file-count weighting for all packages.", + ); + } else if (timings.stale) { + logger.warn( + `[ci-test-shard] WARNING: timing snapshot is ${Math.round(timings.ageDays)} days old ` + + `(> ${TIMINGS_STALENESS_DAYS}d staleness budget; capturedAt ${timings.capturedAt}). ` + + "Shard balance may have drifted. Refresh it from the default branch's CI timing artifacts " + + "via `node scripts/ci-test-shard.mjs --write-timings`.", + ); + } + + const units = []; + let dashboardLanes = []; + const untimedPackages = []; + + for (const pkg of packages) { + if (pkg.name === DASHBOARD_PACKAGE_NAME) { + const { units: laneUnits, lanes, method, untimed } = buildDashboardLaneUnits(pkg, timings, { projectRoot }); + units.push(...laneUnits); + dashboardLanes = lanes; + logger.log( + `[ci-test-shard] dashboard distributed across ${lanes.length} lanes (weights via ${method}).`, + ); + if (untimed.length > 0) { + logger.warn( + `[ci-test-shard] dashboard lanes without timing data (median-scaled fallback): ${untimed.join(", ")}`, + ); + } + continue; + } + + const weighted = computePackageDurationWeight(pkg, timings, { projectRoot }); + if (weighted.fullyUntimed) untimedPackages.push(pkg.name); + units.push({ + name: pkg.name, + weight: weighted.weight, + // Engine remains virtual-sliceable; everything else stays whole unless + // the planner's force-split balance pass decides otherwise. + splittable: true, + }); + } + + if (untimedPackages.length > 0) { + logger.warn( + `[ci-test-shard] no timing data for: ${untimedPackages.join(", ")}; ` + + `using median-scaled (${timings.medianPerFileMs}ms/file) file-count weight.`, + ); + } + + return { units, dashboardLanes, timings }; +} + function entryLabel(entry) { + if (entry.runKind === "dashboard-lane") { + return `${entry.name} run ${entry.lane}`; + } if (entry.shardCount) { return `${entry.name} [${entry.shardIndex}/${entry.shardCount}]`; } @@ -642,6 +1049,56 @@ export function runColdStartProbe(packageName, options = {}) { }; } +/** + * Translate the resolved shard entries into the concrete pnpm command argument + * vectors that execute them. Plain duration-weighted packages run together in + * one `pnpm --filter ... test` invocation; virtual engine slices each get a + * `--shard=i/n` invocation; dashboard lane units each run their own + * `pnpm --filter @fusion/dashboard run `. `timingFlags()` (if provided) + * appends the JSON reporter flags so telemetry keeps flowing (U1/R4). + * + * @param {ShardEntry[]} shardEntries + * @param {{ timingFlags?: () => string[] }} [options] + * @returns {Array<{ kind: string, label: string, args: string[] }>} + */ +export function buildShardCommands(shardEntries, options = {}) { + const timingFlags = options.timingFlags ?? (() => []); + const commands = []; + + const plain = shardEntries.filter((e) => !e.shardCount && e.runKind !== "dashboard-lane"); + const virtual = shardEntries.filter((e) => e.shardCount); + const lanes = shardEntries.filter((e) => e.runKind === "dashboard-lane"); + + if (plain.length > 0) { + const filters = plain.flatMap((e) => ["--filter", e.name]); + commands.push({ + kind: "plain", + label: plain.map((e) => e.name).join(", "), + args: [...filters, "test", ...timingFlags()], + }); + } + + for (const entry of virtual) { + commands.push({ + kind: "virtual", + label: `${entry.name} [${entry.shardIndex}/${entry.shardCount}]`, + // NB: no `--` between `test` and `--shard`; cac would treat the value as a + // positional file filter and silently disable sharding. + args: ["--filter", entry.name, "test", `--shard=${entry.shardIndex}/${entry.shardCount}`, ...timingFlags()], + }); + } + + for (const entry of lanes) { + commands.push({ + kind: "dashboard-lane", + label: `${entry.name} run ${entry.lane}`, + args: ["--filter", entry.name, "run", entry.lane, ...timingFlags()], + }); + } + + return commands; +} + export function main(argv = process.argv.slice(2), env = process.env) { if (argv.includes("--write-timings")) { const dirIdx = argv.indexOf("--inputs-dir"); @@ -650,6 +1107,64 @@ export function main(argv = process.argv.slice(2), env = process.env) { return; } + if (argv.includes("--check-timings-staleness")) { + const timings = loadPlanningTimings(); + if (!timings.present) { + console.error("[ci-test-shard] no timing snapshot present; refresh required."); + process.exitCode = 1; + return; + } + if (timings.stale) { + console.error( + `[ci-test-shard] timing snapshot is stale: ${Math.round(timings.ageDays)} days old ` + + `(> ${TIMINGS_STALENESS_DAYS}d). capturedAt ${timings.capturedAt}. ` + + "Refresh from the default branch via `node scripts/ci-test-shard.mjs --write-timings`.", + ); + process.exitCode = 1; + return; + } + console.log( + `[ci-test-shard] timing snapshot fresh: ${Math.round(timings.ageDays ?? 0)} days old ` + + `(<= ${TIMINGS_STALENESS_DAYS}d). capturedAt ${timings.capturedAt}.`, + ); + return; + } + + if (argv.includes("--dry-run")) { + const total = parsePositiveInteger( + (() => { + const i = argv.indexOf("--total"); + return i >= 0 ? argv[i + 1] : undefined; + })() ?? env.CI_SHARD_TOTAL, + ); + const singleShard = parsePositiveInteger( + (() => { + const i = argv.indexOf("--shard"); + return i >= 0 ? argv[i + 1] : undefined; + })() ?? env.CI_SHARD_INDEX, + ); + if (!total) { + throw new Error("Usage: node scripts/ci-test-shard.mjs --dry-run --total [--shard <1..N>]"); + } + const { units } = buildScheduleUnits(); + const assignments = planShardAssignments(units, total); + const weightOf = (entry) => entry.weight ?? 0; + const shardsToPrint = singleShard ? [singleShard] : Array.from({ length: total }, (_, i) => i + 1); + for (const shardNum of shardsToPrint) { + const entries = assignments[shardNum - 1] ?? []; + const totalMs = entries.reduce((sum, e) => sum + weightOf(e), 0); + console.log( + `\n[ci-test-shard] shard ${shardNum}/${total} — weight ${(totalMs / 1000).toFixed(1)}s, ${entries.length} unit(s):`, + ); + const commands = buildShardCommands(entries); + for (const command of commands) { + console.log(` pnpm ${command.args.join(" ")}`); + } + if (commands.length === 0) console.log(" (no assigned units)"); + } + return; + } + if (argv.includes("--cold-start-probe")) { const pkgIdx = argv.indexOf("--cold-start-probe"); const packageName = argv[pkgIdx + 1]; @@ -666,10 +1181,11 @@ export function main(argv = process.argv.slice(2), env = process.env) { } const { shard, total } = parseShardArgs(argv, env); - const shardEntries = selectShardPackages(listWorkspaceTestPackages(), shard, total); + const { units } = buildScheduleUnits(); + const shardEntries = planShardAssignments(units, total)[shard - 1] || []; if (shardEntries.length === 0) { - console.log(`[ci-test-shard] shard ${shard}/${total} has no assigned packages; skipping.`); + console.log(`[ci-test-shard] shard ${shard}/${total} has no assigned units; skipping.`); return; } @@ -688,9 +1204,6 @@ export function main(argv = process.argv.slice(2), env = process.env) { // Per-shard timing telemetry (U1 / R4): each test invocation also emits a // vitest JSON reporter file under .timings/. These are uploaded as CI // artifacts and consumed by `--write-timings` to refresh the snapshot. - // Reporters are appended as CLI flags following the same no-`--` quirk as the - // virtual `--shard` forwarding; package `test` scripts already pass - // `--reporter=dot`, and vitest accepts multiple `--reporter` flags. const timingsDir = path.join(process.cwd(), ".timings"); mkdirSync(timingsDir, { recursive: true }); let invocationIndex = 0; @@ -699,29 +1212,10 @@ export function main(argv = process.argv.slice(2), env = process.env) { return ["--reporter=json", `--outputFile.json=${outputFile}`]; }; - // Group entries: plain packages run together in one pnpm invocation; - // virtual (sharded) entries each get their own vitest --shard invocation. - const plain = shardEntries.filter((e) => !e.shardCount); - const virtual = shardEntries.filter((e) => e.shardCount); - - if (plain.length > 0) { - const filters = plain.flatMap((e) => ["--filter", e.name]); - run("pnpm", [...filters, "test", ...timingFlags()], { env: shardEnv }); - } - - for (const entry of virtual) { - console.log( - `[ci-test-shard] shard ${shard}/${total}: running ${entry.name} --shard ${entry.shardIndex}/${entry.shardCount}`, - ); - // NB: no `--` between `test` and `--shard`. pnpm 10 forwards extra args to - // the script regardless, and inserting `--` causes vitest's CLI parser - // (cac) to treat `--shard X/Y` as positional file filters → sharding is - // silently disabled and every shard runs the full suite. - run( - "pnpm", - ["--filter", entry.name, "test", `--shard=${entry.shardIndex}/${entry.shardCount}`, ...timingFlags()], - { env: shardEnv }, - ); + const commands = buildShardCommands(shardEntries, { timingFlags }); + for (const command of commands) { + console.log(`[ci-test-shard] shard ${shard}/${total}: running ${command.label}`); + run("pnpm", command.args, { env: shardEnv }); } }