From 7edf24816c83e57425fd5adfc0a975e061a95f30 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 13 Jun 2026 00:40:21 -0700 Subject: [PATCH] feat(test-infra): run CI shard invocations under the watchdog (U1) Convert ci-test-shard.mjs's vitest invocation path from blocking spawnSync to async spawn through scripts/lib/run-vitest-watchdog.mjs. Each shard command gets a per-class budget (shard ceiling 30min) tightened by aggregated timings only when the snapshot is fresh; quick non-test commands stay synchronous. Co-Authored-By: Claude Opus 4.8 --- scripts/ci-test-shard.mjs | 61 +++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/scripts/ci-test-shard.mjs b/scripts/ci-test-shard.mjs index aba7f31500..20e6dccdea 100644 --- a/scripts/ci-test-shard.mjs +++ b/scripts/ci-test-shard.mjs @@ -13,14 +13,17 @@ * keeping slices of the same package on different shards whenever possible. */ -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { globSync, readFileSync, writeFileSync, readdirSync, mkdirSync, renameSync } from "node:fs"; import { cpus } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs"; import { listWorkspacePackageInfos } from "./test-changed.mjs"; +import { deriveBudgetMs, runWithWatchdog } from "./lib/run-vitest-watchdog.mjs"; +// Quick, non-test commands (e.g. skill-sync check) stay synchronous — they have +// no hang risk and no benefit from the watchdog. function run(command, commandArgs, options = {}) { const result = spawnSync(command, commandArgs, { cwd: process.cwd(), @@ -33,6 +36,32 @@ function run(command, commandArgs, options = {}) { } } +// Test invocations run under the L2 wall-clock watchdog so a wedged vitest run +// is SIGTERM/SIGKILLed at its budget instead of blocking to the CI job ceiling. +// Preserves the fail-fast contract of `run` (exit non-zero on failure/timeout). +async function runWatched(command, commandArgs, { env, budgetMs, label } = {}) { + const { code, signal, timedOut } = await runWithWatchdog({ + command, + args: commandArgs, + env: env ?? process.env, + budgetMs, + label: label ?? command, + log: console.error, + spawn, + }); + if (timedOut) { + console.error(`[ci-test-shard] FAILED (timeout): ${label ?? command}`); + process.exit(124); + } + if (signal) { + console.error(`[ci-test-shard] FAILED (signal ${signal}): ${label ?? command}`); + process.exit(1); + } + if (code !== 0) { + process.exit(code ?? 1); + } +} + function parsePositiveInteger(value) { const parsed = Number.parseInt(value ?? "", 10); if (!Number.isInteger(parsed) || parsed <= 0) { @@ -1129,6 +1158,10 @@ export function buildShardCommands(shardEntries, options = {}) { commands.push({ kind: "plain", label: plain.map((e) => e.name).join(", "), + // A single plain command fans out across every packed package, so its + // expected duration is the SUM of their weights — not a per-package value + // (see the watchdog budget aggregation, KTD-2). + weightMs: plain.reduce((sum, e) => sum + (e.weight ?? 0), 0), args: [...filters, "test", ...timingFlags()], }); } @@ -1137,6 +1170,7 @@ export function buildShardCommands(shardEntries, options = {}) { commands.push({ kind: "virtual", label: `${entry.name} [${entry.shardIndex}/${entry.shardCount}]`, + weightMs: entry.weight ?? 0, // NB: no `--` between `test` and `--shard`; cac would treat the value as a // positional file filter and silently disable sharding. args: ["--filter", entry.name, "test", `--shard=${entry.shardIndex}/${entry.shardCount}`, ...timingFlags()], @@ -1147,6 +1181,7 @@ export function buildShardCommands(shardEntries, options = {}) { commands.push({ kind: "dashboard-lane", label: `${entry.name} run ${entry.lane}`, + weightMs: entry.weight ?? 0, args: ["--filter", entry.name, "run", entry.lane, ...timingFlags()], }); } @@ -1154,7 +1189,7 @@ export function buildShardCommands(shardEntries, options = {}) { return commands; } -export function main(argv = process.argv.slice(2), env = process.env) { +export async function main(argv = process.argv.slice(2), env = process.env) { if (argv.includes("--write-timings")) { const dirIdx = argv.indexOf("--inputs-dir"); const inputDir = dirIdx >= 0 ? argv[dirIdx + 1] : undefined; @@ -1236,7 +1271,11 @@ export function main(argv = process.argv.slice(2), env = process.env) { } const { shard, total } = parseShardArgs(argv, env); - const { units } = buildScheduleUnits(); + const { units, timings } = buildScheduleUnits(); + // Only trust timings to TIGHTEN the watchdog budget when the snapshot is + // present and fresh; otherwise deriveBudgetMs falls back to the generous + // per-class ceiling (KTD-2). + const timingsFresh = Boolean(timings?.present) && !timings?.stale; const shardEntries = planShardAssignments(units, total)[shard - 1] || []; if (shardEntries.length === 0) { @@ -1274,12 +1313,22 @@ export function main(argv = process.argv.slice(2), env = process.env) { const commands = buildShardCommands(shardEntries, { timingFlags }); for (const command of commands) { - console.log(`[ci-test-shard] shard ${shard}/${total}: running ${command.label}`); - run("pnpm", command.args, { env: shardEnv }); + const klass = command.kind === "dashboard-lane" ? "dashboard-lane" : "shard"; + const budgetMs = deriveBudgetMs({ + klass, + expectedDurationMs: command.weightMs, + timingsFresh, + }); + const label = `shard ${shard}/${total}: ${command.label}`; + console.log(`[ci-test-shard] ${label} (watchdog budget ${Math.round(budgetMs / 1000)}s)`); + await runWatched("pnpm", command.args, { env: shardEnv, budgetMs, label }); } } const currentFilePath = fileURLToPath(import.meta.url); if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) { - main(); + main().catch((error) => { + console.error(error); + process.exit(1); + }); }