From 414e1a3bfb5fbc2d27971cdddd5d0e1969175e3c Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 13 Jun 2026 00:43:38 -0700 Subject: [PATCH] feat(test-infra): run changed-file test invocations under the watchdog (U1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert test-changed.mjs's invocation path from blocking spawnSync to async spawn through the shared watchdog. runWatchedTest throws on failure/timeout with .exitCode so the existing catch and the runMaybeIsolated finally (prune + isolation post-check) still run — a watchdog kill reaps any leaked isolated HOME instead of leaving it for the guard to flag. Full/affected runs use a generous 60min backstop (dashboard lanes are already inner-watchdog'd); the quick gate uses the changed-class ceiling. Co-Authored-By: Claude Opus 4.8 --- scripts/test-changed.mjs | 69 ++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index 59a07aa1ed..3797c193b6 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -2,7 +2,7 @@ import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync } from "node:fs"; import path from "node:path"; -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; import { cpus, tmpdir } from "node:os"; @@ -10,6 +10,10 @@ import { createRequire } from "node:module"; import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs"; import { isSkillSyncCheckCached } from "./sync-fusion-skill-tools.mjs"; import { computeContentHash, createRepoContentSnapshot } from "./lib/content-hash.mjs"; +import { deriveBudgetMs, runWithWatchdog } from "./lib/run-vitest-watchdog.mjs"; + +/** Generous local full-suite budget (60min): far above a real full run, far below an infinite hang. */ +const FULL_SUITE_BUDGET_MS = 60 * 60 * 1000; const currentFilePath = fileURLToPath(import.meta.url); const scriptDir = path.dirname(currentFilePath); @@ -198,13 +202,38 @@ export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES) { } } -function runMaybeIsolated(command, commandArgs, options = {}) { +// Run a test invocation under the L2 wall-clock watchdog (async). Throws on +// failure/timeout/signal with an `.exitCode` — same shape as `run` — so the +// caller's catch and the `finally` cleanup below behave identically. On a +// watchdog kill the child group is already dead, and the `finally` prune + +// isolation post-check then reap any leaked isolated HOME (no leak slips past +// the guard). +async function runWatchedTest(command, commandArgs, { env, budgetMs, label } = {}) { + const { code, signal, timedOut } = await runWithWatchdog({ + command, + args: commandArgs, + env: env ?? process.env, + budgetMs, + label: label ?? `${command} ${commandArgs.join(" ")}`, + log: console.error, + spawn, + }); + if (timedOut || signal || code !== 0) { + const reason = timedOut ? "watchdog timeout" : signal ? `signal ${signal}` : `exit code ${code}`; + const error = new Error(`${command} ${commandArgs.join(" ")} failed (${reason})`); + error.exitCode = timedOut ? 124 : signal ? 1 : code ?? 1; + throw error; + } +} + +async function runMaybeIsolated(command, commandArgs, options = {}) { const enabled = shouldRunIsolationGuard(); const env = options.env ?? process.env; - const { onBeforeAfterCheck, ...spawnOptions } = options; + const { onBeforeAfterCheck, budgetMs, label, ...spawnOptions } = options; + void spawnOptions; // cwd/stdio defaults live in the watchdog/spawn path now if (enabled) runIsolationCheck(true, env, /* fastBefore */ true); try { - run(command, commandArgs, spawnOptions); + await runWatchedTest(command, commandArgs, { env, budgetMs, label }); } finally { if (typeof onBeforeAfterCheck === "function") { onBeforeAfterCheck(); @@ -1071,7 +1100,7 @@ export function normalizeForwardedArgs(argv) { return normalized; } -export function main(argv = process.argv.slice(2)) { +export async function main(argv = process.argv.slice(2)) { // The full suite is explicit opt-in ONLY (--full / FUSION_TEST_FULL=1). // CI no longer routes through this script (the gate job runs `pnpm // test:gate`; the demoted tier runs `test:ci:shard` in full-suite.yml), so @@ -1202,9 +1231,11 @@ export function main(argv = process.argv.slice(2)) { if (plan.mode === "full") { // Explicit opt-in only ("forced": --full / FUSION_TEST_FULL=1). - runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { + await runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { env: isolatedHomeEnv, onBeforeAfterCheck: cleanupIsolatedHome, + budgetMs: FULL_SUITE_BUDGET_MS, + label: "test:full (-r)", }); return; } @@ -1223,9 +1254,11 @@ export function main(argv = process.argv.slice(2)) { } console.log("[test-changed] need the full sweep instead? run `pnpm test:full` (explicit opt-in)."); - runMaybeIsolated("pnpm", ["test:gate"], { + await runMaybeIsolated("pnpm", ["test:gate"], { env: isolatedHomeEnv, onBeforeAfterCheck: cleanupIsolatedHome, + budgetMs: deriveBudgetMs({ klass: "changed" }), + label: "test:gate", }); return; } @@ -1238,7 +1271,11 @@ export function main(argv = process.argv.slice(2)) { // Run the gate under the same isolation guard as the affected set — a gate // suite leak must trip the checker, not silently become the "before" state // of the later run. - runMaybeIsolated("pnpm", ["test:gate"], { env: isolatedHomeEnv }); + await runMaybeIsolated("pnpm", ["test:gate"], { + env: isolatedHomeEnv, + budgetMs: deriveBudgetMs({ klass: "changed" }), + label: "test:gate (pre-affected)", + }); const filterArgs = activePackages.flatMap((pkg) => ["--filter", pkg]); console.log(`[test-changed] running tests for changed packages: ${activePackages.join(", ")}`); @@ -1246,9 +1283,14 @@ export function main(argv = process.argv.slice(2)) { console.log(`[test-changed] skipping cached packages: ${cachedPackages.join(", ")}`); } - runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { + await runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { env: isolatedHomeEnv, onBeforeAfterCheck: cleanupIsolatedHome, + // Affected sets can include dashboard (13 inner-watchdog'd lanes); use the + // generous full-suite backstop rather than the tight changed ceiling so a + // legitimately long local run is never false-killed. + budgetMs: FULL_SUITE_BUDGET_MS, + label: `affected: ${activePackages.join(", ")}`, }); // Tests passed — record in cache (never cache failures; process.exit on failure above). @@ -1264,12 +1306,11 @@ export function main(argv = process.argv.slice(2)) { } if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) { - try { - main(); - } catch (error) { + main().catch((error) => { if (error?.exitCode) { process.exit(error.exitCode); } - throw error; - } + console.error(error); + process.exit(1); + }); }