feat(test-infra): run changed-file test invocations under the watchdog (U1)

Convert test-changed.mjs's invocation path from blocking spawnSync to async
spawn through the shared watchdog. runWatchedTest throws on failure/timeout with
.exitCode so the existing catch and the runMaybeIsolated finally (prune +
isolation post-check) still run — a watchdog kill reaps any leaked isolated HOME
instead of leaving it for the guard to flag. Full/affected runs use a generous
60min backstop (dashboard lanes are already inner-watchdog'd); the quick gate
uses the changed-class ceiling.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-06-13 00:43:38 -07:00
parent 7edf24816c
commit 414e1a3bfb

View File

@@ -2,7 +2,7 @@
import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync } from "node:fs";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { spawn, spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import { cpus, tmpdir } from "node:os";
@@ -10,6 +10,10 @@ import { createRequire } from "node:module";
import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs";
import { isSkillSyncCheckCached } from "./sync-fusion-skill-tools.mjs";
import { computeContentHash, createRepoContentSnapshot } from "./lib/content-hash.mjs";
import { deriveBudgetMs, runWithWatchdog } from "./lib/run-vitest-watchdog.mjs";
/** Generous local full-suite budget (60min): far above a real full run, far below an infinite hang. */
const FULL_SUITE_BUDGET_MS = 60 * 60 * 1000;
const currentFilePath = fileURLToPath(import.meta.url);
const scriptDir = path.dirname(currentFilePath);
@@ -198,13 +202,38 @@ export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES) {
}
}
function runMaybeIsolated(command, commandArgs, options = {}) {
// Run a test invocation under the L2 wall-clock watchdog (async). Throws on
// failure/timeout/signal with an `.exitCode` — same shape as `run` — so the
// caller's catch and the `finally` cleanup below behave identically. On a
// watchdog kill the child group is already dead, and the `finally` prune +
// isolation post-check then reap any leaked isolated HOME (no leak slips past
// the guard).
async function runWatchedTest(command, commandArgs, { env, budgetMs, label } = {}) {
const { code, signal, timedOut } = await runWithWatchdog({
command,
args: commandArgs,
env: env ?? process.env,
budgetMs,
label: label ?? `${command} ${commandArgs.join(" ")}`,
log: console.error,
spawn,
});
if (timedOut || signal || code !== 0) {
const reason = timedOut ? "watchdog timeout" : signal ? `signal ${signal}` : `exit code ${code}`;
const error = new Error(`${command} ${commandArgs.join(" ")} failed (${reason})`);
error.exitCode = timedOut ? 124 : signal ? 1 : code ?? 1;
throw error;
}
}
async function runMaybeIsolated(command, commandArgs, options = {}) {
const enabled = shouldRunIsolationGuard();
const env = options.env ?? process.env;
const { onBeforeAfterCheck, ...spawnOptions } = options;
const { onBeforeAfterCheck, budgetMs, label, ...spawnOptions } = options;
void spawnOptions; // cwd/stdio defaults live in the watchdog/spawn path now
if (enabled) runIsolationCheck(true, env, /* fastBefore */ true);
try {
run(command, commandArgs, spawnOptions);
await runWatchedTest(command, commandArgs, { env, budgetMs, label });
} finally {
if (typeof onBeforeAfterCheck === "function") {
onBeforeAfterCheck();
@@ -1071,7 +1100,7 @@ export function normalizeForwardedArgs(argv) {
return normalized;
}
export function main(argv = process.argv.slice(2)) {
export async function main(argv = process.argv.slice(2)) {
// The full suite is explicit opt-in ONLY (--full / FUSION_TEST_FULL=1).
// CI no longer routes through this script (the gate job runs `pnpm
// test:gate`; the demoted tier runs `test:ci:shard` in full-suite.yml), so
@@ -1202,9 +1231,11 @@ export function main(argv = process.argv.slice(2)) {
if (plan.mode === "full") {
// Explicit opt-in only ("forced": --full / FUSION_TEST_FULL=1).
runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
await runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
env: isolatedHomeEnv,
onBeforeAfterCheck: cleanupIsolatedHome,
budgetMs: FULL_SUITE_BUDGET_MS,
label: "test:full (-r)",
});
return;
}
@@ -1223,9 +1254,11 @@ export function main(argv = process.argv.slice(2)) {
}
console.log("[test-changed] need the full sweep instead? run `pnpm test:full` (explicit opt-in).");
runMaybeIsolated("pnpm", ["test:gate"], {
await runMaybeIsolated("pnpm", ["test:gate"], {
env: isolatedHomeEnv,
onBeforeAfterCheck: cleanupIsolatedHome,
budgetMs: deriveBudgetMs({ klass: "changed" }),
label: "test:gate",
});
return;
}
@@ -1238,7 +1271,11 @@ export function main(argv = process.argv.slice(2)) {
// Run the gate under the same isolation guard as the affected set — a gate
// suite leak must trip the checker, not silently become the "before" state
// of the later run.
runMaybeIsolated("pnpm", ["test:gate"], { env: isolatedHomeEnv });
await runMaybeIsolated("pnpm", ["test:gate"], {
env: isolatedHomeEnv,
budgetMs: deriveBudgetMs({ klass: "changed" }),
label: "test:gate (pre-affected)",
});
const filterArgs = activePackages.flatMap((pkg) => ["--filter", pkg]);
console.log(`[test-changed] running tests for changed packages: ${activePackages.join(", ")}`);
@@ -1246,9 +1283,14 @@ export function main(argv = process.argv.slice(2)) {
console.log(`[test-changed] skipping cached packages: ${cachedPackages.join(", ")}`);
}
runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
await runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
env: isolatedHomeEnv,
onBeforeAfterCheck: cleanupIsolatedHome,
// Affected sets can include dashboard (13 inner-watchdog'd lanes); use the
// generous full-suite backstop rather than the tight changed ceiling so a
// legitimately long local run is never false-killed.
budgetMs: FULL_SUITE_BUDGET_MS,
label: `affected: ${activePackages.join(", ")}`,
});
// Tests passed — record in cache (never cache failures; process.exit on failure above).
@@ -1264,12 +1306,11 @@ export function main(argv = process.argv.slice(2)) {
}
if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) {
try {
main();
} catch (error) {
main().catch((error) => {
if (error?.exitCode) {
process.exit(error.exitCode);
}
throw error;
}
console.error(error);
process.exit(1);
});
}