feat(test-infra): run changed-file test invocations under the watchdog (U1)
Convert test-changed.mjs's invocation path from blocking spawnSync to async spawn through the shared watchdog. runWatchedTest throws on failure/timeout with .exitCode so the existing catch and the runMaybeIsolated finally (prune + isolation post-check) still run — a watchdog kill reaps any leaked isolated HOME instead of leaving it for the guard to flag. Full/affected runs use a generous 60min backstop (dashboard lanes are already inner-watchdog'd); the quick gate uses the changed-class ceiling. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
|
||||
import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { cpus, tmpdir } from "node:os";
|
||||
@@ -10,6 +10,10 @@ import { createRequire } from "node:module";
|
||||
import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs";
|
||||
import { isSkillSyncCheckCached } from "./sync-fusion-skill-tools.mjs";
|
||||
import { computeContentHash, createRepoContentSnapshot } from "./lib/content-hash.mjs";
|
||||
import { deriveBudgetMs, runWithWatchdog } from "./lib/run-vitest-watchdog.mjs";
|
||||
|
||||
/** Generous local full-suite budget (60min): far above a real full run, far below an infinite hang. */
|
||||
const FULL_SUITE_BUDGET_MS = 60 * 60 * 1000;
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const scriptDir = path.dirname(currentFilePath);
|
||||
@@ -198,13 +202,38 @@ export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES) {
|
||||
}
|
||||
}
|
||||
|
||||
function runMaybeIsolated(command, commandArgs, options = {}) {
|
||||
// Run a test invocation under the L2 wall-clock watchdog (async). Throws on
|
||||
// failure/timeout/signal with an `.exitCode` — same shape as `run` — so the
|
||||
// caller's catch and the `finally` cleanup below behave identically. On a
|
||||
// watchdog kill the child group is already dead, and the `finally` prune +
|
||||
// isolation post-check then reap any leaked isolated HOME (no leak slips past
|
||||
// the guard).
|
||||
async function runWatchedTest(command, commandArgs, { env, budgetMs, label } = {}) {
|
||||
const { code, signal, timedOut } = await runWithWatchdog({
|
||||
command,
|
||||
args: commandArgs,
|
||||
env: env ?? process.env,
|
||||
budgetMs,
|
||||
label: label ?? `${command} ${commandArgs.join(" ")}`,
|
||||
log: console.error,
|
||||
spawn,
|
||||
});
|
||||
if (timedOut || signal || code !== 0) {
|
||||
const reason = timedOut ? "watchdog timeout" : signal ? `signal ${signal}` : `exit code ${code}`;
|
||||
const error = new Error(`${command} ${commandArgs.join(" ")} failed (${reason})`);
|
||||
error.exitCode = timedOut ? 124 : signal ? 1 : code ?? 1;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runMaybeIsolated(command, commandArgs, options = {}) {
|
||||
const enabled = shouldRunIsolationGuard();
|
||||
const env = options.env ?? process.env;
|
||||
const { onBeforeAfterCheck, ...spawnOptions } = options;
|
||||
const { onBeforeAfterCheck, budgetMs, label, ...spawnOptions } = options;
|
||||
void spawnOptions; // cwd/stdio defaults live in the watchdog/spawn path now
|
||||
if (enabled) runIsolationCheck(true, env, /* fastBefore */ true);
|
||||
try {
|
||||
run(command, commandArgs, spawnOptions);
|
||||
await runWatchedTest(command, commandArgs, { env, budgetMs, label });
|
||||
} finally {
|
||||
if (typeof onBeforeAfterCheck === "function") {
|
||||
onBeforeAfterCheck();
|
||||
@@ -1071,7 +1100,7 @@ export function normalizeForwardedArgs(argv) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
// The full suite is explicit opt-in ONLY (--full / FUSION_TEST_FULL=1).
|
||||
// CI no longer routes through this script (the gate job runs `pnpm
|
||||
// test:gate`; the demoted tier runs `test:ci:shard` in full-suite.yml), so
|
||||
@@ -1202,9 +1231,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
|
||||
if (plan.mode === "full") {
|
||||
// Explicit opt-in only ("forced": --full / FUSION_TEST_FULL=1).
|
||||
runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
await runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
env: isolatedHomeEnv,
|
||||
onBeforeAfterCheck: cleanupIsolatedHome,
|
||||
budgetMs: FULL_SUITE_BUDGET_MS,
|
||||
label: "test:full (-r)",
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -1223,9 +1254,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
}
|
||||
console.log("[test-changed] need the full sweep instead? run `pnpm test:full` (explicit opt-in).");
|
||||
|
||||
runMaybeIsolated("pnpm", ["test:gate"], {
|
||||
await runMaybeIsolated("pnpm", ["test:gate"], {
|
||||
env: isolatedHomeEnv,
|
||||
onBeforeAfterCheck: cleanupIsolatedHome,
|
||||
budgetMs: deriveBudgetMs({ klass: "changed" }),
|
||||
label: "test:gate",
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -1238,7 +1271,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
// Run the gate under the same isolation guard as the affected set — a gate
|
||||
// suite leak must trip the checker, not silently become the "before" state
|
||||
// of the later run.
|
||||
runMaybeIsolated("pnpm", ["test:gate"], { env: isolatedHomeEnv });
|
||||
await runMaybeIsolated("pnpm", ["test:gate"], {
|
||||
env: isolatedHomeEnv,
|
||||
budgetMs: deriveBudgetMs({ klass: "changed" }),
|
||||
label: "test:gate (pre-affected)",
|
||||
});
|
||||
|
||||
const filterArgs = activePackages.flatMap((pkg) => ["--filter", pkg]);
|
||||
console.log(`[test-changed] running tests for changed packages: ${activePackages.join(", ")}`);
|
||||
@@ -1246,9 +1283,14 @@ export function main(argv = process.argv.slice(2)) {
|
||||
console.log(`[test-changed] skipping cached packages: ${cachedPackages.join(", ")}`);
|
||||
}
|
||||
|
||||
runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
await runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
env: isolatedHomeEnv,
|
||||
onBeforeAfterCheck: cleanupIsolatedHome,
|
||||
// Affected sets can include dashboard (13 inner-watchdog'd lanes); use the
|
||||
// generous full-suite backstop rather than the tight changed ceiling so a
|
||||
// legitimately long local run is never false-killed.
|
||||
budgetMs: FULL_SUITE_BUDGET_MS,
|
||||
label: `affected: ${activePackages.join(", ")}`,
|
||||
});
|
||||
|
||||
// Tests passed — record in cache (never cache failures; process.exit on failure above).
|
||||
@@ -1264,12 +1306,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) {
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
main().catch((error) => {
|
||||
if (error?.exitCode) {
|
||||
process.exit(error.exitCode);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user