f0fa75b270
- align triage prompt templates on impacted-first verification - guard verification max lifetime when timeout is disabled - share idle semaphore leak recovery across scheduler and triage Fusion-Task-Id: FN-6043
681 lines
24 KiB
TypeScript
681 lines
24 KiB
TypeScript
/**
|
||
* fn_run_verification — a custom executor tool that wraps test/lint/build/typecheck
|
||
* commands with heartbeat protection and timeout safety rails.
|
||
*
|
||
* Problem this solves: agents running `pnpm test` from an unbootstrapped workspace
|
||
* root can sit silently for 20+ minutes, tripping the stuck-task-detector's
|
||
* inactivity watchdog and killing the session. This tool:
|
||
*
|
||
* - Streams stdout/stderr line-by-line and fires a heartbeat on every line so
|
||
* the watchdog sees continuous activity.
|
||
* - Emits a synthetic heartbeat every 60s even when the command is quiet.
|
||
* - Enforces a configurable hard timeout with SIGTERM → SIGKILL escalation.
|
||
* - Auto-detects a missing bootstrap (node_modules/.modules.yaml) and prepends
|
||
* a `pnpm install --prefer-offline` when the command is package-scoped.
|
||
* - Caps captured output at 200 KB, keeping head + tail on overflow.
|
||
*
|
||
* The core logic is in `runVerificationCommand` which is exported for unit-testing
|
||
* without a full agent session.
|
||
*/
|
||
|
||
import { superviseSpawn, type SupervisedChild } from "@fusion/core";
|
||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||
import { isAbsolute, join, relative } from "node:path";
|
||
import { Type, type Static } from "@earendil-works/pi-ai";
|
||
import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
|
||
import { executorLog } from "./logger.js";
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Constants
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const MAX_OUTPUT_BYTES = 200 * 1024; // 200 KB
|
||
const QUIET_HEARTBEAT_INTERVAL_MS = 60_000; // emit synthetic heartbeat after 60s silence
|
||
const SIGKILL_GRACE_MS = 10_000;
|
||
const DEFAULT_TIMEOUT_PACKAGE_SEC = 300;
|
||
const DEFAULT_TIMEOUT_WORKSPACE_SEC = 900;
|
||
const MAX_TIMEOUT_SEC = 1800;
|
||
|
||
const packageDirCache = new Map<string, string | null>();
|
||
|
||
function shellSplit(input: string): string[] | null {
|
||
const tokens: string[] = [];
|
||
let current = "";
|
||
let quote: "'" | "\"" | null = null;
|
||
let escaped = false;
|
||
|
||
for (const char of input) {
|
||
if (escaped) {
|
||
current += char;
|
||
escaped = false;
|
||
continue;
|
||
}
|
||
if (char === "\\" && quote !== "'") {
|
||
escaped = true;
|
||
continue;
|
||
}
|
||
if (quote) {
|
||
if (char === quote) {
|
||
quote = null;
|
||
} else {
|
||
current += char;
|
||
}
|
||
continue;
|
||
}
|
||
if (char === "'" || char === "\"") {
|
||
quote = char;
|
||
continue;
|
||
}
|
||
if (/\s/.test(char)) {
|
||
if (current.length > 0) {
|
||
tokens.push(current);
|
||
current = "";
|
||
}
|
||
continue;
|
||
}
|
||
current += char;
|
||
}
|
||
|
||
if (escaped) current += "\\";
|
||
if (quote !== null) return null;
|
||
if (current.length > 0) tokens.push(current);
|
||
return tokens;
|
||
}
|
||
|
||
function shellQuote(value: string): string {
|
||
if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(value)) return value;
|
||
return `'${value.replace(/'/g, "'\\''")}'`;
|
||
}
|
||
|
||
function findWorkspacePackageDir(rootDir: string, packageName: string): string | null {
|
||
const cacheKey = `${rootDir}\0${packageName}`;
|
||
if (packageDirCache.has(cacheKey)) return packageDirCache.get(cacheKey) ?? null;
|
||
|
||
const lastSegment = packageName.split("/").pop();
|
||
const candidates = [
|
||
lastSegment ? `packages/${lastSegment}` : "",
|
||
lastSegment === "fusion" ? "packages/cli" : "",
|
||
].filter(Boolean);
|
||
|
||
for (const candidate of candidates) {
|
||
if (!candidate) continue;
|
||
try {
|
||
const pkg = JSON.parse(readFileSync(join(rootDir, candidate, "package.json"), "utf8")) as { name?: string };
|
||
if (pkg.name === packageName) {
|
||
packageDirCache.set(cacheKey, candidate);
|
||
return candidate;
|
||
}
|
||
} catch {
|
||
// Keep looking.
|
||
}
|
||
}
|
||
|
||
const queue: Array<{ dir: string; depth: number }> = [
|
||
{ dir: "packages", depth: 0 },
|
||
{ dir: "plugins", depth: 0 },
|
||
];
|
||
while (queue.length > 0) {
|
||
const current = queue.shift();
|
||
if (!current) break;
|
||
const abs = join(rootDir, current.dir);
|
||
let entries: import("node:fs").Dirent[];
|
||
try {
|
||
entries = readdirSync(abs, { withFileTypes: true });
|
||
} catch {
|
||
continue;
|
||
}
|
||
for (const entry of entries) {
|
||
if (!entry.isDirectory()) continue;
|
||
if (entry.name === "node_modules" || entry.name === "dist") continue;
|
||
const child = join(current.dir, entry.name);
|
||
try {
|
||
const pkg = JSON.parse(readFileSync(join(rootDir, child, "package.json"), "utf8")) as { name?: string };
|
||
if (pkg.name === packageName) {
|
||
packageDirCache.set(cacheKey, child);
|
||
return child;
|
||
}
|
||
} catch {
|
||
if (current.depth < 3) queue.push({ dir: child, depth: current.depth + 1 });
|
||
}
|
||
}
|
||
}
|
||
|
||
packageDirCache.set(cacheKey, null);
|
||
return null;
|
||
}
|
||
|
||
function toPackageRelativeFilter(token: string, rootDir: string, packageDir: string): string {
|
||
const normalizedPackageDir = packageDir.replace(/\\/g, "/");
|
||
const normalized = token.replace(/\\/g, "/").replace(/^\.\//, "");
|
||
|
||
if (normalized.startsWith(`${normalizedPackageDir}/`)) {
|
||
return normalized.slice(normalizedPackageDir.length + 1);
|
||
}
|
||
|
||
if (isAbsolute(token)) {
|
||
const rel = relative(join(rootDir, packageDir), token).replace(/\\/g, "/");
|
||
if (!rel.startsWith("../") && rel !== "..") return rel;
|
||
}
|
||
|
||
return token;
|
||
}
|
||
|
||
export function normalizeVerificationCommand(command: string, rootDir: string): { command: string; warnings: string[] } {
|
||
const tokens = shellSplit(command);
|
||
const warnings: string[] = [];
|
||
if (!tokens) return { command, warnings };
|
||
if (tokens[0] !== "pnpm") return { command, warnings };
|
||
|
||
const filterIndex = tokens.findIndex((token) => token === "--filter" || token === "-F");
|
||
if (filterIndex < 0 || !tokens[filterIndex + 1]) return { command, warnings };
|
||
const packageName = tokens[filterIndex + 1]!;
|
||
const separatorIndex = tokens.indexOf("--");
|
||
if (separatorIndex < 0) return { command, warnings };
|
||
|
||
const scriptTokens = tokens.slice(filterIndex + 2, separatorIndex);
|
||
const runsTestScript =
|
||
scriptTokens.includes("test")
|
||
|| (scriptTokens[0] === "run" && scriptTokens[1] === "test");
|
||
if (!runsTestScript) return { command, warnings };
|
||
|
||
const forwarded = tokens.slice(separatorIndex + 1);
|
||
if (!forwarded.includes("--run")) return { command, warnings };
|
||
|
||
const packageDir = findWorkspacePackageDir(rootDir, packageName);
|
||
if (!packageDir) return { command, warnings };
|
||
|
||
const vitestArgs = forwarded
|
||
.filter((token) => token !== "--run")
|
||
.map((token) => (
|
||
token.startsWith("-")
|
||
? token
|
||
: toPackageRelativeFilter(token, rootDir, packageDir)
|
||
));
|
||
|
||
const hasReporter = vitestArgs.some((token) => token === "--reporter" || token.startsWith("--reporter="));
|
||
const hasSilent = vitestArgs.some((token) => token === "--silent" || token.startsWith("--silent="));
|
||
const pnpmGlobalFlags = tokens.slice(1, filterIndex);
|
||
const normalizedTokens = [
|
||
"pnpm",
|
||
...pnpmGlobalFlags,
|
||
"--filter",
|
||
packageName,
|
||
"exec",
|
||
"vitest",
|
||
"run",
|
||
...vitestArgs,
|
||
...(hasSilent ? [] : ["--silent=passed-only"]),
|
||
...(hasReporter ? [] : ["--reporter=dot"]),
|
||
];
|
||
const normalizedCommand = normalizedTokens.map(shellQuote).join(" ");
|
||
|
||
if (normalizedCommand !== command) {
|
||
warnings.push(
|
||
"rewrote package test file filter to direct vitest execution so package test scripts do not expand into broad quality suites",
|
||
);
|
||
}
|
||
|
||
return { command: normalizedCommand, warnings };
|
||
}
|
||
|
||
function killVerificationProcess(supervised: SupervisedChild, signal: NodeJS.Signals): void {
|
||
supervised.kill(signal);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Tool parameter schema
|
||
// ---------------------------------------------------------------------------
|
||
|
||
export const runVerificationParams = Type.Object({
|
||
command: Type.String({
|
||
description:
|
||
"The shell command to run, e.g. \"pnpm --filter @fusion/droid-cli test\", \"pnpm lint\", \"pnpm build\"",
|
||
}),
|
||
cwd: Type.Optional(
|
||
Type.String({
|
||
description:
|
||
"Working directory for the command. Defaults to the task worktree root if omitted or relative.",
|
||
}),
|
||
),
|
||
scope: Type.Union(
|
||
[Type.Literal("package"), Type.Literal("workspace")],
|
||
{
|
||
description:
|
||
"\"package\" for scoped commands like `pnpm --filter <pkg>`, \"workspace\" for root-level commands like `pnpm test`.",
|
||
},
|
||
),
|
||
timeoutSec: Type.Optional(
|
||
Type.Number({
|
||
description:
|
||
"Override the default timeout in seconds. Default: 300 for package scope, 900 for workspace scope. Hard cap: 1800.",
|
||
}),
|
||
),
|
||
expectFailure: Type.Optional(
|
||
Type.Boolean({
|
||
description:
|
||
"If true, a non-zero exit code is reported but not flagged as an error. Default: false.",
|
||
}),
|
||
),
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Result type
|
||
// ---------------------------------------------------------------------------
|
||
|
||
export interface VerificationResult {
|
||
success: boolean;
|
||
exitCode: number | null;
|
||
durationMs: number;
|
||
stdout: string;
|
||
stderr: string;
|
||
timedOut: boolean;
|
||
killed: boolean;
|
||
command: string;
|
||
cwd: string;
|
||
warnings: string[];
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Output buffer helper — keeps head + tail within the byte cap
|
||
//
|
||
// Stores head/tail as chunk arrays rather than concatenated strings.
|
||
// The previous implementation re-encoded the entire ~100 KB tail through
|
||
// `Buffer.from(...).subarray(...).toString()` on *every* appended line once
|
||
// output crossed MAX_OUTPUT_BYTES — for a `pnpm test` run dumping 50k lines
|
||
// that produced gigabytes of GC churn and stalled the dashboard event loop.
|
||
// Now we just push chunks and only compact the tail when its byte size grows
|
||
// past 2× the cap, making the amortized cost per append O(1).
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface OutputBuffer {
|
||
headChunks: string[];
|
||
headBytes: number;
|
||
tailChunks: string[];
|
||
tailBytes: number;
|
||
totalBytes: number;
|
||
}
|
||
|
||
function createBuffer(): OutputBuffer {
|
||
return { headChunks: [], headBytes: 0, tailChunks: [], tailBytes: 0, totalBytes: 0 };
|
||
}
|
||
|
||
function appendToBuffer(buf: OutputBuffer, chunk: string): void {
|
||
const chunkBytes = Buffer.byteLength(chunk, "utf8");
|
||
buf.totalBytes += chunkBytes;
|
||
|
||
if (buf.headBytes + chunkBytes <= MAX_OUTPUT_BYTES) {
|
||
buf.headChunks.push(chunk);
|
||
buf.headBytes += chunkBytes;
|
||
return;
|
||
}
|
||
|
||
// Overflow: funnel into tail. Keep at most half the cap in tail, but only
|
||
// compact when we're well over so per-line cost stays amortized O(1).
|
||
const tailCap = MAX_OUTPUT_BYTES / 2;
|
||
buf.tailChunks.push(chunk);
|
||
buf.tailBytes += chunkBytes;
|
||
if (buf.tailBytes > tailCap * 2) {
|
||
// Drop oldest chunks until under the cap.
|
||
while (buf.tailChunks.length > 1 && buf.tailBytes - Buffer.byteLength(buf.tailChunks[0], "utf8") >= tailCap) {
|
||
const dropped = buf.tailChunks.shift() as string;
|
||
buf.tailBytes -= Buffer.byteLength(dropped, "utf8");
|
||
}
|
||
}
|
||
}
|
||
|
||
function flattenBuffer(buf: OutputBuffer): string {
|
||
const head = buf.headChunks.join("");
|
||
if (buf.tailChunks.length === 0) return head;
|
||
const tail = buf.tailChunks.join("");
|
||
return (
|
||
head +
|
||
`\n\n[... output truncated — ${buf.totalBytes} bytes total, showing head + tail ...]\n\n` +
|
||
tail
|
||
);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Core logic (exported for unit testing)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
export interface RunVerificationOptions {
|
||
command: string;
|
||
cwd: string;
|
||
timeoutMs: number;
|
||
expectFailure?: boolean;
|
||
onHeartbeat: () => void;
|
||
onLine?: (line: string) => void;
|
||
}
|
||
|
||
/**
|
||
* Spawns a shell command with heartbeat protection, quiet-interval synthetic
|
||
* heartbeats, and hard timeout enforcement.
|
||
*
|
||
* Exported so tests can exercise the core logic without a full agent session.
|
||
*/
|
||
export async function runVerificationCommand(
|
||
opts: RunVerificationOptions,
|
||
): Promise<VerificationResult> {
|
||
const { command, cwd, timeoutMs, expectFailure = false, onHeartbeat, onLine } = opts;
|
||
const startMs = Date.now();
|
||
const warnings: string[] = [];
|
||
|
||
const stdoutBuf = createBuffer();
|
||
const stderrBuf = createBuffer();
|
||
|
||
return new Promise<VerificationResult>((resolve) => {
|
||
const supervised = superviseSpawn(command, [], {
|
||
cwd,
|
||
stdio: ["ignore", "pipe", "pipe"],
|
||
env: {
|
||
...process.env,
|
||
// Corepack otherwise prompts interactively before fetching a pinned
|
||
// packageManager version, which hangs the non-TTY child until the
|
||
// hard timeout. Disable the prompt so it proceeds (or errors fast).
|
||
COREPACK_ENABLE_DOWNLOAD_PROMPT: "0",
|
||
},
|
||
shell: true,
|
||
killGraceMs: SIGKILL_GRACE_MS,
|
||
maxLifetimeMs: timeoutMs > 0 ? timeoutMs + SIGKILL_GRACE_MS + 1_000 : undefined,
|
||
});
|
||
const child = supervised.child;
|
||
|
||
let timedOut = false;
|
||
let killed = false;
|
||
let settled = false;
|
||
|
||
// ── Quiet-interval synthetic heartbeat ──────────────────────────────────
|
||
let lastLineMs = Date.now();
|
||
const quietTimer = setInterval(() => {
|
||
const silenceMs = Date.now() - lastLineMs;
|
||
if (silenceMs >= QUIET_HEARTBEAT_INTERVAL_MS) {
|
||
executorLog.log(
|
||
`[fn_run_verification] command quiet for ${Math.round(silenceMs / 1000)}s, still running... (${command})`,
|
||
);
|
||
onHeartbeat();
|
||
}
|
||
}, QUIET_HEARTBEAT_INTERVAL_MS);
|
||
|
||
// ── Hard timeout ────────────────────────────────────────────────────────
|
||
let killTimer: ReturnType<typeof setTimeout> | null = null;
|
||
const hardTimer = setTimeout(() => {
|
||
if (settled) return;
|
||
timedOut = true;
|
||
executorLog.warn(
|
||
`[fn_run_verification] hard timeout (${timeoutMs / 1000}s) — sending SIGTERM to: ${command}`,
|
||
);
|
||
killVerificationProcess(supervised, "SIGTERM");
|
||
|
||
killTimer = setTimeout(() => {
|
||
if (!settled) {
|
||
executorLog.warn(
|
||
`[fn_run_verification] SIGTERM ignored — sending SIGKILL to: ${command}`,
|
||
);
|
||
killVerificationProcess(supervised, "SIGKILL");
|
||
killed = true;
|
||
}
|
||
}, SIGKILL_GRACE_MS);
|
||
}, timeoutMs);
|
||
|
||
// ── stdout ───────────────────────────────────────────────────────────────
|
||
let stdoutRemainder = "";
|
||
child.stdout?.on("data", (chunk: Buffer) => {
|
||
const text = stdoutRemainder + chunk.toString("utf8");
|
||
const lines = text.split("\n");
|
||
stdoutRemainder = lines.pop() ?? "";
|
||
for (const line of lines) {
|
||
const lineWithNewline = line + "\n";
|
||
appendToBuffer(stdoutBuf, lineWithNewline);
|
||
lastLineMs = Date.now();
|
||
onHeartbeat();
|
||
onLine?.(lineWithNewline);
|
||
}
|
||
});
|
||
|
||
// ── stderr ───────────────────────────────────────────────────────────────
|
||
let stderrRemainder = "";
|
||
child.stderr?.on("data", (chunk: Buffer) => {
|
||
const text = stderrRemainder + chunk.toString("utf8");
|
||
const lines = text.split("\n");
|
||
stderrRemainder = lines.pop() ?? "";
|
||
for (const line of lines) {
|
||
const lineWithNewline = line + "\n";
|
||
appendToBuffer(stderrBuf, lineWithNewline);
|
||
lastLineMs = Date.now();
|
||
onHeartbeat();
|
||
onLine?.(lineWithNewline);
|
||
}
|
||
});
|
||
|
||
// ── Process exit ─────────────────────────────────────────────────────────
|
||
child.on("close", (code, signal) => {
|
||
if (settled) return;
|
||
settled = true;
|
||
clearInterval(quietTimer);
|
||
clearTimeout(hardTimer);
|
||
if (killTimer) clearTimeout(killTimer);
|
||
|
||
// Flush remainders
|
||
if (stdoutRemainder) appendToBuffer(stdoutBuf, stdoutRemainder);
|
||
if (stderrRemainder) appendToBuffer(stderrBuf, stderrRemainder);
|
||
|
||
const exitCode = code ?? null;
|
||
const durationMs = Date.now() - startMs;
|
||
const zeroExit = exitCode === 0;
|
||
const success = expectFailure ? true : zeroExit;
|
||
|
||
if (!success && !timedOut) {
|
||
executorLog.warn(
|
||
`[fn_run_verification] command failed (exit=${exitCode}, signal=${signal ?? "none"}): ${command}`,
|
||
);
|
||
}
|
||
|
||
resolve({
|
||
success,
|
||
exitCode,
|
||
durationMs,
|
||
stdout: flattenBuffer(stdoutBuf),
|
||
stderr: flattenBuffer(stderrBuf),
|
||
timedOut,
|
||
killed,
|
||
command,
|
||
cwd,
|
||
warnings,
|
||
});
|
||
});
|
||
|
||
child.on("error", (err) => {
|
||
if (settled) return;
|
||
settled = true;
|
||
clearInterval(quietTimer);
|
||
clearTimeout(hardTimer);
|
||
if (killTimer) clearTimeout(killTimer);
|
||
const durationMs = Date.now() - startMs;
|
||
warnings.push(`Spawn error: ${err.message}`);
|
||
resolve({
|
||
success: false,
|
||
exitCode: null,
|
||
durationMs,
|
||
stdout: flattenBuffer(stdoutBuf),
|
||
stderr: flattenBuffer(stderrBuf) + `\nSpawn error: ${err.message}`,
|
||
timedOut: false,
|
||
killed: false,
|
||
command,
|
||
cwd,
|
||
warnings,
|
||
});
|
||
});
|
||
});
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Tool factory
|
||
// ---------------------------------------------------------------------------
|
||
|
||
export interface CreateRunVerificationToolOpts {
|
||
/** Root of the task's git worktree — used as the default cwd. */
|
||
worktreePath: string;
|
||
/** Repo root — used to check node_modules/.modules.yaml for bootstrap detection. */
|
||
rootDir: string;
|
||
taskId: string;
|
||
/** Called on every output line AND on synthetic quiet-interval heartbeats. */
|
||
recordActivity: () => void;
|
||
log: {
|
||
info: (s: string) => void;
|
||
warn: (s: string) => void;
|
||
error: (s: string) => void;
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Build the `fn_run_verification` custom tool for the executor agent.
|
||
*
|
||
* Wire this into the `customTools` array alongside `createTaskDoneTool`.
|
||
* Pass `recordActivity: () => stuckDetector?.recordActivity(task.id)`.
|
||
*/
|
||
export function createRunVerificationTool(
|
||
opts: CreateRunVerificationToolOpts,
|
||
): ToolDefinition {
|
||
const { worktreePath, rootDir, taskId, recordActivity, log } = opts;
|
||
|
||
return {
|
||
name: "fn_run_verification",
|
||
label: "Run Verification",
|
||
description:
|
||
"Run a verification command (tests, lint, build, typecheck) with timeout and progress " +
|
||
"heartbeat protection. Use this instead of bash for any pnpm/npm test/lint/build commands. " +
|
||
"Prevents the inactivity watchdog from killing your session during long compiles.",
|
||
parameters: runVerificationParams,
|
||
execute: async (
|
||
_toolCallId: string,
|
||
params: Static<typeof runVerificationParams>,
|
||
) => {
|
||
const { command, scope, expectFailure = false } = params;
|
||
const warnings: string[] = [];
|
||
|
||
// ── Scope / command mismatch warning ─────────────────────────────────
|
||
if (scope === "workspace" && command.trimStart().startsWith("pnpm --filter")) {
|
||
const msg =
|
||
"scope is \"workspace\" but command starts with \"pnpm --filter\" — " +
|
||
"consider using scope=\"package\" for scoped commands.";
|
||
warnings.push(msg);
|
||
log.warn(`[fn_run_verification] ${taskId}: ${msg}`);
|
||
}
|
||
|
||
// ── Resolve cwd ───────────────────────────────────────────────────────
|
||
let resolvedCwd: string;
|
||
if (params.cwd && isAbsolute(params.cwd)) {
|
||
resolvedCwd = params.cwd;
|
||
} else if (params.cwd) {
|
||
resolvedCwd = join(worktreePath, params.cwd);
|
||
} else {
|
||
resolvedCwd = worktreePath;
|
||
}
|
||
|
||
// ── Resolve timeout ───────────────────────────────────────────────────
|
||
const defaultTimeoutSec =
|
||
scope === "package"
|
||
? DEFAULT_TIMEOUT_PACKAGE_SEC
|
||
: DEFAULT_TIMEOUT_WORKSPACE_SEC;
|
||
const rawTimeoutSec = params.timeoutSec ?? defaultTimeoutSec;
|
||
const timeoutSec = Math.min(rawTimeoutSec, MAX_TIMEOUT_SEC);
|
||
const timeoutMs = timeoutSec * 1000;
|
||
|
||
if (rawTimeoutSec > MAX_TIMEOUT_SEC) {
|
||
const msg = `timeoutSec ${rawTimeoutSec} exceeds hard cap of ${MAX_TIMEOUT_SEC}s — clamped.`;
|
||
warnings.push(msg);
|
||
log.warn(`[fn_run_verification] ${taskId}: ${msg}`);
|
||
}
|
||
|
||
// ── Bootstrap detection ───────────────────────────────────────────────
|
||
// If the command is package-scoped and the workspace has no .modules.yaml,
|
||
// prepend a pnpm install so the agent doesn't stall on missing node_modules.
|
||
let effectiveCommand = command;
|
||
const normalized = normalizeVerificationCommand(effectiveCommand, rootDir);
|
||
if (normalized.command !== effectiveCommand) {
|
||
effectiveCommand = normalized.command;
|
||
warnings.push(...normalized.warnings);
|
||
}
|
||
|
||
if (effectiveCommand.trimStart().startsWith("pnpm --filter")) {
|
||
const modulesYaml = join(rootDir, "node_modules", ".modules.yaml");
|
||
if (!existsSync(modulesYaml)) {
|
||
const installCmd = "pnpm install --prefer-offline";
|
||
const msg =
|
||
`node_modules/.modules.yaml not found in workspace root — ` +
|
||
`auto-prepending \`${installCmd}\` before running the command.`;
|
||
warnings.push(msg);
|
||
log.warn(`[fn_run_verification] ${taskId}: ${msg}`);
|
||
effectiveCommand = `${installCmd} && ${effectiveCommand}`;
|
||
}
|
||
}
|
||
|
||
log.info(
|
||
`[fn_run_verification] ${taskId}: scope=${scope} timeout=${timeoutSec}s cwd=${resolvedCwd} cmd=${effectiveCommand}`,
|
||
);
|
||
|
||
// ── Run ───────────────────────────────────────────────────────────────
|
||
const result = await runVerificationCommand({
|
||
command: effectiveCommand,
|
||
cwd: resolvedCwd,
|
||
timeoutMs,
|
||
expectFailure,
|
||
onHeartbeat: recordActivity,
|
||
});
|
||
|
||
// ── Merge warnings from auto-bootstrap / scope check ─────────────────
|
||
const allWarnings = [...warnings, ...result.warnings];
|
||
|
||
// ── Build the tool response text ──────────────────────────────────────
|
||
const lines: string[] = [];
|
||
|
||
if (allWarnings.length > 0) {
|
||
lines.push(`Warnings:\n${allWarnings.map((w) => ` - ${w}`).join("\n")}\n`);
|
||
}
|
||
|
||
if (result.timedOut) {
|
||
lines.push(
|
||
`Command timed out after ${timeoutSec}s and was ${result.killed ? "killed (SIGKILL)" : "terminated (SIGTERM)"}.\n`,
|
||
);
|
||
}
|
||
|
||
lines.push(`Exit code: ${result.exitCode ?? "null (signal)"}`);
|
||
lines.push(`Duration: ${(result.durationMs / 1000).toFixed(1)}s`);
|
||
lines.push(`Success: ${result.success}`);
|
||
|
||
if (result.stdout.length > 0) {
|
||
lines.push(`\n--- stdout ---\n${result.stdout}`);
|
||
}
|
||
if (result.stderr.length > 0) {
|
||
lines.push(`\n--- stderr ---\n${result.stderr}`);
|
||
}
|
||
|
||
if (result.timedOut) {
|
||
lines.push(
|
||
"\nDo NOT blindly retry — investigate whether subprocesses are hung, " +
|
||
"test loops are infinite, or dependencies are missing.",
|
||
);
|
||
}
|
||
|
||
const text = lines.join("\n");
|
||
|
||
log.info(
|
||
`[fn_run_verification] ${taskId}: done exit=${result.exitCode} duration=${result.durationMs}ms success=${result.success}`,
|
||
);
|
||
|
||
return {
|
||
content: [{ type: "text" as const, text }],
|
||
details: {
|
||
success: result.success,
|
||
exitCode: result.exitCode,
|
||
durationMs: result.durationMs,
|
||
timedOut: result.timedOut,
|
||
killed: result.killed,
|
||
command: result.command,
|
||
cwd: result.cwd,
|
||
},
|
||
};
|
||
},
|
||
};
|
||
}
|