/** * fn_run_verification — a custom executor tool that wraps test/lint/build/typecheck * commands with heartbeat protection and timeout safety rails. * * Problem this solves: agents running `pnpm test` from an unbootstrapped workspace * root can sit silently for 20+ minutes, tripping the stuck-task-detector's * inactivity watchdog and killing the session. This tool: * * - Streams stdout/stderr line-by-line and fires a heartbeat on every line so * the watchdog sees continuous activity. * - Emits a synthetic heartbeat every 60s even when the command is quiet. * - Enforces a configurable hard timeout with SIGTERM → SIGKILL escalation. * - Auto-detects a missing bootstrap (node_modules/.modules.yaml) and prepends * a `pnpm install --prefer-offline` when the command is package-scoped. * - Caps captured output at 200 KB, keeping head + tail on overflow. * * The core logic is in `runVerificationCommand` which is exported for unit-testing * without a full agent session. */ import { superviseSpawn, type SupervisedChild } from "@fusion/core"; import { existsSync, readdirSync, readFileSync } from "node:fs"; import { isAbsolute, join, relative } from "node:path"; import { Type, type Static } from "@earendil-works/pi-ai"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { executorLog } from "./logger.js"; // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- const MAX_OUTPUT_BYTES = 200 * 1024; // 200 KB const QUIET_HEARTBEAT_INTERVAL_MS = 60_000; // emit synthetic heartbeat after 60s silence const SIGKILL_GRACE_MS = 10_000; const DEFAULT_TIMEOUT_PACKAGE_SEC = 300; const DEFAULT_TIMEOUT_WORKSPACE_SEC = 900; const MAX_TIMEOUT_SEC = 1800; const packageDirCache = new Map(); function shellSplit(input: string): string[] | null { const tokens: string[] = []; let current = ""; let quote: "'" | "\"" | null = null; let escaped = false; for (const char of input) { if (escaped) { current += char; escaped = false; continue; } if (char === "\\" && quote !== "'") { escaped = true; continue; } if (quote) { if (char === quote) { quote = null; } else { current += char; } continue; } if (char === "'" || char === "\"") { quote = char; continue; } if (/\s/.test(char)) { if (current.length > 0) { tokens.push(current); current = ""; } continue; } current += char; } if (escaped) current += "\\"; if (quote !== null) return null; if (current.length > 0) tokens.push(current); return tokens; } function shellQuote(value: string): string { if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(value)) return value; return `'${value.replace(/'/g, "'\\''")}'`; } function findWorkspacePackageDir(rootDir: string, packageName: string): string | null { const cacheKey = `${rootDir}\0${packageName}`; if (packageDirCache.has(cacheKey)) return packageDirCache.get(cacheKey) ?? null; const lastSegment = packageName.split("/").pop(); const candidates = [ lastSegment ? `packages/${lastSegment}` : "", lastSegment === "fusion" ? "packages/cli" : "", ].filter(Boolean); for (const candidate of candidates) { if (!candidate) continue; try { const pkg = JSON.parse(readFileSync(join(rootDir, candidate, "package.json"), "utf8")) as { name?: string }; if (pkg.name === packageName) { packageDirCache.set(cacheKey, candidate); return candidate; } } catch { // Keep looking. } } const queue: Array<{ dir: string; depth: number }> = [ { dir: "packages", depth: 0 }, { dir: "plugins", depth: 0 }, ]; while (queue.length > 0) { const current = queue.shift(); if (!current) break; const abs = join(rootDir, current.dir); let entries: import("node:fs").Dirent[]; try { entries = readdirSync(abs, { withFileTypes: true }); } catch { continue; } for (const entry of entries) { if (!entry.isDirectory()) continue; if (entry.name === "node_modules" || entry.name === "dist") continue; const child = join(current.dir, entry.name); try { const pkg = JSON.parse(readFileSync(join(rootDir, child, "package.json"), "utf8")) as { name?: string }; if (pkg.name === packageName) { packageDirCache.set(cacheKey, child); return child; } } catch { if (current.depth < 3) queue.push({ dir: child, depth: current.depth + 1 }); } } } packageDirCache.set(cacheKey, null); return null; } function toPackageRelativeFilter(token: string, rootDir: string, packageDir: string): string { const normalizedPackageDir = packageDir.replace(/\\/g, "/"); const normalized = token.replace(/\\/g, "/").replace(/^\.\//, ""); if (normalized.startsWith(`${normalizedPackageDir}/`)) { return normalized.slice(normalizedPackageDir.length + 1); } if (isAbsolute(token)) { const rel = relative(join(rootDir, packageDir), token).replace(/\\/g, "/"); if (!rel.startsWith("../") && rel !== "..") return rel; } return token; } export function normalizeVerificationCommand(command: string, rootDir: string): { command: string; warnings: string[] } { const tokens = shellSplit(command); const warnings: string[] = []; if (!tokens) return { command, warnings }; if (tokens[0] !== "pnpm") return { command, warnings }; const filterIndex = tokens.findIndex((token) => token === "--filter" || token === "-F"); if (filterIndex < 0 || !tokens[filterIndex + 1]) return { command, warnings }; const packageName = tokens[filterIndex + 1]!; const separatorIndex = tokens.indexOf("--"); if (separatorIndex < 0) return { command, warnings }; const scriptTokens = tokens.slice(filterIndex + 2, separatorIndex); const runsTestScript = scriptTokens.includes("test") || (scriptTokens[0] === "run" && scriptTokens[1] === "test"); if (!runsTestScript) return { command, warnings }; const forwarded = tokens.slice(separatorIndex + 1); if (!forwarded.includes("--run")) return { command, warnings }; const packageDir = findWorkspacePackageDir(rootDir, packageName); if (!packageDir) return { command, warnings }; const vitestArgs = forwarded .filter((token) => token !== "--run") .map((token) => ( token.startsWith("-") ? token : toPackageRelativeFilter(token, rootDir, packageDir) )); const hasReporter = vitestArgs.some((token) => token === "--reporter" || token.startsWith("--reporter=")); const hasSilent = vitestArgs.some((token) => token === "--silent" || token.startsWith("--silent=")); const pnpmGlobalFlags = tokens.slice(1, filterIndex); const normalizedTokens = [ "pnpm", ...pnpmGlobalFlags, "--filter", packageName, "exec", "vitest", "run", ...vitestArgs, ...(hasSilent ? [] : ["--silent=passed-only"]), ...(hasReporter ? [] : ["--reporter=dot"]), ]; const normalizedCommand = normalizedTokens.map(shellQuote).join(" "); if (normalizedCommand !== command) { warnings.push( "rewrote package test file filter to direct vitest execution so package test scripts do not expand into broad quality suites", ); } return { command: normalizedCommand, warnings }; } function killVerificationProcess(supervised: SupervisedChild, signal: NodeJS.Signals): void { supervised.kill(signal); } // --------------------------------------------------------------------------- // Tool parameter schema // --------------------------------------------------------------------------- export const runVerificationParams = Type.Object({ command: Type.String({ description: "The shell command to run, e.g. \"pnpm --filter @fusion/droid-cli test\", \"pnpm lint\", \"pnpm build\"", }), cwd: Type.Optional( Type.String({ description: "Working directory for the command. Defaults to the task worktree root if omitted or relative.", }), ), scope: Type.Union( [Type.Literal("package"), Type.Literal("workspace")], { description: "\"package\" for scoped commands like `pnpm --filter `, \"workspace\" for root-level commands like `pnpm test`.", }, ), timeoutSec: Type.Optional( Type.Number({ description: "Override the default timeout in seconds. Default: 300 for package scope, 900 for workspace scope. Hard cap: 1800.", }), ), expectFailure: Type.Optional( Type.Boolean({ description: "If true, a non-zero exit code is reported but not flagged as an error. Default: false.", }), ), }); // --------------------------------------------------------------------------- // Result type // --------------------------------------------------------------------------- export interface VerificationResult { success: boolean; exitCode: number | null; durationMs: number; stdout: string; stderr: string; timedOut: boolean; killed: boolean; command: string; cwd: string; warnings: string[]; } // --------------------------------------------------------------------------- // Output buffer helper — keeps head + tail within the byte cap // // Stores head/tail as chunk arrays rather than concatenated strings. // The previous implementation re-encoded the entire ~100 KB tail through // `Buffer.from(...).subarray(...).toString()` on *every* appended line once // output crossed MAX_OUTPUT_BYTES — for a `pnpm test` run dumping 50k lines // that produced gigabytes of GC churn and stalled the dashboard event loop. // Now we just push chunks and only compact the tail when its byte size grows // past 2× the cap, making the amortized cost per append O(1). // --------------------------------------------------------------------------- interface OutputBuffer { headChunks: string[]; headBytes: number; tailChunks: string[]; tailBytes: number; totalBytes: number; } function createBuffer(): OutputBuffer { return { headChunks: [], headBytes: 0, tailChunks: [], tailBytes: 0, totalBytes: 0 }; } function appendToBuffer(buf: OutputBuffer, chunk: string): void { const chunkBytes = Buffer.byteLength(chunk, "utf8"); buf.totalBytes += chunkBytes; if (buf.headBytes + chunkBytes <= MAX_OUTPUT_BYTES) { buf.headChunks.push(chunk); buf.headBytes += chunkBytes; return; } // Overflow: funnel into tail. Keep at most half the cap in tail, but only // compact when we're well over so per-line cost stays amortized O(1). const tailCap = MAX_OUTPUT_BYTES / 2; buf.tailChunks.push(chunk); buf.tailBytes += chunkBytes; if (buf.tailBytes > tailCap * 2) { // Drop oldest chunks until under the cap. while (buf.tailChunks.length > 1 && buf.tailBytes - Buffer.byteLength(buf.tailChunks[0], "utf8") >= tailCap) { const dropped = buf.tailChunks.shift() as string; buf.tailBytes -= Buffer.byteLength(dropped, "utf8"); } } } function flattenBuffer(buf: OutputBuffer): string { const head = buf.headChunks.join(""); if (buf.tailChunks.length === 0) return head; const tail = buf.tailChunks.join(""); return ( head + `\n\n[... output truncated — ${buf.totalBytes} bytes total, showing head + tail ...]\n\n` + tail ); } // --------------------------------------------------------------------------- // Core logic (exported for unit testing) // --------------------------------------------------------------------------- export interface RunVerificationOptions { command: string; cwd: string; timeoutMs: number; expectFailure?: boolean; onHeartbeat: () => void; onLine?: (line: string) => void; } /** * Spawns a shell command with heartbeat protection, quiet-interval synthetic * heartbeats, and hard timeout enforcement. * * Exported so tests can exercise the core logic without a full agent session. */ export async function runVerificationCommand( opts: RunVerificationOptions, ): Promise { const { command, cwd, timeoutMs, expectFailure = false, onHeartbeat, onLine } = opts; const startMs = Date.now(); const warnings: string[] = []; const stdoutBuf = createBuffer(); const stderrBuf = createBuffer(); return new Promise((resolve) => { const supervised = superviseSpawn(command, [], { cwd, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, // Corepack otherwise prompts interactively before fetching a pinned // packageManager version, which hangs the non-TTY child until the // hard timeout. Disable the prompt so it proceeds (or errors fast). COREPACK_ENABLE_DOWNLOAD_PROMPT: "0", }, shell: true, killGraceMs: SIGKILL_GRACE_MS, maxLifetimeMs: timeoutMs > 0 ? timeoutMs + SIGKILL_GRACE_MS + 1_000 : undefined, }); const child = supervised.child; let timedOut = false; let killed = false; let settled = false; // ── Quiet-interval synthetic heartbeat ────────────────────────────────── let lastLineMs = Date.now(); const quietTimer = setInterval(() => { const silenceMs = Date.now() - lastLineMs; if (silenceMs >= QUIET_HEARTBEAT_INTERVAL_MS) { executorLog.log( `[fn_run_verification] command quiet for ${Math.round(silenceMs / 1000)}s, still running... (${command})`, ); onHeartbeat(); } }, QUIET_HEARTBEAT_INTERVAL_MS); // ── Hard timeout ──────────────────────────────────────────────────────── let killTimer: ReturnType | null = null; const hardTimer = setTimeout(() => { if (settled) return; timedOut = true; executorLog.warn( `[fn_run_verification] hard timeout (${timeoutMs / 1000}s) — sending SIGTERM to: ${command}`, ); killVerificationProcess(supervised, "SIGTERM"); killTimer = setTimeout(() => { if (!settled) { executorLog.warn( `[fn_run_verification] SIGTERM ignored — sending SIGKILL to: ${command}`, ); killVerificationProcess(supervised, "SIGKILL"); killed = true; } }, SIGKILL_GRACE_MS); }, timeoutMs); // ── stdout ─────────────────────────────────────────────────────────────── let stdoutRemainder = ""; child.stdout?.on("data", (chunk: Buffer) => { const text = stdoutRemainder + chunk.toString("utf8"); const lines = text.split("\n"); stdoutRemainder = lines.pop() ?? ""; for (const line of lines) { const lineWithNewline = line + "\n"; appendToBuffer(stdoutBuf, lineWithNewline); lastLineMs = Date.now(); onHeartbeat(); onLine?.(lineWithNewline); } }); // ── stderr ─────────────────────────────────────────────────────────────── let stderrRemainder = ""; child.stderr?.on("data", (chunk: Buffer) => { const text = stderrRemainder + chunk.toString("utf8"); const lines = text.split("\n"); stderrRemainder = lines.pop() ?? ""; for (const line of lines) { const lineWithNewline = line + "\n"; appendToBuffer(stderrBuf, lineWithNewline); lastLineMs = Date.now(); onHeartbeat(); onLine?.(lineWithNewline); } }); // ── Process exit ───────────────────────────────────────────────────────── child.on("close", (code, signal) => { if (settled) return; settled = true; clearInterval(quietTimer); clearTimeout(hardTimer); if (killTimer) clearTimeout(killTimer); // Flush remainders if (stdoutRemainder) appendToBuffer(stdoutBuf, stdoutRemainder); if (stderrRemainder) appendToBuffer(stderrBuf, stderrRemainder); const exitCode = code ?? null; const durationMs = Date.now() - startMs; const zeroExit = exitCode === 0; const success = expectFailure ? true : zeroExit; if (!success && !timedOut) { executorLog.warn( `[fn_run_verification] command failed (exit=${exitCode}, signal=${signal ?? "none"}): ${command}`, ); } resolve({ success, exitCode, durationMs, stdout: flattenBuffer(stdoutBuf), stderr: flattenBuffer(stderrBuf), timedOut, killed, command, cwd, warnings, }); }); child.on("error", (err) => { if (settled) return; settled = true; clearInterval(quietTimer); clearTimeout(hardTimer); if (killTimer) clearTimeout(killTimer); const durationMs = Date.now() - startMs; warnings.push(`Spawn error: ${err.message}`); resolve({ success: false, exitCode: null, durationMs, stdout: flattenBuffer(stdoutBuf), stderr: flattenBuffer(stderrBuf) + `\nSpawn error: ${err.message}`, timedOut: false, killed: false, command, cwd, warnings, }); }); }); } // --------------------------------------------------------------------------- // Tool factory // --------------------------------------------------------------------------- export interface CreateRunVerificationToolOpts { /** Root of the task's git worktree — used as the default cwd. */ worktreePath: string; /** Repo root — used to check node_modules/.modules.yaml for bootstrap detection. */ rootDir: string; taskId: string; /** Called on every output line AND on synthetic quiet-interval heartbeats. */ recordActivity: () => void; log: { info: (s: string) => void; warn: (s: string) => void; error: (s: string) => void; }; } /** * Build the `fn_run_verification` custom tool for the executor agent. * * Wire this into the `customTools` array alongside `createTaskDoneTool`. * Pass `recordActivity: () => stuckDetector?.recordActivity(task.id)`. */ export function createRunVerificationTool( opts: CreateRunVerificationToolOpts, ): ToolDefinition { const { worktreePath, rootDir, taskId, recordActivity, log } = opts; return { name: "fn_run_verification", label: "Run Verification", description: "Run a verification command (tests, lint, build, typecheck) with timeout and progress " + "heartbeat protection. Use this instead of bash for any pnpm/npm test/lint/build commands. " + "Prevents the inactivity watchdog from killing your session during long compiles.", parameters: runVerificationParams, execute: async ( _toolCallId: string, params: Static, ) => { const { command, scope, expectFailure = false } = params; const warnings: string[] = []; // ── Scope / command mismatch warning ───────────────────────────────── if (scope === "workspace" && command.trimStart().startsWith("pnpm --filter")) { const msg = "scope is \"workspace\" but command starts with \"pnpm --filter\" — " + "consider using scope=\"package\" for scoped commands."; warnings.push(msg); log.warn(`[fn_run_verification] ${taskId}: ${msg}`); } // ── Resolve cwd ─────────────────────────────────────────────────────── let resolvedCwd: string; if (params.cwd && isAbsolute(params.cwd)) { resolvedCwd = params.cwd; } else if (params.cwd) { resolvedCwd = join(worktreePath, params.cwd); } else { resolvedCwd = worktreePath; } // ── Resolve timeout ─────────────────────────────────────────────────── const defaultTimeoutSec = scope === "package" ? DEFAULT_TIMEOUT_PACKAGE_SEC : DEFAULT_TIMEOUT_WORKSPACE_SEC; const rawTimeoutSec = params.timeoutSec ?? defaultTimeoutSec; const timeoutSec = Math.min(rawTimeoutSec, MAX_TIMEOUT_SEC); const timeoutMs = timeoutSec * 1000; if (rawTimeoutSec > MAX_TIMEOUT_SEC) { const msg = `timeoutSec ${rawTimeoutSec} exceeds hard cap of ${MAX_TIMEOUT_SEC}s — clamped.`; warnings.push(msg); log.warn(`[fn_run_verification] ${taskId}: ${msg}`); } // ── Bootstrap detection ─────────────────────────────────────────────── // If the command is package-scoped and the workspace has no .modules.yaml, // prepend a pnpm install so the agent doesn't stall on missing node_modules. let effectiveCommand = command; const normalized = normalizeVerificationCommand(effectiveCommand, rootDir); if (normalized.command !== effectiveCommand) { effectiveCommand = normalized.command; warnings.push(...normalized.warnings); } if (effectiveCommand.trimStart().startsWith("pnpm --filter")) { const modulesYaml = join(rootDir, "node_modules", ".modules.yaml"); if (!existsSync(modulesYaml)) { const installCmd = "pnpm install --prefer-offline"; const msg = `node_modules/.modules.yaml not found in workspace root — ` + `auto-prepending \`${installCmd}\` before running the command.`; warnings.push(msg); log.warn(`[fn_run_verification] ${taskId}: ${msg}`); effectiveCommand = `${installCmd} && ${effectiveCommand}`; } } log.info( `[fn_run_verification] ${taskId}: scope=${scope} timeout=${timeoutSec}s cwd=${resolvedCwd} cmd=${effectiveCommand}`, ); // ── Run ─────────────────────────────────────────────────────────────── const result = await runVerificationCommand({ command: effectiveCommand, cwd: resolvedCwd, timeoutMs, expectFailure, onHeartbeat: recordActivity, }); // ── Merge warnings from auto-bootstrap / scope check ───────────────── const allWarnings = [...warnings, ...result.warnings]; // ── Build the tool response text ────────────────────────────────────── const lines: string[] = []; if (allWarnings.length > 0) { lines.push(`Warnings:\n${allWarnings.map((w) => ` - ${w}`).join("\n")}\n`); } if (result.timedOut) { lines.push( `Command timed out after ${timeoutSec}s and was ${result.killed ? "killed (SIGKILL)" : "terminated (SIGTERM)"}.\n`, ); } lines.push(`Exit code: ${result.exitCode ?? "null (signal)"}`); lines.push(`Duration: ${(result.durationMs / 1000).toFixed(1)}s`); lines.push(`Success: ${result.success}`); if (result.stdout.length > 0) { lines.push(`\n--- stdout ---\n${result.stdout}`); } if (result.stderr.length > 0) { lines.push(`\n--- stderr ---\n${result.stderr}`); } if (result.timedOut) { lines.push( "\nDo NOT blindly retry — investigate whether subprocesses are hung, " + "test loops are infinite, or dependencies are missing.", ); } const text = lines.join("\n"); log.info( `[fn_run_verification] ${taskId}: done exit=${result.exitCode} duration=${result.durationMs}ms success=${result.success}`, ); return { content: [{ type: "text" as const, text }], details: { success: result.success, exitCode: result.exitCode, durationMs: result.durationMs, timedOut: result.timedOut, killed: result.killed, command: result.command, cwd: result.cwd, }, }; }, }; }