diff --git a/.changeset/compact-verification-failures.md b/.changeset/compact-verification-failures.md new file mode 100644 index 0000000000..c977fcaed9 --- /dev/null +++ b/.changeset/compact-verification-failures.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Keep verification results concise so large failure dumps do not exhaust agent context. +category: fix +dev: Omits routine successful output and extracts bounded high-signal diagnostics from failed verification commands. diff --git a/packages/engine/src/__tests__/run-verification-command.test.ts b/packages/engine/src/__tests__/run-verification-command.test.ts index c6927e79c5..177449c6fe 100644 --- a/packages/engine/src/__tests__/run-verification-command.test.ts +++ b/packages/engine/src/__tests__/run-verification-command.test.ts @@ -9,6 +9,7 @@ import { detectMarathonVerification, normalizeVerificationCommand, runVerificationCommand, + summarizeVerificationFailureOutput, __testOnlyReapVerificationProcessGroup, type RunVerificationOptions, } from "../run-verification-tool.js"; @@ -474,6 +475,210 @@ describe("runVerificationCommand", { timeout: 30000 }, () => { }); }); + describe("tool response output", () => { + const createCompactTool = () => + createRunVerificationTool({ + worktreePath: tempDir, + rootDir: workspaceRoot, + taskId: "FN-COMPACT", + recordActivity: vi.fn(), + onVerificationStart: vi.fn(), + onVerificationEnd: vi.fn(), + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + + it("reduces noisy test failures to counts, failing tests, errors, and source locations", () => { + const stdout = [ + "\u001b[41m FAIL \u001b[0m app/components/Widget.test.tsx > Widget > preserves focus", + "AssertionError: expected false to be true", + "Ignored nodes: comments, script, style", + "", + " ", + "
", + " ", + " DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT", + "
", + " ", + "", + " ❯ app/components/Widget.test.tsx:42:7", + " Test Files 1 failed | 12 passed (13)", + " Tests 1 failed | 650 passed (651)", + ].join("\n"); + + const summary = summarizeVerificationFailureOutput(stdout, ""); + + expect(summary).toContain("FAIL app/components/Widget.test.tsx > Widget > preserves focus"); + expect(summary).toContain("AssertionError: expected false to be true"); + expect(summary).toContain("app/components/Widget.test.tsx:42:7"); + expect(summary).toContain("Test Files 1 failed | 12 passed (13)"); + expect(summary).toContain("Tests 1 failed | 650 passed (651)"); + expect(summary).not.toContain("DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT"); + expect(summary).not.toContain("\u001b["); + }); + + it("keeps compiler diagnostics and caps generic failure output", () => { + const diagnostic = "src/example.ts(12,4): error TS2322: Type 'number' is not assignable to type 'string'."; + const stderr = [ + diagnostic, + ...Array.from( + { length: 200 }, + (_, index) => + `src/example-${index}.ts(12,4): error TS2322: ${"x".repeat(200)}`, + ), + ].join("\n"); + + const summary = summarizeVerificationFailureOutput("", stderr); + + expect(summary).toContain(diagnostic); + expect(summary.length).toBeLessThanOrEqual(8_000); + expect(summary).toContain("output compacted"); + }); + + it("retains actionable lint context ahead of a generic package-manager failure", () => { + const stderr = [ + "/workspace/src/widget.ts", + " 12:3 error Unexpected any. Specify a different type @typescript-eslint/no-explicit-any", + "✖ 1 problem (1 error, 0 warnings)", + "ELIFECYCLE Command failed with exit code 1.", + ].join("\n"); + + const summary = summarizeVerificationFailureOutput("", stderr); + + expect(summary).toContain("/workspace/src/widget.ts"); + expect(summary).toContain("12:3 error Unexpected any"); + expect(summary).toContain("@typescript-eslint/no-explicit-any"); + expect(summary).toContain("ELIFECYCLE Command failed with exit code 1."); + }); + + it("preserves failure details from both streams when one stream exceeds the cap", () => { + const stderr = Array.from( + { length: 100 }, + (_, index) => `src/error-${index}.ts(1,1): error TS2322: diagnostic ${index}`, + ).join("\n"); + const stdout = [ + "[vite]: Rollup failed to resolve import \"missing-package\" from \"src/main.ts\".", + "Command failed with exit code 1.", + ].join("\n"); + + const summary = summarizeVerificationFailureOutput(stdout, stderr); + + expect(summary).toContain("Rollup failed to resolve import"); + expect(summary).toContain("src/error-0.ts"); + expect(summary.length).toBeLessThanOrEqual(8_000); + }); + + it("preserves terminal totals when one stream has more high-signal lines than the cap", () => { + const stdout = [ + ...Array.from( + { length: 100 }, + (_, index) => `src/error-${index}.ts(1,1): error TS2322: diagnostic ${index}`, + ), + "Test Files 20 failed | 2 passed (22)", + "Tests 100 failed | 10 passed (110)", + "ELIFECYCLE Command failed with exit code 1.", + ].join("\n"); + + const summary = summarizeVerificationFailureOutput(stdout, ""); + + expect(summary).toContain("src/error-0.ts"); + expect(summary).toContain("Test Files 20 failed | 2 passed (22)"); + expect(summary).toContain("Tests 100 failed | 10 passed (110)"); + expect(summary).toContain("ELIFECYCLE Command failed with exit code 1."); + expect(summary.length).toBeLessThanOrEqual(8_000); + }); + + it("keeps a bounded assertion diff with an elided assertion headline", () => { + const stdout = [ + "AssertionError: expected { …(5) } to deeply equal { …(5) }", + "- Expected", + "+ Received", + " Object {", + "- \"status\": \"ready\",", + "+ \"status\": \"failed\",", + " }", + " ❯ src/widget.test.ts:18:4", + ].join("\n"); + + const summary = summarizeVerificationFailureOutput(stdout, ""); + + expect(summary).toContain("- Expected"); + expect(summary).toContain("+ Received"); + expect(summary).toContain("\"status\": \"failed\""); + expect(summary).toContain("src/widget.test.ts:18:4"); + }); + + itPosix("omits routine stdout from successful tool responses", async () => { + const tool = createCompactTool(); + + const result = await tool.execute("call-compact-success", { + command: + "printf 'routine build chatter\\nTests 0 failed | 20 passed (20)\\n100%% tests passed, 0 tests failed out of 5\\n'", + scope: "package", + }); + + const text = result.content[0]?.type === "text" ? result.content[0].text : ""; + expect(text).toContain("Success: true"); + expect(text).not.toContain("routine build chatter"); + expect(text).not.toContain("Verification warning:"); + expect(text).not.toContain("--- stdout ---"); + }); + + itPosix("retains zero-work warnings from commands that exit successfully", async () => { + const tool = createCompactTool(); + + const result = await tool.execute("call-compact-no-work", { + command: "printf 'No projects matched the filters\\nroutine chatter\\n'", + scope: "package", + }); + + const text = result.content[0]?.type === "text" ? result.content[0].text : ""; + expect(text).toContain("Success: true"); + expect(text).toContain("Verification warning:"); + expect(text).toContain("No projects matched the filters"); + expect(text).not.toContain("routine chatter"); + }); + + itPosix("warns when an exit-zero command reports failed tests", async () => { + const tool = createCompactTool(); + + const result = await tool.execute("call-compact-green-while-red", { + command: "printf 'Test Files 1 failed | 2 passed (3)\\nroutine chatter\\n'", + scope: "package", + }); + + const text = result.content[0]?.type === "text" ? result.content[0].text : ""; + expect(text).toContain("Success: true"); + expect(text).toContain("Verification warning:"); + expect(text).toContain("Test Files 1 failed | 2 passed (3)"); + expect(text).not.toContain("routine chatter"); + }); + + itPosix("returns only the compact summary for failed tool responses", async () => { + const tool = createCompactTool(); + const script = [ + "console.log('FAIL src/widget.test.ts > Widget > reports the failure');", + "console.log('Test Files 1 failed | 2 passed (3)');", + "console.error('AssertionError: expected 1 to be 2');", + "console.error('
DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT
');", + "process.exit(1);", + ].join(""); + + const result = await tool.execute("call-compact-failure", { + command: `${process.execPath} -e ${JSON.stringify(script)}`, + scope: "package", + }); + + const text = result.content[0]?.type === "text" ? result.content[0].text : ""; + expect(text).toContain("Failure summary:"); + expect(text).toContain("Widget > reports the failure"); + expect(text).toContain("AssertionError: expected 1 to be 2"); + expect(text).toContain("Test Files 1 failed | 2 passed (3)"); + expect(text).not.toContain("DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT"); + expect(text).not.toContain("--- stdout ---"); + expect(text).not.toContain("--- stderr ---"); + }); + }); + describe("heartbeat callbacks", () => { itPosix("fires onHeartbeat for each output line (POSIX shell)", async () => { const onHeartbeat = vi.fn(); diff --git a/packages/engine/src/run-verification-tool.ts b/packages/engine/src/run-verification-tool.ts index 36513dfa06..8cdc183817 100644 --- a/packages/engine/src/run-verification-tool.ts +++ b/packages/engine/src/run-verification-tool.ts @@ -31,6 +31,9 @@ import { withVerificationSlot } from "./verification-concurrency.js"; // --------------------------------------------------------------------------- const MAX_OUTPUT_BYTES = 200 * 1024; // 200 KB +const VERIFICATION_FAILURE_SUMMARY_MAX_CHARS = 8_000; +const VERIFICATION_FAILURE_SUMMARY_MAX_LINES = 40; +const VERIFICATION_FAILURE_SUMMARY_LINE_MAX_CHARS = 500; const QUIET_HEARTBEAT_INTERVAL_MS = 60_000; // emit synthetic heartbeat after 60s silence const SIGKILL_GRACE_MS = 10_000; const NORMAL_EXIT_REAP_GRACE_MS = 500; @@ -455,6 +458,161 @@ function flattenBuffer(buf: OutputBuffer): string { ); } +const ESC = "\\u001b"; +const ANSI_ESCAPE_PATTERN = new RegExp(`${ESC}\\[[0-?]*[ -/]*[@-~]`, "g"); + +function normalizeFailureLine(line: string): string { + const compact = line + .replace(ANSI_ESCAPE_PATTERN, "") + .replace(/\r/g, "") + .trim() + .replace(/\s+/g, " "); + if (compact.length <= VERIFICATION_FAILURE_SUMMARY_LINE_MAX_CHARS) return compact; + return `${compact.slice(0, VERIFICATION_FAILURE_SUMMARY_LINE_MAX_CHARS - 16)} ... (truncated)`; +} + +function isFailureSummaryNoise(line: string): boolean { + return line.length === 0 + || /^(?:Ignored nodes:|<[/!?]?[a-z]|[a-z-]+=(?:"|')|[·.✓✔xX]+$)/i.test(line) + || /^[⎯━─═-]{4,}/.test(line); +} + +function isHighSignalFailureLine(line: string): boolean { + return /^(?:FAIL(?:ED)?\b|Failed (?:Tests?|Suites?)\b|Test Files\b|Tests:?\b|Snapshots:?\b|Ran all test suites\b)/i.test(line) + || /^(?:AssertionError|TypeError|ReferenceError|SyntaxError|RangeError|Error|Fatal|Expected:|Received:)\b/i.test(line) + || /^(?:npm ERR!|ERR_[A-Z0-9_]+|ELIFECYCLE\b|Command failed\b)/i.test(line) + || /^(?:\d+:\d+\s+(?:error|warning)\b|\[(?:vite|rollup)\].*\b(?:error|failed)\b)/i.test(line) + || /^❯\s+\S+/.test(line) + || /\(\d+,\d+\):\s*(?:error|warning)\b/i.test(line) + || /:\d+:\d+\s+(?:error|warning)\b/i.test(line) + || /\berror TS\d+\b/i.test(line); +} + +function normalizeFailureLines(output: string): string[] { + const lines: string[] = []; + let htmlDepth = 0; + + for (const rawLine of output.split("\n")) { + const line = normalizeFailureLine(rawLine); + const tagOnly = line.match(/^<(\/?)([a-z][\w-]*)(?:\s[^>]*)?\s*\/?>$/i); + if (tagOnly) { + const isClosing = tagOnly[1] === "/"; + const isSelfClosing = /\/>$/.test(line) + || /^(?:area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr)$/i.test( + tagOnly[2]!, + ); + if (isClosing) htmlDepth = Math.max(0, htmlDepth - 1); + else if (!isSelfClosing) htmlDepth += 1; + continue; + } + if (htmlDepth > 0 || isFailureSummaryNoise(line)) continue; + lines.push(line); + } + + return lines; +} + +function extractFailureSummaryLines(output: string): string[] { + const lines = normalizeFailureLines(output); + const selectedIndexes = new Set(); + + lines.forEach((line, index) => { + if (!isHighSignalFailureLine(line)) return; + selectedIndexes.add(index); + if (index > 0) selectedIndexes.add(index - 1); + if (/^(?:AssertionError|Expected:|Received:)\b/i.test(line)) { + for (let offset = 1; offset <= 6 && index + offset < lines.length; offset += 1) { + selectedIndexes.add(index + offset); + } + } + if (/^(?:npm ERR!|ELIFECYCLE\b|Command failed\b)/i.test(line)) { + selectedIndexes.add(Math.max(0, index - 2)); + } + }); + + const candidates = selectedIndexes.size > 0 + ? lines.filter((_, index) => selectedIndexes.has(index)) + : lines.slice(-12); + return Array.from(new Set(candidates)); +} + +function interleaveFailureLines(stderrLines: string[], stdoutLines: string[]): string[] { + const lines: string[] = []; + const maxLength = Math.max(stderrLines.length, stdoutLines.length); + for (let index = 0; index < maxLength; index += 1) { + if (index < stderrLines.length) lines.push(stderrLines[index]!); + if (index < stdoutLines.length) lines.push(stdoutLines[index]!); + } + return lines; +} + +function prioritizeTerminalFailureLines(lines: string[]): string[] { + if (lines.length <= VERIFICATION_FAILURE_SUMMARY_MAX_LINES) return lines; + const halfLimit = VERIFICATION_FAILURE_SUMMARY_MAX_LINES / 2; + return [ + ...lines.slice(-halfLimit), + ...lines.slice(0, halfLimit), + ]; +} + +/** + * FNXC:Verification 2026-07-26-14:29: + * Green verification output stays quiet unless it reports a failure or proves that no work ran; those signals must remain visible so agents do not complete tasks on a vacuous success. + */ +function summarizeSuccessfulVerificationWarnings(stdout: string, stderr: string): string[] { + const warnings = new Set(); + for (const rawLine of `${stderr}\n${stdout}`.split("\n")) { + const line = normalizeFailureLine(rawLine); + if ( + /(?:no projects matched|no test files found|no tests found|^(?:FAIL\b|ELIFECYCLE\b|Command failed\b)|(?:Test Files|Tests)\s+[1-9]\d*\s+failed\b)/i.test(line) + ) { + warnings.add(line); + if (warnings.size === 3) break; + } + } + return [...warnings]; +} + +/** + * FNXC:Verification 2026-07-26-14:23: + * Reduce captured verification output to model-safe diagnostics. + * + * The subprocess capture remains unchanged for heartbeat and process-lifecycle + * behavior. This formatter is only for the tool response injected back into the + * agent context, where full Vitest DOM dumps can otherwise consume an entire + * model window and cause an empty stop before fn_task_done. + */ +export function summarizeVerificationFailureOutput(stdout: string, stderr: string): string { + const combined = [stderr, stdout].filter((part) => part.trim().length > 0).join("\n"); + if (combined.length === 0) { + return "No failure output was captured."; + } + + const candidates = prioritizeTerminalFailureLines( + interleaveFailureLines( + extractFailureSummaryLines(stderr), + extractFailureSummaryLines(stdout), + ), + ); + const footer = + `[verification output compacted from ${combined.length.toLocaleString("en-US")} characters; ` + + "rerun one failing file or test for full detail]"; + const bodyBudget = VERIFICATION_FAILURE_SUMMARY_MAX_CHARS - footer.length - 2; + const selected: string[] = []; + let selectedChars = 0; + + for (const line of candidates) { + if (selected.length >= VERIFICATION_FAILURE_SUMMARY_MAX_LINES) break; + const separatorChars = selected.length > 0 ? 1 : 0; + if (selectedChars + separatorChars + line.length > bodyBudget) continue; + selected.push(line); + selectedChars += separatorChars + line.length; + } + + const body = selected.length > 0 ? selected.join("\n") : "No actionable failure lines were detected."; + return `${body}\n\n${footer}`; +} + // --------------------------------------------------------------------------- // Core logic (exported for unit testing) // --------------------------------------------------------------------------- @@ -850,11 +1008,16 @@ export function createRunVerificationTool( lines.push(`Duration: ${(result.durationMs / 1000).toFixed(1)}s`); lines.push(`Success: ${result.success}`); - if (result.stdout.length > 0) { - lines.push(`\n--- stdout ---\n${result.stdout}`); - } - if (result.stderr.length > 0) { - lines.push(`\n--- stderr ---\n${result.stderr}`); + const hasFailureOutput = result.exitCode !== 0 || result.timedOut; + if (hasFailureOutput) { + lines.push( + `\nFailure summary:\n${summarizeVerificationFailureOutput(result.stdout, result.stderr)}`, + ); + } else { + const successWarnings = summarizeSuccessfulVerificationWarnings(result.stdout, result.stderr); + if (successWarnings.length > 0) { + lines.push(`\nVerification warning:\n${successWarnings.join("\n")}`); + } } if (result.timedOut) {