fix(engine): compact verification failure output
Keep successful verification responses quiet and return bounded, high-signal diagnostics for failures without hiding zero-work or green-while-red warnings.
This commit is contained in:
gsxdsm
7
.changeset/compact-verification-failures.md
Normal file
7
.changeset/compact-verification-failures.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
summary: Keep verification results concise so large failure dumps do not exhaust agent context.
|
||||
category: fix
|
||||
dev: Omits routine successful output and extracts bounded high-signal diagnostics from failed verification commands.
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
detectMarathonVerification,
|
||||
normalizeVerificationCommand,
|
||||
runVerificationCommand,
|
||||
summarizeVerificationFailureOutput,
|
||||
__testOnlyReapVerificationProcessGroup,
|
||||
type RunVerificationOptions,
|
||||
} from "../run-verification-tool.js";
|
||||
@@ -474,6 +475,210 @@ describe("runVerificationCommand", { timeout: 30000 }, () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("tool response output", () => {
|
||||
const createCompactTool = () =>
|
||||
createRunVerificationTool({
|
||||
worktreePath: tempDir,
|
||||
rootDir: workspaceRoot,
|
||||
taskId: "FN-COMPACT",
|
||||
recordActivity: vi.fn(),
|
||||
onVerificationStart: vi.fn(),
|
||||
onVerificationEnd: vi.fn(),
|
||||
log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
it("reduces noisy test failures to counts, failing tests, errors, and source locations", () => {
|
||||
const stdout = [
|
||||
"\u001b[41m FAIL \u001b[0m app/components/Widget.test.tsx > Widget > preserves focus",
|
||||
"AssertionError: expected false to be true",
|
||||
"Ignored nodes: comments, script, style",
|
||||
"<html>",
|
||||
" <body>",
|
||||
" <div class=\"entire-rendered-application\">",
|
||||
" <input />",
|
||||
" DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT",
|
||||
" </div>",
|
||||
" </body>",
|
||||
"</html>",
|
||||
" ❯ app/components/Widget.test.tsx:42:7",
|
||||
" Test Files 1 failed | 12 passed (13)",
|
||||
" Tests 1 failed | 650 passed (651)",
|
||||
].join("\n");
|
||||
|
||||
const summary = summarizeVerificationFailureOutput(stdout, "");
|
||||
|
||||
expect(summary).toContain("FAIL app/components/Widget.test.tsx > Widget > preserves focus");
|
||||
expect(summary).toContain("AssertionError: expected false to be true");
|
||||
expect(summary).toContain("app/components/Widget.test.tsx:42:7");
|
||||
expect(summary).toContain("Test Files 1 failed | 12 passed (13)");
|
||||
expect(summary).toContain("Tests 1 failed | 650 passed (651)");
|
||||
expect(summary).not.toContain("DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT");
|
||||
expect(summary).not.toContain("\u001b[");
|
||||
});
|
||||
|
||||
it("keeps compiler diagnostics and caps generic failure output", () => {
|
||||
const diagnostic = "src/example.ts(12,4): error TS2322: Type 'number' is not assignable to type 'string'.";
|
||||
const stderr = [
|
||||
diagnostic,
|
||||
...Array.from(
|
||||
{ length: 200 },
|
||||
(_, index) =>
|
||||
`src/example-${index}.ts(12,4): error TS2322: ${"x".repeat(200)}`,
|
||||
),
|
||||
].join("\n");
|
||||
|
||||
const summary = summarizeVerificationFailureOutput("", stderr);
|
||||
|
||||
expect(summary).toContain(diagnostic);
|
||||
expect(summary.length).toBeLessThanOrEqual(8_000);
|
||||
expect(summary).toContain("output compacted");
|
||||
});
|
||||
|
||||
it("retains actionable lint context ahead of a generic package-manager failure", () => {
|
||||
const stderr = [
|
||||
"/workspace/src/widget.ts",
|
||||
" 12:3 error Unexpected any. Specify a different type @typescript-eslint/no-explicit-any",
|
||||
"✖ 1 problem (1 error, 0 warnings)",
|
||||
"ELIFECYCLE Command failed with exit code 1.",
|
||||
].join("\n");
|
||||
|
||||
const summary = summarizeVerificationFailureOutput("", stderr);
|
||||
|
||||
expect(summary).toContain("/workspace/src/widget.ts");
|
||||
expect(summary).toContain("12:3 error Unexpected any");
|
||||
expect(summary).toContain("@typescript-eslint/no-explicit-any");
|
||||
expect(summary).toContain("ELIFECYCLE Command failed with exit code 1.");
|
||||
});
|
||||
|
||||
it("preserves failure details from both streams when one stream exceeds the cap", () => {
|
||||
const stderr = Array.from(
|
||||
{ length: 100 },
|
||||
(_, index) => `src/error-${index}.ts(1,1): error TS2322: diagnostic ${index}`,
|
||||
).join("\n");
|
||||
const stdout = [
|
||||
"[vite]: Rollup failed to resolve import \"missing-package\" from \"src/main.ts\".",
|
||||
"Command failed with exit code 1.",
|
||||
].join("\n");
|
||||
|
||||
const summary = summarizeVerificationFailureOutput(stdout, stderr);
|
||||
|
||||
expect(summary).toContain("Rollup failed to resolve import");
|
||||
expect(summary).toContain("src/error-0.ts");
|
||||
expect(summary.length).toBeLessThanOrEqual(8_000);
|
||||
});
|
||||
|
||||
it("preserves terminal totals when one stream has more high-signal lines than the cap", () => {
|
||||
const stdout = [
|
||||
...Array.from(
|
||||
{ length: 100 },
|
||||
(_, index) => `src/error-${index}.ts(1,1): error TS2322: diagnostic ${index}`,
|
||||
),
|
||||
"Test Files 20 failed | 2 passed (22)",
|
||||
"Tests 100 failed | 10 passed (110)",
|
||||
"ELIFECYCLE Command failed with exit code 1.",
|
||||
].join("\n");
|
||||
|
||||
const summary = summarizeVerificationFailureOutput(stdout, "");
|
||||
|
||||
expect(summary).toContain("src/error-0.ts");
|
||||
expect(summary).toContain("Test Files 20 failed | 2 passed (22)");
|
||||
expect(summary).toContain("Tests 100 failed | 10 passed (110)");
|
||||
expect(summary).toContain("ELIFECYCLE Command failed with exit code 1.");
|
||||
expect(summary.length).toBeLessThanOrEqual(8_000);
|
||||
});
|
||||
|
||||
it("keeps a bounded assertion diff with an elided assertion headline", () => {
|
||||
const stdout = [
|
||||
"AssertionError: expected { …(5) } to deeply equal { …(5) }",
|
||||
"- Expected",
|
||||
"+ Received",
|
||||
" Object {",
|
||||
"- \"status\": \"ready\",",
|
||||
"+ \"status\": \"failed\",",
|
||||
" }",
|
||||
" ❯ src/widget.test.ts:18:4",
|
||||
].join("\n");
|
||||
|
||||
const summary = summarizeVerificationFailureOutput(stdout, "");
|
||||
|
||||
expect(summary).toContain("- Expected");
|
||||
expect(summary).toContain("+ Received");
|
||||
expect(summary).toContain("\"status\": \"failed\"");
|
||||
expect(summary).toContain("src/widget.test.ts:18:4");
|
||||
});
|
||||
|
||||
itPosix("omits routine stdout from successful tool responses", async () => {
|
||||
const tool = createCompactTool();
|
||||
|
||||
const result = await tool.execute("call-compact-success", {
|
||||
command:
|
||||
"printf 'routine build chatter\\nTests 0 failed | 20 passed (20)\\n100%% tests passed, 0 tests failed out of 5\\n'",
|
||||
scope: "package",
|
||||
});
|
||||
|
||||
const text = result.content[0]?.type === "text" ? result.content[0].text : "";
|
||||
expect(text).toContain("Success: true");
|
||||
expect(text).not.toContain("routine build chatter");
|
||||
expect(text).not.toContain("Verification warning:");
|
||||
expect(text).not.toContain("--- stdout ---");
|
||||
});
|
||||
|
||||
itPosix("retains zero-work warnings from commands that exit successfully", async () => {
|
||||
const tool = createCompactTool();
|
||||
|
||||
const result = await tool.execute("call-compact-no-work", {
|
||||
command: "printf 'No projects matched the filters\\nroutine chatter\\n'",
|
||||
scope: "package",
|
||||
});
|
||||
|
||||
const text = result.content[0]?.type === "text" ? result.content[0].text : "";
|
||||
expect(text).toContain("Success: true");
|
||||
expect(text).toContain("Verification warning:");
|
||||
expect(text).toContain("No projects matched the filters");
|
||||
expect(text).not.toContain("routine chatter");
|
||||
});
|
||||
|
||||
itPosix("warns when an exit-zero command reports failed tests", async () => {
|
||||
const tool = createCompactTool();
|
||||
|
||||
const result = await tool.execute("call-compact-green-while-red", {
|
||||
command: "printf 'Test Files 1 failed | 2 passed (3)\\nroutine chatter\\n'",
|
||||
scope: "package",
|
||||
});
|
||||
|
||||
const text = result.content[0]?.type === "text" ? result.content[0].text : "";
|
||||
expect(text).toContain("Success: true");
|
||||
expect(text).toContain("Verification warning:");
|
||||
expect(text).toContain("Test Files 1 failed | 2 passed (3)");
|
||||
expect(text).not.toContain("routine chatter");
|
||||
});
|
||||
|
||||
itPosix("returns only the compact summary for failed tool responses", async () => {
|
||||
const tool = createCompactTool();
|
||||
const script = [
|
||||
"console.log('FAIL src/widget.test.ts > Widget > reports the failure');",
|
||||
"console.log('Test Files 1 failed | 2 passed (3)');",
|
||||
"console.error('AssertionError: expected 1 to be 2');",
|
||||
"console.error('<div>DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT</div>');",
|
||||
"process.exit(1);",
|
||||
].join("");
|
||||
|
||||
const result = await tool.execute("call-compact-failure", {
|
||||
command: `${process.execPath} -e ${JSON.stringify(script)}`,
|
||||
scope: "package",
|
||||
});
|
||||
|
||||
const text = result.content[0]?.type === "text" ? result.content[0].text : "";
|
||||
expect(text).toContain("Failure summary:");
|
||||
expect(text).toContain("Widget > reports the failure");
|
||||
expect(text).toContain("AssertionError: expected 1 to be 2");
|
||||
expect(text).toContain("Test Files 1 failed | 2 passed (3)");
|
||||
expect(text).not.toContain("DOM-NOISE-THAT-MUST-NOT-REACH-THE-AGENT");
|
||||
expect(text).not.toContain("--- stdout ---");
|
||||
expect(text).not.toContain("--- stderr ---");
|
||||
});
|
||||
});
|
||||
|
||||
describe("heartbeat callbacks", () => {
|
||||
itPosix("fires onHeartbeat for each output line (POSIX shell)", async () => {
|
||||
const onHeartbeat = vi.fn();
|
||||
|
||||
@@ -31,6 +31,9 @@ import { withVerificationSlot } from "./verification-concurrency.js";
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MAX_OUTPUT_BYTES = 200 * 1024; // 200 KB
|
||||
const VERIFICATION_FAILURE_SUMMARY_MAX_CHARS = 8_000;
|
||||
const VERIFICATION_FAILURE_SUMMARY_MAX_LINES = 40;
|
||||
const VERIFICATION_FAILURE_SUMMARY_LINE_MAX_CHARS = 500;
|
||||
const QUIET_HEARTBEAT_INTERVAL_MS = 60_000; // emit synthetic heartbeat after 60s silence
|
||||
const SIGKILL_GRACE_MS = 10_000;
|
||||
const NORMAL_EXIT_REAP_GRACE_MS = 500;
|
||||
@@ -455,6 +458,161 @@ function flattenBuffer(buf: OutputBuffer): string {
|
||||
);
|
||||
}
|
||||
|
||||
const ESC = "\\u001b";
|
||||
const ANSI_ESCAPE_PATTERN = new RegExp(`${ESC}\\[[0-?]*[ -/]*[@-~]`, "g");
|
||||
|
||||
function normalizeFailureLine(line: string): string {
|
||||
const compact = line
|
||||
.replace(ANSI_ESCAPE_PATTERN, "")
|
||||
.replace(/\r/g, "")
|
||||
.trim()
|
||||
.replace(/\s+/g, " ");
|
||||
if (compact.length <= VERIFICATION_FAILURE_SUMMARY_LINE_MAX_CHARS) return compact;
|
||||
return `${compact.slice(0, VERIFICATION_FAILURE_SUMMARY_LINE_MAX_CHARS - 16)} ... (truncated)`;
|
||||
}
|
||||
|
||||
function isFailureSummaryNoise(line: string): boolean {
|
||||
return line.length === 0
|
||||
|| /^(?:Ignored nodes:|<[/!?]?[a-z]|[a-z-]+=(?:"|')|[·.✓✔xX]+$)/i.test(line)
|
||||
|| /^[⎯━─═-]{4,}/.test(line);
|
||||
}
|
||||
|
||||
function isHighSignalFailureLine(line: string): boolean {
|
||||
return /^(?:FAIL(?:ED)?\b|Failed (?:Tests?|Suites?)\b|Test Files\b|Tests:?\b|Snapshots:?\b|Ran all test suites\b)/i.test(line)
|
||||
|| /^(?:AssertionError|TypeError|ReferenceError|SyntaxError|RangeError|Error|Fatal|Expected:|Received:)\b/i.test(line)
|
||||
|| /^(?:npm ERR!|ERR_[A-Z0-9_]+|ELIFECYCLE\b|Command failed\b)/i.test(line)
|
||||
|| /^(?:\d+:\d+\s+(?:error|warning)\b|\[(?:vite|rollup)\].*\b(?:error|failed)\b)/i.test(line)
|
||||
|| /^❯\s+\S+/.test(line)
|
||||
|| /\(\d+,\d+\):\s*(?:error|warning)\b/i.test(line)
|
||||
|| /:\d+:\d+\s+(?:error|warning)\b/i.test(line)
|
||||
|| /\berror TS\d+\b/i.test(line);
|
||||
}
|
||||
|
||||
function normalizeFailureLines(output: string): string[] {
|
||||
const lines: string[] = [];
|
||||
let htmlDepth = 0;
|
||||
|
||||
for (const rawLine of output.split("\n")) {
|
||||
const line = normalizeFailureLine(rawLine);
|
||||
const tagOnly = line.match(/^<(\/?)([a-z][\w-]*)(?:\s[^>]*)?\s*\/?>$/i);
|
||||
if (tagOnly) {
|
||||
const isClosing = tagOnly[1] === "/";
|
||||
const isSelfClosing = /\/>$/.test(line)
|
||||
|| /^(?:area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr)$/i.test(
|
||||
tagOnly[2]!,
|
||||
);
|
||||
if (isClosing) htmlDepth = Math.max(0, htmlDepth - 1);
|
||||
else if (!isSelfClosing) htmlDepth += 1;
|
||||
continue;
|
||||
}
|
||||
if (htmlDepth > 0 || isFailureSummaryNoise(line)) continue;
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
function extractFailureSummaryLines(output: string): string[] {
|
||||
const lines = normalizeFailureLines(output);
|
||||
const selectedIndexes = new Set<number>();
|
||||
|
||||
lines.forEach((line, index) => {
|
||||
if (!isHighSignalFailureLine(line)) return;
|
||||
selectedIndexes.add(index);
|
||||
if (index > 0) selectedIndexes.add(index - 1);
|
||||
if (/^(?:AssertionError|Expected:|Received:)\b/i.test(line)) {
|
||||
for (let offset = 1; offset <= 6 && index + offset < lines.length; offset += 1) {
|
||||
selectedIndexes.add(index + offset);
|
||||
}
|
||||
}
|
||||
if (/^(?:npm ERR!|ELIFECYCLE\b|Command failed\b)/i.test(line)) {
|
||||
selectedIndexes.add(Math.max(0, index - 2));
|
||||
}
|
||||
});
|
||||
|
||||
const candidates = selectedIndexes.size > 0
|
||||
? lines.filter((_, index) => selectedIndexes.has(index))
|
||||
: lines.slice(-12);
|
||||
return Array.from(new Set(candidates));
|
||||
}
|
||||
|
||||
function interleaveFailureLines(stderrLines: string[], stdoutLines: string[]): string[] {
|
||||
const lines: string[] = [];
|
||||
const maxLength = Math.max(stderrLines.length, stdoutLines.length);
|
||||
for (let index = 0; index < maxLength; index += 1) {
|
||||
if (index < stderrLines.length) lines.push(stderrLines[index]!);
|
||||
if (index < stdoutLines.length) lines.push(stdoutLines[index]!);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function prioritizeTerminalFailureLines(lines: string[]): string[] {
|
||||
if (lines.length <= VERIFICATION_FAILURE_SUMMARY_MAX_LINES) return lines;
|
||||
const halfLimit = VERIFICATION_FAILURE_SUMMARY_MAX_LINES / 2;
|
||||
return [
|
||||
...lines.slice(-halfLimit),
|
||||
...lines.slice(0, halfLimit),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* FNXC:Verification 2026-07-26-14:29:
|
||||
* Green verification output stays quiet unless it reports a failure or proves that no work ran; those signals must remain visible so agents do not complete tasks on a vacuous success.
|
||||
*/
|
||||
function summarizeSuccessfulVerificationWarnings(stdout: string, stderr: string): string[] {
|
||||
const warnings = new Set<string>();
|
||||
for (const rawLine of `${stderr}\n${stdout}`.split("\n")) {
|
||||
const line = normalizeFailureLine(rawLine);
|
||||
if (
|
||||
/(?:no projects matched|no test files found|no tests found|^(?:FAIL\b|ELIFECYCLE\b|Command failed\b)|(?:Test Files|Tests)\s+[1-9]\d*\s+failed\b)/i.test(line)
|
||||
) {
|
||||
warnings.add(line);
|
||||
if (warnings.size === 3) break;
|
||||
}
|
||||
}
|
||||
return [...warnings];
|
||||
}
|
||||
|
||||
/**
|
||||
* FNXC:Verification 2026-07-26-14:23:
|
||||
* Reduce captured verification output to model-safe diagnostics.
|
||||
*
|
||||
* The subprocess capture remains unchanged for heartbeat and process-lifecycle
|
||||
* behavior. This formatter is only for the tool response injected back into the
|
||||
* agent context, where full Vitest DOM dumps can otherwise consume an entire
|
||||
* model window and cause an empty stop before fn_task_done.
|
||||
*/
|
||||
export function summarizeVerificationFailureOutput(stdout: string, stderr: string): string {
|
||||
const combined = [stderr, stdout].filter((part) => part.trim().length > 0).join("\n");
|
||||
if (combined.length === 0) {
|
||||
return "No failure output was captured.";
|
||||
}
|
||||
|
||||
const candidates = prioritizeTerminalFailureLines(
|
||||
interleaveFailureLines(
|
||||
extractFailureSummaryLines(stderr),
|
||||
extractFailureSummaryLines(stdout),
|
||||
),
|
||||
);
|
||||
const footer =
|
||||
`[verification output compacted from ${combined.length.toLocaleString("en-US")} characters; ` +
|
||||
"rerun one failing file or test for full detail]";
|
||||
const bodyBudget = VERIFICATION_FAILURE_SUMMARY_MAX_CHARS - footer.length - 2;
|
||||
const selected: string[] = [];
|
||||
let selectedChars = 0;
|
||||
|
||||
for (const line of candidates) {
|
||||
if (selected.length >= VERIFICATION_FAILURE_SUMMARY_MAX_LINES) break;
|
||||
const separatorChars = selected.length > 0 ? 1 : 0;
|
||||
if (selectedChars + separatorChars + line.length > bodyBudget) continue;
|
||||
selected.push(line);
|
||||
selectedChars += separatorChars + line.length;
|
||||
}
|
||||
|
||||
const body = selected.length > 0 ? selected.join("\n") : "No actionable failure lines were detected.";
|
||||
return `${body}\n\n${footer}`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core logic (exported for unit testing)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -850,11 +1008,16 @@ export function createRunVerificationTool(
|
||||
lines.push(`Duration: ${(result.durationMs / 1000).toFixed(1)}s`);
|
||||
lines.push(`Success: ${result.success}`);
|
||||
|
||||
if (result.stdout.length > 0) {
|
||||
lines.push(`\n--- stdout ---\n${result.stdout}`);
|
||||
}
|
||||
if (result.stderr.length > 0) {
|
||||
lines.push(`\n--- stderr ---\n${result.stderr}`);
|
||||
const hasFailureOutput = result.exitCode !== 0 || result.timedOut;
|
||||
if (hasFailureOutput) {
|
||||
lines.push(
|
||||
`\nFailure summary:\n${summarizeVerificationFailureOutput(result.stdout, result.stderr)}`,
|
||||
);
|
||||
} else {
|
||||
const successWarnings = summarizeSuccessfulVerificationWarnings(result.stdout, result.stderr);
|
||||
if (successWarnings.length > 0) {
|
||||
lines.push(`\nVerification warning:\n${successWarnings.join("\n")}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.timedOut) {
|
||||
|
||||
Reference in New Issue
Block a user