diff --git a/.changeset/fn-8707-mission-validator-json-recovery.md b/.changeset/fn-8707-mission-validator-json-recovery.md new file mode 100644 index 0000000000..3c85c1daa8 --- /dev/null +++ b/.changeset/fn-8707-mission-validator-json-recovery.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Recover valid mission validator JSON from ordinary response formatting noise. +category: fix +dev: Bounds candidate parsing to 256 KiB and eight attempts while preserving fail-closed assertion validation. diff --git a/docs/missions.md b/docs/missions.md index cf2fe862da..62282854ff 100644 --- a/docs/missions.md +++ b/docs/missions.md @@ -496,6 +496,8 @@ interface MilestoneValidationRollup { Canonical authored feature criteria live on `MissionFeature.acceptanceCriteria`, and each feature validator derives its verdict only from its **linked feature-scoped assertions**. Validator prompts list each authoritative assertion ID in brackets; responses must return exactly one result keyed by each listed ID. To recover older model output safely, only an exact-count response with zero recognized IDs is matched positionally and recorded in diagnostics. Partial matches, duplicate IDs, and count mismatches remain fail-closed. Model summary prose, milestone prose, and behavioral results that are not mapped to a linked behavioral assertion cannot override that verdict. +Validator formatting recovery examines only the final 256 KiB of an assistant response and at most eight string-aware fenced or balanced-object candidates, preferring the final syntactically valid payload. It first parses exactly, then makes one conservative syntax-only repair for trailing commas or missing closing delimiters. Recovery never supplies or changes assertion IDs, verdicts, evidence, summaries, or aggregate outcomes; responses that remain invalid are recorded as validator errors and generate no remediation. + Milestone prose is synchronized to one canonical milestone-scoped assertion with `origin: "derived_milestone_acceptance"`. PostgreSQL restricts uniqueness to that derived origin per project/milestone; authored, imported, and migrated legacy milestone assertions stay independent, are never inferred from title/text, and require no feature links. The rollup evaluates all milestone-scoped assertions after feature coverage and feature assertion passes are ready; unmet parent criteria therefore block milestone completion without failing an already-passing feature. See [Mission Completion Gate Contract](./missions-completion-contract.md). ### Phase 3: Feature Execution Loop diff --git a/packages/engine/src/__tests__/mission-execution-loop.test.ts b/packages/engine/src/__tests__/mission-execution-loop.test.ts index 635d6ea6c0..48f7a7f446 100644 --- a/packages/engine/src/__tests__/mission-execution-loop.test.ts +++ b/packages/engine/src/__tests__/mission-execution-loop.test.ts @@ -1789,7 +1789,65 @@ describe("MissionExecutionLoop", () => { // ── parseValidationResult JSON extraction ───────────────────────────────── describe("parseValidationResult", () => { - it("should parse pass result from plain JSON", async () => { + const completePassingPayload = JSON.stringify({ + status: "pass", + assertions: [ + { assertionId: "CA-1", verdict: "pass", passed: true }, + { assertionId: "CA-2", verdict: "pass", passed: true }, + ], + summary: "All assertions passed", + }); + + it.each([ + ["prose braces before a trailing payload", "The inspection called render({ value: 1 }).\n" + completePassingPayload], + ["a malformed earlier object before a trailing payload", '{"status": broken}\n' + completePassingPayload], + ["an unlabeled fenced payload", "Reasoning follows.\n```\n" + completePassingPayload + "\n```"], + ["a trailing comma", completePassingPayload.replace("\n", "").replace("}", ",}")], + ["safely truncated closing delimiters", completePassingPayload.slice(0, -1)], + ["oversized leading prose while retaining the trailing payload", "x".repeat(256 * 1024) + completePassingPayload], + ["more than eight earlier object candidates", Array.from({ length: 12 }, (_, index) => `{"example":${index}}`).join("\n") + "\n" + completePassingPayload], + ])("recovers a complete validator payload after %s", async (_name, response) => { + const assertions = makeAssertions(2); + mockSessionHolder.session.state.messages = [{ role: "assistant", content: response }]; + loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" }); + + await expect((loop as any).parseValidationResult(mockSessionHolder.session, assertions)).resolves.toMatchObject({ + status: "pass", + assertions: [{ assertionId: "CA-1", passed: true }, { assertionId: "CA-2", passed: true }], + }); + }); + + it("caps candidate extraction and preserves braces in JSON strings", () => { + const candidate = JSON.stringify({ summary: "literal ,} is evidence", status: "pass" }); + loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" }); + + expect((loop as any).extractJsonCandidates("x".repeat(256 * 1024) + candidate)).toEqual([candidate]); + expect((loop as any).extractJsonCandidates(Array.from({ length: 12 }, (_, index) => `{"example":${index}}`).join("\n") + candidate)).toHaveLength(8); + expect((loop as any).repairJson(candidate)).toBe(candidate); + }); + + it("keeps recovered payload semantic validation fail-closed", async () => { + const assertions = makeAssertions(2); + const cases = [ + { status: "pass", assertions: [{ assertionId: "CA-1", passed: true }, { assertionId: "unknown", passed: true }] }, + { status: "pass", assertions: [{ assertionId: "CA-1", passed: true }] }, + { status: "pass", assertions: [{ assertionId: "CA-1", passed: true }, { assertionId: "CA-1", passed: true }] }, + { status: "unknown", assertions: [{ assertionId: "CA-1", passed: true }, { assertionId: "CA-2", passed: true }] }, + ]; + + for (const candidate of cases) { + mockSessionHolder.session.state.messages = [{ role: "assistant", content: `prose {not JSON}\n${JSON.stringify(candidate)}` }]; + loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" }); + const result = await (loop as any).parseValidationResult(mockSessionHolder.session, assertions); + expect(result.status).not.toBe("pass"); + } + + mockSessionHolder.session.state.messages = [{ role: "assistant", content: 'prose {not JSON}\n{"status":"pass","summary":"unterminated' }]; + loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" }); + await expect((loop as any).parseValidationResult(mockSessionHolder.session, assertions)).resolves.toMatchObject({ status: "error" }); + }); + + it("routes a recovered trailing pass payload through processTaskOutcome", async () => { const assertions = makeAssertions(2); const response = JSON.stringify({ status: "pass", @@ -1803,7 +1861,7 @@ describe("MissionExecutionLoop", () => { // Set up mock session with AI response mockSessionHolder.session.state.messages = [ { role: "user", content: "Validate this" }, - { role: "assistant", content: response }, + { role: "assistant", content: "The implementation uses render({ value: 1 }).\n" + response }, ]; const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" }); @@ -1978,10 +2036,9 @@ describe("MissionExecutionLoop", () => { }); }); - it("should handle malformed JSON gracefully", async () => { + it("routes irreparable JSON to a validator error without generated remediation", async () => { const assertions = makeAssertions(1); - // Malformed JSON with trailing comma - const malformedResponse = '{"status":"blocked","assertions":[{"assertionId":"CA-1","passed":false}],"summary":"Blocked","blockedReason":"API down",}'; + const malformedResponse = '{"status": broken, "assertions": ['; // Set up mock session with malformed JSON mockSessionHolder.session.state.messages = [ @@ -2005,12 +2062,11 @@ describe("MissionExecutionLoop", () => { await loop.processTaskOutcome("FN-001"); - // When JSON is malformed and cannot be repaired, it should result in an error status - // The loop should handle the error gracefully expect(emitSpy).toHaveBeenCalledWith( - expect.stringMatching(/validation:(passed|failed|blocked|error)/), - expect.any(Object), + "validation:error", + expect.objectContaining({ featureId: "F-001" }), ); + expect(missionStore.createGeneratedFixFeature).not.toHaveBeenCalled(); }); it("should handle AI session returning no messages gracefully", async () => { diff --git a/packages/engine/src/mission-execution-loop.ts b/packages/engine/src/mission-execution-loop.ts index f3d3f37c15..0f6a833262 100644 --- a/packages/engine/src/mission-execution-loop.ts +++ b/packages/engine/src/mission-execution-loop.ts @@ -59,6 +59,11 @@ export const loopLog = createLogger("mission-loop"); /** Maximum time (ms) to wait for a validation session to complete. */ const VALIDATION_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes +/** Bound untrusted validator text while preserving its authoritative trailing payload. */ +const MAX_VALIDATION_RESPONSE_BYTES = 256 * 1024; +/** Avoid unbounded parsing when a model emits many JSON-like examples. */ +const MAX_VALIDATION_JSON_CANDIDATES = 8; + /** * FNXC:MissionValidation 2026-08-01-16:21: * FN-8694 hashes the fixed JSON UTF-8 tuple (landed SHA, judge identity, and exact @@ -1270,27 +1275,22 @@ export class MissionExecutionLoop extends EventEmitter { return this.createErrorValidationResult("No response from validation agent", assertions); } - // Extract JSON from the response (handles markdown code blocks) - const jsonCandidate = this.extractJsonCandidate(responseText); - - if (!jsonCandidate) { + // Prefer the final syntactically valid payload without allowing syntax recovery + // to bypass the semantic validation below. + const jsonCandidates = this.extractJsonCandidates(responseText); + if (jsonCandidates.length === 0) { loopLog.warn("No JSON found in validation response"); return this.createErrorValidationResult("Validation agent did not return JSON", assertions); } - // Try to parse the JSON - let parsed: Record; - try { - parsed = JSON.parse(jsonCandidate); - } catch { - // Intentional fallback: initial parse can fail on malformed JSON; try repairJson() next. - const repaired = this.repairJson(jsonCandidate); - try { - parsed = JSON.parse(repaired); - } catch (e) { - loopLog.warn("Failed to parse validation JSON", e); - return this.createErrorValidationResult("Invalid JSON in validation response", assertions); - } + let parsed: Record | undefined; + for (let index = jsonCandidates.length - 1; index >= 0; index -= 1) { + parsed = this.parseJsonCandidate(jsonCandidates[index]); + if (parsed) break; + } + if (!parsed) { + loopLog.warn("Failed to parse bounded validation JSON candidates"); + return this.createErrorValidationResult("Invalid JSON in validation response", assertions); } // Validate the status field @@ -1359,51 +1359,128 @@ export class MissionExecutionLoop extends EventEmitter { } } - /** - * Extract JSON from a text that may contain markdown code blocks. - */ - private extractJsonCandidate(text: string): string | undefined { - // Try to find JSON in markdown code blocks first - const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/); - if (codeBlockMatch) { - return codeBlockMatch[1].trim(); + /* + FNXC:MissionValidation 2026-08-01-20:13: + FN-8707 accepts ordinary formatting noise only through a 256 KiB trailing + response window and eight string-aware object/fence candidates. Exact parsing + precedes one conservative syntax repair; assertion IDs, verdicts, omissions, + duplicates, and aggregate status still fail closed in the semantic validators. + */ + private extractJsonCandidates(responseText: string): string[] { + const responseBytes = Buffer.from(responseText, "utf8"); + const text = responseBytes.byteLength <= MAX_VALIDATION_RESPONSE_BYTES + ? responseText + : responseBytes.subarray(responseBytes.byteLength - MAX_VALIDATION_RESPONSE_BYTES).toString("utf8"); + const candidates = new Map(); + const addCandidate = (start: number, end: number, value: string) => { + const trimmed = value.trim(); + if (trimmed) candidates.set(`${start}:${end}`, { start, text: trimmed }); + }; + + // Fences are candidates too because a safely truncated fence can still hold + // an otherwise recoverable JSON object. + const fencePattern = /```(?:json)?[ \t]*\r?\n?([\s\S]*?)(?:```|$)/gi; + let fenceMatch: RegExpExecArray | null; + while ((fenceMatch = fencePattern.exec(text)) !== null) { + const bodyOffset = fenceMatch[0].indexOf(fenceMatch[1]); + addCandidate(fenceMatch.index + Math.max(bodyOffset, 0), fencePattern.lastIndex, fenceMatch[1]); } - // Try to find JSON directly (starts with { or [) - const jsonStartMatch = text.match(/(\{[\s\S]*\}|\[[\s\S]*\])/); - if (jsonStartMatch) { - return jsonStartMatch[1]; + const starts: number[] = []; + let inString = false; + let escaped = false; + for (let index = 0; index < text.length; index += 1) { + const character = text[index]; + if (inString) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') inString = false; + continue; + } + if (character === '"') inString = true; + else if (character === "{") starts.push(index); + else if (character === "}") { + const start = starts.pop(); + if (start !== undefined && starts.length === 0) addCandidate(start, index + 1, text.slice(start, index + 1)); + } + } + // A final, string-complete object may be truncated only by closing delimiters. + if (!inString && starts.length > 0) { + addCandidate(starts[0], text.length, text.slice(starts[0])); } + return [...candidates.values()] + .sort((left, right) => left.start - right.start) + .slice(-MAX_VALIDATION_JSON_CANDIDATES) + .map((candidate) => candidate.text); + } + + /** Parse exactly first, then apply one bounded syntax-only repair. */ + private parseJsonCandidate(candidate: string): Record | undefined { + for (const value of [candidate, this.repairJson(candidate)]) { + try { + const parsed: unknown = JSON.parse(value); + if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // Try the single conservative repair, then the prior candidate. + } + } return undefined; } - /** - * Repair common JSON issues in AI responses. - */ + /** Repair only trailing commas and string-complete missing closing delimiters. */ private repairJson(json: string): string { - // Remove trailing commas before closing braces/brackets - let repaired = json.replace(/,\s*([\]}])/g, "$1"); + const removeTrailingCommas = (value: string): string => { + let repaired = ""; + let inString = false; + let escaped = false; + for (let index = 0; index < value.length; index += 1) { + const character = value[index]; + if (inString) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') inString = false; + repaired += character; + continue; + } + if (character === '"') { + inString = true; + repaired += character; + continue; + } + if (character === ",") { + let next = index + 1; + while (/\s/.test(value[next] ?? "")) next += 1; + if (value[next] === "}" || value[next] === "]") continue; + } + repaired += character; + } + return repaired; + }; - // Handle unclosed arrays/objects by finding the last balanced close - const openBraces = (repaired.match(/\{/g) || []).length; - const closeBraces = (repaired.match(/\}/g) || []).length; - const openBrackets = (repaired.match(/\[/g) || []).length; - const closeBrackets = (repaired.match(/\]/g) || []).length; - - // Close missing braces - while (closeBraces < openBraces) { - repaired += "}"; + let repaired = removeTrailingCommas(json); + const closingDelimiters: string[] = []; + let inString = false; + let escaped = false; + for (const character of repaired) { + if (inString) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') inString = false; + continue; + } + if (character === '"') inString = true; + else if (character === "{") closingDelimiters.push("}"); + else if (character === "[") closingDelimiters.push("]"); + else if (character === "}" || character === "]") { + if (closingDelimiters.pop() !== character) return json; + } } - // Close missing brackets - while (closeBrackets < openBrackets) { - repaired += "]"; - } - - // Remove any trailing commas - repaired = repaired.replace(/,\s*([\]}])/g, "$1"); - - return repaired; + if (inString) return json; + repaired += closingDelimiters.reverse().join(""); + return removeTrailingCommas(repaired); } /**