diff --git a/.changeset/fn-6976-planning-json-recovery.md b/.changeset/fn-6976-planning-json-recovery.md new file mode 100644 index 0000000000..77fde172ad --- /dev/null +++ b/.changeset/fn-6976-planning-json-recovery.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Keep Planning Mode malformed AI responses retryable instead of stranding sessions. +category: fix +dev: Hardens planning JSON candidate selection and persists bounded parse failures as retryable AI-session errors. diff --git a/packages/cli/src/commands/__tests__/dashboard.test.ts b/packages/cli/src/commands/__tests__/dashboard.test.ts index 67930ef3c6..7b9df34a10 100644 --- a/packages/cli/src/commands/__tests__/dashboard.test.ts +++ b/packages/cli/src/commands/__tests__/dashboard.test.ts @@ -756,6 +756,7 @@ vi.mock("@fusion/engine", async (importOriginal) => { PrMonitor, PrCommentHandler, aiMergeTask, + runAiMerge: aiMergeTask, CronRunner, createAiPromptExecutor, SelfHealingManager, diff --git a/packages/cli/src/commands/__tests__/task.test.ts b/packages/cli/src/commands/__tests__/task.test.ts index 6b4e569ef7..1f2648f95d 100644 --- a/packages/cli/src/commands/__tests__/task.test.ts +++ b/packages/cli/src/commands/__tests__/task.test.ts @@ -96,7 +96,8 @@ vi.mock("@fusion/core", async (importActual) => { }); // Mock @fusion/engine -vi.mock("@fusion/engine", () => ({ aiMergeTask: vi.fn() })); +const { runAiMergeMock } = vi.hoisted(() => ({ runAiMergeMock: vi.fn() })); +vi.mock("@fusion/engine", () => ({ runAiMerge: runAiMergeMock, aiMergeTask: runAiMergeMock })); // Mock @fusion/dashboard vi.mock("@fusion/dashboard", () => ({ @@ -155,7 +156,7 @@ import { import { GitHubClient, generatePrMetadata } from "@fusion/dashboard"; import { createSession, submitResponse } from "@fusion/dashboard/planning"; import { resolveProject } from "../../project-context.js"; -import { aiMergeTask } from "@fusion/engine"; +import { runAiMerge } from "@fusion/engine"; const mockedExec = vi.mocked(exec); @@ -1228,7 +1229,7 @@ describe("project-aware task command behavior", () => { isRegistered: true, store: resolvedStore, }); - vi.mocked(aiMergeTask).mockResolvedValue({ + vi.mocked(runAiMerge).mockResolvedValue({ merged: true, task: makeTask({ id: "FN-123" }), branch: "fusion/fn-123", @@ -1244,7 +1245,7 @@ describe("project-aware task command behavior", () => { expect(updateStep).toHaveBeenCalled(); expect(logEntry).toHaveBeenCalled(); - expect(aiMergeTask).toHaveBeenCalledWith(resolvedStore, "/test", "FN-123", expect.any(Object)); + expect(runAiMerge).toHaveBeenCalledWith(resolvedStore, "/test", "FN-123", expect.any(Object)); expect(duplicateTask).toHaveBeenCalledWith("FN-123"); expect(refineTask).toHaveBeenCalledWith("FN-123", "more tests"); }); diff --git a/packages/dashboard/src/__tests__/session-error-recovery.test.ts b/packages/dashboard/src/__tests__/session-error-recovery.test.ts index 151c46a61d..a1e5495ad5 100644 --- a/packages/dashboard/src/__tests__/session-error-recovery.test.ts +++ b/packages/dashboard/src/__tests__/session-error-recovery.test.ts @@ -20,6 +20,7 @@ import { createSessionWithAgent, GENERATION_TIMEOUT_MS as PLANNING_GENERATION_TIMEOUT_MS, getSession, + parseAgentResponse, planningStreamManager, retrySession, setAiSessionStore as setPlanningAiSessionStore, @@ -138,6 +139,149 @@ describe("session error recovery", () => { await rm(tmpDir, { recursive: true, force: true }); }); + it("recovers a streaming initial-turn prose response when the bounded reformat succeeds", async () => { + const errorEvents: string[] = []; + const questionEvents: string[] = []; + + __setCreateFnAgent( + async () => + createMockAgent([ + "I should ask a question next, but I forgot the JSON wrapper.", + JSON.stringify({ + type: "question", + data: { id: "q-reformatted", type: "text", question: "Recovered initial question" }, + }), + ]), + ); + + const sessionId = await createSessionWithAgent( + "127.0.0.102", + "Streaming malformed first turn", + "/tmp/project", + taskStore, + ); + const unsubscribe = planningStreamManager.subscribe(sessionId, (event) => { + if (event.type === "error") errorEvents.push(String(event.data)); + if (event.type === "question") questionEvents.push(String((event.data as { id?: string }).id)); + }); + + planningStreamManager.consumeInitialTurn(sessionId)?.(); + + await waitFor(() => aiSessionStore.get(sessionId)?.status === "awaiting_input"); + + const persisted = aiSessionStore.get(sessionId); + expect(persisted?.status).toBe("awaiting_input"); + expect(persisted?.error).toBeNull(); + expect(getSession(sessionId)?.currentQuestion?.id).toBe("q-reformatted"); + expect(questionEvents).toContain("q-reformatted"); + expect(errorEvents).toEqual([]); + + unsubscribe(); + }); + + it("keeps a streaming initial-turn parse failure retryable and recovers on retry", async () => { + const errorEvents: string[] = []; + + __setCreateFnAgent( + async () => + createMockAgent([ + "I can help plan this, but this response is prose only.", + "Still prose only after the bounded reformat request.", + ]), + ); + + const sessionId = await createSessionWithAgent( + "127.0.0.103", + "Streaming unrecoverable first turn", + "/tmp/project", + taskStore, + ); + const unsubscribe = planningStreamManager.subscribe(sessionId, (event) => { + if (event.type === "error") { + errorEvents.push(String(event.data)); + } + }); + + planningStreamManager.consumeInitialTurn(sessionId)?.(); + + await waitFor(() => aiSessionStore.get(sessionId)?.status === "error"); + + const persistedError = aiSessionStore.get(sessionId); + expect(persistedError?.status).toBe("error"); + expect(persistedError?.error).toContain("AI returned no valid JSON"); + expect(JSON.parse(persistedError?.conversationHistory ?? "[]")).toHaveLength(0); + expect(errorEvents).toContainEqual(expect.stringContaining("AI returned no valid JSON")); + expect(getSession(sessionId)).toBeDefined(); + + __setCreateFnAgent( + async () => + createMockAgent([ + JSON.stringify({ + type: "question", + data: { id: "q-retry-initial", type: "text", question: "Recovered retry question" }, + }), + ]), + ); + + await retrySession(sessionId, "/tmp/project", undefined, taskStore); + + const persistedRecovered = aiSessionStore.get(sessionId); + expect(persistedRecovered?.status).toBe("awaiting_input"); + expect(persistedRecovered?.error).toBeNull(); + expect(getSession(sessionId)?.currentQuestion?.id).toBe("q-retry-initial"); + + unsubscribe(); + }); + + it("keeps a non-streaming initial-turn parse failure persisted for retry", async () => { + __setCreateFnAgent( + async () => + createMockAgent([ + "This non-streaming first response is prose only.", + "Still not JSON after the bounded reformat request.", + ]), + ); + + await expect( + createSession("127.0.0.104", "Non-streaming unrecoverable first turn", taskStore, "/tmp/project"), + ).rejects.toThrow("Failed to get first question from AI"); + + const failedSession = aiSessionStore.listActive().find((session) => session.type === "planning"); + expect(failedSession?.status).toBe("error"); + expect(failedSession?.id).toBeTruthy(); + const sessionId = failedSession?.id as string; + expect(aiSessionStore.get(sessionId)?.error).toContain("AI returned no valid JSON"); + expect(getSession(sessionId)).toBeDefined(); + + __setCreateFnAgent( + async () => + createMockAgent([ + JSON.stringify({ + type: "question", + data: { id: "q-nonstream-retry", type: "text", question: "Recovered non-streaming retry" }, + }), + ]), + ); + + await retrySession(sessionId, "/tmp/project", undefined, taskStore); + + expect(aiSessionStore.get(sessionId)?.status).toBe("awaiting_input"); + expect(aiSessionStore.get(sessionId)?.error).toBeNull(); + expect(getSession(sessionId)?.currentQuestion?.id).toBe("q-nonstream-retry"); + }); + + it("selects a valid planning JSON object over a larger unrelated JSON candidate", () => { + const parsed = parseAgentResponse(`Here is an unrelated object first: +{"metadata":{"items":[{"label":"not planning","details":"${"x".repeat(200)}"}]}} +The actual planning response is: +{"type":"question","data":{"id":"q-small","type":"text","question":"What should we build?"}}`); + + expect(parsed.type).toBe("question"); + if (parsed.type === "question") { + expect(parsed.data.id).toBe("q-small"); + } + }); + it("captures planning parse failures as error state, preserves history, and allows retry", async () => { const errorEvents: string[] = []; const unsubscribe = planningStreamManager.subscribe("pending", () => { diff --git a/packages/dashboard/src/planning.ts b/packages/dashboard/src/planning.ts index ea29458b80..e1bc75ae54 100644 --- a/packages/dashboard/src/planning.ts +++ b/packages/dashboard/src/planning.ts @@ -1009,11 +1009,8 @@ async function getFirstQuestionFromAgent( } if (!parsed) { - // Clean up the failed startup session and release the underlying agent so - // an unparsable first response cannot leave model/transport handles behind - // in the long-running dashboard process. - sessions.delete(session.id); - unpersistSession(session.id); + const errorMessage = buildRetryableParseErrorMessage(lastError); + setSessionError(session, errorMessage); try { await session.agent.session.dispose?.(); } catch (disposeErr) { @@ -1024,9 +1021,7 @@ async function getFirstQuestionFromAgent( }); } session.agent = undefined; - throw new Error( - `Failed to get first question from AI: ${lastError?.message || "Unknown error"}` - ); + throw new Error(`Failed to get first question from AI: ${errorMessage}`); } if (parsed.type === "complete") { @@ -1598,6 +1593,18 @@ async function maybeNotifyPlanningAwaitingInput(session: Session, question: Plan /** Max number of retry attempts when AI returns unparseable output */ const MAX_PARSE_RETRIES = 1; +/* +FNXC:PlanningJsonRecovery 2026-06-24-20:58: +Planning Mode malformed AI output must either recover through the bounded reformat prompt to a valid planning response or persist a retryable session error. Parser candidate selection therefore prefers valid planning-shaped JSON over unrelated larger JSON blobs embedded in model prose. +*/ + +function buildRetryableParseErrorMessage(error: Error | undefined): string { + const baseMessage = (error?.message || "Failed to parse AI response") + .replace(/\s*Please try again\.?\s*$/i, "") + .trim(); + return `${baseMessage}. Retry this planning session or start a new one.`; +} + /** * Continue the AI conversation with a user message. * @@ -1784,19 +1791,13 @@ async function continueAgentConversation(session: Session, message: string): Pro } if (!parsed) { - // All attempts exhausted — emit actionable error - const errorMsg = `${lastError?.message || "Failed to parse AI response"} You can try responding again or start a new planning session.`; + // All attempts exhausted — emit actionable, retryable error without duplicated "Please try again" suffixes. + const errorMsg = buildRetryableParseErrorMessage(lastError); diagnostics.error( "All parse attempts exhausted for session", { sessionId: session.id, message: errorMsg, operation: "parse-exhausted" } ); - session.error = errorMsg; - session.updatedAt = new Date(); - persistSession(session, "error", errorMsg); - planningStreamManager.broadcast(session.id, { - type: "error", - data: errorMsg, - }); + setSessionError(session, errorMsg); return; } @@ -1846,18 +1847,52 @@ async function continueAgentConversation(session: Session, message: string): Pro * * Returns the extracted JSON string or null if nothing usable is found. */ +function isPlanningResponseShape(parsed: unknown): parsed is PlanningResponse { + if ( + typeof parsed !== "object" || + parsed === null || + !("type" in parsed) || + !("data" in parsed) + ) { + return false; + } + + const typed = parsed as { type: string; data: unknown }; + return ( + (typed.type === "question" || typed.type === "complete") && + typed.data !== null && + typed.data !== undefined + ); +} + +function parseJsonCandidateForShape(candidate: string): unknown | undefined { + try { + return JSON.parse(candidate); + } catch { + try { + return JSON.parse(repairJson(candidate)); + } catch { + return undefined; + } + } +} + function extractJsonCandidate(text: string): string | null { if (!text || !text.trim()) return null; - // 1. Try markdown code blocks first (most reliable) - const codeBlockMatch = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/); - if (codeBlockMatch?.[1]) { - const candidate = codeBlockMatch[1].trim(); - if (candidate.startsWith("{")) return candidate; - } + // 1. Try markdown code blocks first (most reliable when they contain a planning response). + const codeBlockMatches = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)\s*```/g)]; + const codeBlockCandidates = codeBlockMatches + .map((match) => match[1]?.trim()) + .filter((candidate): candidate is string => Boolean(candidate?.startsWith("{"))); + const planningCodeBlock = codeBlockCandidates.find((candidate) => + isPlanningResponseShape(parseJsonCandidateForShape(candidate)), + ); + if (planningCodeBlock) return planningCodeBlock; + if (codeBlockCandidates.length > 0) return codeBlockCandidates[0]; - // 2. Find all top-level brace-delimited objects using balanced brace counting - const candidates: Array<{ start: number; end: number; text: string }> = []; + // 2. Find all top-level brace-delimited objects using balanced brace counting. + const candidates: Array<{ start: number; end: number; text: string; parsed?: unknown }> = []; for (let i = 0; i < text.length; i++) { if (text[i] === "{") { let depth = 0; @@ -1882,26 +1917,24 @@ function extractJsonCandidate(text: string): string | null { if (ch === "}") depth--; if (depth === 0) { const candidate = text.slice(i, j + 1).trim(); - // Only accept candidates that parse as valid JSON - try { - JSON.parse(candidate); - candidates.push({ start: i, end: j, text: candidate }); - } catch { - // Not valid JSON, skip - } + candidates.push({ start: i, end: j, text: candidate, parsed: parseJsonCandidateForShape(candidate) }); break; } } } } - // Pick the largest valid candidate (most likely the full response) - if (candidates.length > 0) { - candidates.sort((a, b) => b.text.length - a.text.length); - return candidates[0].text; + const planningCandidate = candidates.find((candidate) => isPlanningResponseShape(candidate.parsed)); + if (planningCandidate) return planningCandidate.text; + + // Pick the largest valid JSON candidate only after planning-shaped candidates are ruled out. + const validCandidates = candidates.filter((candidate) => candidate.parsed !== undefined); + if (validCandidates.length > 0) { + validCandidates.sort((a, b) => b.text.length - a.text.length || a.start - b.start); + return validCandidates[0].text; } - // 3. Last resort: try the full trimmed text + // 3. Last resort: try the full trimmed text so repairJson can close truncated objects. const trimmed = text.trim(); if (trimmed.startsWith("{")) return trimmed; @@ -2003,20 +2036,8 @@ export function parseAgentResponse(text: string): PlanningResponse { } // Validate structure - if ( - typeof parsed === "object" && - parsed !== null && - "type" in parsed && - "data" in parsed - ) { - const typed = parsed as { type: string; data: unknown }; - if ( - (typed.type === "question" || typed.type === "complete") && - typed.data !== null && - typed.data !== undefined - ) { - return parsed as PlanningResponse; - } + if (isPlanningResponseShape(parsed)) { + return parsed; } diagnostics.error("Invalid response structure from AI", { parsedSnippet: JSON.stringify(parsed).slice(0, 500), operation: "parse-validate" });