diff --git a/.changeset/planning-provider-error-no-hang.md b/.changeset/planning-provider-error-no-hang.md new file mode 100644 index 0000000000..8c7c621d73 --- /dev/null +++ b/.changeset/planning-provider-error-no-hang.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Planning Mode no longer hangs on "Generating plan" after a provider error; it surfaces a retryable error. +category: fix +dev: Provider errors thrown after a planning session persists "generating" (agent rebuild, history replay, legacy sync start) now land the session in a persisted retryable error with an SSE error event; the stream route reconciles stranded generating sessions past the watchdog window via `reconcileStalePlanningGeneration`. diff --git a/packages/dashboard/src/__tests__/planning-provider-error-recovery.test.ts b/packages/dashboard/src/__tests__/planning-provider-error-recovery.test.ts new file mode 100644 index 0000000000..427b7e0305 --- /dev/null +++ b/packages/dashboard/src/__tests__/planning-provider-error-recovery.test.ts @@ -0,0 +1,225 @@ +// @vitest-environment node + +/* +FNXC:PlanningProviderErrors 2026-07-23-20:10: +Regression tests for provider-error handling in dedicated Planning Mode. Reported bug: a +provider failure (auth error, overloaded provider, model-registry stall) thrown between +persistSession("generating") and the turn's own error handling escaped to the route with the +session row left "generating" forever — no persisted error, no SSE error event, no watchdog — +so the Planning modal hung on "Thinking/Generating plan" (its SSE reconnect loop and 8s poll +both treat a persisted "generating" row as healthy). Invariant: once a planning session enters +"generating", every non-abort failure lands it in a retryable persisted "error" state with an +SSE error event, and a stream connect against a settled/stranded session terminates with an +error event instead of waiting forever. +*/ + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { EventEmitter } from "node:events"; +import type { TaskStore } from "@fusion/core"; + +vi.mock("@fusion/engine", () => ({ + listCliAdapterDescriptors: () => [], + resolveMcpServersForStore: async () => ({ servers: [] }), + buildSessionSkillContextSync: () => ({ + skillSelectionContext: undefined, + resolvedSkillNames: ["fusion"], + skillSource: "role-fallback" as const, + }), + createFnAgent: vi.fn(), + createWorkflowAuthoringTools: () => [], + createChatTaskDocumentTools: () => [], + createChatTaskLogsReadTool: () => ({}), +})); + +import { + __resetPlanningState, + __setCreateFnAgent, + createSession, + createSessionWithAgent, + getSession, + PLANNING_INTERRUPTED_ERROR_MESSAGE, + planningStreamManager, + reconcileStalePlanningGeneration, + retrySession, + setAiSessionStore, + submitResponse, +} from "../planning.js"; + +const MOCK_TASK_STORE = { + listTasks: vi.fn(async () => []), + getSettings: vi.fn(async () => ({})), + getTask: vi.fn(async () => { + throw new Error("not found"); + }), +} as unknown as TaskStore; + +const QUESTION_JSON = JSON.stringify({ + type: "question", + data: { id: "q-next", type: "single_select", question: "What next?" }, +}); + +const PROVIDER_ERROR_MESSAGE = "Provider rejected the request: 401 invalid_api_key"; + +function createScriptedAgent(responder: () => string = () => QUESTION_JSON) { + const messages: Array<{ role: string; content: string }> = []; + const prompt = vi.fn(async () => { + messages.push({ role: "assistant", content: responder() }); + }); + return { agent: { session: { state: { messages }, prompt, dispose: vi.fn() } } }; +} + +async function waitFor(predicate: () => Promise | boolean, attempts = 50): Promise { + for (let i = 0; i < attempts; i++) { + if (await predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error("condition not reached"); +} + +async function startSessionAwaitingInput(ip: string): Promise { + const scripted = createScriptedAgent(); + __setCreateFnAgent(vi.fn(async () => scripted.agent) as never); + const sessionId = await createSessionWithAgent(ip, "Plan something small", "/tmp/project", MOCK_TASK_STORE); + planningStreamManager.consumeInitialTurn(sessionId)?.(); + await waitFor(async () => Boolean((await getSession(sessionId))?.currentQuestion)); + return sessionId; +} + +function bufferedErrorEvents(sessionId: string): unknown[] { + return planningStreamManager + .getBufferedEvents(sessionId, 0) + .filter((event: { event: string }) => event.event === "error"); +} + +describe("planning provider-error recovery", () => { + let upsertMock: ReturnType; + let storeGet: ReturnType; + + beforeEach(() => { + __resetPlanningState(); + upsertMock = vi.fn(async () => {}); + storeGet = vi.fn(async () => null); + setAiSessionStore(Object.assign(new EventEmitter(), { + upsert: upsertMock, + get: storeGet, + updateThinking: vi.fn(), + }) as never); + }); + + it("submitResponse persists a retryable error when agent rebuild hits a provider failure after entering generating", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.1"); + const session = (await getSession(sessionId))!; + const question = session.currentQuestion!; + + // Force the ensureSessionAgent path (agent gone, e.g. after retry disposal) with a provider failure. + session.agent = undefined; + __setCreateFnAgent(vi.fn(async () => { + throw new Error(PROVIDER_ERROR_MESSAGE); + }) as never); + + await expect( + submitResponse(sessionId, { [question.id]: "option-1" }, "/tmp/project", undefined, MOCK_TASK_STORE), + ).rejects.toThrow(PROVIDER_ERROR_MESSAGE); + + const after = (await getSession(sessionId))!; + expect(after.error).toContain(PROVIDER_ERROR_MESSAGE); + expect(bufferedErrorEvents(sessionId).length).toBeGreaterThan(0); + await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error")); + }); + + it("retrySession re-persists an error instead of stranding the session in generating", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.2"); + const session = (await getSession(sessionId))!; + session.error = "AI returned no valid JSON. Retry this planning session or start a new one."; + session.agent = undefined; + storeGet.mockImplementation(async () => ({ id: sessionId, type: "planning", status: "error" })); + + __setCreateFnAgent(vi.fn(async () => { + throw new Error(PROVIDER_ERROR_MESSAGE); + }) as never); + + await expect(retrySession(sessionId, "/tmp/project", undefined, MOCK_TASK_STORE)).rejects.toThrow(PROVIDER_ERROR_MESSAGE); + + const after = (await getSession(sessionId))!; + // retrySession clears the prior error before generating; the failure must restore a terminal error. + expect(after.error).toContain(PROVIDER_ERROR_MESSAGE); + await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error")); + }); + + it("legacy synchronous createSession persists an error when the provider fails before the first question", async () => { + __setCreateFnAgent(vi.fn(async () => { + throw new Error(PROVIDER_ERROR_MESSAGE); + }) as never); + + await expect( + createSession("10.1.0.3", "Plan something small", MOCK_TASK_STORE, "/tmp/project"), + ).rejects.toThrow(PROVIDER_ERROR_MESSAGE); + + await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error")); + const errorRows = upsertMock.mock.calls.map((call) => call[0]).filter((row: { status?: string }) => row?.status === "error"); + const errorRow = errorRows[errorRows.length - 1]; + expect(errorRow?.error).toContain(PROVIDER_ERROR_MESSAGE); + }); + + it("surfaces the provider failure from the reformat retry instead of a misleading parse error", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.4"); + const session = (await getSession(sessionId))!; + const question = session.currentQuestion!; + + let calls = 0; + const scripted = createScriptedAgent(() => "definitely not json"); + scripted.agent.session.prompt = vi.fn(async () => { + calls += 1; + if (calls === 1) { + scripted.agent.session.state.messages.push({ role: "assistant", content: "definitely not json" }); + return; + } + throw new Error(PROVIDER_ERROR_MESSAGE); + }); + session.agent = scripted.agent as never; + + await submitResponse(sessionId, { [question.id]: "option-1" }, "/tmp/project", undefined, MOCK_TASK_STORE); + + const after = (await getSession(sessionId))!; + expect(after.error).toContain(PROVIDER_ERROR_MESSAGE); + }); + + describe("reconcileStalePlanningGeneration", () => { + it("returns the persisted terminal error for a settled errored session", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.5"); + const session = (await getSession(sessionId))!; + session.error = PROVIDER_ERROR_MESSAGE; + + expect(reconcileStalePlanningGeneration(sessionId)).toBe(PROVIDER_ERROR_MESSAGE); + }); + + it("converts a stranded generating session past the watchdog window into a retryable error", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.6"); + const session = (await getSession(sessionId))!; + session.generationPurpose = "plan_update"; + session.generationStartedAt = new Date(Date.now() - 10 * 60_000).toISOString(); + + expect(reconcileStalePlanningGeneration(sessionId)).toBe(PLANNING_INTERRUPTED_ERROR_MESSAGE); + expect(session.error).toBe(PLANNING_INTERRUPTED_ERROR_MESSAGE); + expect(bufferedErrorEvents(sessionId).length).toBeGreaterThan(0); + await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error")); + }); + + it("leaves a fresh generating session alone", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.7"); + const session = (await getSession(sessionId))!; + session.generationPurpose = "plan_update"; + session.generationStartedAt = new Date().toISOString(); + + expect(reconcileStalePlanningGeneration(sessionId)).toBeUndefined(); + expect(session.error).toBeUndefined(); + }); + + it("leaves an awaiting-input session alone", async () => { + const sessionId = await startSessionAwaitingInput("10.1.0.8"); + + expect(reconcileStalePlanningGeneration(sessionId)).toBeUndefined(); + expect((await getSession(sessionId))!.error).toBeUndefined(); + }); + }); +}); diff --git a/packages/dashboard/src/planning.ts b/packages/dashboard/src/planning.ts index 3c58667cc7..01453c877f 100644 --- a/packages/dashboard/src/planning.ts +++ b/packages/dashboard/src/planning.ts @@ -332,6 +332,7 @@ export const GENERATION_LOOP_REPEAT_LIMIT = 8; const PLANNING_STUCK_ERROR_MESSAGE = "AI generation appears stuck with no new output. You can retry or start a new session."; const PLANNING_LOOP_ERROR_MESSAGE = "AI generation appears stuck repeating the same output. You can retry or start a new session."; +export const PLANNING_INTERRUPTED_ERROR_MESSAGE = "Planning generation was interrupted before it finished. Retry to continue this session."; // ── Types ─────────────────────────────────────────────────────────────────── @@ -1245,6 +1246,33 @@ export async function createSession( beginPlanningGeneration(session, "initial_plan"); persistSession(session, "generating"); + /* + FNXC:PlanningProviderErrors 2026-07-23-20:10: + Once the session row is persisted "generating", every failure on the way to the first + question (system-prompt resolution, agent construction, the provider prompt itself) must + land the session in a retryable persisted "error" state before rethrowing to the route. + Previously a provider error thrown here left the row "generating" forever with no error + and no watchdog, so the session appeared stuck on "Generating plan" until a server restart. + */ + try { + return await runCreateSessionFirstTurn(session, rootDir, store, promptOverrides, pluginRunner); + } catch (err) { + if (!(err instanceof Error && err.name === "AbortError") && !session.error) { + setSessionError(session, err instanceof Error ? err.message : "Failed to initialize AI agent"); + } + throw err; + } +} + +/** First turn of the legacy synchronous planning start; see the provider-error guard in createSession. */ +async function runCreateSessionFirstTurn( + session: Session, + rootDir: string, + store: TaskStore, + promptOverrides?: PromptOverrideMap, + pluginRunner?: SkillPluginRunner, +): Promise<{ sessionId: string; firstQuestion: PlanningQuestion; summary: PlanningSummary; validated: boolean }> { + const sessionId = session.id; const systemPrompt = await resolvePlanningModeSystemPrompt(store, promptOverrides, session.workflowId); // Create AI agent and get the first question @@ -1293,7 +1321,7 @@ export async function createSession( session.agent = agentResult; session.updatedAt = new Date(); - const firstResponse = await getFirstQuestionFromAgent(session, formatInitialPlanRequestForAgent(initialPlan)); + const firstResponse = await getFirstQuestionFromAgent(session, formatInitialPlanRequestForAgent(session.initialPlan)); const firstQuestion = firstResponse.data; session.currentQuestion = firstQuestion; @@ -1413,7 +1441,9 @@ async function getFirstQuestionFromAgent( } } } - } catch { + } catch (retryPromptErr) { + // FNXC:PlanningProviderErrors 2026-07-23-20:10: a provider failure on the reformat prompt must surface as itself, not as a misleading "no valid JSON" parse error. + lastError = retryPromptErr instanceof Error ? retryPromptErr : new Error(String(retryPromptErr)); break; } } @@ -2197,6 +2227,40 @@ function createAbortError(): Error { return error; } +/* +FNXC:PlanningProviderErrors 2026-07-23-20:10: +Terminal-state reconciliation for SSE stream connects. Two hang shapes ended with the Planning +modal pinned on "Thinking/Generating plan" with a stream that will never emit again: +1. The session already holds a terminal error, but the buffered error event is gone (the + client's reconnect loop treats a persisted "generating" row as healthy and the 8s poll only + reacts to persisted status changes). +2. The session is persisted "generating" with generation metadata set, but no turn is active + or pending and the generation started longer ago than the inactivity watchdog window — a + stranded run (e.g. a pre-fix provider-error escape or a crashed generation promise) that no + watchdog owns anymore. +Returns the terminal error message the stream route must emit before closing, or undefined +when the session is healthy. Only the stale case writes state: it persists the same retryable +error contract as every other planning failure so Retry and the bounded auto-retry recover it. +*/ +export function reconcileStalePlanningGeneration(sessionId: string): string | undefined { + const session = sessions.get(sessionId); + if (!session || session.validated) return undefined; + if (isPlanningTurnActive(sessionId) || planningStreamManager.hasPendingInitialTurn(sessionId)) return undefined; + if (session.error) return session.error; + if (session.generationPurpose === undefined) return undefined; + const startedAtMs = session.generationStartedAt ? Date.parse(session.generationStartedAt) : Number.NaN; + const ageMs = Number.isNaN(startedAtMs) ? Number.POSITIVE_INFINITY : Date.now() - startedAtMs; + if (ageMs < GENERATION_TIMEOUT_MS) return undefined; + diagnostics.warn("Reconciling stranded planning generation to a retryable error", { + sessionId, + generationPurpose: session.generationPurpose, + generationStartedAt: session.generationStartedAt, + operation: "reconcile-stale-generation", + }); + setSessionError(session, PLANNING_INTERRUPTED_ERROR_MESSAGE); + return PLANNING_INTERRUPTED_ERROR_MESSAGE; +} + /* FNXC:PlanningContextCompaction 2026-07-22-22:40: Long planning interviews accumulate the whole Q/A history in one agent session and can hit the @@ -2681,6 +2745,8 @@ async function continueAgentConversation(session: Session, message: string): Pro retryErr, { sessionId: session.id, operation: "retry-prompt" } ); + // FNXC:PlanningProviderErrors 2026-07-23-20:10: report the provider failure itself instead of a misleading "no valid JSON" parse error. + lastError = retryErr instanceof Error ? retryErr : new Error(String(retryErr)); break; } } @@ -3090,6 +3156,17 @@ export async function submitResponse( generation-error case so the modal's submit path keeps its existing SSE-driven recovery. */ let answeredQuestion: PlanningQuestion | undefined; + /* + FNXC:PlanningProviderErrors 2026-07-23-20:10: + ensureSessionAgent (agent construction + history-replay prompt) throws provider errors AFTER + the session row was persisted "generating". continueAgentConversation converts its own + failures to a persisted session error, but a throw between persist("generating") and that + call previously escaped to the route with the row left "generating" forever — no error + event, no watchdog — so the Planning modal hung on "Thinking/Generating plan" (its SSE + reconnect + 8s poll both treat "generating" as healthy). Track the generating transition and + convert any non-abort escape into the same retryable persisted error state. + */ + let enteredGenerating = false; try { const contextualComments = getContextualComments(responses); @@ -3104,6 +3181,7 @@ export async function submitResponse( session.error = undefined; session.pendingContextualComments = contextualComments; await persistSession(session, "generating"); + enteredGenerating = true; await ensureSessionAgent(session, rootDir, session.history, promptOverrides, store); await continueAgentConversation(session, formatContextualCommentsForAgent(session.summary, contextualComments)); } else if (isRefineRequest(responses) && session.summary) { @@ -3113,6 +3191,7 @@ export async function submitResponse( session.currentQuestion = undefined; session.error = undefined; await persistSession(session, "generating"); + enteredGenerating = true; await ensureSessionAgent(session, rootDir, session.history, promptOverrides, store); const focus = typeof responses.focus === "string" ? responses.focus.trim() : undefined; @@ -3154,6 +3233,7 @@ export async function submitResponse( beginPlanningGeneration(session, "plan_update"); session.currentQuestion = undefined; await persistSession(session, "generating"); + enteredGenerating = true; if (!session.agent) { // An edited older answer must be replayed in its original position with every // later answer retained; only a newly appended answer is sent after replay. @@ -3170,6 +3250,11 @@ export async function submitResponse( : formatResponseForAgent(currentQuestion, responses); await continueAgentConversation(session, message); } + } catch (err) { + if (enteredGenerating && !session.error && !(err instanceof Error && err.name === "AbortError")) { + setSessionError(session, err instanceof Error ? err.message : "AI processing failed"); + } + throw err; } finally { releaseTurn(); } @@ -3234,6 +3319,8 @@ export async function retrySession( JSON" failure. Admission is reserved synchronously before any turn state is touched. */ const releaseTurn = reservePlanningTurn(session.id); + // FNXC:PlanningProviderErrors 2026-07-23-20:10: same generating-strand guard as submitResponse — see the comment there. + let enteredGenerating = false; try { disposeSessionAgentForRetry(session); @@ -3253,6 +3340,7 @@ export async function retrySession( session.updatedAt = new Date(); beginPlanningGeneration(session, session.history.length === 0 ? "initial_plan" : "plan_update"); await persistSession(session, "generating"); + enteredGenerating = true; if (pendingContextualComments) { await ensureSessionAgent(session, rootDir, session.history, promptOverrides, store); @@ -3281,6 +3369,11 @@ export async function retrySession( coerceResponseRecord(lastEntry.question, lastEntry.response), ); await continueAgentConversation(session, replayMessage); + } catch (err) { + if (enteredGenerating && !session.error && !(err instanceof Error && err.name === "AbortError")) { + setSessionError(session, err instanceof Error ? err.message : "AI processing failed"); + } + throw err; } finally { releaseTurn(); } diff --git a/packages/dashboard/src/routes/register-planning-subtask-routes.ts b/packages/dashboard/src/routes/register-planning-subtask-routes.ts index 6240f1b543..9a2896bacd 100644 --- a/packages/dashboard/src/routes/register-planning-subtask-routes.ts +++ b/packages/dashboard/src/routes/register-planning-subtask-routes.ts @@ -1743,7 +1743,7 @@ export function registerPlanningSubtaskRoutes(ctx: ApiRoutesContext, deps: Plann res.write(": connected\n\n"); try { - const { planningStreamManager, getSession } = await import("../planning.js"); + const { planningStreamManager, getSession, reconcileStalePlanningGeneration } = await import("../planning.js"); // Verify session exists const session = await getSession(sessionId); @@ -1762,6 +1762,27 @@ export function registerPlanningSubtaskRoutes(ctx: ApiRoutesContext, deps: Plann } } + /* + FNXC:PlanningProviderErrors 2026-07-23-20:10: + A session settled in a terminal error — or stranded in "generating" past the watchdog + window with no live turn — must end this stream with an error event instead of holding + the client on "Thinking/Generating plan" waiting for events that can never arrive + (the client's SSE reconnect loop and 8s poll both treat a persisted "generating" row as + healthy, so without this the hang is permanent). + */ + const terminalError = reconcileStalePlanningGeneration(sessionId); + if (terminalError) { + const existing = planningStreamManager.getBufferedEvents(sessionId, 0); + const lastErrorEvent = [...existing].reverse().find((event) => event.event === "error"); + const errorEventId = lastErrorEvent?.id + ?? planningStreamManager.broadcast(sessionId, { type: "error", data: terminalError }); + if (lastEventId === undefined || errorEventId > lastEventId) { + writeSSEEvent(res, "error", JSON.stringify(terminalError), errorEventId); + } + res.end(); + return; + } + /* FNXC:PlanningStreamTurnIdentity 2026-07-20-10:36: A running summary is persisted after every interview turn, so it is catch-up state rather