fix(dashboard): stop Planning Mode hanging on provider errors mid-generation

Provider errors thrown between persistSession("generating") and the turn's
own error handling (agent rebuild in ensureSessionAgent, history replay,
legacy sync createSession first turn) escaped to the route and left the
session row "generating" forever with no error, no SSE event, and no
watchdog — the modal hung on "Thinking/Generating plan" because its SSE
reconnect loop and 8s poll both treat a persisted "generating" row as
healthy.

- submitResponse/retrySession/createSession now convert any non-abort
  escape after entering "generating" into the standard persisted retryable
  error + SSE error broadcast before rethrowing.
- The SSE stream route reconciles settled/stranded sessions on connect
  (reconcileStalePlanningGeneration): a terminal error is replayed and the
  stream closed; a "generating" session with no live/pending turn past the
  watchdog window is converted to a retryable interrupted error.
- Provider failures on the JSON reformat retry now surface as themselves
  instead of a misleading "no valid JSON" parse error.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-07-23 18:14:29 -07:00
parent dcbf923e07
commit 716e698628
4 changed files with 349 additions and 3 deletions

View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Planning Mode no longer hangs on "Generating plan" after a provider error; it surfaces a retryable error.
category: fix
dev: Provider errors thrown after a planning session persists "generating" (agent rebuild, history replay, legacy sync start) now land the session in a persisted retryable error with an SSE error event; the stream route reconciles stranded generating sessions past the watchdog window via `reconcileStalePlanningGeneration`.

View File

@@ -0,0 +1,225 @@
// @vitest-environment node
/*
FNXC:PlanningProviderErrors 2026-07-23-20:10:
Regression tests for provider-error handling in dedicated Planning Mode. Reported bug: a
provider failure (auth error, overloaded provider, model-registry stall) thrown between
persistSession("generating") and the turn's own error handling escaped to the route with the
session row left "generating" forever — no persisted error, no SSE error event, no watchdog —
so the Planning modal hung on "Thinking/Generating plan" (its SSE reconnect loop and 8s poll
both treat a persisted "generating" row as healthy). Invariant: once a planning session enters
"generating", every non-abort failure lands it in a retryable persisted "error" state with an
SSE error event, and a stream connect against a settled/stranded session terminates with an
error event instead of waiting forever.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { EventEmitter } from "node:events";
import type { TaskStore } from "@fusion/core";
vi.mock("@fusion/engine", () => ({
listCliAdapterDescriptors: () => [],
resolveMcpServersForStore: async () => ({ servers: [] }),
buildSessionSkillContextSync: () => ({
skillSelectionContext: undefined,
resolvedSkillNames: ["fusion"],
skillSource: "role-fallback" as const,
}),
createFnAgent: vi.fn(),
createWorkflowAuthoringTools: () => [],
createChatTaskDocumentTools: () => [],
createChatTaskLogsReadTool: () => ({}),
}));
import {
__resetPlanningState,
__setCreateFnAgent,
createSession,
createSessionWithAgent,
getSession,
PLANNING_INTERRUPTED_ERROR_MESSAGE,
planningStreamManager,
reconcileStalePlanningGeneration,
retrySession,
setAiSessionStore,
submitResponse,
} from "../planning.js";
const MOCK_TASK_STORE = {
listTasks: vi.fn(async () => []),
getSettings: vi.fn(async () => ({})),
getTask: vi.fn(async () => {
throw new Error("not found");
}),
} as unknown as TaskStore;
const QUESTION_JSON = JSON.stringify({
type: "question",
data: { id: "q-next", type: "single_select", question: "What next?" },
});
const PROVIDER_ERROR_MESSAGE = "Provider rejected the request: 401 invalid_api_key";
function createScriptedAgent(responder: () => string = () => QUESTION_JSON) {
const messages: Array<{ role: string; content: string }> = [];
const prompt = vi.fn(async () => {
messages.push({ role: "assistant", content: responder() });
});
return { agent: { session: { state: { messages }, prompt, dispose: vi.fn() } } };
}
async function waitFor(predicate: () => Promise<boolean> | boolean, attempts = 50): Promise<void> {
for (let i = 0; i < attempts; i++) {
if (await predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 5));
}
throw new Error("condition not reached");
}
async function startSessionAwaitingInput(ip: string): Promise<string> {
const scripted = createScriptedAgent();
__setCreateFnAgent(vi.fn(async () => scripted.agent) as never);
const sessionId = await createSessionWithAgent(ip, "Plan something small", "/tmp/project", MOCK_TASK_STORE);
planningStreamManager.consumeInitialTurn(sessionId)?.();
await waitFor(async () => Boolean((await getSession(sessionId))?.currentQuestion));
return sessionId;
}
function bufferedErrorEvents(sessionId: string): unknown[] {
return planningStreamManager
.getBufferedEvents(sessionId, 0)
.filter((event: { event: string }) => event.event === "error");
}
describe("planning provider-error recovery", () => {
let upsertMock: ReturnType<typeof vi.fn>;
let storeGet: ReturnType<typeof vi.fn>;
beforeEach(() => {
__resetPlanningState();
upsertMock = vi.fn(async () => {});
storeGet = vi.fn(async () => null);
setAiSessionStore(Object.assign(new EventEmitter(), {
upsert: upsertMock,
get: storeGet,
updateThinking: vi.fn(),
}) as never);
});
it("submitResponse persists a retryable error when agent rebuild hits a provider failure after entering generating", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.1");
const session = (await getSession(sessionId))!;
const question = session.currentQuestion!;
// Force the ensureSessionAgent path (agent gone, e.g. after retry disposal) with a provider failure.
session.agent = undefined;
__setCreateFnAgent(vi.fn(async () => {
throw new Error(PROVIDER_ERROR_MESSAGE);
}) as never);
await expect(
submitResponse(sessionId, { [question.id]: "option-1" }, "/tmp/project", undefined, MOCK_TASK_STORE),
).rejects.toThrow(PROVIDER_ERROR_MESSAGE);
const after = (await getSession(sessionId))!;
expect(after.error).toContain(PROVIDER_ERROR_MESSAGE);
expect(bufferedErrorEvents(sessionId).length).toBeGreaterThan(0);
await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error"));
});
it("retrySession re-persists an error instead of stranding the session in generating", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.2");
const session = (await getSession(sessionId))!;
session.error = "AI returned no valid JSON. Retry this planning session or start a new one.";
session.agent = undefined;
storeGet.mockImplementation(async () => ({ id: sessionId, type: "planning", status: "error" }));
__setCreateFnAgent(vi.fn(async () => {
throw new Error(PROVIDER_ERROR_MESSAGE);
}) as never);
await expect(retrySession(sessionId, "/tmp/project", undefined, MOCK_TASK_STORE)).rejects.toThrow(PROVIDER_ERROR_MESSAGE);
const after = (await getSession(sessionId))!;
// retrySession clears the prior error before generating; the failure must restore a terminal error.
expect(after.error).toContain(PROVIDER_ERROR_MESSAGE);
await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error"));
});
it("legacy synchronous createSession persists an error when the provider fails before the first question", async () => {
__setCreateFnAgent(vi.fn(async () => {
throw new Error(PROVIDER_ERROR_MESSAGE);
}) as never);
await expect(
createSession("10.1.0.3", "Plan something small", MOCK_TASK_STORE, "/tmp/project"),
).rejects.toThrow(PROVIDER_ERROR_MESSAGE);
await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error"));
const errorRows = upsertMock.mock.calls.map((call) => call[0]).filter((row: { status?: string }) => row?.status === "error");
const errorRow = errorRows[errorRows.length - 1];
expect(errorRow?.error).toContain(PROVIDER_ERROR_MESSAGE);
});
it("surfaces the provider failure from the reformat retry instead of a misleading parse error", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.4");
const session = (await getSession(sessionId))!;
const question = session.currentQuestion!;
let calls = 0;
const scripted = createScriptedAgent(() => "definitely not json");
scripted.agent.session.prompt = vi.fn(async () => {
calls += 1;
if (calls === 1) {
scripted.agent.session.state.messages.push({ role: "assistant", content: "definitely not json" });
return;
}
throw new Error(PROVIDER_ERROR_MESSAGE);
});
session.agent = scripted.agent as never;
await submitResponse(sessionId, { [question.id]: "option-1" }, "/tmp/project", undefined, MOCK_TASK_STORE);
const after = (await getSession(sessionId))!;
expect(after.error).toContain(PROVIDER_ERROR_MESSAGE);
});
describe("reconcileStalePlanningGeneration", () => {
it("returns the persisted terminal error for a settled errored session", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.5");
const session = (await getSession(sessionId))!;
session.error = PROVIDER_ERROR_MESSAGE;
expect(reconcileStalePlanningGeneration(sessionId)).toBe(PROVIDER_ERROR_MESSAGE);
});
it("converts a stranded generating session past the watchdog window into a retryable error", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.6");
const session = (await getSession(sessionId))!;
session.generationPurpose = "plan_update";
session.generationStartedAt = new Date(Date.now() - 10 * 60_000).toISOString();
expect(reconcileStalePlanningGeneration(sessionId)).toBe(PLANNING_INTERRUPTED_ERROR_MESSAGE);
expect(session.error).toBe(PLANNING_INTERRUPTED_ERROR_MESSAGE);
expect(bufferedErrorEvents(sessionId).length).toBeGreaterThan(0);
await waitFor(() => upsertMock.mock.calls.some((call) => call[0]?.status === "error"));
});
it("leaves a fresh generating session alone", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.7");
const session = (await getSession(sessionId))!;
session.generationPurpose = "plan_update";
session.generationStartedAt = new Date().toISOString();
expect(reconcileStalePlanningGeneration(sessionId)).toBeUndefined();
expect(session.error).toBeUndefined();
});
it("leaves an awaiting-input session alone", async () => {
const sessionId = await startSessionAwaitingInput("10.1.0.8");
expect(reconcileStalePlanningGeneration(sessionId)).toBeUndefined();
expect((await getSession(sessionId))!.error).toBeUndefined();
});
});
});

View File

@@ -332,6 +332,7 @@ export const GENERATION_LOOP_REPEAT_LIMIT = 8;
const PLANNING_STUCK_ERROR_MESSAGE = "AI generation appears stuck with no new output. You can retry or start a new session.";
const PLANNING_LOOP_ERROR_MESSAGE = "AI generation appears stuck repeating the same output. You can retry or start a new session.";
export const PLANNING_INTERRUPTED_ERROR_MESSAGE = "Planning generation was interrupted before it finished. Retry to continue this session.";
// ── Types ───────────────────────────────────────────────────────────────────
@@ -1245,6 +1246,33 @@ export async function createSession(
beginPlanningGeneration(session, "initial_plan");
persistSession(session, "generating");
/*
FNXC:PlanningProviderErrors 2026-07-23-20:10:
Once the session row is persisted "generating", every failure on the way to the first
question (system-prompt resolution, agent construction, the provider prompt itself) must
land the session in a retryable persisted "error" state before rethrowing to the route.
Previously a provider error thrown here left the row "generating" forever with no error
and no watchdog, so the session appeared stuck on "Generating plan" until a server restart.
*/
try {
return await runCreateSessionFirstTurn(session, rootDir, store, promptOverrides, pluginRunner);
} catch (err) {
if (!(err instanceof Error && err.name === "AbortError") && !session.error) {
setSessionError(session, err instanceof Error ? err.message : "Failed to initialize AI agent");
}
throw err;
}
}
/** First turn of the legacy synchronous planning start; see the provider-error guard in createSession. */
async function runCreateSessionFirstTurn(
session: Session,
rootDir: string,
store: TaskStore,
promptOverrides?: PromptOverrideMap,
pluginRunner?: SkillPluginRunner,
): Promise<{ sessionId: string; firstQuestion: PlanningQuestion; summary: PlanningSummary; validated: boolean }> {
const sessionId = session.id;
const systemPrompt = await resolvePlanningModeSystemPrompt(store, promptOverrides, session.workflowId);
// Create AI agent and get the first question
@@ -1293,7 +1321,7 @@ export async function createSession(
session.agent = agentResult;
session.updatedAt = new Date();
const firstResponse = await getFirstQuestionFromAgent(session, formatInitialPlanRequestForAgent(initialPlan));
const firstResponse = await getFirstQuestionFromAgent(session, formatInitialPlanRequestForAgent(session.initialPlan));
const firstQuestion = firstResponse.data;
session.currentQuestion = firstQuestion;
@@ -1413,7 +1441,9 @@ async function getFirstQuestionFromAgent(
}
}
}
} catch {
} catch (retryPromptErr) {
// FNXC:PlanningProviderErrors 2026-07-23-20:10: a provider failure on the reformat prompt must surface as itself, not as a misleading "no valid JSON" parse error.
lastError = retryPromptErr instanceof Error ? retryPromptErr : new Error(String(retryPromptErr));
break;
}
}
@@ -2197,6 +2227,40 @@ function createAbortError(): Error {
return error;
}
/*
FNXC:PlanningProviderErrors 2026-07-23-20:10:
Terminal-state reconciliation for SSE stream connects. Two hang shapes ended with the Planning
modal pinned on "Thinking/Generating plan" with a stream that will never emit again:
1. The session already holds a terminal error, but the buffered error event is gone (the
client's reconnect loop treats a persisted "generating" row as healthy and the 8s poll only
reacts to persisted status changes).
2. The session is persisted "generating" with generation metadata set, but no turn is active
or pending and the generation started longer ago than the inactivity watchdog window — a
stranded run (e.g. a pre-fix provider-error escape or a crashed generation promise) that no
watchdog owns anymore.
Returns the terminal error message the stream route must emit before closing, or undefined
when the session is healthy. Only the stale case writes state: it persists the same retryable
error contract as every other planning failure so Retry and the bounded auto-retry recover it.
*/
export function reconcileStalePlanningGeneration(sessionId: string): string | undefined {
const session = sessions.get(sessionId);
if (!session || session.validated) return undefined;
if (isPlanningTurnActive(sessionId) || planningStreamManager.hasPendingInitialTurn(sessionId)) return undefined;
if (session.error) return session.error;
if (session.generationPurpose === undefined) return undefined;
const startedAtMs = session.generationStartedAt ? Date.parse(session.generationStartedAt) : Number.NaN;
const ageMs = Number.isNaN(startedAtMs) ? Number.POSITIVE_INFINITY : Date.now() - startedAtMs;
if (ageMs < GENERATION_TIMEOUT_MS) return undefined;
diagnostics.warn("Reconciling stranded planning generation to a retryable error", {
sessionId,
generationPurpose: session.generationPurpose,
generationStartedAt: session.generationStartedAt,
operation: "reconcile-stale-generation",
});
setSessionError(session, PLANNING_INTERRUPTED_ERROR_MESSAGE);
return PLANNING_INTERRUPTED_ERROR_MESSAGE;
}
/*
FNXC:PlanningContextCompaction 2026-07-22-22:40:
Long planning interviews accumulate the whole Q/A history in one agent session and can hit the
@@ -2681,6 +2745,8 @@ async function continueAgentConversation(session: Session, message: string): Pro
retryErr,
{ sessionId: session.id, operation: "retry-prompt" }
);
// FNXC:PlanningProviderErrors 2026-07-23-20:10: report the provider failure itself instead of a misleading "no valid JSON" parse error.
lastError = retryErr instanceof Error ? retryErr : new Error(String(retryErr));
break;
}
}
@@ -3090,6 +3156,17 @@ export async function submitResponse(
generation-error case so the modal's submit path keeps its existing SSE-driven recovery.
*/
let answeredQuestion: PlanningQuestion | undefined;
/*
FNXC:PlanningProviderErrors 2026-07-23-20:10:
ensureSessionAgent (agent construction + history-replay prompt) throws provider errors AFTER
the session row was persisted "generating". continueAgentConversation converts its own
failures to a persisted session error, but a throw between persist("generating") and that
call previously escaped to the route with the row left "generating" forever — no error
event, no watchdog — so the Planning modal hung on "Thinking/Generating plan" (its SSE
reconnect + 8s poll both treat "generating" as healthy). Track the generating transition and
convert any non-abort escape into the same retryable persisted error state.
*/
let enteredGenerating = false;
try {
const contextualComments = getContextualComments(responses);
@@ -3104,6 +3181,7 @@ export async function submitResponse(
session.error = undefined;
session.pendingContextualComments = contextualComments;
await persistSession(session, "generating");
enteredGenerating = true;
await ensureSessionAgent(session, rootDir, session.history, promptOverrides, store);
await continueAgentConversation(session, formatContextualCommentsForAgent(session.summary, contextualComments));
} else if (isRefineRequest(responses) && session.summary) {
@@ -3113,6 +3191,7 @@ export async function submitResponse(
session.currentQuestion = undefined;
session.error = undefined;
await persistSession(session, "generating");
enteredGenerating = true;
await ensureSessionAgent(session, rootDir, session.history, promptOverrides, store);
const focus = typeof responses.focus === "string" ? responses.focus.trim() : undefined;
@@ -3154,6 +3233,7 @@ export async function submitResponse(
beginPlanningGeneration(session, "plan_update");
session.currentQuestion = undefined;
await persistSession(session, "generating");
enteredGenerating = true;
if (!session.agent) {
// An edited older answer must be replayed in its original position with every
// later answer retained; only a newly appended answer is sent after replay.
@@ -3170,6 +3250,11 @@ export async function submitResponse(
: formatResponseForAgent(currentQuestion, responses);
await continueAgentConversation(session, message);
}
} catch (err) {
if (enteredGenerating && !session.error && !(err instanceof Error && err.name === "AbortError")) {
setSessionError(session, err instanceof Error ? err.message : "AI processing failed");
}
throw err;
} finally {
releaseTurn();
}
@@ -3234,6 +3319,8 @@ export async function retrySession(
JSON" failure. Admission is reserved synchronously before any turn state is touched.
*/
const releaseTurn = reservePlanningTurn(session.id);
// FNXC:PlanningProviderErrors 2026-07-23-20:10: same generating-strand guard as submitResponse — see the comment there.
let enteredGenerating = false;
try {
disposeSessionAgentForRetry(session);
@@ -3253,6 +3340,7 @@ export async function retrySession(
session.updatedAt = new Date();
beginPlanningGeneration(session, session.history.length === 0 ? "initial_plan" : "plan_update");
await persistSession(session, "generating");
enteredGenerating = true;
if (pendingContextualComments) {
await ensureSessionAgent(session, rootDir, session.history, promptOverrides, store);
@@ -3281,6 +3369,11 @@ export async function retrySession(
coerceResponseRecord(lastEntry.question, lastEntry.response),
);
await continueAgentConversation(session, replayMessage);
} catch (err) {
if (enteredGenerating && !session.error && !(err instanceof Error && err.name === "AbortError")) {
setSessionError(session, err instanceof Error ? err.message : "AI processing failed");
}
throw err;
} finally {
releaseTurn();
}

View File

@@ -1743,7 +1743,7 @@ export function registerPlanningSubtaskRoutes(ctx: ApiRoutesContext, deps: Plann
res.write(": connected\n\n");
try {
const { planningStreamManager, getSession } = await import("../planning.js");
const { planningStreamManager, getSession, reconcileStalePlanningGeneration } = await import("../planning.js");
// Verify session exists
const session = await getSession(sessionId);
@@ -1762,6 +1762,27 @@ export function registerPlanningSubtaskRoutes(ctx: ApiRoutesContext, deps: Plann
}
}
/*
FNXC:PlanningProviderErrors 2026-07-23-20:10:
A session settled in a terminal error — or stranded in "generating" past the watchdog
window with no live turn — must end this stream with an error event instead of holding
the client on "Thinking/Generating plan" waiting for events that can never arrive
(the client's SSE reconnect loop and 8s poll both treat a persisted "generating" row as
healthy, so without this the hang is permanent).
*/
const terminalError = reconcileStalePlanningGeneration(sessionId);
if (terminalError) {
const existing = planningStreamManager.getBufferedEvents(sessionId, 0);
const lastErrorEvent = [...existing].reverse().find((event) => event.event === "error");
const errorEventId = lastErrorEvent?.id
?? planningStreamManager.broadcast(sessionId, { type: "error", data: terminalError });
if (lastEventId === undefined || errorEventId > lastEventId) {
writeSSEEvent(res, "error", JSON.stringify(terminalError), errorEventId);
}
res.end();
return;
}
/*
FNXC:PlanningStreamTurnIdentity 2026-07-20-10:36:
A running summary is persisted after every interview turn, so it is catch-up state rather