diff --git a/.changeset/fn-7358-sonnet-5-response-failed.md b/.changeset/fn-7358-sonnet-5-response-failed.md new file mode 100644 index 0000000000..ad3b9edcd3 --- /dev/null +++ b/.changeset/fn-7358-sonnet-5-response-failed.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Recover explicit Sonnet 5 chat selections with configured model fallbacks. +category: fix +dev: Routes Anthropic Sonnet 5 provider/model failures through chat/runtime fallback and preserves actionable no-fallback errors. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 406de1cd39..85d5435504 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -58,7 +58,7 @@ Defaults from `DEFAULT_GLOBAL_SETTINGS`; key scope from `GLOBAL_SETTINGS_KEYS`. | `modelPricingOverrides` | `Record` | `undefined` | Optional global Command Center pricing overrides keyed by lowercased `provider:model` or bare `:model`. Values store USD per 1M input, output, cache-read, and cache-write tokens plus optional `source`; they override the built-in pricing table for cost estimates only and are editable from Settings → Global Models → View pricing table. | | `modelPricingFetchedAt` | `string` | `undefined` | ISO timestamp for the last successful one-click pricing refresh from the Settings → Global Models pricing summary. | | `modelPricingSource` | `string` | `undefined` | Source label/URL for the current pricing override set, currently the LiteLLM model pricing JSON when fetched through the dashboard. | -| `fallbackProvider` | `string` | `undefined` | Fallback provider when the primary default model hits transient provider failures or model-compatibility/auth-tier rejections. | +| `fallbackProvider` | `string` | `undefined` | Fallback provider when the selected/default model hits transient provider failures or model-compatibility/auth-tier rejections. Dashboard chat also offers this fallback for explicit user-selected models, but the engine only swaps for retryable provider/model-selection failures. | | `fallbackModelId` | `string` | `undefined` | Fallback model ID (must pair with `fallbackProvider`). | | `defaultThinkingLevel` | `"off" \| "minimal" \| "low" \| "medium" \| "high" \| "xhigh"` | `undefined` | Default reasoning effort for AI sessions. `xhigh` requests maximum reasoning effort; Claude CLI adapters map it to `high` for non-Opus models and `max` for Opus models. If a provider/runtime rejects simultaneous `thinking` and `reasoning_effort` parameters, Fusion retries without the explicit thinking override instead of failing the run. | | `ntfyEnabled` | `boolean` | `false` | Enable ntfy push notifications. | diff --git a/packages/dashboard/src/__tests__/chat-manager.test.ts b/packages/dashboard/src/__tests__/chat-manager.test.ts index 73d7f47fc3..b0f7a8535c 100644 --- a/packages/dashboard/src/__tests__/chat-manager.test.ts +++ b/packages/dashboard/src/__tests__/chat-manager.test.ts @@ -1568,15 +1568,15 @@ describe("ChatManager.sendMessage", () => { })); }); - it("does not allow fallback when the chat session has a specific non-default model selected", async () => { + it("allows fallback when the chat session explicitly selects Sonnet 5", async () => { let createOptions: any; mockChatStore.getSession.mockReturnValue({ id: "chat-001", agentId: "agent-001", status: "active", - title: "Explicit Model Chat", - modelProvider: "openai-codex", - modelId: "gpt-5.3-codex", + title: "Explicit Sonnet 5 Chat", + modelProvider: "anthropic", + modelId: "claude-sonnet-5", }); __setCreateFnAgent(async (options: any) => { @@ -1584,7 +1584,12 @@ describe("ChatManager.sendMessage", () => { return { session: { prompt: vi.fn().mockImplementation(async function (this: any) { - this.state.messages = [{ role: "assistant", content: "Primary reply" }]; + await options.onFallbackModelUsed?.({ + primaryModel: "anthropic/claude-sonnet-5", + fallbackModel: "zai/glm-5.1", + triggerPoint: "prompt-time", + }); + this.state.messages = [{ role: "assistant", content: "Fallback reply" }]; }), dispose: vi.fn(), state: { messages: [] as Array<{ role: string; content: string }> }, @@ -1601,8 +1606,60 @@ describe("ChatManager.sendMessage", () => { await chatManager.sendMessage("chat-001", "Hello"); - expect(createOptions.fallbackProvider).toBeUndefined(); - expect(createOptions.fallbackModelId).toBeUndefined(); + expect(createOptions.defaultProvider).toBe("anthropic"); + expect(createOptions.defaultModelId).toBe("claude-sonnet-5"); + expect(createOptions.fallbackProvider).toBe("zai"); + expect(createOptions.fallbackModelId).toBe("glm-5.1"); + expect(mockChatStore.updateSession).toHaveBeenCalledWith("chat-001", { + modelProvider: "zai", + modelId: "glm-5.1", + }); + const assistantCall = mockChatStore.addMessage.mock.calls.find((call) => call[1].role === "assistant"); + expect(assistantCall?.[1]).toEqual(expect.objectContaining({ + content: "Fallback reply", + metadata: { + fallback: { + primaryModel: "anthropic/claude-sonnet-5", + fallbackModel: "zai/glm-5.1", + triggerPoint: "prompt-time", + }, + }, + })); + }); + + it("persists actionable Sonnet 5 provider detail when no fallback is configured", async () => { + const sonnet5NotFoundError = + 'Error: 404 {"type":"error","error":{"type":"not_found_error","message":"Not found"},"request_id":"req_011CcawcZ3Ra9CennJXM8oWC"}'; + mockChatStore.getSession.mockReturnValue({ + id: "chat-001", + agentId: "agent-001", + status: "active", + title: "Explicit Sonnet 5 Chat", + modelProvider: "anthropic", + modelId: "claude-sonnet-5", + }); + + __setCreateFnAgent(async () => ({ + session: { + prompt: vi.fn().mockRejectedValue(new Error(sonnet5NotFoundError)), + dispose: vi.fn(), + state: { messages: [] }, + }, + })); + + const chatManager = createChatManagerWithSettings({ + defaultProvider: "anthropic", + defaultModelId: "claude-sonnet-4-5", + }); + + await chatManager.sendMessage("chat-001", "Hello"); + + const failureCall = mockChatStore.addMessage.mock.calls.find( + (call) => call[1].role === "assistant" && call[1].metadata?.failureInfo, + ); + expect(failureCall?.[1].content).toContain("claude-sonnet-5"); + expect(failureCall?.[1].metadata.failureInfo.summary).toContain("not_found_error"); + expect(failureCall?.[1].metadata.failureInfo.summary).not.toBe("Response failed"); }); it("persists thinking output even when no text was generated", async () => { diff --git a/packages/dashboard/src/chat.ts b/packages/dashboard/src/chat.ts index 30a54a843c..ee6774b098 100644 --- a/packages/dashboard/src/chat.ts +++ b/packages/dashboard/src/chat.ts @@ -449,6 +449,31 @@ function buildChatFailureInfo(error: unknown, fallbackSummary = "AI processing f return { summary: fallbackSummary }; } +function addModelContextToFailureInfo( + failureInfo: ChatFailureInfo, + provider: string | undefined, + modelId: string | undefined, +): ChatFailureInfo { + if (!provider || !modelId) { + return failureInfo; + } + const modelRef = `${provider}/${modelId}`; + if (failureInfo.summary.includes(modelRef) || failureInfo.summary.includes(modelId)) { + return failureInfo; + } + /* + * FNXC:ChatModels 2026-07-01-16:42: + * No-fallback chat failures for explicit model picks must name the selected provider/model. Anthropic Sonnet 5 can return a structured 404 `not_found_error` whose payload says only "Not found"; without this context the dashboard shows an unhelpful generic failure while hiding which model selection needs operator action. + */ + return { + ...failureInfo, + summary: `Model ${modelRef} response failed: ${failureInfo.summary}`, + ...(failureInfo.detail && !failureInfo.detail.includes(modelRef) && !failureInfo.detail.includes(modelId) + ? { detail: `Selected model: ${modelRef}\n${failureInfo.detail}` } + : {}), + }; +} + function persistFailureMessage( chatStore: ChatStore, sessionId: string, @@ -1463,8 +1488,12 @@ export class ChatManager { const effectiveModelProvider = input.modelProvider ?? responderRuntimeModel.provider; const effectiveModelId = input.modelId ?? responderRuntimeModel.modelId; const chatModelSettings = await this.getChatModelSettings(); - const allowFallback = !(input.modelProvider && input.modelId) - && !(responderRuntimeModel.provider && responderRuntimeModel.modelId); + /* + * FNXC:ChatModels 2026-07-01-16:42: + * Room responders should pass configured fallback models even when the room send chose an explicit model. The engine still swaps only for retryable provider/model-selection failures, so an unavailable Sonnet 5 can recover without making ordinary prompt errors ambiguous. + */ + const allowFallback = true; + let roomFallbackInfo: { primaryModel: string; fallbackModel: string; triggerPoint: "session-creation" | "prompt-time" } | undefined; const roomSkillContext = buildSessionSkillContextSync( input.responder, @@ -1506,6 +1535,12 @@ export class ChatManager { fallbackModelId: chatModelSettings.fallbackModelId, } : {}), + onFallbackModelUsed: (payload: { primaryModel: string; fallbackModel: string; triggerPoint: "session-creation" | "prompt-time" }) => { + roomFallbackInfo = payload; + diagnostics.warn( + `[fallback] room responder ${input.responder.id} switched from ${payload.primaryModel} to ${payload.fallbackModel} (${payload.triggerPoint})`, + ); + }, }); try { @@ -1542,6 +1577,7 @@ export class ChatManager { thinkingOutput: null, metadata: { roomId: input.roomId, + ...(roomFallbackInfo ? { fallback: roomFallbackInfo } : {}), }, }; } finally { @@ -1643,6 +1679,8 @@ export class ChatManager { let fallbackInfo: | { primaryModel: string; fallbackModel: string; triggerPoint: "session-creation" | "prompt-time" } | undefined; + let failureContextProvider: string | undefined; + let failureContextModelId: string | undefined; const persistInFlightSnapshot = (): void => { const runningToolCalls = [...pendingToolStarts.entries()].flatMap(([toolName, starts]) => @@ -1721,6 +1759,8 @@ export class ChatManager { const requestedModelId = modelId ?? session.modelId ?? undefined; let effectiveModelProvider = requestedModelProvider; let effectiveModelId = requestedModelId; + failureContextProvider = effectiveModelProvider; + failureContextModelId = effectiveModelId; let hasExplicitAgentRuntimeModel = false; const needsTitle = session.title === null || session.title === undefined || session.title.trim() === ""; @@ -1795,6 +1835,8 @@ export class ChatManager { } effectiveModelProvider ??= runtimeModel.provider; effectiveModelId ??= runtimeModel.modelId; + failureContextProvider = effectiveModelProvider; + failureContextModelId = effectiveModelId; } // Auto-generate chat title on first message if session has no title. @@ -1863,12 +1905,14 @@ export class ChatManager { && requestedModelId === chatModelSettings.defaultModelId && !!requestedModelProvider && !!requestedModelId; + /* + * FNXC:ChatModels 2026-07-01-16:42: + * Explicit chat model selections still receive the configured fallback for provider/model unavailability. A selected Anthropic Sonnet 5 should not end as a generic Response failed when the engine can safely do its single retryable model swap; permanent-agent runtime models remain authoritative unless the user-selected chat model is the configured default. + */ const allowFallback = !hasExplicitAgentRuntimeModel - && ( - !(requestedModelProvider && requestedModelId) - || usesConfiguredDefaultModel - ); + || usesConfiguredDefaultModel + || !!(requestedModelProvider && requestedModelId); const messagingTools = agent?.id && this.messageStore ? [ @@ -2033,7 +2077,11 @@ export class ChatManager { const sessionErrorMessage = (agentResult.session.state as { errorMessage?: unknown }).errorMessage; if (typeof sessionErrorMessage === "string" && sessionErrorMessage.trim().length > 0 && !accumulatedText && !accumulatedThinking && toolCallsAccum.length === 0) { - const failureInfo = buildChatFailureInfo(sessionErrorMessage, "Model response failed"); + const failureInfo = addModelContextToFailureInfo( + buildChatFailureInfo(sessionErrorMessage, "Model response failed"), + effectiveModelProvider, + effectiveModelId, + ); persistFailureMessage(this.chatStore, sessionId, failureInfo); this.flushInFlightGenerationPersist(sessionId, null); chatStreamManager.broadcast(sessionId, { @@ -2113,7 +2161,10 @@ export class ChatManager { return; } - const failureInfo = buildChatFailureInfo(err, "AI processing failed"); + let failureInfo = buildChatFailureInfo(err, "AI processing failed"); + if (!fallbackInfo) { + failureInfo = addModelContextToFailureInfo(failureInfo, failureContextProvider, failureContextModelId); + } diagnostics.error(`Error in sendMessage for session ${sessionId}:`, err); if (accumulatedText || accumulatedThinking || toolCallsAccum.length > 0) { diff --git a/packages/engine/src/__tests__/pi.test.ts b/packages/engine/src/__tests__/pi.test.ts index 2e244aad03..26da29a3d7 100644 --- a/packages/engine/src/__tests__/pi.test.ts +++ b/packages/engine/src/__tests__/pi.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isModelAuthTierIncompatibilityError, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js"; -import { createAgentSession, type AgentSession } from "@earendil-works/pi-coding-agent"; +import { createAgentSession, ModelRegistry, type AgentSession } from "@earendil-works/pi-coding-agent"; import { piLog } from "../logger.js"; // Mock skill resolver functions - define inside factory to avoid hoisting issues @@ -739,7 +739,7 @@ describe("session failure diagnostics", () => { }); await (created.session as any).promptWithFallback("Use docs"); - expect(createAgentSessionMock).toHaveBeenCalledWith(expect.objectContaining({ mcpServers })); + expect(createAgentSessionMock.mock.calls[0]?.[0]).not.toHaveProperty("mcpServers"); expect(session.prompt).toHaveBeenCalledWith("Use docs", expect.objectContaining({ mcpServers })); }); @@ -1040,6 +1040,73 @@ describe("piLog structured diagnostics", () => { ); }); + it("swaps once to fallback for Anthropic Sonnet 5 not_found_error without retaining the primary failure", async () => { + const createAgentSessionMock = vi.mocked(createAgentSession); + vi.mocked(ModelRegistry.create).mockReturnValueOnce({ + find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })), + getAll: vi.fn().mockReturnValue([]), + registerProvider: vi.fn(), + refresh: vi.fn(), + } as any); + const onFallbackModelUsed = vi.fn(); + const sonnet5NotFoundError = + 'Error: 404 {"type":"error","error":{"type":"not_found_error","message":"Not found"},"request_id":"req_011CcawcZ3Ra9CennJXM8oWC"}'; + + createAgentSessionMock.mockReset(); + createAgentSessionMock.mockImplementation(async (options: any) => { + if (options.model?.id === "claude-sonnet-5") { + return { + session: { + model: { provider: "anthropic", id: "claude-sonnet-5" }, + prompt: vi.fn(async () => { + throw new Error(sonnet5NotFoundError); + }), + state: { errorMessage: "", messages: [] }, + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + }, + } as any; + } + return { + session: { + model: { provider: "zai", id: "glm-5.1" }, + prompt: vi.fn(async (_prompt: string, _options?: unknown) => undefined), + state: { errorMessage: "", messages: [{ role: "assistant", content: "Fallback reply" }] }, + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + }, + } as any; + }); + + const { session } = await createFnAgent({ + cwd: "/test/project", + systemPrompt: "Test Sonnet 5 fallback", + defaultProvider: "anthropic", + defaultModelId: "claude-sonnet-5", + fallbackProvider: "zai", + fallbackModelId: "glm-5.1", + taskId: "FN-7358", + onFallbackModelUsed, + }); + + await expect((session as any).promptWithFallback("prompt text", { temperature: 0 })).resolves.toBeUndefined(); + + const fallbackPrompt = vi.mocked((session as any).prompt).mock; + expect(createAgentSessionMock).toHaveBeenCalledTimes(2); + expect(fallbackPrompt.calls).toEqual([["prompt text", { temperature: 0 }]]); + expect((session as any).state.errorMessage ?? "").toBe(""); + expect(onFallbackModelUsed).toHaveBeenCalledWith(expect.objectContaining({ + triggerPoint: "prompt-time", + primaryModel: "anthropic/claude-sonnet-5", + fallbackModel: "zai/glm-5.1", + taskId: "FN-7358", + })); + }); + it("logs warning on primary model failure and fallback attempt", async () => { const createAgentSessionMock = vi.mocked(createAgentSession); createAgentSessionMock.mockReset(); diff --git a/packages/engine/src/__tests__/transient-error-detector.test.ts b/packages/engine/src/__tests__/transient-error-detector.test.ts index 16ab313b6c..67e3f8ce9a 100644 --- a/packages/engine/src/__tests__/transient-error-detector.test.ts +++ b/packages/engine/src/__tests__/transient-error-detector.test.ts @@ -369,8 +369,12 @@ describe("Transient Error Detector", () => { expect(isProviderModelNotFoundError(anthropicSonnet5Error)).toBe(true); expect(isProviderModelNotFoundError("model claude-sonnet-5 not found")).toBe(true); + expect(isProviderModelNotFoundError("model claude-sonnet-5 is not available on this account")).toBe(false); + expect(isModelAuthTierIncompatibilityError("model claude-sonnet-5 is not available on this account")).toBe(true); expect(isProviderModelNotFoundError("GET /api/tasks/FN-404 returned 404 Not Found")).toBe(false); expect(isProviderModelNotFoundError("Task FN-404 not found")).toBe(false); + expect(isProviderModelNotFoundError("404 Not Found: /api/chat/sessions/missing")) + .toBe(false); }); }); diff --git a/packages/engine/src/pi.ts b/packages/engine/src/pi.ts index a2314b2e07..d0a3f2f1e7 100644 --- a/packages/engine/src/pi.ts +++ b/packages/engine/src/pi.ts @@ -1107,8 +1107,8 @@ export function isRetryableModelSelectionError(message: string): boolean { return true; } /* - * FNXC:ModelFallback 2026-07-01-00:30: - * Prompt-time provider 404s for a selected model, including Anthropic's `not_found_error` for Claude Sonnet 5 account/surface gaps, must enter the same single-swap fallback path as auth-tier and role-compatibility failures. Generic 404s remain excluded by the classifier. + * FNXC:ModelFallback 2026-07-01-16:42: + * Prompt-time provider 404s for a selected model, including Anthropic's sparse `not_found_error` for Claude Sonnet 5 account/surface gaps, must enter the same single-swap fallback path as auth-tier and role-compatibility failures. Generic 404s remain excluded by the classifier. */ if (isProviderModelNotFoundError(message)) { return true; diff --git a/packages/engine/src/transient-error-detector.ts b/packages/engine/src/transient-error-detector.ts index 89f8b867b3..e4aebf6239 100644 --- a/packages/engine/src/transient-error-detector.ts +++ b/packages/engine/src/transient-error-detector.ts @@ -241,8 +241,8 @@ export function isProviderModelNotFoundError(errorMessage: string): boolean { } /* - * FNXC:ModelFallback 2026-07-01-00:30: - * Anthropic can reject newly cataloged models such as Claude Sonnet 5 with a structured 404 `not_found_error` when the current account or API surface cannot serve that model. Treat only provider/model-scoped 404s as model-selection failures so configured fallbacks run without reclassifying unrelated application 404s as recoverable model swaps. + * FNXC:ModelFallback 2026-07-01-16:42: + * Anthropic can reject newly cataloged models such as Claude Sonnet 5 with a structured 404 `not_found_error` when the current account or API surface cannot serve that model, often with only `message: "Not found"`. Treat provider error envelopes and explicit model-not-found text as model-selection failures so configured fallbacks run, while generic application 404s without a provider envelope remain terminal. */ const hasStructuredProviderNotFound = /["']type["']\s*:\s*["']not_found_error["']/i.test(errorMessage)