diff --git a/.changeset/fn-7437-planner-fallback.md b/.changeset/fn-7437-planner-fallback.md new file mode 100644 index 0000000000..9c075e122d --- /dev/null +++ b/.changeset/fn-7437-planner-fallback.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Stop planner model fallback loops with a clear terminal triage error. +category: fix +dev: Bounds prompt-time/session-creation model fallback exhaustion and persists failed triage state. diff --git a/packages/engine/src/__tests__/fallback-model-observer.test.ts b/packages/engine/src/__tests__/fallback-model-observer.test.ts index 7d418e8d57..ffda9552e7 100644 --- a/packages/engine/src/__tests__/fallback-model-observer.test.ts +++ b/packages/engine/src/__tests__/fallback-model-observer.test.ts @@ -53,6 +53,37 @@ describe("createFallbackModelObserver", () => { }); }); + it("writes fallback events as non-empty rows with readable delimiters", async () => { + const store = { + logEntry: vi.fn().mockResolvedValue(undefined), + appendAgentLog: vi.fn().mockResolvedValue(undefined), + }; + const observer = createFallbackModelObserver({ + agent: "triage", + label: "triage", + store, + taskId: "FN-7437", + }); + + await observer({ + primaryModel: "openai/gpt-4o", + fallbackModel: "anthropic/claude-3-5-haiku-20241022", + triggerPoint: "prompt-time", + }); + await observer({ + primaryModel: "openai/gpt-4o", + fallbackModel: "anthropic/claude-3-5-haiku-20241022", + triggerPoint: "prompt-time", + }); + + const rows = store.appendAgentLog.mock.calls.map((call) => call[1]); + expect(rows).toEqual([ + "[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)", + "[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)", + ]); + expect(rows.every((row) => row.trim() === row && row.includes(" switched from ") && row.includes(" to "))).toBe(true); + }); + it("swallows logging failures and still dispatches a notification", async () => { const store = { logEntry: vi.fn().mockRejectedValue(new Error("log failed")), diff --git a/packages/engine/src/__tests__/pi.test.ts b/packages/engine/src/__tests__/pi.test.ts index 577b260d08..8118bc6143 100644 --- a/packages/engine/src/__tests__/pi.test.ts +++ b/packages/engine/src/__tests__/pi.test.ts @@ -993,6 +993,112 @@ describe("piLog structured diagnostics", () => { ); }); + it("throws a bounded fallback exhaustion error when prompt-time fallback also fails", async () => { + const createAgentSessionMock = vi.mocked(createAgentSession); + vi.mocked(ModelRegistry.create).mockReturnValueOnce({ + find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })), + getAll: vi.fn().mockReturnValue([]), + registerProvider: vi.fn(), + refresh: vi.fn(), + } as any); + const onFallbackModelUsed = vi.fn(); + + const primarySession = { + model: { provider: "openai", id: "gpt-4o" }, + prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")), + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + } as unknown as AgentSession; + + const fallbackSession = { + model: { provider: "anthropic", id: "claude-3-5-haiku-20241022" }, + prompt: vi.fn().mockRejectedValue(new Error("401 invalid api key for fallback")), + state: { errorMessage: "", messages: [] }, + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + } as unknown as AgentSession; + + createAgentSessionMock.mockReset(); + createAgentSessionMock + .mockResolvedValueOnce({ session: primarySession } as any) + .mockResolvedValueOnce({ session: fallbackSession } as any); + + const { session } = await createFnAgent({ + cwd: "/test/project", + systemPrompt: "Test planner fallback exhaustion", + defaultProvider: "openai", + defaultModelId: "gpt-4o", + fallbackProvider: "anthropic", + fallbackModelId: "claude-3-5-haiku-20241022", + taskId: "FN-7437", + onFallbackModelUsed, + }); + + await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({ + name: "ModelFallbackExhaustedError", + attempts: 2, + primaryModel: "openai/gpt-4o", + fallbackModel: "anthropic/claude-3-5-haiku-20241022", + triggerPoint: "prompt-time", + }); + + expect(createAgentSessionMock).toHaveBeenCalledTimes(2); + expect(primarySession.prompt).toHaveBeenCalledTimes(1); + expect(fallbackSession.prompt).toHaveBeenCalledTimes(1); + expect(onFallbackModelUsed).toHaveBeenCalledTimes(1); + expect(onFallbackModelUsed).toHaveBeenCalledWith(expect.objectContaining({ + triggerPoint: "prompt-time", + primaryModel: "openai/gpt-4o", + fallbackModel: "anthropic/claude-3-5-haiku-20241022", + taskId: "FN-7437", + })); + }); + + it("does not create a meaningless prompt-time fallback when primary and fallback match", async () => { + const createAgentSessionMock = vi.mocked(createAgentSession); + vi.mocked(ModelRegistry.create).mockReturnValueOnce({ + find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })), + getAll: vi.fn().mockReturnValue([]), + registerProvider: vi.fn(), + refresh: vi.fn(), + } as any); + const onFallbackModelUsed = vi.fn(); + const primarySession = { + model: { provider: "openai", id: "gpt-4o" }, + prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")), + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + } as unknown as AgentSession; + + createAgentSessionMock.mockReset(); + createAgentSessionMock.mockResolvedValueOnce({ session: primarySession } as any); + + const { session } = await createFnAgent({ + cwd: "/test/project", + systemPrompt: "Test same fallback", + defaultProvider: "openai", + defaultModelId: "gpt-4o", + fallbackProvider: "openai", + fallbackModelId: "gpt-4o", + onFallbackModelUsed, + }); + + await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({ + name: "ModelFallbackExhaustedError", + attempts: 1, + primaryModel: "openai/gpt-4o", + fallbackModel: undefined, + }); + expect(createAgentSessionMock).toHaveBeenCalledTimes(1); + expect(onFallbackModelUsed).not.toHaveBeenCalled(); + }); + it("fires fallback hook on prompt-time model-auth-tier fallback", async () => { const createAgentSessionMock = vi.mocked(createAgentSession); const onFallbackModelUsed = vi.fn(); diff --git a/packages/engine/src/__tests__/triage.test.ts b/packages/engine/src/__tests__/triage.test.ts index 0e5cc23ed2..5531cb2a41 100644 --- a/packages/engine/src/__tests__/triage.test.ts +++ b/packages/engine/src/__tests__/triage.test.ts @@ -29,7 +29,28 @@ vi.mock("../reviewer.js", () => ({ reviewStep: mockReviewStep, })); -vi.mock("../pi.js", () => ({ +vi.mock("../pi.js", () => { + class ModelFallbackExhaustedError extends Error { + readonly primaryModel: string; + readonly fallbackModel?: string; + readonly triggerPoint: "session-creation" | "prompt-time"; + readonly attempts: number; + readonly underlyingReason: string; + + constructor(input: { primaryModel: string; fallbackModel?: string; triggerPoint: "session-creation" | "prompt-time"; attempts: number; underlyingReason: string }) { + const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured"; + super(`Unable to select a usable model after ${input.attempts} attempts (primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`); + this.name = "ModelFallbackExhaustedError"; + this.primaryModel = input.primaryModel; + this.fallbackModel = input.fallbackModel; + this.triggerPoint = input.triggerPoint; + this.attempts = input.attempts; + this.underlyingReason = input.underlyingReason; + } + } + + return { + ModelFallbackExhaustedError, createFnAgent: mockCreateFnAgent, describeModel: vi.fn().mockReturnValue("mock-model"), formatModelMarkerDetails: vi.fn((model: string, thinking?: string | null, annotations: string[] = []) => { @@ -37,7 +58,8 @@ vi.mock("../pi.js", () => ({ return suffixes.length ? `${model} ${suffixes.map((suffix) => `(${suffix})`).join(" ")}` : model; }), promptWithFallback: vi.fn().mockReturnValue("mock-prompt"), -})); + }; +}); vi.mock("@fusion/core", async (importOriginal) => { const { createEngineCoreMock } = await import("../test/mockCore.js"); @@ -3440,6 +3462,13 @@ describe("taskCreate tool model inheritance", () => { }); describe("bounded recovery retries for triage", () => { + beforeEach(async () => { + vi.clearAllMocks(); + const { promptWithFallback } = await import("../pi.js"); + (promptWithFallback as ReturnType).mockReset(); + (promptWithFallback as ReturnType).mockResolvedValue(undefined); + }); + it("requeues triage with backoff when the agent exits without writing PROMPT.md", async () => { const task = { id: "FN-202", @@ -3521,6 +3550,134 @@ describe("taskCreate tool model inheritance", () => { })); }); + it("persists terminal planning error when prompt-time primary and fallback models are exhausted", async () => { + const task = { + id: "FN-7437", + description: "Bug: planner triage fallback loop", + column: "triage", + dependencies: [], + steps: [], + currentStep: 0, + log: [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as unknown as Task; + const onSpecifyError = vi.fn(); + const store = createMockStore({ + getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }), + getSettings: vi.fn().mockResolvedValue({ + maxConcurrent: 2, + maxWorktrees: 4, + pollIntervalMs: 10000, + groupOverlappingFiles: false, + autoMerge: true, + defaultProvider: "openai", + defaultModelId: "gpt-4o", + planningFallbackProvider: "anthropic", + planningFallbackModelId: "claude-3-5-haiku-20241022", + defaultThinkingLevel: "low", + } as Settings), + }); + const mockDispose = vi.fn(); + mockCreateFnAgent.mockResolvedValue({ + session: { + state: {}, + sessionManager: {}, + prompt: vi.fn(), + dispose: mockDispose, + navigateTree: vi.fn(), + }, + }); + const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js"); + (promptWithFallback as ReturnType).mockRejectedValueOnce( + new ModelFallbackExhaustedError({ + primaryModel: "openai/gpt-4o", + fallbackModel: "anthropic/claude-3-5-haiku-20241022", + triggerPoint: "prompt-time", + attempts: 2, + underlyingReason: "401 invalid api key for fallback", + }), + ); + + const processor = new TriageProcessor(store, "/test/root", { + pollIntervalMs: 100_000, + onSpecifyError, + }); + + await processor.specifyTask(task); + + expect(store.logEntry).toHaveBeenCalledWith( + "FN-7437", + "Triage using model: mock-model (thinking effort: low)", + ); + expect(store.appendAgentLog).toHaveBeenCalledWith( + "FN-7437", + "Triage using model: mock-model (thinking effort: low)", + "text", + undefined, + "triage", + ); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-7437", + expect.stringContaining("Triage failed: unable to select a usable model after 2 attempts"), + ); + expect(store.updateTask).toHaveBeenCalledWith("FN-7437", expect.objectContaining({ + status: "failed", + error: expect.stringContaining("openai/gpt-4o"), + recoveryRetryCount: null, + nextRecoveryAt: null, + })); + expect(store.updateTask).not.toHaveBeenCalledWith("FN-7437", expect.objectContaining({ + status: null, + error: null, + })); + expect(mockDispose).toHaveBeenCalledTimes(1); + expect(onSpecifyError).toHaveBeenCalledTimes(1); + }); + + it("persists terminal planning error when session state reports fallback exhaustion", async () => { + const task = { + id: "FN-7437-STATE", + description: "Bug: planner triage fallback loop via state error", + column: "triage", + dependencies: [], + steps: [], + currentStep: 0, + log: [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as unknown as Task; + const store = createMockStore({ + getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }), + }); + mockCreateFnAgent.mockResolvedValue({ + session: { + state: {}, + sessionManager: {}, + prompt: vi.fn(), + dispose: vi.fn(), + navigateTree: vi.fn(), + }, + }); + const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js"); + const exhausted = new ModelFallbackExhaustedError({ + primaryModel: "openai/gpt-4o", + fallbackModel: "anthropic/claude-3-5-haiku-20241022", + triggerPoint: "prompt-time", + attempts: 2, + underlyingReason: "fallback session state error: 403 forbidden", + }); + (promptWithFallback as ReturnType).mockRejectedValueOnce(exhausted); + + const processor = new TriageProcessor(store, "/test/root", { pollIntervalMs: 100_000 }); + await processor.specifyTask(task); + + expect(store.updateTask).toHaveBeenCalledWith("FN-7437-STATE", expect.objectContaining({ + status: "failed", + error: expect.stringContaining("fallback session state error: 403 forbidden"), + })); + }); + it("escalates to error state when triage retries are exhausted via specifyTask", async () => { const task = { id: "FN-201", diff --git a/packages/engine/src/pi.ts b/packages/engine/src/pi.ts index 662253cb47..591ee90b17 100644 --- a/packages/engine/src/pi.ts +++ b/packages/engine/src/pi.ts @@ -967,6 +967,34 @@ export interface FallbackModelUsedPayload { timestamp?: string; } +export class ModelFallbackExhaustedError extends Error { + readonly primaryModel: string; + readonly fallbackModel?: string; + readonly triggerPoint: "session-creation" | "prompt-time"; + readonly attempts: number; + readonly underlyingReason: string; + + constructor(input: { + primaryModel: string; + fallbackModel?: string; + triggerPoint: "session-creation" | "prompt-time"; + attempts: number; + underlyingReason: string; + }) { + const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured"; + super( + `Unable to select a usable model after ${input.attempts} attempt${input.attempts === 1 ? "" : "s"} ` + + `(primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`, + ); + this.name = "ModelFallbackExhaustedError"; + this.primaryModel = input.primaryModel; + this.fallbackModel = input.fallbackModel; + this.triggerPoint = input.triggerPoint; + this.attempts = input.attempts; + this.underlyingReason = input.underlyingReason; + } +} + export type BuiltinWebToolName = "WebSearch" | "WebFetch"; export interface AgentOptions { @@ -2396,8 +2424,34 @@ export async function createFnAgent(options: AgentOptions): Promise } }; + const modelDescription = (model: typeof selectedModel): string => model ? `${model.provider}/${model.id}` : "unknown model"; + const configuredFallbackDiffers = Boolean( + options.fallbackProvider + && options.fallbackModelId + && (options.fallbackProvider !== options.defaultProvider || options.fallbackModelId !== options.defaultModelId), + ); + const hasDistinctFallback = Boolean( + selectedModel + && fallbackModel + && (configuredFallbackDiffers || selectedModel.provider !== fallbackModel.provider || selectedModel.id !== fallbackModel.id), + ); + const makeFallbackExhaustedError = ( + triggerPoint: "session-creation" | "prompt-time", + attempts: number, + underlying: unknown, + ): ModelFallbackExhaustedError => { + const underlyingReason = underlying instanceof Error ? underlying.message : String(underlying); + return new ModelFallbackExhaustedError({ + primaryModel: modelDescription(selectedModel), + fallbackModel: hasDistinctFallback ? modelDescription(fallbackModel) : undefined, + triggerPoint, + attempts, + underlyingReason, + }); + }; + const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise => { - if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel) { + if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel || !hasDistinctFallback) { return; } await options.onFallbackModelUsed({ @@ -2410,19 +2464,27 @@ export async function createFnAgent(options: AgentOptions): Promise }); }; + /* + * FNXC:ModelFallback 2026-07-02-00:00: + * Planner and shared AI lanes may try one distinct fallback model for a logical model-selection failure, but they must then throw ModelFallbackExhaustedError instead of swapping back or relying on scheduler re-pick loops. This keeps transient fallback useful while making exhausted model configuration actionable for operators. + */ let sessionResult; let usingFallback = false; try { sessionResult = await createSessionWithModel(selectedModel); piLog.log(`Session created successfully (model=${selectedModel ? `${selectedModel.provider}/${selectedModel.id}` : "default"})`); } catch (err: any) { - if (!fallbackModel || !selectedModel || !isRetryableModelSelectionError(err?.message || "")) { + if (!fallbackModel || !selectedModel || !hasDistinctFallback || !isRetryableModelSelectionError(err?.message || "")) { piLog.error(`Session creation failed: ${err.message}`); throw err; } piLog.warn(`Primary model failed (${err.message}), trying fallback`); usingFallback = true; - sessionResult = await createSessionWithModel(fallbackModel); + try { + sessionResult = await createSessionWithModel(fallbackModel); + } catch (fallbackErr: unknown) { + throw makeFallbackExhaustedError("session-creation", 2, fallbackErr); + } await emitFallbackUsed("session-creation"); piLog.log("Fallback session created successfully"); } @@ -2565,6 +2627,9 @@ export async function createFnAgent(options: AgentOptions): Promise if (!fallbackModel || usingFallback || !isRetryableModelSelectionError(errorMessage)) { throw err; } + if (!hasDistinctFallback) { + throw makeFallbackExhaustedError("prompt-time", 1, err); + } usingFallback = true; const fallbackSession = await swapPromptSession(fallbackModel); @@ -2617,7 +2682,7 @@ export async function createFnAgent(options: AgentOptions): Promise throw fallbackErr; } } - throw fallbackErr; + throw makeFallbackExhaustedError("prompt-time", 2, fallbackErr); } } }; diff --git a/packages/engine/src/triage.ts b/packages/engine/src/triage.ts index 4c6d8287c8..4e5895d52e 100644 --- a/packages/engine/src/triage.ts +++ b/packages/engine/src/triage.ts @@ -80,7 +80,7 @@ import type { ToolDefinition, AgentSession, } from "@earendil-works/pi-coding-agent"; -import { describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js"; +import { ModelFallbackExhaustedError, describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js"; import { createResolvedAgentSession, extractRuntimeHint, @@ -1396,6 +1396,29 @@ export class TriageProcessor { task.id, errorMessage, ); + } else if (err instanceof ModelFallbackExhaustedError) { + /* + FNXC:TriageModelFallback 2026-07-02-00:00: + Exhausted planner model fallback is terminal and operator-actionable: clearing status lets the scheduler recreate the same primary/fallback pair forever, so triage persists a failed task error with the bounded attempt count and sanitized provider reason. + */ + const failureMessage = + `Triage failed: unable to select a usable model after ${err.attempts} attempt${err.attempts === 1 ? "" : "s"}. ${err.message}`; + planLog.error(`✗ ${task.id} planner model fallback exhausted: ${failureMessage}`); + await this.store.logEntry(task.id, failureMessage).catch((logErr: unknown) => { + const msg = logErr instanceof Error ? logErr.message : String(logErr); + planLog.warn(`${task.id}: failed to log planner fallback exhaustion: ${msg}`); + }); + await this.store.updateTask(task.id, { + status: "failed", + error: failureMessage, + recoveryRetryCount: null, + nextRecoveryAt: null, + }).catch((updateErr: unknown) => { + const msg = updateErr instanceof Error ? updateErr.message : String(updateErr); + planLog.warn(`${task.id}: failed to persist planner fallback exhaustion: ${msg}`); + }); + this.options.onSpecifyError?.(task, err); + return; } else if (isTransientError(errorMessage)) { // Transient network/infrastructure error — use bounded recovery policy const decision = computeRecoveryDecision({