FN-7437: bound planner fallback retries

Bound planner fallback retries so model exhaustion reports a clear terminal error instead of looping.

- Add planner fallback exhaustion state and loop-limit handling for triage and pi planning paths.
- Report exhausted provider/model fallback attempts with actionable error details.
- Cover planner fallback exhaustion with engine regression tests and add a patch changeset.

Files changed:
 .changeset/fn-7437-planner-fallback.md             |   7 +
 .../src/__tests__/fallback-model-observer.test.ts  |  31 ++++
 packages/engine/src/__tests__/pi.test.ts           | 106 ++++++++++++++
 packages/engine/src/__tests__/triage.test.ts       | 161 ++++++++++++++++++++-
 packages/engine/src/pi.ts                          |  73 +++++++++-
 packages/engine/src/triage.ts                      |  25 +++-
 6 files changed, 396 insertions(+), 7 deletions(-)

Fusion-Task-Id: FN-7437

Fusion-Task-Lineage: e2b8361a-c590-4f10-8f95-06dfd77e944c

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-07-02 14:43:32 -07:00
parent dfb6e5270d
commit 1d6bb08e79
6 changed files with 396 additions and 7 deletions

View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Stop planner model fallback loops with a clear terminal triage error.
category: fix
dev: Bounds prompt-time/session-creation model fallback exhaustion and persists failed triage state.

View File

@@ -53,6 +53,37 @@ describe("createFallbackModelObserver", () => {
}); });
}); });
it("writes fallback events as non-empty rows with readable delimiters", async () => {
const store = {
logEntry: vi.fn().mockResolvedValue(undefined),
appendAgentLog: vi.fn().mockResolvedValue(undefined),
};
const observer = createFallbackModelObserver({
agent: "triage",
label: "triage",
store,
taskId: "FN-7437",
});
await observer({
primaryModel: "openai/gpt-4o",
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
triggerPoint: "prompt-time",
});
await observer({
primaryModel: "openai/gpt-4o",
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
triggerPoint: "prompt-time",
});
const rows = store.appendAgentLog.mock.calls.map((call) => call[1]);
expect(rows).toEqual([
"[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)",
"[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)",
]);
expect(rows.every((row) => row.trim() === row && row.includes(" switched from ") && row.includes(" to "))).toBe(true);
});
it("swallows logging failures and still dispatches a notification", async () => { it("swallows logging failures and still dispatches a notification", async () => {
const store = { const store = {
logEntry: vi.fn().mockRejectedValue(new Error("log failed")), logEntry: vi.fn().mockRejectedValue(new Error("log failed")),

View File

@@ -993,6 +993,112 @@ describe("piLog structured diagnostics", () => {
); );
}); });
it("throws a bounded fallback exhaustion error when prompt-time fallback also fails", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession);
vi.mocked(ModelRegistry.create).mockReturnValueOnce({
find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })),
getAll: vi.fn().mockReturnValue([]),
registerProvider: vi.fn(),
refresh: vi.fn(),
} as any);
const onFallbackModelUsed = vi.fn();
const primarySession = {
model: { provider: "openai", id: "gpt-4o" },
prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
sessionFile: undefined,
} as unknown as AgentSession;
const fallbackSession = {
model: { provider: "anthropic", id: "claude-3-5-haiku-20241022" },
prompt: vi.fn().mockRejectedValue(new Error("401 invalid api key for fallback")),
state: { errorMessage: "", messages: [] },
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
sessionFile: undefined,
} as unknown as AgentSession;
createAgentSessionMock.mockReset();
createAgentSessionMock
.mockResolvedValueOnce({ session: primarySession } as any)
.mockResolvedValueOnce({ session: fallbackSession } as any);
const { session } = await createFnAgent({
cwd: "/test/project",
systemPrompt: "Test planner fallback exhaustion",
defaultProvider: "openai",
defaultModelId: "gpt-4o",
fallbackProvider: "anthropic",
fallbackModelId: "claude-3-5-haiku-20241022",
taskId: "FN-7437",
onFallbackModelUsed,
});
await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({
name: "ModelFallbackExhaustedError",
attempts: 2,
primaryModel: "openai/gpt-4o",
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
triggerPoint: "prompt-time",
});
expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
expect(primarySession.prompt).toHaveBeenCalledTimes(1);
expect(fallbackSession.prompt).toHaveBeenCalledTimes(1);
expect(onFallbackModelUsed).toHaveBeenCalledTimes(1);
expect(onFallbackModelUsed).toHaveBeenCalledWith(expect.objectContaining({
triggerPoint: "prompt-time",
primaryModel: "openai/gpt-4o",
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
taskId: "FN-7437",
}));
});
it("does not create a meaningless prompt-time fallback when primary and fallback match", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession);
vi.mocked(ModelRegistry.create).mockReturnValueOnce({
find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })),
getAll: vi.fn().mockReturnValue([]),
registerProvider: vi.fn(),
refresh: vi.fn(),
} as any);
const onFallbackModelUsed = vi.fn();
const primarySession = {
model: { provider: "openai", id: "gpt-4o" },
prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
sessionFile: undefined,
} as unknown as AgentSession;
createAgentSessionMock.mockReset();
createAgentSessionMock.mockResolvedValueOnce({ session: primarySession } as any);
const { session } = await createFnAgent({
cwd: "/test/project",
systemPrompt: "Test same fallback",
defaultProvider: "openai",
defaultModelId: "gpt-4o",
fallbackProvider: "openai",
fallbackModelId: "gpt-4o",
onFallbackModelUsed,
});
await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({
name: "ModelFallbackExhaustedError",
attempts: 1,
primaryModel: "openai/gpt-4o",
fallbackModel: undefined,
});
expect(createAgentSessionMock).toHaveBeenCalledTimes(1);
expect(onFallbackModelUsed).not.toHaveBeenCalled();
});
it("fires fallback hook on prompt-time model-auth-tier fallback", async () => { it("fires fallback hook on prompt-time model-auth-tier fallback", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession); const createAgentSessionMock = vi.mocked(createAgentSession);
const onFallbackModelUsed = vi.fn(); const onFallbackModelUsed = vi.fn();

View File

@@ -29,7 +29,28 @@ vi.mock("../reviewer.js", () => ({
reviewStep: mockReviewStep, reviewStep: mockReviewStep,
})); }));
vi.mock("../pi.js", () => ({ vi.mock("../pi.js", () => {
class ModelFallbackExhaustedError extends Error {
readonly primaryModel: string;
readonly fallbackModel?: string;
readonly triggerPoint: "session-creation" | "prompt-time";
readonly attempts: number;
readonly underlyingReason: string;
constructor(input: { primaryModel: string; fallbackModel?: string; triggerPoint: "session-creation" | "prompt-time"; attempts: number; underlyingReason: string }) {
const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured";
super(`Unable to select a usable model after ${input.attempts} attempts (primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`);
this.name = "ModelFallbackExhaustedError";
this.primaryModel = input.primaryModel;
this.fallbackModel = input.fallbackModel;
this.triggerPoint = input.triggerPoint;
this.attempts = input.attempts;
this.underlyingReason = input.underlyingReason;
}
}
return {
ModelFallbackExhaustedError,
createFnAgent: mockCreateFnAgent, createFnAgent: mockCreateFnAgent,
describeModel: vi.fn().mockReturnValue("mock-model"), describeModel: vi.fn().mockReturnValue("mock-model"),
formatModelMarkerDetails: vi.fn((model: string, thinking?: string | null, annotations: string[] = []) => { formatModelMarkerDetails: vi.fn((model: string, thinking?: string | null, annotations: string[] = []) => {
@@ -37,7 +58,8 @@ vi.mock("../pi.js", () => ({
return suffixes.length ? `${model} ${suffixes.map((suffix) => `(${suffix})`).join(" ")}` : model; return suffixes.length ? `${model} ${suffixes.map((suffix) => `(${suffix})`).join(" ")}` : model;
}), }),
promptWithFallback: vi.fn().mockReturnValue("mock-prompt"), promptWithFallback: vi.fn().mockReturnValue("mock-prompt"),
})); };
});
vi.mock("@fusion/core", async (importOriginal) => { vi.mock("@fusion/core", async (importOriginal) => {
const { createEngineCoreMock } = await import("../test/mockCore.js"); const { createEngineCoreMock } = await import("../test/mockCore.js");
@@ -3440,6 +3462,13 @@ describe("taskCreate tool model inheritance", () => {
}); });
describe("bounded recovery retries for triage", () => { describe("bounded recovery retries for triage", () => {
beforeEach(async () => {
vi.clearAllMocks();
const { promptWithFallback } = await import("../pi.js");
(promptWithFallback as ReturnType<typeof vi.fn>).mockReset();
(promptWithFallback as ReturnType<typeof vi.fn>).mockResolvedValue(undefined);
});
it("requeues triage with backoff when the agent exits without writing PROMPT.md", async () => { it("requeues triage with backoff when the agent exits without writing PROMPT.md", async () => {
const task = { const task = {
id: "FN-202", id: "FN-202",
@@ -3521,6 +3550,134 @@ describe("taskCreate tool model inheritance", () => {
})); }));
}); });
it("persists terminal planning error when prompt-time primary and fallback models are exhausted", async () => {
const task = {
id: "FN-7437",
description: "Bug: planner triage fallback loop",
column: "triage",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
} as unknown as Task;
const onSpecifyError = vi.fn();
const store = createMockStore({
getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
getSettings: vi.fn().mockResolvedValue({
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 10000,
groupOverlappingFiles: false,
autoMerge: true,
defaultProvider: "openai",
defaultModelId: "gpt-4o",
planningFallbackProvider: "anthropic",
planningFallbackModelId: "claude-3-5-haiku-20241022",
defaultThinkingLevel: "low",
} as Settings),
});
const mockDispose = vi.fn();
mockCreateFnAgent.mockResolvedValue({
session: {
state: {},
sessionManager: {},
prompt: vi.fn(),
dispose: mockDispose,
navigateTree: vi.fn(),
},
});
const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js");
(promptWithFallback as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
new ModelFallbackExhaustedError({
primaryModel: "openai/gpt-4o",
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
triggerPoint: "prompt-time",
attempts: 2,
underlyingReason: "401 invalid api key for fallback",
}),
);
const processor = new TriageProcessor(store, "/test/root", {
pollIntervalMs: 100_000,
onSpecifyError,
});
await processor.specifyTask(task);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-7437",
"Triage using model: mock-model (thinking effort: low)",
);
expect(store.appendAgentLog).toHaveBeenCalledWith(
"FN-7437",
"Triage using model: mock-model (thinking effort: low)",
"text",
undefined,
"triage",
);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-7437",
expect.stringContaining("Triage failed: unable to select a usable model after 2 attempts"),
);
expect(store.updateTask).toHaveBeenCalledWith("FN-7437", expect.objectContaining({
status: "failed",
error: expect.stringContaining("openai/gpt-4o"),
recoveryRetryCount: null,
nextRecoveryAt: null,
}));
expect(store.updateTask).not.toHaveBeenCalledWith("FN-7437", expect.objectContaining({
status: null,
error: null,
}));
expect(mockDispose).toHaveBeenCalledTimes(1);
expect(onSpecifyError).toHaveBeenCalledTimes(1);
});
it("persists terminal planning error when session state reports fallback exhaustion", async () => {
const task = {
id: "FN-7437-STATE",
description: "Bug: planner triage fallback loop via state error",
column: "triage",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
} as unknown as Task;
const store = createMockStore({
getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
});
mockCreateFnAgent.mockResolvedValue({
session: {
state: {},
sessionManager: {},
prompt: vi.fn(),
dispose: vi.fn(),
navigateTree: vi.fn(),
},
});
const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js");
const exhausted = new ModelFallbackExhaustedError({
primaryModel: "openai/gpt-4o",
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
triggerPoint: "prompt-time",
attempts: 2,
underlyingReason: "fallback session state error: 403 forbidden",
});
(promptWithFallback as ReturnType<typeof vi.fn>).mockRejectedValueOnce(exhausted);
const processor = new TriageProcessor(store, "/test/root", { pollIntervalMs: 100_000 });
await processor.specifyTask(task);
expect(store.updateTask).toHaveBeenCalledWith("FN-7437-STATE", expect.objectContaining({
status: "failed",
error: expect.stringContaining("fallback session state error: 403 forbidden"),
}));
});
it("escalates to error state when triage retries are exhausted via specifyTask", async () => { it("escalates to error state when triage retries are exhausted via specifyTask", async () => {
const task = { const task = {
id: "FN-201", id: "FN-201",

View File

@@ -967,6 +967,34 @@ export interface FallbackModelUsedPayload {
timestamp?: string; timestamp?: string;
} }
export class ModelFallbackExhaustedError extends Error {
readonly primaryModel: string;
readonly fallbackModel?: string;
readonly triggerPoint: "session-creation" | "prompt-time";
readonly attempts: number;
readonly underlyingReason: string;
constructor(input: {
primaryModel: string;
fallbackModel?: string;
triggerPoint: "session-creation" | "prompt-time";
attempts: number;
underlyingReason: string;
}) {
const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured";
super(
`Unable to select a usable model after ${input.attempts} attempt${input.attempts === 1 ? "" : "s"} `
+ `(primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`,
);
this.name = "ModelFallbackExhaustedError";
this.primaryModel = input.primaryModel;
this.fallbackModel = input.fallbackModel;
this.triggerPoint = input.triggerPoint;
this.attempts = input.attempts;
this.underlyingReason = input.underlyingReason;
}
}
export type BuiltinWebToolName = "WebSearch" | "WebFetch"; export type BuiltinWebToolName = "WebSearch" | "WebFetch";
export interface AgentOptions { export interface AgentOptions {
@@ -2396,8 +2424,34 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
} }
}; };
const modelDescription = (model: typeof selectedModel): string => model ? `${model.provider}/${model.id}` : "unknown model";
const configuredFallbackDiffers = Boolean(
options.fallbackProvider
&& options.fallbackModelId
&& (options.fallbackProvider !== options.defaultProvider || options.fallbackModelId !== options.defaultModelId),
);
const hasDistinctFallback = Boolean(
selectedModel
&& fallbackModel
&& (configuredFallbackDiffers || selectedModel.provider !== fallbackModel.provider || selectedModel.id !== fallbackModel.id),
);
const makeFallbackExhaustedError = (
triggerPoint: "session-creation" | "prompt-time",
attempts: number,
underlying: unknown,
): ModelFallbackExhaustedError => {
const underlyingReason = underlying instanceof Error ? underlying.message : String(underlying);
return new ModelFallbackExhaustedError({
primaryModel: modelDescription(selectedModel),
fallbackModel: hasDistinctFallback ? modelDescription(fallbackModel) : undefined,
triggerPoint,
attempts,
underlyingReason,
});
};
const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise<void> => { const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise<void> => {
if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel) { if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel || !hasDistinctFallback) {
return; return;
} }
await options.onFallbackModelUsed({ await options.onFallbackModelUsed({
@@ -2410,19 +2464,27 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
}); });
}; };
/*
* FNXC:ModelFallback 2026-07-02-00:00:
* Planner and shared AI lanes may try one distinct fallback model for a logical model-selection failure, but they must then throw ModelFallbackExhaustedError instead of swapping back or relying on scheduler re-pick loops. This keeps transient fallback useful while making exhausted model configuration actionable for operators.
*/
let sessionResult; let sessionResult;
let usingFallback = false; let usingFallback = false;
try { try {
sessionResult = await createSessionWithModel(selectedModel); sessionResult = await createSessionWithModel(selectedModel);
piLog.log(`Session created successfully (model=${selectedModel ? `${selectedModel.provider}/${selectedModel.id}` : "default"})`); piLog.log(`Session created successfully (model=${selectedModel ? `${selectedModel.provider}/${selectedModel.id}` : "default"})`);
} catch (err: any) { } catch (err: any) {
if (!fallbackModel || !selectedModel || !isRetryableModelSelectionError(err?.message || "")) { if (!fallbackModel || !selectedModel || !hasDistinctFallback || !isRetryableModelSelectionError(err?.message || "")) {
piLog.error(`Session creation failed: ${err.message}`); piLog.error(`Session creation failed: ${err.message}`);
throw err; throw err;
} }
piLog.warn(`Primary model failed (${err.message}), trying fallback`); piLog.warn(`Primary model failed (${err.message}), trying fallback`);
usingFallback = true; usingFallback = true;
sessionResult = await createSessionWithModel(fallbackModel); try {
sessionResult = await createSessionWithModel(fallbackModel);
} catch (fallbackErr: unknown) {
throw makeFallbackExhaustedError("session-creation", 2, fallbackErr);
}
await emitFallbackUsed("session-creation"); await emitFallbackUsed("session-creation");
piLog.log("Fallback session created successfully"); piLog.log("Fallback session created successfully");
} }
@@ -2565,6 +2627,9 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
if (!fallbackModel || usingFallback || !isRetryableModelSelectionError(errorMessage)) { if (!fallbackModel || usingFallback || !isRetryableModelSelectionError(errorMessage)) {
throw err; throw err;
} }
if (!hasDistinctFallback) {
throw makeFallbackExhaustedError("prompt-time", 1, err);
}
usingFallback = true; usingFallback = true;
const fallbackSession = await swapPromptSession(fallbackModel); const fallbackSession = await swapPromptSession(fallbackModel);
@@ -2617,7 +2682,7 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
throw fallbackErr; throw fallbackErr;
} }
} }
throw fallbackErr; throw makeFallbackExhaustedError("prompt-time", 2, fallbackErr);
} }
} }
}; };

View File

@@ -80,7 +80,7 @@ import type {
ToolDefinition, ToolDefinition,
AgentSession, AgentSession,
} from "@earendil-works/pi-coding-agent"; } from "@earendil-works/pi-coding-agent";
import { describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js"; import { ModelFallbackExhaustedError, describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js";
import { import {
createResolvedAgentSession, createResolvedAgentSession,
extractRuntimeHint, extractRuntimeHint,
@@ -1396,6 +1396,29 @@ export class TriageProcessor {
task.id, task.id,
errorMessage, errorMessage,
); );
} else if (err instanceof ModelFallbackExhaustedError) {
/*
FNXC:TriageModelFallback 2026-07-02-00:00:
Exhausted planner model fallback is terminal and operator-actionable: clearing status lets the scheduler recreate the same primary/fallback pair forever, so triage persists a failed task error with the bounded attempt count and sanitized provider reason.
*/
const failureMessage =
`Triage failed: unable to select a usable model after ${err.attempts} attempt${err.attempts === 1 ? "" : "s"}. ${err.message}`;
planLog.error(`✗ ${task.id} planner model fallback exhausted: ${failureMessage}`);
await this.store.logEntry(task.id, failureMessage).catch((logErr: unknown) => {
const msg = logErr instanceof Error ? logErr.message : String(logErr);
planLog.warn(`${task.id}: failed to log planner fallback exhaustion: ${msg}`);
});
await this.store.updateTask(task.id, {
status: "failed",
error: failureMessage,
recoveryRetryCount: null,
nextRecoveryAt: null,
}).catch((updateErr: unknown) => {
const msg = updateErr instanceof Error ? updateErr.message : String(updateErr);
planLog.warn(`${task.id}: failed to persist planner fallback exhaustion: ${msg}`);
});
this.options.onSpecifyError?.(task, err);
return;
} else if (isTransientError(errorMessage)) { } else if (isTransientError(errorMessage)) {
// Transient network/infrastructure error — use bounded recovery policy // Transient network/infrastructure error — use bounded recovery policy
const decision = computeRecoveryDecision({ const decision = computeRecoveryDecision({