FN-7437: bound planner fallback retries
Bound planner fallback retries so model exhaustion reports a clear terminal error instead of looping. - Add planner fallback exhaustion state and loop-limit handling for triage and pi planning paths. - Report exhausted provider/model fallback attempts with actionable error details. - Cover planner fallback exhaustion with engine regression tests and add a patch changeset. Files changed: .changeset/fn-7437-planner-fallback.md | 7 + .../src/__tests__/fallback-model-observer.test.ts | 31 ++++ packages/engine/src/__tests__/pi.test.ts | 106 ++++++++++++++ packages/engine/src/__tests__/triage.test.ts | 161 ++++++++++++++++++++- packages/engine/src/pi.ts | 73 +++++++++- packages/engine/src/triage.ts | 25 +++- 6 files changed, 396 insertions(+), 7 deletions(-) Fusion-Task-Id: FN-7437 Fusion-Task-Lineage: e2b8361a-c590-4f10-8f95-06dfd77e944c Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
7
.changeset/fn-7437-planner-fallback.md
Normal file
7
.changeset/fn-7437-planner-fallback.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Stop planner model fallback loops with a clear terminal triage error.
|
||||||
|
category: fix
|
||||||
|
dev: Bounds prompt-time/session-creation model fallback exhaustion and persists failed triage state.
|
||||||
@@ -53,6 +53,37 @@ describe("createFallbackModelObserver", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("writes fallback events as non-empty rows with readable delimiters", async () => {
|
||||||
|
const store = {
|
||||||
|
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||||
|
appendAgentLog: vi.fn().mockResolvedValue(undefined),
|
||||||
|
};
|
||||||
|
const observer = createFallbackModelObserver({
|
||||||
|
agent: "triage",
|
||||||
|
label: "triage",
|
||||||
|
store,
|
||||||
|
taskId: "FN-7437",
|
||||||
|
});
|
||||||
|
|
||||||
|
await observer({
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
|
||||||
|
triggerPoint: "prompt-time",
|
||||||
|
});
|
||||||
|
await observer({
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
|
||||||
|
triggerPoint: "prompt-time",
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = store.appendAgentLog.mock.calls.map((call) => call[1]);
|
||||||
|
expect(rows).toEqual([
|
||||||
|
"[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)",
|
||||||
|
"[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)",
|
||||||
|
]);
|
||||||
|
expect(rows.every((row) => row.trim() === row && row.includes(" switched from ") && row.includes(" to "))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("swallows logging failures and still dispatches a notification", async () => {
|
it("swallows logging failures and still dispatches a notification", async () => {
|
||||||
const store = {
|
const store = {
|
||||||
logEntry: vi.fn().mockRejectedValue(new Error("log failed")),
|
logEntry: vi.fn().mockRejectedValue(new Error("log failed")),
|
||||||
|
|||||||
@@ -993,6 +993,112 @@ describe("piLog structured diagnostics", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("throws a bounded fallback exhaustion error when prompt-time fallback also fails", async () => {
|
||||||
|
const createAgentSessionMock = vi.mocked(createAgentSession);
|
||||||
|
vi.mocked(ModelRegistry.create).mockReturnValueOnce({
|
||||||
|
find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })),
|
||||||
|
getAll: vi.fn().mockReturnValue([]),
|
||||||
|
registerProvider: vi.fn(),
|
||||||
|
refresh: vi.fn(),
|
||||||
|
} as any);
|
||||||
|
const onFallbackModelUsed = vi.fn();
|
||||||
|
|
||||||
|
const primarySession = {
|
||||||
|
model: { provider: "openai", id: "gpt-4o" },
|
||||||
|
prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
|
||||||
|
subscribe: vi.fn(),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
setThinkingLevel: vi.fn(),
|
||||||
|
sessionFile: undefined,
|
||||||
|
} as unknown as AgentSession;
|
||||||
|
|
||||||
|
const fallbackSession = {
|
||||||
|
model: { provider: "anthropic", id: "claude-3-5-haiku-20241022" },
|
||||||
|
prompt: vi.fn().mockRejectedValue(new Error("401 invalid api key for fallback")),
|
||||||
|
state: { errorMessage: "", messages: [] },
|
||||||
|
subscribe: vi.fn(),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
setThinkingLevel: vi.fn(),
|
||||||
|
sessionFile: undefined,
|
||||||
|
} as unknown as AgentSession;
|
||||||
|
|
||||||
|
createAgentSessionMock.mockReset();
|
||||||
|
createAgentSessionMock
|
||||||
|
.mockResolvedValueOnce({ session: primarySession } as any)
|
||||||
|
.mockResolvedValueOnce({ session: fallbackSession } as any);
|
||||||
|
|
||||||
|
const { session } = await createFnAgent({
|
||||||
|
cwd: "/test/project",
|
||||||
|
systemPrompt: "Test planner fallback exhaustion",
|
||||||
|
defaultProvider: "openai",
|
||||||
|
defaultModelId: "gpt-4o",
|
||||||
|
fallbackProvider: "anthropic",
|
||||||
|
fallbackModelId: "claude-3-5-haiku-20241022",
|
||||||
|
taskId: "FN-7437",
|
||||||
|
onFallbackModelUsed,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({
|
||||||
|
name: "ModelFallbackExhaustedError",
|
||||||
|
attempts: 2,
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
|
||||||
|
triggerPoint: "prompt-time",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
|
||||||
|
expect(primarySession.prompt).toHaveBeenCalledTimes(1);
|
||||||
|
expect(fallbackSession.prompt).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onFallbackModelUsed).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onFallbackModelUsed).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
triggerPoint: "prompt-time",
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
|
||||||
|
taskId: "FN-7437",
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not create a meaningless prompt-time fallback when primary and fallback match", async () => {
|
||||||
|
const createAgentSessionMock = vi.mocked(createAgentSession);
|
||||||
|
vi.mocked(ModelRegistry.create).mockReturnValueOnce({
|
||||||
|
find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })),
|
||||||
|
getAll: vi.fn().mockReturnValue([]),
|
||||||
|
registerProvider: vi.fn(),
|
||||||
|
refresh: vi.fn(),
|
||||||
|
} as any);
|
||||||
|
const onFallbackModelUsed = vi.fn();
|
||||||
|
const primarySession = {
|
||||||
|
model: { provider: "openai", id: "gpt-4o" },
|
||||||
|
prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
|
||||||
|
subscribe: vi.fn(),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
setThinkingLevel: vi.fn(),
|
||||||
|
sessionFile: undefined,
|
||||||
|
} as unknown as AgentSession;
|
||||||
|
|
||||||
|
createAgentSessionMock.mockReset();
|
||||||
|
createAgentSessionMock.mockResolvedValueOnce({ session: primarySession } as any);
|
||||||
|
|
||||||
|
const { session } = await createFnAgent({
|
||||||
|
cwd: "/test/project",
|
||||||
|
systemPrompt: "Test same fallback",
|
||||||
|
defaultProvider: "openai",
|
||||||
|
defaultModelId: "gpt-4o",
|
||||||
|
fallbackProvider: "openai",
|
||||||
|
fallbackModelId: "gpt-4o",
|
||||||
|
onFallbackModelUsed,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({
|
||||||
|
name: "ModelFallbackExhaustedError",
|
||||||
|
attempts: 1,
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: undefined,
|
||||||
|
});
|
||||||
|
expect(createAgentSessionMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onFallbackModelUsed).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("fires fallback hook on prompt-time model-auth-tier fallback", async () => {
|
it("fires fallback hook on prompt-time model-auth-tier fallback", async () => {
|
||||||
const createAgentSessionMock = vi.mocked(createAgentSession);
|
const createAgentSessionMock = vi.mocked(createAgentSession);
|
||||||
const onFallbackModelUsed = vi.fn();
|
const onFallbackModelUsed = vi.fn();
|
||||||
|
|||||||
@@ -29,7 +29,28 @@ vi.mock("../reviewer.js", () => ({
|
|||||||
reviewStep: mockReviewStep,
|
reviewStep: mockReviewStep,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("../pi.js", () => ({
|
vi.mock("../pi.js", () => {
|
||||||
|
class ModelFallbackExhaustedError extends Error {
|
||||||
|
readonly primaryModel: string;
|
||||||
|
readonly fallbackModel?: string;
|
||||||
|
readonly triggerPoint: "session-creation" | "prompt-time";
|
||||||
|
readonly attempts: number;
|
||||||
|
readonly underlyingReason: string;
|
||||||
|
|
||||||
|
constructor(input: { primaryModel: string; fallbackModel?: string; triggerPoint: "session-creation" | "prompt-time"; attempts: number; underlyingReason: string }) {
|
||||||
|
const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured";
|
||||||
|
super(`Unable to select a usable model after ${input.attempts} attempts (primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`);
|
||||||
|
this.name = "ModelFallbackExhaustedError";
|
||||||
|
this.primaryModel = input.primaryModel;
|
||||||
|
this.fallbackModel = input.fallbackModel;
|
||||||
|
this.triggerPoint = input.triggerPoint;
|
||||||
|
this.attempts = input.attempts;
|
||||||
|
this.underlyingReason = input.underlyingReason;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ModelFallbackExhaustedError,
|
||||||
createFnAgent: mockCreateFnAgent,
|
createFnAgent: mockCreateFnAgent,
|
||||||
describeModel: vi.fn().mockReturnValue("mock-model"),
|
describeModel: vi.fn().mockReturnValue("mock-model"),
|
||||||
formatModelMarkerDetails: vi.fn((model: string, thinking?: string | null, annotations: string[] = []) => {
|
formatModelMarkerDetails: vi.fn((model: string, thinking?: string | null, annotations: string[] = []) => {
|
||||||
@@ -37,7 +58,8 @@ vi.mock("../pi.js", () => ({
|
|||||||
return suffixes.length ? `${model} ${suffixes.map((suffix) => `(${suffix})`).join(" ")}` : model;
|
return suffixes.length ? `${model} ${suffixes.map((suffix) => `(${suffix})`).join(" ")}` : model;
|
||||||
}),
|
}),
|
||||||
promptWithFallback: vi.fn().mockReturnValue("mock-prompt"),
|
promptWithFallback: vi.fn().mockReturnValue("mock-prompt"),
|
||||||
}));
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock("@fusion/core", async (importOriginal) => {
|
vi.mock("@fusion/core", async (importOriginal) => {
|
||||||
const { createEngineCoreMock } = await import("../test/mockCore.js");
|
const { createEngineCoreMock } = await import("../test/mockCore.js");
|
||||||
@@ -3440,6 +3462,13 @@ describe("taskCreate tool model inheritance", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("bounded recovery retries for triage", () => {
|
describe("bounded recovery retries for triage", () => {
|
||||||
|
beforeEach(async () => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
const { promptWithFallback } = await import("../pi.js");
|
||||||
|
(promptWithFallback as ReturnType<typeof vi.fn>).mockReset();
|
||||||
|
(promptWithFallback as ReturnType<typeof vi.fn>).mockResolvedValue(undefined);
|
||||||
|
});
|
||||||
|
|
||||||
it("requeues triage with backoff when the agent exits without writing PROMPT.md", async () => {
|
it("requeues triage with backoff when the agent exits without writing PROMPT.md", async () => {
|
||||||
const task = {
|
const task = {
|
||||||
id: "FN-202",
|
id: "FN-202",
|
||||||
@@ -3521,6 +3550,134 @@ describe("taskCreate tool model inheritance", () => {
|
|||||||
}));
|
}));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("persists terminal planning error when prompt-time primary and fallback models are exhausted", async () => {
|
||||||
|
const task = {
|
||||||
|
id: "FN-7437",
|
||||||
|
description: "Bug: planner triage fallback loop",
|
||||||
|
column: "triage",
|
||||||
|
dependencies: [],
|
||||||
|
steps: [],
|
||||||
|
currentStep: 0,
|
||||||
|
log: [],
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
} as unknown as Task;
|
||||||
|
const onSpecifyError = vi.fn();
|
||||||
|
const store = createMockStore({
|
||||||
|
getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
|
||||||
|
getSettings: vi.fn().mockResolvedValue({
|
||||||
|
maxConcurrent: 2,
|
||||||
|
maxWorktrees: 4,
|
||||||
|
pollIntervalMs: 10000,
|
||||||
|
groupOverlappingFiles: false,
|
||||||
|
autoMerge: true,
|
||||||
|
defaultProvider: "openai",
|
||||||
|
defaultModelId: "gpt-4o",
|
||||||
|
planningFallbackProvider: "anthropic",
|
||||||
|
planningFallbackModelId: "claude-3-5-haiku-20241022",
|
||||||
|
defaultThinkingLevel: "low",
|
||||||
|
} as Settings),
|
||||||
|
});
|
||||||
|
const mockDispose = vi.fn();
|
||||||
|
mockCreateFnAgent.mockResolvedValue({
|
||||||
|
session: {
|
||||||
|
state: {},
|
||||||
|
sessionManager: {},
|
||||||
|
prompt: vi.fn(),
|
||||||
|
dispose: mockDispose,
|
||||||
|
navigateTree: vi.fn(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js");
|
||||||
|
(promptWithFallback as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
|
||||||
|
new ModelFallbackExhaustedError({
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
|
||||||
|
triggerPoint: "prompt-time",
|
||||||
|
attempts: 2,
|
||||||
|
underlyingReason: "401 invalid api key for fallback",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const processor = new TriageProcessor(store, "/test/root", {
|
||||||
|
pollIntervalMs: 100_000,
|
||||||
|
onSpecifyError,
|
||||||
|
});
|
||||||
|
|
||||||
|
await processor.specifyTask(task);
|
||||||
|
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-7437",
|
||||||
|
"Triage using model: mock-model (thinking effort: low)",
|
||||||
|
);
|
||||||
|
expect(store.appendAgentLog).toHaveBeenCalledWith(
|
||||||
|
"FN-7437",
|
||||||
|
"Triage using model: mock-model (thinking effort: low)",
|
||||||
|
"text",
|
||||||
|
undefined,
|
||||||
|
"triage",
|
||||||
|
);
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-7437",
|
||||||
|
expect.stringContaining("Triage failed: unable to select a usable model after 2 attempts"),
|
||||||
|
);
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith("FN-7437", expect.objectContaining({
|
||||||
|
status: "failed",
|
||||||
|
error: expect.stringContaining("openai/gpt-4o"),
|
||||||
|
recoveryRetryCount: null,
|
||||||
|
nextRecoveryAt: null,
|
||||||
|
}));
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith("FN-7437", expect.objectContaining({
|
||||||
|
status: null,
|
||||||
|
error: null,
|
||||||
|
}));
|
||||||
|
expect(mockDispose).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onSpecifyError).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("persists terminal planning error when session state reports fallback exhaustion", async () => {
|
||||||
|
const task = {
|
||||||
|
id: "FN-7437-STATE",
|
||||||
|
description: "Bug: planner triage fallback loop via state error",
|
||||||
|
column: "triage",
|
||||||
|
dependencies: [],
|
||||||
|
steps: [],
|
||||||
|
currentStep: 0,
|
||||||
|
log: [],
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
} as unknown as Task;
|
||||||
|
const store = createMockStore({
|
||||||
|
getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
|
||||||
|
});
|
||||||
|
mockCreateFnAgent.mockResolvedValue({
|
||||||
|
session: {
|
||||||
|
state: {},
|
||||||
|
sessionManager: {},
|
||||||
|
prompt: vi.fn(),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
navigateTree: vi.fn(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js");
|
||||||
|
const exhausted = new ModelFallbackExhaustedError({
|
||||||
|
primaryModel: "openai/gpt-4o",
|
||||||
|
fallbackModel: "anthropic/claude-3-5-haiku-20241022",
|
||||||
|
triggerPoint: "prompt-time",
|
||||||
|
attempts: 2,
|
||||||
|
underlyingReason: "fallback session state error: 403 forbidden",
|
||||||
|
});
|
||||||
|
(promptWithFallback as ReturnType<typeof vi.fn>).mockRejectedValueOnce(exhausted);
|
||||||
|
|
||||||
|
const processor = new TriageProcessor(store, "/test/root", { pollIntervalMs: 100_000 });
|
||||||
|
await processor.specifyTask(task);
|
||||||
|
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith("FN-7437-STATE", expect.objectContaining({
|
||||||
|
status: "failed",
|
||||||
|
error: expect.stringContaining("fallback session state error: 403 forbidden"),
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
it("escalates to error state when triage retries are exhausted via specifyTask", async () => {
|
it("escalates to error state when triage retries are exhausted via specifyTask", async () => {
|
||||||
const task = {
|
const task = {
|
||||||
id: "FN-201",
|
id: "FN-201",
|
||||||
|
|||||||
@@ -967,6 +967,34 @@ export interface FallbackModelUsedPayload {
|
|||||||
timestamp?: string;
|
timestamp?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class ModelFallbackExhaustedError extends Error {
|
||||||
|
readonly primaryModel: string;
|
||||||
|
readonly fallbackModel?: string;
|
||||||
|
readonly triggerPoint: "session-creation" | "prompt-time";
|
||||||
|
readonly attempts: number;
|
||||||
|
readonly underlyingReason: string;
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
primaryModel: string;
|
||||||
|
fallbackModel?: string;
|
||||||
|
triggerPoint: "session-creation" | "prompt-time";
|
||||||
|
attempts: number;
|
||||||
|
underlyingReason: string;
|
||||||
|
}) {
|
||||||
|
const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured";
|
||||||
|
super(
|
||||||
|
`Unable to select a usable model after ${input.attempts} attempt${input.attempts === 1 ? "" : "s"} `
|
||||||
|
+ `(primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`,
|
||||||
|
);
|
||||||
|
this.name = "ModelFallbackExhaustedError";
|
||||||
|
this.primaryModel = input.primaryModel;
|
||||||
|
this.fallbackModel = input.fallbackModel;
|
||||||
|
this.triggerPoint = input.triggerPoint;
|
||||||
|
this.attempts = input.attempts;
|
||||||
|
this.underlyingReason = input.underlyingReason;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export type BuiltinWebToolName = "WebSearch" | "WebFetch";
|
export type BuiltinWebToolName = "WebSearch" | "WebFetch";
|
||||||
|
|
||||||
export interface AgentOptions {
|
export interface AgentOptions {
|
||||||
@@ -2396,8 +2424,34 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const modelDescription = (model: typeof selectedModel): string => model ? `${model.provider}/${model.id}` : "unknown model";
|
||||||
|
const configuredFallbackDiffers = Boolean(
|
||||||
|
options.fallbackProvider
|
||||||
|
&& options.fallbackModelId
|
||||||
|
&& (options.fallbackProvider !== options.defaultProvider || options.fallbackModelId !== options.defaultModelId),
|
||||||
|
);
|
||||||
|
const hasDistinctFallback = Boolean(
|
||||||
|
selectedModel
|
||||||
|
&& fallbackModel
|
||||||
|
&& (configuredFallbackDiffers || selectedModel.provider !== fallbackModel.provider || selectedModel.id !== fallbackModel.id),
|
||||||
|
);
|
||||||
|
const makeFallbackExhaustedError = (
|
||||||
|
triggerPoint: "session-creation" | "prompt-time",
|
||||||
|
attempts: number,
|
||||||
|
underlying: unknown,
|
||||||
|
): ModelFallbackExhaustedError => {
|
||||||
|
const underlyingReason = underlying instanceof Error ? underlying.message : String(underlying);
|
||||||
|
return new ModelFallbackExhaustedError({
|
||||||
|
primaryModel: modelDescription(selectedModel),
|
||||||
|
fallbackModel: hasDistinctFallback ? modelDescription(fallbackModel) : undefined,
|
||||||
|
triggerPoint,
|
||||||
|
attempts,
|
||||||
|
underlyingReason,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise<void> => {
|
const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise<void> => {
|
||||||
if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel) {
|
if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel || !hasDistinctFallback) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await options.onFallbackModelUsed({
|
await options.onFallbackModelUsed({
|
||||||
@@ -2410,19 +2464,27 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* FNXC:ModelFallback 2026-07-02-00:00:
|
||||||
|
* Planner and shared AI lanes may try one distinct fallback model for a logical model-selection failure, but they must then throw ModelFallbackExhaustedError instead of swapping back or relying on scheduler re-pick loops. This keeps transient fallback useful while making exhausted model configuration actionable for operators.
|
||||||
|
*/
|
||||||
let sessionResult;
|
let sessionResult;
|
||||||
let usingFallback = false;
|
let usingFallback = false;
|
||||||
try {
|
try {
|
||||||
sessionResult = await createSessionWithModel(selectedModel);
|
sessionResult = await createSessionWithModel(selectedModel);
|
||||||
piLog.log(`Session created successfully (model=${selectedModel ? `${selectedModel.provider}/${selectedModel.id}` : "default"})`);
|
piLog.log(`Session created successfully (model=${selectedModel ? `${selectedModel.provider}/${selectedModel.id}` : "default"})`);
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (!fallbackModel || !selectedModel || !isRetryableModelSelectionError(err?.message || "")) {
|
if (!fallbackModel || !selectedModel || !hasDistinctFallback || !isRetryableModelSelectionError(err?.message || "")) {
|
||||||
piLog.error(`Session creation failed: ${err.message}`);
|
piLog.error(`Session creation failed: ${err.message}`);
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
piLog.warn(`Primary model failed (${err.message}), trying fallback`);
|
piLog.warn(`Primary model failed (${err.message}), trying fallback`);
|
||||||
usingFallback = true;
|
usingFallback = true;
|
||||||
sessionResult = await createSessionWithModel(fallbackModel);
|
try {
|
||||||
|
sessionResult = await createSessionWithModel(fallbackModel);
|
||||||
|
} catch (fallbackErr: unknown) {
|
||||||
|
throw makeFallbackExhaustedError("session-creation", 2, fallbackErr);
|
||||||
|
}
|
||||||
await emitFallbackUsed("session-creation");
|
await emitFallbackUsed("session-creation");
|
||||||
piLog.log("Fallback session created successfully");
|
piLog.log("Fallback session created successfully");
|
||||||
}
|
}
|
||||||
@@ -2565,6 +2627,9 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
|||||||
if (!fallbackModel || usingFallback || !isRetryableModelSelectionError(errorMessage)) {
|
if (!fallbackModel || usingFallback || !isRetryableModelSelectionError(errorMessage)) {
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
|
if (!hasDistinctFallback) {
|
||||||
|
throw makeFallbackExhaustedError("prompt-time", 1, err);
|
||||||
|
}
|
||||||
|
|
||||||
usingFallback = true;
|
usingFallback = true;
|
||||||
const fallbackSession = await swapPromptSession(fallbackModel);
|
const fallbackSession = await swapPromptSession(fallbackModel);
|
||||||
@@ -2617,7 +2682,7 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
|||||||
throw fallbackErr;
|
throw fallbackErr;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw fallbackErr;
|
throw makeFallbackExhaustedError("prompt-time", 2, fallbackErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ import type {
|
|||||||
ToolDefinition,
|
ToolDefinition,
|
||||||
AgentSession,
|
AgentSession,
|
||||||
} from "@earendil-works/pi-coding-agent";
|
} from "@earendil-works/pi-coding-agent";
|
||||||
import { describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js";
|
import { ModelFallbackExhaustedError, describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js";
|
||||||
import {
|
import {
|
||||||
createResolvedAgentSession,
|
createResolvedAgentSession,
|
||||||
extractRuntimeHint,
|
extractRuntimeHint,
|
||||||
@@ -1396,6 +1396,29 @@ export class TriageProcessor {
|
|||||||
task.id,
|
task.id,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
);
|
);
|
||||||
|
} else if (err instanceof ModelFallbackExhaustedError) {
|
||||||
|
/*
|
||||||
|
FNXC:TriageModelFallback 2026-07-02-00:00:
|
||||||
|
Exhausted planner model fallback is terminal and operator-actionable: clearing status lets the scheduler recreate the same primary/fallback pair forever, so triage persists a failed task error with the bounded attempt count and sanitized provider reason.
|
||||||
|
*/
|
||||||
|
const failureMessage =
|
||||||
|
`Triage failed: unable to select a usable model after ${err.attempts} attempt${err.attempts === 1 ? "" : "s"}. ${err.message}`;
|
||||||
|
planLog.error(`✗ ${task.id} planner model fallback exhausted: ${failureMessage}`);
|
||||||
|
await this.store.logEntry(task.id, failureMessage).catch((logErr: unknown) => {
|
||||||
|
const msg = logErr instanceof Error ? logErr.message : String(logErr);
|
||||||
|
planLog.warn(`${task.id}: failed to log planner fallback exhaustion: ${msg}`);
|
||||||
|
});
|
||||||
|
await this.store.updateTask(task.id, {
|
||||||
|
status: "failed",
|
||||||
|
error: failureMessage,
|
||||||
|
recoveryRetryCount: null,
|
||||||
|
nextRecoveryAt: null,
|
||||||
|
}).catch((updateErr: unknown) => {
|
||||||
|
const msg = updateErr instanceof Error ? updateErr.message : String(updateErr);
|
||||||
|
planLog.warn(`${task.id}: failed to persist planner fallback exhaustion: ${msg}`);
|
||||||
|
});
|
||||||
|
this.options.onSpecifyError?.(task, err);
|
||||||
|
return;
|
||||||
} else if (isTransientError(errorMessage)) {
|
} else if (isTransientError(errorMessage)) {
|
||||||
// Transient network/infrastructure error — use bounded recovery policy
|
// Transient network/infrastructure error — use bounded recovery policy
|
||||||
const decision = computeRecoveryDecision({
|
const decision = computeRecoveryDecision({
|
||||||
|
|||||||
Reference in New Issue
Block a user