FN-7437: bound planner fallback retries

Bound planner fallback retries so model exhaustion reports a clear terminal error instead of looping. - Add planner fallback exhaustion state and loop-limit handling for triage and pi planning paths. - Report exhausted provider/model fallback attempts with actionable error details. - Cover planner fallback exhaustion with engine regression tests and add a patch changeset. Files changed: .changeset/fn-7437-planner-fallback.md | 7 + .../src/__tests__/fallback-model-observer.test.ts | 31 ++++ packages/engine/src/__tests__/pi.test.ts | 106 ++++++++++++++ packages/engine/src/__tests__/triage.test.ts | 161 ++++++++++++++++++++- packages/engine/src/pi.ts | 73 +++++++++- packages/engine/src/triage.ts | 25 +++- 6 files changed, 396 insertions(+), 7 deletions(-) Fusion-Task-Id: FN-7437 Fusion-Task-Lineage: e2b8361a-c590-4f10-8f95-06dfd77e944c Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
2026-07-02 14:43:32 -07:00
parent dfb6e5270d
commit 1d6bb08e79
6 changed files with 396 additions and 7 deletions
--- a/.changeset/fn-7437-planner-fallback.md
+++ b/.changeset/fn-7437-planner-fallback.md
@@ -0,0 +1,7 @@
+---
+"@runfusion/fusion": patch
+---
+
+summary: Stop planner model fallback loops with a clear terminal triage error.
+category: fix
+dev: Bounds prompt-time/session-creation model fallback exhaustion and persists failed triage state.
--- a/packages/engine/src/tests/fallback-model-observer.test.ts
+++ b/packages/engine/src/tests/fallback-model-observer.test.ts
@@ -53,6 +53,37 @@ describe("createFallbackModelObserver", () => {
    });
  });

+  it("writes fallback events as non-empty rows with readable delimiters", async () => {
+    const store = {
+      logEntry: vi.fn().mockResolvedValue(undefined),
+      appendAgentLog: vi.fn().mockResolvedValue(undefined),
+    };
+    const observer = createFallbackModelObserver({
+      agent: "triage",
+      label: "triage",
+      store,
+      taskId: "FN-7437",
+    });
+
+    await observer({
+      primaryModel: "openai/gpt-4o",
+      fallbackModel: "anthropic/claude-3-5-haiku-20241022",
+      triggerPoint: "prompt-time",
+    });
+    await observer({
+      primaryModel: "openai/gpt-4o",
+      fallbackModel: "anthropic/claude-3-5-haiku-20241022",
+      triggerPoint: "prompt-time",
+    });
+
+    const rows = store.appendAgentLog.mock.calls.map((call) => call[1]);
+    expect(rows).toEqual([
+      "[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)",
+      "[fallback] triage switched from openai/gpt-4o to anthropic/claude-3-5-haiku-20241022 (prompt-time)",
+    ]);
+    expect(rows.every((row) => row.trim() === row && row.includes(" switched from ") && row.includes(" to "))).toBe(true);
+  });
+
  it("swallows logging failures and still dispatches a notification", async () => {
    const store = {
      logEntry: vi.fn().mockRejectedValue(new Error("log failed")),
--- a/packages/engine/src/tests/pi.test.ts
+++ b/packages/engine/src/tests/pi.test.ts
@@ -993,6 +993,112 @@ describe("piLog structured diagnostics", () => {
    );
  });

+  it("throws a bounded fallback exhaustion error when prompt-time fallback also fails", async () => {
+    const createAgentSessionMock = vi.mocked(createAgentSession);
+    vi.mocked(ModelRegistry.create).mockReturnValueOnce({
+      find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })),
+      getAll: vi.fn().mockReturnValue([]),
+      registerProvider: vi.fn(),
+      refresh: vi.fn(),
+    } as any);
+    const onFallbackModelUsed = vi.fn();
+
+    const primarySession = {
+      model: { provider: "openai", id: "gpt-4o" },
+      prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
+      subscribe: vi.fn(),
+      dispose: vi.fn(),
+      setThinkingLevel: vi.fn(),
+      sessionFile: undefined,
+    } as unknown as AgentSession;
+
+    const fallbackSession = {
+      model: { provider: "anthropic", id: "claude-3-5-haiku-20241022" },
+      prompt: vi.fn().mockRejectedValue(new Error("401 invalid api key for fallback")),
+      state: { errorMessage: "", messages: [] },
+      subscribe: vi.fn(),
+      dispose: vi.fn(),
+      setThinkingLevel: vi.fn(),
+      sessionFile: undefined,
+    } as unknown as AgentSession;
+
+    createAgentSessionMock.mockReset();
+    createAgentSessionMock
+      .mockResolvedValueOnce({ session: primarySession } as any)
+      .mockResolvedValueOnce({ session: fallbackSession } as any);
+
+    const { session } = await createFnAgent({
+      cwd: "/test/project",
+      systemPrompt: "Test planner fallback exhaustion",
+      defaultProvider: "openai",
+      defaultModelId: "gpt-4o",
+      fallbackProvider: "anthropic",
+      fallbackModelId: "claude-3-5-haiku-20241022",
+      taskId: "FN-7437",
+      onFallbackModelUsed,
+    });
+
+    await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({
+      name: "ModelFallbackExhaustedError",
+      attempts: 2,
+      primaryModel: "openai/gpt-4o",
+      fallbackModel: "anthropic/claude-3-5-haiku-20241022",
+      triggerPoint: "prompt-time",
+    });
+
+    expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
+    expect(primarySession.prompt).toHaveBeenCalledTimes(1);
+    expect(fallbackSession.prompt).toHaveBeenCalledTimes(1);
+    expect(onFallbackModelUsed).toHaveBeenCalledTimes(1);
+    expect(onFallbackModelUsed).toHaveBeenCalledWith(expect.objectContaining({
+      triggerPoint: "prompt-time",
+      primaryModel: "openai/gpt-4o",
+      fallbackModel: "anthropic/claude-3-5-haiku-20241022",
+      taskId: "FN-7437",
+    }));
+  });
+
+  it("does not create a meaningless prompt-time fallback when primary and fallback match", async () => {
+    const createAgentSessionMock = vi.mocked(createAgentSession);
+    vi.mocked(ModelRegistry.create).mockReturnValueOnce({
+      find: vi.fn((provider: string, id: string) => ({ provider, id, name: id })),
+      getAll: vi.fn().mockReturnValue([]),
+      registerProvider: vi.fn(),
+      refresh: vi.fn(),
+    } as any);
+    const onFallbackModelUsed = vi.fn();
+    const primarySession = {
+      model: { provider: "openai", id: "gpt-4o" },
+      prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
+      subscribe: vi.fn(),
+      dispose: vi.fn(),
+      setThinkingLevel: vi.fn(),
+      sessionFile: undefined,
+    } as unknown as AgentSession;
+
+    createAgentSessionMock.mockReset();
+    createAgentSessionMock.mockResolvedValueOnce({ session: primarySession } as any);
+
+    const { session } = await createFnAgent({
+      cwd: "/test/project",
+      systemPrompt: "Test same fallback",
+      defaultProvider: "openai",
+      defaultModelId: "gpt-4o",
+      fallbackProvider: "openai",
+      fallbackModelId: "gpt-4o",
+      onFallbackModelUsed,
+    });
+
+    await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({
+      name: "ModelFallbackExhaustedError",
+      attempts: 1,
+      primaryModel: "openai/gpt-4o",
+      fallbackModel: undefined,
+    });
+    expect(createAgentSessionMock).toHaveBeenCalledTimes(1);
+    expect(onFallbackModelUsed).not.toHaveBeenCalled();
+  });
+
  it("fires fallback hook on prompt-time model-auth-tier fallback", async () => {
    const createAgentSessionMock = vi.mocked(createAgentSession);
    const onFallbackModelUsed = vi.fn();
--- a/packages/engine/src/tests/triage.test.ts
+++ b/packages/engine/src/tests/triage.test.ts
@@ -29,7 +29,28 @@ vi.mock("../reviewer.js", () => ({
  reviewStep: mockReviewStep,
 }));

-vi.mock("../pi.js", () => ({
+vi.mock("../pi.js", () => {
+  class ModelFallbackExhaustedError extends Error {
+    readonly primaryModel: string;
+    readonly fallbackModel?: string;
+    readonly triggerPoint: "session-creation" | "prompt-time";
+    readonly attempts: number;
+    readonly underlyingReason: string;
+
+    constructor(input: { primaryModel: string; fallbackModel?: string; triggerPoint: "session-creation" | "prompt-time"; attempts: number; underlyingReason: string }) {
+      const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured";
+      super(`Unable to select a usable model after ${input.attempts} attempts (primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`);
+      this.name = "ModelFallbackExhaustedError";
+      this.primaryModel = input.primaryModel;
+      this.fallbackModel = input.fallbackModel;
+      this.triggerPoint = input.triggerPoint;
+      this.attempts = input.attempts;
+      this.underlyingReason = input.underlyingReason;
+    }
+  }
+
+  return {
+  ModelFallbackExhaustedError,
  createFnAgent: mockCreateFnAgent,
  describeModel: vi.fn().mockReturnValue("mock-model"),
  formatModelMarkerDetails: vi.fn((model: string, thinking?: string | null, annotations: string[] = []) => {
@@ -37,7 +58,8 @@ vi.mock("../pi.js", () => ({
    return suffixes.length ? `${model} ${suffixes.map((suffix) => `(${suffix})`).join(" ")}` : model;
  }),
  promptWithFallback: vi.fn().mockReturnValue("mock-prompt"),
-}));
+  };
+});

 vi.mock("@fusion/core", async (importOriginal) => {
  const { createEngineCoreMock } = await import("../test/mockCore.js");
@@ -3440,6 +3462,13 @@ describe("taskCreate tool model inheritance", () => {
  });

  describe("bounded recovery retries for triage", () => {
+    beforeEach(async () => {
+      vi.clearAllMocks();
+      const { promptWithFallback } = await import("../pi.js");
+      (promptWithFallback as ReturnType<typeof vi.fn>).mockReset();
+      (promptWithFallback as ReturnType<typeof vi.fn>).mockResolvedValue(undefined);
+    });
+
    it("requeues triage with backoff when the agent exits without writing PROMPT.md", async () => {
      const task = {
        id: "FN-202",
@@ -3521,6 +3550,134 @@ describe("taskCreate tool model inheritance", () => {
      }));
    });

+    it("persists terminal planning error when prompt-time primary and fallback models are exhausted", async () => {
+      const task = {
+        id: "FN-7437",
+        description: "Bug: planner triage fallback loop",
+        column: "triage",
+        dependencies: [],
+        steps: [],
+        currentStep: 0,
+        log: [],
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      } as unknown as Task;
+      const onSpecifyError = vi.fn();
+      const store = createMockStore({
+        getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
+        getSettings: vi.fn().mockResolvedValue({
+          maxConcurrent: 2,
+          maxWorktrees: 4,
+          pollIntervalMs: 10000,
+          groupOverlappingFiles: false,
+          autoMerge: true,
+          defaultProvider: "openai",
+          defaultModelId: "gpt-4o",
+          planningFallbackProvider: "anthropic",
+          planningFallbackModelId: "claude-3-5-haiku-20241022",
+          defaultThinkingLevel: "low",
+        } as Settings),
+      });
+      const mockDispose = vi.fn();
+      mockCreateFnAgent.mockResolvedValue({
+        session: {
+          state: {},
+          sessionManager: {},
+          prompt: vi.fn(),
+          dispose: mockDispose,
+          navigateTree: vi.fn(),
+        },
+      });
+      const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js");
+      (promptWithFallback as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
+        new ModelFallbackExhaustedError({
+          primaryModel: "openai/gpt-4o",
+          fallbackModel: "anthropic/claude-3-5-haiku-20241022",
+          triggerPoint: "prompt-time",
+          attempts: 2,
+          underlyingReason: "401 invalid api key for fallback",
+        }),
+      );
+
+      const processor = new TriageProcessor(store, "/test/root", {
+        pollIntervalMs: 100_000,
+        onSpecifyError,
+      });
+
+      await processor.specifyTask(task);
+
+      expect(store.logEntry).toHaveBeenCalledWith(
+        "FN-7437",
+        "Triage using model: mock-model (thinking effort: low)",
+      );
+      expect(store.appendAgentLog).toHaveBeenCalledWith(
+        "FN-7437",
+        "Triage using model: mock-model (thinking effort: low)",
+        "text",
+        undefined,
+        "triage",
+      );
+      expect(store.logEntry).toHaveBeenCalledWith(
+        "FN-7437",
+        expect.stringContaining("Triage failed: unable to select a usable model after 2 attempts"),
+      );
+      expect(store.updateTask).toHaveBeenCalledWith("FN-7437", expect.objectContaining({
+        status: "failed",
+        error: expect.stringContaining("openai/gpt-4o"),
+        recoveryRetryCount: null,
+        nextRecoveryAt: null,
+      }));
+      expect(store.updateTask).not.toHaveBeenCalledWith("FN-7437", expect.objectContaining({
+        status: null,
+        error: null,
+      }));
+      expect(mockDispose).toHaveBeenCalledTimes(1);
+      expect(onSpecifyError).toHaveBeenCalledTimes(1);
+    });
+
+    it("persists terminal planning error when session state reports fallback exhaustion", async () => {
+      const task = {
+        id: "FN-7437-STATE",
+        description: "Bug: planner triage fallback loop via state error",
+        column: "triage",
+        dependencies: [],
+        steps: [],
+        currentStep: 0,
+        log: [],
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      } as unknown as Task;
+      const store = createMockStore({
+        getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
+      });
+      mockCreateFnAgent.mockResolvedValue({
+        session: {
+          state: {},
+          sessionManager: {},
+          prompt: vi.fn(),
+          dispose: vi.fn(),
+          navigateTree: vi.fn(),
+        },
+      });
+      const { ModelFallbackExhaustedError, promptWithFallback } = await import("../pi.js");
+      const exhausted = new ModelFallbackExhaustedError({
+        primaryModel: "openai/gpt-4o",
+        fallbackModel: "anthropic/claude-3-5-haiku-20241022",
+        triggerPoint: "prompt-time",
+        attempts: 2,
+        underlyingReason: "fallback session state error: 403 forbidden",
+      });
+      (promptWithFallback as ReturnType<typeof vi.fn>).mockRejectedValueOnce(exhausted);
+
+      const processor = new TriageProcessor(store, "/test/root", { pollIntervalMs: 100_000 });
+      await processor.specifyTask(task);
+
+      expect(store.updateTask).toHaveBeenCalledWith("FN-7437-STATE", expect.objectContaining({
+        status: "failed",
+        error: expect.stringContaining("fallback session state error: 403 forbidden"),
+      }));
+    });
+
    it("escalates to error state when triage retries are exhausted via specifyTask", async () => {
      const task = {
        id: "FN-201",
--- a/packages/engine/src/pi.ts
+++ b/packages/engine/src/pi.ts
@@ -967,6 +967,34 @@ export interface FallbackModelUsedPayload {
  timestamp?: string;
 }

+export class ModelFallbackExhaustedError extends Error {
+  readonly primaryModel: string;
+  readonly fallbackModel?: string;
+  readonly triggerPoint: "session-creation" | "prompt-time";
+  readonly attempts: number;
+  readonly underlyingReason: string;
+
+  constructor(input: {
+    primaryModel: string;
+    fallbackModel?: string;
+    triggerPoint: "session-creation" | "prompt-time";
+    attempts: number;
+    underlyingReason: string;
+  }) {
+    const fallbackClause = input.fallbackModel ? `, fallback ${input.fallbackModel}` : ", no fallback configured";
+    super(
+      `Unable to select a usable model after ${input.attempts} attempt${input.attempts === 1 ? "" : "s"} `
+      + `(primary ${input.primaryModel}${fallbackClause}, trigger: ${input.triggerPoint}): ${input.underlyingReason}`,
+    );
+    this.name = "ModelFallbackExhaustedError";
+    this.primaryModel = input.primaryModel;
+    this.fallbackModel = input.fallbackModel;
+    this.triggerPoint = input.triggerPoint;
+    this.attempts = input.attempts;
+    this.underlyingReason = input.underlyingReason;
+  }
+}
+
 export type BuiltinWebToolName = "WebSearch" | "WebFetch";

 export interface AgentOptions {
@@ -2396,8 +2424,34 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
    }
  };

+  const modelDescription = (model: typeof selectedModel): string => model ? `${model.provider}/${model.id}` : "unknown model";
+  const configuredFallbackDiffers = Boolean(
+    options.fallbackProvider
+    && options.fallbackModelId
+    && (options.fallbackProvider !== options.defaultProvider || options.fallbackModelId !== options.defaultModelId),
+  );
+  const hasDistinctFallback = Boolean(
+    selectedModel
+    && fallbackModel
+    && (configuredFallbackDiffers || selectedModel.provider !== fallbackModel.provider || selectedModel.id !== fallbackModel.id),
+  );
+  const makeFallbackExhaustedError = (
+    triggerPoint: "session-creation" | "prompt-time",
+    attempts: number,
+    underlying: unknown,
+  ): ModelFallbackExhaustedError => {
+    const underlyingReason = underlying instanceof Error ? underlying.message : String(underlying);
+    return new ModelFallbackExhaustedError({
+      primaryModel: modelDescription(selectedModel),
+      fallbackModel: hasDistinctFallback ? modelDescription(fallbackModel) : undefined,
+      triggerPoint,
+      attempts,
+      underlyingReason,
+    });
+  };
+
  const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise<void> => {
-    if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel) {
+    if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel || !hasDistinctFallback) {
      return;
    }
    await options.onFallbackModelUsed({
@@ -2410,19 +2464,27 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
    });
  };

+  /*
+   * FNXC:ModelFallback 2026-07-02-00:00:
+   * Planner and shared AI lanes may try one distinct fallback model for a logical model-selection failure, but they must then throw ModelFallbackExhaustedError instead of swapping back or relying on scheduler re-pick loops. This keeps transient fallback useful while making exhausted model configuration actionable for operators.
+   */
  let sessionResult;
  let usingFallback = false;
  try {
    sessionResult = await createSessionWithModel(selectedModel);
    piLog.log(`Session created successfully (model=${selectedModel ? `${selectedModel.provider}/${selectedModel.id}` : "default"})`);
  } catch (err: any) {
-    if (!fallbackModel || !selectedModel || !isRetryableModelSelectionError(err?.message || "")) {
+    if (!fallbackModel || !selectedModel || !hasDistinctFallback || !isRetryableModelSelectionError(err?.message || "")) {
      piLog.error(`Session creation failed: ${err.message}`);
      throw err;
    }
    piLog.warn(`Primary model failed (${err.message}), trying fallback`);
    usingFallback = true;
-    sessionResult = await createSessionWithModel(fallbackModel);
+    try {
+      sessionResult = await createSessionWithModel(fallbackModel);
+    } catch (fallbackErr: unknown) {
+      throw makeFallbackExhaustedError("session-creation", 2, fallbackErr);
+    }
    await emitFallbackUsed("session-creation");
    piLog.log("Fallback session created successfully");
  }
@@ -2565,6 +2627,9 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
      if (!fallbackModel || usingFallback || !isRetryableModelSelectionError(errorMessage)) {
        throw err;
      }
+      if (!hasDistinctFallback) {
+        throw makeFallbackExhaustedError("prompt-time", 1, err);
+      }

      usingFallback = true;
      const fallbackSession = await swapPromptSession(fallbackModel);
@@ -2617,7 +2682,7 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
            throw fallbackErr;
          }
        }
-        throw fallbackErr;
+        throw makeFallbackExhaustedError("prompt-time", 2, fallbackErr);
      }
    }
  };
--- a/packages/engine/src/triage.ts
+++ b/packages/engine/src/triage.ts
@@ -80,7 +80,7 @@ import type {
  ToolDefinition,
  AgentSession,
 } from "@earendil-works/pi-coding-agent";
-import { describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js";
+import { ModelFallbackExhaustedError, describeModel, formatModelMarkerDetails, promptWithFallback } from "./pi.js";
 import {
  createResolvedAgentSession,
  extractRuntimeHint,
@@ -1396,6 +1396,29 @@ export class TriageProcessor {
            task.id,
            errorMessage,
          );
+        } else if (err instanceof ModelFallbackExhaustedError) {
+          /*
+          FNXC:TriageModelFallback 2026-07-02-00:00:
+          Exhausted planner model fallback is terminal and operator-actionable: clearing status lets the scheduler recreate the same primary/fallback pair forever, so triage persists a failed task error with the bounded attempt count and sanitized provider reason.
+          */
+          const failureMessage =
+            `Triage failed: unable to select a usable model after ${err.attempts} attempt${err.attempts === 1 ? "" : "s"}. ${err.message}`;
+          planLog.error(`✗ ${task.id} planner model fallback exhausted: ${failureMessage}`);
+          await this.store.logEntry(task.id, failureMessage).catch((logErr: unknown) => {
+            const msg = logErr instanceof Error ? logErr.message : String(logErr);
+            planLog.warn(`${task.id}: failed to log planner fallback exhaustion: ${msg}`);
+          });
+          await this.store.updateTask(task.id, {
+            status: "failed",
+            error: failureMessage,
+            recoveryRetryCount: null,
+            nextRecoveryAt: null,
+          }).catch((updateErr: unknown) => {
+            const msg = updateErr instanceof Error ? updateErr.message : String(updateErr);
+            planLog.warn(`${task.id}: failed to persist planner fallback exhaustion: ${msg}`);
+          });
+          this.options.onSpecifyError?.(task, err);
+          return;
        } else if (isTransientError(errorMessage)) {
          // Transient network/infrastructure error — use bounded recovery policy
          const decision = computeRecoveryDecision({