FN-6158: handle Codex auth-tier model incompatibility

Gracefully recover from unsupported Codex model selections on ChatGPT-account auth tiers. - detect Codex and general model-auth-tier incompatibility errors as retryable model-selection failures - trigger configured fallback models during session creation and prompt-time retries, with actionable operator guidance when no fallback works - document the expanded fallback behavior and add engine regression coverage plus a CLI changeset Files changed: .changeset/fn-6158-model-compatibility-fallback.md | 5 + docs/settings-reference.md | 2 +- .../pi-prompt-session-and-check-recursion.test.ts | 25 ++++ packages/engine/src/__tests__/pi.test.ts | 136 ++++++++++++++++++++- .../src/__tests__/transient-error-detector.test.ts | 24 ++++ packages/engine/src/pi.ts | 21 +++- packages/engine/src/transient-error-detector.ts | 24 ++++ 7 files changed, 234 insertions(+), 3 deletions(-) Fusion-Task-Id: FN-6158 Fusion-Task-Lineage: 3e91f647-40f1-4296-afa4-40861310e1c1
2026-06-09 21:16:42 -07:00
parent 1c49ae6b7c
commit 4435ca2c54
7 changed files with 234 additions and 3 deletions
--- a/packages/engine/src/tests/pi-prompt-session-and-check-recursion.test.ts
+++ b/packages/engine/src/tests/pi-prompt-session-and-check-recursion.test.ts
@@ -57,6 +57,31 @@ describe("promptSessionAndCheck recursion guard (FN-4930)", () => {
    expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining("failed to inspect transcript"));
  });

+  it("annotates model-auth-tier incompatibility errors with model identity and actionable hint", async () => {
+    const { promptSessionAndCheck } = await import("../pi.js");
+
+    const state = {
+      errorMessage: "",
+      messages: [],
+    };
+
+    const session = {
+      model: { provider: "openai-codex", id: "gpt-5.3-codex" },
+      prompt: vi.fn(async () => {
+        state.errorMessage =
+          "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"";
+      }),
+      state,
+    } as any;
+
+    await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/model=openai-codex\/gpt-5\.3-codex/);
+    await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(
+      /Operator action required: this agent's configured model is not supported by the current authentication tier\./,
+    );
+    await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/configure a fallback model/);
+    await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/use a Codex-supported model \(not a GPT model\)/);
+  });
+
  it("annotates unsupported message-role provider errors with actionable hint", async () => {
    const { promptSessionAndCheck } = await import("../pi.js");

--- a/packages/engine/src/tests/pi.test.ts
+++ b/packages/engine/src/tests/pi.test.ts
@@ -1,5 +1,5 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js";
+import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isModelAuthTierIncompatibilityError, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js";
 import { createAgentSession, type AgentSession } from "@earendil-works/pi-coding-agent";
 import { piLog } from "../logger.js";

@@ -803,6 +803,48 @@ describe("piLog structured diagnostics", () => {
    );
  });

+  it("fires fallback hook on session-creation model-auth-tier fallback", async () => {
+    const createAgentSessionMock = vi.mocked(createAgentSession);
+    const onFallbackModelUsed = vi.fn();
+    createAgentSessionMock.mockReset();
+    createAgentSessionMock
+      .mockRejectedValueOnce(
+        new Error(
+          "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
+        ),
+      )
+      .mockResolvedValueOnce({
+        session: {
+          model: { provider: "test", id: "fallback-model" },
+          prompt: vi.fn(),
+          subscribe: vi.fn(),
+          dispose: vi.fn(),
+          setThinkingLevel: vi.fn(),
+          sessionFile: undefined,
+        },
+      } as any);
+
+    await createFnAgent({
+      cwd: "/test/project",
+      systemPrompt: "Test",
+      defaultProvider: "test",
+      defaultModelId: "primary-model",
+      fallbackProvider: "test",
+      fallbackModelId: "fallback-model",
+      taskId: "FN-1",
+      taskTitle: "My Task",
+      onFallbackModelUsed,
+    });
+
+    expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
+    expect(onFallbackModelUsed).toHaveBeenCalledWith(
+      expect.objectContaining({
+        triggerPoint: "session-creation",
+        taskId: "FN-1",
+      }),
+    );
+  });
+
  it("fires fallback hook on prompt-time fallback", async () => {
    const createAgentSessionMock = vi.mocked(createAgentSession);
    const onFallbackModelUsed = vi.fn();
@@ -851,6 +893,63 @@ describe("piLog structured diagnostics", () => {
    );
  });

+  it("fires fallback hook on prompt-time model-auth-tier fallback", async () => {
+    const createAgentSessionMock = vi.mocked(createAgentSession);
+    const onFallbackModelUsed = vi.fn();
+    const primaryState = { errorMessage: "", messages: [] };
+    const modelAuthTierError =
+      "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"";
+
+    const primarySession = {
+      model: { provider: "test", id: "primary-model" },
+      prompt: vi.fn(async () => {
+        primaryState.errorMessage = modelAuthTierError;
+      }),
+      state: primaryState,
+      subscribe: vi.fn(),
+      dispose: vi.fn(),
+      setThinkingLevel: vi.fn(),
+      sessionFile: undefined,
+    } as unknown as AgentSession;
+
+    const fallbackSession = {
+      model: { provider: "test", id: "fallback-model" },
+      prompt: vi.fn().mockResolvedValue(undefined),
+      state: { errorMessage: "", messages: [] },
+      subscribe: vi.fn(),
+      dispose: vi.fn(),
+      setThinkingLevel: vi.fn(),
+      sessionFile: undefined,
+    } as unknown as AgentSession;
+
+    createAgentSessionMock.mockReset();
+    createAgentSessionMock
+      .mockResolvedValueOnce({ session: primarySession } as any)
+      .mockResolvedValueOnce({ session: fallbackSession } as any);
+
+    const { session } = await createFnAgent({
+      cwd: "/test/project",
+      systemPrompt: "Test",
+      defaultProvider: "test",
+      defaultModelId: "primary-model",
+      fallbackProvider: "test",
+      fallbackModelId: "fallback-model",
+      taskId: "FN-2",
+      onFallbackModelUsed,
+    });
+
+    await (session as any).promptWithFallback("prompt text");
+
+    expect(fallbackSession.prompt).toHaveBeenCalledWith("prompt text");
+    expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
+    expect(onFallbackModelUsed).toHaveBeenCalledWith(
+      expect.objectContaining({
+        triggerPoint: "prompt-time",
+        taskId: "FN-2",
+      }),
+    );
+  });
+
  it("logs warning on primary model failure and fallback attempt", async () => {
    const createAgentSessionMock = vi.mocked(createAgentSession);
    createAgentSessionMock.mockReset();
@@ -913,7 +1012,42 @@ describe("piLog structured diagnostics", () => {
  });
 });

+describe("isModelAuthTierIncompatibilityError", () => {
+  it("matches Codex ChatGPT-account model-auth-tier incompatibility errors", () => {
+    expect(
+      isModelAuthTierIncompatibilityError(
+        "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
+      ),
+    ).toBe(true);
+  });
+
+  it("matches general model compatibility errors", () => {
+    expect(isModelAuthTierIncompatibilityError("The gpt-5.3-codex model is not supported for this account")).toBe(true);
+    expect(isModelAuthTierIncompatibilityError("model gpt-5.3-codex is not available to this organization")).toBe(true);
+  });
+
+  it("matches invalid_request_error model not found compatibility errors", () => {
+    expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: model gpt-5.3-codex was not found")).toBe(true);
+  });
+
+  it("does not match unrelated provider errors", () => {
+    expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: invalid temperature for this request")).toBe(false);
+    expect(isModelAuthTierIncompatibilityError("400 bad request: missing required field messages")).toBe(false);
+    expect(isModelAuthTierIncompatibilityError("ENOENT: no such file or directory")).toBe(false);
+  });
+});
+
 describe("isRetryableModelSelectionError", () => {
+  it("treats model-auth-tier incompatibility as model-selection retryable so the fallback model is tried", () => {
+    expect(
+      isRetryableModelSelectionError(
+        "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
+      ),
+    ).toBe(true);
+    expect(isRetryableModelSelectionError("The gpt-5.3-codex model is not supported for this account")).toBe(true);
+    expect(isRetryableModelSelectionError("400 invalid_request_error: missing required field messages")).toBe(false);
+  });
+
  it("treats an unsupported message-role rejection as model-selection retryable so the fallback model is tried (issue #1261)", () => {
    expect(
      isRetryableModelSelectionError(
--- a/packages/engine/src/tests/transient-error-detector.test.ts
+++ b/packages/engine/src/tests/transient-error-detector.test.ts
@@ -6,6 +6,7 @@ import {
  extractMissingModulePath,
  isOperatorActionableAgentError,
  isStaleWorktreeModuleResolutionError,
+  isModelAuthTierIncompatibilityError,
  isUnsupportedMessageRoleError,
  isNonContinuableSessionError,
  TRANSIENT_ERROR_PATTERNS,
@@ -325,6 +326,19 @@ describe("Transient Error Detector", () => {
    });
  });

+  describe("isModelAuthTierIncompatibilityError", () => {
+    it("matches model compatibility errors without matching generic 400s", () => {
+      expect(
+        isModelAuthTierIncompatibilityError(
+          "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
+        ),
+      ).toBe(true);
+      expect(isModelAuthTierIncompatibilityError("model gpt-5.3-codex is not supported for this account")).toBe(true);
+      expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: model gpt-5.3-codex not found")).toBe(true);
+      expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: invalid temperature")).toBe(false);
+    });
+  });
+
  describe("isOperatorActionableAgentError", () => {
    it("returns true for credential/model/billing errors", () => {
      expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
@@ -341,6 +355,16 @@ describe("Transient Error Detector", () => {
      expect(classifyError(message)).toBe("permanent");
    });

+    it("returns true for model-auth-tier compatibility errors", () => {
+      expect(
+        isOperatorActionableAgentError(
+          "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
+        ),
+      ).toBe(true);
+      expect(isOperatorActionableAgentError("model gpt-5.3-codex is not supported for this account")).toBe(true);
+      expect(isOperatorActionableAgentError("400 invalid_request_error: missing required field messages")).toBe(false);
+    });
+
    it("returns false for transient network errors", () => {
      expect(isOperatorActionableAgentError("socket hang up")).toBe(false);
      expect(isOperatorActionableAgentError("upstream connect error")).toBe(false);
--- a/packages/engine/src/pi.ts
+++ b/packages/engine/src/pi.ts
@@ -59,7 +59,8 @@ import { resolvePermanentAgentToolDecision } from "./permanent-agent-gating.js";
 import type { SystemPromptLayers } from "./prompt-layers.js";
 import { READONLY_ALLOWLIST, filterCustomToolsForReadonly, isReadonlyAllowed } from "./workflow-step-tool-policy.js";
 import { createStreamingDeltaNormalizer } from "./streaming-delta.js";
-import { isUnsupportedMessageRoleError } from "./transient-error-detector.js";
+import { isModelAuthTierIncompatibilityError, isUnsupportedMessageRoleError } from "./transient-error-detector.js";
+export { isModelAuthTierIncompatibilityError } from "./transient-error-detector.js";

 const RTK_ACCEPTED_REWRITE_EXIT_CODES = new Set([0, 3]);
 const RTK_EXPECTED_PASSTHROUGH_EXIT_CODES = new Set([1, 2]);
@@ -389,6 +390,15 @@ export async function promptSessionAndCheck(session: AgentSession, prompt: strin
      piLog.warn(`pi state error — Codex WebSocket transport drop (model=${modelDesc}): ${stateError}`);
      throw new Error(`${stateError} (model=${modelDesc})`);
    }
+    if (isModelAuthTierIncompatibilityError(stateError)) {
+      const modelDesc = describeModel(session);
+      const hint =
+        "Operator action required: this agent's configured model is not supported by the current authentication tier. "
+        + "Update the model selection in Settings → Models or configure a fallback model. "
+        + "If using a ChatGPT account with Codex, use a Codex-supported model (not a GPT model).";
+      piLog.error(`pi state error — model not supported for auth tier (model=${modelDesc}): ${stateError}`);
+      throw new Error(`${stateError} (model=${modelDesc}). ${hint}`);
+    }
    if (isUnsupportedMessageRoleError(stateError)) {
      const modelDesc = describeModel(session);
      const hint =
@@ -1028,6 +1038,15 @@ function resolveConfiguredModel(
 }

 export function isRetryableModelSelectionError(message: string): boolean {
+  // Codex ChatGPT-account auth-tier model incompatibility: the model is valid
+  // but not available for the current auth tier. This is a model-selection
+  // problem — a configured fallback model may work. Treat as retryable so the
+  // fallback path is tried once (the `usingFallback` guard prevents infinite
+  // swaps).
+  if (isModelAuthTierIncompatibilityError(message)) {
+    return true;
+  }
+
  // An unsupported message-role rejection (e.g. a reasoning model sending the
  // "developer" system role to a provider that only accepts
  // system/user/assistant/tool) is fundamentally a model+provider
--- a/packages/engine/src/transient-error-detector.ts
+++ b/packages/engine/src/transient-error-detector.ts
@@ -193,6 +193,15 @@ export function extractMissingModulePath(errorMessage: string): string | null {

 const UNSUPPORTED_MESSAGE_ROLE_PATTERN = /\bmessages\.\[\d+\]\.role\b[\s\S]*\bis not one of\b|\bis not one of\b[\s\S]*\bmessages\.\[\d+\]\.role\b/i;
 const NON_CONTINUABLE_SESSION_PATTERN = /cannot continue from message role\s*[:=-]?\s*(?:['"`]?)(assistant|tool|function|system|user)(?:['"`]?)\b/i;
+const MODEL_AUTH_TIER_INCOMPATIBILITY_PATTERNS: RegExp[] = [
+  // Codex ChatGPT-account auth-tier incompatibility: the model is valid, but
+  // unavailable for the current auth tier.
+  /\bmodel\b[\s\S]{0,160}\bnot\s+supported\s+when\s+using\s+Codex\s+with\s+a\s+ChatGPT\s+account\b/i,
+  // General provider model-compatibility shapes. Keep these model-scoped so
+  // generic 400/invalid_request_error failures are not treated as model swaps.
+  /\bmodel\b[\s\S]{0,160}\b(?:is|was)\s+not\s+(?:supported|available)\b/i,
+  /(?:['"`][^'"`]+['"`]\s+)?\bmodel\b\s+(?:is|was)\s+not\s+(?:supported|available)\b/i,
+];

 export function isUnsupportedMessageRoleError(errorMessage: string): boolean {
  if (!errorMessage || typeof errorMessage !== "string") {
@@ -201,6 +210,20 @@ export function isUnsupportedMessageRoleError(errorMessage: string): boolean {
  return UNSUPPORTED_MESSAGE_ROLE_PATTERN.test(errorMessage);
 }

+export function isModelAuthTierIncompatibilityError(errorMessage: string): boolean {
+  if (!errorMessage || typeof errorMessage !== "string") {
+    return false;
+  }
+
+  const hasModelContext = /\bmodel\b/i.test(errorMessage);
+  const hasCompatibilitySignal = /\bnot\s+(?:supported|available|found)\b/i.test(errorMessage);
+  if (/\binvalid_request_error\b/i.test(errorMessage) && hasModelContext && hasCompatibilitySignal) {
+    return true;
+  }
+
+  return MODEL_AUTH_TIER_INCOMPATIBILITY_PATTERNS.some((pattern) => pattern.test(errorMessage));
+}
+
 export function isNonContinuableSessionError(errorMessage: string): boolean {
  if (!errorMessage || typeof errorMessage !== "string") {
    return false;
@@ -229,6 +252,7 @@ export function isOperatorActionableAgentError(errorMessage: string): boolean {
  }
  return (
    isUnsupportedMessageRoleError(errorMessage) ||
+    isModelAuthTierIncompatibilityError(errorMessage) ||
    OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage))
  );
 }