FN-6158: handle Codex auth-tier model incompatibility

Gracefully recover from unsupported Codex model selections on ChatGPT-account auth tiers.

- detect Codex and general model-auth-tier incompatibility errors as retryable model-selection failures
- trigger configured fallback models during session creation and prompt-time retries, with actionable operator guidance when no fallback works
- document the expanded fallback behavior and add engine regression coverage plus a CLI changeset

Files changed:
 .changeset/fn-6158-model-compatibility-fallback.md |   5 +
 docs/settings-reference.md                         |   2 +-
 .../pi-prompt-session-and-check-recursion.test.ts  |  25 ++++
 packages/engine/src/__tests__/pi.test.ts           | 136 ++++++++++++++++++++-
 .../src/__tests__/transient-error-detector.test.ts |  24 ++++
 packages/engine/src/pi.ts                          |  21 +++-
 packages/engine/src/transient-error-detector.ts    |  24 ++++
 7 files changed, 234 insertions(+), 3 deletions(-)

Fusion-Task-Id: FN-6158
Fusion-Task-Lineage: 3e91f647-40f1-4296-afa4-40861310e1c1
This commit is contained in:
gsxdsm
2026-06-09 21:16:42 -07:00
parent 1c49ae6b7c
commit 4435ca2c54
7 changed files with 234 additions and 3 deletions

View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": patch
---
Detect Codex model-auth-tier incompatibility as a model-selection error, trigger configured fallback models, and surface an actionable diagnostic when no fallback is available.

View File

@@ -37,7 +37,7 @@ Defaults from `DEFAULT_GLOBAL_SETTINGS`; key scope from `GLOBAL_SETTINGS_KEYS`.
| `dashboardFontScalePct` | `number` | `100` | Dashboard font scale percentage used by Appearance settings. Valid range: `85` to `125`; applied pre-hydration via document root font-size so board typography (column headers/counts, task cards, and quick-entry text) scales with the setting from first paint. |
| `defaultProvider` | `string` | `undefined` | Default AI provider. |
| `defaultModelId` | `string` | `undefined` | Default AI model ID. |
| `fallbackProvider` | `string` | `undefined` | Fallback provider when the primary default model hits transient provider failures. |
| `fallbackProvider` | `string` | `undefined` | Fallback provider when the primary default model hits transient provider failures or model-compatibility/auth-tier rejections. |
| `fallbackModelId` | `string` | `undefined` | Fallback model ID (must pair with `fallbackProvider`). |
| `defaultThinkingLevel` | `"off" \| "minimal" \| "low" \| "medium" \| "high"` | `undefined` | Default reasoning effort for AI sessions. If a provider/runtime rejects simultaneous `thinking` and `reasoning_effort` parameters, Fusion retries without the explicit thinking override instead of failing the run. |
| `ntfyEnabled` | `boolean` | `false` | Enable ntfy push notifications. |

View File

@@ -57,6 +57,31 @@ describe("promptSessionAndCheck recursion guard (FN-4930)", () => {
expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining("failed to inspect transcript"));
});
it("annotates model-auth-tier incompatibility errors with model identity and actionable hint", async () => {
const { promptSessionAndCheck } = await import("../pi.js");
const state = {
errorMessage: "",
messages: [],
};
const session = {
model: { provider: "openai-codex", id: "gpt-5.3-codex" },
prompt: vi.fn(async () => {
state.errorMessage =
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"";
}),
state,
} as any;
await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/model=openai-codex\/gpt-5\.3-codex/);
await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(
/Operator action required: this agent's configured model is not supported by the current authentication tier\./,
);
await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/configure a fallback model/);
await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/use a Codex-supported model \(not a GPT model\)/);
});
it("annotates unsupported message-role provider errors with actionable hint", async () => {
const { promptSessionAndCheck } = await import("../pi.js");

View File

@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js";
import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isModelAuthTierIncompatibilityError, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js";
import { createAgentSession, type AgentSession } from "@earendil-works/pi-coding-agent";
import { piLog } from "../logger.js";
@@ -803,6 +803,48 @@ describe("piLog structured diagnostics", () => {
);
});
it("fires fallback hook on session-creation model-auth-tier fallback", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession);
const onFallbackModelUsed = vi.fn();
createAgentSessionMock.mockReset();
createAgentSessionMock
.mockRejectedValueOnce(
new Error(
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
),
)
.mockResolvedValueOnce({
session: {
model: { provider: "test", id: "fallback-model" },
prompt: vi.fn(),
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
sessionFile: undefined,
},
} as any);
await createFnAgent({
cwd: "/test/project",
systemPrompt: "Test",
defaultProvider: "test",
defaultModelId: "primary-model",
fallbackProvider: "test",
fallbackModelId: "fallback-model",
taskId: "FN-1",
taskTitle: "My Task",
onFallbackModelUsed,
});
expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
expect(onFallbackModelUsed).toHaveBeenCalledWith(
expect.objectContaining({
triggerPoint: "session-creation",
taskId: "FN-1",
}),
);
});
it("fires fallback hook on prompt-time fallback", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession);
const onFallbackModelUsed = vi.fn();
@@ -851,6 +893,63 @@ describe("piLog structured diagnostics", () => {
);
});
it("fires fallback hook on prompt-time model-auth-tier fallback", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession);
const onFallbackModelUsed = vi.fn();
const primaryState = { errorMessage: "", messages: [] };
const modelAuthTierError =
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"";
const primarySession = {
model: { provider: "test", id: "primary-model" },
prompt: vi.fn(async () => {
primaryState.errorMessage = modelAuthTierError;
}),
state: primaryState,
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
sessionFile: undefined,
} as unknown as AgentSession;
const fallbackSession = {
model: { provider: "test", id: "fallback-model" },
prompt: vi.fn().mockResolvedValue(undefined),
state: { errorMessage: "", messages: [] },
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
sessionFile: undefined,
} as unknown as AgentSession;
createAgentSessionMock.mockReset();
createAgentSessionMock
.mockResolvedValueOnce({ session: primarySession } as any)
.mockResolvedValueOnce({ session: fallbackSession } as any);
const { session } = await createFnAgent({
cwd: "/test/project",
systemPrompt: "Test",
defaultProvider: "test",
defaultModelId: "primary-model",
fallbackProvider: "test",
fallbackModelId: "fallback-model",
taskId: "FN-2",
onFallbackModelUsed,
});
await (session as any).promptWithFallback("prompt text");
expect(fallbackSession.prompt).toHaveBeenCalledWith("prompt text");
expect(createAgentSessionMock).toHaveBeenCalledTimes(2);
expect(onFallbackModelUsed).toHaveBeenCalledWith(
expect.objectContaining({
triggerPoint: "prompt-time",
taskId: "FN-2",
}),
);
});
it("logs warning on primary model failure and fallback attempt", async () => {
const createAgentSessionMock = vi.mocked(createAgentSession);
createAgentSessionMock.mockReset();
@@ -913,7 +1012,42 @@ describe("piLog structured diagnostics", () => {
});
});
describe("isModelAuthTierIncompatibilityError", () => {
it("matches Codex ChatGPT-account model-auth-tier incompatibility errors", () => {
expect(
isModelAuthTierIncompatibilityError(
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
),
).toBe(true);
});
it("matches general model compatibility errors", () => {
expect(isModelAuthTierIncompatibilityError("The gpt-5.3-codex model is not supported for this account")).toBe(true);
expect(isModelAuthTierIncompatibilityError("model gpt-5.3-codex is not available to this organization")).toBe(true);
});
it("matches invalid_request_error model not found compatibility errors", () => {
expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: model gpt-5.3-codex was not found")).toBe(true);
});
it("does not match unrelated provider errors", () => {
expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: invalid temperature for this request")).toBe(false);
expect(isModelAuthTierIncompatibilityError("400 bad request: missing required field messages")).toBe(false);
expect(isModelAuthTierIncompatibilityError("ENOENT: no such file or directory")).toBe(false);
});
});
describe("isRetryableModelSelectionError", () => {
it("treats model-auth-tier incompatibility as model-selection retryable so the fallback model is tried", () => {
expect(
isRetryableModelSelectionError(
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
),
).toBe(true);
expect(isRetryableModelSelectionError("The gpt-5.3-codex model is not supported for this account")).toBe(true);
expect(isRetryableModelSelectionError("400 invalid_request_error: missing required field messages")).toBe(false);
});
it("treats an unsupported message-role rejection as model-selection retryable so the fallback model is tried (issue #1261)", () => {
expect(
isRetryableModelSelectionError(

View File

@@ -6,6 +6,7 @@ import {
extractMissingModulePath,
isOperatorActionableAgentError,
isStaleWorktreeModuleResolutionError,
isModelAuthTierIncompatibilityError,
isUnsupportedMessageRoleError,
isNonContinuableSessionError,
TRANSIENT_ERROR_PATTERNS,
@@ -325,6 +326,19 @@ describe("Transient Error Detector", () => {
});
});
describe("isModelAuthTierIncompatibilityError", () => {
it("matches model compatibility errors without matching generic 400s", () => {
expect(
isModelAuthTierIncompatibilityError(
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
),
).toBe(true);
expect(isModelAuthTierIncompatibilityError("model gpt-5.3-codex is not supported for this account")).toBe(true);
expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: model gpt-5.3-codex not found")).toBe(true);
expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: invalid temperature")).toBe(false);
});
});
describe("isOperatorActionableAgentError", () => {
it("returns true for credential/model/billing errors", () => {
expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
@@ -341,6 +355,16 @@ describe("Transient Error Detector", () => {
expect(classifyError(message)).toBe("permanent");
});
it("returns true for model-auth-tier compatibility errors", () => {
expect(
isOperatorActionableAgentError(
"Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"",
),
).toBe(true);
expect(isOperatorActionableAgentError("model gpt-5.3-codex is not supported for this account")).toBe(true);
expect(isOperatorActionableAgentError("400 invalid_request_error: missing required field messages")).toBe(false);
});
it("returns false for transient network errors", () => {
expect(isOperatorActionableAgentError("socket hang up")).toBe(false);
expect(isOperatorActionableAgentError("upstream connect error")).toBe(false);

View File

@@ -59,7 +59,8 @@ import { resolvePermanentAgentToolDecision } from "./permanent-agent-gating.js";
import type { SystemPromptLayers } from "./prompt-layers.js";
import { READONLY_ALLOWLIST, filterCustomToolsForReadonly, isReadonlyAllowed } from "./workflow-step-tool-policy.js";
import { createStreamingDeltaNormalizer } from "./streaming-delta.js";
import { isUnsupportedMessageRoleError } from "./transient-error-detector.js";
import { isModelAuthTierIncompatibilityError, isUnsupportedMessageRoleError } from "./transient-error-detector.js";
export { isModelAuthTierIncompatibilityError } from "./transient-error-detector.js";
const RTK_ACCEPTED_REWRITE_EXIT_CODES = new Set([0, 3]);
const RTK_EXPECTED_PASSTHROUGH_EXIT_CODES = new Set([1, 2]);
@@ -389,6 +390,15 @@ export async function promptSessionAndCheck(session: AgentSession, prompt: strin
piLog.warn(`pi state error — Codex WebSocket transport drop (model=${modelDesc}): ${stateError}`);
throw new Error(`${stateError} (model=${modelDesc})`);
}
if (isModelAuthTierIncompatibilityError(stateError)) {
const modelDesc = describeModel(session);
const hint =
"Operator action required: this agent's configured model is not supported by the current authentication tier. "
+ "Update the model selection in Settings → Models or configure a fallback model. "
+ "If using a ChatGPT account with Codex, use a Codex-supported model (not a GPT model).";
piLog.error(`pi state error — model not supported for auth tier (model=${modelDesc}): ${stateError}`);
throw new Error(`${stateError} (model=${modelDesc}). ${hint}`);
}
if (isUnsupportedMessageRoleError(stateError)) {
const modelDesc = describeModel(session);
const hint =
@@ -1028,6 +1038,15 @@ function resolveConfiguredModel(
}
export function isRetryableModelSelectionError(message: string): boolean {
// Codex ChatGPT-account auth-tier model incompatibility: the model is valid
// but not available for the current auth tier. This is a model-selection
// problem — a configured fallback model may work. Treat as retryable so the
// fallback path is tried once (the `usingFallback` guard prevents infinite
// swaps).
if (isModelAuthTierIncompatibilityError(message)) {
return true;
}
// An unsupported message-role rejection (e.g. a reasoning model sending the
// "developer" system role to a provider that only accepts
// system/user/assistant/tool) is fundamentally a model+provider

View File

@@ -193,6 +193,15 @@ export function extractMissingModulePath(errorMessage: string): string | null {
const UNSUPPORTED_MESSAGE_ROLE_PATTERN = /\bmessages\.\[\d+\]\.role\b[\s\S]*\bis not one of\b|\bis not one of\b[\s\S]*\bmessages\.\[\d+\]\.role\b/i;
const NON_CONTINUABLE_SESSION_PATTERN = /cannot continue from message role\s*[:=-]?\s*(?:['"`]?)(assistant|tool|function|system|user)(?:['"`]?)\b/i;
const MODEL_AUTH_TIER_INCOMPATIBILITY_PATTERNS: RegExp[] = [
// Codex ChatGPT-account auth-tier incompatibility: the model is valid, but
// unavailable for the current auth tier.
/\bmodel\b[\s\S]{0,160}\bnot\s+supported\s+when\s+using\s+Codex\s+with\s+a\s+ChatGPT\s+account\b/i,
// General provider model-compatibility shapes. Keep these model-scoped so
// generic 400/invalid_request_error failures are not treated as model swaps.
/\bmodel\b[\s\S]{0,160}\b(?:is|was)\s+not\s+(?:supported|available)\b/i,
/(?:['"`][^'"`]+['"`]\s+)?\bmodel\b\s+(?:is|was)\s+not\s+(?:supported|available)\b/i,
];
export function isUnsupportedMessageRoleError(errorMessage: string): boolean {
if (!errorMessage || typeof errorMessage !== "string") {
@@ -201,6 +210,20 @@ export function isUnsupportedMessageRoleError(errorMessage: string): boolean {
return UNSUPPORTED_MESSAGE_ROLE_PATTERN.test(errorMessage);
}
export function isModelAuthTierIncompatibilityError(errorMessage: string): boolean {
if (!errorMessage || typeof errorMessage !== "string") {
return false;
}
const hasModelContext = /\bmodel\b/i.test(errorMessage);
const hasCompatibilitySignal = /\bnot\s+(?:supported|available|found)\b/i.test(errorMessage);
if (/\binvalid_request_error\b/i.test(errorMessage) && hasModelContext && hasCompatibilitySignal) {
return true;
}
return MODEL_AUTH_TIER_INCOMPATIBILITY_PATTERNS.some((pattern) => pattern.test(errorMessage));
}
export function isNonContinuableSessionError(errorMessage: string): boolean {
if (!errorMessage || typeof errorMessage !== "string") {
return false;
@@ -229,6 +252,7 @@ export function isOperatorActionableAgentError(errorMessage: string): boolean {
}
return (
isUnsupportedMessageRoleError(errorMessage) ||
isModelAuthTierIncompatibilityError(errorMessage) ||
OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage))
);
}