diff --git a/.changeset/fn-6158-model-compatibility-fallback.md b/.changeset/fn-6158-model-compatibility-fallback.md new file mode 100644 index 0000000000..480f788250 --- /dev/null +++ b/.changeset/fn-6158-model-compatibility-fallback.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Detect Codex model-auth-tier incompatibility as a model-selection error, trigger configured fallback models, and surface an actionable diagnostic when no fallback is available. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 0527e03f00..60dfbafcf0 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -37,7 +37,7 @@ Defaults from `DEFAULT_GLOBAL_SETTINGS`; key scope from `GLOBAL_SETTINGS_KEYS`. | `dashboardFontScalePct` | `number` | `100` | Dashboard font scale percentage used by Appearance settings. Valid range: `85` to `125`; applied pre-hydration via document root font-size so board typography (column headers/counts, task cards, and quick-entry text) scales with the setting from first paint. | | `defaultProvider` | `string` | `undefined` | Default AI provider. | | `defaultModelId` | `string` | `undefined` | Default AI model ID. | -| `fallbackProvider` | `string` | `undefined` | Fallback provider when the primary default model hits transient provider failures. | +| `fallbackProvider` | `string` | `undefined` | Fallback provider when the primary default model hits transient provider failures or model-compatibility/auth-tier rejections. | | `fallbackModelId` | `string` | `undefined` | Fallback model ID (must pair with `fallbackProvider`). | | `defaultThinkingLevel` | `"off" \| "minimal" \| "low" \| "medium" \| "high"` | `undefined` | Default reasoning effort for AI sessions. If a provider/runtime rejects simultaneous `thinking` and `reasoning_effort` parameters, Fusion retries without the explicit thinking override instead of failing the run. | | `ntfyEnabled` | `boolean` | `false` | Enable ntfy push notifications. | diff --git a/packages/engine/src/__tests__/pi-prompt-session-and-check-recursion.test.ts b/packages/engine/src/__tests__/pi-prompt-session-and-check-recursion.test.ts index e8bc679d90..1bb0de5005 100644 --- a/packages/engine/src/__tests__/pi-prompt-session-and-check-recursion.test.ts +++ b/packages/engine/src/__tests__/pi-prompt-session-and-check-recursion.test.ts @@ -57,6 +57,31 @@ describe("promptSessionAndCheck recursion guard (FN-4930)", () => { expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining("failed to inspect transcript")); }); + it("annotates model-auth-tier incompatibility errors with model identity and actionable hint", async () => { + const { promptSessionAndCheck } = await import("../pi.js"); + + const state = { + errorMessage: "", + messages: [], + }; + + const session = { + model: { provider: "openai-codex", id: "gpt-5.3-codex" }, + prompt: vi.fn(async () => { + state.errorMessage = + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\""; + }), + state, + } as any; + + await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/model=openai-codex\/gpt-5\.3-codex/); + await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow( + /Operator action required: this agent's configured model is not supported by the current authentication tier\./, + ); + await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/configure a fallback model/); + await expect(promptSessionAndCheck(session, "hello")).rejects.toThrow(/use a Codex-supported model \(not a GPT model\)/); + }); + it("annotates unsupported message-role provider errors with actionable hint", async () => { const { promptSessionAndCheck } = await import("../pi.js"); diff --git a/packages/engine/src/__tests__/pi.test.ts b/packages/engine/src/__tests__/pi.test.ts index d88dae1282..c638a83b2c 100644 --- a/packages/engine/src/__tests__/pi.test.ts +++ b/packages/engine/src/__tests__/pi.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js"; +import { describeModel, compactSessionContext, COMPACTION_FALLBACK_INSTRUCTIONS, createFnAgent, getProjectRootFromWorktree, isModelAuthTierIncompatibilityError, isRetryableModelSelectionError, promptWithFallback, type AgentOptions } from "../pi.js"; import { createAgentSession, type AgentSession } from "@earendil-works/pi-coding-agent"; import { piLog } from "../logger.js"; @@ -803,6 +803,48 @@ describe("piLog structured diagnostics", () => { ); }); + it("fires fallback hook on session-creation model-auth-tier fallback", async () => { + const createAgentSessionMock = vi.mocked(createAgentSession); + const onFallbackModelUsed = vi.fn(); + createAgentSessionMock.mockReset(); + createAgentSessionMock + .mockRejectedValueOnce( + new Error( + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"", + ), + ) + .mockResolvedValueOnce({ + session: { + model: { provider: "test", id: "fallback-model" }, + prompt: vi.fn(), + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + }, + } as any); + + await createFnAgent({ + cwd: "/test/project", + systemPrompt: "Test", + defaultProvider: "test", + defaultModelId: "primary-model", + fallbackProvider: "test", + fallbackModelId: "fallback-model", + taskId: "FN-1", + taskTitle: "My Task", + onFallbackModelUsed, + }); + + expect(createAgentSessionMock).toHaveBeenCalledTimes(2); + expect(onFallbackModelUsed).toHaveBeenCalledWith( + expect.objectContaining({ + triggerPoint: "session-creation", + taskId: "FN-1", + }), + ); + }); + it("fires fallback hook on prompt-time fallback", async () => { const createAgentSessionMock = vi.mocked(createAgentSession); const onFallbackModelUsed = vi.fn(); @@ -851,6 +893,63 @@ describe("piLog structured diagnostics", () => { ); }); + it("fires fallback hook on prompt-time model-auth-tier fallback", async () => { + const createAgentSessionMock = vi.mocked(createAgentSession); + const onFallbackModelUsed = vi.fn(); + const primaryState = { errorMessage: "", messages: [] }; + const modelAuthTierError = + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\""; + + const primarySession = { + model: { provider: "test", id: "primary-model" }, + prompt: vi.fn(async () => { + primaryState.errorMessage = modelAuthTierError; + }), + state: primaryState, + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + } as unknown as AgentSession; + + const fallbackSession = { + model: { provider: "test", id: "fallback-model" }, + prompt: vi.fn().mockResolvedValue(undefined), + state: { errorMessage: "", messages: [] }, + subscribe: vi.fn(), + dispose: vi.fn(), + setThinkingLevel: vi.fn(), + sessionFile: undefined, + } as unknown as AgentSession; + + createAgentSessionMock.mockReset(); + createAgentSessionMock + .mockResolvedValueOnce({ session: primarySession } as any) + .mockResolvedValueOnce({ session: fallbackSession } as any); + + const { session } = await createFnAgent({ + cwd: "/test/project", + systemPrompt: "Test", + defaultProvider: "test", + defaultModelId: "primary-model", + fallbackProvider: "test", + fallbackModelId: "fallback-model", + taskId: "FN-2", + onFallbackModelUsed, + }); + + await (session as any).promptWithFallback("prompt text"); + + expect(fallbackSession.prompt).toHaveBeenCalledWith("prompt text"); + expect(createAgentSessionMock).toHaveBeenCalledTimes(2); + expect(onFallbackModelUsed).toHaveBeenCalledWith( + expect.objectContaining({ + triggerPoint: "prompt-time", + taskId: "FN-2", + }), + ); + }); + it("logs warning on primary model failure and fallback attempt", async () => { const createAgentSessionMock = vi.mocked(createAgentSession); createAgentSessionMock.mockReset(); @@ -913,7 +1012,42 @@ describe("piLog structured diagnostics", () => { }); }); +describe("isModelAuthTierIncompatibilityError", () => { + it("matches Codex ChatGPT-account model-auth-tier incompatibility errors", () => { + expect( + isModelAuthTierIncompatibilityError( + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"", + ), + ).toBe(true); + }); + + it("matches general model compatibility errors", () => { + expect(isModelAuthTierIncompatibilityError("The gpt-5.3-codex model is not supported for this account")).toBe(true); + expect(isModelAuthTierIncompatibilityError("model gpt-5.3-codex is not available to this organization")).toBe(true); + }); + + it("matches invalid_request_error model not found compatibility errors", () => { + expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: model gpt-5.3-codex was not found")).toBe(true); + }); + + it("does not match unrelated provider errors", () => { + expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: invalid temperature for this request")).toBe(false); + expect(isModelAuthTierIncompatibilityError("400 bad request: missing required field messages")).toBe(false); + expect(isModelAuthTierIncompatibilityError("ENOENT: no such file or directory")).toBe(false); + }); +}); + describe("isRetryableModelSelectionError", () => { + it("treats model-auth-tier incompatibility as model-selection retryable so the fallback model is tried", () => { + expect( + isRetryableModelSelectionError( + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"", + ), + ).toBe(true); + expect(isRetryableModelSelectionError("The gpt-5.3-codex model is not supported for this account")).toBe(true); + expect(isRetryableModelSelectionError("400 invalid_request_error: missing required field messages")).toBe(false); + }); + it("treats an unsupported message-role rejection as model-selection retryable so the fallback model is tried (issue #1261)", () => { expect( isRetryableModelSelectionError( diff --git a/packages/engine/src/__tests__/transient-error-detector.test.ts b/packages/engine/src/__tests__/transient-error-detector.test.ts index 64e673ff9d..cc8a6328ad 100644 --- a/packages/engine/src/__tests__/transient-error-detector.test.ts +++ b/packages/engine/src/__tests__/transient-error-detector.test.ts @@ -6,6 +6,7 @@ import { extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError, + isModelAuthTierIncompatibilityError, isUnsupportedMessageRoleError, isNonContinuableSessionError, TRANSIENT_ERROR_PATTERNS, @@ -325,6 +326,19 @@ describe("Transient Error Detector", () => { }); }); + describe("isModelAuthTierIncompatibilityError", () => { + it("matches model compatibility errors without matching generic 400s", () => { + expect( + isModelAuthTierIncompatibilityError( + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"", + ), + ).toBe(true); + expect(isModelAuthTierIncompatibilityError("model gpt-5.3-codex is not supported for this account")).toBe(true); + expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: model gpt-5.3-codex not found")).toBe(true); + expect(isModelAuthTierIncompatibilityError("400 invalid_request_error: invalid temperature")).toBe(false); + }); + }); + describe("isOperatorActionableAgentError", () => { it("returns true for credential/model/billing errors", () => { expect(isOperatorActionableAgentError("invalid api key")).toBe(true); @@ -341,6 +355,16 @@ describe("Transient Error Detector", () => { expect(classifyError(message)).toBe("permanent"); }); + it("returns true for model-auth-tier compatibility errors", () => { + expect( + isOperatorActionableAgentError( + "Codex error: 400 invalid_request_error — \"The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.\"", + ), + ).toBe(true); + expect(isOperatorActionableAgentError("model gpt-5.3-codex is not supported for this account")).toBe(true); + expect(isOperatorActionableAgentError("400 invalid_request_error: missing required field messages")).toBe(false); + }); + it("returns false for transient network errors", () => { expect(isOperatorActionableAgentError("socket hang up")).toBe(false); expect(isOperatorActionableAgentError("upstream connect error")).toBe(false); diff --git a/packages/engine/src/pi.ts b/packages/engine/src/pi.ts index 8f891b2728..c5e7646afd 100644 --- a/packages/engine/src/pi.ts +++ b/packages/engine/src/pi.ts @@ -59,7 +59,8 @@ import { resolvePermanentAgentToolDecision } from "./permanent-agent-gating.js"; import type { SystemPromptLayers } from "./prompt-layers.js"; import { READONLY_ALLOWLIST, filterCustomToolsForReadonly, isReadonlyAllowed } from "./workflow-step-tool-policy.js"; import { createStreamingDeltaNormalizer } from "./streaming-delta.js"; -import { isUnsupportedMessageRoleError } from "./transient-error-detector.js"; +import { isModelAuthTierIncompatibilityError, isUnsupportedMessageRoleError } from "./transient-error-detector.js"; +export { isModelAuthTierIncompatibilityError } from "./transient-error-detector.js"; const RTK_ACCEPTED_REWRITE_EXIT_CODES = new Set([0, 3]); const RTK_EXPECTED_PASSTHROUGH_EXIT_CODES = new Set([1, 2]); @@ -389,6 +390,15 @@ export async function promptSessionAndCheck(session: AgentSession, prompt: strin piLog.warn(`pi state error — Codex WebSocket transport drop (model=${modelDesc}): ${stateError}`); throw new Error(`${stateError} (model=${modelDesc})`); } + if (isModelAuthTierIncompatibilityError(stateError)) { + const modelDesc = describeModel(session); + const hint = + "Operator action required: this agent's configured model is not supported by the current authentication tier. " + + "Update the model selection in Settings → Models or configure a fallback model. " + + "If using a ChatGPT account with Codex, use a Codex-supported model (not a GPT model)."; + piLog.error(`pi state error — model not supported for auth tier (model=${modelDesc}): ${stateError}`); + throw new Error(`${stateError} (model=${modelDesc}). ${hint}`); + } if (isUnsupportedMessageRoleError(stateError)) { const modelDesc = describeModel(session); const hint = @@ -1028,6 +1038,15 @@ function resolveConfiguredModel( } export function isRetryableModelSelectionError(message: string): boolean { + // Codex ChatGPT-account auth-tier model incompatibility: the model is valid + // but not available for the current auth tier. This is a model-selection + // problem — a configured fallback model may work. Treat as retryable so the + // fallback path is tried once (the `usingFallback` guard prevents infinite + // swaps). + if (isModelAuthTierIncompatibilityError(message)) { + return true; + } + // An unsupported message-role rejection (e.g. a reasoning model sending the // "developer" system role to a provider that only accepts // system/user/assistant/tool) is fundamentally a model+provider diff --git a/packages/engine/src/transient-error-detector.ts b/packages/engine/src/transient-error-detector.ts index 37f58b7387..1c86a25c1c 100644 --- a/packages/engine/src/transient-error-detector.ts +++ b/packages/engine/src/transient-error-detector.ts @@ -193,6 +193,15 @@ export function extractMissingModulePath(errorMessage: string): string | null { const UNSUPPORTED_MESSAGE_ROLE_PATTERN = /\bmessages\.\[\d+\]\.role\b[\s\S]*\bis not one of\b|\bis not one of\b[\s\S]*\bmessages\.\[\d+\]\.role\b/i; const NON_CONTINUABLE_SESSION_PATTERN = /cannot continue from message role\s*[:=-]?\s*(?:['"`]?)(assistant|tool|function|system|user)(?:['"`]?)\b/i; +const MODEL_AUTH_TIER_INCOMPATIBILITY_PATTERNS: RegExp[] = [ + // Codex ChatGPT-account auth-tier incompatibility: the model is valid, but + // unavailable for the current auth tier. + /\bmodel\b[\s\S]{0,160}\bnot\s+supported\s+when\s+using\s+Codex\s+with\s+a\s+ChatGPT\s+account\b/i, + // General provider model-compatibility shapes. Keep these model-scoped so + // generic 400/invalid_request_error failures are not treated as model swaps. + /\bmodel\b[\s\S]{0,160}\b(?:is|was)\s+not\s+(?:supported|available)\b/i, + /(?:['"`][^'"`]+['"`]\s+)?\bmodel\b\s+(?:is|was)\s+not\s+(?:supported|available)\b/i, +]; export function isUnsupportedMessageRoleError(errorMessage: string): boolean { if (!errorMessage || typeof errorMessage !== "string") { @@ -201,6 +210,20 @@ export function isUnsupportedMessageRoleError(errorMessage: string): boolean { return UNSUPPORTED_MESSAGE_ROLE_PATTERN.test(errorMessage); } +export function isModelAuthTierIncompatibilityError(errorMessage: string): boolean { + if (!errorMessage || typeof errorMessage !== "string") { + return false; + } + + const hasModelContext = /\bmodel\b/i.test(errorMessage); + const hasCompatibilitySignal = /\bnot\s+(?:supported|available|found)\b/i.test(errorMessage); + if (/\binvalid_request_error\b/i.test(errorMessage) && hasModelContext && hasCompatibilitySignal) { + return true; + } + + return MODEL_AUTH_TIER_INCOMPATIBILITY_PATTERNS.some((pattern) => pattern.test(errorMessage)); +} + export function isNonContinuableSessionError(errorMessage: string): boolean { if (!errorMessage || typeof errorMessage !== "string") { return false; @@ -229,6 +252,7 @@ export function isOperatorActionableAgentError(errorMessage: string): boolean { } return ( isUnsupportedMessageRoleError(errorMessage) || + isModelAuthTierIncompatibilityError(errorMessage) || OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage)) ); }