diff --git a/.changeset/fn-8098-model-fallback.md b/.changeset/fn-8098-model-fallback.md new file mode 100644 index 0000000000..1de17cb92b --- /dev/null +++ b/.changeset/fn-8098-model-fallback.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: Add an executor fallback model and retry the primary model before blocking on fallback exhaustion. +category: feature +dev: Adds executionFallbackProvider, executionFallbackModelId, and executionFallbackThinkingLevel workflow settings. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 77e9276809..90911dc57a 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -561,6 +561,9 @@ Default notes: | `defaultThinkingLevelOverride` | `ThinkingLevel` | `undefined` | Optional project default-lane thinking override used when a task does not set `thinkingLevel`; inherits `defaultThinkingLevel` when unset. | | `executionProvider` | `string` | `undefined` | Provider for task execution agents. | | `executionModelId` | `string` | `undefined` | Model ID for task execution agents. | +| `executionFallbackProvider` | `string` | `undefined` | Workflow fallback provider for executor sessions; paired with `executionFallbackModelId` and resolves before the shared fallback pair. | +| `executionFallbackModelId` | `string` | `undefined` | Workflow fallback model ID for executor sessions. | +| `executionFallbackThinkingLevel` | `ThinkingLevel` | `undefined` | Executor fallback thinking override; inherits shared fallback thinking, then executor primary thinking. | | `validatorProvider` | `string` | `undefined` | Provider for plan/code reviewers. | | `validatorModelId` | `string` | `undefined` | Model ID for plan/code reviewers. | | `validatorFallbackProvider` | `string` | `undefined` | Fallback provider for reviewers; also used by reviewer UNAVAILABLE/error recovery retry before returning terminal UNAVAILABLE. | @@ -1014,7 +1017,9 @@ Fusion resolves task models through workflow-backed lane values first, then glob Direct-chat defaults are project-scoped and independent of task workflow lanes. Configure them in **Settings -> Project Models -> Chat**. `chatDefaultKind: "agent"` resolves only when `chatDefaultAgentId` is set; `chatDefaultKind: "model"` resolves only when both `chatDefaultModelProvider` and `chatDefaultModelId` are set, with optional `chatDefaultThinkingLevel`. If `chatNewSessionMode` is `"always-default"` and that target resolves, every New Chat entry point creates the session directly. If the target is incomplete, or the mode is unset/`"prompt"`, Fusion opens the New Chat dialog instead and preselects the resolved default when one exists. Chat Rooms additionally support a per-room `thinkingLevel` default that applies to every room responder; clearing it inherits the resolved project/global default. -Settings model lanes can also carry optional thinking/reasoning effort overrides in the same model dropdown. Primary workflow lanes declare `executionThinkingLevel`, `planningThinkingLevel`, or `validatorThinkingLevel` per `(workflow, project)`; planning/reviewer fallback lanes declare `planningFallbackThinkingLevel` and `validatorFallbackThinkingLevel`; global fallback uses `fallbackThinkingLevel`; and project title summarization fallback uses `titleSummarizerFallbackThinkingLevel`. Empty thinking values inherit through the lane/global/default chain and explicit values are cleared by the lane reset action. Runtime thinking precedence for task/workflow execution is node/step `config.thinkingLevel` > lane-specific task override (`planningThinkingLevel` or `validatorThinkingLevel`) > shared task `thinkingLevel` > workflow lane thinking override > global lane thinking override > project default thinking override > global `defaultThinkingLevel`; executor sessions continue to use shared task `thinkingLevel` directly. Model-mode Chat sessions use the same executor-lane resolver with session `thinkingLevel` in the task slot, so an empty chat-session value inherits project/global defaults while a concrete New Chat selection wins for that session. The resolved value still flows through pi.ts' existing thinking/reasoning-conflict fallback (Fusion retries without the explicit level when a provider rejects conflicting thinking parameters). +Settings model lanes can also carry optional thinking/reasoning effort overrides in the same model dropdown. Primary workflow lanes declare `executionThinkingLevel`, `planningThinkingLevel`, or `validatorThinkingLevel` per `(workflow, project)`; executor/planning/reviewer fallback lanes declare `executionFallbackThinkingLevel`, `planningFallbackThinkingLevel`, and `validatorFallbackThinkingLevel`; global fallback uses `fallbackThinkingLevel`; and project title summarization fallback uses `titleSummarizerFallbackThinkingLevel`. Empty thinking values inherit through the lane/global/default chain and explicit values are cleared by the lane reset action. Runtime thinking precedence for task/workflow execution is node/step `config.thinkingLevel` > lane-specific task override (`planningThinkingLevel` or `validatorThinkingLevel`) > shared task `thinkingLevel` > workflow lane thinking override > global lane thinking override > project default thinking override > global `defaultThinkingLevel`; executor sessions continue to use shared task `thinkingLevel` directly. Model-mode Chat sessions use the same executor-lane resolver with session `thinkingLevel` in the task slot, so an empty chat-session value inherits project/global defaults while a concrete New Chat selection wins for that session. The resolved value still flows through pi.ts' existing thinking/reasoning-conflict fallback (Fusion retries without the explicit level when a provider rejects conflicting thinking parameters). + +Executor sessions, including workflow-step timeout/malformed-output recovery and durable heartbeats, resolve `executionFallbackProvider`/`executionFallbackModelId` first and otherwise inherit the global `fallbackProvider`/`fallbackModelId` pair. For a distinct complete fallback pair, model-selection recovery is bounded to **primary → fallback → primary**. If all three attempts fail, Fusion raises an operator-actionable terminal failure with the standard retry affordance; missing, incomplete, or equal fallback pairs remain terminal after the initial primary failure. When the planning lane has neither `planningFallback*` nor a global `fallback*` pair configured, triage now derives an **implicit fallback** from the resolved project/global default (execution) model (FN-7719). This lets a retryable primary planner-model failure (e.g. a provider 404/429) recover via one distinct swap instead of permanently failing triage with "no fallback configured" — the operator's chosen primary planner lane is unchanged, and the implicit fallback is skipped when it would equal the primary model or when test mode is active. diff --git a/packages/core/src/__tests__/model-resolution.test.ts b/packages/core/src/__tests__/model-resolution.test.ts index 3abd0d3eb2..141ccdc196 100644 --- a/packages/core/src/__tests__/model-resolution.test.ts +++ b/packages/core/src/__tests__/model-resolution.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { applyTestModeOverrides, resolveExecutionSettingsModel, + resolveExecutorFallbackModel, resolvePlanningSettingsModel, resolveProjectDefaultModel, resolveTaskExecutionModel, @@ -15,6 +16,18 @@ import { } from "../model-resolution.js"; describe("model-resolution", () => { + it("resolves executor fallback before the shared fallback and forces mock in test mode", () => { + expect(resolveExecutorFallbackModel({ + executionFallbackProvider: "executor-provider", + executionFallbackModelId: "executor-model", + fallbackProvider: "global-provider", + fallbackModelId: "global-model", + })).toEqual({ provider: "executor-provider", modelId: "executor-model" }); + expect(resolveExecutorFallbackModel({ fallbackProvider: "global-provider", fallbackModelId: "global-model" })).toEqual({ provider: "global-provider", modelId: "global-model" }); + expect(resolveExecutorFallbackModel({})).toEqual({ provider: undefined, modelId: undefined }); + expect(resolveExecutorFallbackModel({ testMode: true, fallbackProvider: "global-provider", fallbackModelId: "global-model" })).toEqual(TEST_MODE_RESOLVED); + }); + it("prefers the project default override over the global default", () => { expect( resolveProjectDefaultModel({ diff --git a/packages/core/src/__tests__/settings-parity.test.ts b/packages/core/src/__tests__/settings-parity.test.ts index ee33a9e2d8..2cdb21b587 100644 --- a/packages/core/src/__tests__/settings-parity.test.ts +++ b/packages/core/src/__tests__/settings-parity.test.ts @@ -317,6 +317,9 @@ describe("settings key parity", () => { "reflectionEnabled", "executionProvider", "executionModelId", + "executionFallbackProvider", + "executionFallbackModelId", + "executionFallbackThinkingLevel", "planningProvider", "planningModelId", "planningFallbackProvider", @@ -574,6 +577,8 @@ describe("model lane key parity regression (FN-1729)", () => { { provider: "planningProvider", modelId: "planningModelId", expectedScope: "workflow" }, { provider: "planningGlobalProvider", modelId: "planningGlobalModelId", expectedScope: "global" }, { provider: "planningFallbackProvider", modelId: "planningFallbackModelId", expectedScope: "workflow" }, + // Executor fallback lane + { provider: "executionFallbackProvider", modelId: "executionFallbackModelId", expectedScope: "workflow" }, // Validator lane { provider: "validatorProvider", modelId: "validatorModelId", expectedScope: "workflow" }, { provider: "validatorGlobalProvider", modelId: "validatorGlobalModelId", expectedScope: "global" }, diff --git a/packages/core/src/builtin-workflow-settings.ts b/packages/core/src/builtin-workflow-settings.ts index 446251cca4..d4c031cc28 100644 --- a/packages/core/src/builtin-workflow-settings.ts +++ b/packages/core/src/builtin-workflow-settings.ts @@ -229,6 +229,30 @@ export const BUILTIN_MOVED_WORKFLOW_SETTINGS: WorkflowSettingDefinition[] = [ options: THINKING_LEVELS.map((level) => ({ value: level, label: level })), description: "Thinking effort for the execution phase. Empty inherits from the task or default thinking level.", }, + /* + * FNXC:Settings-ExecutorModel 2026-07-16-00:00: + * FN-8098 makes executor recovery workflow-configurable; unset values deliberately + * inherit the shared fallback pair so existing configurations continue to work. + */ + { + id: "executionFallbackProvider", + name: "Executor fallback provider", + type: "string", + description: "Fallback provider for the execution phase.", + }, + { + id: "executionFallbackModelId", + name: "Executor fallback model", + type: "string", + description: "Fallback model id for the execution phase.", + }, + { + id: "executionFallbackThinkingLevel", + name: "Executor fallback thinking level", + type: "enum", + options: THINKING_LEVELS.map((level) => ({ value: level, label: level })), + description: "Thinking effort for the executor fallback model. Empty inherits from shared fallback or executor thinking.", + }, { id: "planningProvider", name: "Planning provider", diff --git a/packages/core/src/index.gate.ts b/packages/core/src/index.gate.ts index 7d2e9777e1..65ad33aa4b 100644 --- a/packages/core/src/index.gate.ts +++ b/packages/core/src/index.gate.ts @@ -1371,6 +1371,7 @@ export { applyTestModeOverrides, isTestModeActive, resolveExecutionSettingsModel, + resolveExecutorFallbackModel, resolveMergerFallbackModel, resolveMergerSettingsModel, resolvePlanningSettingsModel, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d58c2d7e15..8787902481 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1423,6 +1423,7 @@ export { applyTestModeOverrides, isTestModeActive, resolveExecutionSettingsModel, + resolveExecutorFallbackModel, resolveMergerFallbackModel, resolveMergerSettingsModel, resolvePhaseThinkingLevel, diff --git a/packages/core/src/model-resolution.ts b/packages/core/src/model-resolution.ts index 8c8b8d4b7c..38e043ff08 100644 --- a/packages/core/src/model-resolution.ts +++ b/packages/core/src/model-resolution.ts @@ -262,6 +262,27 @@ export function resolveMergerFallbackModel(settings?: Partial): Resolv ); } +/** + * FNXC:Settings-ExecutorModel 2026-07-16-00:00: + * FN-8098 gives executor work a workflow-specific fallback pair while preserving the + * shared fallback as the ultimate default for workflows that leave this lane unset. + */ +export function resolveExecutorFallbackModel(settings?: Partial): ResolvedModelSelection { + return applyTestModeOverrides( + pickFirstModelPair( + { + provider: settings?.executionFallbackProvider, + modelId: settings?.executionFallbackModelId, + }, + { + provider: settings?.fallbackProvider, + modelId: settings?.fallbackModelId, + }, + ), + settings, + ); +} + export function resolveTaskExecutionModel( task: TaskModelLike, settings?: Partial, diff --git a/packages/core/src/settings-schema.ts b/packages/core/src/settings-schema.ts index aa8c2eabd4..dd1f6add67 100644 --- a/packages/core/src/settings-schema.ts +++ b/packages/core/src/settings-schema.ts @@ -43,6 +43,9 @@ type MovedProjectSettingsKey = | "executionProvider" | "executionModelId" | "executionThinkingLevel" + | "executionFallbackProvider" + | "executionFallbackModelId" + | "executionFallbackThinkingLevel" | "planningProvider" | "planningModelId" | "planningThinkingLevel" diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index b7287f59a1..1ffaa0a9f0 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -3377,6 +3377,17 @@ export interface ProjectSettings { executionModelId?: string; /** Workflow-declared execution-lane thinking override. Inherits through task/default thinking when unset. */ executionThinkingLevel?: ThinkingLevel; + /* + * FNXC:Settings-ExecutorModel 2026-07-16-00:00: + * FN-8098 lets execution sessions select their own recovery model before the shared + * fallback pair, so reviewer, merger, planning, and executor lanes can recover independently. + */ + /** Workflow fallback provider for executor sessions. Must pair with `executionFallbackModelId`; resolves before the shared global fallback pair. */ + executionFallbackProvider?: string; + /** Workflow fallback model ID for executor sessions. Must pair with `executionFallbackProvider`; resolves before the shared global fallback pair. */ + executionFallbackModelId?: string; + /** Workflow executor-fallback thinking override. Inherits shared fallback thinking, then executor primary thinking. */ + executionFallbackThinkingLevel?: ThinkingLevel; /** Workflow-declared planning-lane thinking override. Inherits through task/default thinking when unset. */ planningThinkingLevel?: ThinkingLevel; /** AI model provider for validator/reviewer agent. diff --git a/packages/dashboard/app/components/WorkflowSettingsPanel.tsx b/packages/dashboard/app/components/WorkflowSettingsPanel.tsx index 3f65b71946..00af9cff91 100644 --- a/packages/dashboard/app/components/WorkflowSettingsPanel.tsx +++ b/packages/dashboard/app/components/WorkflowSettingsPanel.tsx @@ -541,6 +541,14 @@ export const WORKFLOW_MODEL_LANE_CATALOG: WorkflowModelLanePair[] = [ label: "Reviewer Model", help: "Provider and model used by review and validation agents. Leave unset to inherit from the default lane.", }, + { + id: "execution-fallback", + providerId: "executionFallbackProvider", + modelId: "executionFallbackModelId", + thinkingId: "executionFallbackThinkingLevel", + label: "Executor Fallback Model", + help: "Fallback provider and model used when the primary Executor model cannot be used.", + }, { id: "planning-fallback", providerId: "planningFallbackProvider", diff --git a/packages/dashboard/app/components/settings/sections/ProjectModelsSection.tsx b/packages/dashboard/app/components/settings/sections/ProjectModelsSection.tsx index 0dab0d4521..d0044cf2b1 100644 --- a/packages/dashboard/app/components/settings/sections/ProjectModelsSection.tsx +++ b/packages/dashboard/app/components/settings/sections/ProjectModelsSection.tsx @@ -15,7 +15,7 @@ import { LoadingSpinner } from "../../LoadingSpinner"; import { useAgentsMapCache } from "../../../hooks/useAgentsMapCache"; type LaneStatus = "inherited" | "overridden"; type WorkflowModelPair = { - id: "planning" | "execution" | "validator" | "planning-fallback" | "validator-fallback"; + id: "planning" | "execution" | "validator" | "execution-fallback" | "planning-fallback" | "validator-fallback"; providerId: string; modelId: string; thinkingId?: string; @@ -55,6 +55,14 @@ const WORKFLOW_MODEL_PAIRS: WorkflowModelPair[] = [ label: "Reviewer Model", help: "Provider and model used for workflow review or validation lanes. Leave unset to inherit from the workflow default.", }, + { + id: "execution-fallback", + providerId: "executionFallbackProvider", + modelId: "executionFallbackModelId", + thinkingId: "executionFallbackThinkingLevel", + label: "Executor Fallback Model", + help: "Fallback provider and model used when the primary Executor model cannot be used.", + }, { id: "planning-fallback", providerId: "planningFallbackProvider", diff --git a/packages/engine/src/__tests__/pi.test.ts b/packages/engine/src/__tests__/pi.test.ts index 07aa72a70d..beceabbb22 100644 --- a/packages/engine/src/__tests__/pi.test.ts +++ b/packages/engine/src/__tests__/pi.test.ts @@ -1176,9 +1176,15 @@ describe("piLog structured diagnostics", () => { } as unknown as AgentSession; createAgentSessionMock.mockReset(); + const primaryRetrySession = { + model: { provider: "openai", id: "gpt-4o" }, + prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")), + subscribe: vi.fn(), dispose: vi.fn(), setThinkingLevel: vi.fn(), sessionFile: undefined, + } as unknown as AgentSession; createAgentSessionMock .mockResolvedValueOnce({ session: primarySession } as any) - .mockResolvedValueOnce({ session: fallbackSession } as any); + .mockResolvedValueOnce({ session: fallbackSession } as any) + .mockResolvedValueOnce({ session: primaryRetrySession } as any); const { session } = await createFnAgent({ cwd: "/test/project", @@ -1193,15 +1199,17 @@ describe("piLog structured diagnostics", () => { await expect((session as any).promptWithFallback("prompt text")).rejects.toMatchObject({ name: "ModelFallbackExhaustedError", - attempts: 2, + attempts: 3, primaryModel: "openai/gpt-4o", fallbackModel: "anthropic/claude-3-5-haiku-20241022", triggerPoint: "prompt-time", }); - expect(createAgentSessionMock).toHaveBeenCalledTimes(2); + expect(createAgentSessionMock).toHaveBeenCalledTimes(3); expect(primarySession.prompt).toHaveBeenCalledTimes(1); expect(fallbackSession.prompt).toHaveBeenCalledTimes(1); + expect(primaryRetrySession.prompt).toHaveBeenCalledTimes(1); + expect((createAgentSessionMock.mock.calls[2]?.[0] as any).model.id).toBe("gpt-4o"); expect(onFallbackModelUsed).toHaveBeenCalledTimes(1); expect(onFallbackModelUsed).toHaveBeenCalledWith(expect.objectContaining({ triggerPoint: "prompt-time", diff --git a/packages/engine/src/agent-session-helpers.ts b/packages/engine/src/agent-session-helpers.ts index efc07f7fee..c3db406c15 100644 --- a/packages/engine/src/agent-session-helpers.ts +++ b/packages/engine/src/agent-session-helpers.ts @@ -16,6 +16,7 @@ import { isGrokApiKeyFusionVisible, isTestModeActive, resolveExecutionSettingsModel, + resolveExecutorFallbackModel, resolveMergerSettingsModel, resolvePhaseThinkingLevel, resolveProjectDefaultModel, @@ -285,6 +286,7 @@ export function resolveExecutorFallbackThinkingLevel( settings: Partial | undefined, ): string | undefined { return firstThinkingLevel( + settings?.executionFallbackThinkingLevel, settings?.fallbackThinkingLevel, resolveExecutorThinkingLevel(taskThinkingLevel, settings), ); @@ -567,6 +569,7 @@ export function resolveHeartbeatSessionModels( } const executionSettingsModel = resolveExecutionSettingsModel(settings); + const executorFallbackModel = resolveExecutorFallbackModel(settings); const assignedRuntimeModel = extractRuntimeModel(assignedAgentRuntimeConfig); /* FNXC:AgentHeartbeat 2026-07-14-16:13: @@ -579,8 +582,8 @@ export function resolveHeartbeatSessionModels( return { defaultProvider: resolvedModel.provider, defaultModelId: resolvedModel.modelId, - fallbackProvider: undefined, - fallbackModelId: undefined, + fallbackProvider: executorFallbackModel.provider, + fallbackModelId: executorFallbackModel.modelId, }; } diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index 98f51f429c..e5ffb462f7 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -13,7 +13,7 @@ import { existsSync, lstatSync, realpathSync } from "node:fs"; import { readFile, rm, writeFile } from "node:fs/promises"; import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, AsyncMissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings, MergeResult, WorkflowIrNode, WorkflowIrNodeKind, WorkflowStepResult as CoreWorkflowStepResult, ThinkingLevel } from "@fusion/core"; import { getUnmetSchedulingDependencies } from "./scheduler.js"; -import { RetryStormError, serializeRetryStormError, isExperimentalFeatureEnabled, resolveWorkflowIrForTask, resolveColumnAgentBinding, resolveEffectiveAgent, instanceNodeId, getWorkflowExtensionRegistry, getBuiltinWorkflow, parseNoOpCompletionMarker, allowsAutoMergeProcessing, resolveEffectiveAutoMerge, isLiveSharedBranchGroupMemberIntegration, resolveMaxAutoMergeRetries, resolveOptionalStepRevisionBudget, resolveOptionalReviewRevisionBudget, COMPLETION_SUMMARY_NODE_ID, upsertWorkflowStepResult, AWAITING_APPROVAL_PAUSE_REASON, THINKING_LEVELS, AgentStore } from "@fusion/core"; +import { RetryStormError, serializeRetryStormError, isExperimentalFeatureEnabled, resolveWorkflowIrForTask, resolveColumnAgentBinding, resolveEffectiveAgent, instanceNodeId, getWorkflowExtensionRegistry, getBuiltinWorkflow, parseNoOpCompletionMarker, allowsAutoMergeProcessing, resolveEffectiveAutoMerge, isLiveSharedBranchGroupMemberIntegration, resolveMaxAutoMergeRetries, resolveOptionalStepRevisionBudget, resolveOptionalReviewRevisionBudget, COMPLETION_SUMMARY_NODE_ID, upsertWorkflowStepResult, AWAITING_APPROVAL_PAUSE_REASON, THINKING_LEVELS, AgentStore, resolveExecutorFallbackModel } from "@fusion/core"; import { finalizeProvenAutoMergeTask } from "./auto-merge-finalization.js"; import { mergeEffectiveSettings } from "./effective-settings.js"; import { moveTaskToReplanColumn, resolveReplanTargetColumn } from "./replan-target.js"; @@ -11242,8 +11242,7 @@ export class TaskExecutor { settings, (identityAgent?.runtimeConfig ?? undefined) as Record | undefined, ); - const executorFallbackProvider = settings.fallbackProvider; - const executorFallbackModelId = settings.fallbackModelId; + const { provider: executorFallbackProvider, modelId: executorFallbackModelId } = resolveExecutorFallbackModel(settings); const executorSessionThinkingSource = this.graphSeamThinkingLevel.get(task.id) ?? detail.thinkingLevel; const executorThinkingLevel = resolveExecutorThinkingLevel(executorSessionThinkingSource, settings); const executorFallbackThinkingLevel = resolveExecutorFallbackThinkingLevel(executorSessionThinkingSource, settings); @@ -14947,6 +14946,8 @@ export class TaskExecutor { assignedRuntimeConfig, ); + const executorFallback = resolveExecutorFallbackModel(settings); + // Create the fix agent session const { session } = await createResolvedAgentSession({ sessionPurpose: "executor", @@ -14975,6 +14976,9 @@ Do not refactor, rename broadly, or make opportunistic improvements. onToolEnd: logger.onToolEnd, defaultProvider: executorProvider, defaultModelId: executorModelId, + fallbackProvider: executorFallback.provider, + fallbackModelId: executorFallback.modelId, + fallbackThinkingLevel: resolveExecutorFallbackThinkingLevel(task.thinkingLevel, settings), defaultThinkingLevel: resolveExecutorThinkingLevel(task.thinkingLevel, settings), runAuditor: createRunAuditor(this.store, this.getRunContextFor(task.id)), settings, @@ -15895,8 +15899,8 @@ You have access to the file system to review changes.${inlineFixBlock}${verdictB // own override takes precedence; otherwise use the canonical executor // hierarchy: task override → project execution lane → global execution lane // → project default override → global default. The fallback is the per-step - // override's missing-counterpart settings, then the global validator/fallback - // pair, then the executor's `fallbackProvider`. + // override's missing-counterpart settings, then the executor fallback lane, + // which itself falls through to the shared global fallback pair. // FNXC:ModelResolution 2026-06-25-12:00: FN-7039 requires workflow steps to inherit project execution-lane model settings before default settings so configured Execution models reach step sessions unless the step itself overrides them. const assignedRuntimeConfig = await this.getAssignedAgentRuntimeConfig(task.assignedAgentId); const executorModel = resolveExecutorSessionModel( @@ -15909,15 +15913,11 @@ You have access to the file system to review changes.${inlineFixBlock}${verdictB const primaryModelId = workflowStep.modelId || executorModel.modelId; const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId); - type ModelTuple = { provider?: string; modelId?: string }; - type WorkflowStepFallbackLabel = "validatorFallback" | "globalFallback"; - const fallbackCandidates: Array = [ - { provider: settings.validatorFallbackProvider, modelId: settings.validatorFallbackModelId, label: "validatorFallback" }, - { provider: settings.fallbackProvider, modelId: settings.fallbackModelId, label: "globalFallback" }, - ]; - const fallback = fallbackCandidates.find( - (c) => c.provider && c.modelId && (c.provider !== primaryProvider || c.modelId !== primaryModelId), - ); + const executorFallback = resolveExecutorFallbackModel(settings); + const fallback = executorFallback.provider && executorFallback.modelId + && (executorFallback.provider !== primaryProvider || executorFallback.modelId !== primaryModelId) + ? executorFallback + : undefined; const timeoutMs = Math.max(60_000, settings.workflowStepTimeoutMs ?? 900_000); @@ -16064,9 +16064,7 @@ You have access to the file system to review changes.${inlineFixBlock}${verdictB */ const workflowStepThinkingSource = workflowStep.thinkingLevel ?? task.thinkingLevel; const workflowStepThinkingLevel = attemptLabel === "fallback" - ? (fallback?.label === "validatorFallback" - ? resolveValidatorFallbackThinkingLevel(workflowStepThinkingSource, settings) - : resolveExecutorFallbackThinkingLevel(workflowStepThinkingSource, settings)) + ? resolveExecutorFallbackThinkingLevel(workflowStepThinkingSource, settings) : resolveExecutorThinkingLevel(workflowStepThinkingSource, settings); const workflowStepFallbackThinkingLevel = resolveExecutorFallbackThinkingLevel(workflowStepThinkingSource, settings); const { session } = await createResolvedAgentSession({ @@ -16078,8 +16076,8 @@ You have access to the file system to review changes.${inlineFixBlock}${verdictB tools: toolMode, defaultProvider: provider, defaultModelId: modelId, - fallbackProvider: settings.fallbackProvider, - fallbackModelId: settings.fallbackModelId, + fallbackProvider: executorFallback.provider, + fallbackModelId: executorFallback.modelId, fallbackThinkingLevel: workflowStepFallbackThinkingLevel, defaultThinkingLevel: workflowStepThinkingLevel, runAuditor: createRunAuditor(this.store, this.getRunContextFor(task.id)), @@ -16272,7 +16270,7 @@ You have access to the file system to review changes.${inlineFixBlock}${verdictB if (!retryMalformed) return retryOutcome; await this.store.logEntry( task.id, - `Workflow step '${workflowStep.name}' produced malformed output on both the primary attempt and one self-retry — no fallback model configured (set settings.validatorFallbackProvider/Id or fallbackProvider/Id)`, + `Workflow step '${workflowStep.name}' produced malformed output on both the primary attempt and one self-retry — no fallback model configured (set settings.executionFallbackProvider/Id or fallbackProvider/Id)`, ); return retryOutcome; } @@ -16280,12 +16278,12 @@ You have access to the file system to review changes.${inlineFixBlock}${verdictB executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' ${reason} and no fallback model is configured`); await this.store.logEntry( task.id, - `Workflow step '${workflowStep.name}' ${reason} — no fallback model configured (set settings.validatorFallbackProvider/Id or fallbackProvider/Id)`, + `Workflow step '${workflowStep.name}' ${reason} — no fallback model configured (set settings.executionFallbackProvider/Id or fallbackProvider/Id)`, ); return primaryOutcome; } - executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with fallback ${fallback.provider}/${fallback.modelId} (label=${fallback.label}) after primary ${primaryOutcome.timedOut ? "timeout" : "malformed output"}`); + executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with executor fallback ${fallback.provider}/${fallback.modelId} after primary ${primaryOutcome.timedOut ? "timeout" : "malformed output"}`); return runOnce(fallback.provider, fallback.modelId, "fallback"); } @@ -18945,6 +18943,8 @@ Child agent: ${agent.id} (${name})`; const { provider: childExecutorProvider, modelId: childExecutorModelId } = resolveExecutorSessionModel(undefined, undefined, settings, agent.runtimeConfig as Record | undefined); + const childExecutorFallback = resolveExecutorFallbackModel(settings); + // Create child agent session const { session: childSession } = await createResolvedAgentSession({ sessionPurpose: "executor", @@ -18955,8 +18955,8 @@ Child agent: ${agent.id} (${name})`; tools: "coding", defaultProvider: childExecutorProvider, defaultModelId: childExecutorModelId, - fallbackProvider: settings.fallbackProvider, - fallbackModelId: settings.fallbackModelId, + fallbackProvider: childExecutorFallback.provider, + fallbackModelId: childExecutorFallback.modelId, fallbackThinkingLevel: resolveExecutorFallbackThinkingLevel(undefined, settings), runAuditor: createRunAuditor(this.store, this.getRunContextFor(taskId)), settings, diff --git a/packages/engine/src/pi.ts b/packages/engine/src/pi.ts index 6cde3c70ff..2ce03bef62 100644 --- a/packages/engine/src/pi.ts +++ b/packages/engine/src/pi.ts @@ -2555,8 +2555,10 @@ export async function createFnAgent(options: AgentOptions): Promise }; /* - * FNXC:ModelFallback 2026-07-02-00:00: - * Planner and shared AI lanes may try one distinct fallback model for a logical model-selection failure, but they must then throw ModelFallbackExhaustedError instead of swapping back or relying on scheduler re-pick loops. This keeps transient fallback useful while making exhausted model configuration actionable for operators. + * FNXC:ModelFallback 2026-07-16-00:00: + * FN-8098 requires a distinct fallback failure to get one final primary retry before + * blocking. This bounded primary → fallback → primary sequence prevents fallback loops + * while still surfacing an operator-actionable ModelFallbackExhaustedError after three attempts. */ let sessionResult; let usingFallback = false; @@ -2572,8 +2574,14 @@ export async function createFnAgent(options: AgentOptions): Promise usingFallback = true; try { sessionResult = await createSessionWithModel(fallbackModel); - } catch (fallbackErr: unknown) { - throw makeFallbackExhaustedError("session-creation", 2, fallbackErr); + } catch (_fallbackErr: unknown) { + // The final retry is intentionally primary and terminal even when fallback failed non-retryably. + usingFallback = false; + try { + sessionResult = await createSessionWithModel(selectedModel); + } catch (primaryRetryErr: unknown) { + throw makeFallbackExhaustedError("session-creation", 3, primaryRetryErr); + } } await emitFallbackUsed("session-creation", err); piLog.log("Fallback session created successfully"); @@ -2736,50 +2744,21 @@ export async function createFnAgent(options: AgentOptions): Promise try { await promptSessionAndCheck(fallbackSession, prompt, effectivePromptOptions); return; - } catch (fallbackErr: any) { - const fallbackErrorMessage = fallbackErr?.message || ""; - if (isContextLimitError(fallbackErrorMessage)) { - const promptMemoryRetry = await retryWithCompactedPromptMemory(fallbackSession, prompt, effectivePromptOptions); - if (promptMemoryRetry.recovered) { - return; - } - if (promptMemoryRetry.error) { - const retryMessage = promptMemoryRetry.error instanceof Error ? promptMemoryRetry.error.message : String(promptMemoryRetry.error); - if (!isContextLimitError(retryMessage)) { - throw promptMemoryRetry.error; - } - } - - const promptSectionRetry = await retryWithCompactedPromptSections(fallbackSession, prompt, effectivePromptOptions); - if (promptSectionRetry.recovered) { - return; - } - if (promptSectionRetry.error) { - const retryMessage = promptSectionRetry.error instanceof Error ? promptSectionRetry.error.message : String(promptSectionRetry.error); - if (!isContextLimitError(retryMessage)) { - throw promptSectionRetry.error; - } - } - - piLog.warn("promptWithFallback: fallback session context limit error — attempting auto-compaction"); - await flushMemoryBeforeSessionCompaction(fallbackSession); - const compactResult = await compactSessionContext(fallbackSession); - if (compactResult) { - piLog.log(`promptWithFallback: fallback compaction succeeded (${compactResult.tokensBefore} tokens) — retrying`); - try { - await promptSessionAndCheck(fallbackSession, prompt, effectivePromptOptions); - return; - } catch (retryErr: any) { - const retryErrorMessage = retryErr?.message || ""; - piLog.error(`promptWithFallback: fallback retry after auto-compaction failed: ${retryErrorMessage}`); - throw fallbackErr; // Throw original fallback error - } - } else { - piLog.error("promptWithFallback: fallback compaction unavailable — propagating original error"); - throw fallbackErr; - } + } catch (_fallbackErr: unknown) { + /* + * FNXC:ModelFallback 2026-07-16-00:00: + * Once a distinct fallback has failed, retry the primary exactly once regardless + * of whether the fallback error is retryable or a context/compaction failure. + * Resetting `usingFallback` ensures the final primary receives primary thinking. + */ + usingFallback = false; + try { + const primaryRetrySession = await swapPromptSession(selectedModel); + await promptSessionAndCheck(primaryRetrySession, prompt, effectivePromptOptions); + return; + } catch (primaryRetryErr: unknown) { + throw makeFallbackExhaustedError("prompt-time", 3, primaryRetryErr); } - throw makeFallbackExhaustedError("prompt-time", 2, fallbackErr); } } }; diff --git a/packages/engine/src/step-session-executor.ts b/packages/engine/src/step-session-executor.ts index 36105283da..58bde02618 100644 --- a/packages/engine/src/step-session-executor.ts +++ b/packages/engine/src/step-session-executor.ts @@ -19,7 +19,7 @@ import { existsSync } from "node:fs"; import { rm } from "node:fs/promises"; import type { AgentSession } from "@earendil-works/pi-coding-agent"; import type { AgentHeartbeatRun, AgentStore, MessageStore, PermanentAgentGatingContext, ResolvedMcpServerDefinition, TaskDetail, Settings, SteeringComment, TaskStore } from "@fusion/core"; -import { resolvePersistAgentThinkingLog } from "@fusion/core"; +import { resolvePersistAgentThinkingLog, resolveExecutorFallbackModel } from "@fusion/core"; import { createResolvedAgentSession, @@ -27,6 +27,7 @@ import { promptWithAutoRetry, resolveExecutorSessionModel, resolveExecutorThinkingLevel, + resolveExecutorFallbackThinkingLevel, } from "./agent-session-helpers.js"; import type { AgentActionGateContext } from "./agent-action-gate.js"; import type { SkillSelectionContext } from "./skill-resolver.js"; @@ -1359,8 +1360,9 @@ Your role: Follow instructions precisely and avoid unrelated changes.`, defaultProvider: executorProvider, defaultModelId: executorModelId, - fallbackProvider: settings.fallbackProvider, - fallbackModelId: settings.fallbackModelId, + fallbackProvider: resolveExecutorFallbackModel(settings).provider, + fallbackModelId: resolveExecutorFallbackModel(settings).modelId, + fallbackThinkingLevel: resolveExecutorFallbackThinkingLevel(taskDetail.thinkingLevel, settings), defaultThinkingLevel: effectiveThinkingLevel, runAuditor: createRunAuditor(this.store, { runId: generateSyntheticRunId("workflow-step", taskDetail.id),