diff --git a/.changeset/fn-7039-workflow-execution-model.md b/.changeset/fn-7039-workflow-execution-model.md new file mode 100644 index 0000000000..69cacabf14 --- /dev/null +++ b/.changeset/fn-7039-workflow-execution-model.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Workflow and automation steps now use the configured project Execution model instead of the default. +category: fix +dev: Workflow/AI-prompt step model resolution now consults the execution lane (resolveExecutorSessionModel / resolveExecutionSettingsModel) instead of resolveProjectDefaultModel, fixing executeWorkflowStep (executor.ts), cron-runner.ts, and dashboard routes.ts. FN-7039. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 5461376501..0127bbe983 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -877,6 +877,8 @@ Z.ai's built-in provider uses the existing `zai` auth entry / `ZAI_API_KEY` envi 6. Assigned durable agent runtime model (`runtimeConfig.model` or `runtimeConfig.modelProvider` + `runtimeConfig.modelId`) when both provider and model ID are set and no task/lane/default pair is configured 7. Automatic provider/model resolution +Workflow prompt steps and scheduled/manual AI-prompt automation steps use the same executor lane before falling back to project/global defaults; explicit step-level `modelProvider` + `modelId` values still take precedence for that individual step. + ### Heartbeat model (durable agents) Heartbeat sessions for durable agents use this order: diff --git a/packages/core/src/__tests__/model-resolution.test.ts b/packages/core/src/__tests__/model-resolution.test.ts index c48ef1e1e5..94ff91521e 100644 --- a/packages/core/src/__tests__/model-resolution.test.ts +++ b/packages/core/src/__tests__/model-resolution.test.ts @@ -35,6 +35,18 @@ describe("model-resolution", () => { ).toEqual({ provider: "google", modelId: "gemini-2.5-pro" }); }); + it("selects the project execution lane over the base default for workflow-step callers", () => { + const resolved = resolveExecutionSettingsModel({ + executionProvider: "openai", + executionModelId: "gpt-4o", + defaultProvider: "anthropic", + defaultModelId: "claude-3-5-sonnet", + }); + + expect(resolved).toEqual({ provider: "openai", modelId: "gpt-4o" }); + expect(resolved).not.toEqual({ provider: "anthropic", modelId: "claude-3-5-sonnet" }); + }); + it("falls back from planning global to the project default override", () => { expect( resolvePlanningSettingsModel({ diff --git a/packages/dashboard/src/routes.ts b/packages/dashboard/src/routes.ts index 8cff6f4654..a6cff4824b 100644 --- a/packages/dashboard/src/routes.ts +++ b/packages/dashboard/src/routes.ts @@ -31,7 +31,7 @@ import { readAgentMemoryFile, resolvePlanningSettingsModel, resolvePluginEntryPath, - resolveProjectDefaultModel, + resolveExecutionSettingsModel, resolveTitleSummarizerSettingsModel, writeAgentMemoryFile, } from "@fusion/core"; @@ -5121,7 +5121,9 @@ async function executeAiPromptStep( } const settings = await taskStore.getSettings(); - const defaultModel = resolveProjectDefaultModel(settings); + // Resolve model: step override → project execution lane → global execution lane → project default override → global default + // FNXC:ModelResolution 2026-06-25-12:00: FN-7039 requires manual AI-prompt workflow runs to use execution-lane settings before default settings because these runs have no task/runtime model context. + const defaultModel = resolveExecutionSettingsModel(settings); const modelProvider = step.modelProvider?.trim() || defaultModel.provider; const modelId = step.modelId?.trim() || defaultModel.modelId; let responseText = ""; diff --git a/packages/engine/src/__tests__/executor-test-helpers.ts b/packages/engine/src/__tests__/executor-test-helpers.ts index 8a17d1e403..9eb42edaa9 100644 --- a/packages/engine/src/__tests__/executor-test-helpers.ts +++ b/packages/engine/src/__tests__/executor-test-helpers.ts @@ -79,10 +79,8 @@ vi.mock("../agent-session-helpers.js", async () => { settings: Record | undefined, assignedAgentRuntimeConfig?: Record, ) => { - const model = typeof assignedAgentRuntimeConfig?.model === "string" ? assignedAgentRuntimeConfig.model : ""; - const slash = model.indexOf("/"); - if (slash > 0 && slash < model.length - 1) { - return { provider: model.slice(0, slash), modelId: model.slice(slash + 1) }; + if (settings?.testMode === true || (typeof settings?.defaultProvider === "string" && settings.defaultProvider.trim().toLowerCase() === "mock")) { + return { provider: "mock", modelId: "scripted" }; } if (taskModelProvider && taskModelId) return { provider: taskModelProvider, modelId: taskModelId }; if (typeof settings?.executionProvider === "string" && typeof settings?.executionModelId === "string") { @@ -97,6 +95,11 @@ vi.mock("../agent-session-helpers.js", async () => { if (typeof settings?.defaultProvider === "string" && typeof settings?.defaultModelId === "string") { return { provider: settings.defaultProvider as string, modelId: settings.defaultModelId as string }; } + const model = typeof assignedAgentRuntimeConfig?.model === "string" ? assignedAgentRuntimeConfig.model : ""; + const slash = model.indexOf("/"); + if (slash > 0 && slash < model.length - 1) { + return { provider: model.slice(0, slash), modelId: model.slice(slash + 1) }; + } return { provider: undefined, modelId: undefined }; }, }; diff --git a/packages/engine/src/__tests__/executor-workflow-step-model.test.ts b/packages/engine/src/__tests__/executor-workflow-step-model.test.ts new file mode 100644 index 0000000000..93b66376ef --- /dev/null +++ b/packages/engine/src/__tests__/executor-workflow-step-model.test.ts @@ -0,0 +1,234 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import "./executor-test-helpers.js"; +import { TaskExecutor } from "../executor.js"; +import { + createMockStore, + mockedCreateFnAgent, + mockedExecSync, + resetExecutorMocks, +} from "./executor-test-helpers.js"; + +type CapturedSession = { + defaultProvider?: string; + defaultModelId?: string; +}; + +function captureSession(output = '{"verdict":"APPROVE","notes":""}'): { last?: CapturedSession } { + const holder: { last?: CapturedSession } = {}; + mockedCreateFnAgent.mockImplementation(async (opts: any) => { + holder.last = { + defaultProvider: opts.defaultProvider, + defaultModelId: opts.defaultModelId, + }; + + const listeners: Array<(event: any) => void> = []; + const session: any = { + state: {}, + subscribe: (fn: (event: any) => void) => { + listeners.push(fn); + return () => {}; + }, + prompt: vi.fn(async () => { + for (const fn of listeners) { + fn({ + type: "message_update", + assistantMessageEvent: { + type: "text_delta", + partial: output, + contentIndex: 0, + delta: output, + }, + }); + } + }), + dispose: vi.fn(), + }; + return { session }; + }); + return holder; +} + +function quietGit() { + mockedExecSync.mockImplementation(() => Buffer.from("")); +} + +function makeExecutor(store: ReturnType) { + const agentStore = { getAgent: vi.fn().mockResolvedValue(null), createAgent: vi.fn() }; + return new TaskExecutor(store as any, "/tmp/test", { agentStore } as any); +} + +function baseTask(overrides: Record = {}) { + return { + id: "FN-MODEL-1", + title: "Model resolution", + description: "verify model resolution", + column: "in-progress" as const, + worktree: "/tmp/wt", + branch: "fusion/fn-model-1", + baseCommitSha: "abc123", + dependencies: [], + steps: [{ name: "s", status: "in-progress" as const }], + currentStep: 0, + log: [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + ...overrides, + }; +} + +function workflowStep(overrides: Record = {}) { + const now = new Date().toISOString(); + return { + id: "step:model", + name: "Model Step", + description: "", + mode: "prompt" as const, + phase: "pre-merge" as const, + gateMode: "advisory" as const, + prompt: "Check the model.", + toolMode: "readonly" as const, + enabled: true, + createdAt: now, + updatedAt: now, + ...overrides, + }; +} + +async function runStepWithSettings( + settings: Record, + options: { + task?: Record; + step?: Record; + } = {}, +) { + const store = createMockStore(); + store.getSettings.mockResolvedValue(settings); + const executor = makeExecutor(store); + const captured = captureSession(); + + await (executor as any).executeWorkflowStep( + baseTask(options.task), + workflowStep(options.step), + "/tmp/wt", + settings, + undefined, + ); + + return captured.last; +} + +describe("executor workflow-step model resolution", () => { + beforeEach(() => { + resetExecutorMocks(); + quietGit(); + }); + + it("uses the project execution lane instead of the global default when the step has no override", async () => { + const captured = await runStepWithSettings({ + executionProvider: "openai", + executionModelId: "gpt-4o", + defaultProvider: "anthropic", + defaultModelId: "claude-3-5-sonnet", + }); + + expect(captured).toMatchObject({ + defaultProvider: "openai", + defaultModelId: "gpt-4o", + }); + expect(captured).not.toMatchObject({ + defaultProvider: "anthropic", + defaultModelId: "claude-3-5-sonnet", + }); + }); + + it("keeps step and task overrides ahead of execution-lane settings", async () => { + await expect( + runStepWithSettings( + { + executionProvider: "project-exec-provider", + executionModelId: "project-exec-model", + }, + { + step: { + modelProvider: "step-provider", + modelId: "step-model", + }, + }, + ), + ).resolves.toMatchObject({ + defaultProvider: "step-provider", + defaultModelId: "step-model", + }); + + await expect( + runStepWithSettings( + { + executionProvider: "project-exec-provider", + executionModelId: "project-exec-model", + }, + { + task: { + modelProvider: "task-provider", + modelId: "task-model", + }, + }, + ), + ).resolves.toMatchObject({ + defaultProvider: "task-provider", + defaultModelId: "task-model", + }); + }); + + it("falls through the execution hierarchy without mixing partial pairs", async () => { + await expect( + runStepWithSettings({ + executionProvider: "partial-project-provider", + executionGlobalProvider: "global-exec-provider", + executionGlobalModelId: "global-exec-model", + defaultProvider: "global-default-provider", + defaultModelId: "global-default-model", + }), + ).resolves.toMatchObject({ + defaultProvider: "global-exec-provider", + defaultModelId: "global-exec-model", + }); + + await expect( + runStepWithSettings({ + executionGlobalModelId: "partial-global-model", + defaultProviderOverride: "project-default-provider", + defaultModelIdOverride: "project-default-model", + defaultProvider: "global-default-provider", + defaultModelId: "global-default-model", + }), + ).resolves.toMatchObject({ + defaultProvider: "project-default-provider", + defaultModelId: "project-default-model", + }); + + await expect( + runStepWithSettings({ + defaultProvider: "global-default-provider", + defaultModelId: "global-default-model", + }), + ).resolves.toMatchObject({ + defaultProvider: "global-default-provider", + defaultModelId: "global-default-model", + }); + }); + + it("forces mock/scripted for workflow steps when test mode is active", async () => { + await expect( + runStepWithSettings({ + testMode: true, + executionProvider: "project-exec-provider", + executionModelId: "project-exec-model", + defaultProvider: "anthropic", + defaultModelId: "claude-3-5-sonnet", + }), + ).resolves.toMatchObject({ + defaultProvider: "mock", + defaultModelId: "scripted", + }); + }); +}); diff --git a/packages/engine/src/cron-runner.ts b/packages/engine/src/cron-runner.ts index 1a6b8c4186..38e223c685 100644 --- a/packages/engine/src/cron-runner.ts +++ b/packages/engine/src/cron-runner.ts @@ -1,7 +1,7 @@ import { exec } from "node:child_process"; import { - resolveProjectDefaultModel, + resolveExecutionSettingsModel, runScheduledEvalBatch, resolveTaskEvaluationSettings, isEvalsExperimentalEnabled, @@ -855,9 +855,10 @@ export class CronRunner { }; } - // Resolve model: step override → project default override → global default + // Resolve model: step override → project execution lane → global execution lane → project default override → global default + // FNXC:ModelResolution 2026-06-25-12:00: FN-7039 requires scheduled AI-prompt automation steps to use execution-lane settings before default settings because these steps have no task/runtime model context. const settings = await this.store.getSettings(); - const defaultModel = resolveProjectDefaultModel(settings); + const defaultModel = resolveExecutionSettingsModel(settings); const modelProvider = step.modelProvider?.trim() || defaultModel.provider; const modelId = step.modelId?.trim() || defaultModel.modelId; diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index df189829e6..2e7a5d853f 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -53,7 +53,6 @@ import { resolveAgentPrompt, resolvePersistAgentThinkingLog, resolveEffectiveAgentPermissionPolicy, - resolveProjectDefaultModel, resolveAgentMemoryInclusionMode, loadWorkspaceConfig, type WorkspaceConfig, @@ -13423,13 +13422,21 @@ You have access to the file system to review changes.${verdictBlock}`; }); // Determine primary model and an explicit fallback. The workflow step's - // own override takes precedence; otherwise we use the project default - // override before falling through to the global default. The - // fallback is the per-step override's missing-counterpart settings, then - // the global validator/fallback pair, then the executor's `fallbackProvider`. - const defaultModel = resolveProjectDefaultModel(settings); - const primaryProvider = workflowStep.modelProvider || defaultModel.provider; - const primaryModelId = workflowStep.modelId || defaultModel.modelId; + // own override takes precedence; otherwise use the canonical executor + // hierarchy: task override → project execution lane → global execution lane + // → project default override → global default. The fallback is the per-step + // override's missing-counterpart settings, then the global validator/fallback + // pair, then the executor's `fallbackProvider`. + // FNXC:ModelResolution 2026-06-25-12:00: FN-7039 requires workflow steps to inherit project execution-lane model settings before default settings so configured Execution models reach step sessions unless the step itself overrides them. + const assignedRuntimeConfig = await this.getAssignedAgentRuntimeConfig(task.assignedAgentId); + const executorModel = resolveExecutorSessionModel( + task.modelProvider, + task.modelId, + settings, + assignedRuntimeConfig, + ); + const primaryProvider = workflowStep.modelProvider || executorModel.provider; + const primaryModelId = workflowStep.modelId || executorModel.modelId; const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId); type ModelTuple = { provider?: string; modelId?: string };