diff --git a/.changeset/fn-8681-credential-instance-retarget.md b/.changeset/fn-8681-credential-instance-retarget.md new file mode 100644 index 0000000000..3c4b950097 --- /dev/null +++ b/.changeset/fn-8681-credential-instance-retarget.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Let executor steps use a rotated credential account without interrupting active work. +category: feature +dev: StepSessionExecutor accepts credentialInstanceId and retargetCredentialInstance(ref). diff --git a/docs/settings-reference.md b/docs/settings-reference.md index c2cd7d6431..91be86f14d 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -1880,9 +1880,10 @@ Credential instance ids are validated when authored. Invalid values (including e whitespace-only, bracket-containing, oversized, or non-string values) are rejected for project and global settings, workflow setting values, workflow IR node overrides, and task writes. Settings validation also visits every `modelPresets[]` element: one invalid executor or validator instance -id rejects the entire settings write without changing the stored presets. These values are -persisted-but-inert in this release; runtime credential resolution will consume them in the -follow-up runtime-resolution work. +id rejects the entire settings write without changing the stored presets. Runtime session resolution consumes the selected instance for supported lanes, including +executor step sessions. When an executor-step credential is retargeted after a usage-limit +failure, only sessions created after the active attempt completes use the new instance; work +already in progress continues on its existing session. ### Authentication credential instances diff --git a/packages/engine/src/__tests__/step-session-executor.test.ts b/packages/engine/src/__tests__/step-session-executor.test.ts index 9e6297e733..b6ff8fd4b6 100644 --- a/packages/engine/src/__tests__/step-session-executor.test.ts +++ b/packages/engine/src/__tests__/step-session-executor.test.ts @@ -1031,6 +1031,7 @@ vi.mock("../context-limit-detector.js", () => ({ // Mock usage-limit-detector vi.mock("../usage-limit-detector.js", () => ({ checkSessionError: vi.fn(), + isUsageLimitError: (message: string) => /usage limit|rate limit|\b429\b/i.test(message), })); // Mock worktree-names @@ -1086,8 +1087,10 @@ import { generateWorktreeName } from "../worktree-names.js"; import { execSync } from "node:child_process"; import { AgentSemaphore } from "../concurrency.js"; import { createLogger } from "../logger.js"; +import { promptWithAutoRetry, resolveExecutorSessionModel } from "../agent-session-helpers.js"; const mockedCreateFnAgent = vi.mocked(createFnAgent); +const mockedResolveExecutorSessionModel = vi.mocked(resolveExecutorSessionModel); const mockedExecSync = vi.mocked(execSync); const mockedInstallTaskWorktreeIdentityGuard = vi.mocked(installTaskWorktreeIdentityGuard); const mockedGenerateWorktreeName = vi.mocked(generateWorktreeName); @@ -3281,3 +3284,211 @@ describe("StepSessionExecutor executor model lane hierarchy", () => { }); }); }); + +describe("StepSessionExecutor credential-instance retargeting", () => { + function makeCredentialExecutor(options: { + steps?: number; + runStepsInNewSessions?: boolean; + credentialInstanceId?: string; + maxParallelSteps?: number; + resolveCredentialInstanceRetarget?: () => Promise<{ providerId: string; instanceId: string } | undefined>; + } = {}) { + const stepCount = options.steps ?? 1; + return new StepSessionExecutor({ + taskDetail: makeTaskDetail({ + prompt: makeStepPrompt("FN-CREDENTIAL", stepCount), + steps: Array.from({ length: stepCount }, (_, index) => ({ name: `Step ${index}`, status: "pending" as const })), + }), + worktreePath: "/project/.worktrees/main", + rootDir: "/project", + settings: makeSettings({ maxParallelSteps: options.maxParallelSteps ?? 1, runStepsInNewSessions: options.runStepsInNewSessions }), + credentialInstanceId: options.credentialInstanceId, + resolveCredentialInstanceRetarget: options.resolveCredentialInstanceRetarget, + }); + } + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers({ shouldAdvanceTime: true }); + vi.mocked(promptWithAutoRetry).mockImplementation(async (session: any, prompt: string, options?: unknown) => + session.prompt(prompt, options), + ); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("omits an unset credential instance from legacy session creation", async () => { + mockedCreateFnAgent.mockResolvedValue({ session: makeMockSession() } as any); + const executor = makeCredentialExecutor(); + + await executor.executeAll(); + + expect(mockedCreateFnAgent.mock.calls[0]?.[0]).not.toHaveProperty("credentialInstanceId"); + }); + + it("forwards the initial instance to each newly-created sequential session", async () => { + mockedCreateFnAgent.mockResolvedValue({ session: makeMockSession() } as any); + const executor = makeCredentialExecutor({ steps: 2, runStepsInNewSessions: true, credentialInstanceId: "account-a" }); + + await executor.executeAll(); + + expect(mockedCreateFnAgent).toHaveBeenCalledTimes(2); + expect(mockedCreateFnAgent.mock.calls.map(([options]) => options.credentialInstanceId)).toEqual(["account-a", "account-a"]); + expect(mockedResolveExecutorSessionModel.mock.calls.map((args) => args[4])).toEqual(["account-a", "account-a"]); + }); + + it("forwards the initial instance to every fresh session in a parallel wave", async () => { + mockedCreateFnAgent.mockResolvedValue({ session: makeMockSession() } as any); + const executor = makeCredentialExecutor({ steps: 2, maxParallelSteps: 2, credentialInstanceId: "account-a" }); + + await executor.executeAll(); + + expect(mockedCreateFnAgent).toHaveBeenCalledTimes(2); + expect(mockedCreateFnAgent.mock.calls.map(([options]) => options.credentialInstanceId)).toEqual(["account-a", "account-a"]); + }); + + it("retargets only subsequent fresh sessions and clears back to omitted resolution", async () => { + const first = makeMockSession(); + const second = makeMockSession(); + const third = makeMockSession(); + mockedCreateFnAgent + .mockResolvedValueOnce({ session: first } as any) + .mockResolvedValueOnce({ session: second } as any) + .mockResolvedValueOnce({ session: third } as any); + const executor = makeCredentialExecutor({ steps: 3, runStepsInNewSessions: true, credentialInstanceId: "account-a" }); + + await (executor as any).executeStep(0, "/project/.worktrees/main"); + await executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "account-b" }); + await (executor as any).executeStep(1, "/project/.worktrees/main"); + await executor.retargetCredentialInstance(undefined); + await (executor as any).executeStep(2, "/project/.worktrees/main"); + + expect(mockedCreateFnAgent.mock.calls.map(([options]) => options.credentialInstanceId)).toEqual([ + "account-a", + "account-b", + undefined, + ]); + expect(mockedCreateFnAgent.mock.calls[2]?.[0]).not.toHaveProperty("credentialInstanceId"); + }); + + it("defers reusable-primary disposal until an active prompt completes, then uses the retargeted instance", async () => { + let finishFirstPrompt: (() => void) | undefined; + let firstSession: ReturnType | undefined; + const firstPromptStarted = new Promise((resolve) => { + firstSession = { + ...makeMockSession(), + abortBash: vi.fn(), + prompt: vi.fn(() => new Promise((finish) => { + finishFirstPrompt = finish; + resolve(); + })), + }; + const second = { ...makeMockSession(), abortBash: vi.fn() }; + mockedCreateFnAgent + .mockResolvedValueOnce({ session: firstSession } as any) + .mockResolvedValueOnce({ session: second } as any); + }); + const executor = makeCredentialExecutor({ steps: 2, runStepsInNewSessions: false, credentialInstanceId: "account-a" }); + const execution = executor.executeAll(); + + await firstPromptStarted; + await executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "account-b" }); + + expect(firstSession?.abortBash).not.toHaveBeenCalled(); + expect(firstSession?.dispose).not.toHaveBeenCalled(); + finishFirstPrompt?.(); + + await expect(execution).resolves.toEqual([ + expect.objectContaining({ stepIndex: 0, success: true, retries: 0 }), + expect.objectContaining({ stepIndex: 1, success: true, retries: 0 }), + ]); + expect(firstSession?.dispose).toHaveBeenCalledTimes(1); + expect(mockedCreateFnAgent.mock.calls[1]?.[0]).toMatchObject({ credentialInstanceId: "account-b" }); + }); + + it("clears a deferred retarget during cleanup", async () => { + let finishPrompt: (() => void) | undefined; + const promptStarted = new Promise((resolve) => { + const session = { + ...makeMockSession(), + abortBash: vi.fn(), + prompt: vi.fn(() => new Promise((finish) => { + finishPrompt = finish; + resolve(); + })), + }; + mockedCreateFnAgent.mockResolvedValue({ session } as any); + }); + const executor = makeCredentialExecutor({ runStepsInNewSessions: false, credentialInstanceId: "account-a" }); + const execution = executor.executeAll(); + + await promptStarted; + await executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "account-b" }); + await executor.cleanup(); + + expect((executor as any).reusablePrimaryRetargetPending).toBe(false); + finishPrompt?.(); + await execution; + }); + + it("immediately disposes an idle reusable primary session and ignores equivalent or invalid retargets", async () => { + const session = { ...makeMockSession(), abortBash: vi.fn() }; + mockedCreateFnAgent.mockResolvedValue({ session } as any); + const executor = makeCredentialExecutor({ runStepsInNewSessions: false, credentialInstanceId: "account-a" }); + + await (executor as any).executeStep(0, "/project/.worktrees/main"); + await executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "account-a" }); + expect(session.dispose).not.toHaveBeenCalled(); + await executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "account-b" }); + expect(session.abortBash).toHaveBeenCalledTimes(1); + expect(session.dispose).toHaveBeenCalledTimes(1); + await expect(executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "bad id" })).resolves.toBeUndefined(); + expect(getStepSessionLogger().warn).toHaveBeenCalledWith(expect.stringContaining("Ignoring invalid credential instance id")); + }); + + it("retargets a usage-limit retry through the owning executor's live selection", async () => { + const first = { + ...makeMockSession(), + prompt: vi.fn() + .mockRejectedValueOnce(new Error("429 usage limit reached")) + .mockResolvedValueOnce(undefined), + }; + const second = makeMockSession(); + const resolveCredentialInstanceRetarget = vi.fn().mockResolvedValue({ providerId: "anthropic", instanceId: "account-b" }); + mockedCreateFnAgent + .mockResolvedValueOnce({ session: first } as any) + .mockResolvedValueOnce({ session: second } as any); + const executor = makeCredentialExecutor({ credentialInstanceId: "account-a", resolveCredentialInstanceRetarget }); + + const execution = executor.executeAll(); + await vi.advanceTimersByTimeAsync(10_000); + + await expect(execution).resolves.toEqual([expect.objectContaining({ success: true, retries: 1 })]); + expect(resolveCredentialInstanceRetarget).toHaveBeenCalledTimes(1); + expect(mockedCreateFnAgent.mock.calls[1]?.[0]).toMatchObject({ credentialInstanceId: "account-b" }); + }); + + it("applies a manual retarget before a retry without changing retry accounting", async () => { + let rejectFirstPrompt: ((error: Error) => void) | undefined; + const first = { + ...makeMockSession(), + prompt: vi.fn(() => new Promise((_resolve, reject) => { rejectFirstPrompt = reject; })), + }; + const second = makeMockSession(); + mockedCreateFnAgent + .mockResolvedValueOnce({ session: first } as any) + .mockResolvedValueOnce({ session: second } as any); + const executor = makeCredentialExecutor({ credentialInstanceId: "account-a" }); + const execution = executor.executeAll(); + await vi.waitFor(() => expect(rejectFirstPrompt).toBeTypeOf("function")); + + await executor.retargetCredentialInstance({ providerId: "anthropic", instanceId: "account-b" }); + rejectFirstPrompt?.(new Error("retry me")); + await vi.advanceTimersByTimeAsync(10_000); + + await expect(execution).resolves.toEqual([expect.objectContaining({ success: true, retries: 1 })]); + expect(mockedCreateFnAgent.mock.calls[1]?.[0]).toMatchObject({ credentialInstanceId: "account-b" }); + }); +}); diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index 09711815ee..19a485239e 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -13341,6 +13341,27 @@ export class TaskExecutor { pluginRunner: this.options.pluginRunner, runtimeHint: stepSessionRuntimeHint, assignedAgentRuntimeConfig: (stepIdentityAgent?.runtimeConfig ?? undefined) as Record | undefined, + /* + * FNXC:CredentialInstanceRotation 2026-08-01-10:41: + * Step sessions must start on the task-selected account. On a usage-limit + * retry, re-read the live selection and resolve its provider with the same + * effective column-agent runtime config used to create the session. + */ + credentialInstanceId: detail.credentialInstanceId, + resolveCredentialInstanceRetarget: async () => { + const liveDetail = await this.store.getTask(task.id); + if (!liveDetail) return undefined; + const resolvedModel = resolveExecutorSessionModel( + liveDetail.modelProvider, + liveDetail.modelId, + settings, + (stepIdentityAgent?.runtimeConfig ?? undefined) as Record | undefined, + liveDetail.credentialInstanceId ?? undefined, + ); + return resolvedModel.provider && resolvedModel.credentialInstanceId + ? { providerId: resolvedModel.provider, instanceId: resolvedModel.credentialInstanceId } + : undefined; + }, // Attribute the per-step run auditor to the column agent when it governs // (U4); absent → StepSessionExecutor falls back to assignedAgentId. effectiveAgentId: stepColumnAgent?.agent.id, diff --git a/packages/engine/src/step-session-executor.ts b/packages/engine/src/step-session-executor.ts index 540524ce36..fe5b179b88 100644 --- a/packages/engine/src/step-session-executor.ts +++ b/packages/engine/src/step-session-executor.ts @@ -18,8 +18,8 @@ const execAsync = promisify(exec); import { existsSync } from "node:fs"; import { rm } from "node:fs/promises"; import type { AgentSession } from "@earendil-works/pi-coding-agent"; -import type { AgentHeartbeatRun, AgentStore, MessageStore, PermanentAgentGatingContext, ResolvedMcpServerDefinition, TaskDetail, Settings, SteeringComment, TaskStore } from "@fusion/core"; -import { resolvePersistAgentThinkingLog, resolveExecutorFallbackModel } from "@fusion/core"; +import type { AgentHeartbeatRun, AgentStore, MessageStore, PermanentAgentGatingContext, ProviderInstanceRef, ResolvedMcpServerDefinition, TaskDetail, Settings, SteeringComment, TaskStore } from "@fusion/core"; +import { isValidProviderInstanceId, resolvePersistAgentThinkingLog, resolveExecutorFallbackModel } from "@fusion/core"; import { createResolvedAgentSession, @@ -41,7 +41,7 @@ import { createLogger } from "./logger.js"; import { createFallbackModelObserver } from "./fallback-model-observer.js"; import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js"; import { isContextLimitError } from "./context-limit-detector.js"; -import { checkSessionError } from "./usage-limit-detector.js"; +import { checkSessionError, isUsageLimitError } from "./usage-limit-detector.js"; import { createDelegateTaskTool, createTaskAssignTool, @@ -124,6 +124,20 @@ export interface StepSessionExecutorOptions { runtimeHint?: string; /** Optional assigned-agent runtime config for model override precedence. */ assignedAgentRuntimeConfig?: Record; + /* + * FNXC:CredentialInstanceRotation 2026-08-01-09:58: + * FN-8654 rotates executor-step work after a usage-limit error. This optional + * initial target lets every new session use that selected credential instance; + * when absent, session creation omits the key to preserve legacy resolution. + */ + credentialInstanceId?: string; + /* + * FNXC:CredentialInstanceRotation 2026-08-01-10:41: + * A usage-limit retry must re-read the task's selected account through the + * owning executor, which alone has the live task and effective agent identity. + * Returning undefined deliberately restores provider-default resolution. + */ + resolveCredentialInstanceRetarget?: () => Promise; /** * FNXC:StepLifecycle 2026-07-22-09:53: This awaitable pre-start contract lets the * authoritative lifecycle projection reject execution before session allocation; @@ -741,6 +755,15 @@ export class StepSessionExecutor { private reusablePrimaryHandle: SessionHandle | null = null; private reusableStepTelemetry: { agentLogger: AgentLogger; trackingKey: string } | null = null; private reusablePrimaryLastTokenUsage: StepResult["tokenUsage"] | undefined; + /* + * FNXC:CredentialInstanceRotation 2026-08-01-09:58: + * Credential rotation is mutable run state, never an options mutation. Only + * the reusable primary session can outlive an attempt; fresh and parallel + * sessions naturally read this target when they are created. + */ + private credentialInstanceId: string | undefined; + private reusablePrimaryRetargetPending = false; + private reusablePrimaryAttemptActive = false; private registerActiveStepSession(stepIndex: number, handle: SessionHandle, worktreePath: string): void { this.activeSessions.set(stepIndex, handle); @@ -777,10 +800,17 @@ export class StepSessionExecutor { * FNXC:WorkflowStepSessions 2026-06-29-22:58: * Coding (per-step review) still needs the StepSessionExecutor boundary so the graph can run `step-review` between steps, but the operator's "Each step in a new session" switch must control session freshness. Reuse only the primary sequential worktree when `runStepsInNewSessions` is false; parallel/isolated worktrees always need their own sessions because their cwd differs and they may run concurrently. */ - private shouldReusePrimarySession(worktreePath: string): boolean { + private async shouldReusePrimarySession(worktreePath: string): Promise { + await this.drainReusablePrimaryRetarget(); return this.options.settings.runStepsInNewSessions === false && worktreePath === this.options.worktreePath; } + private async drainReusablePrimaryRetarget(): Promise { + if (!this.reusablePrimaryRetargetPending || this.reusablePrimaryAttemptActive) return; + this.reusablePrimaryRetargetPending = false; + await this.disposeReusablePrimarySession(); + } + private selectReusableTelemetry(fallback: { agentLogger: AgentLogger; trackingKey: string }): { agentLogger: AgentLogger; trackingKey: string } { return this.reusableStepTelemetry ?? fallback; } @@ -807,10 +837,34 @@ export class StepSessionExecutor { constructor(options: StepSessionExecutorOptions) { this.options = options; this.store = options.store ?? (NOOP_TASK_STORE as TaskStore); + this.credentialInstanceId = options.credentialInstanceId; // Clamp maxParallelSteps to 1–4 range this.maxParallel = Math.max(1, Math.min(4, options.settings.maxParallelSteps ?? 2)); } + /* + * FNXC:CredentialInstanceRotation 2026-08-01-09:58: + * FN-8654 may retarget future executor-step sessions after a usage-limit error. + * disposeReusablePrimarySession aborts bash and disposes its session, so an + * active primary attempt must drain normally before disposal; eager disposal + * would change its result and retry lifecycle. Parallel sessions are untouched. + */ + async retargetCredentialInstance(ref: ProviderInstanceRef | undefined): Promise { + if (ref && !isValidProviderInstanceId(ref.instanceId)) { + stepExecLog.warn(`Ignoring invalid credential instance id for task ${this.options.taskDetail.id}`); + return; + } + const nextCredentialInstanceId = ref?.instanceId; + if (nextCredentialInstanceId === this.credentialInstanceId) return; + + this.credentialInstanceId = nextCredentialInstanceId; + if (this.reusablePrimaryAttemptActive) { + this.reusablePrimaryRetargetPending = true; + return; + } + await this.disposeReusablePrimarySession(); + } + /** * Execute all steps in the task, respecting conflict-based parallel waves. * @@ -998,6 +1052,8 @@ export class StepSessionExecutor { activeSessionRegistry.unregisterPath(worktreePath); } this.activeSessions.clear(); + this.reusablePrimaryRetargetPending = false; + this.reusablePrimaryAttemptActive = false; await this.disposeReusablePrimarySession(); } @@ -1012,6 +1068,8 @@ export class StepSessionExecutor { if (this.activeSessions.size > 0) { await this.terminateAllSessions(); } + this.reusablePrimaryRetargetPending = false; + this.reusablePrimaryAttemptActive = false; await this.disposeReusablePrimarySession(); // Remove parallel worktrees @@ -1257,7 +1315,7 @@ export class StepSessionExecutor { // Build reduced step prompt for context-limit recovery (simpler, shorter) const reducedStepPrompt = buildReducedStepPrompt(promptTaskDetail, stepIndex, this.options.rootDir); - const reusePrimarySession = this.shouldReusePrimarySession(worktreePath); + const reusePrimarySession = await this.shouldReusePrimarySession(worktreePath); // Acquire semaphore if provided if (semaphore) { @@ -1379,6 +1437,7 @@ export class StepSessionExecutor { taskDetail.modelId, settings, this.options.assignedAgentRuntimeConfig, + this.credentialInstanceId, ); if (reusePrimarySession && this.reusablePrimarySession) { @@ -1456,6 +1515,13 @@ Follow instructions precisely and avoid unrelated changes.`, telemetry.agentLogger.onToolEnd(name, isError, result); stuckTaskDetector?.recordActivity(telemetry.trackingKey); }, + /* + * FNXC:CredentialInstanceRotation 2026-08-01-09:58: + * The explicit executor target wins over any model-derived instance + * because FN-8654 deliberately retargeted after a usage-limit error. + * Omit the key entirely when unset to preserve legacy resolution. + */ + ...(this.credentialInstanceId ? { credentialInstanceId: this.credentialInstanceId } : {}), // FNXC:PluginSkills 2026-07-12-00:00: Step-session createFnAgent must receive plugin skill body dirs from TaskExecutor; names alone do not make plugin-package SKILL.md files discoverable. ...(this.options.skillSelection ? { skillSelection: this.options.skillSelection } : {}), ...(this.options.additionalSkillPaths && this.options.additionalSkillPaths.length > 0 ? { additionalSkillPaths: this.options.additionalSkillPaths } : {}), @@ -1499,6 +1565,10 @@ Follow instructions precisely and avoid unrelated changes.`, this.reusableStepTelemetry = localTelemetry; } this.registerActiveStepSession(stepIndex, handle, worktreePath); + // FNXC:CredentialInstanceRotation 2026-08-01-09:58: This marker covers + // the whole registered reused-primary attempt so retargeting cannot call + // destructive disposal between registration and prompt completion. + if (reusePrimarySession) this.reusablePrimaryAttemptActive = true; stuckTaskDetector?.trackTask(trackingKey, { dispose: () => session?.dispose() }, taskDetail.id); const sessionModel = await describeAgentModel(session); @@ -1581,6 +1651,18 @@ Follow instructions precisely and avoid unrelated changes.`, } } + /* + * FNXC:CredentialInstanceRotation 2026-08-01-10:41: + * A limited account may be replaced while this workflow is running. + * Ask TaskExecutor for the live, runtime-config-aware target before + * retrying, so the next session does not recreate the limited account. + */ + if (isUsageLimitError(errorMessage) && this.options.resolveCredentialInstanceRetarget) { + await this.retargetCredentialInstance( + await this.options.resolveCredentialInstanceRetarget(), + ); + } + // If this was the last attempt, return failure if (attempt === MAX_STEP_RETRIES) { const result: StepResult = { @@ -1609,6 +1691,8 @@ Follow instructions precisely and avoid unrelated changes.`, this.unregisterActiveStepSession(stepIndex, worktreePath); stuckTaskDetector?.untrackTask(trackingKey); if (reusePrimarySession) { + this.reusablePrimaryAttemptActive = false; + await this.drainReusablePrimaryRetarget(); this.reusableStepTelemetry = null; } else { try {