From c4fad2d7939cfac7c762ddbc177f55b82ab694be Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 12 Jul 2026 13:18:04 -0700 Subject: [PATCH] fix(engine): auto-recover agents from transient OAuth token-rotation 401s A routine Claude Max OAuth token rotation (~8h) fails the in-flight call with 401 authentication_error "Invalid authentication credentials" even though refreshed credentials already exist on disk. Three compounding defects turned that into a fleet-wide operator-action park: - The heartbeat prompt path never ran under withRateLimitRetry (executor/ triage/merger all do), so the 401 immediately failed the run. Now wrapped. - The 401 matched the operator-actionable /credential/ pattern and defaulted to "permanent", so FN-7859 parked agents paused/error-unrecoverable. A new shared isTransientAuthCredentialError classifier (also used by rate-limit-retry) classifies rotation 401s transient + not operator- actionable; OAuth scope-grant and API-key failures still park. - Heartbeat failure classification ran on the stack-bearing error detail; stack frames like "at withRateLimitRetry (.../rate-limit-retry.ts)" match the usage-limit /rate[_\s]?limit/ pattern. Classification and agent.lastError now use the message; stderrExcerpt keeps the full detail. Self-healing additionally un-parks agents previously paused with error-unrecoverable whose lastError now classifies recoverable, bounded by the shared heartbeat error-recovery budget. Co-Authored-By: Claude Fable 5 --- .changeset/auth-rotation-401-auto-recovery.md | 7 + .../heartbeat-error-recovery.test.ts | 80 +++++++++++- .../engine/src/__tests__/self-healing.test.ts | 120 ++++++++++++++++-- .../transient-error-detector.test.ts | 54 ++++++++ packages/engine/src/agent-heartbeat.ts | 26 +++- packages/engine/src/rate-limit-retry.ts | 24 +--- packages/engine/src/self-healing.ts | 25 +++- .../engine/src/transient-error-detector.ts | 35 +++++ 8 files changed, 331 insertions(+), 40 deletions(-) create mode 100644 .changeset/auth-rotation-401-auto-recovery.md diff --git a/.changeset/auth-rotation-401-auto-recovery.md b/.changeset/auth-rotation-401-auto-recovery.md new file mode 100644 index 0000000000..87642aba3e --- /dev/null +++ b/.changeset/auth-rotation-401-auto-recovery.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Agents now auto-recover from transient OAuth token-rotation 401 errors instead of parking for operator action. +category: fix +dev: Adds `isTransientAuthCredentialError` to the shared transient-error classifier (401 `authentication_error` / "Invalid authentication credentials" / token-expired shapes are transient and not operator-actionable; OAuth scope-grant and API-key failures still park). Heartbeat prompts now run under `withRateLimitRetry` so mid-run token rotations retry in-run. Heartbeat failure classification uses the error message instead of the stack-bearing detail. Self-healing un-parks agents previously paused with `error-unrecoverable` whose lastError now classifies recoverable. diff --git a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts index d1328c59be..f3808c8376 100644 --- a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts +++ b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts @@ -177,6 +177,9 @@ describe("heartbeat error-recovery primitives", () => { expect(isErrorRecoveryEligible(baseAgent({ metadata: buildHeartbeatErrorRecoveryMetadata(baseAgent(), 5), lastError: "socket hang up" }), 5)).toBe(false); expect(isErrorRecoveryEligible(baseAgent({ lastError: "invalid api key" }), 5)).toBe(false); expect(isErrorRecoveryEligible(baseAgent({ lastError: "SyntaxError: Unexpected token" }), 5)).toBe(false); + // OAuth token-rotation 401s are transient credential rotations, not operator problems. + expect(isErrorRecoveryEligible(baseAgent({ lastError: 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}' }), 5)).toBe(true); + expect(isErrorRecoveryEligible(baseAgent({ lastError: '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}' }), 5)).toBe(false); expect(isHeartbeatErrorRecoverable({ lastError: "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/tmp/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /tmp/deleted/packages/engine/src/pi.ts" })).toBe(false); }); }); @@ -237,7 +240,9 @@ describe("HeartbeatMonitor error-state recovery", () => { }); it("parks a first-run operator-actionable failure immediately with an explicit reason", async () => { - mockedCreateFnAgent.mockResolvedValueOnce(createSession(async () => { throw new Error("Invalid authentication credentials"); }) as never); + mockedCreateFnAgent.mockResolvedValueOnce(createSession(async () => { + throw new Error('401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}'); + }) as never); const store = createAgentStore(baseAgent({ state: "active", lastError: undefined })); const taskStore = createNoTaskStore(); const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); @@ -245,7 +250,7 @@ describe("HeartbeatMonitor error-state recovery", () => { await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); expect(store.agent.state).toBe("paused"); - expect(store.agent.lastError).toContain("Invalid authentication credentials"); + expect(store.agent.lastError).toContain("OAuth token does not meet scope requirements"); expect(store.agent.pauseReason).toBe(HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON); expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ @@ -255,6 +260,77 @@ describe("HeartbeatMonitor error-state recovery", () => { })); }); + /* + FNXC:AgentHeartbeat 2026-07-12-20:10: + Regression for the OAuth token-rotation incident: a mid-run 401 + "authentication_error: Invalid authentication credentials" must be retried + IN-RUN (withRateLimitRetry transient-auth budget) so a routine ~8h Claude Max + token rotation never fails the heartbeat run, and — if it still escapes — + must classify as a recoverable error (bare `error` + bounded auto-retry), + never an operator-actionable "error-unrecoverable" park. + */ + it("retries a transient OAuth token-rotation 401 in-run and completes without entering error state", async () => { + vi.useFakeTimers(); + try { + let calls = 0; + const session = createSession(async () => { + calls += 1; + if (calls === 1) { + throw new Error('Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}'); + } + }); + mockedCreateFnAgent.mockResolvedValueOnce(session as never); + const store = createAgentStore(baseAgent({ state: "active", lastError: undefined })); + const taskStore = createNoTaskStore(); + const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); + + let settled = false; + const heartbeat = monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }).finally(() => { settled = true; }); + // Flat transient-auth retry delay is ~5s ±10% jitter; advance fake time until the run settles. + for (let i = 0; i < 30 && !settled; i++) { + await vi.advanceTimersByTimeAsync(1_000); + } + await heartbeat; + + expect(session.prompt).toHaveBeenCalledTimes(2); + expect(store.agent.state).toBe("active"); + expect(store.agent.lastError).toBeUndefined(); + expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-parked-unrecoverable", + })); + } finally { + vi.useRealTimers(); + } + }); + + it("keeps a persistent rotation-shaped 401 recoverable (bare error, no unrecoverable park)", async () => { + vi.useFakeTimers(); + try { + const rotation401 = 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}'; + mockedCreateFnAgent.mockResolvedValueOnce(createSession(async () => { throw new Error(rotation401); }) as never); + const store = createAgentStore(baseAgent({ state: "active", lastError: undefined })); + const taskStore = createNoTaskStore(); + const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); + + let settled = false; + const heartbeat = monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }).finally(() => { settled = true; }); + // Exhaust the in-run transient-auth retry budget (2 retries × ~5s) on fake time. + for (let i = 0; i < 30 && !settled; i++) { + await vi.advanceTimersByTimeAsync(1_000); + } + await heartbeat; + + expect(store.agent.state).toBe("error"); + expect(store.agent.pauseReason).toBeUndefined(); + expect(isErrorRecoveryEligible(store.agent, 5)).toBe(true); + expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-parked-unrecoverable", + })); + } finally { + vi.useRealTimers(); + } + }); + it("leaves runtime-disabled operator-actionable error agents excluded from timer recovery", async () => { const session = createSession(async () => undefined); mockedCreateFnAgent.mockResolvedValueOnce(session as never); diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 37f5367876..6c6c28afbd 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -980,12 +980,23 @@ describe("SelfHealingManager", () => { expect(result).toBe(1); expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-1", "active"); - expect(agentStore.updateAgent).toHaveBeenLastCalledWith("report-1", { lastError: undefined }); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("report-1", { lastError: undefined, pauseReason: undefined }); expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("report-1", { reason: "transient-error", attempt: 1 }); managerWithAgents.stop(); }); - it("parks a manager-present agent whose error is operator-actionable (FN-7672/FN-7859 auth-credential cluster shape)", async () => { + /* + * FNXC:AgentHeartbeat 2026-07-12-20:10: + * The FN-7672 auth-credential cluster shape turned out to be a routine Claude + * Max OAuth token rotation (~8 h lifetime): the in-flight call 401s with + * "authentication_error: Invalid authentication credentials" even though + * refreshed credentials already exist, and the next call succeeds. That shape + * is now classified transient/recoverable, so the sweep AUTO-RECOVERS it + * (bounded by the shared retry budget) instead of parking a whole fleet of + * durable agents paused/"error-unrecoverable" for a human. Genuinely + * operator-actionable auth failures (scope grants, bad API keys) still park. + */ + it("auto-recovers a manager-present agent stuck on an OAuth token-rotation 401 (former unrecoverable-park shape)", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); const agentStore = createMockAgentStore([ @@ -996,6 +1007,42 @@ describe("SelfHealingManager", () => { reportsTo: "manager-1", lastError: 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcpL6f3iXHxeHfMUjg9o8"}', + metadata: {}, + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, + ]); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(1); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-auth", "active"); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("report-auth", { lastError: undefined, pauseReason: undefined }); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: "report-auth", + metadata: expect.objectContaining({ agentId: "report-auth", attempt: 1, limit: 5, source: "self-healing" }), + })); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("report-auth", { reason: "transient-error", attempt: 1 }); + managerWithAgents.stop(); + }); + + it("still parks a manager-present agent whose auth error is genuinely operator-actionable (OAuth scope grant)", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { id: "manager-1", state: "active", updatedAt: new Date(now).toISOString() } as Agent, + { + id: "report-scope", + state: "error", + reportsTo: "manager-1", + lastError: + 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}', updatedAt: new Date(now - 120_000).toISOString(), } as Agent, ]); @@ -1004,9 +1051,9 @@ describe("SelfHealingManager", () => { const result = await managerWithAgents.recoverOrphanedAgents(); expect(result).toBe(1); - expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-auth", "paused"); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-scope", "paused"); expect(agentStore.updateAgent).toHaveBeenCalledWith( - "report-auth", + "report-scope", expect.objectContaining({ pauseReason: HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON, metadata: expect.objectContaining({ @@ -1016,12 +1063,66 @@ describe("SelfHealingManager", () => { ); expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "agent:error-parked-unrecoverable", - target: "report-auth", - metadata: expect.objectContaining({ agentId: "report-auth", attempts: 0, limit: 5, source: "self-healing" }), + target: "report-scope", + metadata: expect.objectContaining({ agentId: "report-scope", attempts: 0, limit: 5, source: "self-healing" }), })); managerWithAgents.stop(); }); + it("un-parks an agent previously parked error-unrecoverable whose lastError now classifies recoverable", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { + id: "parked-rotation", + state: "paused", + pauseReason: HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON, + lastError: + 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}', + metadata: {}, + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, + // Same pauseReason but genuinely operator-actionable error: stays parked. + { + id: "parked-scope", + state: "paused", + pauseReason: HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON, + lastError: "OAuth token does not meet scope requirements", + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, + // Different pauseReason (e.g. user/budget pause): never touched. + { + id: "parked-budget", + state: "paused", + pauseReason: "budget-exhausted", + lastError: "Invalid authentication credentials", + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, + ]); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(1); + expect(agentStore.updateAgentState).toHaveBeenCalledTimes(1); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("parked-rotation", "active"); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("parked-rotation", { lastError: undefined, pauseReason: undefined }); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: "parked-rotation", + metadata: expect.objectContaining({ agentId: "parked-rotation", attempt: 1, limit: 5, source: "self-healing" }), + })); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("parked-rotation", { reason: "transient-error", attempt: 1 }); + expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("parked-scope", expect.anything()); + expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("parked-budget", expect.anything()); + managerWithAgents.stop(); + }); + it("recovers only the eligible manager-present agent among a mixed cluster without touching healthy siblings", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); @@ -1057,7 +1158,8 @@ describe("SelfHealingManager", () => { expect(result).toBe(2); expect(agentStore.updateAgentState).toHaveBeenCalledTimes(2); expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-transient", "active"); - expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-auth-1", "paused"); + // Rotation-shaped auth 401s are transient credential rotations — recovered, not parked. + expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-auth-1", "active"); expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("sibling-healthy-1", expect.anything()); expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("sibling-healthy-2", expect.anything()); managerWithAgents.stop(); @@ -1102,7 +1204,7 @@ describe("SelfHealingManager", () => { expect(result).toBe(1); expect(agentStore.updateAgentState).toHaveBeenCalledWith("orphan-1", "active"); - expect(agentStore.updateAgent).toHaveBeenLastCalledWith("orphan-1", { lastError: undefined }); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("orphan-1", { lastError: undefined, pauseReason: undefined }); expect(agentStore.updateAgent).toHaveBeenCalledWith( "orphan-1", expect.objectContaining({ @@ -1483,7 +1585,7 @@ describe("SelfHealingManager", () => { expect(result).toBe(1); expect(agentStore.updateAgentState).toHaveBeenCalledWith("orphan-2", "active"); - expect(agentStore.updateAgent).toHaveBeenCalledWith("orphan-2", { lastError: undefined }); + expect(agentStore.updateAgent).toHaveBeenCalledWith("orphan-2", { lastError: undefined, pauseReason: undefined }); managerWithAgents.stop(); }); diff --git a/packages/engine/src/__tests__/transient-error-detector.test.ts b/packages/engine/src/__tests__/transient-error-detector.test.ts index 67e3f8ce9a..2b3d59b3b5 100644 --- a/packages/engine/src/__tests__/transient-error-detector.test.ts +++ b/packages/engine/src/__tests__/transient-error-detector.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest"; import { isTransientError, + isTransientAuthCredentialError, classifyError, isSilentTransientError, extractMissingModulePath, @@ -415,6 +416,59 @@ describe("Transient Error Detector", () => { }); }); + /* + FNXC:Reliability-ErrorClassification 2026-07-12-20:10: + Regression suite for the durable-agent OAuth token-rotation incident: a routine Claude Max + credential rotation surfaced as `401 {"type":"authentication_error","message":"Invalid + authentication credentials"}`, matched the operator-actionable /credential/ pattern, + classified "permanent", and parked every durable agent paused/"error-unrecoverable". + These 401s must classify transient + NOT operator-actionable on every surface so in-run + retry and heartbeat/self-healing error recovery auto-recover. Scope-grant and API-key + failures stay operator-actionable. + */ + describe("isTransientAuthCredentialError", () => { + const rotation401 = + 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}'; + + it("classifies the OAuth token-rotation 401 as transient and not operator-actionable", () => { + expect(isTransientAuthCredentialError(rotation401)).toBe(true); + expect(isTransientError(rotation401)).toBe(true); + expect(classifyError(rotation401)).toBe("transient"); + expect(isOperatorActionableAgentError(rotation401)).toBe(false); + }); + + it("matches bare rotation shapes (message-only, token expired, envelope-only)", () => { + expect(isTransientAuthCredentialError("Invalid authentication credentials")).toBe(true); + expect(isTransientAuthCredentialError("token_expired: please re-authenticate")).toBe(true); + expect(isTransientAuthCredentialError('{"type":"error","error":{"type":"authentication_error"}}')).toBe(true); + }); + + it("keeps OAuth scope-grant failures operator-actionable even inside an authentication_error envelope", () => { + const scopeError = + '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}'; + expect(isTransientAuthCredentialError(scopeError)).toBe(false); + expect(isTransientError(scopeError)).toBe(false); + expect(classifyError(scopeError)).toBe("permanent"); + }); + + it("keeps API-key misconfiguration operator-actionable even inside an authentication_error envelope", () => { + const badApiKey = + '401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}'; + expect(isTransientAuthCredentialError(badApiKey)).toBe(false); + expect(classifyError(badApiKey)).toBe("permanent"); + expect(isTransientAuthCredentialError("missing ANTHROPIC_API_KEY")).toBe(false); + expect(isOperatorActionableAgentError("invalid api key")).toBe(true); + expect(isOperatorActionableAgentError("missing OPENAI_API_KEY")).toBe(true); + }); + + it("does not match unrelated auth failures or empty input", () => { + expect(isTransientAuthCredentialError("Authentication failed for provider")).toBe(false); + expect(isOperatorActionableAgentError("Authentication failed for provider")).toBe(true); + expect(isTransientAuthCredentialError("")).toBe(false); + expect(isTransientAuthCredentialError(null as unknown as string)).toBe(false); + }); + }); + describe("isSilentTransientError", () => { it("returns true for 'request was aborted'", () => { expect(isSilentTransientError("request was aborted")).toBe(true); diff --git a/packages/engine/src/agent-heartbeat.ts b/packages/engine/src/agent-heartbeat.ts index 2f28e279f1..a1bbcdf53e 100644 --- a/packages/engine/src/agent-heartbeat.ts +++ b/packages/engine/src/agent-heartbeat.ts @@ -67,6 +67,7 @@ export const HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON = "error-unrecoverable"; import { acquireTaskWorktree } from "./worktree-acquisition.js"; import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type EngineRunContext } from "./run-audit.js"; import { promptWithFallback } from "./pi.js"; +import { withRateLimitRetry } from "./rate-limit-retry.js"; import { buildAgentGatedActionSummary } from "./permanent-agent-gating.js"; import { createResolvedAgentSession, extractRuntimeHint, resolveHeartbeatSessionModels, resolveExecutorFallbackThinkingLevel } from "./agent-session-helpers.js"; import { resolveMcpServersForStore } from "./mcp-resolution.js"; @@ -1663,6 +1664,12 @@ export class HeartbeatMonitor { resultJson?: Record; stdoutExcerpt?: string; stderrExcerpt?: string; + /* + FNXC:AgentHeartbeat 2026-07-12-20:10: + Failure classification (recoverable vs unrecoverable park, FN-7835/FN-7859) must run on the provider error MESSAGE, never on a stack-bearing detail string: stack frames contain classifier-triggering identifiers — e.g. `at withRateLimitRetry (.../rate-limit-retry.ts)` matches the usage-limit /rate[_\s]?limit/ pattern and would misclassify EVERY failed heartbeat as usage-limit/unrecoverable. `stderrExcerpt` keeps the full detail for run-detail observability; `errorMessage` (message-only) drives classification and `agent.lastError`. + */ + /** Message-only failure text used for error classification and agent.lastError; falls back to stderrExcerpt. */ + errorMessage?: string; /** When true, preserve current agent state instead of forcing a terminal transition. */ skipStateTransition?: boolean; } @@ -1744,7 +1751,7 @@ export class HeartbeatMonitor { })) : MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS; const retryCount = latestAgent ? readHeartbeatErrorRetryCount(latestAgent) : 0; - const failedError = completionResult.stderrExcerpt ?? "Run failed"; + const failedError = completionResult.errorMessage ?? completionResult.stderrExcerpt ?? "Run failed"; const failedWithRecoverableError = isHeartbeatErrorRecoverable({ lastError: failedError }); const failedWithUnrecoverableError = !failedWithRecoverableError && !isStaleWorktreeModuleResolutionError(failedError); /* @@ -1761,7 +1768,7 @@ export class HeartbeatMonitor { ) { await this.store.updateAgentState(agentId, "paused"); await this.store.updateAgent(agentId, { - lastError: completionResult.stderrExcerpt ?? "Run failed", + lastError: failedError, pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, }); heartbeatLog.warn(`Agent ${agentId} error recovery exhausted after ${retryCount}/${errorRecoveryLimit} attempts — pausing`); @@ -3415,7 +3422,16 @@ export class HeartbeatMonitor { } // Execute - await promptWithFallback(session, executionPrompt); + /* + FNXC:AgentHeartbeat 2026-07-12-20:10: + Heartbeat prompts must run under the same rate-limit + transient-auth retry wrapper as executor/triage/merger work. Claude Max OAuth tokens rotate mid-run (~8 h); the in-flight call 401s ("authentication_error: Invalid authentication credentials") even though refreshed credentials already exist, and the next attempt succeeds. Without this wrapper a routine token rotation failed the run, pushed every durable agent to `error`, and (via FN-7859 unrecoverable classification) parked them paused for operator action. Retrying in-run prevents the error state at the source; the durable-agent error-recovery budget stays the backstop for errors that escape. + */ + await withRateLimitRetry(() => promptWithFallback(session, executionPrompt), { + onRetry: (attempt, delayMs, retryError) => { + const delaySec = Math.round(delayMs / 1000); + heartbeatLog.warn(`Agent ${agentId} heartbeat prompt hit retryable provider error — retry ${attempt} in ${delaySec}s: ${retryError.message}`); + }, + }); // Capture real per-session token counts from pi-coding-agent's // SessionStats. Falls back to a 4-chars-per-token estimate of output @@ -3492,7 +3508,7 @@ export class HeartbeatMonitor { heartbeatLog.log(`Heartbeat completed for ${agentId} (${toolCallCount} tool calls, ${usageInput} input + ${usageOutput} output + ${usageCached} cache-read + ${usageCacheWrite} cache-write tokens)`); } catch (err) { - const errorDetail = formatError(err).detail; + const { message: errorMessage, detail: errorDetail } = formatError(err); heartbeatLog.error(`Heartbeat execution failed for ${agentId}: ${errorDetail}`); await flushAgentLogger(); @@ -3502,6 +3518,7 @@ export class HeartbeatMonitor { await this.completeRun(agentId, run.id, { status: "failed", stderrExcerpt: errorDetail, + errorMessage, stdoutExcerpt: stdoutExcerpt || undefined, }); } @@ -3576,6 +3593,7 @@ export class HeartbeatMonitor { await this.completeRun(agentId, run.id, { status: "failed", stderrExcerpt: errorDetail, + errorMessage, }); } } catch (completeRunErr) { diff --git a/packages/engine/src/rate-limit-retry.ts b/packages/engine/src/rate-limit-retry.ts index 56e9ac2268..4501b02962 100644 --- a/packages/engine/src/rate-limit-retry.ts +++ b/packages/engine/src/rate-limit-retry.ts @@ -23,34 +23,18 @@ */ import { isUsageLimitError } from "./usage-limit-detector.js"; +import { isTransientAuthCredentialError } from "./transient-error-detector.js"; /* FNXC:EngineAuthRetry 2026-07-05-06:07: A long-running agent session holds its OAuth access token in memory. Claude Max access tokens rotate mid-run (~8 h lifetime); the in-flight call fails with a 401 authentication_error even though the credentials file has already been refreshed, and the very next call succeeds. Retry these a few times so a token-boundary rotation does not surface as a spurious task-failure alert. This budget is separate from the rate-limit retry budget and must not consume rate-limit attempts. -*/ -/** - * Matches transient authentication failures caused by credential rotation — - * e.g. a Claude Max OAuth access token expiring mid-run (~8 h lifetime). A - * long-running agent session holds the old token in memory; the very next - * call after the provider refreshes credentials succeeds, so these are worth - * a couple of quick retries before propagating as a task failure. - */ -const TRANSIENT_AUTH_ERROR_RE = - /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired/i; - -/* -FNXC:EngineAuthRetry 2026-07-05-06:07: -OAuth scope/permission-grant failures are NOT transient — the token is valid but lacks required grants, so the operator must re-authorize the connection. Retrying would repeat the failing call for ~10 s before surfacing the real (operator-actionable) error. This exclusion runs BEFORE the transient match because providers wrap scope errors inside a generic {"type":"authentication_error"} envelope that would otherwise match TRANSIENT_AUTH_ERROR_RE and retry pointlessly. +FNXC:EngineAuthRetry 2026-07-12-20:10: +The transient-auth classifier moved to transient-error-detector.ts (isTransientAuthCredentialError) so the same rotation-vs-operator-actionable decision drives this in-run retry AND durable-agent heartbeat error recovery / self-healing (FN-7835/FN-7844/FN-7859). Scope-grant failures and invalid/missing API keys are excluded there — the operator must act, so they surface immediately instead of retrying. */ -const SCOPE_ERROR_RE = - /oauth token does not meet scope|insufficient[_\s-]?scope|invalid[_\s-]?scope/i; function isTransientAuthError(message: string | undefined): boolean { - const msg = message ?? ""; - // Permanent scope failures must surface immediately instead of retrying. - if (SCOPE_ERROR_RE.test(msg)) return false; - return TRANSIENT_AUTH_ERROR_RE.test(msg); + return isTransientAuthCredentialError(message ?? ""); } /** Transient-auth retry budget — separate from the rate-limit `maxRetries`. */ diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 3688ff2604..57e0b49757 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -10475,11 +10475,20 @@ export class SelfHealingManager { const allAgentIds = new Set(allAgents.map((agent) => agent.id)); const now = Date.now(); + /* + FNXC:AgentHeartbeat 2026-07-12-20:10: + An agent parked paused/"error-unrecoverable" whose lastError NOW classifies as recoverable (e.g. transient OAuth token-rotation 401s that were misclassified operator-actionable before isTransientAuthCredentialError existed) must not stay parked forever waiting for a human. Re-admit exactly those parked agents to the error-recovery sweep; user pauses and every other pauseReason are untouched. The shared retry budget, cooldown, and staleness gates below still apply. + */ + const isReclassifiedRecoverableParkedError = (agent: Agent): boolean => + agent.state === "paused" + && agent.pauseReason === HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON + && isHeartbeatErrorRecoverable(agent); + const orphaned = allAgents.filter((agent) => { if (isEphemeralAgent(agent)) { return false; } - if (agent.state !== "running" && agent.state !== "error") { + if (agent.state !== "running" && agent.state !== "error" && !isReclassifiedRecoverableParkedError(agent)) { return false; } /* @@ -10513,7 +10522,7 @@ export class SelfHealingManager { return false; } - if (agent.state === "error") { + if (agent.state === "error" || isReclassifiedRecoverableParkedError(agent)) { const runtimeConfig = (agent.runtimeConfig ?? {}) as Record; if (runtimeConfig.enabled === false) { return false; @@ -10549,8 +10558,11 @@ export class SelfHealingManager { for (const agent of orphaned) { const updatedAt = Date.parse(agent.updatedAt ?? ""); const stuckForMs = Math.max(0, now - updatedAt); + // Reclassified "error-unrecoverable" parked agents run the same recovery + // branch as error-state agents: shared budget, cooldown, audit, restart. + const isErrorRecoveryCandidate = agent.state === "error" || isReclassifiedRecoverableParkedError(agent); try { - if (agent.state === "error") { + if (isErrorRecoveryCandidate) { const recoveryState = this.getDurableAgentRecoveryState(agent); const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? ""); const isUnrecoverableHeartbeatError = !isHeartbeatErrorRecoverable(agent) && !isStaleMissingModule; @@ -10653,9 +10665,12 @@ export class SelfHealingManager { await agentStore.updateAgentState(agent.id, "active"); await agentStore.updateAgent(agent.id, { lastError: undefined, + // Clear the "error-unrecoverable" park marker when a reclassified + // parked agent is re-admitted; harmless no-op for error-state agents. + pauseReason: undefined, }); - if (agent.state === "error") { + if (isErrorRecoveryCandidate) { const attempt = this.getDurableAgentRecoveryState(agent).attempts + 1; await this.emitDurableAgentErrorRecoveryAudit({ agentId: agent.id, @@ -10669,7 +10684,7 @@ export class SelfHealingManager { } } - if (agent.state === "error" && this.options.restartDurableAgentHeartbeat) { + if (isErrorRecoveryCandidate && this.options.restartDurableAgentHeartbeat) { const restartOk = await this.options.restartDurableAgentHeartbeat(agent.id, { reason: "transient-error", attempt: this.getDurableAgentRecoveryState(agent).attempts + 1, diff --git a/packages/engine/src/transient-error-detector.ts b/packages/engine/src/transient-error-detector.ts index e4aebf6239..413dcc52da 100644 --- a/packages/engine/src/transient-error-detector.ts +++ b/packages/engine/src/transient-error-detector.ts @@ -98,9 +98,37 @@ export function isTransientError(errorMessage: string): boolean { if (!errorMessage || typeof errorMessage !== "string") { return false; } + if (isTransientAuthCredentialError(errorMessage)) { + return true; + } return TRANSIENT_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage)); } +/* +FNXC:Reliability-ErrorClassification 2026-07-12-20:10: +A long-running agent session holds its OAuth access token in memory. Claude Max access tokens rotate mid-run (~8 h lifetime); the in-flight call fails with a 401 {"type":"authentication_error","message":"Invalid authentication credentials"} even though the credentials file has already been refreshed, and the very next call succeeds. These must classify as TRANSIENT (retryable) and NOT operator-actionable, so in-run retry (withRateLimitRetry) and durable-agent heartbeat error recovery (FN-7835/FN-7844/FN-7859) auto-recover instead of parking agents paused with pauseReason "error-unrecoverable". Previously the message matched the operator-actionable /credential/ and /unauthorized/ patterns and defaulted to "permanent", so a routine token rotation parked every durable agent for manual operator repair. +Genuinely operator-actionable auth failures are excluded first: OAuth scope/permission-grant errors (token valid but lacks grants) and explicit API-key problems (invalid/missing x-api-key) — retrying those only repeats the failing call. +*/ +const TRANSIENT_AUTH_CREDENTIAL_ROTATION_PATTERN = + /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired/i; +const OPERATOR_ACTIONABLE_AUTH_EXCLUSION_PATTERN = + /oauth token does not meet scope|insufficient[_\s-]?scope|invalid[_\s-]?scope|invalid (?:api[_\s-]?key|x-api-key)|missing\s+(?:\S+\s+)?(?:api[_\s-]?)?key/i; + +/** + * Detect a transient authentication failure caused by credential rotation + * (e.g. a Claude Max OAuth access token expiring mid-run). Scope-grant and + * API-key misconfiguration errors are excluded — those need operator action. + */ +export function isTransientAuthCredentialError(errorMessage: string): boolean { + if (!errorMessage || typeof errorMessage !== "string") { + return false; + } + if (OPERATOR_ACTIONABLE_AUTH_EXCLUSION_PATTERN.test(errorMessage)) { + return false; + } + return TRANSIENT_AUTH_CREDENTIAL_ROTATION_PATTERN.test(errorMessage); +} + /** * Patterns for transient errors that should be silently retried without * logging to task log entries. These errors are extremely noisy (high frequency) @@ -283,6 +311,13 @@ export function isOperatorActionableAgentError(errorMessage: string): boolean { if (!errorMessage || typeof errorMessage !== "string") { return false; } + /* + FNXC:Reliability-ErrorClassification 2026-07-12-20:10: + Transient OAuth token-rotation 401s must NOT be treated as operator-actionable even though the provider message contains "credentials": no operator action fixes them (the refreshed token already exists on disk) and marking them actionable parks durable agents "error-unrecoverable" instead of letting bounded heartbeat error recovery retry. Scope/API-key failures are excluded inside the classifier and still fall through to the actionable patterns below. + */ + if (isTransientAuthCredentialError(errorMessage)) { + return false; + } return ( isUnsupportedMessageRoleError(errorMessage) || isModelAuthTierIncompatibilityError(errorMessage) ||