diff --git a/docs/agents.md b/docs/agents.md index a250677ac..6bd99eec0 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -218,6 +218,26 @@ When the runtime model is present and differs from execution-lane settings, hear If a heartbeat cannot create/run a session due to unavailable provider credentials or missing provider registration, Fusion records `resultJson.reason = "heartbeat_model_unavailable"` with actionable diagnostics in `resultJson.detail`/`stderrExcerpt`. +### Durable-agent transient error auto-recovery + +Self-healing may auto-recover **durable (non-ephemeral)** agents stuck in `state="error"` when all eligibility checks pass: + +- agent is non-ephemeral (`isEphemeralAgent(...) === false`) +- heartbeat runtime is enabled (`runtimeConfig.enabled !== false`) +- no active heartbeat execution is already running for the agent +- `lastError` classifies as transient network/infrastructure failure +- `lastError` is **not** operator-actionable (credentials/model/billing-style failures) + +When eligible, self-healing uses bounded retries with persisted metadata at `agent.metadata.durableErrorRecovery`: + +- exponential cooldown (`30s` base, capped at `15m`) +- retry budget cap (`5` attempts) +- persisted `attempts`, `lastAttemptAt`, `nextRetryAt`, `exhausted`, and `lastReason` + +On restart attempts, the runtime triggers the normal heartbeat pipeline with `source: "automation"` and a structured `contextSnapshot.selfHealing` payload so operators can audit recovery runs in heartbeat history. + +Self-healing intentionally leaves agents in `error` (no auto-restart) when blockers are operator-actionable or non-transient, when cooldown has not elapsed, when active execution is present, or when retry budget is exhausted. + - **Timer trigger:** run completes and the durable agent returns to `state="active"` (recoverable soft-fail). - **Assignment / on-demand trigger:** run completes with `resultJson.actionRequired = true`, then the durable agent is paused with `pauseReason="heartbeat-model-unavailable"` and `lastError` set to actionable credential guidance (including the missing provider name when detectable). diff --git a/packages/engine/src/__tests__/heartbeat-executor.test.ts b/packages/engine/src/__tests__/heartbeat-executor.test.ts index cd0cab87c..1d1988c3d 100644 --- a/packages/engine/src/__tests__/heartbeat-executor.test.ts +++ b/packages/engine/src/__tests__/heartbeat-executor.test.ts @@ -2465,6 +2465,39 @@ describe("executeHeartbeat", () => { }); }); + it("persists automation source recovery context on run records", async () => { + const store = createStoreWithAgentForExec(); + const mockSession = createMockAgentSession(); + mockedCreateFnAgent.mockResolvedValue({ + session: mockSession as any, + }); + + const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" }); + + const result = await monitor.executeHeartbeat({ + agentId: "agent-001", + source: "automation", + triggerDetail: "self-healing durable-agent transient recovery", + contextSnapshot: { + selfHealing: { + reason: "transient-error", + attempt: 1, + source: "durable-agent-transient-error-recovery", + }, + }, + }); + + expect(result.contextSnapshot).toEqual( + expect.objectContaining({ + selfHealing: { + reason: "transient-error", + attempt: 1, + source: "durable-agent-transient-error-recovery", + }, + }), + ); + }); + it("records agent logs, context taskId, and stdoutExcerpt for successful runs", async () => { const store = createStoreWithAgentForExec(); const appendAgentLog = vi.fn().mockResolvedValue(undefined); diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 85a0673b5..04a9bc24e 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -529,19 +529,43 @@ describe("SelfHealingManager", () => { managerWithAgents.stop(); }); - it("recovers orphaned agent in error state", async () => { + it("recovers orphaned agent in transient error state", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); const agentStore = createMockAgentStore([ - { id: "orphan-1", state: "error", updatedAt: new Date(now - 120_000).toISOString() } as Agent, + { + id: "orphan-1", + state: "error", + lastError: "socket hang up", + metadata: {}, + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, ]); - const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); const result = await managerWithAgents.recoverOrphanedAgents(); expect(result).toBe(1); expect(agentStore.updateAgentState).toHaveBeenCalledWith("orphan-1", "active"); - expect(agentStore.updateAgent).toHaveBeenCalledWith("orphan-1", { lastError: undefined }); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("orphan-1", { lastError: undefined }); + expect(agentStore.updateAgent).toHaveBeenCalledWith( + "orphan-1", + expect.objectContaining({ + metadata: expect.objectContaining({ + durableErrorRecovery: expect.objectContaining({ + attempts: 1, + exhausted: false, + lastReason: "transient-error", + }), + }), + }), + ); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("orphan-1", { reason: "transient-error", attempt: 1 }); managerWithAgents.stop(); }); @@ -549,7 +573,7 @@ describe("SelfHealingManager", () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); const agentStore = createMockAgentStore([ - { id: "orphan-1", state: "error", updatedAt: new Date(now - 10_000).toISOString() } as Agent, + { id: "orphan-1", state: "error", lastError: "socket hang up", updatedAt: new Date(now - 10_000).toISOString() } as Agent, ]); const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); @@ -560,6 +584,99 @@ describe("SelfHealingManager", () => { managerWithAgents.stop(); }); + it("skips non-transient/operator-actionable durable errors", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { id: "agent-perm", state: "error", lastError: "invalid api key", updatedAt: new Date(now - 120_000).toISOString() } as Agent, + ]); + const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(0); + expect(agentStore.updateAgentState).not.toHaveBeenCalled(); + managerWithAgents.stop(); + }); + + it("suppresses transient recovery while cooldown is active", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { + id: "agent-cooldown", + state: "error", + lastError: "socket hang up", + updatedAt: new Date(now - 120_000).toISOString(), + metadata: { durableErrorRecovery: { attempts: 2, nextRetryAt: new Date(now + 5 * 60_000).toISOString() } }, + } as unknown as Agent, + ]); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(0); + expect(agentStore.updateAgent).not.toHaveBeenCalled(); + managerWithAgents.stop(); + }); + + it("suppresses transient recovery when active agent execution is present", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { id: "agent-active", state: "error", lastError: "socket hang up", updatedAt: new Date(now - 120_000).toISOString() } as Agent, + ]); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + hasActiveAgentExecution: (agentId) => agentId === "agent-active", + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(0); + expect(agentStore.updateAgentState).not.toHaveBeenCalled(); + managerWithAgents.stop(); + }); + + it("suppresses transient recovery when retry budget is exhausted", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { + id: "agent-exhausted", + state: "error", + lastError: "socket hang up", + updatedAt: new Date(now - 120_000).toISOString(), + metadata: { durableErrorRecovery: { attempts: 4 } }, + } as unknown as Agent, + ]); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(0); + expect(agentStore.updateAgentState).not.toHaveBeenCalled(); + expect(agentStore.updateAgent).toHaveBeenCalledWith( + "agent-exhausted", + expect.objectContaining({ + metadata: expect.objectContaining({ + durableErrorRecovery: expect.objectContaining({ + exhausted: true, + lastReason: "retry-budget-exhausted", + }), + }), + }), + ); + managerWithAgents.stop(); + }); + it("skips ephemeral agents", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); diff --git a/packages/engine/src/__tests__/transient-error-detector.test.ts b/packages/engine/src/__tests__/transient-error-detector.test.ts index be10b29f0..b190e2589 100644 --- a/packages/engine/src/__tests__/transient-error-detector.test.ts +++ b/packages/engine/src/__tests__/transient-error-detector.test.ts @@ -3,6 +3,7 @@ import { isTransientError, classifyError, isSilentTransientError, + isOperatorActionableAgentError, TRANSIENT_ERROR_PATTERNS, } from "../transient-error-detector.js"; import { isUsageLimitError } from "../usage-limit-detector.js"; @@ -238,6 +239,21 @@ describe("Transient Error Detector", () => { }); }); + describe("isOperatorActionableAgentError", () => { + it("returns true for credential/model/billing errors", () => { + expect(isOperatorActionableAgentError("invalid api key")).toBe(true); + expect(isOperatorActionableAgentError("Authentication failed for provider")).toBe(true); + expect(isOperatorActionableAgentError("model gpt-x not found")).toBe(true); + expect(isOperatorActionableAgentError("missing OPENAI_API_KEY")).toBe(true); + expect(isOperatorActionableAgentError("billing issue: quota exceeded")).toBe(true); + }); + + it("returns false for transient network errors", () => { + expect(isOperatorActionableAgentError("socket hang up")).toBe(false); + expect(isOperatorActionableAgentError("upstream connect error")).toBe(false); + }); + }); + describe("isSilentTransientError", () => { it("returns true for 'request was aborted'", () => { expect(isSilentTransientError("request was aborted")).toBe(true); diff --git a/packages/engine/src/runtimes/in-process-runtime.ts b/packages/engine/src/runtimes/in-process-runtime.ts index c7a2aacdb..e540a7e89 100644 --- a/packages/engine/src/runtimes/in-process-runtime.ts +++ b/packages/engine/src/runtimes/in-process-runtime.ts @@ -635,6 +635,25 @@ export class InProcessRuntime enqueueMerge: this.mergeEnqueuer ? (taskId: string) => this.mergeEnqueuer?.(taskId) : undefined, getActiveMergeTaskId: () => this.activeMergeTaskIdProvider?.() ?? null, leaseManager: this.leaseManager, + hasActiveAgentExecution: (agentId: string) => this.heartbeatMonitor?.getTrackedAgents().includes(agentId) ?? false, + restartDurableAgentHeartbeat: async (agentId: string, context: { reason: string; attempt: number }) => { + if (!this.heartbeatMonitor) { + return false; + } + const run = await this.heartbeatMonitor.executeHeartbeat({ + agentId, + source: "automation", + triggerDetail: `self-healing durable-agent transient recovery (${context.reason}, attempt ${context.attempt})`, + contextSnapshot: { + selfHealing: { + reason: context.reason, + attempt: context.attempt, + source: "durable-agent-transient-error-recovery", + }, + }, + }); + return !!run; + }, }); this.selfHealingManager.start(); this.stuckTaskDetector.start(); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 52cd79d1f..2dc8eb439 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -22,6 +22,7 @@ import type { MeshLeaseManager } from "./mesh-lease-manager.js"; import { createLogger } from "./logger.js"; import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js"; import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js"; +import { classifyError, isOperatorActionableAgentError } from "./transient-error-detector.js"; const log = createLogger("self-healing"); const execAsync = promisify(exec); @@ -91,6 +92,8 @@ export interface SelfHealingOptions { * Used to avoid clearing a transient merge status mid-merge. */ getActiveMergeTaskId?: () => string | null; + hasActiveAgentExecution?: (agentId: string) => boolean; + restartDurableAgentHeartbeat?: (agentId: string, context: { reason: string; attempt: number }) => Promise; } const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000; @@ -124,6 +127,9 @@ const MAX_TASK_DONE_RETRIES = 3; const MAX_AUTO_MERGE_RETRIES = 3; const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000; const DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS = 5 * 60_000; +const DURABLE_ERROR_RECOVERY_MAX_RETRIES = 5; +const DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS = 30_000; +const DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS = 15 * 60_000; interface LandedTaskCommit { sha: string; @@ -2182,6 +2188,33 @@ export class SelfHealingManager { } } + private getDurableAgentRecoveryState(agent: { metadata?: Record | null }): { + attempts: number; + nextRetryAt?: string; + exhausted?: boolean; + } { + const metadata = agent.metadata ?? {}; + const raw = metadata.durableErrorRecovery; + if (!raw || typeof raw !== "object") { + return { attempts: 0 }; + } + const record = raw as Record; + const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts) + ? Math.max(0, Math.floor(record.attempts)) + : 0; + return { + attempts, + nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined, + exhausted: record.exhausted === true, + }; + } + + private computeDurableAgentRecoveryCooldownMs(attempts: number): number { + const clampedAttempts = Math.max(1, attempts); + const exponential = DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS * Math.pow(2, clampedAttempts - 1); + return Math.min(exponential, DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS); + } + async recoverOrphanedAgents(): Promise { const agentStore = this.options.agentStore; if (!agentStore) { @@ -2212,10 +2245,39 @@ export class SelfHealingManager { return false; } const updatedAt = Date.parse(agent.updatedAt ?? ""); - if (!Number.isFinite(updatedAt)) { + if (!Number.isFinite(updatedAt) || now - updatedAt < recoveryTimeoutMs) { return false; } - return now - updatedAt >= recoveryTimeoutMs; + + if (agent.state === "error") { + const runtimeConfig = (agent.runtimeConfig ?? {}) as Record; + if (runtimeConfig.enabled === false) { + return false; + } + if (this.options.hasActiveAgentExecution?.(agent.id) === true) { + return false; + } + if (classifyError(agent.lastError ?? "") !== "transient") { + return false; + } + if (isOperatorActionableAgentError(agent.lastError ?? "")) { + return false; + } + + const recoveryState = this.getDurableAgentRecoveryState(agent); + if (recoveryState.exhausted) { + return false; + } + if (recoveryState.nextRetryAt) { + const nextRetryMs = Date.parse(recoveryState.nextRetryAt); + if (Number.isFinite(nextRetryMs) && nextRetryMs > now) { + log.log(`Durable agent ${agent.id} transient recovery delayed until ${recoveryState.nextRetryAt}`); + return false; + } + } + } + + return true; }); if (orphaned.length === 0) { @@ -2227,10 +2289,44 @@ export class SelfHealingManager { const updatedAt = Date.parse(agent.updatedAt ?? ""); const stuckForMs = Math.max(0, now - updatedAt); try { + if (agent.state === "error") { + const recoveryState = this.getDurableAgentRecoveryState(agent); + const nextAttempts = recoveryState.attempts + 1; + const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES; + const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString(); + await agentStore.updateAgent(agent.id, { + metadata: { + ...(agent.metadata ?? {}), + durableErrorRecovery: { + attempts: nextAttempts, + lastAttemptAt: new Date().toISOString(), + nextRetryAt, + exhausted, + lastReason: exhausted ? "retry-budget-exhausted" : "transient-error", + }, + }, + }); + if (exhausted) { + log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: retry budget exhausted`); + continue; + } + } + await agentStore.updateAgentState(agent.id, "active"); await agentStore.updateAgent(agent.id, { lastError: undefined, }); + + if (agent.state === "error" && this.options.restartDurableAgentHeartbeat) { + const restartOk = await this.options.restartDurableAgentHeartbeat(agent.id, { + reason: "transient-error", + attempt: this.getDurableAgentRecoveryState(agent).attempts + 1, + }); + if (!restartOk) { + log.warn(`Durable-agent transient recovery heartbeat restart skipped for ${agent.id}`); + } + } + log.log( `Auto-recovered: orphaned agent ${agent.id} stuck in ${agent.state} for ${Math.round(stuckForMs / 1000)}s — reset to active`, ); diff --git a/packages/engine/src/transient-error-detector.ts b/packages/engine/src/transient-error-detector.ts index a70c42ac7..10d31db8a 100644 --- a/packages/engine/src/transient-error-detector.ts +++ b/packages/engine/src/transient-error-detector.ts @@ -169,3 +169,25 @@ export function classifyError(errorMessage: string): "transient" | "usage-limit" // Default to permanent (mark as failed) return "permanent"; } + +const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [ + /invalid api key/i, + /authentication failed/i, + /unauthorized/i, + /forbidden/i, + /insufficient permissions?/i, + /model .* not found/i, + /unknown model/i, + /no such model/i, + /credential/i, + /missing .*key/i, + /billing/i, + /quota exceeded/i, +]; + +export function isOperatorActionableAgentError(errorMessage: string): boolean { + if (!errorMessage || typeof errorMessage !== "string") { + return false; + } + return OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage)); +}