diff --git a/.changeset/fn-7844-error-recovery-coordination.md b/.changeset/fn-7844-error-recovery-coordination.md new file mode 100644 index 0000000000..bbaa807f76 --- /dev/null +++ b/.changeset/fn-7844-error-recovery-coordination.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Coordinate durable-agent error recovery across heartbeat and self-healing. +category: fix +dev: Reconciles heartbeatErrorRecovery with recoverOrphanedAgents so timer and self-healing paths share one retry budget, use consistent transient/operator-actionable eligibility, and emit a source-discriminated audit surface (FN-7844). diff --git a/AGENTS.md b/AGENTS.md index dcfb5e4edc..9586d0f978 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,7 +224,7 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme - FN-7158: agent performance reflections emit `reflection:generated`, `reflection:skipped`, and `reflection:failed` with ids/counts/outcomes-only metadata; never persist reflection prose or prompt text in run-audit. - FN-7528: a deterministic, non-LLM post-task performance capture (`AgentReflectionService.captureTaskPerformance`) runs once per completed task and emits `reflection:captured` with ids/counts/outcomes-only metadata (`retryReworkCount?`, `filesTouchedCount?`, `packagesTouchedCount?`, `verificationFileScoped?`, `durationMs?`); never persists `verificationScopeReason` free-text or summary prose in run-audit. - FN-7787: `createResolvedAgentSession` enriches `session:runtime-resolved` with `noModelResolved: true` and `runtimeBuiltInFallbackModel` when a non-mock/non-test session reaches runtime creation without a complete provider/model pair; this is a visibility signal for runtime built-in fallback usage, not a fabricated model-resolution verdict. -- FN-7835: heartbeat error-state recovery emits `agent:auto-recover-error-state` when a durable heartbeat-managed agent with transient, non-operator-actionable `lastError` clears `error` and retries on the next heartbeat; metadata stays ids/counts/outcomes-only (`agentId`, attempt, limit, source). It emits `agent:error-retry-exhausted` when the bounded recovery budget is exhausted and the agent is parked `paused` with `pauseReason:"error-retry-exhausted"`. Operator-actionable errors remain parked for human repair. +- FN-7835/FN-7844: durable-agent error-state recovery emits `agent:auto-recover-error-state` when either the heartbeat timer or the self-healing sweep clears a transient, non-operator-actionable `error` and retries; metadata stays ids/counts/outcomes-only (`agentId`, attempt, limit, source), where `source` is `timer`/`automation`/`self-healing`. Both entry paths share the `heartbeatErrorRecovery` budget (self-healing keeps `durableErrorRecovery` only for cooldown/stale-path bookkeeping) and emit `agent:error-retry-exhausted` when the shared budget is exhausted and the agent is parked `paused` with `pauseReason:"error-retry-exhausted"`. Operator-actionable and stale-worktree/module-resolution errors remain parked for human repair. - FN-7802: self-healing emits `task:reconcile-missing-worktree-merge-active` when it proves an `in-review` merge-active task (`merging`/`merging-pr`/`merging-fix`) is stranded by an unusable-worktree session-start failure, clears stale `worktree`/`branch`/`sessionFile`, resets the worktree-session retry budget, increments `recoveryRetryCount` as the bounded stale-metadata clear counter, and requeues to `todo`; it emits `task:reconcile-missing-worktree-merge-active-no-action` when `autoMerge:false`, workspace-task ownership, or triple-proof blocks the backward move. - FN-7011: self-healing emits `task:reconcile-engine-downtime-active-timing` when startup recovery shifts active task segment anchors to exclude proven engine-process downtime, and `task:reconcile-engine-downtime-active-timing-no-action` when no active task qualifies. - FN-5419: git run-audit now includes `pull:fast-forward` and `stash:pop-conflict`; dashboard git surfaces now include the extended `POST /api/git/pull` integration-worktree path plus companion `POST /api/git/stash-resolve`, `POST /api/git/stash-drop`, and `POST /api/git/stash-apply` routes. diff --git a/docs/agents.md b/docs/agents.md index d21a2ce852..c5e11ba045 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -351,25 +351,25 @@ If a heartbeat cannot create/run a session due to unavailable provider credentia ### Durable-agent transient error auto-recovery -Self-healing may auto-recover **durable (non-ephemeral)** agents stuck in `state="error"` when all eligibility checks pass: +Durable-agent error recovery is coordinated between the heartbeat timer path and the self-healing sweep. Either entry path may auto-recover **durable (non-ephemeral)** agents stuck in `state="error"` only when all eligibility checks pass: - agent is non-ephemeral (`isEphemeralAgent(...) === false`) - heartbeat runtime is enabled (`runtimeConfig.enabled !== false`) - no active heartbeat execution is already running for the agent - `lastError` classifies as transient network/infrastructure failure - `lastError` is **not** operator-actionable (credentials/model/billing-style failures) +- stale worktree/module-resolution failures remain suppressed instead of auto-restarted -When eligible, self-healing uses bounded retries with persisted metadata at `agent.metadata.durableErrorRecovery`: +Both paths use the same persisted retry budget, `agent.metadata.heartbeatErrorRecovery.consecutiveAttempts`, with the default cap of `5` attempts (settings-overridable through `heartbeatErrorRecoveryAttempts`). The timer path provides fast recovery on the agent's own interval; the self-healing sweep is the backstop for stale `error` agents whose timer was lost, delayed, or did not re-tick. Self-healing still persists `agent.metadata.durableErrorRecovery` for sweep-specific cooldown and stale-path details: - exponential cooldown (`30s` base, capped at `15m`) -- retry budget cap (`5` attempts) -- persisted `attempts`, `lastAttemptAt`, `nextRetryAt`, `exhausted`, and `lastReason` +- persisted `attempts`, `lastAttemptAt`, `nextRetryAt`, `exhausted`, `lastReason`, and stale missing-module path counters -On restart attempts, the runtime triggers the normal heartbeat pipeline with `source: "automation"` and a structured `contextSnapshot.selfHealing` payload so operators can audit recovery runs in heartbeat history. +On restart attempts, the runtime triggers the normal heartbeat pipeline with `source: "automation"` and a structured `contextSnapshot.selfHealing` payload so operators can audit recovery runs in heartbeat history. The sweep flips `error → active` before calling `executeHeartbeat`, so the heartbeat run does not re-enter run-entry error recovery or double-count the same recovery. -Self-healing intentionally leaves agents in `error` (no auto-restart) when blockers are operator-actionable or non-transient, when cooldown has not elapsed, when active execution is present, or when retry budget is exhausted. +Self-healing intentionally leaves agents in `error` (or parks them `paused` when the shared retry budget is exhausted) when blockers are operator-actionable or non-transient, when stale worktree/module-resolution suppression applies, when cooldown has not elapsed, when active execution is present, or when retry budget is exhausted. -**Manager presence does not gate this sweep (FN-7672):** eligibility for durable `state="error"` recovery does *not* depend on whether the agent's `reportsTo` manager is present/active. `HeartbeatTriggerScheduler` clears timers entirely once an agent enters `state="error"`, so this recovery sweep is the *only* path back to a healthy heartbeat for a durable agent stuck in `error` — a present manager does not make the agent any less stuck. (A separate, unrelated `managerMissing` check still gates recovery of orphaned `state="running"` agents — a different failure mode where a live process's manager row was deleted.) FN-7672 root-caused a correlated 4-agent error cluster reporting to one active manager (a transient upstream auth/session blip) that could never have self-healed under the old manager-missing-only gate, even once the underlying cause resolved. +**Manager presence does not gate this sweep (FN-7672/FN-7844):** eligibility for durable `state="error"` recovery does *not* depend on whether the agent's `reportsTo` manager is present/active. The timer path is now the fast path for heartbeat-managed error agents, while this recovery sweep remains the maintenance backstop for durable agents that are still stale in `error`; a present manager does not make the agent any less stuck. (A separate, unrelated `managerMissing` check still gates recovery of orphaned `state="running"` agents — a different failure mode where a live process's manager row was deleted.) FN-7672 root-caused a correlated 4-agent error cluster reporting to one active manager (a transient upstream auth/session blip) that could never have self-healed under the old manager-missing-only gate, even once the underlying cause resolved. - **Timer trigger:** run completes and the durable agent returns to `state="active"` (recoverable soft-fail). - **Assignment / on-demand trigger:** run completes with `resultJson.actionRequired = true`, then the durable agent is paused with `pauseReason="heartbeat-model-unavailable"` and `lastError` set to actionable credential guidance (including the missing provider name when detectable). diff --git a/docs/architecture.md b/docs/architecture.md index b543e55f0f..9a1e40cf39 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -675,7 +675,7 @@ Runtime action-gate flow (v1): - `StuckTaskDetector` (`stuck-task-detector.ts`) — inactivity/loop stall detection - `GridlockDetector` (`gridlock-detector.ts`) — detects all-blocked todo pipelines and emits notification events (plus explicit clear signals when gridlock resolves) - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification -- Durable agent heartbeat recovery (FN-7835): a heartbeat-managed, runtime-enabled non-ephemeral agent that lands in `state:"error"` remains timer-eligible and clears `lastError` by transitioning `error → active` at the next heartbeat run entry. Recovery is bounded by `MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS` (settings-overridable through the engine's optional cast-based knob); success resets the metadata counter, while budget exhaustion parks the agent `paused` with `pauseReason:"error-retry-exhausted"` and emits `agent:error-retry-exhausted`. +- Durable agent error recovery (FN-7835/FN-7844): a heartbeat-managed, runtime-enabled non-ephemeral agent that lands in `state:"error"` remains timer-eligible and clears `lastError` by transitioning `error → active` at the next heartbeat run entry when `lastError` is transient and non-operator-actionable. Recovery is bounded by one shared `heartbeatErrorRecovery` attempt budget (`MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS`, settings-overridable through the engine's optional cast-based knob) across both the timer path and `SelfHealingManager.recoverOrphanedAgents()`. Self-healing is the stale-agent backstop and still stores `durableErrorRecovery` cooldown/stale-module metadata, but it writes/reads the shared heartbeat counter and emits the same `agent:auto-recover-error-state` / `agent:error-retry-exhausted` audit surface with `source:"self-healing"`. The sweep flips `error → active` before `restartDurableAgentHeartbeat()` calls `executeHeartbeat()`, preventing run-entry recovery from re-counting or double-emitting for the same recovery. Success resets the shared counter and clears legacy sweep retry state; budget exhaustion parks the agent `paused` with `pauseReason:"error-retry-exhausted"`. - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - Batch 1 maintenance now includes `reconcile-orphaned-task-dirs` (FN-6783), a paused-safe housekeeping step that calls `TaskStore.reconcileOrphanedTaskDirs()` so valid live `.fusion/tasks/{ID}/task.json` records missing from the SQLite index become visible without waiting for process restart. The store-level guard skips any ID already present in active, soft-deleted, archived, or tombstoned storage and emits `task:reconcile-orphaned-task-dir` only for recovered rows. - Batch 1 maintenance also includes `reconcile-phantom-committed-reservations` (FN-7069), which calls `TaskStore.reconcilePhantomCommittedReservations()` for committed task-ID reservations that have no live/soft-deleted/archived task row and no `.fusion/tasks/{ID}/task.json`. The sweep prunes orphaned `activityLog` rows and `agents`/cascaded `agentRuns`, preserves `runAuditEvents`, and keeps the reservation `committed` per FN-5105 so the ID is permanently reserved rather than resurrected or handed out again. diff --git a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts index b4196681d5..12aa3c1160 100644 --- a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts +++ b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts @@ -42,6 +42,7 @@ import { HeartbeatTriggerScheduler, incrementHeartbeatErrorRecoveryMetadata, isErrorRecoveryEligible, + isHeartbeatErrorRecoverable, readHeartbeatErrorRetryCount, resetHeartbeatErrorRecoveryMetadata, resolveErrorRecoveryLimit, @@ -143,7 +144,7 @@ describe("heartbeat error-recovery primitives", () => { expect(resolveErrorRecoveryLimit({ heartbeatErrorRecoveryAttempts: Number.NaN } as never)).toBe(5); }); - it("reads, increments, and resets the counter without clobbering unrelated metadata", () => { + it("reads, increments, and resets the shared counter without clobbering unrelated metadata", () => { const agent = baseAgent({ metadata: { heartbeatTimerRepair: { repairedAt: "now" } } }); expect(readHeartbeatErrorRetryCount(agent)).toBe(0); @@ -154,8 +155,15 @@ describe("heartbeat error-recovery primitives", () => { const forced = buildHeartbeatErrorRecoveryMetadata({ metadata: incremented }, 4); expect(readHeartbeatErrorRetryCount({ metadata: forced })).toBe(4); - const reset = resetHeartbeatErrorRecoveryMetadata({ metadata: forced }); + const legacySweepMetadata = { + ...forced, + durableErrorRecovery: { attempts: 5, exhausted: true }, + }; + expect(readHeartbeatErrorRetryCount({ metadata: legacySweepMetadata })).toBe(5); + + const reset = resetHeartbeatErrorRecoveryMetadata({ metadata: legacySweepMetadata }); expect(reset.heartbeatTimerRepair).toEqual({ repairedAt: "now" }); + expect(reset.durableErrorRecovery).toBeUndefined(); expect(readHeartbeatErrorRetryCount({ metadata: reset })).toBe(0); expect(reset[HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]).toMatchObject({ consecutiveAttempts: 0 }); }); @@ -168,6 +176,7 @@ describe("heartbeat error-recovery primitives", () => { expect(isErrorRecoveryEligible(baseAgent({ metadata: buildHeartbeatErrorRecoveryMetadata(baseAgent(), 5), lastError: "socket hang up" }), 5)).toBe(false); expect(isErrorRecoveryEligible(baseAgent({ lastError: "invalid api key" }), 5)).toBe(false); expect(isErrorRecoveryEligible(baseAgent({ lastError: "SyntaxError: Unexpected token" }), 5)).toBe(false); + expect(isHeartbeatErrorRecoverable({ lastError: "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/tmp/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /tmp/deleted/packages/engine/src/pi.ts" })).toBe(false); }); }); diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index e3fc237404..2ccfc70233 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -112,6 +112,7 @@ vi.mock("../merger.js", () => ({ })); import { SelfHealingManager, isBranchAheadOfBase, MAX_AUTO_MERGE_RETRIES } from "../self-healing.js"; +import { HEARTBEAT_ERROR_RECOVERY_METADATA_KEY, HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON } from "../agent-heartbeat.js"; import type { TaskStore, Settings, Task, AgentStore, Agent, NotificationProvider } from "@fusion/core"; import { EventEmitter } from "node:events"; import { execSync } from "node:child_process"; @@ -1091,6 +1092,7 @@ describe("SelfHealingManager", () => { "orphan-1", expect.objectContaining({ metadata: expect.objectContaining({ + [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: expect.objectContaining({ consecutiveAttempts: 1 }), durableErrorRecovery: expect.objectContaining({ attempts: 1, exhausted: false, @@ -1099,6 +1101,11 @@ describe("SelfHealingManager", () => { }), }), ); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: "orphan-1", + metadata: expect.objectContaining({ agentId: "orphan-1", attempt: 1, limit: 5, source: "self-healing" }), + })); expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("orphan-1", { reason: "transient-error", attempt: 1 }); managerWithAgents.stop(); }); @@ -1317,18 +1324,67 @@ describe("SelfHealingManager", () => { const result = await managerWithAgents.recoverOrphanedAgents(); expect(result).toBe(0); - expect(agentStore.updateAgentState).not.toHaveBeenCalled(); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("agent-exhausted", "paused"); expect(agentStore.updateAgent).toHaveBeenCalledWith( "agent-exhausted", expect.objectContaining({ metadata: expect.objectContaining({ + [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: expect.objectContaining({ consecutiveAttempts: 5 }), durableErrorRecovery: expect.objectContaining({ + attempts: 5, exhausted: true, lastReason: "retry-budget-exhausted", }), }), }), ); + expect(agentStore.updateAgent).toHaveBeenCalledWith("agent-exhausted", { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON }); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-retry-exhausted", + target: "agent-exhausted", + metadata: expect.objectContaining({ agentId: "agent-exhausted", attempts: 5, limit: 5, source: "self-healing" }), + })); + managerWithAgents.stop(); + }); + + it("honors heartbeat timer recovery attempts when the self-healing sweep checks exhaustion", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { + id: "agent-shared-budget", + state: "error", + lastError: "socket hang up", + updatedAt: new Date(now - 120_000).toISOString(), + metadata: { [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: { consecutiveAttempts: 4 } }, + } as unknown as Agent, + ]); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(0); + expect(restartDurableAgentHeartbeat).not.toHaveBeenCalled(); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("agent-shared-budget", "paused"); + expect(agentStore.updateAgent).toHaveBeenCalledWith( + "agent-shared-budget", + expect.objectContaining({ + metadata: expect.objectContaining({ + [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: expect.objectContaining({ consecutiveAttempts: 5 }), + durableErrorRecovery: expect.objectContaining({ attempts: 5, exhausted: true }), + }), + }), + ); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-retry-exhausted", + target: "agent-shared-budget", + metadata: expect.objectContaining({ attempts: 5, limit: 5, source: "self-healing" }), + })); managerWithAgents.stop(); }); diff --git a/packages/engine/src/agent-heartbeat.ts b/packages/engine/src/agent-heartbeat.ts index 36e1d8c575..c56b60be16 100644 --- a/packages/engine/src/agent-heartbeat.ts +++ b/packages/engine/src/agent-heartbeat.ts @@ -4087,18 +4087,30 @@ export function resolveErrorRecoveryLimit(settings: Settings | null | undefined) return Math.max(1, Math.floor(raw)); } -export function readHeartbeatErrorRetryCount(agent: Pick): number { +export function readHeartbeatErrorRetryCount(agent: { metadata?: Record | null }): number { + /* + FNXC:AgentHeartbeat 2026-07-11-22:42: + FN-7844 requires the heartbeat timer and self-healing sweep to honor one durable-agent error-recovery budget. Read the legacy durableErrorRecovery attempt count as part of the shared budget so agents recovered by either entry path cannot receive separate retry pools. + */ const metadata = (agent.metadata ?? {}) as Record; const raw = metadata[HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]; - if (!raw || typeof raw !== "object") { - return 0; - } - const candidate = raw as Record; - const count = candidate.consecutiveAttempts; - return typeof count === "number" && Number.isFinite(count) && count > 0 ? Math.floor(count) : 0; + const heartbeatCount = raw && typeof raw === "object" + ? (raw as Record).consecutiveAttempts + : 0; + const legacyRaw = metadata.durableErrorRecovery; + const legacyCount = legacyRaw && typeof legacyRaw === "object" + ? (legacyRaw as Record).attempts + : 0; + const normalizedHeartbeatCount = typeof heartbeatCount === "number" && Number.isFinite(heartbeatCount) && heartbeatCount > 0 + ? Math.floor(heartbeatCount) + : 0; + const normalizedLegacyCount = typeof legacyCount === "number" && Number.isFinite(legacyCount) && legacyCount > 0 + ? Math.floor(legacyCount) + : 0; + return Math.max(normalizedHeartbeatCount, normalizedLegacyCount); } -export function buildHeartbeatErrorRecoveryMetadata(agent: Pick, consecutiveAttempts: number): Record { +export function buildHeartbeatErrorRecoveryMetadata(agent: { metadata?: Record | null }, consecutiveAttempts: number): Record { return { ...(agent.metadata ?? {}), [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: { @@ -4108,12 +4120,13 @@ export function buildHeartbeatErrorRecoveryMetadata(agent: Pick): Record { +export function incrementHeartbeatErrorRecoveryMetadata(agent: { metadata?: Record | null }): Record { return buildHeartbeatErrorRecoveryMetadata(agent, readHeartbeatErrorRetryCount(agent) + 1); } -export function resetHeartbeatErrorRecoveryMetadata(agent: Pick): Record { - return buildHeartbeatErrorRecoveryMetadata(agent, 0); +export function resetHeartbeatErrorRecoveryMetadata(agent: { metadata?: Record | null }): Record { + const { durableErrorRecovery: _legacyDurableErrorRecovery, ...metadata } = (agent.metadata ?? {}) as Record; + return buildHeartbeatErrorRecoveryMetadata({ metadata }, 0); } export function isHeartbeatErrorRecoverable(agent: Pick): boolean { diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 99dab5910f..e6787f4455 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -44,7 +44,14 @@ import { isRecoverableMissingWorktreeReviewFailureWithProgress, MERGE_ACTIVE_MISSING_WORKTREE_STATUSES, } from "./restart-recovery-coordinator.js"; -import { classifyError, extractMissingModulePath, isNonContinuableSessionError, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js"; +import { extractMissingModulePath, isNonContinuableSessionError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js"; +import { + buildHeartbeatErrorRecoveryMetadata, + HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, + isHeartbeatErrorRecoverable, + readHeartbeatErrorRetryCount, + resolveErrorRecoveryLimit, +} from "./agent-heartbeat.js"; import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js"; import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type RunAuditor } from "./run-audit.js"; import { finalizeProvenAutoMergeTask, validateWorkflowDoneMergeProof } from "./auto-merge-finalization.js"; @@ -509,7 +516,6 @@ const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000; const DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS = 5 * 60_000; const DEFAULT_STALE_MERGING_FANOUT_MIN_AGE_MS = 15 * 60_000; const DEFAULT_UNBACKED_MERGING_FANOUT_GRACE_MS = 60_000; -const DURABLE_ERROR_RECOVERY_MAX_RETRIES = 5; const DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS = 30_000; const DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS = 15 * 60_000; const RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS = PARKED_AGENT_LINK_FRESH_RUN_MS; @@ -10231,13 +10237,11 @@ export class SelfHealingManager { } { const metadata = agent.metadata ?? {}; const raw = metadata.durableErrorRecovery; - if (!raw || typeof raw !== "object") { - return { attempts: 0, consecutiveMissingModulePathCount: 0 }; - } - const record = raw as Record; - const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts) + const record = raw && typeof raw === "object" ? raw as Record : {}; + const durableAttempts = typeof record.attempts === "number" && Number.isFinite(record.attempts) ? Math.max(0, Math.floor(record.attempts)) : 0; + const attempts = Math.max(durableAttempts, readHeartbeatErrorRetryCount(agent)); const consecutiveMissingModulePathCount = typeof record.consecutiveMissingModulePathCount === "number" && Number.isFinite(record.consecutiveMissingModulePathCount) ? Math.max(0, Math.floor(record.consecutiveMissingModulePathCount)) @@ -10257,6 +10261,36 @@ export class SelfHealingManager { return Math.min(exponential, DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS); } + private async emitDurableAgentErrorRecoveryAudit(options: { + agentId: string; + type: "agent:auto-recover-error-state" | "agent:error-retry-exhausted"; + attempt?: number; + attempts?: number; + limit: number; + source: "self-healing"; + }): Promise { + try { + await createRunAuditor(this.store, { + runId: generateSyntheticRunId("durable-agent-error-recovery", options.agentId), + agentId: "self-healing", + phase: "durable-agent-error-recovery", + source: options.source, + }).database({ + type: options.type as DatabaseMutationType, + target: options.agentId, + metadata: { + agentId: options.agentId, + ...(options.attempt !== undefined ? { attempt: options.attempt } : {}), + ...(options.attempts !== undefined ? { attempts: options.attempts } : {}), + limit: options.limit, + source: options.source, + }, + }); + } catch (error) { + log.warn(`Failed to emit durable-agent error recovery audit for ${options.agentId}: ${error instanceof Error ? error.message : String(error)}`); + } + } + private async emitStaleAgentAssignmentAudit(options: { agent: Pick; taskId: string; @@ -10429,6 +10463,7 @@ export class SelfHealingManager { try { const settings = await this.store.getSettings(); + const errorRecoveryLimit = resolveErrorRecoveryLimit(settings); const timeoutMs = settings.taskStuckTimeoutMs; if (!Number.isFinite(timeoutMs) || timeoutMs === undefined || timeoutMs <= 0) { return 0; @@ -10485,10 +10520,7 @@ export class SelfHealingManager { if (this.options.hasActiveAgentExecution?.(agent.id) === true) { return false; } - if (classifyError(agent.lastError ?? "") !== "transient" && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) { - return false; - } - if (isOperatorActionableAgentError(agent.lastError ?? "")) { + if (!isHeartbeatErrorRecoverable(agent) && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) { return false; } @@ -10549,11 +10581,15 @@ export class SelfHealingManager { continue; } const nextAttempts = recoveryState.attempts + 1; - const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES; + const exhausted = nextAttempts >= errorRecoveryLimit; const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString(); + /* + FNXC:AgentHeartbeat 2026-07-11-22:42: + FN-7844 consolidates durable-agent error recovery accounting across the heartbeat timer and self-healing sweep. The sweep keeps its cooldown/stale-path metadata, but writes the shared heartbeatErrorRecovery counter and audit event so a single retry budget applies regardless of which recovery entry path fires. + */ await agentStore.updateAgent(agent.id, { metadata: { - ...(agent.metadata ?? {}), + ...buildHeartbeatErrorRecoveryMetadata(agent, nextAttempts), durableErrorRecovery: { attempts: nextAttempts, lastAttemptAt: new Date().toISOString(), @@ -10566,6 +10602,15 @@ export class SelfHealingManager { }, }); if (exhausted) { + await this.emitDurableAgentErrorRecoveryAudit({ + agentId: agent.id, + type: "agent:error-retry-exhausted", + attempts: nextAttempts, + limit: errorRecoveryLimit, + source: "self-healing", + }); + await agentStore.updateAgentState(agent.id, "paused"); + await agentStore.updateAgent(agent.id, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON }); log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: retry budget exhausted`); continue; } @@ -10576,6 +10621,20 @@ export class SelfHealingManager { lastError: undefined, }); + if (agent.state === "error") { + const attempt = this.getDurableAgentRecoveryState(agent).attempts + 1; + await this.emitDurableAgentErrorRecoveryAudit({ + agentId: agent.id, + type: "agent:auto-recover-error-state", + attempt, + limit: errorRecoveryLimit, + source: "self-healing", + }); + if (!this.options.restartDurableAgentHeartbeat) { + log.log(`Durable-agent transient recovery heartbeat restart unavailable for ${agent.id}; state reset only`); + } + } + if (agent.state === "error" && this.options.restartDurableAgentHeartbeat) { const restartOk = await this.options.restartDurableAgentHeartbeat(agent.id, { reason: "transient-error",