From 504dc69f02308ea33254c0ed8d60ab08cfd94f8d Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 12 Jul 2026 16:25:48 -0700 Subject: [PATCH] FN-7878: default heartbeat error recovery to recoverable for generic durable-agent failures Durable agents were parking as error-unrecoverable on any non-transient-pattern failure, even generic/unknown blips that manual Retry immediately fixed; this changes the default to recoverable and reserves immediate unrecoverable parking for operator-actionable errors. - isHeartbeatErrorRecoverable now returns true unless the error is operator-actionable (auth/model/billing/scope) or a stale worktree/module-resolution error, instead of requiring a transient-pattern match via classifyError - Add OAuth scope-requirement and insufficient-scope patterns to the operator-actionable error detector so those still park immediately - Update heartbeat-error-recovery, heartbeat-executor, self-healing, and transient-error-detector tests to cover the new default-recoverable behavior - Update AGENTS.md and docs/architecture.md durable-agent error recovery notes to describe the new recoverable-by-default policy - Add changeset documenting the fix Files changed: .changeset/fn-7878-recoverable-default.md | 7 ++ AGENTS.md | 2 +- docs/architecture.md | 4 +- .../src/__tests__/heartbeat-error-recovery.test.ts | 90 +++++++++++++++++++--- .../src/__tests__/heartbeat-executor.test.ts | 17 ++-- packages/engine/src/__tests__/self-healing.test.ts | 45 ++++++----- .../src/__tests__/transient-error-detector.test.ts | 7 +- packages/engine/src/agent-heartbeat.ts | 8 +- packages/engine/src/transient-error-detector.ts | 2 + 9 files changed, 137 insertions(+), 45 deletions(-) Fusion-Task-Id: FN-7878 Fusion-Task-Lineage: 6f929af9-ceef-404f-95c9-98f26478f020 Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-7878-recoverable-default.md | 7 ++ AGENTS.md | 2 +- docs/architecture.md | 4 +- .../heartbeat-error-recovery.test.ts | 90 ++++++++++++++++--- .../src/__tests__/heartbeat-executor.test.ts | 17 ++-- .../engine/src/__tests__/self-healing.test.ts | 45 +++++----- .../transient-error-detector.test.ts | 7 +- packages/engine/src/agent-heartbeat.ts | 8 +- .../engine/src/transient-error-detector.ts | 2 + 9 files changed, 137 insertions(+), 45 deletions(-) create mode 100644 .changeset/fn-7878-recoverable-default.md diff --git a/.changeset/fn-7878-recoverable-default.md b/.changeset/fn-7878-recoverable-default.md new file mode 100644 index 0000000000..dd0eeca4e2 --- /dev/null +++ b/.changeset/fn-7878-recoverable-default.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Durable agents retry generic heartbeat failures instead of parking as unrecoverable on first error. +category: fix +dev: `isHeartbeatErrorRecoverable` now gates on operator-actionable and stale-module errors rather than requiring a transient-pattern match. diff --git a/AGENTS.md b/AGENTS.md index 592cbe738d..ad78d662fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,7 +224,7 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme - FN-7158: agent performance reflections emit `reflection:generated`, `reflection:skipped`, and `reflection:failed` with ids/counts/outcomes-only metadata; never persist reflection prose or prompt text in run-audit. - FN-7528: a deterministic, non-LLM post-task performance capture (`AgentReflectionService.captureTaskPerformance`) runs once per completed task and emits `reflection:captured` with ids/counts/outcomes-only metadata (`retryReworkCount?`, `filesTouchedCount?`, `packagesTouchedCount?`, `verificationFileScoped?`, `durationMs?`); never persists `verificationScopeReason` free-text or summary prose in run-audit. - FN-7787: `createResolvedAgentSession` enriches `session:runtime-resolved` with `noModelResolved: true` and `runtimeBuiltInFallbackModel` when a non-mock/non-test session reaches runtime creation without a complete provider/model pair; this is a visibility signal for runtime built-in fallback usage, not a fabricated model-resolution verdict. -- FN-7835/FN-7844/FN-7859: durable-agent error-state recovery emits `agent:auto-recover-error-state` when either the heartbeat timer or the self-healing sweep clears a transient, non-operator-actionable `error` and retries; metadata stays ids/counts/outcomes-only (`agentId`, attempt, limit, source), where `source` is `timer`/`automation`/`self-healing`. Both entry paths share the `heartbeatErrorRecovery` budget (self-healing keeps `durableErrorRecovery` only for cooldown/stale-path bookkeeping) and emit `agent:error-retry-exhausted` when the shared budget is exhausted and the agent is parked `paused` with `pauseReason:"error-retry-exhausted"`. Non-recoverable durable heartbeat errors (permanent/usage-limit/operator-actionable, excluding stale worktree/module-resolution suppression and disabled/ephemeral/actively-executing agents) emit `agent:error-parked-unrecoverable` with ids/counts/outcomes-only metadata (`agentId`, `source`, optional `attempts`, `limit`) and park `paused` with `pauseReason:"error-unrecoverable"` for human repair. +- FN-7835/FN-7844/FN-7859/FN-7878: durable-agent error-state recovery emits `agent:auto-recover-error-state` when either the heartbeat timer or the self-healing sweep clears a recoverable, non-operator-actionable `error` and retries; metadata stays ids/counts/outcomes-only (`agentId`, attempt, limit, source), where `source` is `timer`/`automation`/`self-healing`. Generic/unknown heartbeat failures are recoverable by default because manual Retry often proves they were transient; both entry paths share the `heartbeatErrorRecovery` budget (self-healing keeps `durableErrorRecovery` only for cooldown/stale-path bookkeeping) and emit `agent:error-retry-exhausted` when the shared budget is exhausted and the agent is parked `paused` with `pauseReason:"error-retry-exhausted"`. Only operator-actionable durable heartbeat errors (credentials/OAuth scope, model access, billing/quota, excluding transient auth rotation), plus stale worktree/module-resolution errors handled by their dedicated suppression path, skip the retry budget and emit `agent:error-parked-unrecoverable` with ids/counts/outcomes-only metadata (`agentId`, `source`, optional `attempts`, `limit`) before parking `paused` with `pauseReason:"error-unrecoverable"` for human repair. - FN-7802: self-healing emits `task:reconcile-missing-worktree-merge-active` when it proves an `in-review` merge-active task (`merging`/`merging-pr`/`merging-fix`) is stranded by an unusable-worktree session-start failure, clears stale `worktree`/`branch`/`sessionFile`, resets the worktree-session retry budget, increments `recoveryRetryCount` as the bounded stale-metadata clear counter, and requeues to `todo`; it emits `task:reconcile-missing-worktree-merge-active-no-action` when `autoMerge:false`, workspace-task ownership, or triple-proof blocks the backward move. - FN-7863: executor emits `task:execution-dispatch-loop-terminalized` when an execute-node self-requeue loop reaches `MAX_EXECUTE_REQUEUE_LOOP_CYCLES` with an unchanged progress signature; metadata stays ids/counts/outcomes-only (`taskId`, `cycleCount`, `maxCycles`, `progressSignature`, `failureValue`) and the task is visibly failed with `EXECUTION_DISPATCH_LOOP_EXHAUSTED:` while preserving worktree/branch/step progress. - FN-7011: self-healing emits `task:reconcile-engine-downtime-active-timing` when startup recovery shifts active task segment anchors to exclude proven engine-process downtime, and `task:reconcile-engine-downtime-active-timing-no-action` when no active task qualifies. diff --git a/docs/architecture.md b/docs/architecture.md index b015628a46..d9441f618b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -676,7 +676,7 @@ Runtime action-gate flow (v1): - `StuckTaskDetector` (`stuck-task-detector.ts`) — inactivity/loop stall detection - `GridlockDetector` (`gridlock-detector.ts`) — detects all-blocked todo pipelines and emits notification events (plus explicit clear signals when gridlock resolves) - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification -- Durable agent error recovery (FN-7835/FN-7844/FN-7859): a heartbeat-managed, runtime-enabled non-ephemeral agent that lands in `state:"error"` remains timer-eligible and clears `lastError` by transitioning `error → active` at the next heartbeat run entry when `lastError` is transient and non-operator-actionable. Recovery is bounded by one shared `heartbeatErrorRecovery` attempt budget (`MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS`, settings-overridable through the engine's optional cast-based knob) across both the timer path and `SelfHealingManager.recoverOrphanedAgents()`. Self-healing is the stale-agent backstop and still stores `durableErrorRecovery` cooldown/stale-module metadata, but it writes/reads the shared heartbeat counter and emits the same `agent:auto-recover-error-state` / `agent:error-retry-exhausted` audit surface with `source:"self-healing"`. The sweep flips `error → active` before `restartDurableAgentHeartbeat()` calls `executeHeartbeat()`, preventing run-entry recovery from re-counting or double-emitting for the same recovery. Success resets the shared counter and clears legacy sweep retry state; budget exhaustion parks the agent `paused` with `pauseReason:"error-retry-exhausted"`. Non-recoverable durable heartbeat errors are not restarted; both timer and sweep paths park them `paused` with `pauseReason:"error-unrecoverable"` and emit `agent:error-parked-unrecoverable`, while preserving exclusions for disabled runtime agents, ephemeral agents, active executions, and stale worktree/module-resolution suppression. +- Durable agent error recovery (FN-7835/FN-7844/FN-7859/FN-7878): a heartbeat-managed, runtime-enabled non-ephemeral agent that lands in `state:"error"` remains timer-eligible and clears `lastError` by transitioning `error → active` at the next heartbeat run entry when `lastError` is recoverable. Generic/unknown errors are recoverable by default; immediate `error-unrecoverable` parking is reserved for operator-actionable auth/model/billing/quota failures, while stale worktree/module-resolution errors stay on their dedicated self-healing suppression/rebuild path. Recovery is bounded by one shared `heartbeatErrorRecovery` attempt budget (`MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS`, settings-overridable through the engine's optional cast-based knob) across both the timer path and `SelfHealingManager.recoverOrphanedAgents()`. Self-healing is the stale-agent backstop and still stores `durableErrorRecovery` cooldown/stale-module metadata, but it writes/reads the shared heartbeat counter and emits the same `agent:auto-recover-error-state` / `agent:error-retry-exhausted` audit surface with `source:"self-healing"`. The sweep flips `error → active` before `restartDurableAgentHeartbeat()` calls `executeHeartbeat()`, preventing run-entry recovery from re-counting or double-emitting for the same recovery. Success resets the shared counter and clears legacy sweep retry state; budget exhaustion parks the agent `paused` with `pauseReason:"error-retry-exhausted"`. Non-recoverable durable heartbeat errors are not restarted; both timer and sweep paths park them `paused` with `pauseReason:"error-unrecoverable"` and emit `agent:error-parked-unrecoverable`, while preserving exclusions for disabled runtime agents, ephemeral agents, active executions, and stale worktree/module-resolution suppression. - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - Batch 1 maintenance now includes `reconcile-orphaned-task-dirs` (FN-6783), a paused-safe housekeeping step that calls `TaskStore.reconcileOrphanedTaskDirs()` so valid live `.fusion/tasks/{ID}/task.json` records missing from the SQLite index become visible without waiting for process restart. The store-level guard skips any ID already present in active, soft-deleted, archived, or tombstoned storage and emits `task:reconcile-orphaned-task-dir` only for recovered rows. - Batch 1 maintenance also includes `reconcile-phantom-committed-reservations` (FN-7069), which calls `TaskStore.reconcilePhantomCommittedReservations()` for committed task-ID reservations that have no live/soft-deleted/archived task row and no `.fusion/tasks/{ID}/task.json`. The sweep prunes orphaned `activityLog` rows and `agents`/cascaded `agentRuns`, preserves `runAuditEvents`, and keeps the reservation `committed` per FN-5105 so the ID is permanently reserved rather than resurrected or handed out again. @@ -2162,7 +2162,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f - **Completion fan-out is synchronous**: `SelfHealingManager.reconcileCompletedTask()` runs on `in-review → done`. Downstream stale `blockedBy` links and residual `fusion/` branch/worktree artifacts are reconciled immediately, not on a periodic sweep. - **In-review stall deadlock**: identical stalls (same code + reason) repeated past `inReviewStallDeadlockThreshold` (default 3) auto-pause with `pausedReason: "in-review-stall-deadlock"` and `status: "failed"`. User-initiated retry paths (dashboard retry, `fn_task_retry`, and CLI `task retry`) clear that automatic deadlock pause so the retry can execute, but they never override explicit/manual pauses or unrelated automatic pause reasons. - **Restart recovery**: `RestartRecoveryCoordinator` classifies interrupted `in-progress` runs. Unusable-worktree session-start failures (`missing`, `incomplete`, `unregistered git worktree`) are recoverable; retries are capped at `MAX_WORKTREE_SESSION_RETRIES=3` before escalating. `recoverMissingWorktreeReviewFailures` also owns durable unusable-worktree session-start failures that reached `in-review` with a merge-active status (`merging`, `merging-pr`, or `merging-fix`): before interrupted/deadlocked merge sweeps can re-drive the same phantom path, it applies auto-merge eligibility, workspace-task exclusion, and triple-proof, clears stale `worktree`/`branch`/`sessionFile`, resets the exhausted worktree-session retry budget, increments `recoveryRetryCount` as the bounded merge-active stale-metadata clear counter, and requeues to `todo` preserving progress. Operator retry surfaces (`fn_task_retry`, CLI `task retry`, dashboard retry) have the same signature-only reset primitive so a missing-worktree failure does not require a valid `merging` transition. -- **Durable agent heartbeat error recovery (FN-7835/FN-7859)**: `HeartbeatTriggerScheduler` keeps timers armed for durable heartbeat-managed agents in `state:"error"` only when `lastError` classifies as transient and is not operator-actionable (credential, quota, model-access, or permanent configuration failures do not restart). `HeartbeatMonitor.executeHeartbeat()` clears recoverable errors at run entry (`error → active`, clears `lastError`), increments a consecutive `metadata.heartbeatErrorRecovery` counter, and emits `agent:auto-recover-error-state`; success resets the counter. Once the bounded budget (`MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS`, settings-overridable) is exhausted, the agent is parked `paused` with `pauseReason:"error-retry-exhausted"` and `agent:error-retry-exhausted` is emitted. Non-recoverable durable heartbeat errors are parked `paused` with `pauseReason:"error-unrecoverable"` and `agent:error-parked-unrecoverable` rather than remaining indefinitely in bare `error`. +- **Durable agent heartbeat error recovery (FN-7835/FN-7859/FN-7878)**: `HeartbeatTriggerScheduler` keeps timers armed for durable heartbeat-managed agents in `state:"error"` when `lastError` is recoverable: generic/unknown errors and transient credential-rotation failures get the bounded retry budget, while operator-actionable credential/scope, quota/billing, and model-access failures do not restart. Stale worktree/module-resolution errors also skip naive retry recovery so the dedicated stale-host/worktree suppression path can handle them. `HeartbeatMonitor.executeHeartbeat()` clears recoverable errors at run entry (`error → active`, clears `lastError`), increments a consecutive `metadata.heartbeatErrorRecovery` counter, and emits `agent:auto-recover-error-state`; success resets the counter. Once the bounded budget (`MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS`, settings-overridable) is exhausted, the agent is parked `paused` with `pauseReason:"error-retry-exhausted"` and `agent:error-retry-exhausted` is emitted. Non-recoverable durable heartbeat errors are parked `paused` with `pauseReason:"error-unrecoverable"` and `agent:error-parked-unrecoverable` rather than remaining indefinitely in bare `error`. - **Executor pre-session liveness gate (FN-4935/FN-6861)**: the gate now skips for fresh acquisitions (`acquisition.source === "fresh"`), emits structured `not_usable_task_worktree:` diagnostics (including canonicalized registered-path snapshots) and a `worktree:incomplete-detected` audit event with `source: "executor-liveness-gate"`, while preserving the existing `taskDoneRetryCount` / `MAX_TASK_DONE_REQUEUE_RETRIES` requeue contract. The project repo root is never a usable task worktree even though it is a legitimately registered Git worktree; `classifyTaskWorktree` returns `repo-root` for canonical root-equal paths, and resume acquisition treats that as self-healable stale metadata by clearing `task.worktree` and creating a fresh checkout under the configured worktrees directory. FN-5772 adds a bounded nested-root self-heal: when `task.worktree` points at a strict descendant of a registered worktree root inside the configured worktrees dir, executor re-anchors `task.worktree` to the git top-level, emits `worktree:reanchored` (`fromPath`, `toPath`, `source`), and proceeds; repo-root/outside-dir/unregistered top-level mismatches still fail. FN-4651 `worktreeSessionRetryCount` remains scoped to the in-review/session-start recovery path. - **Stale self-owned active-session reconcile on conflict cleanup (FN-4973)**: when executor worktree-conflict cleanup finds only a same-task stale `activeSessionRegistry` entry and no live in-memory `activeWorktrees` binding for that task/path, it must unregister the stale entry before `removeWorktree` (plus one-shot backstop reconcile on same-task `ActiveSessionWorktreeRemovalError` races). Foreign-task entries remain protected by FN-4811 and must never be reconciled by the requesting task. - **Same-task stale removal canonical helper (FN-5346)**: executor same-task cleanup paths now route pre-removal reconciliation through `reconcileSelfOwnedActiveSessionForRemoval` (via executor helper wiring), so stale self-owned `activeSessionRegistry` residues are cleared only when no live in-memory binding exists, while FN-4811 foreign-owner refusals and live-owner protections remain intact. diff --git a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts index f3808c8376..a3ce429c37 100644 --- a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts +++ b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts @@ -176,7 +176,11 @@ describe("heartbeat error-recovery primitives", () => { expect(isErrorRecoveryEligible(baseAgent({ metadata: { agentKind: "task-worker" }, lastError: "socket hang up" }), 5)).toBe(false); expect(isErrorRecoveryEligible(baseAgent({ metadata: buildHeartbeatErrorRecoveryMetadata(baseAgent(), 5), lastError: "socket hang up" }), 5)).toBe(false); expect(isErrorRecoveryEligible(baseAgent({ lastError: "invalid api key" }), 5)).toBe(false); - expect(isErrorRecoveryEligible(baseAgent({ lastError: "SyntaxError: Unexpected token" }), 5)).toBe(false); + expect(isErrorRecoveryEligible(baseAgent({ lastError: "model gpt-x not found" }), 5)).toBe(false); + expect(isErrorRecoveryEligible(baseAgent({ lastError: "quota exceeded" }), 5)).toBe(false); + expect(isErrorRecoveryEligible(baseAgent({ lastError: "billing issue: payment required" }), 5)).toBe(false); + expect(isErrorRecoveryEligible(baseAgent({ lastError: "SyntaxError: Unexpected token" }), 5)).toBe(true); + expect(isErrorRecoveryEligible(baseAgent({ lastError: "" }), 5)).toBe(true); // OAuth token-rotation 401s are transient credential rotations, not operator problems. expect(isErrorRecoveryEligible(baseAgent({ lastError: 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}' }), 5)).toBe(true); expect(isErrorRecoveryEligible(baseAgent({ lastError: '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}' }), 5)).toBe(false); @@ -215,10 +219,48 @@ describe("HeartbeatMonitor error-state recovery", () => { })); }); - it("parks an operator-actionable error state instead of auto-recovering or stranding it", async () => { + it("treats a generic first-run heartbeat failure as recoverable and auto-recovers on the next heartbeat", async () => { + const genericError = "Failed to start agent session: spawn ENOENT"; + const firstSession = createSession(async () => { throw new Error(genericError); }); + const secondSession = createSession(async () => undefined); + mockedCreateFnAgent.mockResolvedValueOnce(firstSession as never).mockResolvedValueOnce(secondSession as never); + const store = createAgentStore(baseAgent({ state: "active", lastError: undefined })); + const taskStore = createNoTaskStore(); + const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); + + await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); + + expect(store.agent.state).toBe("error"); + expect(store.agent.lastError).toContain(genericError); + expect(store.agent.pauseReason).toBeUndefined(); + expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); + expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-parked-unrecoverable", + })); + + await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); + + expect(secondSession.prompt).toHaveBeenCalledTimes(1); + expect(store.agent.state).toBe("active"); + expect(store.agent.lastError).toBeUndefined(); + expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: store.agent.id, + metadata: expect.objectContaining({ attempt: 1, limit: 5, source: "timer" }), + })); + }); + + it.each([ + "invalid api key", + '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}', + "model gpt-x not found", + "quota exceeded", + "billing issue: payment method required", + ])("parks an operator-actionable error state instead of auto-recovering or stranding it: %s", async (lastError) => { const session = createSession(async () => undefined); mockedCreateFnAgent.mockResolvedValueOnce(session as never); - const store = createAgentStore(baseAgent({ lastError: "invalid api key" })); + const store = createAgentStore(baseAgent({ lastError })); const taskStore = createNoTaskStore(); const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); @@ -226,7 +268,7 @@ describe("HeartbeatMonitor error-state recovery", () => { expect(session.prompt).not.toHaveBeenCalled(); expect(store.agent.state).toBe("paused"); - expect(store.agent.lastError).toBe("invalid api key"); + expect(store.agent.lastError).toBe(lastError); expect(store.agent.pauseReason).toBe(HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON); expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ @@ -239,9 +281,13 @@ describe("HeartbeatMonitor error-state recovery", () => { })); }); - it("parks a first-run operator-actionable failure immediately with an explicit reason", async () => { + it.each([ + ["invalid api key", "invalid api key"], + ["OAuth scope", '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}'], + ["model not found", "model gpt-x not found"], + ])("parks a first-run operator-actionable failure immediately with an explicit reason: %s", async (_name, errorMessage) => { mockedCreateFnAgent.mockResolvedValueOnce(createSession(async () => { - throw new Error('401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}'); + throw new Error(errorMessage); }) as never); const store = createAgentStore(baseAgent({ state: "active", lastError: undefined })); const taskStore = createNoTaskStore(); @@ -250,7 +296,7 @@ describe("HeartbeatMonitor error-state recovery", () => { await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); expect(store.agent.state).toBe("paused"); - expect(store.agent.lastError).toContain("OAuth token does not meet scope requirements"); + expect(store.agent.lastError).toContain(errorMessage); expect(store.agent.pauseReason).toBe(HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON); expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ @@ -348,11 +394,11 @@ describe("HeartbeatMonitor error-state recovery", () => { })); }); - it("parks the agent paused after the bounded recovery budget is exhausted", async () => { + it("parks the agent paused as retry-exhausted after a persistent generic error consumes the bounded recovery budget", async () => { mockedCreateFnAgent - .mockResolvedValueOnce(createSession(async () => { throw new Error("socket hang up 1"); }) as never) - .mockResolvedValueOnce(createSession(async () => { throw new Error("socket hang up 2"); }) as never); - const store = createAgentStore(baseAgent({ metadata: {}, lastError: "socket hang up" })); + .mockResolvedValueOnce(createSession(async () => { throw new Error("Unexpected end of JSON input 1"); }) as never) + .mockResolvedValueOnce(createSession(async () => { throw new Error("Unexpected end of JSON input 2"); }) as never); + const store = createAgentStore(baseAgent({ metadata: {}, lastError: "Unexpected end of JSON input" })); const taskStore = createNoTaskStore({ heartbeatErrorRecoveryAttempts: 2 }); const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); @@ -431,6 +477,28 @@ describe("HeartbeatMonitor error-state recovery", () => { } }); + it("suppresses stale worktree module-resolution errors instead of naively retrying or parking unrecoverable", async () => { + const staleError = "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/tmp/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /tmp/deleted/packages/engine/src/pi.ts"; + const session = createSession(async () => undefined); + mockedCreateFnAgent.mockResolvedValueOnce(session as never); + const store = createAgentStore(baseAgent({ lastError: staleError })); + const taskStore = createNoTaskStore(); + const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); + + await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); + + expect(session.prompt).not.toHaveBeenCalled(); + expect(store.agent.state).toBe("error"); + expect(store.agent.pauseReason).toBeUndefined(); + expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); + expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + })); + expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-parked-unrecoverable", + })); + }); + it("resets the recovery budget after a successful run", async () => { const session = createSession(async () => undefined); mockedCreateFnAgent.mockResolvedValueOnce(session as never); diff --git a/packages/engine/src/__tests__/heartbeat-executor.test.ts b/packages/engine/src/__tests__/heartbeat-executor.test.ts index fb8aeb3776..346b3b1630 100644 --- a/packages/engine/src/__tests__/heartbeat-executor.test.ts +++ b/packages/engine/src/__tests__/heartbeat-executor.test.ts @@ -884,10 +884,10 @@ describe("executeHeartbeat", () => { expect(result).toBeDefined(); expect(result.status).toBe("completed"); - // FNXC:HeartbeatTests 2026-07-12-FN7835: FN-7835 reclassified the error-state-not-recovery-eligible resultJson reason from "invalid_state" to "error-unrecoverable" (HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON) so parked-unrecoverable agents are distinguishable from other invalid states. The paused-state case above still uses "invalid_state". - expect(result.resultJson).toEqual({ reason: "error-unrecoverable", recoveryEligible: false, state: "error" }); + // FNXC:HeartbeatTests 2026-07-12-16:10: FN-7878 makes absent/generic durable-agent lastError recoverable, but this executor-harness agent is not recovery-eligible; it should remain a normal invalid-state exit instead of fabricating an unrecoverable park. + expect(result.resultJson).toEqual({ reason: "invalid_state", state: "error" }); expect(mockedCreateFnAgent).not.toHaveBeenCalled(); - expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "active"); + expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "active"); }); it("keeps terminated as a run status while pausing the agent", async () => { @@ -920,9 +920,9 @@ describe("executeHeartbeat", () => { status: "completed", }); - // FNXC:HeartbeatTests 2026-07-12-FN7835: FN-7835/FN-7859 park non-recoverable heartbeat failures as "paused" (pauseReason: error-unrecoverable) instead of bare "error". The subsequent successful run still clears the stale lastError and returns to active. - expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "paused"); - expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "error"); + // FNXC:HeartbeatTests 2026-07-12-16:10: FN-7878 changed generic heartbeat failures such as "Prompt failed" from immediate `error-unrecoverable` parking to bare `error` so the bounded retry budget can run. The subsequent successful run still clears the stale lastError and returns to active. + expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "error"); + expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "paused"); expect(store.updateAgent).toHaveBeenCalledWith("agent-001", expect.objectContaining({ lastError: "Prompt failed" })); expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "active"); // FNXC:HeartbeatTests 2026-07-12-10:10: FN-7835's success path also resets error-recovery metadata alongside lastError, so use objectContaining to tolerate the extra metadata key. @@ -3695,8 +3695,9 @@ describe("executeHeartbeat", () => { expect(result).toBeDefined(); expect(result.status).toBe("failed"); expect(result.stderrExcerpt).toContain("Model unavailable"); - // FNXC:HeartbeatTests 2026-07-12-FN7835: FN-7835/FN-7859 park non-recoverable failures (e.g. "Model unavailable") as "paused" with pauseReason error-unrecoverable instead of bare "error". - expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "paused"); + // FNXC:HeartbeatTests 2026-07-12-16:10: FN-7878 treats generic session startup failures such as "Model unavailable" as recoverable unless an operator-actionable auth/model/billing signal is present. + expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "error"); + expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "paused"); }); it("fails soft on timer heartbeat when model provider credentials are unavailable", async () => { diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 6c6c28afbd..a4a37643fd 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -921,25 +921,31 @@ describe("SelfHealingManager", () => { expect(result).toBe(0); }); - it("parks a manager-present error-state agent whose lastError is absent/non-transient", async () => { + it("recovers a manager-present error-state agent whose lastError is absent", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); const agentStore = createMockAgentStore([ { id: "manager-1", state: "active", updatedAt: new Date(now).toISOString() } as Agent, - { id: "report-1", state: "error", reportsTo: "manager-1", updatedAt: new Date(now - 120_000).toISOString() } as Agent, + { id: "report-1", state: "error", reportsTo: "manager-1", updatedAt: new Date(now - 120_000).toISOString(), metadata: {} } as Agent, ]); - const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); const result = await managerWithAgents.recoverOrphanedAgents(); - // No lastError at all classifies as "permanent" (default). FN-7859 parks - // this terminal non-recoverable shape instead of leaving it in bare error. expect(result).toBe(1); - expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-1", "paused"); - expect(agentStore.updateAgent).toHaveBeenCalledWith( - "report-1", - expect.objectContaining({ pauseReason: HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON }), - ); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("report-1", "active"); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("report-1", { lastError: undefined, pauseReason: undefined }); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: "report-1", + metadata: expect.objectContaining({ agentId: "report-1", attempt: 1, limit: 5, source: "self-healing" }), + })); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("report-1", { reason: "transient-error", attempt: 1 }); managerWithAgents.stop(); }); @@ -955,7 +961,7 @@ describe("SelfHealingManager", () => { * the manager-present path is now considered (subject to all existing * guards, unweakened). */ - it("recovers a transient error-state agent even when its manager is present and active", async () => { + it("recovers a generic error-state agent even when its manager is present and active", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); const agentStore = createMockAgentStore([ @@ -964,7 +970,7 @@ describe("SelfHealingManager", () => { id: "report-1", state: "error", reportsTo: "manager-1", - lastError: "socket hang up", + lastError: "Unexpected end of JSON input", metadata: {}, updatedAt: new Date(now - 120_000).toISOString(), } as Agent, @@ -1074,11 +1080,10 @@ describe("SelfHealingManager", () => { const now = Date.now(); const agentStore = createMockAgentStore([ { - id: "parked-rotation", + id: "parked-generic", state: "paused", pauseReason: HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON, - lastError: - 'Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"},"request_id":"req_011CcxRi9mwx1NrZmX9qN7p2"}', + lastError: "Failed to start agent session: spawn ENOENT", metadata: {}, updatedAt: new Date(now - 120_000).toISOString(), } as Agent, @@ -1110,14 +1115,14 @@ describe("SelfHealingManager", () => { expect(result).toBe(1); expect(agentStore.updateAgentState).toHaveBeenCalledTimes(1); - expect(agentStore.updateAgentState).toHaveBeenCalledWith("parked-rotation", "active"); - expect(agentStore.updateAgent).toHaveBeenLastCalledWith("parked-rotation", { lastError: undefined, pauseReason: undefined }); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("parked-generic", "active"); + expect(agentStore.updateAgent).toHaveBeenLastCalledWith("parked-generic", { lastError: undefined, pauseReason: undefined }); expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "agent:auto-recover-error-state", - target: "parked-rotation", - metadata: expect.objectContaining({ agentId: "parked-rotation", attempt: 1, limit: 5, source: "self-healing" }), + target: "parked-generic", + metadata: expect.objectContaining({ agentId: "parked-generic", attempt: 1, limit: 5, source: "self-healing" }), })); - expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("parked-rotation", { reason: "transient-error", attempt: 1 }); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("parked-generic", { reason: "transient-error", attempt: 1 }); expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("parked-scope", expect.anything()); expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("parked-budget", expect.anything()); managerWithAgents.stop(); diff --git a/packages/engine/src/__tests__/transient-error-detector.test.ts b/packages/engine/src/__tests__/transient-error-detector.test.ts index 2d3d63e67f..21087c813c 100644 --- a/packages/engine/src/__tests__/transient-error-detector.test.ts +++ b/packages/engine/src/__tests__/transient-error-detector.test.ts @@ -386,6 +386,8 @@ describe("Transient Error Detector", () => { expect(isOperatorActionableAgentError("model gpt-x not found")).toBe(true); expect(isOperatorActionableAgentError("missing OPENAI_API_KEY")).toBe(true); expect(isOperatorActionableAgentError("billing issue: quota exceeded")).toBe(true); + expect(isOperatorActionableAgentError("OAuth token does not meet scope requirements")).toBe(true); + expect(isOperatorActionableAgentError("insufficient_scope: missing repo grant")).toBe(true); expect( isOperatorActionableAgentError( 'Error: 404 {"type":"error","error":{"type":"not_found_error","message":"Not found"},"request_id":"req_011CcawcZ3Ra9CennJXM8oWC"}', @@ -410,9 +412,11 @@ describe("Transient Error Detector", () => { expect(isOperatorActionableAgentError("400 invalid_request_error: missing required field messages")).toBe(false); }); - it("returns false for transient network errors", () => { + it("returns false for transient and generic retryable errors", () => { expect(isOperatorActionableAgentError("socket hang up")).toBe(false); expect(isOperatorActionableAgentError("upstream connect error")).toBe(false); + expect(isOperatorActionableAgentError("Failed to start agent session: spawn ENOENT")).toBe(false); + expect(isOperatorActionableAgentError("Unexpected end of JSON input")).toBe(false); }); }); @@ -449,6 +453,7 @@ describe("Transient Error Detector", () => { expect(isTransientAuthCredentialError(scopeError)).toBe(false); expect(isTransientError(scopeError)).toBe(false); expect(classifyError(scopeError)).toBe("permanent"); + expect(isOperatorActionableAgentError(scopeError)).toBe(true); }); it("keeps API-key misconfiguration operator-actionable even inside an authentication_error envelope", () => { diff --git a/packages/engine/src/agent-heartbeat.ts b/packages/engine/src/agent-heartbeat.ts index bdb9e8811d..3ef0f2ff11 100644 --- a/packages/engine/src/agent-heartbeat.ts +++ b/packages/engine/src/agent-heartbeat.ts @@ -34,7 +34,7 @@ import { resolveHeartbeatPromptTemplate, resolveHeartbeatScopeDisciplineMode, se import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js"; import { resolveAndEmitGoalContext } from "./goal-injection-diagnostics.js"; import { createLogger, heartbeatLog, formatError } from "./logger.js"; -import { classifyError, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js"; +import { isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js"; /** * FNXC:WorktreeAcquisition 2026-07-09-00:00: @@ -4224,7 +4224,11 @@ export function resetHeartbeatErrorRecoveryMetadata(agent: { metadata?: Record): boolean { const lastError = agent.lastError ?? ""; - return classifyError(lastError) === "transient" && !isOperatorActionableAgentError(lastError); + /* + FNXC:Reliability-ErrorClassification 2026-07-12-16:09: + FN-7878: a generic durable-agent heartbeat failure that manual Retry immediately fixes is recoverable by policy, even when it does not match curated transient patterns. Give unknown/session/spawn/stream blips the bounded heartbeat retry budget and re-park persistent failures as `error-retry-exhausted`; only operator-actionable auth/model/billing errors park immediately as `error-unrecoverable`. Stale worktree module-resolution errors stay out of naive retry recovery because self-healing has a dedicated stale-host/worktree suppression path. + */ + return !isStaleWorktreeModuleResolutionError(lastError) && !isOperatorActionableAgentError(lastError); } export function isErrorRecoveryEligible(agent: Agent, limit: number): boolean { diff --git a/packages/engine/src/transient-error-detector.ts b/packages/engine/src/transient-error-detector.ts index 334efd1f7a..80a3c0433b 100644 --- a/packages/engine/src/transient-error-detector.ts +++ b/packages/engine/src/transient-error-detector.ts @@ -302,6 +302,8 @@ const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [ /unauthorized/i, /forbidden/i, /insufficient permissions?/i, + /(?:oauth token )?does not meet scope requirements?/i, + /insufficient[_\s-]?scope/i, /model .* not found/i, /unknown model/i, /no such model/i,