From 7a4a9c82293af713a653d4657c08ac3cb8e0a2ca Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 15 Jul 2026 08:58:33 -0700 Subject: [PATCH] fix(engine): auto-recover false-positive heartbeat-model-unavailable parks Admit under-budget paused/heartbeat-model-unavailable agents to the shared heartbeatErrorRecovery budget so timer, self-healing, and startup paths retry without a manual Retry. Keep the pause reason when the budget is exhausted so operators still see credential guidance. --- docs/agents.md | 2 +- .../heartbeat-error-recovery.test.ts | 73 ++++++++ ...healing-model-unavailable-recovery.test.ts | 170 ++++++++++++++++++ .../engine/src/__tests__/self-healing.test.ts | 66 ++++++- packages/engine/src/agent-heartbeat.ts | 86 +++++++-- packages/engine/src/self-healing.ts | 82 +++++---- 6 files changed, 422 insertions(+), 57 deletions(-) create mode 100644 packages/engine/src/__tests__/self-healing-model-unavailable-recovery.test.ts diff --git a/docs/agents.md b/docs/agents.md index c48bbf314e..22f3873e69 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -379,7 +379,7 @@ Self-healing intentionally refuses to auto-restart agents when blockers are oper - **Timer trigger:** run completes and the durable agent returns to `state="active"` (recoverable soft-fail). - **Assignment / on-demand trigger:** run completes with `resultJson.actionRequired = true`, then the durable agent is paused with `pauseReason="heartbeat-model-unavailable"` and `lastError` set to actionable credential guidance (including the missing provider name when detectable). -After credentials are fixed, operators should resume the paused durable agent; subsequent heartbeats proceed normally. +False-positive `heartbeat-model-unavailable` parks (session/registry/credential-probe blips that a manual Retry would clear without config changes) are admitted to the same bounded `heartbeatErrorRecovery` budget as error-state recovery. The heartbeat timer re-arms while budget remains, the run-entry path clears the park and retries, and the self-healing sweep plus engine-startup reset are backstops. Genuine missing credentials re-park after each failed attempt and stay parked once the budget is exhausted (keeping `pauseReason="heartbeat-model-unavailable"` for operator guidance). Manual resume still works at any time. ### Assigned-agent identity + planning model precedence for task triage diff --git a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts index a3ce429c37..174c5c3c90 100644 --- a/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts +++ b/packages/engine/src/__tests__/heartbeat-error-recovery.test.ts @@ -39,11 +39,13 @@ import { HEARTBEAT_ERROR_RECOVERY_METADATA_KEY, HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON, + HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, HeartbeatMonitor, HeartbeatTriggerScheduler, incrementHeartbeatErrorRecoveryMetadata, isErrorRecoveryEligible, isHeartbeatErrorRecoverable, + isModelUnavailableParkRecoveryEligible, readHeartbeatErrorRetryCount, resetHeartbeatErrorRecoveryMetadata, resolveErrorRecoveryLimit, @@ -186,6 +188,25 @@ describe("heartbeat error-recovery primitives", () => { expect(isErrorRecoveryEligible(baseAgent({ lastError: '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}' }), 5)).toBe(false); expect(isHeartbeatErrorRecoverable({ lastError: "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/tmp/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /tmp/deleted/packages/engine/src/pi.ts" })).toBe(false); }); + + it("admits under-budget heartbeat-model-unavailable parks for auto-recovery even when lastError is operator-actionable", () => { + const parked = baseAgent({ + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + }); + expect(isModelUnavailableParkRecoveryEligible(parked, 5)).toBe(true); + expect(isErrorRecoveryEligible(parked, 5)).toBe(true); + expect(isHeartbeatErrorRecoverable(parked)).toBe(false); + expect(isErrorRecoveryEligible({ + ...parked, + metadata: buildHeartbeatErrorRecoveryMetadata(parked, 5), + }, 5)).toBe(false); + expect(isErrorRecoveryEligible({ + ...parked, + pauseReason: "manual", + }, 5)).toBe(false); + }); }); describe("HeartbeatMonitor error-state recovery", () => { @@ -219,6 +240,58 @@ describe("HeartbeatMonitor error-state recovery", () => { })); }); + it("auto-retries a false heartbeat-model-unavailable park on the next heartbeat without operator Retry", async () => { + const session = createSession(async () => undefined); + mockedCreateFnAgent.mockResolvedValueOnce(session as never); + const store = createAgentStore(baseAgent({ + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + })); + const taskStore = createNoTaskStore(); + const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); + + await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); + + expect(session.prompt).toHaveBeenCalledTimes(1); + expect(store.agent.state).toBe("active"); + expect(store.agent.lastError).toBeUndefined(); + expect(store.agent.pauseReason).toBeUndefined(); + expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: store.agent.id, + metadata: expect.objectContaining({ attempt: 1, limit: 5, source: "timer" }), + })); + }); + + it("keeps an exhausted heartbeat-model-unavailable park parked with the same pause reason", async () => { + const store = createAgentStore(baseAgent({ + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + metadata: buildHeartbeatErrorRecoveryMetadata(baseAgent(), 5), + })); + const taskStore = createNoTaskStore(); + const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() }); + + const result = await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" }); + + expect(result.status).toBe("completed"); + expect(result.resultJson).toMatchObject({ + reason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + attempts: 5, + limit: 5, + }); + expect(store.agent.state).toBe("paused"); + expect(store.agent.pauseReason).toBe(HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON); + expect(mockedCreateFnAgent).not.toHaveBeenCalled(); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:error-retry-exhausted", + target: store.agent.id, + })); + }); + it("treats a generic first-run heartbeat failure as recoverable and auto-recovers on the next heartbeat", async () => { const genericError = "Failed to start agent session: spawn ENOENT"; const firstSession = createSession(async () => { throw new Error(genericError); }); diff --git a/packages/engine/src/__tests__/self-healing-model-unavailable-recovery.test.ts b/packages/engine/src/__tests__/self-healing-model-unavailable-recovery.test.ts new file mode 100644 index 0000000000..de0c5da9d9 --- /dev/null +++ b/packages/engine/src/__tests__/self-healing-model-unavailable-recovery.test.ts @@ -0,0 +1,170 @@ +/* +FNXC:HeartbeatRecovery 2026-07-15-08:50: +Focused suite for false-positive heartbeat-model-unavailable parks. Kept out of the large +quarantined self-healing.test.ts so default vitest project covers the auto-retry contract. +*/ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { Agent, AgentStore, Settings, TaskStore } from "@fusion/core"; +import { SelfHealingManager } from "../self-healing.js"; +import { + HEARTBEAT_ERROR_RECOVERY_METADATA_KEY, + HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + readHeartbeatErrorRetryCount, +} from "../agent-heartbeat.js"; + +vi.mock("../logger.js", () => ({ + createLogger: vi.fn(() => ({ + log: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + })), + schedulerLog: { log: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +function createStatefulMockAgentStore(agents: Agent[]): AgentStore & { getAgent(id: string): Agent | undefined } { + const agentMap = new Map( + agents.map((agent) => [agent.id, { ...agent, metadata: agent.metadata ? { ...agent.metadata } : agent.metadata }]), + ); + return { + getAgent: (id: string) => agentMap.get(id), + listAgents: vi.fn().mockImplementation(async () => Array.from(agentMap.values())), + updateAgentState: vi.fn().mockImplementation(async (id: string, state: Agent["state"]) => { + const agent = agentMap.get(id); + if (agent) agentMap.set(id, { ...agent, state }); + }), + updateAgent: vi.fn().mockImplementation(async (id: string, patch: Partial) => { + const agent = agentMap.get(id); + if (agent) agentMap.set(id, { ...agent, ...patch }); + }), + } as unknown as AgentStore & { getAgent(id: string): Agent | undefined }; +} + +describe("SelfHealingManager heartbeat-model-unavailable recovery", () => { + let store: TaskStore; + + beforeEach(() => { + vi.clearAllMocks(); + store = { + getSettings: vi.fn().mockResolvedValue({ + globalPause: false, + enginePaused: false, + taskStuckTimeoutMs: 60_000, + } as unknown as Settings), + recordRunAuditEvent: vi.fn().mockResolvedValue(undefined), + listTasks: vi.fn().mockResolvedValue([]), + } as unknown as TaskStore; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("startup resets both misattributed and same-provider heartbeat-model-unavailable parks", async () => { + const now = Date.now(); + const agentStore = createStatefulMockAgentStore([ + { + id: "misattributed-heartbeat-model", + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true, modelProvider: "grok-cli", modelId: "grok-4.5", model: "grok-cli/grok-4.5" }, + updatedAt: new Date(now).toISOString(), + } as unknown as Agent, + { + id: "genuine-heartbeat-model", + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true, modelProvider: "anthropic", modelId: "claude-opus-4-8", model: "anthropic/claude-opus-4-8" }, + updatedAt: new Date(now).toISOString(), + } as unknown as Agent, + { + id: "user-paused", + state: "paused", + pauseReason: "manual", + lastError: "socket hang up", + updatedAt: new Date(now).toISOString(), + } as unknown as Agent, + ]); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const manager = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); + + const resetCount = await manager.resetDurableAgentErrorStateOnStartup(); + + expect(resetCount).toBe(2); + for (const agentId of ["misattributed-heartbeat-model", "genuine-heartbeat-model"]) { + const agent = agentStore.getAgent(agentId)!; + expect(agent.state).toBe("active"); + expect(agent.lastError).toBeUndefined(); + expect(agent.pauseReason).toBeUndefined(); + expect(readHeartbeatErrorRetryCount(agent)).toBe(0); + } + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("misattributed-heartbeat-model", { + reason: "startup-error-reset", + attempt: 1, + }); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("genuine-heartbeat-model", { + reason: "startup-error-reset", + attempt: 1, + }); + expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("user-paused", expect.anything()); + manager.stop(); + }); + + it("recoverOrphanedAgents auto-recovers a stale under-budget model-unavailable park", async () => { + const now = Date.now(); + const agentStore = createStatefulMockAgentStore([ + { + id: "false-model-park", + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true }, + metadata: {}, + updatedAt: new Date(now - 120_000).toISOString(), + } as unknown as Agent, + { + id: "fresh-model-park", + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true }, + updatedAt: new Date(now).toISOString(), + } as unknown as Agent, + { + id: "exhausted-model-park", + state: "paused", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true }, + metadata: { [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: { consecutiveAttempts: 5 } }, + updatedAt: new Date(now - 120_000).toISOString(), + } as unknown as Agent, + ]); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const manager = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); + + const result = await manager.recoverOrphanedAgents(); + + expect(result).toBe(1); + expect(agentStore.getAgent("false-model-park")?.state).toBe("active"); + expect(agentStore.getAgent("false-model-park")?.pauseReason).toBeUndefined(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: "false-model-park", + metadata: expect.objectContaining({ agentId: "false-model-park", attempt: 1, limit: 5, source: "self-healing" }), + })); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("false-model-park", { reason: "transient-error", attempt: 1 }); + expect(agentStore.getAgent("fresh-model-park")?.state).toBe("paused"); + expect(agentStore.getAgent("exhausted-model-park")?.state).toBe("paused"); + manager.stop(); + }); +}); diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index dd83a74b72..e812e81e60 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -1018,7 +1018,7 @@ describe("SelfHealingManager", () => { await managerWithAgents.runStartupRecovery(); - for (const agentId of ["fresh-error", "exhausted-parked", "misattributed-heartbeat-model"]) { + for (const agentId of ["fresh-error", "exhausted-parked", "misattributed-heartbeat-model", "genuine-heartbeat-model"]) { const agent = agentStore.getAgent(agentId)!; expect(agent.state).toBe("active"); expect(agent.lastError).toBeUndefined(); @@ -1031,24 +1031,26 @@ describe("SelfHealingManager", () => { } expect(agentStore.getAgent("fresh-error")?.metadata?.unrelated).toBe("keep"); expect(agentStore.getAgent("exhausted-parked")?.metadata?.unrelated).toBe("keep-too"); - expect(restartDurableAgentHeartbeat).toHaveBeenCalledTimes(3); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledTimes(4); expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("fresh-error", { reason: "startup-error-reset", attempt: 1 }); expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("exhausted-parked", { reason: "startup-error-reset", attempt: 1 }); expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("misattributed-heartbeat-model", { reason: "startup-error-reset", attempt: 1 }); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("genuine-heartbeat-model", { reason: "startup-error-reset", attempt: 1 }); const resetAudits = recordRunAuditEvent.mock.calls .map(([event]) => event) .filter((event) => event.mutationType === "agent:reset-error-state-on-startup"); - expect(resetAudits).toHaveLength(3); + expect(resetAudits).toHaveLength(4); expect(resetAudits).toEqual(expect.arrayContaining([ expect.objectContaining({ target: "fresh-error", metadata: expect.objectContaining({ agentId: "fresh-error", priorState: "error", source: "self-healing" }) }), expect.objectContaining({ target: "exhausted-parked", metadata: expect.objectContaining({ agentId: "exhausted-parked", priorState: "paused", priorPauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, source: "self-healing" }) }), expect.objectContaining({ target: "misattributed-heartbeat-model", metadata: expect.objectContaining({ agentId: "misattributed-heartbeat-model", priorState: "paused", priorPauseReason: "heartbeat-model-unavailable", source: "self-healing" }) }), + expect.objectContaining({ target: "genuine-heartbeat-model", metadata: expect.objectContaining({ agentId: "genuine-heartbeat-model", priorState: "paused", priorPauseReason: "heartbeat-model-unavailable", source: "self-healing" }) }), ])); expect(recordRunAuditEvent.mock.calls.map(([event]) => event.mutationType).filter((type) => type === "agent:auto-recover-error-state")).toHaveLength(0); - expect(agentStore.updateAgentState).toHaveBeenCalledTimes(3); + expect(agentStore.updateAgentState).toHaveBeenCalledTimes(4); - for (const untouchedId of ["operator-actionable", "stale-module", "error-unrecoverable", "genuine-heartbeat-model", "user-paused", "ephemeral", "disabled", "live-agent", "healthy-active", "healthy-idle"]) { + for (const untouchedId of ["operator-actionable", "stale-module", "error-unrecoverable", "user-paused", "ephemeral", "disabled", "live-agent", "healthy-active", "healthy-idle"]) { expect(agentStore.updateAgentState).not.toHaveBeenCalledWith(untouchedId, expect.anything()); expect(agentStore.updateAgent).not.toHaveBeenCalledWith(untouchedId, expect.anything()); } @@ -1280,6 +1282,60 @@ describe("SelfHealingManager", () => { managerWithAgents.stop(); }); + it("auto-recovers a stale heartbeat-model-unavailable park even when lastError looks operator-actionable", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); + const now = Date.now(); + const agentStore = createMockAgentStore([ + { + id: "false-model-park", + state: "paused", + pauseReason: "heartbeat-model-unavailable", + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true }, + metadata: {}, + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, + { + id: "fresh-model-park", + state: "paused", + pauseReason: "heartbeat-model-unavailable", + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true }, + updatedAt: new Date(now).toISOString(), + } as Agent, + { + id: "exhausted-model-park", + state: "paused", + pauseReason: "heartbeat-model-unavailable", + lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.', + runtimeConfig: { enabled: true }, + metadata: { [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: { consecutiveAttempts: 5 } }, + updatedAt: new Date(now - 120_000).toISOString(), + } as Agent, + ]); + const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true); + const managerWithAgents = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + agentStore, + restartDurableAgentHeartbeat, + }); + + const result = await managerWithAgents.recoverOrphanedAgents(); + + expect(result).toBe(1); + expect(agentStore.updateAgentState).toHaveBeenCalledWith("false-model-park", "active"); + expect(agentStore.updateAgent).toHaveBeenCalledWith("false-model-park", { lastError: undefined, pauseReason: undefined }); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "agent:auto-recover-error-state", + target: "false-model-park", + metadata: expect.objectContaining({ agentId: "false-model-park", attempt: 1, limit: 5, source: "self-healing" }), + })); + expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("false-model-park", { reason: "transient-error", attempt: 1 }); + expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("fresh-model-park", expect.anything()); + expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("exhausted-model-park", expect.anything()); + managerWithAgents.stop(); + }); + it("recovers only the eligible manager-present agent among a mixed cluster without touching healthy siblings", async () => { vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings); const now = Date.now(); diff --git a/packages/engine/src/agent-heartbeat.ts b/packages/engine/src/agent-heartbeat.ts index 4c037e9fcb..d8b4487abe 100644 --- a/packages/engine/src/agent-heartbeat.ts +++ b/packages/engine/src/agent-heartbeat.ts @@ -73,11 +73,15 @@ FN-7835 requires durable heartbeat-managed agents in error state to retry on the FNXC:HeartbeatRecovery 2026-07-11-00:00: FN-7672 requires durable agent error recovery to stay classification-gated: only transient, non-operator-actionable lastError values may be retried automatically. Credential, quota, model-access, and permanent configuration failures must remain parked for operator action instead of burning heartbeat retries. + +FNXC:HeartbeatRecovery 2026-07-15-08:50: +heartbeat-model-unavailable parks from assignment/on-demand runs were terminal until a human Retry, even when the next attempt succeeds with unchanged credentials (false "model unavailable" / registry / credential-probe blips). Admit those parks to the same bounded heartbeatErrorRecovery budget as error-state recovery so the engine auto-retries like operator Retry, while genuine missing credentials re-park after the budget exhausts. */ export const MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS = 5; export const HEARTBEAT_ERROR_RECOVERY_METADATA_KEY = "heartbeatErrorRecovery"; export const HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON = "error-retry-exhausted"; export const HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON = "error-unrecoverable"; +export const HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON = "heartbeat-model-unavailable"; import { acquireTaskWorktree } from "./worktree-acquisition.js"; import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type EngineRunContext } from "./run-audit.js"; import { promptWithFallback } from "./pi.js"; @@ -2382,35 +2386,56 @@ export class HeartbeatMonitor { return (await this.store.getRunDetail(agentId, run.id))!; } - if (agent.state === "error") { + /* + FNXC:HeartbeatRecovery 2026-07-15-08:50: + Include paused/heartbeat-model-unavailable in the same run-entry recovery gate as bare error. Assignment/on-demand model-unavailable parks previously never re-entered the timer path, so false positives stayed parked until a human Retry even though the next session start would succeed. + */ + if (agent.state === "error" || isModelUnavailablePark(agent)) { const errorRecoveryLimit = resolveErrorRecoveryLimit(heartbeatModelSettings); const currentRetryCount = readHeartbeatErrorRetryCount(agent); const canAttemptErrorRecovery = isErrorRecoveryEligible(agent, errorRecoveryLimit); const recoveryBudgetExhausted = isHeartbeatManaged(agent) && agent.runtimeConfig?.enabled !== false - && isHeartbeatErrorRecoverable(agent) - && currentRetryCount >= errorRecoveryLimit; + && currentRetryCount >= errorRecoveryLimit + && (isHeartbeatErrorRecoverable(agent) || isModelUnavailablePark(agent)); if (canAttemptErrorRecovery) { const attempt = currentRetryCount + 1; const metadata = incrementHeartbeatErrorRecoveryMetadata(agent); try { await this.store.updateAgentState(agentId, "active"); - await this.store.updateAgent(agentId, { lastError: undefined, metadata }); - heartbeatLog.log(`Agent ${agentId} auto-recovered from error state for heartbeat retry attempt ${attempt}/${errorRecoveryLimit}`); + await this.store.updateAgent(agentId, { + lastError: undefined, + pauseReason: undefined, + metadata, + }); + heartbeatLog.log(`Agent ${agentId} auto-recovered from ${agent.state === "error" ? "error state" : "heartbeat-model-unavailable park"} for heartbeat retry attempt ${attempt}/${errorRecoveryLimit}`); await audit.database({ type: "agent:auto-recover-error-state", target: agentId, metadata: { agentId, attempt, limit: errorRecoveryLimit, source }, }); - agent = (await this.store.getAgent(agentId)) ?? { ...agent, state: "active", lastError: undefined, metadata }; + agent = (await this.store.getAgent(agentId)) ?? { + ...agent, + state: "active", + lastError: undefined, + pauseReason: undefined, + metadata, + }; } catch (recoveryErr) { heartbeatLog.warn(`Agent ${agentId} error-state recovery bookkeeping failed: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)} — continuing with existing state`); } } else if (recoveryBudgetExhausted) { try { + // startRun may have flipped the agent to "running"; restore a parked terminal state. + // Exhausted model-unavailable parks keep their pause reason so the UI still + // points at credentials/model config rather than a generic retry-exhausted label. await this.store.updateAgentState(agentId, "paused"); - await this.store.updateAgent(agentId, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON }); + await this.store.updateAgent(agentId, { + pauseReason: isModelUnavailablePark(agent) + ? HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON + : HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, + }); heartbeatLog.warn(`Agent ${agentId} error recovery exhausted after ${currentRetryCount}/${errorRecoveryLimit} attempts — pausing`); await audit.database({ type: "agent:error-retry-exhausted", @@ -2422,12 +2447,19 @@ export class HeartbeatMonitor { } await this.completeRun(agentId, run.id, { status: "completed", - resultJson: { reason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, attempts: currentRetryCount, limit: errorRecoveryLimit }, + resultJson: { + reason: isModelUnavailablePark(agent) + ? HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON + : HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, + attempts: currentRetryCount, + limit: errorRecoveryLimit, + }, skipStateTransition: true, }); return (await this.store.getRunDetail(agentId, run.id))!; } else if ( - isHeartbeatManaged(agent) + agent.state === "error" + && isHeartbeatManaged(agent) && agent.runtimeConfig?.enabled !== false && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "") ) { @@ -2454,9 +2486,11 @@ export class HeartbeatMonitor { }); return (await this.store.getRunDetail(agentId, run.id))!; } else { - heartbeatLog.log(`Agent ${agentId} state is "error" but lastError is not eligible for heartbeat recovery — graceful exit`); + heartbeatLog.log(`Agent ${agentId} state is "${agent.state}" but is not eligible for heartbeat recovery — graceful exit`); try { - await this.store.updateAgentState(agentId, "error"); + if (agent.state === "error") { + await this.store.updateAgentState(agentId, "error"); + } } catch (restoreErr) { heartbeatLog.warn(`Agent ${agentId} non-recoverable error-state restore failed: ${restoreErr instanceof Error ? restoreErr.message : String(restoreErr)} — preserving run completion`); } @@ -2999,7 +3033,7 @@ export class HeartbeatMonitor { await this.store.updateAgentState(agentId, "paused"); await this.store.updateAgent(agentId, { - pauseReason: "heartbeat-model-unavailable", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, lastError: detail, }); }; @@ -3708,7 +3742,7 @@ export class HeartbeatMonitor { }); await this.store.updateAgentState(agentId, "paused"); await this.store.updateAgent(agentId, { - pauseReason: "heartbeat-model-unavailable", + pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON, lastError: detail, }); } @@ -4387,7 +4421,31 @@ export function isHeartbeatErrorRecoverable(agent: Pick): bo return !isStaleWorktreeModuleResolutionError(lastError) && !isOperatorActionableAgentError(lastError); } +export function isModelUnavailablePark(agent: Pick): boolean { + // Key on pauseReason, not only state=paused: startRun flips the agent to + // "running" before the run-entry recovery gate reads it, and a failed preload + // can load the post-startRun store row. Matching only state=paused would miss + // budgeted auto-retry for those paths. + return agent.pauseReason === HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON + && agent.state !== "active" + && agent.state !== "idle"; +} + +/* +FNXC:HeartbeatRecovery 2026-07-15-08:50: +False-positive heartbeat-model-unavailable parks must stay on the timer path with a bounded budget. Operator-actionable lastError text (no API key / registry miss) would otherwise exclude them from isHeartbeatErrorRecoverable forever, so this park reason is an explicit second recovery admission path independent of lastError classification. +*/ +export function isModelUnavailableParkRecoveryEligible(agent: Agent, limit: number): boolean { + return isModelUnavailablePark(agent) + && isHeartbeatManaged(agent) + && agent.runtimeConfig?.enabled !== false + && readHeartbeatErrorRetryCount(agent) < Math.max(1, Math.floor(limit)); +} + export function isErrorRecoveryEligible(agent: Agent, limit: number): boolean { + if (isModelUnavailableParkRecoveryEligible(agent, limit)) { + return true; + } return agent.state === "error" && isHeartbeatManaged(agent) && agent.runtimeConfig?.enabled !== false @@ -4400,7 +4458,7 @@ export function isErrorRecoveryEligible(agent: Agent, limit: number): boolean { * * Timers are armed only for durable agents where all of the following hold: * - `runtimeConfig.enabled !== false` - * - `state ∈ {active, running, idle}` or `state === "error"` with retry budget remaining + * - `state ∈ {active, running, idle}`, or `state === "error"` / `paused`+`heartbeat-model-unavailable` with retry budget remaining * * Any other state, or any ephemeral/task-worker agent, clears the timer. * State changes and heartbeat config updates are observed via AgentStore diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index a5f2651618..7f33e244c3 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -50,6 +50,8 @@ import { HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON, isHeartbeatErrorRecoverable, + isModelUnavailablePark, + isModelUnavailableParkRecoveryEligible, readHeartbeatErrorRetryCount, resetHeartbeatErrorRecoveryMetadata, resolveErrorRecoveryLimit, @@ -91,32 +93,9 @@ import type { GhostBugDecision } from "./triage-preflight.js"; import { DependencyBlockedTodoReporter } from "./dependency-blocked-todo-reporter.js"; import { filterPathsByIgnoreList, getUnmetSchedulingDependencies, isCoordinationOnlyTask, pathsOverlap, shouldHoldActiveFileScopeLease } from "./scheduler.js"; import { evaluateParkedAgentTaskLink, PARKED_AGENT_LINK_FRESH_RUN_MS } from "./task-agent-sync.js"; -import { extractRuntimeModel } from "./agent-session-helpers.js"; const log = createLogger("self-healing"); const OPTIONAL_STEP_REVISION_KEY_MARKER = "Workflow revision key:"; -const HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON = "heartbeat-model-unavailable"; - -function extractHeartbeatUnavailableProvider(error: string | undefined): string | undefined { - if (!error) return undefined; - const rawProvider = /no api key for provider:\s*([^\s)]+)/i.exec(error)?.[1] - ?? /configured primary model\s+([^/\s]+)\//i.exec(error)?.[1]; - return rawProvider?.replace(/["'.,:;]+$/g, "").trim() || undefined; -} - -function isMisattributedHeartbeatModelPark(agent: Agent): boolean { - if (agent.state !== "paused" || agent.pauseReason !== HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON) { - return false; - } - const assignedModel = extractRuntimeModel((agent.runtimeConfig ?? {}) as Record); - const failedProvider = extractHeartbeatUnavailableProvider(agent.lastError); - return Boolean( - assignedModel.provider - && assignedModel.modelId - && failedProvider - && assignedModel.provider.toLowerCase() !== failedProvider.toLowerCase(), - ); -} function normalizeOptionalStepRevisionKey(value: string | undefined): string { return (value ?? "").trim().toLowerCase(); @@ -10600,6 +10579,9 @@ export class SelfHealingManager { FNXC:AgentHeartbeat 2026-07-14-16:13: Startup must also recover a `heartbeat-model-unavailable` park when its recorded failing provider differs from the agent's complete assigned runtime model. This repairs agents falsely parked by the former shared-project-model precedence while preserving genuine assigned-provider authentication failures for operator action. + + FNXC:AgentHeartbeat 2026-07-15-08:50: + Startup now recovers every `heartbeat-model-unavailable` park (not only misattributed providers). Engine restart is treated like operator Retry: false model-unavailable/credential-probe parks clear immediately, and genuine missing credentials re-park on the next failing heartbeat. */ async resetDurableAgentErrorStateOnStartup(): Promise { const agentStore = this.options.agentStore; @@ -10613,8 +10595,8 @@ export class SelfHealingManager { for (const agent of allAgents) { const isErrorRetryExhaustedPark = agent.state === "paused" && agent.pauseReason === HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON; - const isMisattributedModelPark = isMisattributedHeartbeatModelPark(agent); - if (agent.state !== "error" && !isErrorRetryExhaustedPark && !isMisattributedModelPark) { + const isModelUnavailableParked = isModelUnavailablePark(agent); + if (agent.state !== "error" && !isErrorRetryExhaustedPark && !isModelUnavailableParked) { continue; } if (isEphemeralAgent(agent)) { @@ -10627,7 +10609,7 @@ export class SelfHealingManager { if (this.options.hasActiveAgentExecution?.(agent.id) === true) { continue; } - if ((!isMisattributedModelPark && !isHeartbeatErrorRecoverable(agent)) || isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) { + if ((!isModelUnavailableParked && !isHeartbeatErrorRecoverable(agent)) || isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) { log.warn(`Startup durable-agent error reset suppressed for ${agent.id}: unrecoverable or stale-module error requires existing recovery path`); continue; } @@ -10696,6 +10678,9 @@ export class SelfHealingManager { /* FNXC:AgentHeartbeat 2026-07-12-20:10: An agent parked paused/"error-unrecoverable" whose lastError NOW classifies as recoverable (e.g. transient OAuth token-rotation 401s that were misclassified operator-actionable before isTransientAuthCredentialError existed) must not stay parked forever waiting for a human. Re-admit exactly those parked agents to the error-recovery sweep; user pauses and every other pauseReason are untouched. The shared retry budget, cooldown, and staleness gates below still apply. + + FNXC:AgentHeartbeat 2026-07-15-08:50: + Also re-admit stale paused/heartbeat-model-unavailable parks when shared retry budget remains. Manual Retry already proves many of these are false positives; the timer path is the fast recovery, and this sweep is the backstop when timers were cleared on park. */ const isReclassifiedRecoverableParkedError = (agent: Agent): boolean => agent.state === "paused" @@ -10706,7 +10691,13 @@ export class SelfHealingManager { if (isEphemeralAgent(agent)) { return false; } - if (agent.state !== "running" && agent.state !== "error" && !isReclassifiedRecoverableParkedError(agent)) { + const isModelUnavailableRecoveryCandidate = isModelUnavailableParkRecoveryEligible(agent, errorRecoveryLimit); + if ( + agent.state !== "running" + && agent.state !== "error" + && !isReclassifiedRecoverableParkedError(agent) + && !isModelUnavailableRecoveryCandidate + ) { return false; } /* @@ -10740,7 +10731,11 @@ export class SelfHealingManager { return false; } - if (agent.state === "error" || isReclassifiedRecoverableParkedError(agent)) { + if ( + agent.state === "error" + || isReclassifiedRecoverableParkedError(agent) + || isModelUnavailableRecoveryCandidate + ) { const runtimeConfig = (agent.runtimeConfig ?? {}) as Record; if (runtimeConfig.enabled === false) { return false; @@ -10748,7 +10743,7 @@ export class SelfHealingManager { if (this.options.hasActiveAgentExecution?.(agent.id) === true) { return false; } - const isRecoverableHeartbeatError = isHeartbeatErrorRecoverable(agent); + const isRecoverableHeartbeatError = isHeartbeatErrorRecoverable(agent) || isModelUnavailableRecoveryCandidate; const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? ""); const isUnrecoverableHeartbeatError = !isRecoverableHeartbeatError && !isStaleMissingModule; @@ -10781,14 +10776,23 @@ export class SelfHealingManager { for (const agent of orphaned) { const updatedAt = Date.parse(agent.updatedAt ?? ""); const stuckForMs = Math.max(0, now - updatedAt); - // Reclassified "error-unrecoverable" parked agents run the same recovery - // branch as error-state agents: shared budget, cooldown, audit, restart. - const isErrorRecoveryCandidate = agent.state === "error" || isReclassifiedRecoverableParkedError(agent); + // Reclassified "error-unrecoverable" and heartbeat-model-unavailable parked + // agents run the same recovery branch as error-state agents: shared budget, + // cooldown, audit, restart. + const isErrorRecoveryCandidate = + agent.state === "error" + || isReclassifiedRecoverableParkedError(agent) + || isModelUnavailablePark(agent); try { if (isErrorRecoveryCandidate) { const recoveryState = this.getDurableAgentRecoveryState(agent); const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? ""); - const isUnrecoverableHeartbeatError = !isHeartbeatErrorRecoverable(agent) && !isStaleMissingModule; + // Model-unavailable parks are intentionally operator-actionable by lastError text + // but still budget-retryable; do not reclassify them as error-unrecoverable here. + const isUnrecoverableHeartbeatError = + !isModelUnavailablePark(agent) + && !isHeartbeatErrorRecoverable(agent) + && !isStaleMissingModule; if (isUnrecoverableHeartbeatError) { /* FNXC:AgentHeartbeat 2026-07-12-18:34: @@ -10878,8 +10882,12 @@ export class SelfHealingManager { limit: errorRecoveryLimit, source: "self-healing", }); - await agentStore.updateAgentState(agent.id, "paused"); - await agentStore.updateAgent(agent.id, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON }); + // Keep heartbeat-model-unavailable labeling when that park exhausted so + // operators still see credential/model guidance rather than a generic label. + if (!isModelUnavailablePark(agent)) { + await agentStore.updateAgentState(agent.id, "paused"); + await agentStore.updateAgent(agent.id, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON }); + } log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: retry budget exhausted`); continue; } @@ -10888,8 +10896,8 @@ export class SelfHealingManager { await agentStore.updateAgentState(agent.id, "active"); await agentStore.updateAgent(agent.id, { lastError: undefined, - // Clear the "error-unrecoverable" park marker when a reclassified - // parked agent is re-admitted; harmless no-op for error-state agents. + // Clear error-unrecoverable / heartbeat-model-unavailable park markers when + // a parked agent is re-admitted; harmless no-op for bare error-state agents. pauseReason: undefined, });