fix(engine): auto-recover false-positive heartbeat-model-unavailable parks
Admit under-budget paused/heartbeat-model-unavailable agents to the shared heartbeatErrorRecovery budget so timer, self-healing, and startup paths retry without a manual Retry. Keep the pause reason when the budget is exhausted so operators still see credential guidance.
This commit is contained in:
gsxdsm
@@ -379,7 +379,7 @@ Self-healing intentionally refuses to auto-restart agents when blockers are oper
|
||||
- **Timer trigger:** run completes and the durable agent returns to `state="active"` (recoverable soft-fail).
|
||||
- **Assignment / on-demand trigger:** run completes with `resultJson.actionRequired = true`, then the durable agent is paused with `pauseReason="heartbeat-model-unavailable"` and `lastError` set to actionable credential guidance (including the missing provider name when detectable).
|
||||
|
||||
After credentials are fixed, operators should resume the paused durable agent; subsequent heartbeats proceed normally.
|
||||
False-positive `heartbeat-model-unavailable` parks (session/registry/credential-probe blips that a manual Retry would clear without config changes) are admitted to the same bounded `heartbeatErrorRecovery` budget as error-state recovery. The heartbeat timer re-arms while budget remains, the run-entry path clears the park and retries, and the self-healing sweep plus engine-startup reset are backstops. Genuine missing credentials re-park after each failed attempt and stay parked once the budget is exhausted (keeping `pauseReason="heartbeat-model-unavailable"` for operator guidance). Manual resume still works at any time.
|
||||
|
||||
### Assigned-agent identity + planning model precedence for task triage
|
||||
|
||||
|
||||
@@ -39,11 +39,13 @@ import {
|
||||
HEARTBEAT_ERROR_RECOVERY_METADATA_KEY,
|
||||
HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON,
|
||||
HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON,
|
||||
HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
HeartbeatMonitor,
|
||||
HeartbeatTriggerScheduler,
|
||||
incrementHeartbeatErrorRecoveryMetadata,
|
||||
isErrorRecoveryEligible,
|
||||
isHeartbeatErrorRecoverable,
|
||||
isModelUnavailableParkRecoveryEligible,
|
||||
readHeartbeatErrorRetryCount,
|
||||
resetHeartbeatErrorRecoveryMetadata,
|
||||
resolveErrorRecoveryLimit,
|
||||
@@ -186,6 +188,25 @@ describe("heartbeat error-recovery primitives", () => {
|
||||
expect(isErrorRecoveryEligible(baseAgent({ lastError: '401 {"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}' }), 5)).toBe(false);
|
||||
expect(isHeartbeatErrorRecoverable({ lastError: "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/tmp/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /tmp/deleted/packages/engine/src/pi.ts" })).toBe(false);
|
||||
});
|
||||
|
||||
it("admits under-budget heartbeat-model-unavailable parks for auto-recovery even when lastError is operator-actionable", () => {
|
||||
const parked = baseAgent({
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
});
|
||||
expect(isModelUnavailableParkRecoveryEligible(parked, 5)).toBe(true);
|
||||
expect(isErrorRecoveryEligible(parked, 5)).toBe(true);
|
||||
expect(isHeartbeatErrorRecoverable(parked)).toBe(false);
|
||||
expect(isErrorRecoveryEligible({
|
||||
...parked,
|
||||
metadata: buildHeartbeatErrorRecoveryMetadata(parked, 5),
|
||||
}, 5)).toBe(false);
|
||||
expect(isErrorRecoveryEligible({
|
||||
...parked,
|
||||
pauseReason: "manual",
|
||||
}, 5)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HeartbeatMonitor error-state recovery", () => {
|
||||
@@ -219,6 +240,58 @@ describe("HeartbeatMonitor error-state recovery", () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it("auto-retries a false heartbeat-model-unavailable park on the next heartbeat without operator Retry", async () => {
|
||||
const session = createSession(async () => undefined);
|
||||
mockedCreateFnAgent.mockResolvedValueOnce(session as never);
|
||||
const store = createAgentStore(baseAgent({
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
}));
|
||||
const taskStore = createNoTaskStore();
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() });
|
||||
|
||||
await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" });
|
||||
|
||||
expect(session.prompt).toHaveBeenCalledTimes(1);
|
||||
expect(store.agent.state).toBe("active");
|
||||
expect(store.agent.lastError).toBeUndefined();
|
||||
expect(store.agent.pauseReason).toBeUndefined();
|
||||
expect(readHeartbeatErrorRetryCount(store.agent)).toBe(0);
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
mutationType: "agent:auto-recover-error-state",
|
||||
target: store.agent.id,
|
||||
metadata: expect.objectContaining({ attempt: 1, limit: 5, source: "timer" }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("keeps an exhausted heartbeat-model-unavailable park parked with the same pause reason", async () => {
|
||||
const store = createAgentStore(baseAgent({
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
metadata: buildHeartbeatErrorRecoveryMetadata(baseAgent(), 5),
|
||||
}));
|
||||
const taskStore = createNoTaskStore();
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: process.cwd() });
|
||||
|
||||
const result = await monitor.executeHeartbeat({ agentId: store.agent.id, source: "timer" });
|
||||
|
||||
expect(result.status).toBe("completed");
|
||||
expect(result.resultJson).toMatchObject({
|
||||
reason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
attempts: 5,
|
||||
limit: 5,
|
||||
});
|
||||
expect(store.agent.state).toBe("paused");
|
||||
expect(store.agent.pauseReason).toBe(HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON);
|
||||
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
mutationType: "agent:error-retry-exhausted",
|
||||
target: store.agent.id,
|
||||
}));
|
||||
});
|
||||
|
||||
it("treats a generic first-run heartbeat failure as recoverable and auto-recovers on the next heartbeat", async () => {
|
||||
const genericError = "Failed to start agent session: spawn ENOENT";
|
||||
const firstSession = createSession(async () => { throw new Error(genericError); });
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
FNXC:HeartbeatRecovery 2026-07-15-08:50:
|
||||
Focused suite for false-positive heartbeat-model-unavailable parks. Kept out of the large
|
||||
quarantined self-healing.test.ts so default vitest project covers the auto-retry contract.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import type { Agent, AgentStore, Settings, TaskStore } from "@fusion/core";
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
import {
|
||||
HEARTBEAT_ERROR_RECOVERY_METADATA_KEY,
|
||||
HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
readHeartbeatErrorRetryCount,
|
||||
} from "../agent-heartbeat.js";
|
||||
|
||||
vi.mock("../logger.js", () => ({
|
||||
createLogger: vi.fn(() => ({
|
||||
log: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
})),
|
||||
schedulerLog: { log: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
function createStatefulMockAgentStore(agents: Agent[]): AgentStore & { getAgent(id: string): Agent | undefined } {
|
||||
const agentMap = new Map<string, Agent>(
|
||||
agents.map((agent) => [agent.id, { ...agent, metadata: agent.metadata ? { ...agent.metadata } : agent.metadata }]),
|
||||
);
|
||||
return {
|
||||
getAgent: (id: string) => agentMap.get(id),
|
||||
listAgents: vi.fn().mockImplementation(async () => Array.from(agentMap.values())),
|
||||
updateAgentState: vi.fn().mockImplementation(async (id: string, state: Agent["state"]) => {
|
||||
const agent = agentMap.get(id);
|
||||
if (agent) agentMap.set(id, { ...agent, state });
|
||||
}),
|
||||
updateAgent: vi.fn().mockImplementation(async (id: string, patch: Partial<Agent>) => {
|
||||
const agent = agentMap.get(id);
|
||||
if (agent) agentMap.set(id, { ...agent, ...patch });
|
||||
}),
|
||||
} as unknown as AgentStore & { getAgent(id: string): Agent | undefined };
|
||||
}
|
||||
|
||||
describe("SelfHealingManager heartbeat-model-unavailable recovery", () => {
|
||||
let store: TaskStore;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
store = {
|
||||
getSettings: vi.fn().mockResolvedValue({
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
taskStuckTimeoutMs: 60_000,
|
||||
} as unknown as Settings),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
listTasks: vi.fn().mockResolvedValue([]),
|
||||
} as unknown as TaskStore;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("startup resets both misattributed and same-provider heartbeat-model-unavailable parks", async () => {
|
||||
const now = Date.now();
|
||||
const agentStore = createStatefulMockAgentStore([
|
||||
{
|
||||
id: "misattributed-heartbeat-model",
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true, modelProvider: "grok-cli", modelId: "grok-4.5", model: "grok-cli/grok-4.5" },
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
} as unknown as Agent,
|
||||
{
|
||||
id: "genuine-heartbeat-model",
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true, modelProvider: "anthropic", modelId: "claude-opus-4-8", model: "anthropic/claude-opus-4-8" },
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
} as unknown as Agent,
|
||||
{
|
||||
id: "user-paused",
|
||||
state: "paused",
|
||||
pauseReason: "manual",
|
||||
lastError: "socket hang up",
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
} as unknown as Agent,
|
||||
]);
|
||||
const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
agentStore,
|
||||
restartDurableAgentHeartbeat,
|
||||
});
|
||||
|
||||
const resetCount = await manager.resetDurableAgentErrorStateOnStartup();
|
||||
|
||||
expect(resetCount).toBe(2);
|
||||
for (const agentId of ["misattributed-heartbeat-model", "genuine-heartbeat-model"]) {
|
||||
const agent = agentStore.getAgent(agentId)!;
|
||||
expect(agent.state).toBe("active");
|
||||
expect(agent.lastError).toBeUndefined();
|
||||
expect(agent.pauseReason).toBeUndefined();
|
||||
expect(readHeartbeatErrorRetryCount(agent)).toBe(0);
|
||||
}
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("misattributed-heartbeat-model", {
|
||||
reason: "startup-error-reset",
|
||||
attempt: 1,
|
||||
});
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("genuine-heartbeat-model", {
|
||||
reason: "startup-error-reset",
|
||||
attempt: 1,
|
||||
});
|
||||
expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("user-paused", expect.anything());
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("recoverOrphanedAgents auto-recovers a stale under-budget model-unavailable park", async () => {
|
||||
const now = Date.now();
|
||||
const agentStore = createStatefulMockAgentStore([
|
||||
{
|
||||
id: "false-model-park",
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true },
|
||||
metadata: {},
|
||||
updatedAt: new Date(now - 120_000).toISOString(),
|
||||
} as unknown as Agent,
|
||||
{
|
||||
id: "fresh-model-park",
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true },
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
} as unknown as Agent,
|
||||
{
|
||||
id: "exhausted-model-park",
|
||||
state: "paused",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true },
|
||||
metadata: { [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: { consecutiveAttempts: 5 } },
|
||||
updatedAt: new Date(now - 120_000).toISOString(),
|
||||
} as unknown as Agent,
|
||||
]);
|
||||
const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
agentStore,
|
||||
restartDurableAgentHeartbeat,
|
||||
});
|
||||
|
||||
const result = await manager.recoverOrphanedAgents();
|
||||
|
||||
expect(result).toBe(1);
|
||||
expect(agentStore.getAgent("false-model-park")?.state).toBe("active");
|
||||
expect(agentStore.getAgent("false-model-park")?.pauseReason).toBeUndefined();
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
mutationType: "agent:auto-recover-error-state",
|
||||
target: "false-model-park",
|
||||
metadata: expect.objectContaining({ agentId: "false-model-park", attempt: 1, limit: 5, source: "self-healing" }),
|
||||
}));
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("false-model-park", { reason: "transient-error", attempt: 1 });
|
||||
expect(agentStore.getAgent("fresh-model-park")?.state).toBe("paused");
|
||||
expect(agentStore.getAgent("exhausted-model-park")?.state).toBe("paused");
|
||||
manager.stop();
|
||||
});
|
||||
});
|
||||
@@ -1018,7 +1018,7 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
await managerWithAgents.runStartupRecovery();
|
||||
|
||||
for (const agentId of ["fresh-error", "exhausted-parked", "misattributed-heartbeat-model"]) {
|
||||
for (const agentId of ["fresh-error", "exhausted-parked", "misattributed-heartbeat-model", "genuine-heartbeat-model"]) {
|
||||
const agent = agentStore.getAgent(agentId)!;
|
||||
expect(agent.state).toBe("active");
|
||||
expect(agent.lastError).toBeUndefined();
|
||||
@@ -1031,24 +1031,26 @@ describe("SelfHealingManager", () => {
|
||||
}
|
||||
expect(agentStore.getAgent("fresh-error")?.metadata?.unrelated).toBe("keep");
|
||||
expect(agentStore.getAgent("exhausted-parked")?.metadata?.unrelated).toBe("keep-too");
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledTimes(3);
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledTimes(4);
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("fresh-error", { reason: "startup-error-reset", attempt: 1 });
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("exhausted-parked", { reason: "startup-error-reset", attempt: 1 });
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("misattributed-heartbeat-model", { reason: "startup-error-reset", attempt: 1 });
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("genuine-heartbeat-model", { reason: "startup-error-reset", attempt: 1 });
|
||||
|
||||
const resetAudits = recordRunAuditEvent.mock.calls
|
||||
.map(([event]) => event)
|
||||
.filter((event) => event.mutationType === "agent:reset-error-state-on-startup");
|
||||
expect(resetAudits).toHaveLength(3);
|
||||
expect(resetAudits).toHaveLength(4);
|
||||
expect(resetAudits).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ target: "fresh-error", metadata: expect.objectContaining({ agentId: "fresh-error", priorState: "error", source: "self-healing" }) }),
|
||||
expect.objectContaining({ target: "exhausted-parked", metadata: expect.objectContaining({ agentId: "exhausted-parked", priorState: "paused", priorPauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, source: "self-healing" }) }),
|
||||
expect.objectContaining({ target: "misattributed-heartbeat-model", metadata: expect.objectContaining({ agentId: "misattributed-heartbeat-model", priorState: "paused", priorPauseReason: "heartbeat-model-unavailable", source: "self-healing" }) }),
|
||||
expect.objectContaining({ target: "genuine-heartbeat-model", metadata: expect.objectContaining({ agentId: "genuine-heartbeat-model", priorState: "paused", priorPauseReason: "heartbeat-model-unavailable", source: "self-healing" }) }),
|
||||
]));
|
||||
expect(recordRunAuditEvent.mock.calls.map(([event]) => event.mutationType).filter((type) => type === "agent:auto-recover-error-state")).toHaveLength(0);
|
||||
expect(agentStore.updateAgentState).toHaveBeenCalledTimes(3);
|
||||
expect(agentStore.updateAgentState).toHaveBeenCalledTimes(4);
|
||||
|
||||
for (const untouchedId of ["operator-actionable", "stale-module", "error-unrecoverable", "genuine-heartbeat-model", "user-paused", "ephemeral", "disabled", "live-agent", "healthy-active", "healthy-idle"]) {
|
||||
for (const untouchedId of ["operator-actionable", "stale-module", "error-unrecoverable", "user-paused", "ephemeral", "disabled", "live-agent", "healthy-active", "healthy-idle"]) {
|
||||
expect(agentStore.updateAgentState).not.toHaveBeenCalledWith(untouchedId, expect.anything());
|
||||
expect(agentStore.updateAgent).not.toHaveBeenCalledWith(untouchedId, expect.anything());
|
||||
}
|
||||
@@ -1280,6 +1282,60 @@ describe("SelfHealingManager", () => {
|
||||
managerWithAgents.stop();
|
||||
});
|
||||
|
||||
it("auto-recovers a stale heartbeat-model-unavailable park even when lastError looks operator-actionable", async () => {
|
||||
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
||||
const now = Date.now();
|
||||
const agentStore = createMockAgentStore([
|
||||
{
|
||||
id: "false-model-park",
|
||||
state: "paused",
|
||||
pauseReason: "heartbeat-model-unavailable",
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true },
|
||||
metadata: {},
|
||||
updatedAt: new Date(now - 120_000).toISOString(),
|
||||
} as Agent,
|
||||
{
|
||||
id: "fresh-model-park",
|
||||
state: "paused",
|
||||
pauseReason: "heartbeat-model-unavailable",
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true },
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
} as Agent,
|
||||
{
|
||||
id: "exhausted-model-park",
|
||||
state: "paused",
|
||||
pauseReason: "heartbeat-model-unavailable",
|
||||
lastError: 'No API key for provider: anthropic. Configure credentials for provider "anthropic" in settings, then resume the agent.',
|
||||
runtimeConfig: { enabled: true },
|
||||
metadata: { [HEARTBEAT_ERROR_RECOVERY_METADATA_KEY]: { consecutiveAttempts: 5 } },
|
||||
updatedAt: new Date(now - 120_000).toISOString(),
|
||||
} as Agent,
|
||||
]);
|
||||
const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true);
|
||||
const managerWithAgents = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
agentStore,
|
||||
restartDurableAgentHeartbeat,
|
||||
});
|
||||
|
||||
const result = await managerWithAgents.recoverOrphanedAgents();
|
||||
|
||||
expect(result).toBe(1);
|
||||
expect(agentStore.updateAgentState).toHaveBeenCalledWith("false-model-park", "active");
|
||||
expect(agentStore.updateAgent).toHaveBeenCalledWith("false-model-park", { lastError: undefined, pauseReason: undefined });
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
mutationType: "agent:auto-recover-error-state",
|
||||
target: "false-model-park",
|
||||
metadata: expect.objectContaining({ agentId: "false-model-park", attempt: 1, limit: 5, source: "self-healing" }),
|
||||
}));
|
||||
expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("false-model-park", { reason: "transient-error", attempt: 1 });
|
||||
expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("fresh-model-park", expect.anything());
|
||||
expect(agentStore.updateAgentState).not.toHaveBeenCalledWith("exhausted-model-park", expect.anything());
|
||||
managerWithAgents.stop();
|
||||
});
|
||||
|
||||
it("recovers only the eligible manager-present agent among a mixed cluster without touching healthy siblings", async () => {
|
||||
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
||||
const now = Date.now();
|
||||
|
||||
@@ -73,11 +73,15 @@ FN-7835 requires durable heartbeat-managed agents in error state to retry on the
|
||||
|
||||
FNXC:HeartbeatRecovery 2026-07-11-00:00:
|
||||
FN-7672 requires durable agent error recovery to stay classification-gated: only transient, non-operator-actionable lastError values may be retried automatically. Credential, quota, model-access, and permanent configuration failures must remain parked for operator action instead of burning heartbeat retries.
|
||||
|
||||
FNXC:HeartbeatRecovery 2026-07-15-08:50:
|
||||
heartbeat-model-unavailable parks from assignment/on-demand runs were terminal until a human Retry, even when the next attempt succeeds with unchanged credentials (false "model unavailable" / registry / credential-probe blips). Admit those parks to the same bounded heartbeatErrorRecovery budget as error-state recovery so the engine auto-retries like operator Retry, while genuine missing credentials re-park after the budget exhausts.
|
||||
*/
|
||||
export const MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS = 5;
|
||||
export const HEARTBEAT_ERROR_RECOVERY_METADATA_KEY = "heartbeatErrorRecovery";
|
||||
export const HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON = "error-retry-exhausted";
|
||||
export const HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON = "error-unrecoverable";
|
||||
export const HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON = "heartbeat-model-unavailable";
|
||||
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||
import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type EngineRunContext } from "./run-audit.js";
|
||||
import { promptWithFallback } from "./pi.js";
|
||||
@@ -2382,35 +2386,56 @@ export class HeartbeatMonitor {
|
||||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||
}
|
||||
|
||||
if (agent.state === "error") {
|
||||
/*
|
||||
FNXC:HeartbeatRecovery 2026-07-15-08:50:
|
||||
Include paused/heartbeat-model-unavailable in the same run-entry recovery gate as bare error. Assignment/on-demand model-unavailable parks previously never re-entered the timer path, so false positives stayed parked until a human Retry even though the next session start would succeed.
|
||||
*/
|
||||
if (agent.state === "error" || isModelUnavailablePark(agent)) {
|
||||
const errorRecoveryLimit = resolveErrorRecoveryLimit(heartbeatModelSettings);
|
||||
const currentRetryCount = readHeartbeatErrorRetryCount(agent);
|
||||
const canAttemptErrorRecovery = isErrorRecoveryEligible(agent, errorRecoveryLimit);
|
||||
const recoveryBudgetExhausted = isHeartbeatManaged(agent)
|
||||
&& agent.runtimeConfig?.enabled !== false
|
||||
&& isHeartbeatErrorRecoverable(agent)
|
||||
&& currentRetryCount >= errorRecoveryLimit;
|
||||
&& currentRetryCount >= errorRecoveryLimit
|
||||
&& (isHeartbeatErrorRecoverable(agent) || isModelUnavailablePark(agent));
|
||||
|
||||
if (canAttemptErrorRecovery) {
|
||||
const attempt = currentRetryCount + 1;
|
||||
const metadata = incrementHeartbeatErrorRecoveryMetadata(agent);
|
||||
try {
|
||||
await this.store.updateAgentState(agentId, "active");
|
||||
await this.store.updateAgent(agentId, { lastError: undefined, metadata });
|
||||
heartbeatLog.log(`Agent ${agentId} auto-recovered from error state for heartbeat retry attempt ${attempt}/${errorRecoveryLimit}`);
|
||||
await this.store.updateAgent(agentId, {
|
||||
lastError: undefined,
|
||||
pauseReason: undefined,
|
||||
metadata,
|
||||
});
|
||||
heartbeatLog.log(`Agent ${agentId} auto-recovered from ${agent.state === "error" ? "error state" : "heartbeat-model-unavailable park"} for heartbeat retry attempt ${attempt}/${errorRecoveryLimit}`);
|
||||
await audit.database({
|
||||
type: "agent:auto-recover-error-state",
|
||||
target: agentId,
|
||||
metadata: { agentId, attempt, limit: errorRecoveryLimit, source },
|
||||
});
|
||||
agent = (await this.store.getAgent(agentId)) ?? { ...agent, state: "active", lastError: undefined, metadata };
|
||||
agent = (await this.store.getAgent(agentId)) ?? {
|
||||
...agent,
|
||||
state: "active",
|
||||
lastError: undefined,
|
||||
pauseReason: undefined,
|
||||
metadata,
|
||||
};
|
||||
} catch (recoveryErr) {
|
||||
heartbeatLog.warn(`Agent ${agentId} error-state recovery bookkeeping failed: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)} — continuing with existing state`);
|
||||
}
|
||||
} else if (recoveryBudgetExhausted) {
|
||||
try {
|
||||
// startRun may have flipped the agent to "running"; restore a parked terminal state.
|
||||
// Exhausted model-unavailable parks keep their pause reason so the UI still
|
||||
// points at credentials/model config rather than a generic retry-exhausted label.
|
||||
await this.store.updateAgentState(agentId, "paused");
|
||||
await this.store.updateAgent(agentId, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON });
|
||||
await this.store.updateAgent(agentId, {
|
||||
pauseReason: isModelUnavailablePark(agent)
|
||||
? HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON
|
||||
: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON,
|
||||
});
|
||||
heartbeatLog.warn(`Agent ${agentId} error recovery exhausted after ${currentRetryCount}/${errorRecoveryLimit} attempts — pausing`);
|
||||
await audit.database({
|
||||
type: "agent:error-retry-exhausted",
|
||||
@@ -2422,12 +2447,19 @@ export class HeartbeatMonitor {
|
||||
}
|
||||
await this.completeRun(agentId, run.id, {
|
||||
status: "completed",
|
||||
resultJson: { reason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON, attempts: currentRetryCount, limit: errorRecoveryLimit },
|
||||
resultJson: {
|
||||
reason: isModelUnavailablePark(agent)
|
||||
? HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON
|
||||
: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON,
|
||||
attempts: currentRetryCount,
|
||||
limit: errorRecoveryLimit,
|
||||
},
|
||||
skipStateTransition: true,
|
||||
});
|
||||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||
} else if (
|
||||
isHeartbeatManaged(agent)
|
||||
agent.state === "error"
|
||||
&& isHeartbeatManaged(agent)
|
||||
&& agent.runtimeConfig?.enabled !== false
|
||||
&& !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")
|
||||
) {
|
||||
@@ -2454,9 +2486,11 @@ export class HeartbeatMonitor {
|
||||
});
|
||||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||
} else {
|
||||
heartbeatLog.log(`Agent ${agentId} state is "error" but lastError is not eligible for heartbeat recovery — graceful exit`);
|
||||
heartbeatLog.log(`Agent ${agentId} state is "${agent.state}" but is not eligible for heartbeat recovery — graceful exit`);
|
||||
try {
|
||||
await this.store.updateAgentState(agentId, "error");
|
||||
if (agent.state === "error") {
|
||||
await this.store.updateAgentState(agentId, "error");
|
||||
}
|
||||
} catch (restoreErr) {
|
||||
heartbeatLog.warn(`Agent ${agentId} non-recoverable error-state restore failed: ${restoreErr instanceof Error ? restoreErr.message : String(restoreErr)} — preserving run completion`);
|
||||
}
|
||||
@@ -2999,7 +3033,7 @@ export class HeartbeatMonitor {
|
||||
|
||||
await this.store.updateAgentState(agentId, "paused");
|
||||
await this.store.updateAgent(agentId, {
|
||||
pauseReason: "heartbeat-model-unavailable",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: detail,
|
||||
});
|
||||
};
|
||||
@@ -3708,7 +3742,7 @@ export class HeartbeatMonitor {
|
||||
});
|
||||
await this.store.updateAgentState(agentId, "paused");
|
||||
await this.store.updateAgent(agentId, {
|
||||
pauseReason: "heartbeat-model-unavailable",
|
||||
pauseReason: HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON,
|
||||
lastError: detail,
|
||||
});
|
||||
}
|
||||
@@ -4387,7 +4421,31 @@ export function isHeartbeatErrorRecoverable(agent: Pick<Agent, "lastError">): bo
|
||||
return !isStaleWorktreeModuleResolutionError(lastError) && !isOperatorActionableAgentError(lastError);
|
||||
}
|
||||
|
||||
export function isModelUnavailablePark(agent: Pick<Agent, "state" | "pauseReason">): boolean {
|
||||
// Key on pauseReason, not only state=paused: startRun flips the agent to
|
||||
// "running" before the run-entry recovery gate reads it, and a failed preload
|
||||
// can load the post-startRun store row. Matching only state=paused would miss
|
||||
// budgeted auto-retry for those paths.
|
||||
return agent.pauseReason === HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON
|
||||
&& agent.state !== "active"
|
||||
&& agent.state !== "idle";
|
||||
}
|
||||
|
||||
/*
|
||||
FNXC:HeartbeatRecovery 2026-07-15-08:50:
|
||||
False-positive heartbeat-model-unavailable parks must stay on the timer path with a bounded budget. Operator-actionable lastError text (no API key / registry miss) would otherwise exclude them from isHeartbeatErrorRecoverable forever, so this park reason is an explicit second recovery admission path independent of lastError classification.
|
||||
*/
|
||||
export function isModelUnavailableParkRecoveryEligible(agent: Agent, limit: number): boolean {
|
||||
return isModelUnavailablePark(agent)
|
||||
&& isHeartbeatManaged(agent)
|
||||
&& agent.runtimeConfig?.enabled !== false
|
||||
&& readHeartbeatErrorRetryCount(agent) < Math.max(1, Math.floor(limit));
|
||||
}
|
||||
|
||||
export function isErrorRecoveryEligible(agent: Agent, limit: number): boolean {
|
||||
if (isModelUnavailableParkRecoveryEligible(agent, limit)) {
|
||||
return true;
|
||||
}
|
||||
return agent.state === "error"
|
||||
&& isHeartbeatManaged(agent)
|
||||
&& agent.runtimeConfig?.enabled !== false
|
||||
@@ -4400,7 +4458,7 @@ export function isErrorRecoveryEligible(agent: Agent, limit: number): boolean {
|
||||
*
|
||||
* Timers are armed only for durable agents where all of the following hold:
|
||||
* - `runtimeConfig.enabled !== false`
|
||||
* - `state ∈ {active, running, idle}` or `state === "error"` with retry budget remaining
|
||||
* - `state ∈ {active, running, idle}`, or `state === "error"` / `paused`+`heartbeat-model-unavailable` with retry budget remaining
|
||||
*
|
||||
* Any other state, or any ephemeral/task-worker agent, clears the timer.
|
||||
* State changes and heartbeat config updates are observed via AgentStore
|
||||
|
||||
@@ -50,6 +50,8 @@ import {
|
||||
HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON,
|
||||
HEARTBEAT_ERROR_UNRECOVERABLE_PAUSE_REASON,
|
||||
isHeartbeatErrorRecoverable,
|
||||
isModelUnavailablePark,
|
||||
isModelUnavailableParkRecoveryEligible,
|
||||
readHeartbeatErrorRetryCount,
|
||||
resetHeartbeatErrorRecoveryMetadata,
|
||||
resolveErrorRecoveryLimit,
|
||||
@@ -91,32 +93,9 @@ import type { GhostBugDecision } from "./triage-preflight.js";
|
||||
import { DependencyBlockedTodoReporter } from "./dependency-blocked-todo-reporter.js";
|
||||
import { filterPathsByIgnoreList, getUnmetSchedulingDependencies, isCoordinationOnlyTask, pathsOverlap, shouldHoldActiveFileScopeLease } from "./scheduler.js";
|
||||
import { evaluateParkedAgentTaskLink, PARKED_AGENT_LINK_FRESH_RUN_MS } from "./task-agent-sync.js";
|
||||
import { extractRuntimeModel } from "./agent-session-helpers.js";
|
||||
|
||||
const log = createLogger("self-healing");
|
||||
const OPTIONAL_STEP_REVISION_KEY_MARKER = "Workflow revision key:";
|
||||
const HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON = "heartbeat-model-unavailable";
|
||||
|
||||
function extractHeartbeatUnavailableProvider(error: string | undefined): string | undefined {
|
||||
if (!error) return undefined;
|
||||
const rawProvider = /no api key for provider:\s*([^\s)]+)/i.exec(error)?.[1]
|
||||
?? /configured primary model\s+([^/\s]+)\//i.exec(error)?.[1];
|
||||
return rawProvider?.replace(/["'.,:;]+$/g, "").trim() || undefined;
|
||||
}
|
||||
|
||||
function isMisattributedHeartbeatModelPark(agent: Agent): boolean {
|
||||
if (agent.state !== "paused" || agent.pauseReason !== HEARTBEAT_MODEL_UNAVAILABLE_PAUSE_REASON) {
|
||||
return false;
|
||||
}
|
||||
const assignedModel = extractRuntimeModel((agent.runtimeConfig ?? {}) as Record<string, unknown>);
|
||||
const failedProvider = extractHeartbeatUnavailableProvider(agent.lastError);
|
||||
return Boolean(
|
||||
assignedModel.provider
|
||||
&& assignedModel.modelId
|
||||
&& failedProvider
|
||||
&& assignedModel.provider.toLowerCase() !== failedProvider.toLowerCase(),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeOptionalStepRevisionKey(value: string | undefined): string {
|
||||
return (value ?? "").trim().toLowerCase();
|
||||
@@ -10600,6 +10579,9 @@ export class SelfHealingManager {
|
||||
|
||||
FNXC:AgentHeartbeat 2026-07-14-16:13:
|
||||
Startup must also recover a `heartbeat-model-unavailable` park when its recorded failing provider differs from the agent's complete assigned runtime model. This repairs agents falsely parked by the former shared-project-model precedence while preserving genuine assigned-provider authentication failures for operator action.
|
||||
|
||||
FNXC:AgentHeartbeat 2026-07-15-08:50:
|
||||
Startup now recovers every `heartbeat-model-unavailable` park (not only misattributed providers). Engine restart is treated like operator Retry: false model-unavailable/credential-probe parks clear immediately, and genuine missing credentials re-park on the next failing heartbeat.
|
||||
*/
|
||||
async resetDurableAgentErrorStateOnStartup(): Promise<number> {
|
||||
const agentStore = this.options.agentStore;
|
||||
@@ -10613,8 +10595,8 @@ export class SelfHealingManager {
|
||||
for (const agent of allAgents) {
|
||||
const isErrorRetryExhaustedPark =
|
||||
agent.state === "paused" && agent.pauseReason === HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON;
|
||||
const isMisattributedModelPark = isMisattributedHeartbeatModelPark(agent);
|
||||
if (agent.state !== "error" && !isErrorRetryExhaustedPark && !isMisattributedModelPark) {
|
||||
const isModelUnavailableParked = isModelUnavailablePark(agent);
|
||||
if (agent.state !== "error" && !isErrorRetryExhaustedPark && !isModelUnavailableParked) {
|
||||
continue;
|
||||
}
|
||||
if (isEphemeralAgent(agent)) {
|
||||
@@ -10627,7 +10609,7 @@ export class SelfHealingManager {
|
||||
if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
|
||||
continue;
|
||||
}
|
||||
if ((!isMisattributedModelPark && !isHeartbeatErrorRecoverable(agent)) || isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) {
|
||||
if ((!isModelUnavailableParked && !isHeartbeatErrorRecoverable(agent)) || isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) {
|
||||
log.warn(`Startup durable-agent error reset suppressed for ${agent.id}: unrecoverable or stale-module error requires existing recovery path`);
|
||||
continue;
|
||||
}
|
||||
@@ -10696,6 +10678,9 @@ export class SelfHealingManager {
|
||||
/*
|
||||
FNXC:AgentHeartbeat 2026-07-12-20:10:
|
||||
An agent parked paused/"error-unrecoverable" whose lastError NOW classifies as recoverable (e.g. transient OAuth token-rotation 401s that were misclassified operator-actionable before isTransientAuthCredentialError existed) must not stay parked forever waiting for a human. Re-admit exactly those parked agents to the error-recovery sweep; user pauses and every other pauseReason are untouched. The shared retry budget, cooldown, and staleness gates below still apply.
|
||||
|
||||
FNXC:AgentHeartbeat 2026-07-15-08:50:
|
||||
Also re-admit stale paused/heartbeat-model-unavailable parks when shared retry budget remains. Manual Retry already proves many of these are false positives; the timer path is the fast recovery, and this sweep is the backstop when timers were cleared on park.
|
||||
*/
|
||||
const isReclassifiedRecoverableParkedError = (agent: Agent): boolean =>
|
||||
agent.state === "paused"
|
||||
@@ -10706,7 +10691,13 @@ export class SelfHealingManager {
|
||||
if (isEphemeralAgent(agent)) {
|
||||
return false;
|
||||
}
|
||||
if (agent.state !== "running" && agent.state !== "error" && !isReclassifiedRecoverableParkedError(agent)) {
|
||||
const isModelUnavailableRecoveryCandidate = isModelUnavailableParkRecoveryEligible(agent, errorRecoveryLimit);
|
||||
if (
|
||||
agent.state !== "running"
|
||||
&& agent.state !== "error"
|
||||
&& !isReclassifiedRecoverableParkedError(agent)
|
||||
&& !isModelUnavailableRecoveryCandidate
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
/*
|
||||
@@ -10740,7 +10731,11 @@ export class SelfHealingManager {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (agent.state === "error" || isReclassifiedRecoverableParkedError(agent)) {
|
||||
if (
|
||||
agent.state === "error"
|
||||
|| isReclassifiedRecoverableParkedError(agent)
|
||||
|| isModelUnavailableRecoveryCandidate
|
||||
) {
|
||||
const runtimeConfig = (agent.runtimeConfig ?? {}) as Record<string, unknown>;
|
||||
if (runtimeConfig.enabled === false) {
|
||||
return false;
|
||||
@@ -10748,7 +10743,7 @@ export class SelfHealingManager {
|
||||
if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
|
||||
return false;
|
||||
}
|
||||
const isRecoverableHeartbeatError = isHeartbeatErrorRecoverable(agent);
|
||||
const isRecoverableHeartbeatError = isHeartbeatErrorRecoverable(agent) || isModelUnavailableRecoveryCandidate;
|
||||
const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? "");
|
||||
const isUnrecoverableHeartbeatError = !isRecoverableHeartbeatError && !isStaleMissingModule;
|
||||
|
||||
@@ -10781,14 +10776,23 @@ export class SelfHealingManager {
|
||||
for (const agent of orphaned) {
|
||||
const updatedAt = Date.parse(agent.updatedAt ?? "");
|
||||
const stuckForMs = Math.max(0, now - updatedAt);
|
||||
// Reclassified "error-unrecoverable" parked agents run the same recovery
|
||||
// branch as error-state agents: shared budget, cooldown, audit, restart.
|
||||
const isErrorRecoveryCandidate = agent.state === "error" || isReclassifiedRecoverableParkedError(agent);
|
||||
// Reclassified "error-unrecoverable" and heartbeat-model-unavailable parked
|
||||
// agents run the same recovery branch as error-state agents: shared budget,
|
||||
// cooldown, audit, restart.
|
||||
const isErrorRecoveryCandidate =
|
||||
agent.state === "error"
|
||||
|| isReclassifiedRecoverableParkedError(agent)
|
||||
|| isModelUnavailablePark(agent);
|
||||
try {
|
||||
if (isErrorRecoveryCandidate) {
|
||||
const recoveryState = this.getDurableAgentRecoveryState(agent);
|
||||
const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? "");
|
||||
const isUnrecoverableHeartbeatError = !isHeartbeatErrorRecoverable(agent) && !isStaleMissingModule;
|
||||
// Model-unavailable parks are intentionally operator-actionable by lastError text
|
||||
// but still budget-retryable; do not reclassify them as error-unrecoverable here.
|
||||
const isUnrecoverableHeartbeatError =
|
||||
!isModelUnavailablePark(agent)
|
||||
&& !isHeartbeatErrorRecoverable(agent)
|
||||
&& !isStaleMissingModule;
|
||||
if (isUnrecoverableHeartbeatError) {
|
||||
/*
|
||||
FNXC:AgentHeartbeat 2026-07-12-18:34:
|
||||
@@ -10878,8 +10882,12 @@ export class SelfHealingManager {
|
||||
limit: errorRecoveryLimit,
|
||||
source: "self-healing",
|
||||
});
|
||||
await agentStore.updateAgentState(agent.id, "paused");
|
||||
await agentStore.updateAgent(agent.id, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON });
|
||||
// Keep heartbeat-model-unavailable labeling when that park exhausted so
|
||||
// operators still see credential/model guidance rather than a generic label.
|
||||
if (!isModelUnavailablePark(agent)) {
|
||||
await agentStore.updateAgentState(agent.id, "paused");
|
||||
await agentStore.updateAgent(agent.id, { pauseReason: HEARTBEAT_ERROR_RETRY_EXHAUSTED_PAUSE_REASON });
|
||||
}
|
||||
log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: retry budget exhausted`);
|
||||
continue;
|
||||
}
|
||||
@@ -10888,8 +10896,8 @@ export class SelfHealingManager {
|
||||
await agentStore.updateAgentState(agent.id, "active");
|
||||
await agentStore.updateAgent(agent.id, {
|
||||
lastError: undefined,
|
||||
// Clear the "error-unrecoverable" park marker when a reclassified
|
||||
// parked agent is re-admitted; harmless no-op for error-state agents.
|
||||
// Clear error-unrecoverable / heartbeat-model-unavailable park markers when
|
||||
// a parked agent is re-admitted; harmless no-op for bare error-state agents.
|
||||
pauseReason: undefined,
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user