feat(FN-4020): surface timer repair stale health in agent health utilities

Adds health-surfacing logic for timer-triggered stale heartbeats in the engine, a new `agentHealth.tsx` dashboard utility with tests, and documentation updates for the repair behavior.

Fusion-Task-Id: FN-4020
This commit is contained in:
Fusion
2026-05-11 11:45:54 -07:00
committed by gsxdsm
parent 1cf987208c
commit a8ef54c7f9
5 changed files with 158 additions and 2 deletions

View File

@@ -510,6 +510,26 @@ describe("AgentHealthStatus reason field", () => {
expect(status.reason).toContain("threshold:");
});
it("surfaces unresponsive status when timer repair metadata marks stale and no newer heartbeat exists", () => {
const repairTime = new Date(FIXED_NOW - 2 * 60 * 1000).toISOString();
const agent = makeAgent({
state: "active",
lastHeartbeatAt: new Date(FIXED_NOW - 20 * 60 * 1000).toISOString(),
runtimeConfig: { heartbeatIntervalMs: 60 * 60 * 1000 },
metadata: {
heartbeatTimerRepair: {
repairedAt: repairTime,
staleAtRepair: true,
staleRepairReason: "No heartbeat before repair",
},
},
});
const status = getAgentHealthStatus(agent);
expect(status.label).toBe("Unresponsive");
expect(status.reason).toBe("No heartbeat before repair");
});
it("formats reason with elapsed time and threshold", () => {
const agent = makeAgent({
state: "active",

View File

@@ -106,6 +106,22 @@ function isTaskWorkerAgent(agent: AgentHealthInput): boolean {
* @param agent - The agent object (partial Agent shape is accepted)
* @returns A health status object with label, icon, color, and stateDerived metadata
*/
function getHeartbeatRepairMetadata(agent: AgentHealthInput): {
repairedAt?: string;
staleAtRepair?: boolean;
staleRepairReason?: string;
} {
const metadata = agent.metadata as Record<string, unknown> | null | undefined;
const raw = metadata?.heartbeatTimerRepair;
if (!raw || typeof raw !== "object") return {};
const value = raw as Record<string, unknown>;
return {
repairedAt: typeof value.repairedAt === "string" ? value.repairedAt : undefined,
staleAtRepair: typeof value.staleAtRepair === "boolean" ? value.staleAtRepair : undefined,
staleRepairReason: typeof value.staleRepairReason === "string" ? value.staleRepairReason : undefined,
};
}
export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus {
const { state, lastHeartbeatAt, lastError, pauseReason, runtimeConfig } = agent;
const isTaskWorker = isTaskWorkerAgent(agent);
@@ -159,6 +175,21 @@ export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus
};
}
const heartbeatRepair = getHeartbeatRepairMetadata(agent);
if (heartbeatRepair.staleAtRepair && heartbeatRepair.repairedAt) {
const repairedMs = Date.parse(heartbeatRepair.repairedAt);
const lastHeartbeatMs = Date.parse(lastHeartbeatAt);
if (Number.isFinite(repairedMs) && Number.isFinite(lastHeartbeatMs) && lastHeartbeatMs < repairedMs) {
return {
label: "Unresponsive",
icon: <Activity size={14} />,
color: "var(--state-error-text)",
stateDerived: false,
reason: heartbeatRepair.staleRepairReason ?? "Heartbeat scheduler repaired a missing timer; waiting for recovery heartbeat",
};
}
}
// Every non-task-worker agent has an effective interval — either explicitly
// configured, or the scheduler's 1h default. Compare elapsed time to that
// interval (with grace) rather than to `heartbeatTimeoutMs`, which is the