feat(FN-4020): surface timer repair stale health in agent health utilities
Adds health-surfacing logic for timer-triggered stale heartbeats in the engine, a new `agentHealth.tsx` dashboard utility with tests, and documentation updates for the repair behavior. Fusion-Task-Id: FN-4020
This commit is contained in:
@@ -510,6 +510,26 @@ describe("AgentHealthStatus reason field", () => {
|
||||
expect(status.reason).toContain("threshold:");
|
||||
});
|
||||
|
||||
it("surfaces unresponsive status when timer repair metadata marks stale and no newer heartbeat exists", () => {
|
||||
const repairTime = new Date(FIXED_NOW - 2 * 60 * 1000).toISOString();
|
||||
const agent = makeAgent({
|
||||
state: "active",
|
||||
lastHeartbeatAt: new Date(FIXED_NOW - 20 * 60 * 1000).toISOString(),
|
||||
runtimeConfig: { heartbeatIntervalMs: 60 * 60 * 1000 },
|
||||
metadata: {
|
||||
heartbeatTimerRepair: {
|
||||
repairedAt: repairTime,
|
||||
staleAtRepair: true,
|
||||
staleRepairReason: "No heartbeat before repair",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const status = getAgentHealthStatus(agent);
|
||||
expect(status.label).toBe("Unresponsive");
|
||||
expect(status.reason).toBe("No heartbeat before repair");
|
||||
});
|
||||
|
||||
it("formats reason with elapsed time and threshold", () => {
|
||||
const agent = makeAgent({
|
||||
state: "active",
|
||||
|
||||
@@ -106,6 +106,22 @@ function isTaskWorkerAgent(agent: AgentHealthInput): boolean {
|
||||
* @param agent - The agent object (partial Agent shape is accepted)
|
||||
* @returns A health status object with label, icon, color, and stateDerived metadata
|
||||
*/
|
||||
function getHeartbeatRepairMetadata(agent: AgentHealthInput): {
|
||||
repairedAt?: string;
|
||||
staleAtRepair?: boolean;
|
||||
staleRepairReason?: string;
|
||||
} {
|
||||
const metadata = agent.metadata as Record<string, unknown> | null | undefined;
|
||||
const raw = metadata?.heartbeatTimerRepair;
|
||||
if (!raw || typeof raw !== "object") return {};
|
||||
const value = raw as Record<string, unknown>;
|
||||
return {
|
||||
repairedAt: typeof value.repairedAt === "string" ? value.repairedAt : undefined,
|
||||
staleAtRepair: typeof value.staleAtRepair === "boolean" ? value.staleAtRepair : undefined,
|
||||
staleRepairReason: typeof value.staleRepairReason === "string" ? value.staleRepairReason : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus {
|
||||
const { state, lastHeartbeatAt, lastError, pauseReason, runtimeConfig } = agent;
|
||||
const isTaskWorker = isTaskWorkerAgent(agent);
|
||||
@@ -159,6 +175,21 @@ export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus
|
||||
};
|
||||
}
|
||||
|
||||
const heartbeatRepair = getHeartbeatRepairMetadata(agent);
|
||||
if (heartbeatRepair.staleAtRepair && heartbeatRepair.repairedAt) {
|
||||
const repairedMs = Date.parse(heartbeatRepair.repairedAt);
|
||||
const lastHeartbeatMs = Date.parse(lastHeartbeatAt);
|
||||
if (Number.isFinite(repairedMs) && Number.isFinite(lastHeartbeatMs) && lastHeartbeatMs < repairedMs) {
|
||||
return {
|
||||
label: "Unresponsive",
|
||||
icon: <Activity size={14} />,
|
||||
color: "var(--state-error-text)",
|
||||
stateDerived: false,
|
||||
reason: heartbeatRepair.staleRepairReason ?? "Heartbeat scheduler repaired a missing timer; waiting for recovery heartbeat",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Every non-task-worker agent has an effective interval — either explicitly
|
||||
// configured, or the scheduler's 1h default. Compare elapsed time to that
|
||||
// interval (with grace) rather than to `heartbeatTimeoutMs`, which is the
|
||||
|
||||
Reference in New Issue
Block a user