FN-8020: harden agent heartbeat health classification
Validate the four-interval heartbeat grace window and classify invalid persisted heartbeats safely. - Cover reported field heartbeat ages in dashboard and engine health checks. - Mark unparseable heartbeat timestamps as unresponsive and clamp future timestamps to fresh. - Align dashboard health documentation with the existing four-interval grace window. Files changed: .../app/utils/__tests__/agentHealth.test.tsx | 48 +++++++++++++++++++++- packages/dashboard/app/utils/agentHealth.tsx | 26 ++++++++++-- .../src/__tests__/heartbeat-executor.test.ts | 31 ++++++++++++++ 3 files changed, 100 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-8020 Fusion-Task-Lineage: 2bc0df78-d68c-489b-8bfb-9b09da10cdfa Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
@@ -272,6 +272,34 @@ describe("getAgentHealthStatus", () => {
|
||||
expect(getAgentHealthStatus(agent).label).toBe("Unresponsive");
|
||||
});
|
||||
|
||||
it("reproduces the FN-8018 field ages against the default 4h grace boundary", () => {
|
||||
const fieldAgents = [
|
||||
{ name: "Backend Engineer", ageMs: 6 * 3_600_000 + 32 * 60_000 },
|
||||
{ name: "Frontend Engineer", ageMs: 5 * 3_600_000 + 59 * 60_000 },
|
||||
{ name: "Technical Writer", ageMs: 6 * 3_600_000 + 34 * 60_000 },
|
||||
];
|
||||
|
||||
for (const fieldAgent of fieldAgents) {
|
||||
expect(getAgentHealthStatus(makeAgent({
|
||||
name: fieldAgent.name,
|
||||
state: "active",
|
||||
lastHeartbeatAt: new Date(FIXED_NOW - fieldAgent.ageMs).toISOString(),
|
||||
runtimeConfig: {},
|
||||
})).label).toBe("Unresponsive");
|
||||
}
|
||||
|
||||
expect(getAgentHealthStatus(makeAgent({
|
||||
state: "active",
|
||||
lastHeartbeatAt: new Date(FIXED_NOW - 4 * 3_600_000).toISOString(),
|
||||
runtimeConfig: {},
|
||||
})).label).toBe("Healthy");
|
||||
expect(getAgentHealthStatus(makeAgent({
|
||||
state: "active",
|
||||
lastHeartbeatAt: new Date(FIXED_NOW - 4 * 3_600_000 - 1).toISOString(),
|
||||
runtimeConfig: {},
|
||||
})).label).toBe("Unresponsive");
|
||||
});
|
||||
|
||||
it("clamps invalid intervals (0/negative) to the dashboard minimum (5m)", () => {
|
||||
// 0 clamp to 300000ms (5m minimum) → threshold = max(300000 × 4, 300000) = 1,200,000ms (20 minutes).
|
||||
// A heartbeat 21 minutes old is stale.
|
||||
@@ -403,8 +431,26 @@ describe("getAgentHealthStatus", () => {
|
||||
expect(status.stateDerived).toBe(false);
|
||||
});
|
||||
|
||||
it("treats an unparseable persisted heartbeat as Unresponsive instead of Healthy", () => {
|
||||
const status = getAgentHealthStatus(makeAgent({
|
||||
state: "active",
|
||||
lastHeartbeatAt: "not-a-timestamp",
|
||||
runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 },
|
||||
}));
|
||||
expect(status.label).toBe("Unresponsive");
|
||||
expect(status.reason).toBe("Last heartbeat timestamp is invalid");
|
||||
});
|
||||
|
||||
it("clamps a future heartbeat timestamp to fresh rather than stale", () => {
|
||||
expect(getAgentHealthStatus(makeAgent({
|
||||
state: "active",
|
||||
lastHeartbeatAt: new Date(FIXED_NOW + 24 * 60 * 60_000).toISOString(),
|
||||
runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 },
|
||||
})).label).toBe("Healthy");
|
||||
});
|
||||
|
||||
it("100s stale heartbeat with no explicit interval → Healthy (default 1h applies)", () => {
|
||||
// 1h default interval → 2h threshold, so 100s is well within range.
|
||||
// 1h default interval → 4h threshold, so 100s is well within range.
|
||||
const agent = makeAgent({
|
||||
state: "active",
|
||||
lastHeartbeatAt: new Date(FIXED_NOW - 100_000).toISOString(),
|
||||
|
||||
@@ -100,8 +100,8 @@ function isTaskWorkerAgent(agent: AgentHealthInput): boolean {
|
||||
* - "Heartbeat Disabled" — durable agent with `runtimeConfig.enabled === false`
|
||||
* - "Starting..." — state === "active" && no lastHeartbeatAt
|
||||
* - "Idle" — state !== "active" && no lastHeartbeatAt
|
||||
* - "Healthy" — heartbeat is fresh within 2× the configured interval
|
||||
* - "Unresponsive" — heartbeat exceeded 2× the configured interval
|
||||
* - "Healthy" — heartbeat is fresh within the configured interval's 4× grace window
|
||||
* - "Unresponsive" — heartbeat exceeded the configured interval's 4× grace window
|
||||
*
|
||||
* @param agent - The agent object (partial Agent shape is accepted)
|
||||
* @returns A health status object with label, icon, color, and stateDerived metadata
|
||||
@@ -194,10 +194,28 @@ export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus
|
||||
// configured, or the scheduler's 1h default. Compare elapsed time to that
|
||||
// interval (with grace) rather than to `heartbeatTimeoutMs`, which is the
|
||||
// per-run work budget and has nothing to do with between-tick freshness.
|
||||
const lastHeartbeat = new Date(lastHeartbeatAt).getTime();
|
||||
const elapsed = Date.now() - lastHeartbeat;
|
||||
const lastHeartbeat = Date.parse(lastHeartbeatAt);
|
||||
const stalenessThresholdMs = getStalenessThresholdMs(runtimeConfig);
|
||||
|
||||
/*
|
||||
FNXC:AgentHeartbeat 2026-07-15-18:00:
|
||||
A persisted but unparseable heartbeat cannot prove agent freshness. Treat it
|
||||
as Unresponsive rather than letting NaN bypass the elapsed-time comparison,
|
||||
matching the engine's persisted-heartbeat classification surfaces. Future
|
||||
timestamps clamp to zero so clock skew does not create a false stale label.
|
||||
*/
|
||||
if (!Number.isFinite(lastHeartbeat)) {
|
||||
return {
|
||||
label: "Unresponsive",
|
||||
icon: <Activity size={14} />,
|
||||
color: "var(--state-error-text)",
|
||||
stateDerived: false,
|
||||
reason: "Last heartbeat timestamp is invalid",
|
||||
};
|
||||
}
|
||||
|
||||
const elapsed = Math.max(0, Date.now() - lastHeartbeat);
|
||||
|
||||
if (elapsed > stalenessThresholdMs) {
|
||||
const reason = `No heartbeat for ${formatDuration(elapsed)} (threshold: ${formatDuration(stalenessThresholdMs)})`;
|
||||
return {
|
||||
|
||||
@@ -465,6 +465,37 @@ describe("executeHeartbeat", () => {
|
||||
expect(section).toContain("**stale**");
|
||||
});
|
||||
|
||||
it("classifies the FN-8018 field ages as stale using persisted timestamps", async () => {
|
||||
const now = Date.now();
|
||||
const store = createStoreWithAgentForExec();
|
||||
vi.mocked(store.getCachedAgent).mockImplementation((id: string) => ({
|
||||
id,
|
||||
runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 },
|
||||
}) as unknown as Agent);
|
||||
vi.mocked(store.getAgentsByReportsTo).mockResolvedValue([
|
||||
{ id: "agent-backend", name: "Backend Engineer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (6 * 60 + 32) * 60_000).toISOString(), updatedAt: new Date(now - (6 * 60 + 32) * 60_000).toISOString() } as Agent,
|
||||
{ id: "agent-frontend", name: "Frontend Engineer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (5 * 60 + 59) * 60_000).toISOString(), updatedAt: new Date(now - (5 * 60 + 59) * 60_000).toISOString() } as Agent,
|
||||
{ id: "agent-writer", name: "Technical Writer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (6 * 60 + 34) * 60_000).toISOString(), updatedAt: new Date(now - (6 * 60 + 34) * 60_000).toISOString() } as Agent,
|
||||
]);
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
|
||||
|
||||
const section = await (monitor as any).buildReportsHealthSection("agent-001", store);
|
||||
expect(section).toMatch(/\| Backend Engineer \| active \| — \| .* \| \*\*stale\*\* \|/);
|
||||
expect(section).toMatch(/\| Frontend Engineer \| active \| — \| .* \| \*\*stale\*\* \|/);
|
||||
expect(section).toMatch(/\| Technical Writer \| active \| — \| .* \| \*\*stale\*\* \|/);
|
||||
});
|
||||
|
||||
it("classifies an invalid persisted heartbeat as stale", async () => {
|
||||
const store = createStoreWithAgentForExec();
|
||||
vi.mocked(store.getAgentsByReportsTo).mockResolvedValue([
|
||||
{ id: "agent-invalid-heartbeat", name: "Invalid Heartbeat", state: "active", taskId: null, lastHeartbeatAt: "not-a-timestamp", updatedAt: new Date().toISOString() } as Agent,
|
||||
]);
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
|
||||
|
||||
const section = await (monitor as any).buildReportsHealthSection("agent-001", store);
|
||||
expect(section).toMatch(/\| Invalid Heartbeat \| active \| — \| unknown \| \*\*stale\*\* \|/);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "60-minute interval stays healthy at 45 minutes",
|
||||
|
||||
Reference in New Issue
Block a user