From 959a7877c8dec136c9a2f9b0964cb9b52e22061a Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 15 Jul 2026 22:11:26 -0700 Subject: [PATCH] FN-8020: harden agent heartbeat health classification Validate the four-interval heartbeat grace window and classify invalid persisted heartbeats safely. - Cover reported field heartbeat ages in dashboard and engine health checks. - Mark unparseable heartbeat timestamps as unresponsive and clamp future timestamps to fresh. - Align dashboard health documentation with the existing four-interval grace window. Files changed: .../app/utils/__tests__/agentHealth.test.tsx | 48 +++++++++++++++++++++- packages/dashboard/app/utils/agentHealth.tsx | 26 ++++++++++-- .../src/__tests__/heartbeat-executor.test.ts | 31 ++++++++++++++ 3 files changed, 100 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-8020 Fusion-Task-Lineage: 2bc0df78-d68c-489b-8bfb-9b09da10cdfa Co-authored-by: Fusion (runfusion.ai) --- .../app/utils/__tests__/agentHealth.test.tsx | 48 ++++++++++++++++++- packages/dashboard/app/utils/agentHealth.tsx | 26 ++++++++-- .../src/__tests__/heartbeat-executor.test.ts | 31 ++++++++++++ 3 files changed, 100 insertions(+), 5 deletions(-) diff --git a/packages/dashboard/app/utils/__tests__/agentHealth.test.tsx b/packages/dashboard/app/utils/__tests__/agentHealth.test.tsx index 5af18690ad..82736f8f75 100644 --- a/packages/dashboard/app/utils/__tests__/agentHealth.test.tsx +++ b/packages/dashboard/app/utils/__tests__/agentHealth.test.tsx @@ -272,6 +272,34 @@ describe("getAgentHealthStatus", () => { expect(getAgentHealthStatus(agent).label).toBe("Unresponsive"); }); + it("reproduces the FN-8018 field ages against the default 4h grace boundary", () => { + const fieldAgents = [ + { name: "Backend Engineer", ageMs: 6 * 3_600_000 + 32 * 60_000 }, + { name: "Frontend Engineer", ageMs: 5 * 3_600_000 + 59 * 60_000 }, + { name: "Technical Writer", ageMs: 6 * 3_600_000 + 34 * 60_000 }, + ]; + + for (const fieldAgent of fieldAgents) { + expect(getAgentHealthStatus(makeAgent({ + name: fieldAgent.name, + state: "active", + lastHeartbeatAt: new Date(FIXED_NOW - fieldAgent.ageMs).toISOString(), + runtimeConfig: {}, + })).label).toBe("Unresponsive"); + } + + expect(getAgentHealthStatus(makeAgent({ + state: "active", + lastHeartbeatAt: new Date(FIXED_NOW - 4 * 3_600_000).toISOString(), + runtimeConfig: {}, + })).label).toBe("Healthy"); + expect(getAgentHealthStatus(makeAgent({ + state: "active", + lastHeartbeatAt: new Date(FIXED_NOW - 4 * 3_600_000 - 1).toISOString(), + runtimeConfig: {}, + })).label).toBe("Unresponsive"); + }); + it("clamps invalid intervals (0/negative) to the dashboard minimum (5m)", () => { // 0 clamp to 300000ms (5m minimum) → threshold = max(300000 × 4, 300000) = 1,200,000ms (20 minutes). // A heartbeat 21 minutes old is stale. @@ -403,8 +431,26 @@ describe("getAgentHealthStatus", () => { expect(status.stateDerived).toBe(false); }); + it("treats an unparseable persisted heartbeat as Unresponsive instead of Healthy", () => { + const status = getAgentHealthStatus(makeAgent({ + state: "active", + lastHeartbeatAt: "not-a-timestamp", + runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 }, + })); + expect(status.label).toBe("Unresponsive"); + expect(status.reason).toBe("Last heartbeat timestamp is invalid"); + }); + + it("clamps a future heartbeat timestamp to fresh rather than stale", () => { + expect(getAgentHealthStatus(makeAgent({ + state: "active", + lastHeartbeatAt: new Date(FIXED_NOW + 24 * 60 * 60_000).toISOString(), + runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 }, + })).label).toBe("Healthy"); + }); + it("100s stale heartbeat with no explicit interval → Healthy (default 1h applies)", () => { - // 1h default interval → 2h threshold, so 100s is well within range. + // 1h default interval → 4h threshold, so 100s is well within range. const agent = makeAgent({ state: "active", lastHeartbeatAt: new Date(FIXED_NOW - 100_000).toISOString(), diff --git a/packages/dashboard/app/utils/agentHealth.tsx b/packages/dashboard/app/utils/agentHealth.tsx index 41cebb12a1..5cdfb2c7b0 100644 --- a/packages/dashboard/app/utils/agentHealth.tsx +++ b/packages/dashboard/app/utils/agentHealth.tsx @@ -100,8 +100,8 @@ function isTaskWorkerAgent(agent: AgentHealthInput): boolean { * - "Heartbeat Disabled" — durable agent with `runtimeConfig.enabled === false` * - "Starting..." — state === "active" && no lastHeartbeatAt * - "Idle" — state !== "active" && no lastHeartbeatAt - * - "Healthy" — heartbeat is fresh within 2× the configured interval - * - "Unresponsive" — heartbeat exceeded 2× the configured interval + * - "Healthy" — heartbeat is fresh within the configured interval's 4× grace window + * - "Unresponsive" — heartbeat exceeded the configured interval's 4× grace window * * @param agent - The agent object (partial Agent shape is accepted) * @returns A health status object with label, icon, color, and stateDerived metadata @@ -194,10 +194,28 @@ export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus // configured, or the scheduler's 1h default. Compare elapsed time to that // interval (with grace) rather than to `heartbeatTimeoutMs`, which is the // per-run work budget and has nothing to do with between-tick freshness. - const lastHeartbeat = new Date(lastHeartbeatAt).getTime(); - const elapsed = Date.now() - lastHeartbeat; + const lastHeartbeat = Date.parse(lastHeartbeatAt); const stalenessThresholdMs = getStalenessThresholdMs(runtimeConfig); + /* + FNXC:AgentHeartbeat 2026-07-15-18:00: + A persisted but unparseable heartbeat cannot prove agent freshness. Treat it + as Unresponsive rather than letting NaN bypass the elapsed-time comparison, + matching the engine's persisted-heartbeat classification surfaces. Future + timestamps clamp to zero so clock skew does not create a false stale label. + */ + if (!Number.isFinite(lastHeartbeat)) { + return { + label: "Unresponsive", + icon: , + color: "var(--state-error-text)", + stateDerived: false, + reason: "Last heartbeat timestamp is invalid", + }; + } + + const elapsed = Math.max(0, Date.now() - lastHeartbeat); + if (elapsed > stalenessThresholdMs) { const reason = `No heartbeat for ${formatDuration(elapsed)} (threshold: ${formatDuration(stalenessThresholdMs)})`; return { diff --git a/packages/engine/src/__tests__/heartbeat-executor.test.ts b/packages/engine/src/__tests__/heartbeat-executor.test.ts index 5e01706377..076e4ac488 100644 --- a/packages/engine/src/__tests__/heartbeat-executor.test.ts +++ b/packages/engine/src/__tests__/heartbeat-executor.test.ts @@ -465,6 +465,37 @@ describe("executeHeartbeat", () => { expect(section).toContain("**stale**"); }); + it("classifies the FN-8018 field ages as stale using persisted timestamps", async () => { + const now = Date.now(); + const store = createStoreWithAgentForExec(); + vi.mocked(store.getCachedAgent).mockImplementation((id: string) => ({ + id, + runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 }, + }) as unknown as Agent); + vi.mocked(store.getAgentsByReportsTo).mockResolvedValue([ + { id: "agent-backend", name: "Backend Engineer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (6 * 60 + 32) * 60_000).toISOString(), updatedAt: new Date(now - (6 * 60 + 32) * 60_000).toISOString() } as Agent, + { id: "agent-frontend", name: "Frontend Engineer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (5 * 60 + 59) * 60_000).toISOString(), updatedAt: new Date(now - (5 * 60 + 59) * 60_000).toISOString() } as Agent, + { id: "agent-writer", name: "Technical Writer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (6 * 60 + 34) * 60_000).toISOString(), updatedAt: new Date(now - (6 * 60 + 34) * 60_000).toISOString() } as Agent, + ]); + const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" }); + + const section = await (monitor as any).buildReportsHealthSection("agent-001", store); + expect(section).toMatch(/\| Backend Engineer \| active \| — \| .* \| \*\*stale\*\* \|/); + expect(section).toMatch(/\| Frontend Engineer \| active \| — \| .* \| \*\*stale\*\* \|/); + expect(section).toMatch(/\| Technical Writer \| active \| — \| .* \| \*\*stale\*\* \|/); + }); + + it("classifies an invalid persisted heartbeat as stale", async () => { + const store = createStoreWithAgentForExec(); + vi.mocked(store.getAgentsByReportsTo).mockResolvedValue([ + { id: "agent-invalid-heartbeat", name: "Invalid Heartbeat", state: "active", taskId: null, lastHeartbeatAt: "not-a-timestamp", updatedAt: new Date().toISOString() } as Agent, + ]); + const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" }); + + const section = await (monitor as any).buildReportsHealthSection("agent-001", store); + expect(section).toMatch(/\| Invalid Heartbeat \| active \| — \| unknown \| \*\*stale\*\* \|/); + }); + it.each([ { name: "60-minute interval stays healthy at 45 minutes",