FN-8020: harden agent heartbeat health classification

Validate the four-interval heartbeat grace window and classify invalid persisted heartbeats safely.

- Cover reported field heartbeat ages in dashboard and engine health checks.
- Mark unparseable heartbeat timestamps as unresponsive and clamp future timestamps to fresh.
- Align dashboard health documentation with the existing four-interval grace window.

Files changed:
 .../app/utils/__tests__/agentHealth.test.tsx       | 48 +++++++++++++++++++++-
 packages/dashboard/app/utils/agentHealth.tsx       | 26 ++++++++++--
 .../src/__tests__/heartbeat-executor.test.ts       | 31 ++++++++++++++
 3 files changed, 100 insertions(+), 5 deletions(-)

Fusion-Task-Id: FN-8020

Fusion-Task-Lineage: 2bc0df78-d68c-489b-8bfb-9b09da10cdfa

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-07-15 22:11:26 -07:00
parent e57f52bb0d
commit 959a7877c8
3 changed files with 100 additions and 5 deletions

View File

@@ -272,6 +272,34 @@ describe("getAgentHealthStatus", () => {
expect(getAgentHealthStatus(agent).label).toBe("Unresponsive");
});
it("reproduces the FN-8018 field ages against the default 4h grace boundary", () => {
const fieldAgents = [
{ name: "Backend Engineer", ageMs: 6 * 3_600_000 + 32 * 60_000 },
{ name: "Frontend Engineer", ageMs: 5 * 3_600_000 + 59 * 60_000 },
{ name: "Technical Writer", ageMs: 6 * 3_600_000 + 34 * 60_000 },
];
for (const fieldAgent of fieldAgents) {
expect(getAgentHealthStatus(makeAgent({
name: fieldAgent.name,
state: "active",
lastHeartbeatAt: new Date(FIXED_NOW - fieldAgent.ageMs).toISOString(),
runtimeConfig: {},
})).label).toBe("Unresponsive");
}
expect(getAgentHealthStatus(makeAgent({
state: "active",
lastHeartbeatAt: new Date(FIXED_NOW - 4 * 3_600_000).toISOString(),
runtimeConfig: {},
})).label).toBe("Healthy");
expect(getAgentHealthStatus(makeAgent({
state: "active",
lastHeartbeatAt: new Date(FIXED_NOW - 4 * 3_600_000 - 1).toISOString(),
runtimeConfig: {},
})).label).toBe("Unresponsive");
});
it("clamps invalid intervals (0/negative) to the dashboard minimum (5m)", () => {
// 0 clamp to 300000ms (5m minimum) → threshold = max(300000 × 4, 300000) = 1,200,000ms (20 minutes).
// A heartbeat 21 minutes old is stale.
@@ -403,8 +431,26 @@ describe("getAgentHealthStatus", () => {
expect(status.stateDerived).toBe(false);
});
it("treats an unparseable persisted heartbeat as Unresponsive instead of Healthy", () => {
const status = getAgentHealthStatus(makeAgent({
state: "active",
lastHeartbeatAt: "not-a-timestamp",
runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 },
}));
expect(status.label).toBe("Unresponsive");
expect(status.reason).toBe("Last heartbeat timestamp is invalid");
});
it("clamps a future heartbeat timestamp to fresh rather than stale", () => {
expect(getAgentHealthStatus(makeAgent({
state: "active",
lastHeartbeatAt: new Date(FIXED_NOW + 24 * 60 * 60_000).toISOString(),
runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 },
})).label).toBe("Healthy");
});
it("100s stale heartbeat with no explicit interval → Healthy (default 1h applies)", () => {
// 1h default interval → 2h threshold, so 100s is well within range.
// 1h default interval → 4h threshold, so 100s is well within range.
const agent = makeAgent({
state: "active",
lastHeartbeatAt: new Date(FIXED_NOW - 100_000).toISOString(),

View File

@@ -100,8 +100,8 @@ function isTaskWorkerAgent(agent: AgentHealthInput): boolean {
* - "Heartbeat Disabled" — durable agent with `runtimeConfig.enabled === false`
* - "Starting..." — state === "active" && no lastHeartbeatAt
* - "Idle" — state !== "active" && no lastHeartbeatAt
* - "Healthy" — heartbeat is fresh within 2× the configured interval
* - "Unresponsive" — heartbeat exceeded 2× the configured interval
* - "Healthy" — heartbeat is fresh within the configured interval's 4× grace window
* - "Unresponsive" — heartbeat exceeded the configured interval's 4× grace window
*
* @param agent - The agent object (partial Agent shape is accepted)
* @returns A health status object with label, icon, color, and stateDerived metadata
@@ -194,10 +194,28 @@ export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus
// configured, or the scheduler's 1h default. Compare elapsed time to that
// interval (with grace) rather than to `heartbeatTimeoutMs`, which is the
// per-run work budget and has nothing to do with between-tick freshness.
const lastHeartbeat = new Date(lastHeartbeatAt).getTime();
const elapsed = Date.now() - lastHeartbeat;
const lastHeartbeat = Date.parse(lastHeartbeatAt);
const stalenessThresholdMs = getStalenessThresholdMs(runtimeConfig);
/*
FNXC:AgentHeartbeat 2026-07-15-18:00:
A persisted but unparseable heartbeat cannot prove agent freshness. Treat it
as Unresponsive rather than letting NaN bypass the elapsed-time comparison,
matching the engine's persisted-heartbeat classification surfaces. Future
timestamps clamp to zero so clock skew does not create a false stale label.
*/
if (!Number.isFinite(lastHeartbeat)) {
return {
label: "Unresponsive",
icon: <Activity size={14} />,
color: "var(--state-error-text)",
stateDerived: false,
reason: "Last heartbeat timestamp is invalid",
};
}
const elapsed = Math.max(0, Date.now() - lastHeartbeat);
if (elapsed > stalenessThresholdMs) {
const reason = `No heartbeat for ${formatDuration(elapsed)} (threshold: ${formatDuration(stalenessThresholdMs)})`;
return {

View File

@@ -465,6 +465,37 @@ describe("executeHeartbeat", () => {
expect(section).toContain("**stale**");
});
it("classifies the FN-8018 field ages as stale using persisted timestamps", async () => {
const now = Date.now();
const store = createStoreWithAgentForExec();
vi.mocked(store.getCachedAgent).mockImplementation((id: string) => ({
id,
runtimeConfig: { heartbeatIntervalMs: 60 * 60_000 },
}) as unknown as Agent);
vi.mocked(store.getAgentsByReportsTo).mockResolvedValue([
{ id: "agent-backend", name: "Backend Engineer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (6 * 60 + 32) * 60_000).toISOString(), updatedAt: new Date(now - (6 * 60 + 32) * 60_000).toISOString() } as Agent,
{ id: "agent-frontend", name: "Frontend Engineer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (5 * 60 + 59) * 60_000).toISOString(), updatedAt: new Date(now - (5 * 60 + 59) * 60_000).toISOString() } as Agent,
{ id: "agent-writer", name: "Technical Writer", state: "active", taskId: null, lastHeartbeatAt: new Date(now - (6 * 60 + 34) * 60_000).toISOString(), updatedAt: new Date(now - (6 * 60 + 34) * 60_000).toISOString() } as Agent,
]);
const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
const section = await (monitor as any).buildReportsHealthSection("agent-001", store);
expect(section).toMatch(/\| Backend Engineer \| active \| — \| .* \| \*\*stale\*\* \|/);
expect(section).toMatch(/\| Frontend Engineer \| active \| — \| .* \| \*\*stale\*\* \|/);
expect(section).toMatch(/\| Technical Writer \| active \| — \| .* \| \*\*stale\*\* \|/);
});
it("classifies an invalid persisted heartbeat as stale", async () => {
const store = createStoreWithAgentForExec();
vi.mocked(store.getAgentsByReportsTo).mockResolvedValue([
{ id: "agent-invalid-heartbeat", name: "Invalid Heartbeat", state: "active", taskId: null, lastHeartbeatAt: "not-a-timestamp", updatedAt: new Date().toISOString() } as Agent,
]);
const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
const section = await (monitor as any).buildReportsHealthSection("agent-001", store);
expect(section).toMatch(/\| Invalid Heartbeat \| active \| — \| unknown \| \*\*stale\*\* \|/);
});
it.each([
{
name: "60-minute interval stays healthy at 45 minutes",