feat(FN-4020): surface timer repair stale health in agent health utilities

Adds health-surfacing logic for timer-triggered stale heartbeats in the engine, a new `agentHealth.tsx` dashboard utility with tests, and documentation updates for the repair behavior.

Fusion-Task-Id: FN-4020

docs/agents.md

@@ -1024,6 +1024,7 @@ Effects:
 - Repair target: durable, heartbeat-enabled agents in tickable states (`active`, `running`, `idle`) that are missing a timer entry
 - Safety guards: skip ephemeral/task-worker agents, skip disabled agents, skip non-tickable states, and skip agents with an active heartbeat run
 - Existing timer entries are left untouched (no interval reset/jitter churn)
+- Repair metadata: each audit re-arm writes `metadata.heartbeatTimerRepair` with `repairedAt` and a stale-at-repair indicator when the agent had already missed its expected cadence
 
 This covers the untracked timer-loss failure mode where no `agent:updated` event fires after a timer entry disappears. Manual stop/start is no longer required to re-arm the timer in that case.
 
@@ -1046,7 +1047,7 @@ The dashboard displays agent health status in AgentsView, AgentListModal, and Ag
 | **Starting...** | State is "active" with no lastHeartbeatAt |
 | **Idle** | Non-active state with no lastHeartbeatAt |
 | **Healthy** | Heartbeat is fresh within the resolved interval-based staleness threshold |
-| **Unresponsive** | Heartbeat exceeded the resolved interval-based staleness threshold |
+| **Unresponsive** | Heartbeat exceeded the resolved interval-based staleness threshold, or timer-repair metadata indicates scheduler-detected stale drift before the next successful heartbeat |
 
 ### Timeout Configuration
 
@@ -1062,6 +1063,7 @@ Health status uses interval-based staleness evaluation:
 - **Consistent across views**: All dashboard surfaces use the same centralized utility, ensuring consistent health labels everywhere
 - **Auto-refresh**: Health status is refreshed every 30 seconds while views are open to keep status current
 - **State-first evaluation**: Explicit non-idle states (error, paused, running) take priority over timeout-based evaluation
+- **Repair-aware surfacing**: If scheduler audit repairs a missing timer and marks the agent stale, dashboard surfaces `Unresponsive` immediately until a newer heartbeat arrives
 
 ## Heartbeat Run Lifecycle
 

packages/dashboard/app/utils/__tests__/agentHealth.test.tsx

@@ -510,6 +510,26 @@ describe("AgentHealthStatus reason field", () => {
     expect(status.reason).toContain("threshold:");
   });
 
+  it("surfaces unresponsive status when timer repair metadata marks stale and no newer heartbeat exists", () => {
+    const repairTime = new Date(FIXED_NOW - 2 * 60 * 1000).toISOString();
+    const agent = makeAgent({
+      state: "active",
+      lastHeartbeatAt: new Date(FIXED_NOW - 20 * 60 * 1000).toISOString(),
+      runtimeConfig: { heartbeatIntervalMs: 60 * 60 * 1000 },
+      metadata: {
+        heartbeatTimerRepair: {
+          repairedAt: repairTime,
+          staleAtRepair: true,
+          staleRepairReason: "No heartbeat before repair",
+        },
+      },
+    });
+
+    const status = getAgentHealthStatus(agent);
+    expect(status.label).toBe("Unresponsive");
+    expect(status.reason).toBe("No heartbeat before repair");
+  });
+
   it("formats reason with elapsed time and threshold", () => {
     const agent = makeAgent({
       state: "active",

packages/dashboard/app/utils/agentHealth.tsx

@@ -106,6 +106,22 @@ function isTaskWorkerAgent(agent: AgentHealthInput): boolean {
  * @param agent - The agent object (partial Agent shape is accepted)
  * @returns A health status object with label, icon, color, and stateDerived metadata
  */
+function getHeartbeatRepairMetadata(agent: AgentHealthInput): {
+  repairedAt?: string;
+  staleAtRepair?: boolean;
+  staleRepairReason?: string;
+} {
+  const metadata = agent.metadata as Record<string, unknown> | null | undefined;
+  const raw = metadata?.heartbeatTimerRepair;
+  if (!raw || typeof raw !== "object") return {};
+  const value = raw as Record<string, unknown>;
+  return {
+    repairedAt: typeof value.repairedAt === "string" ? value.repairedAt : undefined,
+    staleAtRepair: typeof value.staleAtRepair === "boolean" ? value.staleAtRepair : undefined,
+    staleRepairReason: typeof value.staleRepairReason === "string" ? value.staleRepairReason : undefined,
+  };
+}
+
 export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus {
   const { state, lastHeartbeatAt, lastError, pauseReason, runtimeConfig } = agent;
   const isTaskWorker = isTaskWorkerAgent(agent);
@@ -159,6 +175,21 @@ export function getAgentHealthStatus(agent: AgentHealthInput): AgentHealthStatus
     };
   }
 
+  const heartbeatRepair = getHeartbeatRepairMetadata(agent);
+  if (heartbeatRepair.staleAtRepair && heartbeatRepair.repairedAt) {
+    const repairedMs = Date.parse(heartbeatRepair.repairedAt);
+    const lastHeartbeatMs = Date.parse(lastHeartbeatAt);
+    if (Number.isFinite(repairedMs) && Number.isFinite(lastHeartbeatMs) && lastHeartbeatMs < repairedMs) {
+      return {
+        label: "Unresponsive",
+        icon: <Activity size={14} />,
+        color: "var(--state-error-text)",
+        stateDerived: false,
+        reason: heartbeatRepair.staleRepairReason ?? "Heartbeat scheduler repaired a missing timer; waiting for recovery heartbeat",
+      };
+    }
+  }
+
   // Every non-task-worker agent has an effective interval — either explicitly
   // configured, or the scheduler's 1h default. Compare elapsed time to that
   // interval (with grace) rather than to `heartbeatTimeoutMs`, which is the

packages/engine/src/__tests__/heartbeat-scheduler.test.ts

@@ -38,6 +38,10 @@ describe("HeartbeatTriggerScheduler", () => {
       listAgents: vi.fn().mockResolvedValue([]),
       on: vi.fn(),
       off: vi.fn(),
+      updateAgent: vi.fn().mockImplementation(async (_id: string, updates: { metadata: Record<string, unknown> }) => ({
+        id: "agent-001",
+        metadata: updates.metadata,
+      })),
     } as unknown as AgentStore;
   });
 
@@ -104,6 +108,37 @@ describe("HeartbeatTriggerScheduler", () => {
       expect(scheduler.getRegisteredAgents()).toContain("agent-001");
     });
 
+    it("marks repaired agent metadata as stale when last heartbeat is old", async () => {
+      vi.useFakeTimers();
+      vi.setSystemTime(new Date("2026-01-01T02:00:00.000Z"));
+      const agent = {
+        id: "agent-001",
+        name: "Agent 001",
+        role: "executor",
+        state: "active",
+        lastHeartbeatAt: "2026-01-01T00:00:00.000Z",
+        runtimeConfig: { enabled: true, heartbeatIntervalMs: 30_000 },
+        createdAt: "2026-01-01T00:00:00.000Z",
+        updatedAt: "2026-01-01T00:00:00.000Z",
+        metadata: {},
+      } as Agent;
+      vi.mocked(store.listAgents).mockResolvedValue([agent]);
+      vi.mocked(store.getActiveHeartbeatRun).mockResolvedValue(null);
+
+      scheduler = new HeartbeatTriggerScheduler(store, callback);
+      scheduler.start();
+      await vi.advanceTimersByTimeAsync(0);
+
+      expect(store.updateAgent).toHaveBeenCalledWith(
+        "agent-001",
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            heartbeatTimerRepair: expect.objectContaining({ staleAtRepair: true }),
+          }),
+        }),
+      );
+    });
+
     it("skips audit re-arm when the agent already has an active heartbeat run", async () => {
       vi.useFakeTimers();
       const agent = {

packages/engine/src/agent-heartbeat.ts

@@ -2855,6 +2855,26 @@ function isHeartbeatManaged(agent: Agent): boolean {
  * - `heartbeatIntervalMs`: Timer interval (default 1h)
  * - `maxConcurrentRuns`: Skip tick if agent already has an active run
  */
+type HeartbeatTimerRepairMetadata = {
+  repairedAt?: string;
+  staleAtRepair?: boolean;
+  staleRepairReason?: string;
+};
+
+function readHeartbeatTimerRepairMetadata(agent: Agent): HeartbeatTimerRepairMetadata {
+  const metadata = (agent.metadata ?? {}) as Record<string, unknown>;
+  const raw = metadata.heartbeatTimerRepair;
+  if (!raw || typeof raw !== "object") {
+    return {};
+  }
+  const candidate = raw as Record<string, unknown>;
+  return {
+    repairedAt: typeof candidate.repairedAt === "string" ? candidate.repairedAt : undefined,
+    staleAtRepair: typeof candidate.staleAtRepair === "boolean" ? candidate.staleAtRepair : undefined,
+    staleRepairReason: typeof candidate.staleRepairReason === "string" ? candidate.staleRepairReason : undefined,
+  };
+}
+
 export class HeartbeatTriggerScheduler {
   private store: AgentStore;
   private callback: TriggerCallback;
@@ -2871,6 +2891,7 @@ export class HeartbeatTriggerScheduler {
   private timerAuditIntervalHandle: ReturnType<typeof setInterval> | null = null;
 
   private static readonly TIMER_AUDIT_INTERVAL_MS = 60_000;
+  private static readonly REPAIR_STALE_GRACE_MULTIPLIER = 1.5;
 
   constructor(store: AgentStore, callback: TriggerCallback, taskStore?: TaskStore, options?: { isTaskExecuting?: (taskId: string) => boolean }) {
     this.store = store;
@@ -3364,6 +3385,43 @@ export class HeartbeatTriggerScheduler {
     }
   }
 
+  private getRepairStaleThresholdMs(agent: Agent): number {
+    const config = this.getAgentTimerConfig(agent);
+    let rawIntervalMs = config.heartbeatIntervalMs;
+    if (!rawIntervalMs || typeof rawIntervalMs !== "number" || !Number.isFinite(rawIntervalMs) || rawIntervalMs <= 0) {
+      rawIntervalMs = HeartbeatTriggerScheduler.DEFAULT_HEARTBEAT_INTERVAL_MS;
+    }
+    const intervalMs = Math.max(1000, Math.round(rawIntervalMs));
+    return Math.round(intervalMs * HeartbeatTriggerScheduler.REPAIR_STALE_GRACE_MULTIPLIER);
+  }
+
+  private async markRepairMetadata(agent: Agent, staleAtRepair: boolean, staleRepairReason?: string): Promise<void> {
+    const updater = (this.store as { updateAgent?: (agentId: string, updates: { metadata: Record<string, unknown> }) => Promise<unknown> }).updateAgent;
+    if (typeof updater !== "function") {
+      return;
+    }
+
+    const existing = readHeartbeatTimerRepairMetadata(agent);
+    const repairedAt = new Date().toISOString();
+    const nextRepair: HeartbeatTimerRepairMetadata = {
+      repairedAt,
+      staleAtRepair,
+      ...(staleAtRepair && staleRepairReason ? { staleRepairReason } : {}),
+    };
+
+    const didChange =
+      existing.repairedAt !== nextRepair.repairedAt ||
+      existing.staleAtRepair !== nextRepair.staleAtRepair ||
+      existing.staleRepairReason !== nextRepair.staleRepairReason;
+    if (!didChange) {
+      return;
+    }
+
+    const metadata = { ...(agent.metadata ?? {}) } as Record<string, unknown>;
+    metadata.heartbeatTimerRepair = nextRepair;
+    await updater.call(this.store, agent.id, { metadata });
+  }
+
   async auditTimerRegistrations(reason: "start" | "interval" = "interval"): Promise<void> {
     if (!this.running) return;
 
@@ -3383,8 +3441,18 @@ export class HeartbeatTriggerScheduler {
         this.registerAgent(agent.id, this.getAgentTimerConfig(agent), {
           lastHeartbeatAt: agent.lastHeartbeatAt,
         });
+
+        const staleThresholdMs = this.getRepairStaleThresholdMs(agent);
+        const lastHeartbeatMs = agent.lastHeartbeatAt ? Date.parse(agent.lastHeartbeatAt) : Number.NaN;
+        const elapsedMs = Number.isFinite(lastHeartbeatMs) ? Date.now() - lastHeartbeatMs : Number.NaN;
+        const staleAtRepair = Number.isFinite(elapsedMs) && elapsedMs > staleThresholdMs;
+        const staleRepairReason = staleAtRepair
+          ? `No heartbeat for ${Math.round(elapsedMs / 1000)}s before timer audit repair (threshold ${Math.round(staleThresholdMs / 1000)}s)`
+          : undefined;
+        await this.markRepairMetadata(agent, staleAtRepair, staleRepairReason);
+
         rearmedCount++;
-        heartbeatLog.log(`Timer re-armed for ${agent.id} (audit:${reason})`);
+        heartbeatLog.log(`Timer re-armed for ${agent.id} (audit:${reason}${staleAtRepair ? ", stale" : ""})`);
       }
 
       if (rearmedCount > 0) {