feat(FN-3958): add heartbeat timer reconciliation self-healing

Adds scheduler heartbeat timer reconciliation with automatic self-healing when timers drift, including tests for tracked-only monitor recovery and documentation in the agents reference.

Fusion-Task-Id: FN-3958

packages/engine/src/__tests__/heartbeat-monitor.test.ts

@@ -719,7 +719,7 @@ describe("missed heartbeat detection", () => {
 });
 
 describe("unresponsive agent recovery", () => {
-  it("disposes session and pauses/resumes agent after 2x timeout", async () => {
+  it("recovers only tracked stale sessions via pause/resume restart", async () => {
     const onTerminated = vi.fn();
     const session = createMockSession();
     const localStore = createMockStore({
@@ -751,6 +751,34 @@ describe("unresponsive agent recovery", () => {
     vi.useRealTimers();
   });
 
+  it("does not attempt stale recovery for untracked agents", async () => {
+    const localStore = createMockStore({
+      getAgent: vi.fn().mockResolvedValue({
+        id: "agent-001",
+        state: "active",
+        lastHeartbeatAt: new Date(Date.now() - 60_000).toISOString(),
+        runtimeConfig: { enabled: true },
+      }),
+      updateAgentState: vi.fn().mockResolvedValue(undefined),
+      updateAgent: vi.fn().mockResolvedValue(undefined),
+    });
+    const customMonitor = new HeartbeatMonitor({
+      store: localStore,
+      heartbeatTimeoutMs: 5_000,
+      pollIntervalMs: 1_000,
+    });
+
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    customMonitor.start();
+    await vi.advanceTimersByTimeAsync(12_000);
+
+    expect(localStore.updateAgentState).not.toHaveBeenCalledWith("agent-001", "paused");
+    expect(localStore.updateAgentState).not.toHaveBeenCalledWith("agent-001", "active");
+
+    customMonitor.stop();
+    vi.useRealTimers();
+  });
+
   it("logs recovery warnings when dispose and pause fail", async () => {
     const warnSpy = vi.mocked(heartbeatLog.warn);
     warnSpy.mockClear();

packages/engine/src/__tests__/heartbeat-scheduler.test.ts

@@ -35,6 +35,7 @@ describe("HeartbeatTriggerScheduler", () => {
       }),
       getActiveHeartbeatRun: vi.fn().mockResolvedValue(null),
       getBudgetStatus: vi.fn().mockResolvedValue(createBudgetStatus()),
+      listAgents: vi.fn().mockResolvedValue([]),
       on: vi.fn(),
       off: vi.fn(),
     } as unknown as AgentStore;
@@ -73,6 +74,61 @@ describe("HeartbeatTriggerScheduler", () => {
     });
   });
 
+  describe("scheduler timer audit", () => {
+    it("re-arms a tickable durable agent when timer entry is missing and no lifecycle event fires", async () => {
+      vi.useFakeTimers();
+      const agent = {
+        id: "agent-001",
+        name: "Agent 001",
+        role: "executor",
+        state: "active",
+        lastHeartbeatAt: "2026-01-01T00:00:00.000Z",
+        runtimeConfig: { enabled: true, heartbeatIntervalMs: 30_000 },
+        createdAt: "2026-01-01T00:00:00.000Z",
+        updatedAt: "2026-01-01T00:00:00.000Z",
+        metadata: {},
+      } as Agent;
+      vi.mocked(store.listAgents).mockResolvedValue([agent]);
+      vi.mocked(store.getActiveHeartbeatRun).mockResolvedValue(null);
+
+      scheduler = new HeartbeatTriggerScheduler(store, callback);
+      scheduler.start();
+      await vi.advanceTimersByTimeAsync(0);
+
+      expect(scheduler.getRegisteredAgents()).toContain("agent-001");
+      scheduler.unregisterAgent("agent-001");
+      expect(scheduler.getRegisteredAgents()).not.toContain("agent-001");
+
+      await vi.advanceTimersByTimeAsync(60_000);
+
+      expect(scheduler.getRegisteredAgents()).toContain("agent-001");
+    });
+
+    it("skips audit re-arm when the agent already has an active heartbeat run", async () => {
+      vi.useFakeTimers();
+      const agent = {
+        id: "agent-001",
+        name: "Agent 001",
+        role: "executor",
+        state: "active",
+        runtimeConfig: { enabled: true, heartbeatIntervalMs: 30_000 },
+        createdAt: "2026-01-01T00:00:00.000Z",
+        updatedAt: "2026-01-01T00:00:00.000Z",
+        metadata: {},
+      } as Agent;
+      vi.mocked(store.listAgents).mockResolvedValue([agent]);
+      vi.mocked(store.getActiveHeartbeatRun).mockResolvedValue({ id: "run-1" } as any);
+
+      scheduler = new HeartbeatTriggerScheduler(store, callback);
+      scheduler.start();
+      await vi.advanceTimersByTimeAsync(0);
+      scheduler.unregisterAgent("agent-001");
+
+      await vi.advanceTimersByTimeAsync(60_000);
+      expect(scheduler.getRegisteredAgents()).not.toContain("agent-001");
+    });
+  });
+
   describe("registerAgent", () => {
     beforeEach(() => {
       scheduler = new HeartbeatTriggerScheduler(store, callback);

packages/engine/src/agent-heartbeat.ts

@@ -2856,6 +2856,9 @@ export class HeartbeatTriggerScheduler {
   private configRevisionListener: ((agentId: string, revision: AgentConfigRevision) => void) | null = null;
   private deletedListener: ((agentId: string) => void) | null = null;
   private isTaskExecuting?: (taskId: string) => boolean;
+  private timerAuditIntervalHandle: ReturnType<typeof setInterval> | null = null;
+
+  private static readonly TIMER_AUDIT_INTERVAL_MS = 60_000;
 
   constructor(store: AgentStore, callback: TriggerCallback, taskStore?: TaskStore, options?: { isTaskExecuting?: (taskId: string) => boolean }) {
     this.store = store;
@@ -2873,6 +2876,10 @@ export class HeartbeatTriggerScheduler {
     this.running = true;
     this.watchAssignments();
     this.watchAgentLifecycle();
+    void this.auditTimerRegistrations("start");
+    this.timerAuditIntervalHandle = setInterval(() => {
+      void this.auditTimerRegistrations("interval");
+    }, HeartbeatTriggerScheduler.TIMER_AUDIT_INTERVAL_MS);
     heartbeatLog.log("HeartbeatTriggerScheduler started");
   }
 
@@ -2898,6 +2905,11 @@ export class HeartbeatTriggerScheduler {
     }
     this.timers.clear();
 
+    if (this.timerAuditIntervalHandle) {
+      clearInterval(this.timerAuditIntervalHandle);
+      this.timerAuditIntervalHandle = null;
+    }
+
     heartbeatLog.log("HeartbeatTriggerScheduler stopped");
   }
 
@@ -3340,6 +3352,37 @@ export class HeartbeatTriggerScheduler {
     }
   }
 
+  async auditTimerRegistrations(reason: "start" | "interval" = "interval"): Promise<void> {
+    if (!this.running) return;
+
+    try {
+      const agents = await this.store.listAgents();
+      let rearmedCount = 0;
+      for (const agent of agents) {
+        if (!this.isTimerEligibleAgent(agent)) continue;
+        if (this.timers.has(agent.id)) continue;
+
+        const activeRun = await this.store.getActiveHeartbeatRun(agent.id);
+        if (activeRun) {
+          heartbeatLog.log(`Timer audit skipped re-arm for ${agent.id} (active run)`);
+          continue;
+        }
+
+        this.registerAgent(agent.id, this.getAgentTimerConfig(agent), {
+          lastHeartbeatAt: agent.lastHeartbeatAt,
+        });
+        rearmedCount++;
+        heartbeatLog.log(`Timer re-armed for ${agent.id} (audit:${reason})`);
+      }
+
+      if (rearmedCount > 0) {
+        heartbeatLog.log(`Timer audit repaired ${rearmedCount} missing registration(s) (${reason})`);
+      }
+    } catch (error) {
+      heartbeatLog.warn(`Timer audit failed (${reason}): ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
   /**
    * Handle a timer tick for an agent.
    * Checks for active runs before invoking the callback.

packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts

@@ -1181,6 +1181,28 @@ describe("InProcessRuntime", () => {
       );
     });
 
+    it("reconciles a missing timer for a tickable durable agent without state changes", async () => {
+      const store = getAgentStore(runtime);
+      const scheduler = runtime.getTriggerScheduler();
+      expect(scheduler).toBeDefined();
+
+      const agent = await store.createAgent({
+        name: "audit-rearm-agent",
+        role: "executor",
+        runtimeConfig: {
+          enabled: true,
+          heartbeatIntervalMs: 1_000,
+        },
+      });
+
+      expect(scheduler!.getRegisteredAgents()).toContain(agent.id);
+      scheduler!.unregisterAgent(agent.id);
+      expect(scheduler!.getRegisteredAgents()).not.toContain(agent.id);
+
+      await vi.advanceTimersByTimeAsync(60_000);
+      expect(scheduler!.getRegisteredAgents()).toContain(agent.id);
+    });
+
     it("unregisters an agent when enabled is set to false in update", async () => {
       // Create a new agent with heartbeat enabled
       const store = getAgentStore(runtime);