feat(FN-1049): add per-agent heartbeat configuration via runtimeConfig

- Define AgentHeartbeatConfig interface in core types (heartbeatIntervalMs, heartbeatTimeoutMs, maxConcurrentRuns) - Update HeartbeatMonitor to resolve per-agent config from AgentStore with validated min/max clamping - Wire AgentStore into HeartbeatMonitor via InProcessRuntime initialization - Add heartbeat settings section to dashboard AgentDetailView ConfigTab - Add PATCH /api/agents/:id endpoint accepting runtimeConfig updates - Add comprehensive tests for per-agent heartbeat config resolution and validation - Document per-agent heartbeat configuration in AGENTS.md
2026-04-07 11:50:17 -07:00
parent 39f4f4929f
commit 825ae1061b
10 changed files with 579 additions and 40 deletions
--- a/packages/engine/src/agent-heartbeat.test.ts
+++ b/packages/engine/src/agent-heartbeat.test.ts
@@ -408,4 +408,315 @@ describe("HeartbeatMonitor", () => {
      expect(() => monitor.untrackAgent("agent-001")).not.toThrow();
    });
  });
+
+  // ── Per-Agent Config Tests ──────────────────────────────────────────────
+
+  describe("per-agent heartbeat config", () => {
+    /** Create a mock store that returns a specific agent from getCachedAgent */
+    function createStoreWithAgent(agent: { id: string; runtimeConfig?: Record<string, unknown> }): AgentStore {
+      return {
+        recordHeartbeat: vi.fn().mockResolvedValue(undefined),
+        updateAgentState: vi.fn().mockResolvedValue(undefined),
+        getCachedAgent: vi.fn().mockReturnValue(agent),
+      } as unknown as AgentStore;
+    }
+
+    describe("getAgentHeartbeatConfig", () => {
+      it("returns monitor defaults when agentStore is not provided", () => {
+        const monitor = new HeartbeatMonitor({
+          store,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 10000,
+          maxConcurrentRuns: 2,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(5000);
+        expect(config.heartbeatTimeoutMs).toBe(10000);
+        expect(config.maxConcurrentRuns).toBe(2);
+      });
+
+      it("returns monitor defaults when agent has no runtimeConfig", () => {
+        const agentStore = createStoreWithAgent({ id: "agent-001" });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 10000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(5000);
+        expect(config.heartbeatTimeoutMs).toBe(10000);
+      });
+
+      it("returns per-agent values when runtimeConfig is set", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: {
+            heartbeatIntervalMs: 2000,
+            heartbeatTimeoutMs: 30000,
+            maxConcurrentRuns: 3,
+          },
+        });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 10000,
+          maxConcurrentRuns: 1,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(2000);
+        expect(config.heartbeatTimeoutMs).toBe(30000);
+        expect(config.maxConcurrentRuns).toBe(3);
+      });
+
+      it("clamps heartbeatIntervalMs to minimum of 1000", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { heartbeatIntervalMs: 100 },
+        });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(1000);
+      });
+
+      it("clamps heartbeatTimeoutMs to minimum of 5000", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { heartbeatTimeoutMs: 1000 },
+        });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          heartbeatTimeoutMs: 60000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.heartbeatTimeoutMs).toBe(5000);
+      });
+
+      it("clamps maxConcurrentRuns to minimum of 1", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { maxConcurrentRuns: 0 },
+        });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          maxConcurrentRuns: 1,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.maxConcurrentRuns).toBe(1);
+      });
+
+      it("falls back to monitor defaults when runtimeConfig values are NaN", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: {
+            heartbeatIntervalMs: NaN,
+            heartbeatTimeoutMs: "not a number" as any,
+          },
+        });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 10000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(5000);
+        expect(config.heartbeatTimeoutMs).toBe(10000);
+      });
+
+      it("falls back to monitor defaults when agent is not found", () => {
+        const agentStore = createStoreWithAgent({ id: "agent-001" });
+        (agentStore.getCachedAgent as ReturnType<typeof vi.fn>).mockReturnValue(null);
+
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 10000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-999");
+        expect(config.pollIntervalMs).toBe(5000);
+        expect(config.heartbeatTimeoutMs).toBe(10000);
+      });
+
+      it("returns monitor defaults when getCachedAgent throws", () => {
+        const agentStore = createStoreWithAgent({ id: "agent-001" });
+        (agentStore.getCachedAgent as ReturnType<typeof vi.fn>).mockImplementation(() => {
+          throw new Error("Read error");
+        });
+
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 10000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(5000);
+        expect(config.heartbeatTimeoutMs).toBe(10000);
+      });
+
+      it("returns partial overrides when only some runtimeConfig keys are set", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { heartbeatTimeoutMs: 120000 },
+        });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 5000,
+          heartbeatTimeoutMs: 60000,
+          maxConcurrentRuns: 1,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.pollIntervalMs).toBe(5000); // fallback
+        expect(config.heartbeatTimeoutMs).toBe(120000); // overridden
+        expect(config.maxConcurrentRuns).toBe(1); // fallback
+      });
+    });
+
+    describe("isAgentHealthy with per-agent config", () => {
+      it("uses per-agent timeout for health check", () => {
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { heartbeatTimeoutMs: 30000 },
+        });
+        const session = createMockSession();
+
+        vi.useFakeTimers({ shouldAdvanceTime: true });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          heartbeatTimeoutMs: 5000, // Global default is 5000
+        });
+        monitor.trackAgent("agent-001", session, "run-001");
+
+        // Advance 10s — past the global 5s default, but within the per-agent 30s
+        vi.advanceTimersByTime(10000);
+        expect(monitor.isAgentHealthy("agent-001")).toBe(true);
+
+        // Advance past per-agent 30s timeout
+        vi.advanceTimersByTime(25000);
+        expect(monitor.isAgentHealthy("agent-001")).toBe(false);
+
+        vi.useRealTimers();
+      });
+    });
+
+    describe("checkMissedHeartbeats with per-agent config", () => {
+      it("detects missed heartbeat using per-agent timeout", async () => {
+        const onMissed = vi.fn();
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { heartbeatTimeoutMs: 10000 },
+        });
+        const session = createMockSession();
+
+        vi.useFakeTimers({ shouldAdvanceTime: true });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 1000,
+          heartbeatTimeoutMs: 5000, // Global default 5s — agent overrides to 10s
+          onMissed,
+        });
+        monitor.start();
+        monitor.trackAgent("agent-001", session, "run-001");
+
+        // Advance 6s — past global 5s but within per-agent 10s
+        vi.advanceTimersByTime(6000);
+        await vi.advanceTimersByTimeAsync(100);
+
+        // Should NOT have triggered onMissed because per-agent timeout is 10s
+        expect(onMissed).not.toHaveBeenCalled();
+
+        // Advance past the 10s per-agent timeout
+        vi.advanceTimersByTime(5000);
+        await vi.advanceTimersByTimeAsync(100);
+
+        expect(onMissed).toHaveBeenCalledWith("agent-001");
+
+        monitor.stop();
+        vi.useRealTimers();
+      });
+
+      it("terminates unresponsive agent using per-agent timeout", async () => {
+        const onTerminated = vi.fn();
+        const agentStore = createStoreWithAgent({
+          id: "agent-001",
+          runtimeConfig: { heartbeatTimeoutMs: 5000 },
+        });
+        const session = createMockSession();
+
+        vi.useFakeTimers({ shouldAdvanceTime: true });
+        const monitor = new HeartbeatMonitor({
+          store,
+          agentStore,
+          pollIntervalMs: 1000,
+          heartbeatTimeoutMs: 60000, // Global default 60s — agent overrides to 5s
+          onTerminated,
+        });
+        monitor.start();
+        monitor.trackAgent("agent-001", session, "run-001");
+
+        // Wait for missed (5s) + termination at 2x timeout (10s)
+        vi.advanceTimersByTime(12000);
+        await vi.advanceTimersByTimeAsync(100);
+
+        expect(session.dispose).toHaveBeenCalled();
+        expect(onTerminated).toHaveBeenCalledWith("agent-001");
+
+        monitor.stop();
+        vi.useRealTimers();
+      });
+    });
+
+    describe("backward compatibility", () => {
+      it("works without agentStore (no per-agent config)", () => {
+        const monitor = new HeartbeatMonitor({
+          store,
+          heartbeatTimeoutMs: 5000,
+        });
+
+        const config = monitor.getAgentHeartbeatConfig("agent-001");
+        expect(config.heartbeatTimeoutMs).toBe(5000);
+        expect(config.pollIntervalMs).toBe(30000); // default
+        expect(config.maxConcurrentRuns).toBe(1); // default
+      });
+
+      it("existing isAgentHealthy works without per-agent config", () => {
+        const session = createMockSession();
+        vi.useFakeTimers({ shouldAdvanceTime: true });
+
+        const monitor = new HeartbeatMonitor({
+          store,
+          heartbeatTimeoutMs: 5000,
+        });
+        monitor.trackAgent("agent-001", session, "run-001");
+        expect(monitor.isAgentHealthy("agent-001")).toBe(true);
+
+        vi.advanceTimersByTime(6000);
+        expect(monitor.isAgentHealthy("agent-001")).toBe(false);
+
+        vi.useRealTimers();
+      });
+    });
+  });
 });
--- a/packages/engine/src/agent-heartbeat.ts
+++ b/packages/engine/src/agent-heartbeat.ts
@@ -10,12 +10,22 @@
 * - onTerminated: Called when an unresponsive agent is terminated
 */

-import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource } from "@fusion/core";
+import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig } from "@fusion/core";
+
+/** Resolved per-agent heartbeat config after validation and fallback */
+interface ResolvedHeartbeatConfig {
+  pollIntervalMs: number;
+  heartbeatTimeoutMs: number;
+  maxConcurrentRuns: number;
+}

 /** Options for HeartbeatMonitor constructor */
 export interface HeartbeatMonitorOptions {
  /** AgentStore instance for persistence */
  store: AgentStore;
+  /** Optional separate AgentStore reference for reading per-agent runtimeConfig.
+   *  If not provided, falls back to `store`. */
+  agentStore?: AgentStore;
  /** Polling interval in milliseconds (default: 30000) */
  pollIntervalMs?: number;
  /** Heartbeat timeout in milliseconds (default: 60000) */
@@ -67,6 +77,7 @@ interface TrackedAgent {
 */
 export class HeartbeatMonitor {
  private store: AgentStore;
+  private configStore: AgentStore;
  private pollIntervalMs: number;
  private heartbeatTimeoutMs: number;
  private maxConcurrentRuns: number;
@@ -83,6 +94,7 @@ export class HeartbeatMonitor {

  constructor(options: HeartbeatMonitorOptions) {
    this.store = options.store;
+    this.configStore = options.agentStore ?? options.store;
    this.pollIntervalMs = options.pollIntervalMs ?? 30000;
    this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? 60000;
    this.maxConcurrentRuns = options.maxConcurrentRuns ?? 1;
@@ -301,6 +313,8 @@ export class HeartbeatMonitor {

  /**
   * Check if an agent is healthy (heartbeat within timeout window).
+   * Uses per-agent heartbeatTimeoutMs from runtimeConfig if available,
+   * otherwise falls back to the monitor-level default.
   * @param agentId - The agent ID
   * @returns true if healthy, false if missed heartbeat or not tracked
   */
@@ -308,8 +322,9 @@ export class HeartbeatMonitor {
    const tracked = this.trackedAgents.get(agentId);
    if (!tracked) return false;

+    const config = this.getAgentConfig(agentId);
    const elapsed = Date.now() - tracked.lastSeen;
-    return elapsed < this.heartbeatTimeoutMs;
+    return elapsed < config.heartbeatTimeoutMs;
  }

  /**
@@ -333,13 +348,62 @@ export class HeartbeatMonitor {
  // Private methods
  // ─────────────────────────────────────────────────────────────────────────

+  /**
+   * Get the resolved heartbeat configuration for an agent.
+   * Reads per-agent config from runtimeConfig with fallback to monitor defaults.
+   * @param agentId - The agent ID
+   * @returns Resolved config with validated values
+   */
+  getAgentHeartbeatConfig(agentId: string): ResolvedHeartbeatConfig {
+    return this.getAgentConfig(agentId);
+  }
+
+  /**
+   * Resolve per-agent heartbeat config from runtimeConfig with validation and fallbacks.
+   */
+  private getAgentConfig(agentId: string): ResolvedHeartbeatConfig {
+    // Defaults from monitor-level construction
+    const result: ResolvedHeartbeatConfig = {
+      pollIntervalMs: this.pollIntervalMs,
+      heartbeatTimeoutMs: this.heartbeatTimeoutMs,
+      maxConcurrentRuns: this.maxConcurrentRuns,
+    };
+
+    try {
+      // Synchronous read — AgentStore.getAgent is async, but we can't make this
+      // method async without changing the call chain. Instead, we'll resolve
+      // per-agent config on the checkMissedHeartbeats path (which is async).
+      // For synchronous callers (isAgentHealthy), we use a cached approach.
+      // For simplicity, we read from the store's underlying agent data.
+      const agent = this.configStore.getCachedAgent?.(agentId);
+      if (agent?.runtimeConfig) {
+        const rc = agent.runtimeConfig;
+
+        if (typeof rc.heartbeatIntervalMs === "number" && Number.isFinite(rc.heartbeatIntervalMs)) {
+          result.pollIntervalMs = Math.max(1000, rc.heartbeatIntervalMs);
+        }
+        if (typeof rc.heartbeatTimeoutMs === "number" && Number.isFinite(rc.heartbeatTimeoutMs)) {
+          result.heartbeatTimeoutMs = Math.max(5000, rc.heartbeatTimeoutMs);
+        }
+        if (typeof rc.maxConcurrentRuns === "number" && Number.isFinite(rc.maxConcurrentRuns)) {
+          result.maxConcurrentRuns = Math.max(1, Math.round(rc.maxConcurrentRuns));
+        }
+      }
+    } catch {
+      // If agent lookup fails, use monitor defaults
+    }
+
+    return result;
+  }
+
  private async checkMissedHeartbeats(): Promise<void> {
    const now = Date.now();

    for (const tracked of this.trackedAgents.values()) {
+      const config = this.getAgentConfig(tracked.agentId);
      const elapsed = now - tracked.lastSeen;

-      if (elapsed >= this.heartbeatTimeoutMs) {
+      if (elapsed >= config.heartbeatTimeoutMs) {
        // Missed heartbeat detected
        if (!tracked.missedHeartbeatReported) {
          tracked.missedHeartbeatReported = true;
@@ -347,7 +411,7 @@ export class HeartbeatMonitor {
        } else {
          // Already reported - check if we should terminate
          // Give 2x timeout for recovery before auto-terminate
-          if (elapsed >= this.heartbeatTimeoutMs * 2) {
+          if (elapsed >= config.heartbeatTimeoutMs * 2) {
            await this.terminateUnresponsive(tracked);
          }
        }
--- a/packages/engine/src/runtimes/in-process-runtime.ts
+++ b/packages/engine/src/runtimes/in-process-runtime.ts
@@ -213,6 +213,7 @@ export class InProcessRuntime

        this.heartbeatMonitor = new HeartbeatMonitor({
          store: this.agentStore,
+          agentStore: this.agentStore, // enables per-agent config resolution
          onMissed: (agentId) => {
            runtimeLog.warn(`Agent ${agentId} missed heartbeat`);
          },