/** * HeartbeatMonitor - Runtime monitoring and execution for agents * * Monitors agents via periodic polling, detects missed heartbeats, * and provides the Paperclip-style heartbeat execution engine: * * wake → check inbox → work → exit * * When `executeHeartbeat()` is called (via API, timer, or assignment), * the system wakes the agent, checks its assigned task from AgentStore, * executes work in a lightweight agent session with `fn_task_create` capability, * records results, and transitions the run to completed. * * Callback pattern (not EventEmitter): * - onMissed: Called when an agent misses its heartbeat * - onRecovered: Called when an agent recovers after a missed heartbeat * - onTerminated: Called when an unresponsive agent is terminated */ import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig, AgentBudgetStatus, Message, MessageStore, TaskStore, TaskDetail, AgentRole, Agent, InboxTask, BlockedStateSnapshot, RunMutationContext, Settings } from "@fusion/core"; import { buildExecutionMemoryInstructions, isEphemeralAgent, hasAgentIdentity } from "@fusion/core"; import type { ToolDefinition } from "@mariozechner/pi-coding-agent"; import { Type, type Static } from "@mariozechner/pi-ai"; import { createTaskCreateTool, createTaskLogToolWithContext, createTaskDocumentWriteTool, createTaskDocumentReadTool, createListAgentsTool, createDelegateTaskTool, createSendMessageTool, createReadMessagesTool, createMemoryTools, taskCreateParams } from "./agent-tools.js"; import { AgentLogger } from "./agent-logger.js"; import { resolveAgentInstructionsWithRatings, buildSystemPromptWithInstructions } from "./agent-instructions.js"; import { heartbeatLog, formatError } from "./logger.js"; import { createRunAuditor, type EngineRunContext } from "./run-audit.js"; // Lazy import for pi — avoids pulling the pi SDK into the module graph // when heartbeat execution isn't needed. /** Resolved per-agent heartbeat config after validation and fallback */ interface ResolvedHeartbeatConfig { pollIntervalMs: number; heartbeatTimeoutMs: number; maxConcurrentRuns: number; } /** Options for HeartbeatMonitor constructor */ export interface HeartbeatMonitorOptions { /** AgentStore instance for persistence */ store: AgentStore; /** Optional separate AgentStore reference for reading per-agent runtimeConfig. * If not provided, falls back to `store`. */ agentStore?: AgentStore; /** Optional MessageStore for wake-on-message behavior */ messageStore?: MessageStore; /** Polling interval in milliseconds (default: 3600000) */ pollIntervalMs?: number; /** Heartbeat timeout in milliseconds (default: 60000) */ heartbeatTimeoutMs?: number; /** Max concurrent runs per agent (default: 1) */ maxConcurrentRuns?: number; /** Callback when an agent misses its heartbeat */ onMissed?: (agentId: string) => void; /** Callback when an agent recovers after a missed heartbeat */ onRecovered?: (agentId: string) => void; /** Callback when an unresponsive agent is terminated */ onTerminated?: (agentId: string) => void; /** Callback when a run starts */ onRunStarted?: (agentId: string, run: AgentHeartbeatRun) => void; /** Callback when a run completes */ onRunCompleted?: (agentId: string, run: AgentHeartbeatRun) => void; /** TaskStore for fn_task_create and fn_task_log tools during heartbeat execution. * When not provided, executeHeartbeat() will throw. */ taskStore?: TaskStore; /** Project root directory for agent session CWD. * When not provided, executeHeartbeat() will throw. */ rootDir?: string; /** Plugin runner for runtime selection. When provided, enables plugin runtime lookup. */ pluginRunner?: import("./plugin-runner.js").PluginRunner; } /** Options for waking up an agent */ export interface WakeupOptions { /** What triggered the wakeup */ source: HeartbeatInvocationSource; /** Detail about the trigger (manual, ping, scheduler, system) */ triggerDetail?: string; /** Context snapshot for the run */ contextSnapshot?: Record; } /** Options for executing a heartbeat run */ export interface HeartbeatExecutionOptions { /** Agent ID to execute heartbeat for */ agentId: string; /** What triggered this heartbeat */ source: HeartbeatInvocationSource; /** Human-readable trigger detail */ triggerDetail?: string; /** Optional task ID override (uses agent.taskId if not set) */ taskId?: string; /** IDs of comments that triggered this wake (if any) */ triggeringCommentIds?: string[]; /** Type of comment that triggered this wake */ triggeringCommentType?: "steering" | "task" | "pr"; /** Optional structured context persisted on the run record */ contextSnapshot?: Record; } /** Session interface for disposing agent resources */ export interface AgentSession { /** Dispose the agent session (stop execution, cleanup resources) */ dispose(): void; } /** In-memory tracking data for a monitored agent */ interface TrackedAgent { agentId: string; session: AgentSession; runId: string; lastSeen: number; // timestamp from Date.now() missedHeartbeatReported: boolean; /** Session ID before this execution started */ sessionIdBefore?: string; } /** Compare blocked-state snapshots to decide whether blocked messaging is duplicate noise. */ export function isBlockedStateDuplicate(current: BlockedStateSnapshot, previous: BlockedStateSnapshot): boolean { return current.blockedBy === previous.blockedBy && current.contextHash === previous.contextHash; } /** * System prompt for heartbeat agent sessions. * Instructs the agent to perform a single-pass check on its assigned task * and use `fn_task_create` / `fn_task_log` / `fn_task_document_*` tools to record findings or spawn follow-up work. */ export const HEARTBEAT_SYSTEM_PROMPT = `You are a heartbeat agent running in a short execution window. Your job: 1. Check your assigned task — read the description and PROMPT.md if present. 2. Do ONE useful action: analyze, review, create follow-up tasks, or log findings. 3. Use fn_task_create to spawn follow-up work, fn_task_log to record observations. 4. Use fn_task_document_write to save durable findings, plans, or research notes. 5. Call fn_heartbeat_done when finished with an optional summary of what was accomplished. Keep work lightweight — this is a single-pass check, not a full implementation run. You have readonly file access plus fn_task_create, fn_task_log, and fn_task_document tools. **Task Documents:** Save important findings with fn_task_document_write(key="...", content="..."). Documents persist across sessions and are visible in the dashboard's Documents tab. ## Memory Boundaries You may receive an Agent Memory section and a Project Memory section. - Agent Memory is specific to you, including imported and user-created agents such as CEO-style coordinator agents. It has its own long-term memory, daily notes, dreams, and qmd-backed retrieval under .fusion/agent-memory/{agentId}/. - Project Memory is the workspace memory system under .fusion/memory/ with long-term memory, daily notes, dreams, and qmd-backed retrieval. - Keep these separate: do not copy personal agent operating notes into Project Memory unless they are genuinely useful to every future agent in this workspace. ## Processing Messages When you are woken by an incoming message (source includes "wake-on-message"), you should: 1. Use fn_read_messages to check your inbox for unread messages. 2. Review each message and determine the appropriate action: - If the message requires a response, use fn_send_message to reply. - When replying, include 'reply_to_message_id' with the original message ID from fn_read_messages output. - If the message is informational, acknowledge it by logging with fn_task_log. - If the message requests work, create a follow-up task with fn_task_create or handle it directly. 3. After processing messages, continue with your normal heartbeat duties. When sending messages: - Be concise and clear about what you need or what you've done. - Use 'reply_to_message_id' when replying so threaded conversations stay linked. - Include relevant context (task IDs, file paths) in metadata when applicable. - Use agent-to-agent for inter-agent communication.`; /** * System prompt for no-task heartbeat agent sessions. * Instructs the agent to perform ambient work only with tools that do not require task context. */ export const HEARTBEAT_NO_TASK_SYSTEM_PROMPT = `You are a heartbeat agent running in a short execution window with no task assignment. Your job: 1. Review your context — check messages, memory, and project state. 2. Do ONE useful action: analyze, create follow-up tasks, delegate work, or update memory. 3. Use fn_task_create to spawn follow-up work. 4. Use fn_list_agents and fn_delegate_task to coordinate with other agents. 5. Call fn_heartbeat_done when finished with an optional summary of what was accomplished. Keep work lightweight — this is a single-pass ambient check, not a full implementation run. You have readonly file access plus: - fn_task_create - fn_list_agents and fn_delegate_task - fn_memory_search, fn_memory_get, and fn_memory_append - fn_heartbeat_done - fn_send_message and fn_read_messages when messaging is enabled for this run (they may not always be available) ## Memory Boundaries You may receive an Agent Memory section and a Project Memory section. - Agent Memory is specific to you, including imported and user-created agents such as CEO-style coordinator agents. It has its own long-term memory, daily notes, dreams, and qmd-backed retrieval under .fusion/agent-memory/{agentId}/. - Project Memory is the workspace memory system under .fusion/memory/ with long-term memory, daily notes, dreams, and qmd-backed retrieval. - Keep these separate: do not copy personal agent operating notes into Project Memory unless they are genuinely useful to every future agent in this workspace. ## Processing Messages When you are woken by an incoming message (source includes "wake-on-message"), you should: 1. If fn_read_messages is available, use it to check your inbox for unread messages. 2. Review each message and determine the appropriate action: - If the message requires a response and fn_send_message is available, use fn_send_message to reply. - When replying, include 'reply_to_message_id' with the original message ID from fn_read_messages output. - If the message is informational, acknowledge it and respond via fn_send_message when appropriate. - If the message requests work, create a follow-up task with fn_task_create. 3. After processing messages, continue with your ambient work. When sending messages: - Be concise and clear about what you need or what you've done. - Use 'reply_to_message_id' when replying so threaded conversations stay linked. - Include relevant context (task IDs, file paths) in metadata when applicable. - Use agent-to-agent for inter-agent communication.`; // Backward-compatible alias; prefer HEARTBEAT_NO_TASK_SYSTEM_PROMPT. export const HEARTBEAT_SYSTEM_PROMPT_NO_TASK = HEARTBEAT_NO_TASK_SYSTEM_PROMPT; /** Parameter schema for the fn_heartbeat_done tool */ const heartbeatDoneParams = Type.Object({ summary: Type.Optional(Type.String({ description: "Summary of what was accomplished this heartbeat" })), }); async function getHeartbeatMemorySettings(taskStore: TaskStore): Promise { const maybeGetSettings = (taskStore as { getSettings?: () => Promise }).getSettings; if (!maybeGetSettings) { return undefined; } return maybeGetSettings.call(taskStore); } /** * HeartbeatMonitor monitors agents via periodic polling. * Detects missed heartbeats, auto-terminates unresponsive agents, * and provides the Paperclip-style execution engine via executeHeartbeat(). */ export class HeartbeatMonitor { private store: AgentStore; private configStore: AgentStore; private pollIntervalMs: number; private heartbeatTimeoutMs: number; private maxConcurrentRuns: number; private onMissed?: (agentId: string) => void; private onRecovered?: (agentId: string) => void; private onTerminated?: (agentId: string) => void; private onRunStarted?: (agentId: string, run: AgentHeartbeatRun) => void; private onRunCompleted?: (agentId: string, run: AgentHeartbeatRun) => void; private taskStore?: TaskStore; private rootDir?: string; private messageStore?: MessageStore; private pluginRunner?: import("./plugin-runner.js").PluginRunner; private trackedAgents: Map = new Map(); private agentStartLocks: Map> = new Map(); private pollInterval: NodeJS.Timeout | null = null; private isRunning = false; /** Tasks created per agent during heartbeat runs (keyed by agentId) */ private runCreatedTasks: Map> = new Map(); constructor(options: HeartbeatMonitorOptions) { this.store = options.store; this.configStore = options.agentStore ?? options.store; this.pollIntervalMs = options.pollIntervalMs ?? 3_600_000; this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? 60000; this.maxConcurrentRuns = options.maxConcurrentRuns ?? 1; this.onMissed = options.onMissed; this.onRecovered = options.onRecovered; this.onTerminated = options.onTerminated; this.onRunStarted = options.onRunStarted; this.onRunCompleted = options.onRunCompleted; this.taskStore = options.taskStore; this.rootDir = options.rootDir; this.messageStore = options.messageStore; this.pluginRunner = options.pluginRunner; this.onRecovered = options.onRecovered; this.onTerminated = options.onTerminated; this.onRunStarted = options.onRunStarted; this.onRunCompleted = options.onRunCompleted; this.taskStore = options.taskStore; this.rootDir = options.rootDir; this.messageStore = options.messageStore; } /** * Start the heartbeat monitoring loop. * Safe to call multiple times - no-op if already running. */ start(): void { if (this.isRunning) return; this.isRunning = true; if (this.messageStore) { this.messageStore.setMessageToAgentHook(this.handleMessageToAgent.bind(this)); } this.pollInterval = setInterval(() => { void this.checkMissedHeartbeats(); }, this.pollIntervalMs); } /** * Stop the heartbeat monitoring loop. * Does not untrack agents - they remain in memory. */ stop(): void { if (this.messageStore) { this.messageStore.setMessageToAgentHook(() => {}); } if (!this.isRunning) return; this.isRunning = false; if (this.pollInterval) { clearInterval(this.pollInterval); this.pollInterval = null; } } /** * Check if the monitor is currently running. */ isActive(): boolean { return this.isRunning; } /** * Get the project root directory this monitor is bound to. * Returns undefined when not configured for execution. */ getRootDir(): string | undefined { return this.rootDir; } /** * Register an agent for monitoring with optional session context. * @param agentId - The agent ID * @param session - Session with dispose() for cleanup * @param runId - The heartbeat run ID * @param sessionIdBefore - Optional session ID from before execution */ trackAgent(agentId: string, session: AgentSession, runId: string, sessionIdBefore?: string): void { const tracked: TrackedAgent = { agentId, session, runId, lastSeen: Date.now(), missedHeartbeatReported: false, sessionIdBefore, }; this.trackedAgents.set(agentId, tracked); // Record initial heartbeat void this.store.recordHeartbeat(agentId, "ok", runId); } /** * Serialize run starts per agent to prevent concurrent execution. * @param agentId - The agent ID * @param fn - Function to execute with the lock */ async withAgentStartLock(agentId: string, fn: () => Promise): Promise { const existing = this.agentStartLocks.get(agentId) ?? Promise.resolve(); const operation = existing.then( async () => { try { return await fn(); } finally { // Clean up accumulated run state for this agent at end of each serialized run. // This guarantees cleanup even when the run path throws without calling completeRun // (e.g., execution error before completeRun is reached, or completeRun itself throws). // Because withAgentStartLock serializes runs per agent, the finally runs after each // run completes but before the next concurrent call's callback starts. this.clearRunState(agentId); } }, async (err) => { try { throw err; } finally { this.clearRunState(agentId); } }, ); this.agentStartLocks.set(agentId, operation); return operation as Promise; } /** * Start a rich heartbeat run with full context capture. * Creates a structured run record and saves it to the run store. * @param agentId - The agent ID * @param options - Wakeup options with trigger context * @returns The created run */ async startRun(agentId: string, options?: WakeupOptions): Promise { // Safety net: fail any existing active runs for this agent before creating a new one. // This prevents accumulation of zombie runs when startRun is called multiple times // (e.g., concurrent timer + on-demand triggers, or retries after crashes). try { const existingRun = await this.store.getActiveHeartbeatRun(agentId); if (existingRun) { heartbeatLog.warn( `Agent ${agentId} has active run ${existingRun.id} — marking failed before starting new run`, ); try { const existingDetail = await this.store.getRunDetail(agentId, existingRun.id); if (existingDetail) { await this.store.saveRun({ ...existingDetail, endedAt: new Date().toISOString(), status: "terminated", stderrExcerpt: "Superseded by new heartbeat run (previous run was stale)", }); } await this.store.endHeartbeatRun(existingRun.id, "terminated"); this.clearRunState(agentId); } catch (failErr) { const failErrMessage = failErr instanceof Error ? failErr.message : String(failErr); heartbeatLog.warn( `Failed to terminate stale active run ${existingRun.id} for ${agentId}: ${failErrMessage} — continuing anyway`, ); } } } catch (activeRunCheckErr) { const msg = activeRunCheckErr instanceof Error ? activeRunCheckErr.message : String(activeRunCheckErr); heartbeatLog.warn(`Failed to check for existing active run for ${agentId}: ${msg} — continuing with new run`); } const run = await this.store.startHeartbeatRun(agentId); // Enrich with execution context const enrichedRun: AgentHeartbeatRun = { ...run, invocationSource: options?.source ?? "on_demand", triggerDetail: options?.triggerDetail ?? "manual", contextSnapshot: options?.contextSnapshot, processPid: process.pid, }; // Save rich run data await this.store.saveRun(enrichedRun); // Transition agent to running state try { await this.store.updateAgentState(agentId, "running"); } catch (startRunErr) { heartbeatLog.warn(`updateAgentState(running) failed for ${agentId}: ${startRunErr instanceof Error ? startRunErr.message : String(startRunErr)} — continuing`); } this.onRunStarted?.(agentId, enrichedRun); return enrichedRun; } /** * Complete a heartbeat run with results. * @param agentId - The agent ID * @param runId - The run ID to complete * @param result - Execution results */ async completeRun( agentId: string, runId: string, result: { status: "completed" | "failed" | "terminated"; exitCode?: number; sessionIdAfter?: string; usageJson?: { inputTokens: number; outputTokens: number; cachedTokens: number }; resultJson?: Record; stdoutExcerpt?: string; stderrExcerpt?: string; /** When true, preserve current agent state instead of forcing a terminal transition. */ skipStateTransition?: boolean; } ): Promise { // Load and update the run const run = await this.store.getRunDetail(agentId, runId); if (!run) return; const tracked = this.trackedAgents.get(agentId); let completionResult = result; // Merge accumulated task creations into resultJson const createdTasks = this.runCreatedTasks.get(agentId); const enrichedResultJson = createdTasks?.length ? { ...completionResult.resultJson, tasksCreated: createdTasks } : completionResult.resultJson; const completedRun: AgentHeartbeatRun = { ...run, endedAt: new Date().toISOString(), status: completionResult.status, exitCode: completionResult.exitCode, sessionIdBefore: tracked?.sessionIdBefore, sessionIdAfter: completionResult.sessionIdAfter, usageJson: completionResult.usageJson, resultJson: enrichedResultJson, stdoutExcerpt: completionResult.stdoutExcerpt, stderrExcerpt: completionResult.stderrExcerpt, }; await this.store.saveRun(completedRun); // Clear accumulated run state for this agent. // Safe to call even when runCreatedTasks was already cleared by withAgentStartLock's // finally block (idempotent Map.delete), and necessary for direct completeRun calls // that bypass the lock (e.g., test scenarios, edge-case error paths). this.clearRunState(agentId); // Update cumulative usage on agent if (completionResult.usageJson) { try { const agent = await this.store.getAgent(agentId); if (agent) { await this.store.updateAgent(agentId, { totalInputTokens: (agent.totalInputTokens ?? 0) + completionResult.usageJson.inputTokens, totalOutputTokens: (agent.totalOutputTokens ?? 0) + completionResult.usageJson.outputTokens, }); } } catch (usageUpdateErr) { heartbeatLog.warn(`Agent ${agentId} usage update failed: ${usageUpdateErr instanceof Error ? usageUpdateErr.message : String(usageUpdateErr)} — continuing`); } } // Budget governance: pause agent if over budget after usage update if (completionResult.usageJson && completionResult.status !== "failed" && completionResult.status !== "terminated") { try { const budgetStatus = await this.store.getBudgetStatus(agentId); if (budgetStatus.isOverBudget) { heartbeatLog.log(`Agent ${agentId} is over budget — pausing with reason "budget-exhausted"`); await this.store.updateAgentState(agentId, "paused"); await this.store.updateAgent(agentId, { pauseReason: "budget-exhausted" }); // Skip the normal state transition below since we already set the correct state completionResult = { ...completionResult, skipStateTransition: true }; } } catch (budgetCheckErr) { heartbeatLog.warn(`Agent ${agentId} budget check failed: ${budgetCheckErr instanceof Error ? budgetCheckErr.message : String(budgetCheckErr)} — proceeding with normal state transition`); } } // Transition agent state based on result if (!completionResult.skipStateTransition) { try { if (completionResult.status === "failed") { await this.store.updateAgentState(agentId, "error"); await this.store.updateAgent(agentId, { lastError: completionResult.stderrExcerpt ?? "Run failed" }); } else if (completionResult.status === "terminated") { await this.store.updateAgentState(agentId, "terminated"); } else { // Completed successfully - back to active await this.store.updateAgentState(agentId, "active"); } } catch (stateTransErr) { heartbeatLog.warn(`Agent ${agentId} state transition failed: ${stateTransErr instanceof Error ? stateTransErr.message : String(stateTransErr)} — continuing`); } } // End the heartbeat run tracking await this.store.endHeartbeatRun(runId, completionResult.status === "completed" ? "completed" : "terminated"); this.onRunCompleted?.(agentId, completedRun); } /** * Stop an active heartbeat run for an agent. * * If an in-memory tracked session exists, dispose it and complete the run as terminated. * If no tracked session exists, fall back to persisted active-run state and terminate that run record. * * No-op when no active run exists. */ async stopRun(agentId: string): Promise { const tracked = this.trackedAgents.get(agentId); if (tracked) { heartbeatLog.log(`Stopping tracked run ${tracked.runId} for ${agentId}`); try { tracked.session.dispose(); } catch (error) { heartbeatLog.warn(`Failed to dispose tracked session while stopping run for ${agentId}: ${error instanceof Error ? error.message : String(error)}`); } this.untrackAgent(agentId); await this.completeRun(agentId, tracked.runId, { status: "terminated", stderrExcerpt: "Run stopped by user", }); try { await this.store.updateAgentState(agentId, "active"); } catch (stopStateErr) { heartbeatLog.warn(`Agent ${agentId} updateAgentState(active) failed during stop: ${stopStateErr instanceof Error ? stopStateErr.message : String(stopStateErr)}`); } this.clearRunState(agentId); return; } const activeRun = await this.store.getActiveHeartbeatRun(agentId); if (!activeRun) { this.clearRunState(agentId); return; } heartbeatLog.log(`Stopping persisted run ${activeRun.id} for ${agentId} (no tracked session)`); const existingRun = await this.store.getRunDetail(agentId, activeRun.id); if (existingRun) { await this.store.saveRun({ ...existingRun, endedAt: new Date().toISOString(), status: "terminated", stderrExcerpt: existingRun.stderrExcerpt ?? "Run stopped by user", }); } await this.store.endHeartbeatRun(activeRun.id, "terminated"); try { await this.store.updateAgentState(agentId, "active"); } catch (stopPersistErr) { heartbeatLog.warn(`Agent ${agentId} updateAgentState(active) failed during persisted-run stop: ${stopPersistErr instanceof Error ? stopPersistErr.message : String(stopPersistErr)}`); } this.clearRunState(agentId); } /** * Remove an agent from monitoring. * Does NOT end the heartbeat run - caller's responsibility. * @param agentId - The agent ID */ untrackAgent(agentId: string): void { this.trackedAgents.delete(agentId); } /** * Record a heartbeat for a tracked agent. * @param agentId - The agent ID */ recordHeartbeat(agentId: string): void { const tracked = this.trackedAgents.get(agentId); if (!tracked) return; tracked.lastSeen = Date.now(); // If recovering from a missed heartbeat if (tracked.missedHeartbeatReported) { tracked.missedHeartbeatReported = false; void this.store.recordHeartbeat(agentId, "recovered", tracked.runId); this.onRecovered?.(agentId); } else { void this.store.recordHeartbeat(agentId, "ok", tracked.runId); } } /** * Check if an agent is healthy (heartbeat within timeout window). * Uses per-agent heartbeatTimeoutMs from runtimeConfig if available, * otherwise falls back to the monitor-level default. * @param agentId - The agent ID * @returns true if healthy, false if missed heartbeat or not tracked */ isAgentHealthy(agentId: string): boolean { const tracked = this.trackedAgents.get(agentId); if (!tracked) return false; const config = this.resolveAgentConfig(agentId); const elapsed = Date.now() - tracked.lastSeen; return elapsed < config.heartbeatTimeoutMs; } /** * Get list of currently tracked agent IDs. * Useful for testing and debugging. */ getTrackedAgents(): string[] { return Array.from(this.trackedAgents.keys()); } /** * Get the last seen timestamp for a tracked agent. * @param agentId - The agent ID * @returns Last seen timestamp, or undefined if not tracked */ getLastSeen(agentId: string): number | undefined { return this.trackedAgents.get(agentId)?.lastSeen; } private handleMessageToAgent(message: Message): void { if (message.toType !== "agent") { return; } const agent = this.configStore.getCachedAgent(message.toId); if (!agent) { return; } const runtimeConfig = agent.runtimeConfig as AgentHeartbeatConfig | undefined; if (runtimeConfig?.messageResponseMode !== "immediate") { return; } const validStates = new Set(["active", "idle", "running"]); if (!validStates.has(agent.state)) { return; } void this.executeHeartbeat({ agentId: message.toId, source: "on_demand", triggerDetail: "wake-on-message", }).catch((error) => { const errorMessage = error instanceof Error ? error.message : String(error); heartbeatLog.warn(`Wake-on-message heartbeat failed for ${message.toId}: ${errorMessage}`); }); } // ───────────────────────────────────────────────────────────────────────── // Heartbeat execution (Paperclip wake → check → work → exit) // ───────────────────────────────────────────────────────────────────────── /** * Execute a heartbeat run for an agent. * * Implements the Paperclip-style execution model: * 1. Wake — start a heartbeat run record * 2. Check inbox — resolve the agent's assigned task * 3. Work — run a lightweight agent session with readonly tools + fn_task_create/fn_task_log * 4. Exit — record results and complete the run * * Budget governance: * - Skip all triggers when the agent is over budget (`isOverBudget`) * - Skip timer triggers when over the warning threshold (`isOverThreshold`) * - Continue normal execution for critical triggers (assignment/on_demand) when only over threshold * * Per-agent execution is serialized via `withAgentStartLock` — concurrent calls * for the same agent wait for the previous run to complete. * * @param options - Execution options (agent ID, source, optional task override) * @returns The completed heartbeat run, or null if the monitor isn't configured for execution * @throws Error if taskStore or rootDir are not configured */ async executeHeartbeat(options: HeartbeatExecutionOptions): Promise { const { agentId, source, triggerDetail, taskId: explicitTaskId, contextSnapshot, triggeringCommentIds, triggeringCommentType, } = options; // Validate execution dependencies if (!this.taskStore || !this.rootDir) { throw new Error("HeartbeatMonitor not configured for execution (missing taskStore/rootDir)"); } const taskStore = this.taskStore; const rootDir = this.rootDir; // Serialize per-agent return this.withAgentStartLock(agentId, async () => { heartbeatLog.log(`Executing heartbeat for ${agentId} (source=${source})`); let preloadedAgent: Agent | null = null; try { preloadedAgent = await this.store.getAgent(agentId); } catch (preloadErr) { heartbeatLog.warn(`Agent ${agentId} agent preloading failed: ${preloadErr instanceof Error ? preloadErr.message : String(preloadErr)} — will resolve in execution path`); } const resolvedTaskId = explicitTaskId ?? preloadedAgent?.taskId; const contextTriggeringCommentIds = Array.isArray(contextSnapshot?.triggeringCommentIds) ? contextSnapshot.triggeringCommentIds.filter((id): id is string => typeof id === "string" && id.length > 0) : undefined; const contextTriggeringCommentType = contextSnapshot?.triggeringCommentType === "steering" || contextSnapshot?.triggeringCommentType === "task" || contextSnapshot?.triggeringCommentType === "pr" ? contextSnapshot.triggeringCommentType : undefined; const effectiveTriggeringCommentIds = triggeringCommentIds ?? contextTriggeringCommentIds; const effectiveTriggeringCommentType = triggeringCommentType ?? contextTriggeringCommentType; const runContextSnapshot = { ...(contextSnapshot ?? {}), ...(resolvedTaskId ? { taskId: resolvedTaskId } : {}), ...(effectiveTriggeringCommentIds?.length ? { triggeringCommentIds: effectiveTriggeringCommentIds } : {}), ...(effectiveTriggeringCommentType ? { triggeringCommentType: effectiveTriggeringCommentType } : {}), }; // Start run const run = await this.startRun(agentId, { source, triggerDetail, contextSnapshot: Object.keys(runContextSnapshot).length > 0 ? runContextSnapshot : undefined, }); // Build run context for mutation correlation const runContext: RunMutationContext = { runId: run.id, agentId, source, }; // Build engine run context for audit instrumentation const engineRunContext: EngineRunContext = { runId: run.id, agentId, source, phase: "heartbeat", }; // Create run auditor for audit trail (FN-1404) // Uses TaskStore.recordRunAuditEvent when available; no-ops otherwise const audit = createRunAuditor(taskStore, engineRunContext); let agentLogger: AgentLogger | null = null; const flushAgentLogger = async (): Promise => { if (!agentLogger) { return; } try { await agentLogger.flush(); } catch (error) { heartbeatLog.warn(`Failed to flush heartbeat logs for ${agentId}: ${error instanceof Error ? error.message : String(error)}`); } }; try { // Budget governance: check if agent can run try { const budgetStatus = await this.store.getBudgetStatus(agentId); if (budgetStatus.isOverBudget) { heartbeatLog.log(`Agent ${agentId} budget exhausted — heartbeat skipped`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "budget_exhausted", budgetStatus }, skipStateTransition: true, }); return (await this.store.getRunDetail(agentId, run.id))!; } // Above threshold: only allow critical triggers (assignment, on_demand) if (budgetStatus.isOverThreshold && source === "timer") { heartbeatLog.log(`Agent ${agentId} over budget threshold (${budgetStatus.usagePercent}%) — timer heartbeat skipped`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "budget_threshold_exceeded", budgetStatus }, skipStateTransition: true, }); return (await this.store.getRunDetail(agentId, run.id))!; } } catch (budgetErr) { heartbeatLog.warn(`Agent ${agentId} budget status check failed: ${budgetErr instanceof Error ? budgetErr.message : String(budgetErr)} — proceeding without budget check`); } // Resolve agent const agent = preloadedAgent ?? await this.store.getAgent(agentId); if (!agent) { heartbeatLog.warn(`Agent ${agentId} not found — completing run as failed`); await this.completeRun(agentId, run.id, { status: "failed", stderrExcerpt: `Agent ${agentId} not found`, }); return (await this.store.getRunDetail(agentId, run.id))!; } // Check if agent has identity (used later for no-task run decisions) const agentHasIdentity = hasAgentIdentity(agent); const isAgentEphemeral = isEphemeralAgent(agent); // Resolve task assignment (explicit override → existing assignment → inbox-lite selection) let taskId = explicitTaskId ?? agent.taskId; let inboxSelection: InboxTask | null = null; if (!taskId) { inboxSelection = await taskStore.selectNextTaskForAgent(agentId); if (inboxSelection) { taskId = inboxSelection.task.id; heartbeatLog.log(`Inbox selected task ${taskId} (priority: ${inboxSelection.priority}) for agent ${agentId}`); // Persist assignment to AgentStore so subsequent runs retain linkage. if (agent.taskId !== taskId) { await this.store.assignTask(agentId, taskId, runContext); // Audit trail: record assignment mutation (FN-1404) await audit.database({ type: "task:assign", target: taskId }); } // FN-1253 compatibility: if checkout API is available on TaskStore, // try to claim the lease. On conflict, skip this task gracefully. const checkoutTask = (taskStore as TaskStore & { checkoutTask?: (taskId: string, agentId: string, runContext?: RunMutationContext) => Promise; }).checkoutTask; if (typeof checkoutTask === "function") { try { await checkoutTask.call(taskStore, taskId, agentId, runContext); // Audit trail: record checkout mutation (FN-1404) await audit.database({ type: "task:checkout", target: taskId }); } catch (checkoutErr) { heartbeatLog.warn(`Task ${taskId} checkout failed: ${checkoutErr instanceof Error ? checkoutErr.message : String(checkoutErr)} — skipping`); taskId = undefined; inboxSelection = null; } } } } if (taskId && run.contextSnapshot?.taskId !== taskId) { const updatedRun: AgentHeartbeatRun = { ...run, contextSnapshot: { ...(run.contextSnapshot ?? {}), taskId, }, }; await this.store.saveRun(updatedRun); // Update engine run context with resolved taskId for audit trail (FN-1404) engineRunContext.taskId = taskId; } if (!taskId) { // Agents with identity (soul, instructions, memory) should run a full heartbeat // session even without a task, so they can do ambient work like messaging, // memory management, task creation, and delegation. // Ephemeral agents and agents without identity still exit gracefully. if (!agentHasIdentity || isAgentEphemeral) { heartbeatLog.log(`Agent ${agentId} has no task assignment — graceful exit`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "no_assignment" }, }); return (await this.store.getRunDetail(agentId, run.id))!; } heartbeatLog.log(`Agent ${agentId} has no task but has identity — running no-task heartbeat`); } const isNoTaskRun = !taskId; // Validate agent state (only for task-scoped runs) if (!isNoTaskRun) { const validStates = ["active", "running", "idle"]; if (!validStates.includes(agent.state)) { heartbeatLog.log(`Agent ${agentId} state is "${agent.state}" — graceful exit`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "invalid_state", state: agent.state }, skipStateTransition: true, }); return (await this.store.getRunDetail(agentId, run.id))!; } } // Fetch task context (only for task-scoped runs) let taskDetail: TaskDetail | undefined; if (!isNoTaskRun) { // taskId is guaranteed to be defined here because isNoTaskRun = !taskId const resolvedTaskId = taskId!; try { taskDetail = await taskStore.getTask(resolvedTaskId); } catch (taskDetailErr) { heartbeatLog.warn(`Task ${resolvedTaskId} fetch failed: ${taskDetailErr instanceof Error ? taskDetailErr.message : String(taskDetailErr)} — graceful exit`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "task_not_found", taskId: resolvedTaskId }, }); return (await this.store.getRunDetail(agentId, run.id))!; } // Checkout enforcement: agent must hold the lease to work on this task. // The heartbeat only validates existing checkout state — it does NOT attempt // to acquire a checkout itself. The calling system (scheduler, API trigger) // is responsible for checking out the task before the heartbeat starts. if (taskDetail.checkedOutBy && taskDetail.checkedOutBy !== agentId) { heartbeatLog.warn( `Agent ${agentId} does not hold checkout for ${resolvedTaskId} (held by ${taskDetail.checkedOutBy}) — graceful exit` ); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "checkout_conflict", taskId: resolvedTaskId, checkedOutBy: taskDetail.checkedOutBy, }, }); return (await this.store.getRunDetail(agentId, run.id))!; } const blockedBy = typeof taskDetail.blockedBy === "string" ? taskDetail.blockedBy.trim() : ""; const isBlockedTask = taskDetail.status === "queued" && blockedBy.length > 0; if (isBlockedTask) { const commentCount = (taskDetail.comments?.length ?? 0) + (taskDetail.steeringComments?.length ?? 0); const lastCommentId = taskDetail.comments?.at(-1)?.id; const lastSteeringCommentId = taskDetail.steeringComments?.at(-1)?.id; const contextHash = Buffer.from( JSON.stringify({ commentCount, lastCommentId, lastSteeringCommentId, blockedBy }), ) .toString("base64") .slice(0, 16); const currentBlockedState: BlockedStateSnapshot = { taskId: resolvedTaskId, blockedBy, recordedAt: new Date().toISOString(), contextHash, }; const previousBlockedState = await this.store.getLastBlockedState(agentId); if (previousBlockedState && isBlockedStateDuplicate(currentBlockedState, previousBlockedState)) { heartbeatLog.log(`Task ${resolvedTaskId} is still blocked by ${blockedBy} (duplicate state) — skipping comment`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "blocked_duplicate", taskId: resolvedTaskId, blockedBy }, }); return (await this.store.getRunDetail(agentId, run.id))!; } const blockedMessage = `Task is blocked by ${blockedBy}; waiting for dependency/context changes before retrying.`; await taskStore.addComment(resolvedTaskId, blockedMessage, "agent", undefined, runContext); // Audit trail: record comment mutation (FN-1404) await audit.database({ type: "task:comment:add", target: resolvedTaskId, metadata: { blockedBy } }); await this.store.setLastBlockedState(agentId, currentBlockedState); heartbeatLog.log(`Task ${resolvedTaskId} is blocked by ${blockedBy} — recorded blocked state`); await this.completeRun(agentId, run.id, { status: "completed", resultJson: { reason: "blocked", taskId: resolvedTaskId, blockedBy }, }); return (await this.store.getRunDetail(agentId, run.id))!; } } // Clear blocked state when task is no longer blocked (only for task-scoped runs) if (!isNoTaskRun) { await this.store.clearLastBlockedState(agentId); } // Track usage via callbacks const STDOUT_EXCERPT_LIMIT = 4000; let outputLength = 0; let toolCallCount = 0; let heartbeatSummary: string | undefined; let stdoutExcerpt = ""; const appendStdoutExcerpt = (delta: string): void => { if (stdoutExcerpt.length >= STDOUT_EXCERPT_LIMIT) { return; } const remaining = STDOUT_EXCERPT_LIMIT - stdoutExcerpt.length; stdoutExcerpt += delta.slice(0, remaining); }; // Create fn_heartbeat_done tool const heartbeatDoneTool: ToolDefinition = { name: "fn_heartbeat_done", label: "Heartbeat Done", description: "Signal that the heartbeat execution is complete. Call when finished.", parameters: heartbeatDoneParams, execute: async (_id: string, params: Static) => { if (params.summary) { heartbeatSummary = params.summary; } return { content: [{ type: "text" as const, text: `Heartbeat complete.${params.summary ? ` Summary: ${params.summary}` : ""}`, }], details: {}, }; }, }; // Lazy-load promptWithFallback const { promptWithFallback } = await import("./pi.js"); const { createResolvedAgentSession } = await import("./agent-session-helpers.js"); const { buildSessionSkillContextSync } = await import("./session-skill-context.js"); // Build tools with task creation tracking and run context for mutation correlation // For no-task runs, exclude fn_task_log and document tools (they require a taskId) let heartbeatTools: ToolDefinition[]; if (isNoTaskRun) { // No-task runs: fn_task_create, fn_list_agents, fn_delegate_task, messaging, memory, fn_heartbeat_done heartbeatTools = []; // fn_task_create tool (no tracking needed for no-task runs) heartbeatTools.push(createTaskCreateTool(taskStore)); // Agent delegation tools heartbeatTools.push(createListAgentsTool(this.store)); heartbeatTools.push(createDelegateTaskTool(this.store, taskStore)); // Messaging tools — when MessageStore is available if (this.messageStore) { heartbeatTools.push(createSendMessageTool(this.messageStore, agentId)); heartbeatTools.push(createReadMessagesTool(this.messageStore, agentId)); } } else { // Task-scoped runs: full tool set including fn_task_log and document tools // taskId is guaranteed to be defined here because isNoTaskRun = !taskId heartbeatTools = this.createHeartbeatTools(agentId, taskStore, taskId!, runContext, audit, this.messageStore); } let memorySettings: Settings | undefined; try { memorySettings = await getHeartbeatMemorySettings(taskStore); heartbeatTools.push(...createMemoryTools(rootDir, memorySettings, { agentMemory: { agentId: agent.id, agentName: agent.name, memory: agent.memory, }, })); } catch (memorySettingsError) { const message = memorySettingsError instanceof Error ? memorySettingsError.message : String(memorySettingsError); heartbeatLog.warn(`Failed to configure heartbeat memory tools for ${agentId}: ${message}`); } heartbeatTools.push(heartbeatDoneTool); // AgentLogger requires a taskId — only create for task-scoped runs if (!isNoTaskRun && taskId) { agentLogger = new AgentLogger({ store: taskStore, taskId, agent: agent.role as AgentRole, }); } // Build skill selection context for heartbeat session (uses waking agent's skills, no role fallback) const skillContext = buildSessionSkillContextSync(agent, "heartbeat", rootDir); let systemPrompt = isNoTaskRun ? HEARTBEAT_NO_TASK_SYSTEM_PROMPT : HEARTBEAT_SYSTEM_PROMPT; const baseHeartbeatSystemPrompt = systemPrompt; try { const agentInstructions = await resolveAgentInstructionsWithRatings(agent, rootDir, this.store); const memoryInstructions = memorySettings?.memoryEnabled === false ? "" : buildExecutionMemoryInstructions(rootDir, memorySettings); systemPrompt = buildSystemPromptWithInstructions( baseHeartbeatSystemPrompt, [agentInstructions, memoryInstructions].filter((part) => part.trim()).join("\n\n"), ); } catch (instructionError) { systemPrompt = baseHeartbeatSystemPrompt; const message = instructionError instanceof Error ? instructionError.message : String(instructionError); heartbeatLog.warn(`Failed to enrich heartbeat system prompt for ${agentId}: ${message}`); } // Create agent session const { session } = await createResolvedAgentSession({ sessionPurpose: "heartbeat", pluginRunner: this.pluginRunner, cwd: rootDir, systemPrompt, tools: "readonly", customTools: heartbeatTools, defaultProvider: agent.runtimeConfig?.modelProvider as string | undefined, defaultModelId: agent.runtimeConfig?.modelId as string | undefined, onText: (delta) => { outputLength += delta.length; appendStdoutExcerpt(delta); agentLogger?.onText(delta); }, onThinking: (delta) => { agentLogger?.onThinking(delta); }, onToolStart: (name, args) => { agentLogger?.onToolStart(name, args); }, onToolEnd: (name, isError, result) => { toolCallCount++; agentLogger?.onToolEnd(name, isError, result); }, // Skill selection: use waking agent's skills (heartbeat has no role fallback) ...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}), }); // Track for monitoring this.trackAgent(agentId, { dispose: () => session.dispose() }, run.id); try { // Build execution prompt let pendingMessages: Message[] = []; let executionPrompt: string; if (isNoTaskRun) { // No-task heartbeat: agent has identity but no assigned task // Fetch unread messages when messageStore is available (for all trigger types) if (this.messageStore) { try { pendingMessages = this.messageStore.getInbox(agentId, "agent", { read: false, limit: 10 }); } catch (inboxErr) { heartbeatLog.warn(`Failed to fetch inbox messages for ${agentId}: ${inboxErr instanceof Error ? inboxErr.message : String(inboxErr)}`); } } // Build pending messages section const pendingMessagesLines: string[] = []; if (pendingMessages.length > 0) { pendingMessagesLines.push( "", "Pending Messages:", ...pendingMessages.map((msg) => { const timestamp = new Date(msg.createdAt).toLocaleString(); return `- [id: ${msg.id}] [from: ${msg.fromType}:${msg.fromId}] ${msg.content} (${timestamp})`; }), ); } executionPrompt = [ `Heartbeat execution for agent "${agent.name}" (ID: ${agent.id})`, `Source: ${source}${triggerDetail ? ` (${triggerDetail})` : ""}`, "", "**No assigned task** — This heartbeat run has no task assignment.", "", "You have identity (soul, instructions, and/or memory) loaded, which means you can perform", "useful ambient work. Here are some things you can do:", "", "1. **Check your messages** — Use fn_read_messages to review any pending messages", " and use fn_send_message with reply_to_message_id when responding.", "", "2. **Create new tasks** — Use fn_task_create to spawn follow-up work that needs", " to be done. This is useful for surfacing issues or ideas you discover.", "", "3. **Delegate work** — Use fn_list_agents to discover available agents and", " fn_delegate_task to assign work to them.", "", "4. **Update your memory** — Use fn_memory_append to persist important learnings", " or context that will help you in future sessions.", "", "5. **Monitor the project** — Review the task board and identify any issues", " or opportunities that should be addressed.", ...pendingMessagesLines, "", "Your soul, instructions, and memory are already loaded in the system prompt.", "Focus on work that benefits the project without requiring a specific task context.", "Call fn_heartbeat_done when finished.", ].join("\n"); } else { // Task-scoped heartbeat: agent has an assigned task const taskTitle = taskDetail!.title ?? taskDetail!.description.slice(0, 100); // Fetch unread messages when messageStore is available (for all trigger types) if (this.messageStore) { try { pendingMessages = this.messageStore.getInbox(agentId, "agent", { read: false, limit: 10 }); } catch (inboxErr) { heartbeatLog.warn(`Failed to fetch inbox messages for ${agentId}: ${inboxErr instanceof Error ? inboxErr.message : String(inboxErr)}`); } } const triggeringCommentLines: string[] = []; if (effectiveTriggeringCommentIds && effectiveTriggeringCommentIds.length > 0) { const commentLookup = new Map(); for (const comment of taskDetail!.comments ?? []) { commentLookup.set(comment.id, { author: comment.author, text: comment.text }); } for (const steeringComment of taskDetail!.steeringComments ?? []) { commentLookup.set(steeringComment.id, { author: steeringComment.author, text: steeringComment.text }); } const formatCommentText = (text: string): string => text.replace(/\s+/g, " ").trim(); for (const commentId of effectiveTriggeringCommentIds) { const comment = commentLookup.get(commentId); if (comment) { triggeringCommentLines.push(`- [${comment.author}]: "${formatCommentText(comment.text)}"`); } } if (triggeringCommentLines.length > 0) { triggeringCommentLines.unshift( "", "You were woken because of new comments on this task. Review them and take appropriate action.", `Triggering comment type: ${effectiveTriggeringCommentType ?? "task"}`, "New comments since last run:", ); } } // Build pending messages section const pendingMessagesLines: string[] = []; if (pendingMessages.length > 0) { pendingMessagesLines.push( "", "Pending Messages:", ...pendingMessages.map((msg) => { const timestamp = new Date(msg.createdAt).toLocaleString(); return `- [id: ${msg.id}] [from: ${msg.fromType}:${msg.fromId}] ${msg.content} (${timestamp})`; }), ); } executionPrompt = [ `Heartbeat execution for agent "${agent.name}" (ID: ${agent.id})`, `Source: ${source}${triggerDetail ? ` (${triggerDetail})` : ""}`, `Assigned task: ${taskId} — ${taskTitle}`, "", "Task description:", taskDetail!.description, "", taskDetail!.prompt ? `PROMPT.md:\n${taskDetail!.prompt}` : "No PROMPT.md available.", ...triggeringCommentLines, ...pendingMessagesLines, "", "Review the task status and take appropriate action. Call fn_heartbeat_done when finished.", ].join("\n"); } // Execute await promptWithFallback(session, executionPrompt); // Estimate output tokens (rough: ~4 chars per token) const estimatedOutputTokens = Math.ceil(outputLength / 4); await flushAgentLogger(); // Mark messages as read after successful processing (only if messages were included in prompt) if (pendingMessages.length > 0 && this.messageStore) { try { this.messageStore.markAllAsRead(agentId, "agent"); } catch (markReadErr) { heartbeatLog.warn(`Failed to mark messages as read for ${agentId}: ${markReadErr instanceof Error ? markReadErr.message : String(markReadErr)}`); } } // Complete run successfully const completionResultJson: Record = { summary: heartbeatSummary, toolCallCount, }; if (isNoTaskRun) { // Identity agents without tasks get a special reason for observability completionResultJson.reason = "no_assignment_identity_run"; } else if (inboxSelection) { completionResultJson.reason = "inbox_selected"; completionResultJson.priority = inboxSelection.priority; completionResultJson.taskId = taskId; } await this.completeRun(agentId, run.id, { status: "completed", usageJson: { inputTokens: 0, outputTokens: estimatedOutputTokens, cachedTokens: 0 }, resultJson: completionResultJson, stdoutExcerpt: stdoutExcerpt || undefined, }); heartbeatLog.log(`Heartbeat completed for ${agentId} (${toolCallCount} tool calls, ~${estimatedOutputTokens} output tokens)`); } catch (err) { const errorDetail = formatError(err).detail; heartbeatLog.error(`Heartbeat execution failed for ${agentId}: ${errorDetail}`); await flushAgentLogger(); await this.completeRun(agentId, run.id, { status: "failed", stderrExcerpt: errorDetail, stdoutExcerpt: stdoutExcerpt || undefined, }); } finally { await flushAgentLogger(); // Defensively untrack the agent — wrap in try/catch to guarantee cleanup // can't be blocked by an exception in untrackAgent itself. try { this.untrackAgent(agentId); } catch (untrackErr) { heartbeatLog.warn(`untrackAgent failed for ${agentId}: ${untrackErr instanceof Error ? untrackErr.message : String(untrackErr)}`); } try { session.dispose(); } catch (disposeErr: unknown) { const errorMessage = disposeErr instanceof Error ? disposeErr.message : String(disposeErr); heartbeatLog.warn(`session.dispose() failed for ${agentId}: ${errorMessage}`); } } return (await this.store.getRunDetail(agentId, run.id))!; } catch (err) { const errorDetail = formatError(err).detail; const errorMessage = err instanceof Error ? err.message : String(err); heartbeatLog.error(`Heartbeat execution error for ${agentId}: ${errorDetail}`); await flushAgentLogger(); // Attempt to complete the run as failed if it's still active. // If completeRun also fails, fall back to a direct DB update to ensure // the run is not permanently stuck in "active" state. try { await this.completeRun(agentId, run.id, { status: "failed", stderrExcerpt: errorDetail, }); } catch (completeRunErr) { const completeRunErrMsg = completeRunErr instanceof Error ? completeRunErr.message : String(completeRunErr); heartbeatLog.error(`completeRun failed for ${agentId}/${run.id}: ${completeRunErrMsg} — attempting safety-net completion`); // Safety net: directly update the run record to prevent zombie run state. // This runs only when completeRun itself threw, guaranteeing the run // doesn't remain permanently stuck in "active" state. try { const runDetail = await this.store.getRunDetail(agentId, run.id); if (runDetail && runDetail.status !== "completed" && runDetail.status !== "failed" && runDetail.status !== "terminated") { await this.store.saveRun({ ...runDetail, endedAt: new Date().toISOString(), status: "failed", stderrExcerpt: `Heartbeat execution failed: ${errorMessage}. Run completion also failed: ${completeRunErrMsg}`, }); await this.store.endHeartbeatRun(run.id, "terminated"); // Also clean up run state accumulator this.clearRunState(agentId); heartbeatLog.log(`Safety-net run completion for ${agentId}/${run.id} — run terminated`); } } catch (safetyNetErr) { const safetyNetErrMsg = safetyNetErr instanceof Error ? safetyNetErr.message : String(safetyNetErr); heartbeatLog.error(`Safety-net run completion also failed for ${agentId}/${run.id}: ${safetyNetErrMsg} — run may be stuck permanently`); } } return (await this.store.getRunDetail(agentId, run.id))!; } }); } // ───────────────────────────────────────────────────────────────────────── // Heartbeat tools: createHeartbeatTools / clearRunState // ───────────────────────────────────────────────────────────────────────── /** * Create the tool set for a heartbeat agent session. * * Returns tools with tracking wrappers that record task creations * so they can be included in the run's `resultJson.tasksCreated`. * * @param agentId - The agent ID (used for tracking and logging) * @param taskStore - TaskStore for task creation and logging * @param taskId - The assigned task ID (for fn_task_log context) * @param runContext - Optional run context for mutation correlation * @param audit - Optional run auditor for audit trail (FN-1404) * @param messageStore - Optional MessageStore for messaging tools * @returns Array of ToolDefinitions for the heartbeat session */ createHeartbeatTools( agentId: string, taskStore: TaskStore, taskId: string, runContext?: RunMutationContext, audit?: ReturnType, messageStore?: MessageStore, ): ToolDefinition[] { const tools: ToolDefinition[] = []; // Wrap createTaskCreateTool with tracking and agent-link logging const baseCreateTool = createTaskCreateTool(taskStore); const trackedCreateTool: ToolDefinition = { ...baseCreateTool, execute: async (id: string, params: Static, signal, onUpdate, ctx) => { const result = await baseCreateTool.execute(id, params, signal, onUpdate, ctx); const createdTaskId = (result.details as { taskId?: string })?.taskId ?? "unknown"; // Log agent link on the created task with run context for correlation try { await taskStore.logEntry(createdTaskId, `Created by agent ${agentId} during heartbeat run`, undefined, runContext); } catch (taskCreateLogErr) { heartbeatLog.warn(`Task ${createdTaskId} agent-link log failed: ${taskCreateLogErr instanceof Error ? taskCreateLogErr.message : String(taskCreateLogErr)}`); } // Audit trail: record task creation (FN-1404) await audit?.database({ type: "task:create", target: createdTaskId }); // Accumulate for inclusion in run resultJson if (!this.runCreatedTasks.has(agentId)) { this.runCreatedTasks.set(agentId, []); } this.runCreatedTasks.get(agentId)!.push({ id: createdTaskId, description: params.description, }); return result; }, }; tools.push(trackedCreateTool); // fn_task_log tool (with run context for mutation correlation) tools.push(createTaskLogToolWithContext(taskStore, taskId, runContext)); // Document tools for persisting durable findings tools.push(createTaskDocumentWriteTool(taskStore, taskId)); tools.push(createTaskDocumentReadTool(taskStore, taskId)); // Agent delegation tools — discover and delegate work to other agents tools.push(createListAgentsTool(this.store)); tools.push(createDelegateTaskTool(this.store, taskStore)); // Messaging tools — when MessageStore is available, agents can send and receive messages if (messageStore) { tools.push(createSendMessageTool(messageStore, agentId)); tools.push(createReadMessagesTool(messageStore, agentId)); } return tools; } /** * Clear accumulated run state for an agent. * Called after completing a run to reset the `runCreatedTasks` accumulator. * @param agentId - The agent ID */ clearRunState(agentId: string): void { this.runCreatedTasks.delete(agentId); } // ───────────────────────────────────────────────────────────────────────── // Private methods // ───────────────────────────────────────────────────────────────────────── /** * Get the resolved heartbeat configuration for an agent. * Reads per-agent config from runtimeConfig with fallback to monitor defaults. * @param agentId - The agent ID * @returns Resolved config with validated values */ async getAgentHeartbeatConfig(agentId: string): Promise { return this.getAgentConfig(agentId); } /** * Resolve per-agent heartbeat config from runtimeConfig with validation and fallbacks. */ private resolveAgentConfig(agentId: string): ResolvedHeartbeatConfig { // Defaults from monitor-level construction const result: ResolvedHeartbeatConfig = { pollIntervalMs: this.pollIntervalMs, heartbeatTimeoutMs: this.heartbeatTimeoutMs, maxConcurrentRuns: this.maxConcurrentRuns, }; try { const agent = this.configStore.getCachedAgent?.(agentId); if (agent?.runtimeConfig) { const rc = agent.runtimeConfig; if (typeof rc.heartbeatIntervalMs === "number" && Number.isFinite(rc.heartbeatIntervalMs)) { result.pollIntervalMs = Math.max(1000, rc.heartbeatIntervalMs); } if (typeof rc.heartbeatTimeoutMs === "number" && Number.isFinite(rc.heartbeatTimeoutMs)) { result.heartbeatTimeoutMs = Math.max(5000, rc.heartbeatTimeoutMs); } if (typeof rc.maxConcurrentRuns === "number" && Number.isFinite(rc.maxConcurrentRuns)) { result.maxConcurrentRuns = Math.max(1, Math.round(rc.maxConcurrentRuns)); } } } catch (agentLookupErr) { heartbeatLog.warn(`getAgentConfig(${agentId}) agent lookup failed: ${agentLookupErr instanceof Error ? agentLookupErr.message : String(agentLookupErr)} — using monitor defaults`); } return result; } private async getAgentConfig(agentId: string): Promise { const result = this.resolveAgentConfig(agentId); if (!this.taskStore) { return result; } try { const settings = await getHeartbeatMemorySettings(this.taskStore); const rawMultiplier = settings?.heartbeatMultiplier; const multiplier = typeof rawMultiplier === "number" && Number.isFinite(rawMultiplier) && rawMultiplier > 0 ? rawMultiplier : 1; result.pollIntervalMs = Math.max(1000, Math.round(result.pollIntervalMs * multiplier)); } catch (settingsErr) { heartbeatLog.warn(`getAgentConfig(${agentId}) settings lookup failed: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)} — using base interval`); } return result; } private async checkMissedHeartbeats(): Promise { const now = Date.now(); for (const tracked of this.trackedAgents.values()) { const config = await this.getAgentConfig(tracked.agentId); const elapsed = now - tracked.lastSeen; if (elapsed >= config.heartbeatTimeoutMs) { // Missed heartbeat detected if (!tracked.missedHeartbeatReported) { tracked.missedHeartbeatReported = true; await this.handleMissedHeartbeat(tracked); } else { // Already reported - check if we should terminate // Give 2x timeout for recovery before auto-terminate if (elapsed >= config.heartbeatTimeoutMs * 2) { await this.terminateUnresponsive(tracked); } } } } } private async handleMissedHeartbeat(tracked: TrackedAgent): Promise { // Record missed heartbeat await this.store.recordHeartbeat(tracked.agentId, "missed", tracked.runId); // Notify callback this.onMissed?.(tracked.agentId); } private async terminateUnresponsive(tracked: TrackedAgent): Promise { // Dispose the session try { tracked.session.dispose(); } catch (err) { // Log but don't stop termination heartbeatLog.warn(`Error disposing session for ${tracked.agentId}: ${err instanceof Error ? err.message : String(err)}`); } // Update agent state to terminated try { await this.store.updateAgentState(tracked.agentId, "terminated"); } catch (err) { heartbeatLog.warn(`Error terminating agent ${tracked.agentId}: ${err instanceof Error ? err.message : String(err)}`); } // Remove from tracking this.trackedAgents.delete(tracked.agentId); // Notify callback this.onTerminated?.(tracked.agentId); } } // ───────────────────────────────────────────────────────────────────────── // HeartbeatTriggerScheduler — timer, assignment, and on-demand triggers // ───────────────────────────────────────────────────────────────────────── /** Structured context passed when a trigger fires. */ export interface WakeContext { /** Optional task ID associated with this trigger */ taskId?: string; /** Why the agent was woken */ wakeReason: string; /** Detail about the specific trigger */ triggerDetail: string; /** IDs of comments that triggered this wake (if any) */ triggeringCommentIds?: string[]; /** Type of comment that triggered this wake */ triggeringCommentType?: "steering" | "task" | "pr"; /** Budget governance status for the agent at trigger time */ budgetStatus?: AgentBudgetStatus; /** Additional context (intervalMs, etc.) */ [key: string]: unknown; } /** Callback invoked when a trigger fires. */ export type TriggerCallback = ( agentId: string, source: HeartbeatInvocationSource, context: WakeContext, ) => Promise; /** Per-agent timer state */ interface AgentTimer { intervalMs: number; handle: ReturnType; } /** * True when an agent's state indicates it should be ticking right now. * Heartbeats track liveness while the agent is meant to be doing work. * * States where timers should remain armed: * - "active" — Agent is working * - "running" — Agent has an active heartbeat run * - "idle" — Agent is between tasks, waiting for work (FN-2289 fix) * * States where timers should be cleared: * - "terminated" — Agent has completed/failed * - "error" — Agent encountered an error * - "paused" — Agent is paused by budget exhaustion or manual action */ function isTickableState(state: Agent["state"]): boolean { return state === "active" || state === "running" || state === "idle"; } /** * True when the scheduler should manage this agent at all. Ephemeral * (task-worker) agents are driven directly by TaskExecutor and must never * acquire a scheduler timer. */ function isHeartbeatManaged(agent: Agent): boolean { return !isEphemeralAgent(agent); } /** * HeartbeatTriggerScheduler manages timer-based heartbeat triggers for agents. * * State is the source of truth: state ∈ {active, running} on a non-ephemeral * agent arms the timer; any other state or any ephemeral agent doesn't. The * `runtimeConfig.enabled` flag is no longer consulted here — pause/resume * happens through `agent.state`, and the `agent:updated` listener arms or * clears the timer on transitions. * * Other config knobs still apply: * - `heartbeatIntervalMs`: Timer interval (default 1h) * - `maxConcurrentRuns`: Skip tick if agent already has an active run */ export class HeartbeatTriggerScheduler { private store: AgentStore; private callback: TriggerCallback; private taskStore?: TaskStore; private timers: Map = new Map(); private registrationEpochs: Map = new Map(); private running = false; private assignedListener: ((agent: import("@fusion/core").Agent, taskId: string) => void) | null = null; private updatedListener: ((agent: import("@fusion/core").Agent) => void) | null = null; private deletedListener: ((agentId: string) => void) | null = null; constructor(store: AgentStore, callback: TriggerCallback, taskStore?: TaskStore) { this.store = store; this.callback = callback; this.taskStore = taskStore; } /** * Start the scheduler. Enables assignment watching. * Individual agents must be registered separately via registerAgent(). */ start(): void { if (this.running) return; this.running = true; this.watchAssignments(); this.watchAgentLifecycle(); heartbeatLog.log("HeartbeatTriggerScheduler started"); } /** * Stop the scheduler and clear all timers. */ stop(): void { if (!this.running) return; this.running = false; // Unwatch assignments this.unwatchAssignments(); this.unwatchAgentLifecycle(); // Clear all timers for (const [agentId, timer] of this.timers) { clearInterval(timer.handle); heartbeatLog.log(`Cleared timer for ${agentId}`); } this.timers.clear(); heartbeatLog.log("HeartbeatTriggerScheduler stopped"); } /** * Check if the scheduler is running. */ isActive(): boolean { return this.running; } /** Default heartbeat interval when not explicitly configured (3600 seconds / 1 hour) */ private static readonly DEFAULT_HEARTBEAT_INTERVAL_MS = 3_600_000; /** * Register an agent for timer-based heartbeat triggers. * @param agentId - The agent ID * @param config - Per-agent heartbeat config */ registerAgent(agentId: string, config: AgentHeartbeatConfig): void { // State drives whether an agent ticks; this method no longer honors // `config.enabled` as a registration gate. Callers filter based on // state + ephemeral classification before calling through. // Apply default interval if not explicitly configured // This ensures agents with heartbeat monitoring enabled but no explicit interval // still get periodic timer triggers (matching HeartbeatMonitor constructor default) let rawIntervalMs = config.heartbeatIntervalMs; let usingDefaultInterval = false; if (!rawIntervalMs || typeof rawIntervalMs !== "number" || !Number.isFinite(rawIntervalMs) || rawIntervalMs <= 0) { rawIntervalMs = HeartbeatTriggerScheduler.DEFAULT_HEARTBEAT_INTERVAL_MS; usingDefaultInterval = true; } const intervalMs = Math.max(1000, Math.round(rawIntervalMs)); const registrationEpoch = (this.registrationEpochs.get(agentId) ?? 0) + 1; this.registrationEpochs.set(agentId, registrationEpoch); // Register immediately with multiplier=1 so agents don't wait for async settings I/O. this.applyTimerRegistration(agentId, intervalMs, 1, usingDefaultInterval); // If project settings are available, refresh registration with the current multiplier. if (this.taskStore && typeof (this.taskStore as { getSettings?: () => Promise }).getSettings === "function") { void this.applyProjectMultiplierRegistration(agentId, intervalMs, usingDefaultInterval, registrationEpoch); } } private async applyProjectMultiplierRegistration( agentId: string, baseIntervalMs: number, usingDefaultInterval: boolean, expectedEpoch: number, ): Promise { let multiplier = 1; try { const settings = await getHeartbeatMemorySettings(this.taskStore!); multiplier = HeartbeatTriggerScheduler.resolveHeartbeatMultiplier(settings?.heartbeatMultiplier); } catch (settingsErr) { heartbeatLog.warn( `Failed to read heartbeatMultiplier for ${agentId}: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)} — using 1x`, ); multiplier = 1; } // Guard against stale async completions after subsequent register/unregister calls. if (this.registrationEpochs.get(agentId) !== expectedEpoch) { return; } this.applyTimerRegistration(agentId, baseIntervalMs, multiplier, usingDefaultInterval); } private applyTimerRegistration( agentId: string, baseIntervalMs: number, multiplier: number, usingDefaultInterval: boolean, ): void { const effectiveIntervalMs = Math.max(1000, Math.round(baseIntervalMs * multiplier)); this.clearAgentTimer(agentId); const handle = setInterval(() => { void this.onTimerTick(agentId, effectiveIntervalMs); }, effectiveIntervalMs); this.timers.set(agentId, { intervalMs: effectiveIntervalMs, handle }); if (multiplier !== 1) { heartbeatLog.log( `Registered timer for ${agentId} (every ${baseIntervalMs}ms, multiplier ${multiplier} → ${effectiveIntervalMs}ms effective)`, ); return; } heartbeatLog.log( usingDefaultInterval ? `Registered timer for ${agentId} (every ${effectiveIntervalMs}ms, default interval)` : `Registered timer for ${agentId} (every ${effectiveIntervalMs}ms)`, ); } private clearAgentTimer(agentId: string): void { const timer = this.timers.get(agentId); if (!timer) { return; } clearInterval(timer.handle); this.timers.delete(agentId); } private static resolveHeartbeatMultiplier(rawMultiplier: unknown): number { if (typeof rawMultiplier !== "number" || !Number.isFinite(rawMultiplier) || rawMultiplier <= 0) { return 1; } return rawMultiplier; } /** * Unregister an agent, clearing its timer. * @param agentId - The agent ID */ unregisterAgent(agentId: string): void { this.registrationEpochs.set(agentId, (this.registrationEpochs.get(agentId) ?? 0) + 1); if (this.timers.has(agentId)) { this.clearAgentTimer(agentId); heartbeatLog.log(`Unregistered timer for ${agentId}`); } } /** * Get the set of currently registered agent IDs. * Useful for testing. */ getRegisteredAgents(): string[] { return Array.from(this.timers.keys()); } /** * Subscribe to agent:assigned events on the AgentStore. * When a task is assigned to an agent, the trigger callback fires * with source "assignment" and the task ID in the context. */ watchAssignments(): void { if (this.assignedListener) return; // Already watching this.assignedListener = async (agent, taskId) => { if (!this.running) return; try { if (!isHeartbeatManaged(agent)) { heartbeatLog.log(`Assignment trigger skipped for ${agent.id} (ephemeral/internal)`); return; } const runtimeConfig = (agent.runtimeConfig ?? {}) as { enabled?: boolean }; if (runtimeConfig.enabled === false) { heartbeatLog.log(`Assignment trigger skipped for ${agent.id} (disabled)`); return; } // Guard: skip if agent already has an active run const activeRun = await this.store.getActiveHeartbeatRun(agent.id); if (activeRun) { heartbeatLog.log(`Assignment trigger skipped for ${agent.id} (active run)`); return; } let budgetStatus: AgentBudgetStatus | undefined; // Budget governance: block even critical triggers when budget is fully exhausted try { budgetStatus = await this.store.getBudgetStatus(agent.id); if (budgetStatus.isOverBudget) { heartbeatLog.log(`Agent ${agent.id} budget exhausted — assignment trigger skipped`); return; } } catch (budgetErr) { heartbeatLog.warn(`Assignment trigger budget check failed for ${agent.id}: ${budgetErr instanceof Error ? budgetErr.message : String(budgetErr)} — proceeding without budget check`); } let triggeringCommentIds: string[] | undefined; if (this.taskStore && typeof this.taskStore.getTask === "function") { try { const [task, recentRuns] = await Promise.all([ this.taskStore.getTask(taskId), this.store.getRecentRuns(agent.id, 1), ]); const lastRunAt = recentRuns[0]?.startedAt; const newSteeringComments = (task.steeringComments ?? []).filter((comment) => !lastRunAt || comment.createdAt > lastRunAt, ); if (newSteeringComments.length > 0) { triggeringCommentIds = newSteeringComments.map((comment) => comment.id); } } catch (error) { heartbeatLog.warn( `Failed to resolve triggering steering comments for assignment wake (${agent.id}/${taskId}): ${error instanceof Error ? error.message : String(error)}`, ); } } heartbeatLog.log(`Assignment trigger for ${agent.id} (task: ${taskId})`); await this.callback(agent.id, "assignment", { taskId, wakeReason: "assignment", triggerDetail: "task-assigned", ...(triggeringCommentIds?.length ? { triggeringCommentIds, triggeringCommentType: "steering" as const, } : {}), ...(budgetStatus && { budgetStatus }), }); } catch (err) { heartbeatLog.error(`Assignment trigger error for ${agent.id}: ${err instanceof Error ? err.message : err}`); } }; this.store.on("agent:assigned", this.assignedListener); heartbeatLog.log("Watching agent:assigned events"); } /** * Unsubscribe from agent:assigned events. */ unwatchAssignments(): void { if (this.assignedListener) { this.store.off("agent:assigned", this.assignedListener); this.assignedListener = null; heartbeatLog.log("Stopped watching agent:assigned events"); } } private watchAgentLifecycle(): void { if (this.updatedListener || this.deletedListener) return; // State-driven registration: when an agent transitions into a tickable // state (active/running) arm the timer; transitioning out clears it. this.updatedListener = (agent) => { if (!isHeartbeatManaged(agent) || !isTickableState(agent.state)) { this.unregisterAgent(agent.id); return; } if (this.timers.has(agent.id)) { // Already ticking — re-registering would reset the interval mid-cycle // on every unrelated agent update. return; } const rc = (agent.runtimeConfig ?? {}) as { heartbeatIntervalMs?: number; maxConcurrentRuns?: number; }; this.registerAgent(agent.id, { heartbeatIntervalMs: rc.heartbeatIntervalMs, maxConcurrentRuns: rc.maxConcurrentRuns, }); heartbeatLog.log( `State-driven registration: ${agent.id} is ${agent.state} — timer armed`, ); }; this.deletedListener = (agentId) => { this.unregisterAgent(agentId); }; this.store.on("agent:updated", this.updatedListener); this.store.on("agent:deleted", this.deletedListener); } private unwatchAgentLifecycle(): void { if (this.updatedListener) { this.store.off("agent:updated", this.updatedListener); this.updatedListener = null; } if (this.deletedListener) { this.store.off("agent:deleted", this.deletedListener); this.deletedListener = null; } } /** * Handle a timer tick for an agent. * Checks for active runs before invoking the callback. */ private async onTimerTick(agentId: string, intervalMs: number): Promise { if (!this.running) return; try { const agent = await this.store.getAgent(agentId); if (!agent) { heartbeatLog.log(`Timer tick skipped for ${agentId} (agent missing)`); this.unregisterAgent(agentId); return; } if (!isHeartbeatManaged(agent) || !isTickableState(agent.state)) { heartbeatLog.log(`Timer tick skipped for ${agentId} (state=${agent.state})`); this.unregisterAgent(agentId); return; } // Check for active runs const activeRun = await this.store.getActiveHeartbeatRun(agentId); if (activeRun) { heartbeatLog.log(`Timer tick skipped for ${agentId} (active run)`); return; } // Budget enforcement is handled in HeartbeatMonitor.executeHeartbeat() for timer sources. // The scheduler dispatches the callback regardless of budget status so that executeHeartbeat() // can create explicit run records with budget_exhausted/budget_threshold_exceeded reasons. // This makes timer budget skips observable rather than silent drops. await this.callback(agentId, "timer", { wakeReason: "timer", triggerDetail: "scheduled", intervalMs, }); } catch (err) { heartbeatLog.error(`Timer tick error for ${agentId}: ${err instanceof Error ? err.message : err}`); } } }