- Add shared agent role policy helpers for implementation task detection and executor-role enforcement - Provide a standardized role-mismatch error formatter for delegation and assignment paths - Update test isolation leak detection to ignore ephemeral fusion-test-home-root temp directories while preserving baseline checks Fusion-Task-Id: FN-3716
3254 lines
142 KiB
TypeScript
3254 lines
142 KiB
TypeScript
/**
|
||
* HeartbeatMonitor - Runtime monitoring and execution for agents
|
||
*
|
||
* Monitors agents via periodic polling, detects missed heartbeats,
|
||
* and provides the Paperclip-style heartbeat execution engine:
|
||
*
|
||
* wake → check inbox → work → exit
|
||
*
|
||
* When `executeHeartbeat()` is called (via API, timer, or assignment),
|
||
* the system wakes the agent, checks its assigned task from AgentStore,
|
||
* executes work in a lightweight agent session with `fn_task_create` capability,
|
||
* records results, and transitions the run to completed.
|
||
*
|
||
* Callback pattern (not EventEmitter):
|
||
* - onMissed: Called when an agent misses its heartbeat
|
||
* - onRecovered: Called when an agent recovers after a missed heartbeat
|
||
* - onTerminated: Called when a heartbeat run is terminated
|
||
*/
|
||
|
||
import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig, AgentBudgetStatus, Message, MessageStore, TaskStore, TaskDetail, AgentRole, Agent, InboxTask, RunMutationContext, Settings, AgentConfigRevision, ReflectionStore } from "@fusion/core";
|
||
import { ApprovalRequestStore, buildExecutionMemoryInstructions, isEphemeralAgent, hasAgentIdentity, resolveEffectiveAgentPermissionPolicy, canAgentTakeImplementationTask } from "@fusion/core";
|
||
import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
|
||
import { Type, type Static } from "@mariozechner/pi-ai";
|
||
import { createHash } from "node:crypto";
|
||
import { createTaskCreateTool, createTaskLogToolWithContext, createTaskDocumentWriteTool, createTaskDocumentReadTool, createListAgentsTool, createDelegateTaskTool, createGetAgentConfigTool, createUpdateAgentConfigTool, createSendMessageTool, createReadMessagesTool, createMemoryTools, createReadEvaluationsTool, createUpdateIdentityTool, createReflectOnPerformanceTool, readAgentMemoryWorkspaceLongTerm, taskCreateParams } from "./agent-tools.js";
|
||
import { AgentLogger } from "./agent-logger.js";
|
||
import {
|
||
resolveAgentInstructionsWithRatings,
|
||
buildSystemPromptWithInstructions,
|
||
buildPluginPromptSection,
|
||
resolveAgentHeartbeatProcedure,
|
||
} from "./agent-instructions.js";
|
||
import { heartbeatLog, formatError } from "./logger.js";
|
||
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
|
||
import { promptWithFallback } from "./pi.js";
|
||
import { createResolvedAgentSession, extractRuntimeHint, extractRuntimeModel } from "./agent-session-helpers.js";
|
||
import type { AgentActionGateContext } from "./agent-action-gate.js";
|
||
import { buildSessionSkillContextSync } from "./session-skill-context.js";
|
||
import type { AgentReflectionService } from "./agent-reflection.js";
|
||
|
||
interface SelfImproveServiceLike {
|
||
shouldRunSelfImprove(agentId: string): Promise<boolean>;
|
||
getSelfImprovePrompt(agentId: string): Promise<string>;
|
||
recordSelfImprove(agentId: string): Promise<void>;
|
||
}
|
||
|
||
/** Resolved per-agent heartbeat config after validation and fallback */
|
||
interface ResolvedHeartbeatConfig {
|
||
pollIntervalMs: number;
|
||
heartbeatTimeoutMs: number;
|
||
maxConcurrentRuns: number;
|
||
}
|
||
|
||
/** Options for HeartbeatMonitor constructor */
|
||
export interface HeartbeatMonitorOptions {
|
||
/** AgentStore instance for persistence */
|
||
store: AgentStore;
|
||
/** Optional separate AgentStore reference for reading per-agent runtimeConfig.
|
||
* If not provided, falls back to `store`. */
|
||
agentStore?: AgentStore;
|
||
/** Optional MessageStore for wake-on-message behavior */
|
||
messageStore?: MessageStore;
|
||
/** Polling interval in milliseconds (default: 3600000) */
|
||
pollIntervalMs?: number;
|
||
/** Heartbeat timeout in milliseconds (default: 60000) */
|
||
heartbeatTimeoutMs?: number;
|
||
/** Max concurrent runs per agent (default: 1) */
|
||
maxConcurrentRuns?: number;
|
||
/** Callback when an agent misses its heartbeat */
|
||
onMissed?: (agentId: string, reason: string) => void;
|
||
/** Callback when an agent recovers after a missed heartbeat */
|
||
onRecovered?: (agentId: string) => void;
|
||
/** Callback when a heartbeat run is terminated (run status only; agent state is handled separately). */
|
||
onTerminated?: (agentId: string, reason: string) => void;
|
||
/** Callback when a run starts */
|
||
onRunStarted?: (agentId: string, run: AgentHeartbeatRun) => void;
|
||
/** Callback when a run completes */
|
||
onRunCompleted?: (agentId: string, run: AgentHeartbeatRun) => void;
|
||
/** TaskStore for fn_task_create and fn_task_log tools during heartbeat execution.
|
||
* When not provided, executeHeartbeat() will throw. */
|
||
taskStore?: TaskStore;
|
||
/** Project root directory for agent session CWD.
|
||
* When not provided, executeHeartbeat() will throw. */
|
||
rootDir?: string;
|
||
/** Plugin runner for runtime selection. When provided, enables plugin runtime lookup. */
|
||
pluginRunner?: import("./plugin-runner.js").PluginRunner;
|
||
/** Optional ReflectionStore for evaluation-reading tools */
|
||
reflectionStore?: ReflectionStore;
|
||
/** Optional AgentReflectionService for fn_reflect_on_performance tool */
|
||
reflectionService?: AgentReflectionService;
|
||
/** Optional self-improvement service for periodic self-improve injection */
|
||
selfImproveService?: SelfImproveServiceLike;
|
||
}
|
||
|
||
/** Options for waking up an agent */
|
||
export interface WakeupOptions {
|
||
/** What triggered the wakeup */
|
||
source: HeartbeatInvocationSource;
|
||
/** Detail about the trigger (manual, ping, scheduler, system) */
|
||
triggerDetail?: string;
|
||
/** Context snapshot for the run */
|
||
contextSnapshot?: Record<string, unknown>;
|
||
}
|
||
|
||
/** Options for executing a heartbeat run */
|
||
export interface HeartbeatExecutionOptions {
|
||
/** Agent ID to execute heartbeat for */
|
||
agentId: string;
|
||
/** What triggered this heartbeat */
|
||
source: HeartbeatInvocationSource;
|
||
/** Human-readable trigger detail */
|
||
triggerDetail?: string;
|
||
/** Optional task ID override (uses agent.taskId if not set) */
|
||
taskId?: string;
|
||
/** IDs of comments that triggered this wake (if any) */
|
||
triggeringCommentIds?: string[];
|
||
/** Type of comment that triggered this wake */
|
||
triggeringCommentType?: "steering" | "task" | "pr";
|
||
/** Optional structured context persisted on the run record */
|
||
contextSnapshot?: Record<string, unknown>;
|
||
}
|
||
|
||
export interface PauseAgentOptions {
|
||
pauseReason?: string;
|
||
stopActiveRun?: boolean;
|
||
/**
|
||
* When true (default), assigned tasks are also paused with `pausedByAgentId`
|
||
* set to this agent. Set to false for internal/recovery flows that should
|
||
* not visibly pause user-facing tasks (e.g. heartbeat-unresponsive recovery,
|
||
* which immediately calls resumeAgent afterward).
|
||
*/
|
||
cascadeToTasks?: boolean;
|
||
}
|
||
|
||
export interface ResumeAgentOptions {
|
||
triggerDetail?: string;
|
||
triggerSource?: string;
|
||
clearPauseReason?: boolean;
|
||
/** When true (default), unpauses tasks paused by this agent. */
|
||
cascadeToTasks?: boolean;
|
||
}
|
||
|
||
/** Session interface for disposing agent resources */
|
||
export interface AgentSession {
|
||
/** Dispose the agent session (stop execution, cleanup resources) */
|
||
dispose(): void;
|
||
}
|
||
|
||
/** In-memory tracking data for a monitored agent */
|
||
interface TrackedAgent {
|
||
agentId: string;
|
||
session: AgentSession;
|
||
runId: string;
|
||
lastSeen: number; // timestamp from Date.now()
|
||
missedHeartbeatReported: boolean;
|
||
/** Session ID before this execution started */
|
||
sessionIdBefore?: string;
|
||
}
|
||
|
||
/** Format milliseconds into a human-readable duration string (e.g. "5m", "1h 20m", "2h"). */
|
||
export function formatDuration(ms: number): string {
|
||
const totalMinutes = Math.floor(ms / 60_000);
|
||
if (totalMinutes < 1) return "<1m";
|
||
const hours = Math.floor(totalMinutes / 60);
|
||
const minutes = totalMinutes % 60;
|
||
if (hours > 0 && minutes > 0) return `${hours}h ${minutes}m`;
|
||
if (hours > 0) return `${hours}h`;
|
||
return `${minutes}m`;
|
||
}
|
||
|
||
function formatRelativeTime(iso?: string | null): string {
|
||
if (!iso) return "never";
|
||
const parsed = Date.parse(iso);
|
||
if (!Number.isFinite(parsed)) return "unknown";
|
||
const elapsed = Date.now() - parsed;
|
||
if (elapsed < 0) return "just now";
|
||
return `${formatDuration(elapsed)} ago`;
|
||
}
|
||
|
||
function isAutoClaimRelevantTasksEnabled(agent: Agent): boolean {
|
||
const runtimeConfig = (agent.runtimeConfig ?? {}) as Record<string, unknown>;
|
||
return runtimeConfig.autoClaimRelevantTasks !== false;
|
||
}
|
||
|
||
function taskRelevanceScore(agent: Agent, task: TaskDetail): number {
|
||
const haystack = `${task.title ?? ""} ${task.description}`.toLowerCase();
|
||
let score = 0;
|
||
|
||
const role = agent.role.toLowerCase();
|
||
if (haystack.includes(role)) {
|
||
score += 3;
|
||
}
|
||
|
||
const soulWords = (agent.soul ?? "")
|
||
.toLowerCase()
|
||
.split(/[^a-z0-9]+/)
|
||
.filter((word) => word.length >= 4)
|
||
.slice(0, 8);
|
||
|
||
for (const word of soulWords) {
|
||
if (haystack.includes(word)) {
|
||
score += 1;
|
||
}
|
||
}
|
||
|
||
return score;
|
||
}
|
||
|
||
/**
|
||
* System prompt for heartbeat agent sessions.
|
||
* This is an ambient heartbeat: task implementation runs in a separate executor path.
|
||
* The heartbeat handles coordination, communication, memory, and routing only.
|
||
*/
|
||
export const HEARTBEAT_SYSTEM_PROMPT = `You are a heartbeat agent running in a short execution window.
|
||
|
||
## Your Role
|
||
|
||
This is an ambient heartbeat. Task implementation work (coding, running tests, making commits) runs in a separate
|
||
execution path handled by the executor. Do NOT do task body work or implementation in this heartbeat.
|
||
|
||
Your purpose is to keep momentum through coordination: surface blockers, respond to messages, manage memory,
|
||
delegate, and route work to the right place. Think in single-pass interventions, not coding sessions.
|
||
|
||
Your job:
|
||
1. Check your assigned task context — review its state, blockedBy field, and any new comments.
|
||
2. Do ONE useful coordination action.
|
||
3. Use fn_task_create to spawn follow-up work, fn_task_log to record observations, and fn_task_document_write for durable artifacts.
|
||
4. Use fn_list_agents + fn_delegate_task when work should be assigned to a specific capable agent now.
|
||
5. Use fn_get_agent_config and fn_update_agent_config to tune direct reports before delegating recurring work.
|
||
6. Call fn_heartbeat_done when finished with an optional summary of what was accomplished.
|
||
|
||
**If your bound task is blocked** (blockedBy is set in the task context):
|
||
- Surface the blocker concretely with fn_task_log.
|
||
- Chase the dependency: comment on the blocking task, send a message to the responsible agent, or ping an owner.
|
||
- Look for unblocking work you can spawn or delegate right now.
|
||
- Pivot to other relevant coordination work if the blocker cannot be immediately resolved.
|
||
|
||
**If your bound task is not blocked:**
|
||
- Surface progress, status, or coordination needs with fn_task_log or fn_task_document_write.
|
||
- Create follow-up tasks for discovered risks or gaps.
|
||
- Respond to new steering comments or user messages.
|
||
|
||
Examples of ONE useful coordination action:
|
||
- DO: log a concrete blocker with next steps and message the agent responsible for unblocking.
|
||
- DO: create a focused follow-up task when a missing dependency is discovered.
|
||
- DO: delegate a well-scoped task to an appropriate idle specialist agent.
|
||
- DO: save a short investigation note with fn_task_document_write when the analysis is reusable.
|
||
- DON'T: attempt full implementation, run tests, commit code, or do multi-step coding work.
|
||
- DON'T: create vague tasks like "investigate stuff" without actionable scope.
|
||
|
||
Keep work lightweight — this is a single-pass coordination check, not an implementation run.
|
||
You have workspace read tools (for context gathering) plus fn_task_create, fn_task_log, fn_task_document tools,
|
||
fn_send_message, fn_read_messages, fn_list_agents, fn_delegate_task, and memory tools.
|
||
|
||
**Task Documents:** Save important findings with fn_task_document_write(key="...", content="...").
|
||
Documents persist across sessions and are visible in the dashboard's Documents tab.
|
||
|
||
## Triage and Routing Decisions
|
||
|
||
Use this decision rule:
|
||
- **Log only (fn_task_log):** when the information is contextual, transient, or tied to this task's current state.
|
||
- **Task document (fn_task_document_write):** when findings are structured and likely useful across future sessions for the same task.
|
||
- **Create task (fn_task_create):** when someone must do new executable work.
|
||
- **Delegate task (fn_delegate_task):** when that new work should go to a specific agent based on role/availability.
|
||
- **Manage report config (fn_get_agent_config / fn_update_agent_config):** when direct reports need heartbeat, instruction, or personality tuning.
|
||
|
||
Prefer fn_task_create when assignment is unclear and scheduler routing is fine.
|
||
Prefer fn_delegate_task when immediate ownership by a specific agent materially reduces latency or risk.
|
||
|
||
## Common Patterns
|
||
|
||
- **Blocked task:** log the concrete blocker, chase the dependency via fn_send_message, create a narrowly scoped unblocker task if needed.
|
||
- **Stuck task with no blockedBy:** log the observation and create a follow-up task to investigate the root cause.
|
||
- **Completed task with follow-up risk:** create explicit follow-up task(s) for residual risk instead of burying notes in a long log.
|
||
- **New user/agent comments:** summarize what changed, identify required action, and route via task creation/delegation.
|
||
- **Dependency drift:** log the mismatch and create reconciliation tasks with clear dependencies.
|
||
|
||
## Memory Boundaries
|
||
|
||
You may receive an Agent Memory section and a Project Memory section.
|
||
- Agent Memory is specific to you, including imported and user-created agents such as CEO-style coordinator agents. It has its own long-term memory, daily notes, dreams, and qmd-backed retrieval under .fusion/agent-memory/{agentId}/.
|
||
- Project Memory is the workspace memory system under .fusion/memory/ with long-term memory, daily notes, dreams, and qmd-backed retrieval.
|
||
- Keep these separate: do not copy personal agent operating notes into Project Memory unless they are genuinely useful to every future agent in this workspace.
|
||
- Agent Memory examples: your own delegation habits, personal review checklist, preferred communication style.
|
||
- Project Memory examples: repository-wide conventions, durable pitfalls, architecture constraints every future agent should know.
|
||
|
||
## Processing Messages
|
||
|
||
When you are woken by an incoming message (source includes "wake-on-message"), you should:
|
||
1. Use fn_read_messages to check your inbox for unread messages.
|
||
2. For each message, classify it: informational, question, request, or escalation.
|
||
3. Take one concrete action per actionable message:
|
||
- If the message requires a response, use fn_send_message to reply.
|
||
- When replying, include 'reply_to_message_id' with the original message ID from fn_read_messages output.
|
||
- If the message is informational, acknowledge it by logging with fn_task_log.
|
||
- If the message requests net-new work, create a follow-up task with fn_task_create.
|
||
- If ownership is clear and an agent is available, delegate using fn_delegate_task.
|
||
4. After processing messages, continue with your normal heartbeat duties.
|
||
|
||
Example flow:
|
||
- Read unread messages → identify "needs action" item → reply with intent (reply_to_message_id) → create/delegate task if execution is needed → log key decision.
|
||
|
||
When sending messages:
|
||
- Be concise and clear about what you need or what you've done.
|
||
- Use 'reply_to_message_id' when replying so threaded conversations stay linked.
|
||
- Include relevant context (task IDs, file paths) in metadata when applicable.
|
||
- Use agent-to-agent for inter-agent communication.`;
|
||
|
||
/**
|
||
* System prompt for no-task heartbeat agent sessions.
|
||
* Instructs the agent to perform ambient work only with tools that do not require task context.
|
||
*/
|
||
export const HEARTBEAT_NO_TASK_SYSTEM_PROMPT = `You are a heartbeat agent running in a short execution window with no task assignment.
|
||
|
||
## Your Role
|
||
|
||
You are an ambient coordinator. You scan signals (messages, memory, board state), make one high-leverage move, and hand execution to the right workflow.
|
||
You are not expected to implement large code changes in no-task mode.
|
||
|
||
Your job:
|
||
1. Review your context — check messages, memory, and project state.
|
||
2. Do ONE useful action: analyze, create follow-up tasks, delegate work, or update memory.
|
||
3. Use fn_task_create to spawn follow-up work.
|
||
4. Use fn_list_agents and fn_delegate_task to coordinate with other agents.
|
||
5. Use fn_get_agent_config and fn_update_agent_config to read/tune direct-report agents for better routing outcomes.
|
||
5. Call fn_heartbeat_done when finished with an optional summary of what was accomplished.
|
||
|
||
Examples of ONE useful action:
|
||
- DO: create a clearly scoped task for a newly discovered reliability issue.
|
||
- DO: delegate a ready-to-run task to an idle specialist agent.
|
||
- DO: append durable cross-task conventions to memory.
|
||
- DON'T: open multiple loosely defined tasks in one run.
|
||
- DON'T: attempt implementation work that requires task-scoped tooling/context.
|
||
|
||
Keep work lightweight — this is a single-pass ambient check, not a full implementation run.
|
||
You have coding-capable workspace tools (read/write/edit/bash within worktree boundaries) plus:
|
||
- fn_task_create
|
||
- fn_list_agents and fn_delegate_task
|
||
- fn_get_agent_config and fn_update_agent_config (for direct reports only)
|
||
- fn_memory_search, fn_memory_get, and fn_memory_append
|
||
- fn_heartbeat_done
|
||
- fn_send_message and fn_read_messages when messaging is enabled for this run (they may not always be available)
|
||
|
||
## Triage and Routing Decisions
|
||
|
||
Use this decision rule:
|
||
- **fn_task_create:** create executable work when ownership is not predetermined.
|
||
- **fn_delegate_task:** assign immediately when a specific agent should own the work now.
|
||
- **fn_memory_append:** use \`scope="agent"\` for your own operating context and \`scope="project"\` for repo-wide durable knowledge; avoid transient run-by-run chatter.
|
||
|
||
If unsure who should do the work, prefer fn_task_create and let scheduler routing happen naturally.
|
||
|
||
## Common Patterns
|
||
|
||
- **Unowned risk discovered:** create one focused task with concrete acceptance language.
|
||
- **Known specialist needed:** list agents, then delegate to matching role/capability.
|
||
- **Repeated confusion across runs:** append a concise memory entry so future agents avoid the same mistake.
|
||
- **Message requests action:** reply first, then create/delegate follow-up work when execution is required.
|
||
|
||
## Memory Boundaries
|
||
|
||
You may receive an Agent Memory section and a Project Memory section.
|
||
- Agent Memory is specific to you, including imported and user-created agents such as CEO-style coordinator agents. It has its own long-term memory, daily notes, dreams, and qmd-backed retrieval under .fusion/agent-memory/{agentId}/.
|
||
- Project Memory is the workspace memory system under .fusion/memory/ with long-term memory, daily notes, dreams, and qmd-backed retrieval.
|
||
- Keep these separate: do not copy personal agent operating notes into Project Memory unless they are genuinely useful to every future agent in this workspace.
|
||
- Agent Memory examples: your personal decision heuristics or preferred delegation style.
|
||
- Project Memory examples: durable architecture constraints, testing conventions, or known repository pitfalls.
|
||
|
||
## Processing Messages
|
||
|
||
When you are woken by an incoming message (source includes "wake-on-message"), you should:
|
||
1. If fn_read_messages is available, use it to check your inbox for unread messages.
|
||
2. Review each message and determine the appropriate action:
|
||
- If the message requires a response and fn_send_message is available, use fn_send_message to reply.
|
||
- When replying, include 'reply_to_message_id' with the original message ID from fn_read_messages output.
|
||
- If the message is informational, acknowledge it and respond via fn_send_message when appropriate.
|
||
- If the message requests work, create a follow-up task with fn_task_create.
|
||
- If the request has a clear owner and fn_delegate_task is available, delegate it directly.
|
||
3. After processing messages, continue with your ambient work.
|
||
|
||
Example flow:
|
||
- Read inbox → classify message → reply with reply_to_message_id → create/delegate follow-up if needed → finish with fn_heartbeat_done.
|
||
|
||
When sending messages:
|
||
- Be concise and clear about what you need or what you've done.
|
||
- Use 'reply_to_message_id' when replying so threaded conversations stay linked.
|
||
- Include relevant context (task IDs, file paths) in metadata when applicable.
|
||
- Use agent-to-agent for inter-agent communication.`;
|
||
|
||
// Backward-compatible alias; prefer HEARTBEAT_NO_TASK_SYSTEM_PROMPT.
|
||
export const HEARTBEAT_SYSTEM_PROMPT_NO_TASK = HEARTBEAT_NO_TASK_SYSTEM_PROMPT;
|
||
|
||
/**
|
||
* Per-tick heartbeat procedure appended to every execution prompt. Forces the
|
||
* agent to re-anchor on its own operating procedure each wake instead of
|
||
* silently grinding on a previously assigned task.
|
||
*/
|
||
export const HEARTBEAT_PROCEDURE = `## Heartbeat Procedure (run every tick, in order)
|
||
|
||
1. **Identity & context** — review the **Identity Snapshot** at the top of
|
||
this prompt. Confirm your role, soul, instructions, and memory match what
|
||
you expect, and surface any anomalies in your first text output before
|
||
doing anything else. The full content is in the Custom Instructions
|
||
section of your system prompt.
|
||
2. **Inbox** — when fn_read_messages is available, call it immediately and
|
||
process unread/pending messages before any other action; reply with
|
||
reply_to_message_id when answering.
|
||
3. **Wake delta** — read the Wake Delta block above. The wake reason is the
|
||
highest-priority change for this heartbeat. If you were woken by a comment
|
||
or a message, acknowledge it before doing anything else.
|
||
4. **Assignment review** — if you have an assigned task, re-read its current
|
||
description, latest comments, and any task documents. Decide whether the
|
||
prior plan is still valid given the wake delta. Do not assume yesterday's
|
||
plan is still correct.
|
||
5. **Pick the next concrete action** — exactly ONE useful action this heartbeat:
|
||
advance the task, create a follow-up, log findings, delegate, or update
|
||
memory. Don't stop at planning unless the task is a planning task.
|
||
6. **Persist progress** — fn_task_log for observations, fn_task_document_write
|
||
for durable findings, status updates only when the work warrants it.
|
||
7. **Exit** — call fn_heartbeat_done with a one-line summary of what changed
|
||
this tick. If you took no action, say so and explain why.
|
||
|
||
Critical: a heartbeat without observable progress (a log, a document write, a
|
||
status change, a comment, a delegation, or an explicit "no-op with reason") is
|
||
a bug. Do not loop on the same plan across heartbeats without recording why.`;
|
||
|
||
/**
|
||
* No-task variant of HEARTBEAT_PROCEDURE. Keep this aligned with the ambient
|
||
* tool set (no fn_task_log / fn_task_document_* in no-task runs).
|
||
*/
|
||
export const HEARTBEAT_NO_TASK_PROCEDURE = `## Heartbeat Procedure (run every tick, in order)
|
||
|
||
1. **Identity & context** — review the **Identity Snapshot** at the top of
|
||
this prompt. Confirm your role, soul, instructions, and memory match what
|
||
you expect, and surface any anomalies in your first text output before
|
||
doing anything else. The full content is in the Custom Instructions
|
||
section of your system prompt.
|
||
2. **Inbox** — when fn_read_messages is available, call it immediately and
|
||
process unread/pending messages before any other action; reply with
|
||
reply_to_message_id when answering.
|
||
3. **Wake delta** — read the Wake Delta block above. The wake reason is the
|
||
highest-priority change for this heartbeat. If you were woken by a comment
|
||
or a message, acknowledge it before doing anything else.
|
||
4. **Ambient review** — since you have no assigned task, review board/project
|
||
signals and recent memory context before acting.
|
||
5. **Pick the next concrete action** — exactly ONE useful action this heartbeat:
|
||
create a focused task, delegate work, send/reply to a message, or append
|
||
durable memory.
|
||
6. **Persist progress** — use available ambient tools only:
|
||
fn_task_create, fn_delegate_task, fn_send_message, fn_memory_append.
|
||
7. **Exit** — call fn_heartbeat_done with a one-line summary of what changed
|
||
this tick. If you took no action, say so and explain why.
|
||
|
||
Critical: a heartbeat without observable progress (a created task, delegation,
|
||
message reply, memory append, or explicit "no-op with reason") is a bug. Do
|
||
not loop on the same plan across heartbeats without recording why.`;
|
||
|
||
/** Parameter schema for the fn_heartbeat_done tool */
|
||
const heartbeatDoneParams = Type.Object({
|
||
summary: Type.Optional(Type.String({ description: "Summary of what was accomplished this heartbeat" })),
|
||
});
|
||
|
||
/**
|
||
* Truncate a string to `maxChars`, appending a marker so callers can see
|
||
* content was clipped. Returns the original string unchanged when it fits.
|
||
*/
|
||
function truncatePrompt(text: string, maxChars: number): string {
|
||
if (text.length <= maxChars) return text;
|
||
return `${text.slice(0, maxChars)}\n\n... (truncated, ${text.length} chars)`;
|
||
}
|
||
|
||
/**
|
||
* Build the per-tick **Identity Snapshot** block injected into every
|
||
* heartbeat execution prompt.
|
||
*
|
||
* Why inline (not just a tool): plugin runtimes (openclaw, hermes, paperclip)
|
||
* wrap external CLIs and may not propagate JS `customTools` callbacks to the
|
||
* underlying agent. Embedding the snapshot in the prompt body guarantees the
|
||
* agent always sees its identity regardless of runtime tool support.
|
||
*
|
||
* The full soul/instructions/memory content is already loaded in the system
|
||
* prompt's Custom Instructions section. The snapshot intentionally carries
|
||
* only presence flags + 8-char content hashes — enough to detect drift or
|
||
* misload, without paying a multi-KB preview tax on every tick.
|
||
*/
|
||
function shortContentHash(value: string): string {
|
||
return createHash("sha256").update(value).digest("hex").slice(0, 8);
|
||
}
|
||
|
||
function buildIdentitySnapshot(args: {
|
||
agent: Agent;
|
||
resolvedInstructions: string;
|
||
workspaceMemory: string;
|
||
}): string {
|
||
const { agent, resolvedInstructions, workspaceMemory } = args;
|
||
|
||
const soulTrimmed = typeof agent.soul === "string" ? agent.soul.trim() : "";
|
||
const instrTrimmed = resolvedInstructions.trim();
|
||
const inlineMemoryTrimmed = typeof agent.memory === "string" ? agent.memory.trim() : "";
|
||
const workspaceMemoryTrimmed = workspaceMemory.trim();
|
||
const memorySource = inlineMemoryTrimmed ? "inline" : workspaceMemoryTrimmed ? "workspace" : null;
|
||
const memTrimmed = inlineMemoryTrimmed || workspaceMemoryTrimmed;
|
||
|
||
const formatField = (trimmed: string, source?: "inline" | "workspace"): string => {
|
||
if (!trimmed) return "absent";
|
||
const sourceLabel = source ? `, source: ${source}` : "";
|
||
return `loaded (${trimmed.length} chars, sha256:${shortContentHash(trimmed)}${sourceLabel})`;
|
||
};
|
||
|
||
return [
|
||
"## Identity Snapshot",
|
||
"",
|
||
"Full content is in the Custom Instructions section of your system prompt. Surface anomalies in your first text output before acting.",
|
||
"",
|
||
`- agentId: ${agent.id}`,
|
||
`- name: ${agent.name}`,
|
||
`- role: ${agent.role}`,
|
||
`- soul: ${formatField(soulTrimmed)}`,
|
||
`- instructions: ${formatField(instrTrimmed)}`,
|
||
`- memory: ${formatField(memTrimmed, memorySource ?? undefined)}`,
|
||
].join("\n");
|
||
}
|
||
|
||
async function getHeartbeatMemorySettings(taskStore: TaskStore): Promise<Settings | undefined> {
|
||
const maybeGetSettings = (taskStore as { getSettings?: () => Promise<Settings> }).getSettings;
|
||
if (!maybeGetSettings) {
|
||
return undefined;
|
||
}
|
||
return maybeGetSettings.call(taskStore);
|
||
}
|
||
|
||
/**
|
||
* HeartbeatMonitor monitors agents via periodic polling.
|
||
* Detects missed heartbeats, auto-terminates unresponsive agents,
|
||
* and provides the Paperclip-style execution engine via executeHeartbeat().
|
||
*/
|
||
export class HeartbeatMonitor {
|
||
private store: AgentStore;
|
||
private configStore: AgentStore;
|
||
private pollIntervalMs: number;
|
||
private heartbeatTimeoutMs: number;
|
||
private maxConcurrentRuns: number;
|
||
private onMissed?: (agentId: string, reason: string) => void;
|
||
private onRecovered?: (agentId: string) => void;
|
||
private onTerminated?: (agentId: string, reason: string) => void;
|
||
private onRunStarted?: (agentId: string, run: AgentHeartbeatRun) => void;
|
||
private onRunCompleted?: (agentId: string, run: AgentHeartbeatRun) => void;
|
||
private taskStore?: TaskStore;
|
||
private rootDir?: string;
|
||
private messageStore?: MessageStore;
|
||
private pluginRunner?: import("./plugin-runner.js").PluginRunner;
|
||
private reflectionStore?: ReflectionStore;
|
||
private reflectionService?: AgentReflectionService;
|
||
private selfImproveService?: SelfImproveServiceLike;
|
||
private approvalRequestStore?: ApprovalRequestStore;
|
||
|
||
private trackedAgents: Map<string, TrackedAgent> = new Map();
|
||
private agentStartLocks: Map<string, Promise<unknown>> = new Map();
|
||
private pollInterval: NodeJS.Timeout | null = null;
|
||
private isRunning = false;
|
||
|
||
/** Tasks created per agent during heartbeat runs (keyed by agentId) */
|
||
private runCreatedTasks: Map<string, Array<{ id: string; description: string }>> = new Map();
|
||
|
||
constructor(options: HeartbeatMonitorOptions) {
|
||
this.store = options.store;
|
||
this.configStore = options.agentStore ?? options.store;
|
||
this.pollIntervalMs = options.pollIntervalMs ?? 3_600_000;
|
||
this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? 60000;
|
||
this.maxConcurrentRuns = options.maxConcurrentRuns ?? 1;
|
||
this.onMissed = options.onMissed;
|
||
this.onRecovered = options.onRecovered;
|
||
this.onTerminated = options.onTerminated;
|
||
this.onRunStarted = options.onRunStarted;
|
||
this.onRunCompleted = options.onRunCompleted;
|
||
this.taskStore = options.taskStore;
|
||
this.rootDir = options.rootDir;
|
||
this.messageStore = options.messageStore;
|
||
this.pluginRunner = options.pluginRunner;
|
||
this.reflectionStore = options.reflectionStore;
|
||
this.reflectionService = options.reflectionService;
|
||
this.selfImproveService = options.selfImproveService;
|
||
}
|
||
|
||
private getApprovalRequestStore(): ApprovalRequestStore {
|
||
if (!this.approvalRequestStore) {
|
||
if (!this.taskStore) {
|
||
throw new Error("HeartbeatMonitor missing taskStore for approval request persistence");
|
||
}
|
||
this.approvalRequestStore = new ApprovalRequestStore(this.taskStore.getDatabase());
|
||
}
|
||
return this.approvalRequestStore;
|
||
}
|
||
|
||
private buildActionGateContext(agent: Agent, taskId?: string, runId?: string): AgentActionGateContext | undefined {
|
||
if (isEphemeralAgent(agent)) {
|
||
return undefined;
|
||
}
|
||
const policy = resolveEffectiveAgentPermissionPolicy(agent.permissionPolicy);
|
||
return {
|
||
agentId: agent.id,
|
||
agentName: agent.name,
|
||
isEphemeral: false,
|
||
taskId,
|
||
runId,
|
||
permissionPolicy: policy,
|
||
createApprovalRequest: async (decision, args) => this.getApprovalRequestStore().create({
|
||
requester: { actorId: agent.id, actorType: "agent", actorName: agent.name },
|
||
taskId,
|
||
runId,
|
||
targetAction: {
|
||
category: decision.category === "exempt" ? "command_execution" : decision.category,
|
||
action: decision.operation,
|
||
summary: decision.summary,
|
||
resourceType: decision.resourceType,
|
||
resourceId: decision.resourceId ?? "",
|
||
context: { ...decision.metadata, approvalDedupeKey: decision.approvalDedupeKey, toolName: decision.toolName, toolArgs: args },
|
||
},
|
||
}),
|
||
findPendingApprovalByDedupeKey: async (dedupeKey) => {
|
||
const pending = this.getApprovalRequestStore().list({ status: "pending", requesterActorId: agent.id, taskId, limit: 100 });
|
||
return pending.find((request) => request.targetAction.context?.approvalDedupeKey === dedupeKey) ?? null;
|
||
},
|
||
};
|
||
}
|
||
|
||
private buildPermanentAgentGatingContext(agent: Agent, taskId?: string, runId?: string): import("@fusion/core").PermanentAgentGatingContext | undefined {
|
||
if (isEphemeralAgent(agent)) {
|
||
return undefined;
|
||
}
|
||
|
||
return {
|
||
permissionPolicy: resolveEffectiveAgentPermissionPolicy(agent.permissionPolicy),
|
||
requester: { actorId: agent.id, actorType: "agent", actorName: agent.name },
|
||
taskId,
|
||
runId,
|
||
createApprovalRequest: async ({ category, toolName, args }) => this.getApprovalRequestStore().create({
|
||
requester: { actorId: agent.id, actorType: "agent", actorName: agent.name },
|
||
taskId,
|
||
runId,
|
||
targetAction: {
|
||
category,
|
||
action: toolName,
|
||
summary: `Permanent-agent gated action for ${toolName}`,
|
||
resourceType: "tool",
|
||
resourceId: toolName,
|
||
context: {
|
||
toolName,
|
||
toolArgs: args,
|
||
source: "permanent-agent-gating",
|
||
},
|
||
},
|
||
}),
|
||
findPendingApprovalRequest: async (dedupeKey) => {
|
||
const pending = this.getApprovalRequestStore().list({ status: "pending", requesterActorId: agent.id, taskId, limit: 100 });
|
||
return pending.find((request) => request.targetAction.context?.approvalDedupeKey === dedupeKey) ?? null;
|
||
},
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Start the heartbeat monitoring loop.
|
||
* Safe to call multiple times - no-op if already running.
|
||
*/
|
||
start(): void {
|
||
if (this.isRunning) return;
|
||
|
||
this.isRunning = true;
|
||
if (this.messageStore) {
|
||
this.messageStore.setMessageToAgentHook(this.handleMessageToAgent.bind(this));
|
||
}
|
||
// Reconcile any agents stuck in `state="running"` with no active run.
|
||
// Past versions of governance-skip paths (budget/global-pause) called
|
||
// completeRun with skipStateTransition=true after startRun had already
|
||
// moved the agent to "running", leaving the row stuck. New runs no
|
||
// longer leak this way, but pre-existing rows need a one-shot fix.
|
||
void this.reconcileOrphanedRunningAgents();
|
||
this.pollInterval = setInterval(() => {
|
||
void this.checkMissedHeartbeats();
|
||
}, this.pollIntervalMs);
|
||
}
|
||
|
||
/**
|
||
* Find agents in `state="running"` that are not actually running and flip
|
||
* them to `"active"`. An agent is considered orphaned when either:
|
||
* (a) it has no active heartbeat run record, or
|
||
* (b) it is not in this monitor's in-memory tracked set AND its
|
||
* lastHeartbeatAt is older than 3× the configured timeout.
|
||
*
|
||
* Case (a) covers historical bypass paths (governance-skip, supersede-on-
|
||
* startRun, safety-net run termination) that ended the run record but
|
||
* never propagated the agent-state transition. Case (b) covers a process
|
||
* that crashed mid-run, leaving both the run row and the agent row stuck.
|
||
*
|
||
* Called on monitor start AND periodically from the polling loop to keep
|
||
* the system self-healing across versions. Best-effort — failures are
|
||
* logged but do not block the caller.
|
||
*/
|
||
private async reconcileOrphanedRunningAgents(): Promise<void> {
|
||
try {
|
||
const runningAgents = await this.store.listAgents({ state: "running", includeEphemeral: true });
|
||
const now = Date.now();
|
||
for (const agent of runningAgents) {
|
||
let reason: string | null = null;
|
||
const activeRun = await this.store.getActiveHeartbeatRun(agent.id);
|
||
if (!activeRun) {
|
||
reason = "no active run";
|
||
} else if (!this.trackedAgents.has(agent.id)) {
|
||
const timeoutMs = this.resolveAgentConfig(agent.id).heartbeatTimeoutMs;
|
||
const lastTs = agent.lastHeartbeatAt ? Date.parse(agent.lastHeartbeatAt) : NaN;
|
||
const heartbeatAgeMs = Number.isFinite(lastTs) ? Math.max(0, now - lastTs) : Infinity;
|
||
if (heartbeatAgeMs > timeoutMs * 3) {
|
||
try {
|
||
const detail = await this.store.getRunDetail(agent.id, activeRun.id);
|
||
if (detail && detail.status !== "completed" && detail.status !== "failed" && detail.status !== "terminated") {
|
||
await this.store.saveRun({
|
||
...detail,
|
||
endedAt: new Date().toISOString(),
|
||
status: "terminated",
|
||
stderrExcerpt: `Reconciled stale run (no heartbeat for ${formatDuration(heartbeatAgeMs)}; threshold ${formatDuration(timeoutMs * 3)})`,
|
||
});
|
||
}
|
||
await this.store.endHeartbeatRun(activeRun.id, "terminated");
|
||
} catch (runEndErr) {
|
||
heartbeatLog.warn(`Failed to terminate stale run ${activeRun.id} for ${agent.id}: ${runEndErr instanceof Error ? runEndErr.message : String(runEndErr)}`);
|
||
}
|
||
reason = `stale heartbeat (${formatDuration(heartbeatAgeMs)} since lastHeartbeatAt)`;
|
||
}
|
||
}
|
||
if (!reason) continue;
|
||
try {
|
||
await this.store.updateAgentState(agent.id, "active");
|
||
this.clearRunState(agent.id);
|
||
heartbeatLog.log(`Reconciled orphaned running agent ${agent.id} → active (${reason})`);
|
||
} catch (err) {
|
||
heartbeatLog.warn(`Failed to reconcile orphaned running agent ${agent.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
} catch (err) {
|
||
heartbeatLog.warn(`reconcileOrphanedRunningAgents scan failed: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Stop the heartbeat monitoring loop.
|
||
* Does not untrack agents - they remain in memory.
|
||
*/
|
||
stop(): void {
|
||
if (this.messageStore) {
|
||
this.messageStore.setMessageToAgentHook(() => {});
|
||
}
|
||
if (!this.isRunning) return;
|
||
|
||
this.isRunning = false;
|
||
if (this.pollInterval) {
|
||
clearInterval(this.pollInterval);
|
||
this.pollInterval = null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if the monitor is currently running.
|
||
*/
|
||
isActive(): boolean {
|
||
return this.isRunning;
|
||
}
|
||
|
||
/**
|
||
* Get the project root directory this monitor is bound to.
|
||
* Returns undefined when not configured for execution.
|
||
*/
|
||
getRootDir(): string | undefined {
|
||
return this.rootDir;
|
||
}
|
||
|
||
/**
|
||
* Register an agent for monitoring with optional session context.
|
||
* @param agentId - The agent ID
|
||
* @param session - Session with dispose() for cleanup
|
||
* @param runId - The heartbeat run ID
|
||
* @param sessionIdBefore - Optional session ID from before execution
|
||
*/
|
||
trackAgent(agentId: string, session: AgentSession, runId: string, sessionIdBefore?: string): void {
|
||
const tracked: TrackedAgent = {
|
||
agentId,
|
||
session,
|
||
runId,
|
||
lastSeen: Date.now(),
|
||
missedHeartbeatReported: false,
|
||
sessionIdBefore,
|
||
};
|
||
|
||
this.trackedAgents.set(agentId, tracked);
|
||
|
||
// Record initial heartbeat
|
||
void this.store.recordHeartbeat(agentId, "ok", runId);
|
||
}
|
||
|
||
/**
|
||
* Serialize run starts per agent to prevent concurrent execution.
|
||
* @param agentId - The agent ID
|
||
* @param fn - Function to execute with the lock
|
||
*/
|
||
async withAgentStartLock<T>(agentId: string, fn: () => Promise<T>): Promise<T> {
|
||
const existing = this.agentStartLocks.get(agentId) ?? Promise.resolve();
|
||
const operation = existing.then(
|
||
async () => {
|
||
try {
|
||
return await fn();
|
||
} finally {
|
||
// Clean up accumulated run state for this agent at end of each serialized run.
|
||
// This guarantees cleanup even when the run path throws without calling completeRun
|
||
// (e.g., execution error before completeRun is reached, or completeRun itself throws).
|
||
// Because withAgentStartLock serializes runs per agent, the finally runs after each
|
||
// run completes but before the next concurrent call's callback starts.
|
||
this.clearRunState(agentId);
|
||
}
|
||
},
|
||
async (err) => {
|
||
try {
|
||
throw err;
|
||
} finally {
|
||
this.clearRunState(agentId);
|
||
}
|
||
},
|
||
);
|
||
this.agentStartLocks.set(agentId, operation);
|
||
return operation as Promise<T>;
|
||
}
|
||
|
||
/**
|
||
* Start a rich heartbeat run with full context capture.
|
||
* Creates a structured run record and saves it to the run store.
|
||
* @param agentId - The agent ID
|
||
* @param options - Wakeup options with trigger context
|
||
* @returns The created run
|
||
*/
|
||
async startRun(agentId: string, options?: WakeupOptions): Promise<AgentHeartbeatRun> {
|
||
// Safety net: fail any existing active runs for this agent before creating a new one.
|
||
// This prevents accumulation of zombie runs when startRun is called multiple times
|
||
// (e.g., concurrent timer + on-demand triggers, or retries after crashes).
|
||
try {
|
||
const existingRun = await this.store.getActiveHeartbeatRun(agentId);
|
||
if (existingRun) {
|
||
heartbeatLog.warn(
|
||
`Agent ${agentId} has active run ${existingRun.id} — marking failed before starting new run`,
|
||
);
|
||
try {
|
||
const existingDetail = await this.store.getRunDetail(agentId, existingRun.id);
|
||
if (existingDetail) {
|
||
await this.store.saveRun({
|
||
...existingDetail,
|
||
endedAt: new Date().toISOString(),
|
||
status: "terminated",
|
||
stderrExcerpt: "Superseded by new heartbeat run (previous run was stale)",
|
||
});
|
||
}
|
||
await this.store.endHeartbeatRun(existingRun.id, "terminated");
|
||
this.clearRunState(agentId);
|
||
} catch (failErr) {
|
||
const failErrMessage = failErr instanceof Error ? failErr.message : String(failErr);
|
||
heartbeatLog.warn(
|
||
`Failed to terminate stale active run ${existingRun.id} for ${agentId}: ${failErrMessage} — continuing anyway`,
|
||
);
|
||
}
|
||
}
|
||
} catch (activeRunCheckErr) {
|
||
const msg = activeRunCheckErr instanceof Error ? activeRunCheckErr.message : String(activeRunCheckErr);
|
||
heartbeatLog.warn(`Failed to check for existing active run for ${agentId}: ${msg} — continuing with new run`);
|
||
}
|
||
|
||
const run = await this.store.startHeartbeatRun(agentId);
|
||
|
||
// Enrich with execution context
|
||
const enrichedRun: AgentHeartbeatRun = {
|
||
...run,
|
||
invocationSource: options?.source ?? "on_demand",
|
||
triggerDetail: options?.triggerDetail ?? "manual",
|
||
contextSnapshot: options?.contextSnapshot,
|
||
processPid: process.pid,
|
||
};
|
||
|
||
// Save rich run data
|
||
await this.store.saveRun(enrichedRun);
|
||
|
||
// Transition agent to running state
|
||
try {
|
||
await this.store.updateAgentState(agentId, "running");
|
||
} catch (startRunErr) {
|
||
heartbeatLog.warn(`updateAgentState(running) failed for ${agentId}: ${startRunErr instanceof Error ? startRunErr.message : String(startRunErr)} — continuing`);
|
||
}
|
||
|
||
this.onRunStarted?.(agentId, enrichedRun);
|
||
return enrichedRun;
|
||
}
|
||
|
||
/**
|
||
* Complete a heartbeat run with results.
|
||
* @param agentId - The agent ID
|
||
* @param runId - The run ID to complete
|
||
* @param result - Execution results
|
||
*/
|
||
async completeRun(
|
||
agentId: string,
|
||
runId: string,
|
||
result: {
|
||
status: "completed" | "failed" | "terminated";
|
||
exitCode?: number;
|
||
sessionIdAfter?: string;
|
||
usageJson?: { inputTokens: number; outputTokens: number; cachedTokens: number };
|
||
resultJson?: Record<string, unknown>;
|
||
stdoutExcerpt?: string;
|
||
stderrExcerpt?: string;
|
||
/** When true, preserve current agent state instead of forcing a terminal transition. */
|
||
skipStateTransition?: boolean;
|
||
}
|
||
): Promise<void> {
|
||
// Load and update the run
|
||
const run = await this.store.getRunDetail(agentId, runId);
|
||
if (!run) return;
|
||
|
||
const tracked = this.trackedAgents.get(agentId);
|
||
let completionResult = result;
|
||
|
||
// Merge accumulated task creations into resultJson
|
||
const createdTasks = this.runCreatedTasks.get(agentId);
|
||
const enrichedResultJson = createdTasks?.length
|
||
? { ...completionResult.resultJson, tasksCreated: createdTasks }
|
||
: completionResult.resultJson;
|
||
|
||
const completedRun: AgentHeartbeatRun = {
|
||
...run,
|
||
endedAt: new Date().toISOString(),
|
||
status: completionResult.status,
|
||
exitCode: completionResult.exitCode,
|
||
sessionIdBefore: tracked?.sessionIdBefore,
|
||
sessionIdAfter: completionResult.sessionIdAfter,
|
||
usageJson: completionResult.usageJson,
|
||
resultJson: enrichedResultJson,
|
||
stdoutExcerpt: completionResult.stdoutExcerpt,
|
||
stderrExcerpt: completionResult.stderrExcerpt,
|
||
};
|
||
|
||
await this.store.saveRun(completedRun);
|
||
|
||
// Clear accumulated run state for this agent.
|
||
// Safe to call even when runCreatedTasks was already cleared by withAgentStartLock's
|
||
// finally block (idempotent Map.delete), and necessary for direct completeRun calls
|
||
// that bypass the lock (e.g., test scenarios, edge-case error paths).
|
||
this.clearRunState(agentId);
|
||
|
||
// Update cumulative usage on agent
|
||
if (completionResult.usageJson) {
|
||
try {
|
||
const agent = await this.store.getAgent(agentId);
|
||
if (agent) {
|
||
await this.store.updateAgent(agentId, {
|
||
totalInputTokens: (agent.totalInputTokens ?? 0) + completionResult.usageJson.inputTokens,
|
||
totalOutputTokens: (agent.totalOutputTokens ?? 0) + completionResult.usageJson.outputTokens,
|
||
});
|
||
}
|
||
} catch (usageUpdateErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} usage update failed: ${usageUpdateErr instanceof Error ? usageUpdateErr.message : String(usageUpdateErr)} — continuing`);
|
||
}
|
||
}
|
||
|
||
// Budget governance: pause agent if over budget after usage update
|
||
if (completionResult.usageJson && completionResult.status !== "failed" && completionResult.status !== "terminated") {
|
||
try {
|
||
const budgetStatus = await this.store.getBudgetStatus(agentId);
|
||
if (budgetStatus.isOverBudget) {
|
||
heartbeatLog.log(`Agent ${agentId} is over budget — pausing with reason "budget-exhausted"`);
|
||
await this.store.updateAgentState(agentId, "paused");
|
||
await this.store.updateAgent(agentId, { pauseReason: "budget-exhausted" });
|
||
// Skip the normal state transition below since we already set the correct state
|
||
completionResult = { ...completionResult, skipStateTransition: true };
|
||
}
|
||
} catch (budgetCheckErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} budget check failed: ${budgetCheckErr instanceof Error ? budgetCheckErr.message : String(budgetCheckErr)} — proceeding with normal state transition`);
|
||
}
|
||
}
|
||
|
||
// Transition agent state based on result
|
||
if (!completionResult.skipStateTransition) {
|
||
try {
|
||
if (completionResult.status === "failed") {
|
||
await this.store.updateAgentState(agentId, "error");
|
||
await this.store.updateAgent(agentId, { lastError: completionResult.stderrExcerpt ?? "Run failed" });
|
||
} else if (completionResult.status === "terminated") {
|
||
await this.store.updateAgentState(agentId, "paused");
|
||
} else {
|
||
// Completed successfully - back to active and clear any stale failure marker.
|
||
await this.store.updateAgentState(agentId, "active");
|
||
await this.store.updateAgent(agentId, { lastError: undefined });
|
||
}
|
||
} catch (stateTransErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} state transition failed: ${stateTransErr instanceof Error ? stateTransErr.message : String(stateTransErr)} — continuing`);
|
||
}
|
||
}
|
||
|
||
// End the heartbeat run tracking
|
||
await this.store.endHeartbeatRun(runId, completionResult.status === "completed" ? "completed" : "terminated");
|
||
|
||
if (completionResult.status === "terminated") {
|
||
this.onTerminated?.(agentId, completionResult.stderrExcerpt ?? "Run terminated");
|
||
}
|
||
this.onRunCompleted?.(agentId, completedRun);
|
||
}
|
||
|
||
/**
|
||
* Stop an active heartbeat run for an agent.
|
||
*
|
||
* If an in-memory tracked session exists, dispose it and complete the run as terminated.
|
||
* If no tracked session exists, fall back to persisted active-run state and terminate that run record.
|
||
*
|
||
* No-op when no active run exists.
|
||
*/
|
||
async stopRun(agentId: string): Promise<void> {
|
||
const tracked = this.trackedAgents.get(agentId);
|
||
|
||
if (tracked) {
|
||
heartbeatLog.log(`Stopping tracked run ${tracked.runId} for ${agentId}`);
|
||
|
||
try {
|
||
tracked.session.dispose();
|
||
} catch (error) {
|
||
heartbeatLog.warn(`Failed to dispose tracked session while stopping run for ${agentId}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
|
||
this.untrackAgent(agentId);
|
||
|
||
await this.completeRun(agentId, tracked.runId, {
|
||
status: "terminated",
|
||
stderrExcerpt: "Run stopped by user",
|
||
});
|
||
|
||
try {
|
||
await this.store.updateAgentState(agentId, "active");
|
||
} catch (stopStateErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} updateAgentState(active) failed during stop: ${stopStateErr instanceof Error ? stopStateErr.message : String(stopStateErr)}`);
|
||
}
|
||
|
||
this.clearRunState(agentId);
|
||
return;
|
||
}
|
||
|
||
const activeRun = await this.store.getActiveHeartbeatRun(agentId);
|
||
if (!activeRun) {
|
||
this.clearRunState(agentId);
|
||
return;
|
||
}
|
||
|
||
heartbeatLog.log(`Stopping persisted run ${activeRun.id} for ${agentId} (no tracked session)`);
|
||
|
||
const existingRun = await this.store.getRunDetail(agentId, activeRun.id);
|
||
if (existingRun) {
|
||
await this.store.saveRun({
|
||
...existingRun,
|
||
endedAt: new Date().toISOString(),
|
||
status: "terminated",
|
||
stderrExcerpt: existingRun.stderrExcerpt ?? "Run stopped by user",
|
||
});
|
||
}
|
||
|
||
await this.store.endHeartbeatRun(activeRun.id, "terminated");
|
||
|
||
try {
|
||
await this.store.updateAgentState(agentId, "active");
|
||
} catch (stopPersistErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} updateAgentState(active) failed during persisted-run stop: ${stopPersistErr instanceof Error ? stopPersistErr.message : String(stopPersistErr)}`);
|
||
}
|
||
|
||
this.clearRunState(agentId);
|
||
}
|
||
|
||
async pauseAgent(agentId: string, options: PauseAgentOptions = {}): Promise<Agent> {
|
||
const { pauseReason, stopActiveRun = false, cascadeToTasks = true } = options;
|
||
|
||
if (stopActiveRun) {
|
||
try {
|
||
await this.stopRun(agentId);
|
||
} catch (error) {
|
||
heartbeatLog.warn(`pauseAgent(${agentId}) stopRun failed: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
|
||
const current = await this.store.getAgent(agentId);
|
||
if (!current) {
|
||
throw new Error(`Agent ${agentId} not found`);
|
||
}
|
||
|
||
let updated = current;
|
||
if (current.state !== "paused") {
|
||
updated = await this.store.updateAgentState(agentId, "paused");
|
||
}
|
||
|
||
if (pauseReason !== undefined && updated.pauseReason !== pauseReason) {
|
||
updated = await this.store.updateAgent(agentId, { pauseReason });
|
||
}
|
||
|
||
if (this.taskStore && cascadeToTasks) {
|
||
const assignedTasks = await this.taskStore.getTasksByAssignedAgent(agentId, { excludeArchived: true });
|
||
const toPause = assignedTasks.filter((task) => task.paused !== true);
|
||
const results = await Promise.allSettled(
|
||
toPause.map((task) => this.taskStore!.pauseTask(task.id, true, undefined, { pausedByAgentId: agentId })),
|
||
);
|
||
results.forEach((result, index) => {
|
||
if (result.status === "rejected") {
|
||
heartbeatLog.warn(`pauseAgent(${agentId}) failed to pause assigned task ${toPause[index]?.id}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`);
|
||
}
|
||
});
|
||
}
|
||
|
||
return updated;
|
||
}
|
||
|
||
async resumeAgent(agentId: string, options: ResumeAgentOptions = {}): Promise<Agent> {
|
||
const {
|
||
triggerDetail = "Triggered from state resume",
|
||
triggerSource = "state-resume",
|
||
clearPauseReason = true,
|
||
cascadeToTasks = true,
|
||
} = options;
|
||
|
||
const current = await this.store.getAgent(agentId);
|
||
if (!current) {
|
||
throw new Error(`Agent ${agentId} not found`);
|
||
}
|
||
|
||
let updated = current;
|
||
if (current.state !== "active") {
|
||
updated = await this.store.updateAgentState(agentId, "active");
|
||
}
|
||
|
||
if (clearPauseReason && updated.pauseReason !== undefined) {
|
||
updated = await this.store.updateAgent(agentId, { pauseReason: undefined });
|
||
}
|
||
|
||
if (this.taskStore && cascadeToTasks) {
|
||
const pausedTasks = await this.taskStore.getTasksByAssignedAgent(agentId, {
|
||
pausedOnly: true,
|
||
excludeArchived: true,
|
||
});
|
||
const toUnpause = pausedTasks.filter((task) => task.pausedByAgentId === agentId);
|
||
const results = await Promise.allSettled(toUnpause.map((task) => this.taskStore!.pauseTask(task.id, false)));
|
||
results.forEach((result, index) => {
|
||
if (result.status === "rejected") {
|
||
heartbeatLog.warn(`resumeAgent(${agentId}) failed to unpause assigned task ${toUnpause[index]?.id}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`);
|
||
}
|
||
});
|
||
}
|
||
|
||
const latest = await this.store.getAgent(agentId);
|
||
const isHeartbeatEnabled = latest?.runtimeConfig?.enabled !== false;
|
||
if (isHeartbeatEnabled) {
|
||
try {
|
||
await this.executeHeartbeat({
|
||
agentId,
|
||
source: "on_demand",
|
||
triggerDetail,
|
||
contextSnapshot: {
|
||
wakeReason: "on_demand",
|
||
triggerDetail,
|
||
triggerSource,
|
||
},
|
||
});
|
||
} catch (error) {
|
||
heartbeatLog.warn(`resumeAgent(${agentId}) executeHeartbeat failed: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
|
||
return (await this.store.getAgent(agentId)) ?? updated;
|
||
}
|
||
|
||
/**
|
||
* Remove an agent from monitoring.
|
||
* Does NOT end the heartbeat run - caller's responsibility.
|
||
* @param agentId - The agent ID
|
||
*/
|
||
untrackAgent(agentId: string): void {
|
||
this.trackedAgents.delete(agentId);
|
||
}
|
||
|
||
/**
|
||
* Record a heartbeat for a tracked agent.
|
||
* @param agentId - The agent ID
|
||
*/
|
||
recordHeartbeat(agentId: string): void {
|
||
const tracked = this.trackedAgents.get(agentId);
|
||
if (!tracked) return;
|
||
|
||
tracked.lastSeen = Date.now();
|
||
|
||
// If recovering from a missed heartbeat
|
||
if (tracked.missedHeartbeatReported) {
|
||
tracked.missedHeartbeatReported = false;
|
||
void this.store.recordHeartbeat(agentId, "recovered", tracked.runId);
|
||
this.onRecovered?.(agentId);
|
||
} else {
|
||
void this.store.recordHeartbeat(agentId, "ok", tracked.runId);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if an agent is healthy (heartbeat within timeout window).
|
||
* Uses per-agent heartbeatTimeoutMs from runtimeConfig if available,
|
||
* otherwise falls back to the monitor-level default.
|
||
* @param agentId - The agent ID
|
||
* @returns true if healthy, false if missed heartbeat or not tracked
|
||
*/
|
||
isAgentHealthy(agentId: string): boolean {
|
||
const tracked = this.trackedAgents.get(agentId);
|
||
if (!tracked) return false;
|
||
|
||
const config = this.resolveAgentConfig(agentId);
|
||
const elapsed = Date.now() - tracked.lastSeen;
|
||
return elapsed < config.heartbeatTimeoutMs;
|
||
}
|
||
|
||
/**
|
||
* Get list of currently tracked agent IDs.
|
||
* Useful for testing and debugging.
|
||
*/
|
||
getTrackedAgents(): string[] {
|
||
return Array.from(this.trackedAgents.keys());
|
||
}
|
||
|
||
/**
|
||
* Get the last seen timestamp for a tracked agent.
|
||
* @param agentId - The agent ID
|
||
* @returns Last seen timestamp, or undefined if not tracked
|
||
*/
|
||
getLastSeen(agentId: string): number | undefined {
|
||
return this.trackedAgents.get(agentId)?.lastSeen;
|
||
}
|
||
|
||
private handleMessageToAgent(message: Message): void {
|
||
if (message.toType !== "agent") {
|
||
return;
|
||
}
|
||
|
||
const agent = this.configStore.getCachedAgent(message.toId);
|
||
if (!agent) {
|
||
return;
|
||
}
|
||
|
||
const runtimeConfig = agent.runtimeConfig as AgentHeartbeatConfig | undefined;
|
||
// Only human-originated (user) messages may override an agent's
|
||
// messageResponseMode setting. Agent-to-agent traffic must respect the
|
||
// recipient's configured behavior to prevent agents from forcing wakes
|
||
// on each other.
|
||
const senderForcedWake =
|
||
message.metadata?.wakeRecipient === true && message.fromType === "user";
|
||
if (!senderForcedWake && runtimeConfig?.messageResponseMode !== "immediate") {
|
||
return;
|
||
}
|
||
|
||
const validStates = new Set(["active", "idle", "running"]);
|
||
if (!validStates.has(agent.state)) {
|
||
return;
|
||
}
|
||
|
||
void this.executeHeartbeat({
|
||
agentId: message.toId,
|
||
source: "on_demand",
|
||
triggerDetail: senderForcedWake ? "wake-on-message-forced" : "wake-on-message",
|
||
}).catch((error) => {
|
||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||
heartbeatLog.warn(`Wake-on-message heartbeat failed for ${message.toId}: ${errorMessage}`);
|
||
});
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
// Heartbeat execution (Paperclip wake → check → work → exit)
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Execute a heartbeat run for an agent.
|
||
*
|
||
* Implements the Paperclip-style execution model:
|
||
* 1. Wake — start a heartbeat run record
|
||
* 2. Check inbox — resolve the agent's assigned task
|
||
* 3. Work — run a lightweight agent session with coding-capable tools + fn_task_create/fn_task_log
|
||
* 4. Exit — record results and complete the run
|
||
*
|
||
* Budget governance:
|
||
* - Skip all triggers when the agent is over budget (`isOverBudget`)
|
||
* - Skip timer triggers when over the warning threshold (`isOverThreshold`)
|
||
* - Continue normal execution for critical triggers (assignment/on_demand) when only over threshold
|
||
*
|
||
* Per-agent execution is serialized via `withAgentStartLock` — concurrent calls
|
||
* for the same agent wait for the previous run to complete.
|
||
*
|
||
* @param options - Execution options (agent ID, source, optional task override)
|
||
* @returns The completed heartbeat run, or null if the monitor isn't configured for execution
|
||
* @throws Error if taskStore or rootDir are not configured
|
||
*/
|
||
async executeHeartbeat(options: HeartbeatExecutionOptions): Promise<AgentHeartbeatRun> {
|
||
const {
|
||
agentId,
|
||
source,
|
||
triggerDetail,
|
||
taskId: explicitTaskId,
|
||
contextSnapshot,
|
||
triggeringCommentIds,
|
||
triggeringCommentType,
|
||
} = options;
|
||
|
||
// Validate execution dependencies
|
||
if (!this.taskStore || !this.rootDir) {
|
||
throw new Error("HeartbeatMonitor not configured for execution (missing taskStore/rootDir)");
|
||
}
|
||
const taskStore = this.taskStore;
|
||
const rootDir = this.rootDir;
|
||
|
||
// Serialize per-agent
|
||
return this.withAgentStartLock(agentId, async () => {
|
||
heartbeatLog.log(`Executing heartbeat for ${agentId} (source=${source})`);
|
||
|
||
let preloadedAgent: Agent | null = null;
|
||
try {
|
||
preloadedAgent = await this.store.getAgent(agentId);
|
||
} catch (preloadErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} agent preloading failed: ${preloadErr instanceof Error ? preloadErr.message : String(preloadErr)} — will resolve in execution path`);
|
||
}
|
||
|
||
const resolvedTaskId = explicitTaskId ?? preloadedAgent?.taskId;
|
||
const contextTriggeringCommentIds = Array.isArray(contextSnapshot?.triggeringCommentIds)
|
||
? contextSnapshot.triggeringCommentIds.filter((id): id is string => typeof id === "string" && id.length > 0)
|
||
: undefined;
|
||
const contextTriggeringCommentType =
|
||
contextSnapshot?.triggeringCommentType === "steering"
|
||
|| contextSnapshot?.triggeringCommentType === "task"
|
||
|| contextSnapshot?.triggeringCommentType === "pr"
|
||
? contextSnapshot.triggeringCommentType
|
||
: undefined;
|
||
const effectiveTriggeringCommentIds = triggeringCommentIds ?? contextTriggeringCommentIds;
|
||
const effectiveTriggeringCommentType = triggeringCommentType ?? contextTriggeringCommentType;
|
||
|
||
const runContextSnapshot = {
|
||
...(contextSnapshot ?? {}),
|
||
...(resolvedTaskId ? { taskId: resolvedTaskId } : {}),
|
||
...(effectiveTriggeringCommentIds?.length
|
||
? { triggeringCommentIds: effectiveTriggeringCommentIds }
|
||
: {}),
|
||
...(effectiveTriggeringCommentType ? { triggeringCommentType: effectiveTriggeringCommentType } : {}),
|
||
};
|
||
|
||
// Start run
|
||
const run = await this.startRun(agentId, {
|
||
source,
|
||
triggerDetail,
|
||
contextSnapshot: Object.keys(runContextSnapshot).length > 0 ? runContextSnapshot : undefined,
|
||
});
|
||
|
||
// Build run context for mutation correlation
|
||
const runContext: RunMutationContext = {
|
||
runId: run.id,
|
||
agentId,
|
||
source,
|
||
};
|
||
|
||
// Build engine run context for audit instrumentation
|
||
const engineRunContext: EngineRunContext = {
|
||
runId: run.id,
|
||
agentId,
|
||
source,
|
||
phase: "heartbeat",
|
||
};
|
||
|
||
// Create run auditor for audit trail (FN-1404)
|
||
// Uses TaskStore.recordRunAuditEvent when available; no-ops otherwise
|
||
const audit = createRunAuditor(taskStore, engineRunContext);
|
||
|
||
let agentLogger: AgentLogger | null = null;
|
||
const flushAgentLogger = async (): Promise<void> => {
|
||
if (!agentLogger) {
|
||
return;
|
||
}
|
||
try {
|
||
await agentLogger.flush();
|
||
} catch (error) {
|
||
heartbeatLog.warn(`Failed to flush heartbeat logs for ${agentId}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
};
|
||
|
||
try {
|
||
// Budget governance: check if agent can run
|
||
try {
|
||
const budgetStatus = await this.store.getBudgetStatus(agentId);
|
||
if (budgetStatus.isOverBudget) {
|
||
heartbeatLog.log(`Agent ${agentId} budget exhausted — heartbeat skipped`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "budget_exhausted", budgetStatus },
|
||
skipStateTransition: true,
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
// Above threshold: only allow critical triggers (assignment, on_demand)
|
||
if (budgetStatus.isOverThreshold && source === "timer") {
|
||
heartbeatLog.log(`Agent ${agentId} over budget threshold (${budgetStatus.usagePercent}%) — timer heartbeat skipped`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "budget_threshold_exceeded", budgetStatus },
|
||
skipStateTransition: true,
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
} catch (budgetErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} budget status check failed: ${budgetErr instanceof Error ? budgetErr.message : String(budgetErr)} — proceeding without budget check`);
|
||
}
|
||
|
||
// Pause governance: globalPause blocks all heartbeat sources;
|
||
// enginePaused is a soft pause that only blocks timer ticks.
|
||
try {
|
||
const settings = await taskStore.getSettings();
|
||
if (settings.globalPause) {
|
||
heartbeatLog.log(`Agent ${agentId} heartbeat skipped — global pause active (source=${source})`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "global_pause", source },
|
||
skipStateTransition: true,
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
if (settings.enginePaused && source === "timer") {
|
||
heartbeatLog.log(`Agent ${agentId} timer heartbeat skipped — engine paused (soft pause)`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "engine_paused", source },
|
||
skipStateTransition: true,
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
} catch (pauseErr) {
|
||
heartbeatLog.warn(`Pause status check failed for ${agentId}: ${pauseErr instanceof Error ? pauseErr.message : String(pauseErr)} — proceeding`);
|
||
}
|
||
|
||
// Resolve agent
|
||
const agent = preloadedAgent ?? await this.store.getAgent(agentId);
|
||
if (!agent) {
|
||
heartbeatLog.warn(`Agent ${agentId} not found — completing run as failed`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "failed",
|
||
stderrExcerpt: `Agent ${agentId} not found`,
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
|
||
// Check if agent has identity (used later for no-task run decisions)
|
||
const agentHasIdentity = hasAgentIdentity(agent);
|
||
const isAgentEphemeral = isEphemeralAgent(agent);
|
||
const canRunNoTaskHeartbeat = agentHasIdentity && !isAgentEphemeral;
|
||
|
||
// Resolve task assignment (explicit override → existing assignment → inbox-lite selection)
|
||
let taskId = explicitTaskId ?? agent.taskId;
|
||
let inboxSelection: InboxTask | null = null;
|
||
|
||
if (!taskId) {
|
||
inboxSelection = await taskStore.selectNextTaskForAgent(agentId, { id: agent.id, role: agent.role });
|
||
if (inboxSelection && !canAgentTakeImplementationTask(agent, inboxSelection.task)) {
|
||
heartbeatLog.log(
|
||
`Agent ${agentId} (role=${agent.role}) skipped inbox-selected task ${inboxSelection.task.id} due to executor-role assignment policy`,
|
||
);
|
||
inboxSelection = null;
|
||
}
|
||
if (inboxSelection) {
|
||
taskId = inboxSelection.task.id;
|
||
heartbeatLog.log(`Inbox selected task ${taskId} (priority: ${inboxSelection.priority}) for agent ${agentId}`);
|
||
|
||
// Persist assignment to AgentStore so subsequent runs retain linkage.
|
||
if (agent.taskId !== taskId) {
|
||
await this.store.assignTask(agentId, taskId, runContext);
|
||
// Audit trail: record assignment mutation (FN-1404)
|
||
await audit.database({ type: "task:assign", target: taskId });
|
||
}
|
||
|
||
// FN-1253 compatibility: if checkout API is available on TaskStore,
|
||
// try to claim the lease. On conflict, skip this task gracefully.
|
||
const checkoutTask = (taskStore as TaskStore & {
|
||
checkoutTask?: (taskId: string, agentId: string, runContext?: RunMutationContext) => Promise<unknown>;
|
||
}).checkoutTask;
|
||
if (typeof checkoutTask === "function") {
|
||
try {
|
||
await checkoutTask.call(taskStore, taskId, agentId, runContext);
|
||
// Audit trail: record checkout mutation (FN-1404)
|
||
await audit.database({ type: "task:checkout", target: taskId });
|
||
} catch (checkoutErr) {
|
||
heartbeatLog.warn(`Task ${taskId} checkout failed: ${checkoutErr instanceof Error ? checkoutErr.message : String(checkoutErr)} — skipping`);
|
||
taskId = undefined;
|
||
inboxSelection = null;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (taskId && run.contextSnapshot?.taskId !== taskId) {
|
||
const updatedRun: AgentHeartbeatRun = {
|
||
...run,
|
||
contextSnapshot: {
|
||
...(run.contextSnapshot ?? {}),
|
||
taskId,
|
||
},
|
||
};
|
||
await this.store.saveRun(updatedRun);
|
||
|
||
// Update engine run context with resolved taskId for audit trail (FN-1404)
|
||
engineRunContext.taskId = taskId;
|
||
}
|
||
|
||
let autoClaimCandidates: TaskDetail[] = [];
|
||
const autoClaimEnabled = isAutoClaimRelevantTasksEnabled(agent);
|
||
if (!taskId && canRunNoTaskHeartbeat && autoClaimEnabled) {
|
||
const listTasks = (taskStore as TaskStore & { listTasks?: (options?: { slim?: boolean }) => Promise<TaskDetail[]> }).listTasks;
|
||
if (typeof listTasks === "function") {
|
||
try {
|
||
const allTasks = await listTasks.call(taskStore, { slim: true });
|
||
const tasksById = new Map(allTasks.map((candidate) => [candidate.id, candidate]));
|
||
const openCandidates = allTasks
|
||
.filter((candidate) => (
|
||
candidate.column === "todo"
|
||
&& candidate.paused !== true
|
||
&& !candidate.assignedAgentId
|
||
&& !candidate.checkedOutBy
|
||
&& candidate.dependencies.every((dependencyId) => {
|
||
const dependency = tasksById.get(dependencyId);
|
||
return dependency?.column === "done" || dependency?.column === "archived";
|
||
})
|
||
))
|
||
.sort((a, b) => {
|
||
const aSortAt = a.columnMovedAt ?? a.createdAt;
|
||
const bSortAt = b.columnMovedAt ?? b.createdAt;
|
||
return aSortAt.localeCompare(bSortAt);
|
||
})
|
||
.slice(0, 10);
|
||
|
||
const roleCompatibleCandidates = openCandidates.filter((candidate) => canAgentTakeImplementationTask(agent, candidate));
|
||
const skippedIncompatibleCount = openCandidates.length - roleCompatibleCandidates.length;
|
||
if (skippedIncompatibleCount > 0) {
|
||
heartbeatLog.log(
|
||
`Agent ${agentId} (role=${agent.role}) skipped auto-claim of ${skippedIncompatibleCount} implementation task(s) — only executor agents may claim implementation work`,
|
||
);
|
||
}
|
||
|
||
autoClaimCandidates = roleCompatibleCandidates;
|
||
const ranked = roleCompatibleCandidates
|
||
.map((candidate) => ({ candidate, score: taskRelevanceScore(agent, candidate as TaskDetail) }))
|
||
.filter((entry) => entry.score > 0)
|
||
.sort((a, b) => b.score - a.score || (a.candidate.columnMovedAt ?? a.candidate.createdAt).localeCompare(b.candidate.columnMovedAt ?? b.candidate.createdAt));
|
||
|
||
if (ranked.length > 0) {
|
||
const claimResult = await this.store.claimTaskForAgent(agentId, ranked[0].candidate.id, runContext);
|
||
if (claimResult.ok) {
|
||
taskId = ranked[0].candidate.id;
|
||
heartbeatLog.log(`Agent ${agentId} auto-claimed relevant task ${taskId}`);
|
||
} else {
|
||
heartbeatLog.log(`Agent ${agentId} auto-claim skipped (${claimResult.reason})`);
|
||
}
|
||
}
|
||
} catch (autoClaimError) {
|
||
heartbeatLog.warn(`Auto-claim scan failed for ${agentId}: ${autoClaimError instanceof Error ? autoClaimError.message : String(autoClaimError)}`);
|
||
}
|
||
}
|
||
}
|
||
if (!taskId) {
|
||
// Agents with identity (soul, instructions, memory) should run a full heartbeat
|
||
// session even without a task, so they can do ambient work like messaging,
|
||
// memory management, task creation, and delegation.
|
||
// Ephemeral agents and agents without identity still exit gracefully.
|
||
if (!canRunNoTaskHeartbeat) {
|
||
heartbeatLog.log(`Agent ${agentId} has no task assignment — graceful exit`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "no_assignment" },
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
heartbeatLog.log(`Agent ${agentId} has no task but has identity — running no-task heartbeat`);
|
||
}
|
||
let isNoTaskRun = !taskId;
|
||
|
||
// Validate agent state (only for task-scoped runs)
|
||
if (!isNoTaskRun) {
|
||
const validStates = ["active", "running", "idle"];
|
||
if (!validStates.includes(agent.state)) {
|
||
heartbeatLog.log(`Agent ${agentId} state is "${agent.state}" — graceful exit`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "invalid_state", state: agent.state },
|
||
skipStateTransition: true,
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
}
|
||
|
||
// Fetch task context (only for task-scoped runs)
|
||
let taskDetail: TaskDetail | undefined;
|
||
if (!isNoTaskRun) {
|
||
// taskId is guaranteed to be defined here because isNoTaskRun = !taskId
|
||
const resolvedTaskId = taskId!;
|
||
try {
|
||
taskDetail = await taskStore.getTask(resolvedTaskId);
|
||
} catch (taskDetailErr) {
|
||
heartbeatLog.warn(`Task ${resolvedTaskId} fetch failed: ${taskDetailErr instanceof Error ? taskDetailErr.message : String(taskDetailErr)} — graceful exit`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "task_not_found", taskId: resolvedTaskId },
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
|
||
if (taskDetail.column === "done" || taskDetail.column === "archived") {
|
||
if (agent.taskId === resolvedTaskId) {
|
||
heartbeatLog.log(
|
||
`Agent ${agentId} linked task ${resolvedTaskId} is ${taskDetail.column} — clearing assignment and running heartbeat without task context`,
|
||
);
|
||
try {
|
||
await this.store.assignTask(agentId, undefined, runContext);
|
||
} catch (clearErr) {
|
||
heartbeatLog.warn(
|
||
`Failed to clear terminal task assignment ${resolvedTaskId} for ${agentId}: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`,
|
||
);
|
||
}
|
||
|
||
taskId = undefined;
|
||
taskDetail = undefined;
|
||
isNoTaskRun = true;
|
||
|
||
if (!canRunNoTaskHeartbeat) {
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "no_assignment" },
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
} else {
|
||
heartbeatLog.log(
|
||
`Heartbeat for ${agentId} targeted terminal task ${resolvedTaskId} (${taskDetail.column}) — graceful exit`,
|
||
);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "terminal_task", taskId: resolvedTaskId, column: taskDetail.column },
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
}
|
||
|
||
if (isNoTaskRun) {
|
||
heartbeatLog.log(`Agent ${agentId} terminal task assignment resolved into no-task heartbeat`);
|
||
} else {
|
||
const liveTaskDetail = taskDetail;
|
||
if (!liveTaskDetail) {
|
||
heartbeatLog.warn(`Task ${resolvedTaskId} lost detail after terminal-assignment handling — graceful exit`);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: { reason: "task_not_found", taskId: resolvedTaskId },
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
|
||
// Checkout enforcement: agent must hold the lease to work on this task.
|
||
// The heartbeat only validates existing checkout state — it does NOT attempt
|
||
// to acquire a checkout itself. The calling system (scheduler, API trigger)
|
||
// is responsible for checking out the task before the heartbeat starts.
|
||
if (liveTaskDetail.checkedOutBy && liveTaskDetail.checkedOutBy !== agentId) {
|
||
heartbeatLog.warn(
|
||
`Agent ${agentId} does not hold checkout for ${resolvedTaskId} (held by ${liveTaskDetail.checkedOutBy}) — graceful exit`
|
||
);
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
resultJson: {
|
||
reason: "checkout_conflict",
|
||
taskId: resolvedTaskId,
|
||
checkedOutBy: liveTaskDetail.checkedOutBy,
|
||
},
|
||
});
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
|
||
}
|
||
}
|
||
|
||
// Track usage via callbacks
|
||
const STDOUT_EXCERPT_LIMIT = 4000;
|
||
let outputLength = 0;
|
||
let toolCallCount = 0;
|
||
let heartbeatSummary: string | undefined;
|
||
let stdoutExcerpt = "";
|
||
|
||
const appendStdoutExcerpt = (delta: string): void => {
|
||
if (stdoutExcerpt.length >= STDOUT_EXCERPT_LIMIT) {
|
||
return;
|
||
}
|
||
const remaining = STDOUT_EXCERPT_LIMIT - stdoutExcerpt.length;
|
||
stdoutExcerpt += delta.slice(0, remaining);
|
||
};
|
||
|
||
// Create fn_heartbeat_done tool
|
||
const heartbeatDoneTool: ToolDefinition = {
|
||
name: "fn_heartbeat_done",
|
||
label: "Heartbeat Done",
|
||
description: "Signal that the heartbeat execution is complete. Call when finished.",
|
||
parameters: heartbeatDoneParams,
|
||
execute: async (_id: string, params: Static<typeof heartbeatDoneParams>) => {
|
||
if (params.summary) {
|
||
heartbeatSummary = params.summary;
|
||
}
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Heartbeat complete.${params.summary ? ` Summary: ${params.summary}` : ""}`,
|
||
}],
|
||
details: {},
|
||
};
|
||
},
|
||
};
|
||
|
||
// Build tools with task creation tracking and run context for mutation correlation
|
||
// For no-task runs, exclude fn_task_log and document tools (they require a taskId)
|
||
let heartbeatTools: ToolDefinition[];
|
||
if (isNoTaskRun) {
|
||
// No-task runs: fn_task_create, fn_list_agents, fn_delegate_task, fn_get_agent_config, fn_update_agent_config, messaging, memory, fn_heartbeat_done
|
||
heartbeatTools = [];
|
||
|
||
// fn_task_create tool
|
||
heartbeatTools.push(createTaskCreateTool(taskStore, {
|
||
sourceType: "agent_heartbeat",
|
||
sourceAgentId: agentId,
|
||
sourceRunId: runContext?.runId,
|
||
}, { rootDir: this.rootDir }));
|
||
|
||
// Agent delegation tools
|
||
heartbeatTools.push(createListAgentsTool(this.store));
|
||
heartbeatTools.push(createDelegateTaskTool(this.store, taskStore, { rootDir: this.rootDir }));
|
||
heartbeatTools.push(createGetAgentConfigTool(this.store, agentId));
|
||
heartbeatTools.push(createUpdateAgentConfigTool(this.store, agentId));
|
||
|
||
// Messaging tools — when MessageStore is available
|
||
if (this.messageStore) {
|
||
heartbeatTools.push(createSendMessageTool(this.messageStore, agentId));
|
||
heartbeatTools.push(createReadMessagesTool(this.messageStore, agentId));
|
||
}
|
||
|
||
heartbeatTools.push(createReadEvaluationsTool(this.store, this.reflectionStore, agentId));
|
||
heartbeatTools.push(createUpdateIdentityTool(this.store, agentId));
|
||
if (this.reflectionService) {
|
||
heartbeatTools.push(createReflectOnPerformanceTool(this.reflectionService, agentId));
|
||
}
|
||
} else {
|
||
// Task-scoped runs: full tool set including fn_task_log and document tools
|
||
// taskId is guaranteed to be defined here because isNoTaskRun = !taskId
|
||
heartbeatTools = this.createHeartbeatTools(agentId, taskStore, taskId!, runContext, audit, this.messageStore);
|
||
}
|
||
|
||
let memorySettings: Settings | undefined;
|
||
try {
|
||
memorySettings = await getHeartbeatMemorySettings(taskStore);
|
||
heartbeatTools.push(...createMemoryTools(rootDir, memorySettings, {
|
||
agentMemory: {
|
||
agentId: agent.id,
|
||
agentName: agent.name,
|
||
memory: agent.memory,
|
||
},
|
||
}));
|
||
} catch (memorySettingsError) {
|
||
const message = memorySettingsError instanceof Error ? memorySettingsError.message : String(memorySettingsError);
|
||
heartbeatLog.warn(`Failed to configure heartbeat memory tools for ${agentId}: ${message}`);
|
||
}
|
||
// Build skill selection context for heartbeat session (uses waking agent's skills, no role fallback)
|
||
const skillContext = buildSessionSkillContextSync(agent, "heartbeat", rootDir, this.pluginRunner);
|
||
|
||
const baseHeartbeatSystemPrompt = isNoTaskRun
|
||
? HEARTBEAT_NO_TASK_SYSTEM_PROMPT
|
||
: HEARTBEAT_SYSTEM_PROMPT;
|
||
let resolvedInstructionsForIdentity = "";
|
||
let workspaceMemoryForIdentity = "";
|
||
try {
|
||
resolvedInstructionsForIdentity = await resolveAgentInstructionsWithRatings(agent, rootDir, this.store);
|
||
} catch (instructionError) {
|
||
const message = instructionError instanceof Error ? instructionError.message : String(instructionError);
|
||
heartbeatLog.warn(`Failed to resolve agent instructions for heartbeat ${agentId}: ${message}`);
|
||
}
|
||
|
||
try {
|
||
workspaceMemoryForIdentity = await readAgentMemoryWorkspaceLongTerm(rootDir, agent.id);
|
||
} catch (memoryReadErr) {
|
||
const message = memoryReadErr instanceof Error ? memoryReadErr.message : String(memoryReadErr);
|
||
heartbeatLog.warn(`Failed to resolve workspace memory for heartbeat ${agentId}: ${message}`);
|
||
}
|
||
|
||
let memoryInstructions = "";
|
||
if (memorySettings?.memoryEnabled !== false) {
|
||
try {
|
||
memoryInstructions = buildExecutionMemoryInstructions(rootDir, memorySettings);
|
||
} catch (memoryInstructionErr) {
|
||
const message = memoryInstructionErr instanceof Error ? memoryInstructionErr.message : String(memoryInstructionErr);
|
||
heartbeatLog.warn(`Failed to resolve project memory instructions for heartbeat ${agentId}: ${message}`);
|
||
}
|
||
}
|
||
|
||
let selfImprovePrompt = "";
|
||
let shouldRecordSelfImprove = false;
|
||
if (this.selfImproveService) {
|
||
try {
|
||
const shouldSelfImprove = await this.selfImproveService.shouldRunSelfImprove(agentId);
|
||
if (shouldSelfImprove) {
|
||
selfImprovePrompt = await this.selfImproveService.getSelfImprovePrompt(agentId);
|
||
shouldRecordSelfImprove = true;
|
||
}
|
||
} catch (selfImproveErr) {
|
||
heartbeatLog.warn(`Failed to resolve self-improvement prompt for ${agentId}: ${selfImproveErr instanceof Error ? selfImproveErr.message : String(selfImproveErr)}`);
|
||
}
|
||
}
|
||
|
||
const systemPrompt = buildSystemPromptWithInstructions(
|
||
baseHeartbeatSystemPrompt,
|
||
[resolvedInstructionsForIdentity, memoryInstructions, selfImprovePrompt].filter((part) => part.trim()).join("\n\n"),
|
||
);
|
||
const heartbeatContributions = this.pluginRunner
|
||
?.getPromptContributionsForSurface("heartbeat")
|
||
?? [];
|
||
if (heartbeatContributions.length > 0) {
|
||
heartbeatLog.log(`applied ${heartbeatContributions.length} plugin prompt contributions for heartbeat surface`);
|
||
}
|
||
const heartbeatPluginContributions = buildPluginPromptSection(
|
||
"heartbeat",
|
||
this.pluginRunner,
|
||
);
|
||
const systemPromptFinal = heartbeatPluginContributions
|
||
? `${systemPrompt}\n\n${heartbeatPluginContributions}`
|
||
: systemPrompt;
|
||
|
||
// fn_heartbeat_done must be the last tool in the array (stable terminal signal)
|
||
heartbeatTools.push(heartbeatDoneTool);
|
||
|
||
// Always-on AgentLogger: no-task runs use the callback sink wired to run-scoped JSONL;
|
||
// task-scoped runs write to both the task store AND the run-scoped JSONL.
|
||
if (isNoTaskRun) {
|
||
agentLogger = new AgentLogger({
|
||
appendLog: (entry) => this.store.appendRunLog(agentId, run.id, entry),
|
||
agent: agent.role as AgentRole,
|
||
persistAgentToolOutput: memorySettings?.persistAgentToolOutput,
|
||
});
|
||
} else if (taskId) {
|
||
agentLogger = new AgentLogger({
|
||
store: taskStore,
|
||
taskId,
|
||
agent: agent.role as AgentRole,
|
||
appendLog: (entry) => this.store.appendRunLog(agentId, run.id, entry),
|
||
persistAgentToolOutput: memorySettings?.persistAgentToolOutput,
|
||
});
|
||
}
|
||
|
||
// Create agent session
|
||
const { session } = await createResolvedAgentSession({
|
||
sessionPurpose: "heartbeat",
|
||
runtimeHint: extractRuntimeHint(agent.runtimeConfig),
|
||
pluginRunner: this.pluginRunner,
|
||
cwd: rootDir,
|
||
systemPrompt: systemPromptFinal,
|
||
tools: "coding",
|
||
customTools: heartbeatTools,
|
||
...(() => {
|
||
const { provider, modelId } = extractRuntimeModel(agent.runtimeConfig);
|
||
return { defaultProvider: provider, defaultModelId: modelId };
|
||
})(),
|
||
onText: (delta) => {
|
||
outputLength += delta.length;
|
||
appendStdoutExcerpt(delta);
|
||
agentLogger?.onText(delta);
|
||
},
|
||
onThinking: (delta) => {
|
||
agentLogger?.onThinking(delta);
|
||
},
|
||
onToolStart: (name, args) => {
|
||
agentLogger?.onToolStart(name, args);
|
||
},
|
||
onToolEnd: (name, isError, result) => {
|
||
toolCallCount++;
|
||
agentLogger?.onToolEnd(name, isError, result);
|
||
},
|
||
// Skill selection: use waking agent's skills (heartbeat has no role fallback)
|
||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||
actionGateContext: this.buildActionGateContext(agent, taskId, run.id),
|
||
permanentAgentGating: this.buildPermanentAgentGatingContext(agent, taskId, run.id),
|
||
});
|
||
|
||
// Track for monitoring
|
||
this.trackAgent(agentId, { dispose: () => session.dispose() }, run.id);
|
||
|
||
try {
|
||
// Build execution prompt
|
||
let pendingMessages: Message[] = [];
|
||
let executionPrompt: string;
|
||
|
||
// Derive a stable wake reason from source, triggerDetail, and trigger
|
||
// type so the agent can change its strategy based on *why* it woke up.
|
||
// Mirrors paperclip's PAPERCLIP_WAKE_REASON (see plan: wake delta).
|
||
const deriveWakeReason = (): string => {
|
||
if (effectiveTriggeringCommentType) return `comment_${effectiveTriggeringCommentType}`;
|
||
if (triggerDetail === "wake-on-message") return "message_received";
|
||
if (triggerDetail === "wake-on-message-forced") return "message_received_urgent";
|
||
if (triggerDetail === "wake-on-comment") return "comment_mention";
|
||
if (triggerDetail === "task-assigned") return "task_assigned";
|
||
if (source === "timer") return "timer";
|
||
if (source === "assignment") return "task_assigned";
|
||
if (source === "automation") return "automation";
|
||
if (source === "routine") return "routine";
|
||
return triggerDetail || source;
|
||
};
|
||
const wakeReason = deriveWakeReason();
|
||
|
||
// Per-agent override of the default HEARTBEAT_PROCEDURE: if the agent
|
||
// configured a heartbeatProcedurePath pointing to a markdown file in
|
||
// the project, use that instead. Reloaded fresh each tick (matches the
|
||
// existing instructionsPath/instructionsText reload contract) so an
|
||
// operator can iterate on procedure text without restarting agents.
|
||
const customProcedure = await resolveAgentHeartbeatProcedure(agent, rootDir);
|
||
const heartbeatProcedureText = customProcedure
|
||
?? (isNoTaskRun ? HEARTBEAT_NO_TASK_PROCEDURE : HEARTBEAT_PROCEDURE);
|
||
const reportsHealthSection = await this.buildReportsHealthSection(agent.id, this.store);
|
||
|
||
if (isNoTaskRun) {
|
||
// No-task heartbeat: agent has identity but no assigned task
|
||
// Fetch unread messages when messageStore is available (for all trigger types)
|
||
if (this.messageStore) {
|
||
try {
|
||
pendingMessages = this.messageStore.getInbox(agentId, "agent", { read: false, limit: 10 });
|
||
} catch (inboxErr) {
|
||
heartbeatLog.warn(`Failed to fetch inbox messages for ${agentId}: ${inboxErr instanceof Error ? inboxErr.message : String(inboxErr)}`);
|
||
}
|
||
}
|
||
|
||
// Build pending messages section
|
||
const pendingMessagesLines: string[] = [];
|
||
if (pendingMessages.length > 0) {
|
||
pendingMessagesLines.push(
|
||
"",
|
||
"Pending Messages:",
|
||
...pendingMessages.map((msg) => {
|
||
const timestamp = new Date(msg.createdAt).toLocaleString();
|
||
return `- [id: ${msg.id}] [from: ${msg.fromType}:${msg.fromId}] ${msg.content} (${timestamp})`;
|
||
}),
|
||
);
|
||
}
|
||
|
||
const candidateLines = autoClaimCandidates.length > 0
|
||
? [
|
||
"",
|
||
"Open Task Candidates (auto-claim scan):",
|
||
...autoClaimCandidates.slice(0, 10).map((candidate) => `- ${candidate.id}: ${candidate.title ?? candidate.description.slice(0, 80)}`),
|
||
]
|
||
: ["", "Open Task Candidates (auto-claim scan): none found"];
|
||
|
||
executionPrompt = [
|
||
`Heartbeat execution for agent "${agent.name}" (ID: ${agent.id})`,
|
||
`Source: ${source}${triggerDetail ? ` (${triggerDetail})` : ""}`,
|
||
"",
|
||
buildIdentitySnapshot({
|
||
agent,
|
||
resolvedInstructions: resolvedInstructionsForIdentity,
|
||
workspaceMemory: workspaceMemoryForIdentity,
|
||
}),
|
||
"",
|
||
"## Wake Delta",
|
||
`- source: ${source}${triggerDetail ? ` (${triggerDetail})` : ""}`,
|
||
`- wake reason: ${wakeReason}`,
|
||
`- assigned task: none`,
|
||
`- pending messages: ${pendingMessages.length}`,
|
||
`- auto-claim relevant tasks: ${autoClaimEnabled ? "enabled" : "disabled"}`,
|
||
"",
|
||
"Treat this wake delta as the highest-priority change for this heartbeat.",
|
||
"This is an autonomous heartbeat run (manual or automatic): re-anchor on",
|
||
"identity, process wake context, then complete ONE concrete action.",
|
||
"Run the Heartbeat Procedure (below) before doing anything else — even a",
|
||
"timer-only wake should re-check messages, memory, and project state.",
|
||
"",
|
||
heartbeatProcedureText,
|
||
"",
|
||
"**No assigned task** — This heartbeat run has no task assignment.",
|
||
"",
|
||
"You have identity (soul, instructions, and/or memory) loaded, which means you can perform",
|
||
"useful ambient work. Pick ONE high-value action and finish it clearly before ending:",
|
||
"",
|
||
"1. **Check your messages** — Use fn_read_messages to review pending messages.",
|
||
" If replying, use fn_send_message and include reply_to_message_id so threads stay linked.",
|
||
"",
|
||
"2. **Create new tasks** — Use fn_task_create for net-new executable work.",
|
||
" Prefer concrete tasks with clear outcomes; avoid vague placeholders.",
|
||
"",
|
||
"3. **Delegate work** — Use fn_list_agents to find available specialists, then",
|
||
" fn_delegate_task when immediate ownership by a specific agent is beneficial.",
|
||
"",
|
||
"4. **Update memory** — Use fn_memory_append for durable, reusable learnings",
|
||
" (conventions, pitfalls, architecture constraints), not transient chatter.",
|
||
"",
|
||
"5. **Monitor project flow** — Review board/project signals and surface issues",
|
||
" by creating or delegating follow-up work as appropriate.",
|
||
"",
|
||
"When auto-claim relevant tasks is enabled, review Open Task Candidates above and",
|
||
"prioritize tasks that align with your role and soul before creating net-new tasks.",
|
||
...candidateLines,
|
||
...pendingMessagesLines,
|
||
"",
|
||
"Your soul, instructions, and memory are already loaded in the system prompt.",
|
||
"Focus on work that benefits the project without requiring a specific task context.",
|
||
...(reportsHealthSection ? ["", reportsHealthSection] : []),
|
||
"",
|
||
"Call fn_heartbeat_done when finished.",
|
||
].join("\n");
|
||
} else {
|
||
// Task-scoped heartbeat: agent has an assigned task
|
||
const taskTitle = taskDetail!.title ?? taskDetail!.description.slice(0, 100);
|
||
|
||
// Fetch unread messages when messageStore is available (for all trigger types)
|
||
if (this.messageStore) {
|
||
try {
|
||
pendingMessages = this.messageStore.getInbox(agentId, "agent", { read: false, limit: 10 });
|
||
} catch (inboxErr) {
|
||
heartbeatLog.warn(`Failed to fetch inbox messages for ${agentId}: ${inboxErr instanceof Error ? inboxErr.message : String(inboxErr)}`);
|
||
}
|
||
}
|
||
|
||
const triggeringCommentLines: string[] = [];
|
||
if (effectiveTriggeringCommentIds && effectiveTriggeringCommentIds.length > 0) {
|
||
const commentLookup = new Map<string, { author: string; text: string }>();
|
||
for (const comment of taskDetail!.comments ?? []) {
|
||
commentLookup.set(comment.id, { author: comment.author, text: comment.text });
|
||
}
|
||
for (const steeringComment of taskDetail!.steeringComments ?? []) {
|
||
commentLookup.set(steeringComment.id, { author: steeringComment.author, text: steeringComment.text });
|
||
}
|
||
|
||
const formatCommentText = (text: string): string => text.replace(/\s+/g, " ").trim();
|
||
|
||
for (const commentId of effectiveTriggeringCommentIds) {
|
||
const comment = commentLookup.get(commentId);
|
||
if (comment) {
|
||
triggeringCommentLines.push(`- [${comment.author}]: "${formatCommentText(comment.text)}"`);
|
||
}
|
||
}
|
||
|
||
if (triggeringCommentLines.length > 0) {
|
||
triggeringCommentLines.unshift(
|
||
"",
|
||
"You were woken because of new comments on this task. Review them and take appropriate action.",
|
||
`Triggering comment type: ${effectiveTriggeringCommentType ?? "task"}`,
|
||
"New comments since last run:",
|
||
);
|
||
}
|
||
}
|
||
|
||
// Build pending messages section
|
||
const pendingMessagesLines: string[] = [];
|
||
if (pendingMessages.length > 0) {
|
||
pendingMessagesLines.push(
|
||
"",
|
||
"Pending Messages:",
|
||
...pendingMessages.map((msg) => {
|
||
const timestamp = new Date(msg.createdAt).toLocaleString();
|
||
return `- [id: ${msg.id}] [from: ${msg.fromType}:${msg.fromId}] ${msg.content} (${timestamp})`;
|
||
}),
|
||
);
|
||
}
|
||
|
||
executionPrompt = [
|
||
`Heartbeat execution for agent "${agent.name}" (ID: ${agent.id})`,
|
||
`Source: ${source}${triggerDetail ? ` (${triggerDetail})` : ""}`,
|
||
`Assigned task: ${taskId} — ${taskTitle}`,
|
||
"",
|
||
buildIdentitySnapshot({
|
||
agent,
|
||
resolvedInstructions: resolvedInstructionsForIdentity,
|
||
workspaceMemory: workspaceMemoryForIdentity,
|
||
}),
|
||
"",
|
||
"## Wake Delta",
|
||
`- source: ${source}${triggerDetail ? ` (${triggerDetail})` : ""}`,
|
||
`- wake reason: ${wakeReason}`,
|
||
`- assigned task: ${taskId}`,
|
||
`- pending messages: ${pendingMessages.length}`,
|
||
`- triggering comments: ${effectiveTriggeringCommentIds?.length ?? 0}`,
|
||
"",
|
||
"Treat this wake delta as the highest-priority change for this heartbeat.",
|
||
"This is an autonomous heartbeat run (manual or automatic): re-anchor on",
|
||
"identity, process wake context, then complete ONE concrete action.",
|
||
"Before resuming prior task work, run the Heartbeat Procedure (below) and",
|
||
"decide what action this delta requires. Your assigned task is one input",
|
||
"to the procedure — not the only thing to consider.",
|
||
"",
|
||
heartbeatProcedureText,
|
||
"",
|
||
"Task description:",
|
||
taskDetail!.description,
|
||
"",
|
||
taskDetail!.prompt ? `PROMPT.md:\n${taskDetail!.prompt}` : "No PROMPT.md available.",
|
||
...triggeringCommentLines,
|
||
...pendingMessagesLines,
|
||
...(reportsHealthSection ? ["", reportsHealthSection] : []),
|
||
"",
|
||
"Run the Heartbeat Procedure above. Call fn_heartbeat_done when finished.",
|
||
].join("\n");
|
||
}
|
||
|
||
// Persist prompts on the run record before executing so they are
|
||
// observable in the dashboard even if execution fails partway through.
|
||
try {
|
||
const runWithPrompts: AgentHeartbeatRun = {
|
||
...run,
|
||
systemPrompt: truncatePrompt(systemPromptFinal, 100_000),
|
||
executionPrompt: truncatePrompt(executionPrompt, 100_000),
|
||
heartbeatProcedureSource: customProcedure ? "custom" : "default",
|
||
};
|
||
await this.store.saveRun(runWithPrompts);
|
||
// Update local run reference so completeRun merges correctly
|
||
Object.assign(run, { systemPrompt: runWithPrompts.systemPrompt, executionPrompt: runWithPrompts.executionPrompt, heartbeatProcedureSource: runWithPrompts.heartbeatProcedureSource });
|
||
} catch (promptPersistErr) {
|
||
heartbeatLog.warn(`Failed to persist prompts for ${agentId}/${run.id}: ${promptPersistErr instanceof Error ? promptPersistErr.message : String(promptPersistErr)}`);
|
||
}
|
||
|
||
// Execute
|
||
await promptWithFallback(session, executionPrompt);
|
||
|
||
// Capture real per-session token counts from pi-coding-agent's
|
||
// SessionStats. Falls back to a 4-chars-per-token estimate of output
|
||
// when the runtime doesn't expose stats. When this heartbeat is
|
||
// task-scoped, also accumulate the delta onto task.tokenUsage so the
|
||
// stats panel reflects heartbeat-driven runs.
|
||
let usageInput = 0;
|
||
let usageOutput = Math.ceil(outputLength / 4);
|
||
let usageCached = 0;
|
||
try {
|
||
const sessionStats = (session as unknown as {
|
||
getSessionStats?: () => { tokens?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number } };
|
||
}).getSessionStats?.();
|
||
const tokens = sessionStats?.tokens;
|
||
if (tokens) {
|
||
usageInput = (tokens.input ?? 0) + (tokens.cacheWrite ?? 0);
|
||
usageOutput = tokens.output ?? usageOutput;
|
||
usageCached = tokens.cacheRead ?? 0;
|
||
}
|
||
} catch (statsErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} session stats read failed: ${statsErr instanceof Error ? statsErr.message : String(statsErr)} — using estimated tokens`);
|
||
}
|
||
|
||
if (!isNoTaskRun && taskId) {
|
||
try {
|
||
const { accumulateSessionTokenUsage } = await import("./session-token-usage.js");
|
||
await accumulateSessionTokenUsage(taskStore, taskId, session);
|
||
} catch (accumulateErr) {
|
||
heartbeatLog.warn(`Agent ${agentId} task token usage accumulate failed: ${accumulateErr instanceof Error ? accumulateErr.message : String(accumulateErr)}`);
|
||
}
|
||
}
|
||
|
||
await flushAgentLogger();
|
||
|
||
// Mark messages as read after successful processing (only if messages were included in prompt)
|
||
if (pendingMessages.length > 0 && this.messageStore) {
|
||
try {
|
||
this.messageStore.markAllAsRead(agentId, "agent");
|
||
} catch (markReadErr) {
|
||
heartbeatLog.warn(`Failed to mark messages as read for ${agentId}: ${markReadErr instanceof Error ? markReadErr.message : String(markReadErr)}`);
|
||
}
|
||
}
|
||
|
||
// Complete run successfully
|
||
const completionResultJson: Record<string, unknown> = {
|
||
summary: heartbeatSummary,
|
||
toolCallCount,
|
||
};
|
||
if (isNoTaskRun) {
|
||
// Identity agents without tasks get a special reason for observability
|
||
completionResultJson.reason = "no_assignment_identity_run";
|
||
} else if (inboxSelection) {
|
||
completionResultJson.reason = "inbox_selected";
|
||
completionResultJson.priority = inboxSelection.priority;
|
||
completionResultJson.taskId = taskId;
|
||
}
|
||
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "completed",
|
||
usageJson: { inputTokens: usageInput, outputTokens: usageOutput, cachedTokens: usageCached },
|
||
resultJson: completionResultJson,
|
||
stdoutExcerpt: stdoutExcerpt || undefined,
|
||
});
|
||
|
||
if (shouldRecordSelfImprove && this.selfImproveService) {
|
||
try {
|
||
await this.selfImproveService.recordSelfImprove(agentId);
|
||
} catch (selfImproveRecordErr) {
|
||
heartbeatLog.warn(`Failed to record self-improvement checkpoint for ${agentId}: ${selfImproveRecordErr instanceof Error ? selfImproveRecordErr.message : String(selfImproveRecordErr)}`);
|
||
}
|
||
}
|
||
|
||
heartbeatLog.log(`Heartbeat completed for ${agentId} (${toolCallCount} tool calls, ${usageInput} input + ${usageOutput} output + ${usageCached} cached tokens)`);
|
||
} catch (err) {
|
||
const errorDetail = formatError(err).detail;
|
||
heartbeatLog.error(`Heartbeat execution failed for ${agentId}: ${errorDetail}`);
|
||
await flushAgentLogger();
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "failed",
|
||
stderrExcerpt: errorDetail,
|
||
stdoutExcerpt: stdoutExcerpt || undefined,
|
||
});
|
||
} finally {
|
||
await flushAgentLogger();
|
||
// Defensively untrack the agent — wrap in try/catch to guarantee cleanup
|
||
// can't be blocked by an exception in untrackAgent itself.
|
||
try { this.untrackAgent(agentId); } catch (untrackErr) {
|
||
heartbeatLog.warn(`untrackAgent failed for ${agentId}: ${untrackErr instanceof Error ? untrackErr.message : String(untrackErr)}`);
|
||
}
|
||
try {
|
||
session.dispose();
|
||
} catch (disposeErr: unknown) {
|
||
const errorMessage = disposeErr instanceof Error ? disposeErr.message : String(disposeErr);
|
||
heartbeatLog.warn(`session.dispose() failed for ${agentId}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
} catch (err) {
|
||
const errorDetail = formatError(err).detail;
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
heartbeatLog.error(`Heartbeat execution error for ${agentId}: ${errorDetail}`);
|
||
await flushAgentLogger();
|
||
|
||
// Attempt to complete the run as failed if it's still active.
|
||
// If completeRun also fails, fall back to a direct DB update to ensure
|
||
// the run is not permanently stuck in "active" state.
|
||
try {
|
||
await this.completeRun(agentId, run.id, {
|
||
status: "failed",
|
||
stderrExcerpt: errorDetail,
|
||
});
|
||
} catch (completeRunErr) {
|
||
const completeRunErrMsg = completeRunErr instanceof Error ? completeRunErr.message : String(completeRunErr);
|
||
heartbeatLog.error(`completeRun failed for ${agentId}/${run.id}: ${completeRunErrMsg} — attempting safety-net completion`);
|
||
|
||
// Safety net: directly update the run record to prevent zombie run state.
|
||
// This runs only when completeRun itself threw, guaranteeing the run
|
||
// doesn't remain permanently stuck in "active" state.
|
||
try {
|
||
const runDetail = await this.store.getRunDetail(agentId, run.id);
|
||
if (runDetail && runDetail.status !== "completed" && runDetail.status !== "failed" && runDetail.status !== "terminated") {
|
||
await this.store.saveRun({
|
||
...runDetail,
|
||
endedAt: new Date().toISOString(),
|
||
status: "failed",
|
||
stderrExcerpt: `Heartbeat execution failed: ${errorMessage}. Run completion also failed: ${completeRunErrMsg}`,
|
||
});
|
||
await this.store.endHeartbeatRun(run.id, "terminated");
|
||
// Also clean up run state accumulator
|
||
this.clearRunState(agentId);
|
||
heartbeatLog.log(`Safety-net run completion for ${agentId}/${run.id} — run terminated`);
|
||
}
|
||
} catch (safetyNetErr) {
|
||
const safetyNetErrMsg = safetyNetErr instanceof Error ? safetyNetErr.message : String(safetyNetErr);
|
||
heartbeatLog.error(`Safety-net run completion also failed for ${agentId}/${run.id}: ${safetyNetErrMsg} — run may be stuck permanently`);
|
||
}
|
||
}
|
||
|
||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||
}
|
||
});
|
||
}
|
||
|
||
private async buildReportsHealthSection(agentId: string, agentStore: AgentStore): Promise<string | null> {
|
||
const getReports = (agentStore as AgentStore & { getAgentsByReportsTo?: (id: string) => Promise<Agent[]> }).getAgentsByReportsTo;
|
||
if (typeof getReports !== "function") {
|
||
return null;
|
||
}
|
||
|
||
let reports: Agent[];
|
||
try {
|
||
reports = await getReports(agentId);
|
||
} catch (err) {
|
||
heartbeatLog.warn(`Failed to load reports for ${agentId}: ${err instanceof Error ? err.message : String(err)}`);
|
||
return null;
|
||
}
|
||
if (reports.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
const now = Date.now();
|
||
const rows = reports.map((report) => {
|
||
const timeoutMs = this.resolveAgentConfig(report.id).heartbeatTimeoutMs;
|
||
const lastHeartbeatTs = report.lastHeartbeatAt ? Date.parse(report.lastHeartbeatAt) : NaN;
|
||
const heartbeatAgeMs = Number.isFinite(lastHeartbeatTs) ? Math.max(0, now - lastHeartbeatTs) : Infinity;
|
||
|
||
let health = "healthy";
|
||
if (report.state === "paused") {
|
||
health = report.pauseReason ? `paused (${report.pauseReason})` : "paused";
|
||
} else if (report.state === "error") {
|
||
health = "**stuck**";
|
||
} else if (report.state === "running") {
|
||
health = heartbeatAgeMs <= timeoutMs * 2 ? "healthy" : "**stuck**";
|
||
} else if ((report.state === "active" || report.state === "idle") && heartbeatAgeMs > timeoutMs * 3) {
|
||
health = "**stale**";
|
||
}
|
||
|
||
const task = report.taskId ?? "—";
|
||
const state = report.state;
|
||
const heartbeat = formatRelativeTime(report.lastHeartbeatAt);
|
||
return `| ${report.name} | ${state} | ${task} | ${heartbeat} | ${health} |`;
|
||
});
|
||
|
||
const hasStuck = rows.some((row) => row.includes("**stuck**"));
|
||
const hasStale = rows.some((row) => row.includes("**stale**"));
|
||
|
||
const actionLines = ["### Actions for Unresponsive Reports"];
|
||
if (hasStuck) {
|
||
actionLines.push("- For **stuck** reports: consider sending a message via fn_send_message asking for status, or reassigning their task via fn_delegate_task to a healthy agent.");
|
||
}
|
||
if (hasStale) {
|
||
actionLines.push("- For **stale** reports: the agent may have lost its heartbeat trigger — create a follow-up task to investigate.");
|
||
}
|
||
|
||
return [
|
||
"## Reports Health Check",
|
||
"",
|
||
`You have ${reports.length} agent(s) reporting to you. Review their status and intervene if any are unresponsive.`,
|
||
"",
|
||
"| Name | State | Task | Last Heartbeat | Health |",
|
||
"|------|-------|------|----------------|--------|",
|
||
...rows,
|
||
"",
|
||
...actionLines,
|
||
].join("\n");
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
// Heartbeat tools: createHeartbeatTools / clearRunState
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Create the tool set for a heartbeat agent session.
|
||
*
|
||
* Returns tools with tracking wrappers that record task creations
|
||
* so they can be included in the run's `resultJson.tasksCreated`.
|
||
*
|
||
* @param agentId - The agent ID (used for tracking and logging)
|
||
* @param taskStore - TaskStore for task creation and logging
|
||
* @param taskId - The assigned task ID (for fn_task_log context)
|
||
* @param runContext - Optional run context for mutation correlation
|
||
* @param audit - Optional run auditor for audit trail (FN-1404)
|
||
* @param messageStore - Optional MessageStore for messaging tools
|
||
* @returns Array of ToolDefinitions for the heartbeat session
|
||
*/
|
||
createHeartbeatTools(
|
||
agentId: string,
|
||
taskStore: TaskStore,
|
||
taskId: string,
|
||
runContext?: RunMutationContext,
|
||
audit?: ReturnType<typeof createRunAuditor>,
|
||
messageStore?: MessageStore,
|
||
): ToolDefinition[] {
|
||
const tools: ToolDefinition[] = [];
|
||
|
||
// Wrap createTaskCreateTool with tracking and agent-link logging
|
||
const baseCreateTool = createTaskCreateTool(taskStore, {
|
||
sourceType: "agent_heartbeat",
|
||
sourceAgentId: agentId,
|
||
}, { rootDir: this.rootDir });
|
||
const trackedCreateTool: ToolDefinition = {
|
||
...baseCreateTool,
|
||
execute: async (id: string, params: Static<typeof taskCreateParams>, signal, onUpdate, ctx) => {
|
||
const result = await baseCreateTool.execute(id, params, signal, onUpdate, ctx);
|
||
|
||
const createdTaskId = (result.details as { taskId?: string })?.taskId ?? "unknown";
|
||
|
||
// Log agent link on the created task with run context for correlation
|
||
try {
|
||
await taskStore.logEntry(createdTaskId, `Created by agent ${agentId} during heartbeat run`, undefined, runContext);
|
||
} catch (taskCreateLogErr) {
|
||
heartbeatLog.warn(`Task ${createdTaskId} agent-link log failed: ${taskCreateLogErr instanceof Error ? taskCreateLogErr.message : String(taskCreateLogErr)}`);
|
||
}
|
||
|
||
// Audit trail: record task creation (FN-1404)
|
||
await audit?.database({ type: "task:create", target: createdTaskId });
|
||
|
||
// Accumulate for inclusion in run resultJson
|
||
if (!this.runCreatedTasks.has(agentId)) {
|
||
this.runCreatedTasks.set(agentId, []);
|
||
}
|
||
this.runCreatedTasks.get(agentId)!.push({
|
||
id: createdTaskId,
|
||
description: params.description,
|
||
});
|
||
|
||
return result;
|
||
},
|
||
};
|
||
tools.push(trackedCreateTool);
|
||
|
||
// fn_task_log tool (with run context for mutation correlation)
|
||
tools.push(createTaskLogToolWithContext(taskStore, taskId, runContext));
|
||
|
||
// Document tools for persisting durable findings
|
||
tools.push(createTaskDocumentWriteTool(taskStore, taskId));
|
||
tools.push(createTaskDocumentReadTool(taskStore, taskId));
|
||
// Agent delegation tools — discover and delegate work to other agents
|
||
tools.push(createListAgentsTool(this.store));
|
||
tools.push(createDelegateTaskTool(this.store, taskStore, { rootDir: this.rootDir }));
|
||
tools.push(createGetAgentConfigTool(this.store, agentId));
|
||
tools.push(createUpdateAgentConfigTool(this.store, agentId));
|
||
|
||
// Messaging tools — when MessageStore is available, agents can send and receive messages
|
||
if (messageStore) {
|
||
tools.push(createSendMessageTool(messageStore, agentId));
|
||
tools.push(createReadMessagesTool(messageStore, agentId));
|
||
}
|
||
|
||
tools.push(createReadEvaluationsTool(this.store, this.reflectionStore, agentId));
|
||
tools.push(createUpdateIdentityTool(this.store, agentId));
|
||
if (this.reflectionService) {
|
||
tools.push(createReflectOnPerformanceTool(this.reflectionService, agentId));
|
||
}
|
||
|
||
return tools;
|
||
}
|
||
|
||
/**
|
||
* Clear accumulated run state for an agent.
|
||
* Called after completing a run to reset the `runCreatedTasks` accumulator.
|
||
* @param agentId - The agent ID
|
||
*/
|
||
clearRunState(agentId: string): void {
|
||
this.runCreatedTasks.delete(agentId);
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
// Private methods
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Get the resolved heartbeat configuration for an agent.
|
||
* Reads per-agent config from runtimeConfig with fallback to monitor defaults.
|
||
* @param agentId - The agent ID
|
||
* @returns Resolved config with validated values
|
||
*/
|
||
async getAgentHeartbeatConfig(agentId: string): Promise<ResolvedHeartbeatConfig> {
|
||
return this.getAgentConfig(agentId);
|
||
}
|
||
|
||
/**
|
||
* Resolve per-agent heartbeat config from runtimeConfig with validation and fallbacks.
|
||
*/
|
||
private resolveAgentConfig(agentId: string): ResolvedHeartbeatConfig {
|
||
// Defaults from monitor-level construction
|
||
const result: ResolvedHeartbeatConfig = {
|
||
pollIntervalMs: this.pollIntervalMs,
|
||
heartbeatTimeoutMs: this.heartbeatTimeoutMs,
|
||
maxConcurrentRuns: this.maxConcurrentRuns,
|
||
};
|
||
|
||
try {
|
||
const agent = this.configStore.getCachedAgent?.(agentId);
|
||
if (agent?.runtimeConfig) {
|
||
const rc = agent.runtimeConfig;
|
||
|
||
if (typeof rc.heartbeatIntervalMs === "number" && Number.isFinite(rc.heartbeatIntervalMs)) {
|
||
result.pollIntervalMs = Math.max(1000, rc.heartbeatIntervalMs);
|
||
}
|
||
if (typeof rc.heartbeatTimeoutMs === "number" && Number.isFinite(rc.heartbeatTimeoutMs)) {
|
||
result.heartbeatTimeoutMs = Math.max(5000, rc.heartbeatTimeoutMs);
|
||
}
|
||
if (typeof rc.maxConcurrentRuns === "number" && Number.isFinite(rc.maxConcurrentRuns)) {
|
||
result.maxConcurrentRuns = Math.max(1, Math.round(rc.maxConcurrentRuns));
|
||
}
|
||
}
|
||
} catch (agentLookupErr) {
|
||
heartbeatLog.warn(`getAgentConfig(${agentId}) agent lookup failed: ${agentLookupErr instanceof Error ? agentLookupErr.message : String(agentLookupErr)} — using monitor defaults`);
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
private async getAgentConfig(agentId: string): Promise<ResolvedHeartbeatConfig> {
|
||
const result = this.resolveAgentConfig(agentId);
|
||
|
||
if (!this.taskStore) {
|
||
return result;
|
||
}
|
||
|
||
try {
|
||
const settings = await getHeartbeatMemorySettings(this.taskStore);
|
||
const rawMultiplier = settings?.heartbeatMultiplier;
|
||
const multiplier =
|
||
typeof rawMultiplier === "number" && Number.isFinite(rawMultiplier) && rawMultiplier > 0
|
||
? rawMultiplier
|
||
: 1;
|
||
|
||
result.pollIntervalMs = Math.max(1000, Math.round(result.pollIntervalMs * multiplier));
|
||
} catch (settingsErr) {
|
||
heartbeatLog.warn(`getAgentConfig(${agentId}) settings lookup failed: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)} — using base interval`);
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
private async checkMissedHeartbeats(): Promise<void> {
|
||
const now = Date.now();
|
||
|
||
for (const tracked of this.trackedAgents.values()) {
|
||
const config = await this.getAgentConfig(tracked.agentId);
|
||
const elapsed = now - tracked.lastSeen;
|
||
|
||
if (elapsed >= config.heartbeatTimeoutMs) {
|
||
const reason = `No heartbeat for ${formatDuration(elapsed)} (threshold: ${formatDuration(config.heartbeatTimeoutMs)})`;
|
||
// Missed heartbeat detected
|
||
if (!tracked.missedHeartbeatReported) {
|
||
tracked.missedHeartbeatReported = true;
|
||
await this.handleMissedHeartbeat(tracked, reason);
|
||
} else {
|
||
// Already reported - check if we should terminate
|
||
// Give 2x timeout for recovery before auto-terminate
|
||
if (elapsed >= config.heartbeatTimeoutMs * 2) {
|
||
await this.recoverUnresponsiveAgent(tracked, config.heartbeatTimeoutMs);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Periodically scan for orphaned `state="running"` rows so that a single
|
||
// missed termination can't leave an agent permanently stuck. Cheap query
|
||
// (indexed by state) so running it every poll is fine.
|
||
await this.reconcileOrphanedRunningAgents();
|
||
}
|
||
|
||
private async handleMissedHeartbeat(tracked: TrackedAgent, reason: string): Promise<void> {
|
||
// Record missed heartbeat
|
||
await this.store.recordHeartbeat(tracked.agentId, "missed", tracked.runId);
|
||
|
||
// Notify callback
|
||
this.onMissed?.(tracked.agentId, reason);
|
||
}
|
||
|
||
private async recoverUnresponsiveAgent(tracked: TrackedAgent, heartbeatTimeoutMs: number): Promise<void> {
|
||
const now = Date.now();
|
||
const elapsed = now - tracked.lastSeen;
|
||
const reason = `No heartbeat for ${formatDuration(elapsed)} (2× timeout threshold: ${formatDuration(heartbeatTimeoutMs * 2)})`;
|
||
|
||
heartbeatLog.warn(`Recovering unresponsive agent ${tracked.agentId}: ${reason}`);
|
||
|
||
const runIdToTerminate = tracked.runId;
|
||
|
||
try {
|
||
tracked.session.dispose();
|
||
} catch (err) {
|
||
heartbeatLog.warn(`Error disposing session for ${tracked.agentId}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
this.untrackAgent(tracked.agentId);
|
||
|
||
// Canonically end the run record. Without this, dispose() relies on the
|
||
// in-flight execution self-completing — which never happens when the run
|
||
// is actually hung. completeRun also updates agent state, but we still
|
||
// call pauseAgent below to set `pauseReason="heartbeat-unresponsive"`.
|
||
// We pass cascadeToTasks:false on both pause and resume — this is an
|
||
// internal recovery cycle, not a user-initiated pause, and shouldn't
|
||
// visibly toggle the user's task pause state.
|
||
try {
|
||
await this.completeRun(tracked.agentId, runIdToTerminate, {
|
||
status: "terminated",
|
||
stderrExcerpt: reason,
|
||
});
|
||
} catch (err) {
|
||
heartbeatLog.warn(`completeRun(terminated) failed for ${tracked.agentId}/${runIdToTerminate}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
try {
|
||
await this.pauseAgent(tracked.agentId, {
|
||
pauseReason: "heartbeat-unresponsive",
|
||
stopActiveRun: false,
|
||
cascadeToTasks: false,
|
||
});
|
||
} catch (err) {
|
||
heartbeatLog.warn(`Error pausing unresponsive agent ${tracked.agentId}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
try {
|
||
await this.resumeAgent(tracked.agentId, {
|
||
triggerDetail: "unresponsive-recovery",
|
||
triggerSource: "heartbeat-unresponsive",
|
||
clearPauseReason: true,
|
||
cascadeToTasks: false,
|
||
});
|
||
} catch (err) {
|
||
heartbeatLog.warn(`Error resuming unresponsive agent ${tracked.agentId}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
// HeartbeatTriggerScheduler — timer, assignment, and on-demand triggers
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/** Structured context passed when a trigger fires. */
|
||
export interface WakeContext {
|
||
/** Optional task ID associated with this trigger */
|
||
taskId?: string;
|
||
/** Why the agent was woken */
|
||
wakeReason: string;
|
||
/** Detail about the specific trigger */
|
||
triggerDetail: string;
|
||
/** IDs of comments that triggered this wake (if any) */
|
||
triggeringCommentIds?: string[];
|
||
/** Type of comment that triggered this wake */
|
||
triggeringCommentType?: "steering" | "task" | "pr";
|
||
/** Budget governance status for the agent at trigger time */
|
||
budgetStatus?: AgentBudgetStatus;
|
||
/** Additional context (intervalMs, etc.) */
|
||
[key: string]: unknown;
|
||
}
|
||
|
||
/** Callback invoked when a trigger fires. */
|
||
export type TriggerCallback = (
|
||
agentId: string,
|
||
source: HeartbeatInvocationSource,
|
||
context: WakeContext,
|
||
) => Promise<void>;
|
||
|
||
/** Per-agent timer state. The active handle is either the initial
|
||
* phase-aligned `setTimeout` waiting for the first overdue tick, or the
|
||
* steady-state `setInterval` installed once that first tick fires.
|
||
*/
|
||
interface AgentTimer {
|
||
intervalMs: number;
|
||
kind: "timeout" | "interval";
|
||
handle: ReturnType<typeof setInterval>;
|
||
}
|
||
|
||
/** Optional context passed to registerAgent, used to phase-align the
|
||
* initial timer fire to the agent's persisted heartbeat history.
|
||
*/
|
||
export interface RegisterAgentOptions {
|
||
/** ISO timestamp of the agent's last heartbeat. When set, the initial
|
||
* fire is scheduled at `lastHeartbeatAt + intervalMs` rather than
|
||
* `now + intervalMs`, so a process restart does not cost agents up to
|
||
* one full interval of silence.
|
||
*/
|
||
lastHeartbeatAt?: string | null;
|
||
}
|
||
|
||
/** Maximum random jitter (ms) added to the initial fire when an agent's
|
||
* next tick is already overdue. Prevents a thundering herd when the
|
||
* scheduler boots and many agents want to fire immediately.
|
||
*/
|
||
const OVERDUE_FIRE_JITTER_MS = 5_000;
|
||
|
||
/**
|
||
* True when an agent's state indicates it should be ticking right now.
|
||
* Heartbeats track liveness while the agent is meant to be doing work.
|
||
*
|
||
* States where timers should remain armed:
|
||
* - "active" — Agent is working
|
||
* - "running" — Agent has an active heartbeat run
|
||
* - "idle" — Agent is between tasks, waiting for work (FN-2289 fix)
|
||
*
|
||
* States where timers should be cleared:
|
||
* - "paused" — Agent is paused by budget exhaustion or manual action
|
||
* - "error" — Agent encountered an error
|
||
*/
|
||
function isTickableState(state: Agent["state"]): boolean {
|
||
return state === "active" || state === "running" || state === "idle";
|
||
}
|
||
|
||
/**
|
||
* True when the scheduler should manage this agent at all. Ephemeral
|
||
* (task-worker) agents are driven directly by TaskExecutor and must never
|
||
* acquire a scheduler timer.
|
||
*/
|
||
function isHeartbeatManaged(agent: Agent): boolean {
|
||
return !isEphemeralAgent(agent);
|
||
}
|
||
|
||
/**
|
||
* HeartbeatTriggerScheduler manages timer-based heartbeat triggers for agents.
|
||
*
|
||
* Timers are armed only for durable agents where all of the following hold:
|
||
* - `runtimeConfig.enabled !== false`
|
||
* - `state ∈ {active, running, idle}`
|
||
*
|
||
* Any other state, or any ephemeral/task-worker agent, clears the timer.
|
||
* State changes and heartbeat config updates are observed via AgentStore
|
||
* lifecycle events, while callers can still explicitly register existing
|
||
* agents during startup bootstrap.
|
||
*
|
||
* Other config knobs still apply:
|
||
* - `heartbeatIntervalMs`: Timer interval (default 1h)
|
||
* - `maxConcurrentRuns`: Skip tick if agent already has an active run
|
||
*/
|
||
export class HeartbeatTriggerScheduler {
|
||
private store: AgentStore;
|
||
private callback: TriggerCallback;
|
||
private taskStore?: TaskStore;
|
||
private timers: Map<string, AgentTimer> = new Map();
|
||
private registrationEpochs: Map<string, number> = new Map();
|
||
private running = false;
|
||
private assignedListener: ((agent: import("@fusion/core").Agent, taskId: string) => void) | null = null;
|
||
private createdListener: ((agent: import("@fusion/core").Agent) => void) | null = null;
|
||
private updatedListener: ((agent: import("@fusion/core").Agent) => void) | null = null;
|
||
private configRevisionListener: ((agentId: string, revision: AgentConfigRevision) => void) | null = null;
|
||
private deletedListener: ((agentId: string) => void) | null = null;
|
||
private isTaskExecuting?: (taskId: string) => boolean;
|
||
|
||
constructor(store: AgentStore, callback: TriggerCallback, taskStore?: TaskStore, options?: { isTaskExecuting?: (taskId: string) => boolean }) {
|
||
this.store = store;
|
||
this.callback = callback;
|
||
this.taskStore = taskStore;
|
||
this.isTaskExecuting = options?.isTaskExecuting;
|
||
}
|
||
|
||
/**
|
||
* Start the scheduler. Enables assignment watching.
|
||
* Existing agents still need one startup bootstrap pass via registerAgent().
|
||
*/
|
||
start(): void {
|
||
if (this.running) return;
|
||
this.running = true;
|
||
this.watchAssignments();
|
||
this.watchAgentLifecycle();
|
||
heartbeatLog.log("HeartbeatTriggerScheduler started");
|
||
}
|
||
|
||
/**
|
||
* Stop the scheduler and clear all timers.
|
||
*/
|
||
stop(): void {
|
||
if (!this.running) return;
|
||
this.running = false;
|
||
|
||
// Unwatch assignments
|
||
this.unwatchAssignments();
|
||
this.unwatchAgentLifecycle();
|
||
|
||
// Clear all timers (mix of phase-alignment timeouts and steady intervals).
|
||
for (const [agentId, timer] of this.timers) {
|
||
if (timer.kind === "timeout") {
|
||
clearTimeout(timer.handle as unknown as ReturnType<typeof setTimeout>);
|
||
} else {
|
||
clearInterval(timer.handle);
|
||
}
|
||
heartbeatLog.log(`Cleared timer for ${agentId}`);
|
||
}
|
||
this.timers.clear();
|
||
|
||
heartbeatLog.log("HeartbeatTriggerScheduler stopped");
|
||
}
|
||
|
||
/**
|
||
* Check if the scheduler is running.
|
||
*/
|
||
isActive(): boolean {
|
||
return this.running;
|
||
}
|
||
|
||
/** Default heartbeat interval when not explicitly configured (3600 seconds / 1 hour) */
|
||
private static readonly DEFAULT_HEARTBEAT_INTERVAL_MS = 3_600_000;
|
||
|
||
/**
|
||
* Register an agent for timer-based heartbeat triggers.
|
||
*
|
||
* The first fire is phase-aligned to `options.lastHeartbeatAt + intervalMs`
|
||
* when supplied. This means a process restart resumes each agent's
|
||
* existing schedule rather than waiting up to a full interval before the
|
||
* first tick — the previous behavior caused agents on long intervals
|
||
* (e.g. 1h) to appear "overdue" in the UI for nearly a full interval after
|
||
* every dashboard restart even though nothing was actually wrong with them.
|
||
*
|
||
* @param agentId - The agent ID
|
||
* @param config - Per-agent heartbeat config
|
||
* @param options - Optional registration context (e.g., lastHeartbeatAt)
|
||
*/
|
||
registerAgent(agentId: string, config: AgentHeartbeatConfig, options?: RegisterAgentOptions): void {
|
||
if (config.enabled === false) {
|
||
this.unregisterAgent(agentId);
|
||
return;
|
||
}
|
||
|
||
// Apply default interval if not explicitly configured
|
||
// This ensures agents with heartbeat monitoring enabled but no explicit interval
|
||
// still get periodic timer triggers (matching HeartbeatMonitor constructor default)
|
||
let rawIntervalMs = config.heartbeatIntervalMs;
|
||
let usingDefaultInterval = false;
|
||
if (!rawIntervalMs || typeof rawIntervalMs !== "number" || !Number.isFinite(rawIntervalMs) || rawIntervalMs <= 0) {
|
||
rawIntervalMs = HeartbeatTriggerScheduler.DEFAULT_HEARTBEAT_INTERVAL_MS;
|
||
usingDefaultInterval = true;
|
||
}
|
||
|
||
const intervalMs = Math.max(1000, Math.round(rawIntervalMs));
|
||
const registrationEpoch = (this.registrationEpochs.get(agentId) ?? 0) + 1;
|
||
this.registrationEpochs.set(agentId, registrationEpoch);
|
||
|
||
const lastHeartbeatAt = options?.lastHeartbeatAt ?? null;
|
||
|
||
// Register immediately with multiplier=1 so agents don't wait for async settings I/O.
|
||
this.applyTimerRegistration(agentId, intervalMs, 1, usingDefaultInterval, lastHeartbeatAt);
|
||
|
||
// If project settings are available, refresh registration with the current multiplier.
|
||
if (this.taskStore && typeof (this.taskStore as { getSettings?: () => Promise<Settings> }).getSettings === "function") {
|
||
void this.applyProjectMultiplierRegistration(agentId, intervalMs, usingDefaultInterval, registrationEpoch, lastHeartbeatAt);
|
||
}
|
||
}
|
||
|
||
private async applyProjectMultiplierRegistration(
|
||
agentId: string,
|
||
baseIntervalMs: number,
|
||
usingDefaultInterval: boolean,
|
||
expectedEpoch: number,
|
||
lastHeartbeatAt: string | null,
|
||
): Promise<void> {
|
||
let multiplier = 1;
|
||
|
||
try {
|
||
const settings = await getHeartbeatMemorySettings(this.taskStore!);
|
||
multiplier = HeartbeatTriggerScheduler.resolveHeartbeatMultiplier(settings?.heartbeatMultiplier);
|
||
} catch (settingsErr) {
|
||
heartbeatLog.warn(
|
||
`Failed to read heartbeatMultiplier for ${agentId}: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)} — using 1x`,
|
||
);
|
||
multiplier = 1;
|
||
}
|
||
|
||
// Guard against stale async completions after subsequent register/unregister calls.
|
||
if (this.registrationEpochs.get(agentId) !== expectedEpoch) {
|
||
return;
|
||
}
|
||
|
||
this.applyTimerRegistration(agentId, baseIntervalMs, multiplier, usingDefaultInterval, lastHeartbeatAt);
|
||
}
|
||
|
||
/**
|
||
* Compute the delay until the agent's next scheduled fire, given when it
|
||
* last heartbeat. When `lastHeartbeatAt` is missing or unparseable, falls
|
||
* back to a full-interval delay (matching the original behavior for
|
||
* agents that have never ticked). When the next fire is already overdue,
|
||
* returns a small randomized jitter to spread thundering herds at boot.
|
||
*/
|
||
private static computeInitialDelayMs(
|
||
intervalMs: number,
|
||
lastHeartbeatAt: string | null,
|
||
now: number = Date.now(),
|
||
): number {
|
||
if (!lastHeartbeatAt) {
|
||
return intervalMs;
|
||
}
|
||
const lastMs = Date.parse(lastHeartbeatAt);
|
||
if (!Number.isFinite(lastMs)) {
|
||
return intervalMs;
|
||
}
|
||
const remaining = lastMs + intervalMs - now;
|
||
if (remaining <= 0) {
|
||
return Math.floor(Math.random() * OVERDUE_FIRE_JITTER_MS);
|
||
}
|
||
return Math.min(remaining, intervalMs);
|
||
}
|
||
|
||
private applyTimerRegistration(
|
||
agentId: string,
|
||
baseIntervalMs: number,
|
||
multiplier: number,
|
||
usingDefaultInterval: boolean,
|
||
lastHeartbeatAt: string | null,
|
||
): void {
|
||
const effectiveIntervalMs = Math.max(1000, Math.round(baseIntervalMs * multiplier));
|
||
const initialDelayMs = HeartbeatTriggerScheduler.computeInitialDelayMs(
|
||
effectiveIntervalMs,
|
||
lastHeartbeatAt,
|
||
);
|
||
|
||
this.clearAgentTimer(agentId);
|
||
|
||
const armSteadyInterval = () => {
|
||
// The setTimeout fired and was consumed; replace it with the long-lived
|
||
// setInterval that drives every subsequent tick. Use the same
|
||
// effectiveIntervalMs so the cadence remains correct.
|
||
const intervalHandle = setInterval(() => {
|
||
void this.onTimerTick(agentId, effectiveIntervalMs);
|
||
}, effectiveIntervalMs);
|
||
this.timers.set(agentId, {
|
||
intervalMs: effectiveIntervalMs,
|
||
kind: "interval",
|
||
handle: intervalHandle,
|
||
});
|
||
};
|
||
|
||
if (initialDelayMs >= effectiveIntervalMs) {
|
||
// No phase-shift needed (agent has never ticked, or the saved
|
||
// lastHeartbeatAt is somehow in the future). Skip the timeout hop and
|
||
// arm the steady-state interval directly so the behavior matches the
|
||
// pre-phase-alignment scheduler.
|
||
armSteadyInterval();
|
||
} else {
|
||
const timeoutHandle = setTimeout(() => {
|
||
// Fire the overdue/phase-aligned tick first, then transition to the
|
||
// steady cadence. The tick fires regardless of whether the steady
|
||
// interval install succeeds, so a missed tick can never silently
|
||
// happen here.
|
||
void this.onTimerTick(agentId, effectiveIntervalMs);
|
||
armSteadyInterval();
|
||
}, initialDelayMs);
|
||
this.timers.set(agentId, {
|
||
intervalMs: effectiveIntervalMs,
|
||
kind: "timeout",
|
||
handle: timeoutHandle as unknown as ReturnType<typeof setInterval>,
|
||
});
|
||
}
|
||
|
||
const phaseSuffix = lastHeartbeatAt
|
||
? `, first fire in ${initialDelayMs}ms (phase-aligned to lastHeartbeatAt)`
|
||
: "";
|
||
|
||
if (multiplier !== 1) {
|
||
heartbeatLog.log(
|
||
`Registered timer for ${agentId} (every ${baseIntervalMs}ms, multiplier ${multiplier} → ${effectiveIntervalMs}ms effective${phaseSuffix})`,
|
||
);
|
||
return;
|
||
}
|
||
|
||
heartbeatLog.log(
|
||
usingDefaultInterval
|
||
? `Registered timer for ${agentId} (every ${effectiveIntervalMs}ms, default interval${phaseSuffix})`
|
||
: `Registered timer for ${agentId} (every ${effectiveIntervalMs}ms${phaseSuffix})`,
|
||
);
|
||
}
|
||
|
||
private clearAgentTimer(agentId: string): void {
|
||
const timer = this.timers.get(agentId);
|
||
if (!timer) {
|
||
return;
|
||
}
|
||
// Both kinds share the same opaque handle type at runtime, but we route
|
||
// through the matching clear function for clarity and to satisfy strict
|
||
// type narrowing on platforms that distinguish the two.
|
||
if (timer.kind === "timeout") {
|
||
clearTimeout(timer.handle as unknown as ReturnType<typeof setTimeout>);
|
||
} else {
|
||
clearInterval(timer.handle);
|
||
}
|
||
this.timers.delete(agentId);
|
||
}
|
||
|
||
private static resolveHeartbeatMultiplier(rawMultiplier: unknown): number {
|
||
if (typeof rawMultiplier !== "number" || !Number.isFinite(rawMultiplier) || rawMultiplier <= 0) {
|
||
return 1;
|
||
}
|
||
return rawMultiplier;
|
||
}
|
||
|
||
/**
|
||
* Unregister an agent, clearing its timer.
|
||
* @param agentId - The agent ID
|
||
*/
|
||
unregisterAgent(agentId: string): void {
|
||
this.registrationEpochs.set(agentId, (this.registrationEpochs.get(agentId) ?? 0) + 1);
|
||
if (this.timers.has(agentId)) {
|
||
this.clearAgentTimer(agentId);
|
||
heartbeatLog.log(`Unregistered timer for ${agentId}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get the set of currently registered agent IDs.
|
||
* Useful for testing.
|
||
*/
|
||
getRegisteredAgents(): string[] {
|
||
return Array.from(this.timers.keys());
|
||
}
|
||
|
||
/**
|
||
* Subscribe to agent:assigned events on the AgentStore.
|
||
* When a task is assigned to an agent, the trigger callback fires
|
||
* with source "assignment" and the task ID in the context.
|
||
*/
|
||
watchAssignments(): void {
|
||
if (this.assignedListener) return; // Already watching
|
||
|
||
this.assignedListener = async (agent, taskId) => {
|
||
if (!this.running) return;
|
||
|
||
try {
|
||
if (!isHeartbeatManaged(agent)) {
|
||
heartbeatLog.log(`Assignment trigger skipped for ${agent.id} (ephemeral/internal)`);
|
||
return;
|
||
}
|
||
|
||
const runtimeConfig = (agent.runtimeConfig ?? {}) as { enabled?: boolean; allowParallelExecution?: boolean };
|
||
if (runtimeConfig.enabled === false) {
|
||
heartbeatLog.log(`Assignment trigger skipped for ${agent.id} (disabled)`);
|
||
return;
|
||
}
|
||
|
||
// Guard: skip if agent already has an active run
|
||
const activeRun = await this.store.getActiveHeartbeatRun(agent.id);
|
||
if (activeRun) {
|
||
heartbeatLog.log(`Assignment trigger skipped for ${agent.id} (active run)`);
|
||
return;
|
||
}
|
||
|
||
// Guard: when parallel execution is disabled, skip if the bound task is actively executing
|
||
if (runtimeConfig.allowParallelExecution === false && this.isTaskExecuting?.(taskId)) {
|
||
heartbeatLog.log(`Assignment tick skipped for ${agent.id} (parallel execution disabled, task ${taskId} executing)`);
|
||
return;
|
||
}
|
||
|
||
let budgetStatus: AgentBudgetStatus | undefined;
|
||
// Budget governance: block even critical triggers when budget is fully exhausted
|
||
try {
|
||
budgetStatus = await this.store.getBudgetStatus(agent.id);
|
||
if (budgetStatus.isOverBudget) {
|
||
heartbeatLog.log(`Agent ${agent.id} budget exhausted — assignment trigger skipped`);
|
||
return;
|
||
}
|
||
} catch (budgetErr) {
|
||
heartbeatLog.warn(`Assignment trigger budget check failed for ${agent.id}: ${budgetErr instanceof Error ? budgetErr.message : String(budgetErr)} — proceeding without budget check`);
|
||
}
|
||
|
||
let triggeringCommentIds: string[] | undefined;
|
||
if (this.taskStore && typeof this.taskStore.getTask === "function") {
|
||
try {
|
||
const [task, recentRuns] = await Promise.all([
|
||
this.taskStore.getTask(taskId),
|
||
this.store.getRecentRuns(agent.id, 1),
|
||
]);
|
||
|
||
const lastRunAt = recentRuns[0]?.startedAt;
|
||
const newSteeringComments = (task.steeringComments ?? []).filter((comment) =>
|
||
!lastRunAt || comment.createdAt > lastRunAt,
|
||
);
|
||
if (newSteeringComments.length > 0) {
|
||
triggeringCommentIds = newSteeringComments.map((comment) => comment.id);
|
||
}
|
||
} catch (error) {
|
||
heartbeatLog.warn(
|
||
`Failed to resolve triggering steering comments for assignment wake (${agent.id}/${taskId}): ${error instanceof Error ? error.message : String(error)}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
heartbeatLog.log(`Assignment trigger for ${agent.id} (task: ${taskId})`);
|
||
await this.callback(agent.id, "assignment", {
|
||
taskId,
|
||
wakeReason: "assignment",
|
||
triggerDetail: "task-assigned",
|
||
...(triggeringCommentIds?.length
|
||
? {
|
||
triggeringCommentIds,
|
||
triggeringCommentType: "steering" as const,
|
||
}
|
||
: {}),
|
||
...(budgetStatus && { budgetStatus }),
|
||
});
|
||
} catch (err) {
|
||
heartbeatLog.error(`Assignment trigger error for ${agent.id}: ${err instanceof Error ? err.message : err}`);
|
||
}
|
||
};
|
||
|
||
this.store.on("agent:assigned", this.assignedListener);
|
||
heartbeatLog.log("Watching agent:assigned events");
|
||
}
|
||
|
||
/**
|
||
* Unsubscribe from agent:assigned events.
|
||
*/
|
||
unwatchAssignments(): void {
|
||
if (this.assignedListener) {
|
||
this.store.off("agent:assigned", this.assignedListener);
|
||
this.assignedListener = null;
|
||
heartbeatLog.log("Stopped watching agent:assigned events");
|
||
}
|
||
}
|
||
|
||
private isTimerEligibleAgent(agent: Agent): boolean {
|
||
return isHeartbeatManaged(agent)
|
||
&& agent.runtimeConfig?.enabled !== false
|
||
&& isTickableState(agent.state);
|
||
}
|
||
|
||
private getAgentTimerConfig(agent: Agent): AgentHeartbeatConfig {
|
||
const rc = (agent.runtimeConfig ?? {}) as {
|
||
enabled?: boolean;
|
||
heartbeatIntervalMs?: number;
|
||
maxConcurrentRuns?: number;
|
||
};
|
||
return {
|
||
enabled: rc.enabled,
|
||
heartbeatIntervalMs: rc.heartbeatIntervalMs,
|
||
maxConcurrentRuns: rc.maxConcurrentRuns,
|
||
};
|
||
}
|
||
|
||
private syncTimerForAgent(agent: Agent, reason: string): void {
|
||
if (!this.isTimerEligibleAgent(agent)) {
|
||
this.unregisterAgent(agent.id);
|
||
return;
|
||
}
|
||
|
||
if (this.timers.has(agent.id)) {
|
||
// Already ticking — non-config updates should not reset the interval.
|
||
return;
|
||
}
|
||
|
||
this.registerAgent(agent.id, this.getAgentTimerConfig(agent), {
|
||
lastHeartbeatAt: agent.lastHeartbeatAt,
|
||
});
|
||
heartbeatLog.log(`Timer armed for ${agent.id} (${reason})`);
|
||
}
|
||
|
||
private async syncTimerForAgentFromStore(agentId: string, reason: string): Promise<void> {
|
||
const agent = await this.store.getAgent(agentId);
|
||
if (!agent) {
|
||
this.unregisterAgent(agentId);
|
||
return;
|
||
}
|
||
|
||
if (!this.isTimerEligibleAgent(agent)) {
|
||
this.unregisterAgent(agentId);
|
||
return;
|
||
}
|
||
|
||
this.registerAgent(agent.id, this.getAgentTimerConfig(agent), {
|
||
lastHeartbeatAt: agent.lastHeartbeatAt,
|
||
});
|
||
heartbeatLog.log(`Timer refreshed for ${agent.id} (${reason})`);
|
||
}
|
||
|
||
private didHeartbeatScheduleChange(revision: AgentConfigRevision): boolean {
|
||
const before = (revision.before.runtimeConfig ?? {}) as Record<string, unknown>;
|
||
const after = (revision.after.runtimeConfig ?? {}) as Record<string, unknown>;
|
||
|
||
const pickScheduleFields = (runtimeConfig: Record<string, unknown>) => ({
|
||
enabled: runtimeConfig.enabled,
|
||
heartbeatIntervalMs: runtimeConfig.heartbeatIntervalMs,
|
||
maxConcurrentRuns: runtimeConfig.maxConcurrentRuns,
|
||
});
|
||
|
||
return JSON.stringify(pickScheduleFields(before)) !== JSON.stringify(pickScheduleFields(after));
|
||
}
|
||
|
||
private watchAgentLifecycle(): void {
|
||
if (this.createdListener || this.updatedListener || this.configRevisionListener || this.deletedListener) return;
|
||
|
||
this.createdListener = (agent) => {
|
||
this.syncTimerForAgent(agent, `created:${agent.state}`);
|
||
};
|
||
|
||
// State-driven registration: when an agent transitions into a tickable
|
||
// state arm the timer; transitioning out clears it. Existing timers are
|
||
// left alone here so unrelated agent updates do not reset the interval.
|
||
this.updatedListener = (agent) => {
|
||
this.syncTimerForAgent(agent, `state:${agent.state}`);
|
||
};
|
||
this.configRevisionListener = (agentId, revision) => {
|
||
if (!this.didHeartbeatScheduleChange(revision)) {
|
||
return;
|
||
}
|
||
|
||
void this.syncTimerForAgentFromStore(agentId, "runtime-config-updated");
|
||
};
|
||
this.deletedListener = (agentId) => {
|
||
this.unregisterAgent(agentId);
|
||
};
|
||
|
||
this.store.on("agent:created", this.createdListener);
|
||
this.store.on("agent:updated", this.updatedListener);
|
||
this.store.on("agent:configRevision", this.configRevisionListener);
|
||
this.store.on("agent:deleted", this.deletedListener);
|
||
}
|
||
|
||
private unwatchAgentLifecycle(): void {
|
||
if (this.createdListener) {
|
||
this.store.off("agent:created", this.createdListener);
|
||
this.createdListener = null;
|
||
}
|
||
if (this.updatedListener) {
|
||
this.store.off("agent:updated", this.updatedListener);
|
||
this.updatedListener = null;
|
||
}
|
||
if (this.configRevisionListener) {
|
||
this.store.off("agent:configRevision", this.configRevisionListener);
|
||
this.configRevisionListener = null;
|
||
}
|
||
if (this.deletedListener) {
|
||
this.store.off("agent:deleted", this.deletedListener);
|
||
this.deletedListener = null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Handle a timer tick for an agent.
|
||
* Checks for active runs before invoking the callback.
|
||
*/
|
||
private async onTimerTick(agentId: string, intervalMs: number): Promise<void> {
|
||
if (!this.running) return;
|
||
|
||
try {
|
||
const agent = await this.store.getAgent(agentId);
|
||
if (!agent) {
|
||
heartbeatLog.log(`Timer tick skipped for ${agentId} (agent missing)`);
|
||
this.unregisterAgent(agentId);
|
||
return;
|
||
}
|
||
if (!isHeartbeatManaged(agent) || !isTickableState(agent.state)) {
|
||
heartbeatLog.log(`Timer tick skipped for ${agentId} (state=${agent.state})`);
|
||
this.unregisterAgent(agentId);
|
||
return;
|
||
}
|
||
|
||
// Check for active runs
|
||
const activeRun = await this.store.getActiveHeartbeatRun(agentId);
|
||
if (activeRun) {
|
||
heartbeatLog.log(`Timer tick skipped for ${agentId} (active run)`);
|
||
return;
|
||
}
|
||
|
||
// Guard: when parallel execution is disabled, skip if the agent's bound task is actively executing
|
||
const timerRc = (agent.runtimeConfig ?? {}) as { allowParallelExecution?: boolean };
|
||
if (timerRc.allowParallelExecution === false && agent.taskId && this.isTaskExecuting?.(agent.taskId)) {
|
||
heartbeatLog.log(`Timer tick skipped for ${agentId} (parallel execution disabled, task ${agent.taskId} executing)`);
|
||
return;
|
||
}
|
||
|
||
// Global/engine pause guard: scheduler should not dispatch timer callbacks
|
||
// while globally paused (hard stop) or engine paused (soft stop for timers).
|
||
if (this.taskStore) {
|
||
const settings = await this.taskStore.getSettings();
|
||
if (settings.globalPause) {
|
||
heartbeatLog.log(`Timer tick skipped for ${agentId} (global pause active)`);
|
||
return;
|
||
}
|
||
if (settings.enginePaused) {
|
||
heartbeatLog.log(`Timer tick skipped for ${agentId} (engine paused)`);
|
||
return;
|
||
}
|
||
}
|
||
|
||
// Budget enforcement is handled in HeartbeatMonitor.executeHeartbeat() for timer sources.
|
||
// The scheduler dispatches the callback regardless of budget status so that executeHeartbeat()
|
||
// can create explicit run records with budget_exhausted/budget_threshold_exceeded reasons.
|
||
// This makes timer budget skips observable rather than silent drops.
|
||
|
||
await this.callback(agentId, "timer", {
|
||
wakeReason: "timer",
|
||
triggerDetail: "scheduled",
|
||
intervalMs,
|
||
});
|
||
} catch (err) {
|
||
heartbeatLog.error(`Timer tick error for ${agentId}: ${err instanceof Error ? err.message : err}`);
|
||
}
|
||
}
|
||
}
|