feat(FN-3958): add heartbeat timer reconciliation self-healing

Adds scheduler heartbeat timer reconciliation with automatic self-healing when timers drift, including tests for tracked-only monitor recovery and documentation in the agents reference.

Fusion-Task-Id: FN-3958
This commit is contained in:
Fusion
2026-05-10 23:24:27 -07:00
committed by gsxdsm
parent baed1db0c7
commit be404e5967
7 changed files with 176 additions and 5 deletions

View File

@@ -2856,6 +2856,9 @@ export class HeartbeatTriggerScheduler {
private configRevisionListener: ((agentId: string, revision: AgentConfigRevision) => void) | null = null;
private deletedListener: ((agentId: string) => void) | null = null;
private isTaskExecuting?: (taskId: string) => boolean;
private timerAuditIntervalHandle: ReturnType<typeof setInterval> | null = null;
private static readonly TIMER_AUDIT_INTERVAL_MS = 60_000;
constructor(store: AgentStore, callback: TriggerCallback, taskStore?: TaskStore, options?: { isTaskExecuting?: (taskId: string) => boolean }) {
this.store = store;
@@ -2873,6 +2876,10 @@ export class HeartbeatTriggerScheduler {
this.running = true;
this.watchAssignments();
this.watchAgentLifecycle();
void this.auditTimerRegistrations("start");
this.timerAuditIntervalHandle = setInterval(() => {
void this.auditTimerRegistrations("interval");
}, HeartbeatTriggerScheduler.TIMER_AUDIT_INTERVAL_MS);
heartbeatLog.log("HeartbeatTriggerScheduler started");
}
@@ -2898,6 +2905,11 @@ export class HeartbeatTriggerScheduler {
}
this.timers.clear();
if (this.timerAuditIntervalHandle) {
clearInterval(this.timerAuditIntervalHandle);
this.timerAuditIntervalHandle = null;
}
heartbeatLog.log("HeartbeatTriggerScheduler stopped");
}
@@ -3340,6 +3352,37 @@ export class HeartbeatTriggerScheduler {
}
}
async auditTimerRegistrations(reason: "start" | "interval" = "interval"): Promise<void> {
if (!this.running) return;
try {
const agents = await this.store.listAgents();
let rearmedCount = 0;
for (const agent of agents) {
if (!this.isTimerEligibleAgent(agent)) continue;
if (this.timers.has(agent.id)) continue;
const activeRun = await this.store.getActiveHeartbeatRun(agent.id);
if (activeRun) {
heartbeatLog.log(`Timer audit skipped re-arm for ${agent.id} (active run)`);
continue;
}
this.registerAgent(agent.id, this.getAgentTimerConfig(agent), {
lastHeartbeatAt: agent.lastHeartbeatAt,
});
rearmedCount++;
heartbeatLog.log(`Timer re-armed for ${agent.id} (audit:${reason})`);
}
if (rearmedCount > 0) {
heartbeatLog.log(`Timer audit repaired ${rearmedCount} missing registration(s) (${reason})`);
}
} catch (error) {
heartbeatLog.warn(`Timer audit failed (${reason}): ${error instanceof Error ? error.message : String(error)}`);
}
}
/**
* Handle a timer tick for an agent.
* Checks for active runs before invoking the callback.