feat(FN-3958): add heartbeat timer reconciliation self-healing
Adds scheduler heartbeat timer reconciliation with automatic self-healing when timers drift, including tests for tracked-only monitor recovery and documentation in the agents reference. Fusion-Task-Id: FN-3958
This commit is contained in:
@@ -2856,6 +2856,9 @@ export class HeartbeatTriggerScheduler {
|
||||
private configRevisionListener: ((agentId: string, revision: AgentConfigRevision) => void) | null = null;
|
||||
private deletedListener: ((agentId: string) => void) | null = null;
|
||||
private isTaskExecuting?: (taskId: string) => boolean;
|
||||
private timerAuditIntervalHandle: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
private static readonly TIMER_AUDIT_INTERVAL_MS = 60_000;
|
||||
|
||||
constructor(store: AgentStore, callback: TriggerCallback, taskStore?: TaskStore, options?: { isTaskExecuting?: (taskId: string) => boolean }) {
|
||||
this.store = store;
|
||||
@@ -2873,6 +2876,10 @@ export class HeartbeatTriggerScheduler {
|
||||
this.running = true;
|
||||
this.watchAssignments();
|
||||
this.watchAgentLifecycle();
|
||||
void this.auditTimerRegistrations("start");
|
||||
this.timerAuditIntervalHandle = setInterval(() => {
|
||||
void this.auditTimerRegistrations("interval");
|
||||
}, HeartbeatTriggerScheduler.TIMER_AUDIT_INTERVAL_MS);
|
||||
heartbeatLog.log("HeartbeatTriggerScheduler started");
|
||||
}
|
||||
|
||||
@@ -2898,6 +2905,11 @@ export class HeartbeatTriggerScheduler {
|
||||
}
|
||||
this.timers.clear();
|
||||
|
||||
if (this.timerAuditIntervalHandle) {
|
||||
clearInterval(this.timerAuditIntervalHandle);
|
||||
this.timerAuditIntervalHandle = null;
|
||||
}
|
||||
|
||||
heartbeatLog.log("HeartbeatTriggerScheduler stopped");
|
||||
}
|
||||
|
||||
@@ -3340,6 +3352,37 @@ export class HeartbeatTriggerScheduler {
|
||||
}
|
||||
}
|
||||
|
||||
async auditTimerRegistrations(reason: "start" | "interval" = "interval"): Promise<void> {
|
||||
if (!this.running) return;
|
||||
|
||||
try {
|
||||
const agents = await this.store.listAgents();
|
||||
let rearmedCount = 0;
|
||||
for (const agent of agents) {
|
||||
if (!this.isTimerEligibleAgent(agent)) continue;
|
||||
if (this.timers.has(agent.id)) continue;
|
||||
|
||||
const activeRun = await this.store.getActiveHeartbeatRun(agent.id);
|
||||
if (activeRun) {
|
||||
heartbeatLog.log(`Timer audit skipped re-arm for ${agent.id} (active run)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
this.registerAgent(agent.id, this.getAgentTimerConfig(agent), {
|
||||
lastHeartbeatAt: agent.lastHeartbeatAt,
|
||||
});
|
||||
rearmedCount++;
|
||||
heartbeatLog.log(`Timer re-armed for ${agent.id} (audit:${reason})`);
|
||||
}
|
||||
|
||||
if (rearmedCount > 0) {
|
||||
heartbeatLog.log(`Timer audit repaired ${rearmedCount} missing registration(s) (${reason})`);
|
||||
}
|
||||
} catch (error) {
|
||||
heartbeatLog.warn(`Timer audit failed (${reason}): ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a timer tick for an agent.
|
||||
* Checks for active runs before invoking the callback.
|
||||
|
||||
Reference in New Issue
Block a user