feat(FN-2658): enforce paused-agent guards in heartbeat flows
- Skip timer-triggered heartbeat ticks when the target agent is paused - Add executeHeartbeat pause checks so paused agents do not start task work - Expand agent-heartbeat tests to cover pause guards across scheduler and execution paths - Document paused-agent heartbeat behavior in docs and add a patch changeset for @runfusion/fusion
This commit is contained in:
5
.changeset/pause-guard-heartbeat.md
Normal file
5
.changeset/pause-guard-heartbeat.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
Respect globalPause/enginePaused in heartbeat trigger scheduler and monitor to prevent agents from running when the engine is paused at startup.
|
||||
@@ -415,6 +415,10 @@ Fusion's `HeartbeatTriggerScheduler` supports five trigger types:
|
||||
|
||||
All triggers respect per-agent `maxConcurrentRuns` and produce structured wake context metadata.
|
||||
|
||||
Pause governance for heartbeat execution:
|
||||
- `globalPause` is a hard stop: timer, assignment, and on-demand heartbeats are skipped with observable run reasons.
|
||||
- `enginePaused` is a soft stop for heartbeat timers: timer triggers are skipped, while assignment/on-demand triggers remain allowed for critical responsiveness paths.
|
||||
|
||||
### Control-Plane Lane (No Task Concurrency Gating)
|
||||
|
||||
Heartbeat runs from the Agents panel run on a **separate control-plane lane** that is independent of task execution concurrency limits. This ensures agent responsiveness is preserved even when task pipelines are saturated.
|
||||
@@ -576,7 +580,7 @@ Budget enforcement is centralized in `HeartbeatMonitor.executeHeartbeat()`:
|
||||
- **Timer triggers**: Budget is enforced in `executeHeartbeat()` which creates explicit run records with `budget_exhausted` or `budget_threshold_exceeded` reasons. This makes timer budget skips observable rather than silent drops — users see explicit "skipped" run records in the dashboard instead of timer ticks that appear to "not run".
|
||||
- **Assignment and on-demand triggers**: Budget is enforced in `executeHeartbeat()` with the same outcome recording. These triggers are allowed when over threshold (but not over budget) to maintain responsiveness.
|
||||
|
||||
The `HeartbeatTriggerScheduler` always dispatches timer callbacks regardless of budget status, delegating budget enforcement to the execution layer. This ensures every timer tick produces a heartbeat run record that is visible in the agent's run history.
|
||||
When the engine is not paused, the `HeartbeatTriggerScheduler` dispatches timer callbacks regardless of budget status, delegating budget enforcement to the execution layer. This ensures every eligible timer tick produces a heartbeat run record that is visible in the agent's run history.
|
||||
|
||||
Agents can be paused by budget exhaustion. Timer-triggered heartbeats skip when over threshold to avoid runaway costs, but assignment-triggered and on-demand runs may still execute for responsiveness.
|
||||
|
||||
|
||||
@@ -3462,6 +3462,87 @@ describe("HeartbeatMonitor", () => {
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Pause Governance", () => {
|
||||
it("skips heartbeat on global pause for timer source", async () => {
|
||||
const store = createStoreWithAgentForExec();
|
||||
const pauseAwareTaskStore = createMockTaskStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: true, enginePaused: false }),
|
||||
});
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
|
||||
|
||||
const result = await monitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
|
||||
|
||||
expect(result.status).toBe("completed");
|
||||
expect(result.resultJson).toMatchObject({ reason: "global_pause", source: "timer" });
|
||||
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips heartbeat on global pause for assignment source", async () => {
|
||||
const store = createStoreWithAgentForExec();
|
||||
const pauseAwareTaskStore = createMockTaskStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: true, enginePaused: false }),
|
||||
});
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
|
||||
|
||||
const result = await monitor.executeHeartbeat({ agentId: "agent-001", source: "assignment" });
|
||||
|
||||
expect(result.status).toBe("completed");
|
||||
expect(result.resultJson).toMatchObject({ reason: "global_pause", source: "assignment" });
|
||||
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips timer heartbeat on engine pause but allows assignment", async () => {
|
||||
const timerStore = createStoreWithAgentForExec();
|
||||
const pauseAwareTaskStore = createMockTaskStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: true }),
|
||||
});
|
||||
const timerMonitor = new HeartbeatMonitor({ store: timerStore, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
|
||||
|
||||
const timerResult = await timerMonitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
|
||||
|
||||
expect(timerResult.status).toBe("completed");
|
||||
expect(timerResult.resultJson).toMatchObject({ reason: "engine_paused", source: "timer" });
|
||||
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
|
||||
|
||||
const assignmentStore = createStoreWithAgentForExec();
|
||||
const mockSession = createMockAgentSession();
|
||||
mockedCreateFnAgent.mockResolvedValue({ session: mockSession as any });
|
||||
const assignmentMonitor = new HeartbeatMonitor({
|
||||
store: assignmentStore,
|
||||
taskStore: pauseAwareTaskStore,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
const assignmentResult = await assignmentMonitor.executeHeartbeat({
|
||||
agentId: "agent-001",
|
||||
source: "assignment",
|
||||
});
|
||||
|
||||
expect(assignmentResult.status).toBe("completed");
|
||||
expect((assignmentResult.resultJson as Record<string, unknown>)?.reason).not.toBe("engine_paused");
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("proceeds when pause flags are false", async () => {
|
||||
const store = createStoreWithAgentForExec();
|
||||
const mockSession = createMockAgentSession();
|
||||
mockedCreateFnAgent.mockResolvedValue({ session: mockSession as any });
|
||||
const pauseAwareTaskStore = createMockTaskStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false }),
|
||||
});
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
|
||||
|
||||
const timerResult = await monitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
|
||||
const onDemandResult = await monitor.executeHeartbeat({ agentId: "agent-001", source: "on_demand" });
|
||||
const assignmentResult = await monitor.executeHeartbeat({ agentId: "agent-001", source: "assignment" });
|
||||
|
||||
expect(timerResult.status).toBe("completed");
|
||||
expect(onDemandResult.status).toBe("completed");
|
||||
expect(assignmentResult.status).toBe("completed");
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── Task Creation Tracking Tests ──────────────────────────────────────
|
||||
@@ -4372,6 +4453,54 @@ describe("HeartbeatTriggerScheduler", () => {
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips timer dispatch when global pause is active", async () => {
|
||||
scheduler.stop();
|
||||
const taskStore = {
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: true, enginePaused: false }),
|
||||
} as unknown as TaskStore;
|
||||
scheduler = new HeartbeatTriggerScheduler(store, callback, taskStore);
|
||||
scheduler.start();
|
||||
|
||||
scheduler.registerAgent("agent-001", { heartbeatIntervalMs: 5000 });
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
expect(heartbeatLog.log).toHaveBeenCalledWith("Timer tick skipped for agent-001 (global pause active)");
|
||||
});
|
||||
|
||||
it("skips timer dispatch when engine pause is active", async () => {
|
||||
scheduler.stop();
|
||||
const taskStore = {
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: true }),
|
||||
} as unknown as TaskStore;
|
||||
scheduler = new HeartbeatTriggerScheduler(store, callback, taskStore);
|
||||
scheduler.start();
|
||||
|
||||
scheduler.registerAgent("agent-001", { heartbeatIntervalMs: 5000 });
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dispatches timer callback when pause flags are false", async () => {
|
||||
scheduler.stop();
|
||||
const taskStore = {
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false }),
|
||||
} as unknown as TaskStore;
|
||||
scheduler = new HeartbeatTriggerScheduler(store, callback, taskStore);
|
||||
scheduler.start();
|
||||
|
||||
scheduler.registerAgent("agent-001", { heartbeatIntervalMs: 5000 });
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
expect(callback).toHaveBeenCalledOnce();
|
||||
expect(callback).toHaveBeenCalledWith("agent-001", "timer", {
|
||||
wakeReason: "timer",
|
||||
triggerDetail: "scheduled",
|
||||
intervalMs: 5000,
|
||||
});
|
||||
});
|
||||
|
||||
it("respects maxConcurrentRuns from config", async () => {
|
||||
// Agent with active run should be skipped
|
||||
(store.getActiveHeartbeatRun as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
|
||||
@@ -857,6 +857,32 @@ export class HeartbeatMonitor {
|
||||
heartbeatLog.warn(`Agent ${agentId} budget status check failed: ${budgetErr instanceof Error ? budgetErr.message : String(budgetErr)} — proceeding without budget check`);
|
||||
}
|
||||
|
||||
// Pause governance: globalPause blocks all heartbeat sources;
|
||||
// enginePaused is a soft pause that only blocks timer ticks.
|
||||
try {
|
||||
const settings = await taskStore.getSettings();
|
||||
if (settings.globalPause) {
|
||||
heartbeatLog.log(`Agent ${agentId} heartbeat skipped — global pause active (source=${source})`);
|
||||
await this.completeRun(agentId, run.id, {
|
||||
status: "completed",
|
||||
resultJson: { reason: "global_pause", source },
|
||||
skipStateTransition: true,
|
||||
});
|
||||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||
}
|
||||
if (settings.enginePaused && source === "timer") {
|
||||
heartbeatLog.log(`Agent ${agentId} timer heartbeat skipped — engine paused (soft pause)`);
|
||||
await this.completeRun(agentId, run.id, {
|
||||
status: "completed",
|
||||
resultJson: { reason: "engine_paused", source },
|
||||
skipStateTransition: true,
|
||||
});
|
||||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||
}
|
||||
} catch (pauseErr) {
|
||||
heartbeatLog.warn(`Pause status check failed for ${agentId}: ${pauseErr instanceof Error ? pauseErr.message : String(pauseErr)} — proceeding`);
|
||||
}
|
||||
|
||||
// Resolve agent
|
||||
const agent = preloadedAgent ?? await this.store.getAgent(agentId);
|
||||
if (!agent) {
|
||||
@@ -2065,6 +2091,20 @@ export class HeartbeatTriggerScheduler {
|
||||
return;
|
||||
}
|
||||
|
||||
// Global/engine pause guard: scheduler should not dispatch timer callbacks
|
||||
// while globally paused (hard stop) or engine paused (soft stop for timers).
|
||||
if (this.taskStore) {
|
||||
const settings = await this.taskStore.getSettings();
|
||||
if (settings.globalPause) {
|
||||
heartbeatLog.log(`Timer tick skipped for ${agentId} (global pause active)`);
|
||||
return;
|
||||
}
|
||||
if (settings.enginePaused) {
|
||||
heartbeatLog.log(`Timer tick skipped for ${agentId} (engine paused)`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Budget enforcement is handled in HeartbeatMonitor.executeHeartbeat() for timer sources.
|
||||
// The scheduler dispatches the callback regardless of budget status so that executeHeartbeat()
|
||||
// can create explicit run records with budget_exhausted/budget_threshold_exceeded reasons.
|
||||
|
||||
Reference in New Issue
Block a user