feat(FN-2658): enforce paused-agent guards in heartbeat flows

- Skip timer-triggered heartbeat ticks when the target agent is paused
- Add executeHeartbeat pause checks so paused agents do not start task work
- Expand agent-heartbeat tests to cover pause guards across scheduler and execution paths
- Document paused-agent heartbeat behavior in docs and add a patch changeset for @runfusion/fusion
This commit is contained in:
Fusion
2026-04-27 00:53:23 -07:00
committed by gsxdsm
parent 0bd30f7b58
commit 7691babb23
4 changed files with 179 additions and 1 deletions

View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": patch
---
Respect globalPause/enginePaused in heartbeat trigger scheduler and monitor to prevent agents from running when the engine is paused at startup.

View File

@@ -415,6 +415,10 @@ Fusion's `HeartbeatTriggerScheduler` supports five trigger types:
All triggers respect per-agent `maxConcurrentRuns` and produce structured wake context metadata.
Pause governance for heartbeat execution:
- `globalPause` is a hard stop: timer, assignment, and on-demand heartbeats are skipped with observable run reasons.
- `enginePaused` is a soft stop for heartbeat timers: timer triggers are skipped, while assignment/on-demand triggers remain allowed for critical responsiveness paths.
### Control-Plane Lane (No Task Concurrency Gating)
Heartbeat runs from the Agents panel run on a **separate control-plane lane** that is independent of task execution concurrency limits. This ensures agent responsiveness is preserved even when task pipelines are saturated.
@@ -576,7 +580,7 @@ Budget enforcement is centralized in `HeartbeatMonitor.executeHeartbeat()`:
- **Timer triggers**: Budget is enforced in `executeHeartbeat()` which creates explicit run records with `budget_exhausted` or `budget_threshold_exceeded` reasons. This makes timer budget skips observable rather than silent drops — users see explicit "skipped" run records in the dashboard instead of timer ticks that appear to "not run".
- **Assignment and on-demand triggers**: Budget is enforced in `executeHeartbeat()` with the same outcome recording. These triggers are allowed when over threshold (but not over budget) to maintain responsiveness.
The `HeartbeatTriggerScheduler` always dispatches timer callbacks regardless of budget status, delegating budget enforcement to the execution layer. This ensures every timer tick produces a heartbeat run record that is visible in the agent's run history.
When the engine is not paused, the `HeartbeatTriggerScheduler` dispatches timer callbacks regardless of budget status, delegating budget enforcement to the execution layer. This ensures every eligible timer tick produces a heartbeat run record that is visible in the agent's run history.
Agents can be paused by budget exhaustion. Timer-triggered heartbeats skip when over threshold to avoid runaway costs, but assignment-triggered and on-demand runs may still execute for responsiveness.

View File

@@ -3462,6 +3462,87 @@ describe("HeartbeatMonitor", () => {
expect(mockedCreateFnAgent).toHaveBeenCalledOnce();
});
});
describe("Pause Governance", () => {
it("skips heartbeat on global pause for timer source", async () => {
const store = createStoreWithAgentForExec();
const pauseAwareTaskStore = createMockTaskStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: true, enginePaused: false }),
});
const monitor = new HeartbeatMonitor({ store, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
const result = await monitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
expect(result.status).toBe("completed");
expect(result.resultJson).toMatchObject({ reason: "global_pause", source: "timer" });
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
});
it("skips heartbeat on global pause for assignment source", async () => {
const store = createStoreWithAgentForExec();
const pauseAwareTaskStore = createMockTaskStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: true, enginePaused: false }),
});
const monitor = new HeartbeatMonitor({ store, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
const result = await monitor.executeHeartbeat({ agentId: "agent-001", source: "assignment" });
expect(result.status).toBe("completed");
expect(result.resultJson).toMatchObject({ reason: "global_pause", source: "assignment" });
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
});
it("skips timer heartbeat on engine pause but allows assignment", async () => {
const timerStore = createStoreWithAgentForExec();
const pauseAwareTaskStore = createMockTaskStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: true }),
});
const timerMonitor = new HeartbeatMonitor({ store: timerStore, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
const timerResult = await timerMonitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
expect(timerResult.status).toBe("completed");
expect(timerResult.resultJson).toMatchObject({ reason: "engine_paused", source: "timer" });
expect(mockedCreateFnAgent).not.toHaveBeenCalled();
const assignmentStore = createStoreWithAgentForExec();
const mockSession = createMockAgentSession();
mockedCreateFnAgent.mockResolvedValue({ session: mockSession as any });
const assignmentMonitor = new HeartbeatMonitor({
store: assignmentStore,
taskStore: pauseAwareTaskStore,
rootDir: "/tmp",
});
const assignmentResult = await assignmentMonitor.executeHeartbeat({
agentId: "agent-001",
source: "assignment",
});
expect(assignmentResult.status).toBe("completed");
expect((assignmentResult.resultJson as Record<string, unknown>)?.reason).not.toBe("engine_paused");
expect(mockedCreateFnAgent).toHaveBeenCalledOnce();
});
it("proceeds when pause flags are false", async () => {
const store = createStoreWithAgentForExec();
const mockSession = createMockAgentSession();
mockedCreateFnAgent.mockResolvedValue({ session: mockSession as any });
const pauseAwareTaskStore = createMockTaskStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false }),
});
const monitor = new HeartbeatMonitor({ store, taskStore: pauseAwareTaskStore, rootDir: "/tmp" });
const timerResult = await monitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
const onDemandResult = await monitor.executeHeartbeat({ agentId: "agent-001", source: "on_demand" });
const assignmentResult = await monitor.executeHeartbeat({ agentId: "agent-001", source: "assignment" });
expect(timerResult.status).toBe("completed");
expect(onDemandResult.status).toBe("completed");
expect(assignmentResult.status).toBe("completed");
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(3);
});
});
});
// ── Task Creation Tracking Tests ──────────────────────────────────────
@@ -4372,6 +4453,54 @@ describe("HeartbeatTriggerScheduler", () => {
expect(callback).not.toHaveBeenCalled();
});
it("skips timer dispatch when global pause is active", async () => {
scheduler.stop();
const taskStore = {
getSettings: vi.fn().mockResolvedValue({ globalPause: true, enginePaused: false }),
} as unknown as TaskStore;
scheduler = new HeartbeatTriggerScheduler(store, callback, taskStore);
scheduler.start();
scheduler.registerAgent("agent-001", { heartbeatIntervalMs: 5000 });
await vi.advanceTimersByTimeAsync(5000);
expect(callback).not.toHaveBeenCalled();
expect(heartbeatLog.log).toHaveBeenCalledWith("Timer tick skipped for agent-001 (global pause active)");
});
it("skips timer dispatch when engine pause is active", async () => {
scheduler.stop();
const taskStore = {
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: true }),
} as unknown as TaskStore;
scheduler = new HeartbeatTriggerScheduler(store, callback, taskStore);
scheduler.start();
scheduler.registerAgent("agent-001", { heartbeatIntervalMs: 5000 });
await vi.advanceTimersByTimeAsync(5000);
expect(callback).not.toHaveBeenCalled();
});
it("dispatches timer callback when pause flags are false", async () => {
scheduler.stop();
const taskStore = {
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false }),
} as unknown as TaskStore;
scheduler = new HeartbeatTriggerScheduler(store, callback, taskStore);
scheduler.start();
scheduler.registerAgent("agent-001", { heartbeatIntervalMs: 5000 });
await vi.advanceTimersByTimeAsync(5000);
expect(callback).toHaveBeenCalledOnce();
expect(callback).toHaveBeenCalledWith("agent-001", "timer", {
wakeReason: "timer",
triggerDetail: "scheduled",
intervalMs: 5000,
});
});
it("respects maxConcurrentRuns from config", async () => {
// Agent with active run should be skipped
(store.getActiveHeartbeatRun as ReturnType<typeof vi.fn>).mockResolvedValue({

View File

@@ -857,6 +857,32 @@ export class HeartbeatMonitor {
heartbeatLog.warn(`Agent ${agentId} budget status check failed: ${budgetErr instanceof Error ? budgetErr.message : String(budgetErr)} — proceeding without budget check`);
}
// Pause governance: globalPause blocks all heartbeat sources;
// enginePaused is a soft pause that only blocks timer ticks.
try {
const settings = await taskStore.getSettings();
if (settings.globalPause) {
heartbeatLog.log(`Agent ${agentId} heartbeat skipped — global pause active (source=${source})`);
await this.completeRun(agentId, run.id, {
status: "completed",
resultJson: { reason: "global_pause", source },
skipStateTransition: true,
});
return (await this.store.getRunDetail(agentId, run.id))!;
}
if (settings.enginePaused && source === "timer") {
heartbeatLog.log(`Agent ${agentId} timer heartbeat skipped — engine paused (soft pause)`);
await this.completeRun(agentId, run.id, {
status: "completed",
resultJson: { reason: "engine_paused", source },
skipStateTransition: true,
});
return (await this.store.getRunDetail(agentId, run.id))!;
}
} catch (pauseErr) {
heartbeatLog.warn(`Pause status check failed for ${agentId}: ${pauseErr instanceof Error ? pauseErr.message : String(pauseErr)} — proceeding`);
}
// Resolve agent
const agent = preloadedAgent ?? await this.store.getAgent(agentId);
if (!agent) {
@@ -2065,6 +2091,20 @@ export class HeartbeatTriggerScheduler {
return;
}
// Global/engine pause guard: scheduler should not dispatch timer callbacks
// while globally paused (hard stop) or engine paused (soft stop for timers).
if (this.taskStore) {
const settings = await this.taskStore.getSettings();
if (settings.globalPause) {
heartbeatLog.log(`Timer tick skipped for ${agentId} (global pause active)`);
return;
}
if (settings.enginePaused) {
heartbeatLog.log(`Timer tick skipped for ${agentId} (engine paused)`);
return;
}
}
// Budget enforcement is handled in HeartbeatMonitor.executeHeartbeat() for timer sources.
// The scheduler dispatches the callback regardless of budget status so that executeHeartbeat()
// can create explicit run records with budget_exhausted/budget_threshold_exceeded reasons.