root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(engine): unstick agents in running and clean up ephemeral worker pile-up

Browse Source 
Two related leaks in the agent lifecycle plus a refactor:

- Governance-skip paths in executeHeartbeat (budget/global-pause/engine-paused)
  were leaving agents permanently stuck in `running` because they ran startRun
  first and then short-circuited with skipStateTransition: true. Removed the
  flag from those four paths so they flow through running → active. Added
  HeartbeatMonitor.reconcileOrphanedRunningAgents() on start to recover any
  rows already trapped in this state.
- Ephemeral task-workers piled up across runtime restarts because taskAgentMap
  was in-memory only and the startup sweep ignored ephemerals with no taskId.
  Now: spawn dedup via findAgentByName before create, on-disk fallback in
  finalize when the in-memory map is empty, and the sweep deletes any
  ephemeral not bound to an in-progress task.
- Extracted the lifecycle into EphemeralWorkerManager
  (packages/engine/src/ephemeral-worker-manager.ts). InProcessRuntime drops
  ~140 lines and delegates via onTaskStart/onTaskComplete/onTaskError/
  attachStateChangeListener/reconcileOrphaned. ChildProcessRuntime and
  RemoteNodeRuntime inherit the fix because they delegate execution to a
  worker that runs InProcessRuntime.

Durable assigned agents now return to `active` after task completion (was
`terminated` in the old contract).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-05-05 19:11:49 -07:00
parent 9d8589a149
commit d47501feeb
5 changed files with 450 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts
View File

@@ -739,7 +739,7 @@ describe("InProcessRuntime", () => {
await vi.advanceTimersByTimeAsync(5000);
const updated = await store.getAgent(durable.id);
expect(updated?.state).toBe("terminated");
expect(updated?.state).toBe("active");
expect(updated?.taskId).toBeUndefined();
expect(deleteAgentSpy).not.toHaveBeenCalledWith(durable.id);
} finally {
@@ -807,7 +807,7 @@ describe("InProcessRuntime", () => {
await vi.advanceTimersByTimeAsync(5000);
const updated = await store.getAgent(durable.id);
expect(updated?.state).toBe("terminated");
expect(updated?.state).toBe("error");
expect(updated?.taskId).toBeUndefined();
expect(deleteAgentSpy).not.toHaveBeenCalledWith(durable.id);
} finally {
@@ -860,7 +860,7 @@ describe("InProcessRuntime", () => {
const store = getAgentStore(runtime);
const updateStateSpy = vi.spyOn(store, "updateAgentState").mockImplementation(async (_agentId, state) => {
if (state === "terminated") {
if (state === "active") {
throw new Error("state update failed");
}
return {} as Agent;
@@ -885,7 +885,7 @@ describe("InProcessRuntime", () => {
expect.stringContaining("Failed to update agent"),
);
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("terminated (completion)"),
expect.stringContaining("active (completion)"),
);
warnSpy.mockRestore();
@@ -1260,7 +1260,7 @@ describe("InProcessRuntime", () => {
expect(agents.some((a: Agent) => a.id === agent.id)).toBe(true);
// Emit agent:stateChanged event to trigger termination
store.emit("agent:stateChanged", agent.id, "running", "terminated");
store.emit("agent:stateChanged", agent.id, "running", "paused");
// Wait for async handler
await vi.advanceTimersByTimeAsync(0);
@@ -1299,7 +1299,7 @@ describe("InProcessRuntime", () => {
expect(agents.some((a: Agent) => a.id === agent.id)).toBe(true);
// Emit agent:stateChanged event to trigger termination
store.emit("agent:stateChanged", agent.id, "active", "terminated");
store.emit("agent:stateChanged", agent.id, "active", "paused");
// Wait for async handler
await vi.advanceTimersByTimeAsync(0);
@@ -1339,8 +1339,8 @@ describe("InProcessRuntime", () => {
});
// Emit termination event multiple times
store.emit("agent:stateChanged", agent.id, "running", "terminated");
store.emit("agent:stateChanged", agent.id, "terminated", "terminated"); // Already terminated
store.emit("agent:stateChanged", agent.id, "running", "paused");
store.emit("agent:stateChanged", agent.id, "paused", "paused"); // Already halted
// Wait for async handlers
await vi.advanceTimersByTimeAsync(0);
@@ -1378,7 +1378,7 @@ describe("InProcessRuntime", () => {
.mockRejectedValueOnce(new Error(`Agent ${agent.id} not found`));
// Emit termination event
store.emit("agent:stateChanged", agent.id, "running", "terminated");
store.emit("agent:stateChanged", agent.id, "running", "paused");
// Wait for async handler
await vi.advanceTimersByTimeAsync(0);
@@ -1419,7 +1419,7 @@ describe("InProcessRuntime", () => {
});
// Emit termination event
store.emit("agent:stateChanged", agent.id, "running", "terminated");
store.emit("agent:stateChanged", agent.id, "running", "paused");
// Wait for async handler
await vi.advanceTimersByTimeAsync(0);
@@ -1463,7 +1463,7 @@ describe("InProcessRuntime", () => {
});
// Emit termination event
store.emit("agent:stateChanged", agent.id, "running", "terminated");
store.emit("agent:stateChanged", agent.id, "running", "paused");
// Wait for async handler
await vi.advanceTimersByTimeAsync(0);
@@ -1497,7 +1497,7 @@ describe("InProcessRuntime", () => {
});
// Emit termination event
store.emit("agent:stateChanged", agent.id, "running", "terminated");
store.emit("agent:stateChanged", agent.id, "running", "paused");
// Wait for async handler
await vi.advanceTimersByTimeAsync(0);
@@ -1535,7 +1535,7 @@ describe("InProcessRuntime", () => {
});
executorOptions.onComplete?.({ id: "FN-DUP-COMPLETE" } as Task);
store.emit("agent:stateChanged", worker!.id, "running", "terminated");
store.emit("agent:stateChanged", worker!.id, "running", "paused");
await vi.waitFor(() => {
expect(deleteAgentSpy).toHaveBeenCalledTimes(1);
@@ -1579,7 +1579,7 @@ describe("InProcessRuntime", () => {
runtimeConfig: { enabled: false },
});
await preStore.updateAgentState(orphan.id, "active");
await preStore.updateAgentState(orphan.id, "terminated");
await preStore.updateAgentState(orphan.id, "paused");
await runtime.start();
const store = getAgentStore(runtime);
@@ -1621,9 +1621,9 @@ describe("InProcessRuntime", () => {
const a1 = await preStore.createAgent({ name: "orphan-a1", role: "executor", metadata: { agentKind: "task-worker" }, runtimeConfig: { enabled: false } });
const a2 = await preStore.createAgent({ name: "orphan-a2", role: "executor", metadata: { agentKind: "task-worker" }, runtimeConfig: { enabled: false } });
await preStore.updateAgentState(a1.id, "active");
await preStore.updateAgentState(a1.id, "terminated");
await preStore.updateAgentState(a1.id, "paused");
await preStore.updateAgentState(a2.id, "active");
await preStore.updateAgentState(a2.id, "terminated");
await preStore.updateAgentState(a2.id, "paused");
await runtime.start();
const store = getAgentStore(runtime);