diff --git a/.changeset/fix-phantom-executor-kills-live-ephemeral-tasks.md b/.changeset/fix-phantom-executor-kills-live-ephemeral-tasks.md new file mode 100644 index 0000000000..6c3bde0dc3 --- /dev/null +++ b/.changeset/fix-phantom-executor-kills-live-ephemeral-tasks.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Stop self-healing from killing actively-running tasks after ~30 minutes. +category: fix +dev: FN-7566. isPhantomExecutorBinding's liveness gate (heartbeat/checkout/runAudit) was blind to ephemeral executor agents, leaving only the age>graceMs*3 threshold, so any ephemeral-executor task running longer than ~30 min was reclaimed to `todo` mid-flight. Adds the in-process live-session veto (activeSessionRegistry path / executingTaskLock / isTaskActive), mirroring the isWorkspaceTaskLive/sessionDead predicate, and honors clearPhantomExecutorBinding's live-session refusal in reclaimSelfOwnedBranchConflicts. diff --git a/packages/engine/src/__tests__/reliability-interactions/reclaim-phantom-executor-binding.test.ts b/packages/engine/src/__tests__/reliability-interactions/reclaim-phantom-executor-binding.test.ts index 0f9dac0136..0bf14b9c36 100644 --- a/packages/engine/src/__tests__/reliability-interactions/reclaim-phantom-executor-binding.test.ts +++ b/packages/engine/src/__tests__/reliability-interactions/reclaim-phantom-executor-binding.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import type { Settings, Task, TaskStore } from "@fusion/core"; import { SelfHealingManager, STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS } from "../../self-healing.js"; -import { activeSessionRegistry } from "../../active-session-registry.js"; +import { activeSessionRegistry, executingTaskLock } from "../../active-session-registry.js"; import * as branchConflictModule from "../../branch-conflicts.js"; import * as worktreePoolModule from "../../worktree-pool.js"; @@ -84,6 +84,12 @@ function makeHarness(overrides: Partial = {}, options: { recentAuditRows?: AuditRow[]; activeHeartbeat?: boolean; missingWorktree?: boolean; + // FN-7566: liveness signals that DO track ephemeral executors (agentId: "executor"), + // which never emit heartbeat runs, never take a checkout lease, and write no runAuditEvents. + liveSessionPath?: boolean; + executingTaskLockHeld?: boolean; + isTaskActive?: boolean; + clearReturns?: boolean; } = {}): Harness { const rootDir = mkdtempSync(join(tmpdir(), "fn-6736-")); const worktree = join(rootDir, ".worktrees", "crisp-lotus"); @@ -92,15 +98,24 @@ function makeHarness(overrides: Partial = {}, options: { } const task = makeTask({ worktree, ...overrides }); const store = makeStore(task, { recentAuditRows: options.recentAuditRows }); - const clearPhantomExecutorBinding = vi.fn(); + const clearPhantomExecutorBinding = options.clearReturns === undefined + ? vi.fn() + : vi.fn(() => options.clearReturns); const agentStore = options.activeHeartbeat ? { listActiveHeartbeatRuns: vi.fn(async () => [{ startedAt: new Date(NOW.getTime() - 60_000).toISOString(), contextSnapshot: { taskId: task.id } }]) } : { listActiveHeartbeatRuns: vi.fn(async () => []) }; + if (options.liveSessionPath) { + activeSessionRegistry.registerPath(worktree, { taskId: task.id, kind: "executor", ownerKey: task.id }); + } + if (options.executingTaskLockHeld) { + executingTaskLock.tryClaim(task.id); + } const manager = new SelfHealingManager(store as any, { rootDir, getExecutingTaskIds: () => new Set([task.id]), clearPhantomExecutorBinding, agentStore, + ...(options.isTaskActive !== undefined ? { isTaskActive: () => options.isTaskActive } : {}), } as any); return { rootDir, @@ -126,12 +141,14 @@ describe("FN-6736: phantom executor binding reclaim", () => { vi.setSystemTime(NOW); vi.restoreAllMocks(); activeSessionRegistry.clear(); + executingTaskLock._clearForTest(); vi.spyOn(worktreePoolModule, "isUsableTaskWorktree").mockResolvedValue(true); vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "stale" } as any); }); afterEach(() => { activeSessionRegistry.clear(); + executingTaskLock._clearForTest(); vi.useRealTimers(); }); @@ -230,6 +247,72 @@ describe("FN-6736: phantom executor binding reclaim", () => { h.cleanup(); }); + // FN-7566: the FN-6736 durable-agent liveness gate (heartbeat/checkout/runAudit) is + // structurally blind to ephemeral executors. A live ephemeral executor keeps its worktree + // registered in activeSessionRegistry / holds the executing lock / reports isTaskActive — those + // are the in-process signals that MUST veto the phantom verdict even past the age multiplier, + // otherwise any ephemeral executor task running >30 min is killed mid-flight. Surface + // enumeration: all three live-session signals + the clearPhantomExecutorBinding refusal path. + it("does NOT reclaim a live ephemeral executor whose worktree is registered as an active session", async () => { + const h = makeHarness({}, { liveSessionPath: true }); + expect(existsSync(h.worktree)).toBe(true); + + const recovered = await h.manager.reclaimSelfOwnedBranchConflicts(); + + expect(recovered).toBe(0); + expect(h.clearPhantomExecutorBinding).not.toHaveBeenCalled(); + expect(h.store.moveTask).not.toHaveBeenCalled(); + expect(findAudit(h.store, "task:reclaim-phantom-executor-binding")).toBeUndefined(); + expect(findAudit(h.store, "task:reclaim-self-owned-branch-conflict-no-action")?.metadata).toEqual( + expect.objectContaining({ reason: "executor-active" }), + ); + expect(h.task.column).toBe("in-progress"); + h.cleanup(); + }); + + it("does NOT reclaim a live ephemeral executor that still holds the executing lock", async () => { + const h = makeHarness({}, { executingTaskLockHeld: true }); + + await h.manager.reclaimSelfOwnedBranchConflicts(); + + expect(h.clearPhantomExecutorBinding).not.toHaveBeenCalled(); + expect(h.store.moveTask).not.toHaveBeenCalled(); + expect(findAudit(h.store, "task:reclaim-phantom-executor-binding")).toBeUndefined(); + expect(h.task.column).toBe("in-progress"); + h.cleanup(); + }); + + it("does NOT reclaim a live ephemeral executor that reports isTaskActive", async () => { + const h = makeHarness({}, { isTaskActive: true }); + + await h.manager.reclaimSelfOwnedBranchConflicts(); + + expect(h.clearPhantomExecutorBinding).not.toHaveBeenCalled(); + expect(h.store.moveTask).not.toHaveBeenCalled(); + expect(findAudit(h.store, "task:reclaim-phantom-executor-binding")).toBeUndefined(); + expect(h.task.column).toBe("in-progress"); + h.cleanup(); + }); + + it("honors clearPhantomExecutorBinding's live-session refusal instead of hard-cancelling to todo", async () => { + // Defense-in-depth: even if the phantom verdict slips past isPhantomExecutorBinding, a clear that + // refuses (returns false — a live session surface is still registered) must NOT be followed by the + // destructive moveTask(→todo). Mirrors reapLeakedConcurrencySlots' `released !== true` guard. + const h = makeHarness({}, { clearReturns: false }); + + const recovered = await h.manager.reclaimSelfOwnedBranchConflicts(); + + expect(recovered).toBe(0); + expect(h.clearPhantomExecutorBinding).toHaveBeenCalledWith(h.task.id, { preserveWorktrees: true }); + expect(h.store.moveTask).not.toHaveBeenCalled(); + expect(h.task.column).toBe("in-progress"); + expect(findAudit(h.store, "task:reclaim-phantom-executor-binding")).toBeUndefined(); + expect(findAudit(h.store, "task:reclaim-self-owned-branch-conflict-no-action")?.metadata).toEqual( + expect.objectContaining({ reason: "phantom-clear-refused-live-session" }), + ); + h.cleanup(); + }); + it("does not increment FN-5704 resume-limbo counters on the phantom-binding requeue", async () => { const h = makeHarness({ resumeLimboCount: 1 } as Partial); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 2545357c80..b4a1c7e423 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -1103,6 +1103,10 @@ export class SelfHealingManager { /** * FNXC:SelfHealingReclaim 2026-06-19-00:00: * FN-6736 requires self-healing to stop treating an in-memory `executor-active` binding as live when the owner is demonstrably dead. Preserve FN-4811 by requiring every live-owner signal to be absent, leave the FN-5219 missing-worktree path untouched, and avoid FN-5704 resume-limbo counters because this path only clears a stale binding and requeues once with progress/worktree preserved. + * + * FNXC:SelfHealingReclaim 2026-07-05-08:15: + * FN-7566: the FN-6736 liveness gate (`agentPresent` heartbeat, `checkedOutBy` lease, `hasRecentRunAudit`) is structurally blind to EPHEMERAL EXECUTOR agents (`agentId: "executor"`): they never emit heartbeat runs (so `activeHeartbeatTaskIds` never contains them), never acquire a checkout lease (`checkedOutBy` stays null), and normal execution activity (sandbox:run / task:log / verification) writes no `runAuditEvents` rows (so `getRecentRunAuditActivityAgeMs` stays null). With all three permanently false, the ONLY surviving gate was age > graceMs*3 (~30 min), so any ephemeral executor task running longer than 30 minutes — a heavy foreach workflow, a slow model — was killed mid-flight on the next self-healing sweep and hard-moved to `todo`, corrupting overlapping-worktree/task-link state. + * The fix adds the in-process live-session truth that DOES track ephemeral executors: a worktree path registered as active in `activeSessionRegistry` (the executor/step-session/workflow-step session holds it for the whole run), the `executingTaskLock`, or `isTaskActive`. This mirrors the canonical `isWorkspaceTaskLive` / `sessionDead` predicate. A genuinely leaked binding (FN-6736) still has an EMPTY registry / no lock / inactive task, so legitimate phantom recovery is preserved; a live ephemeral executor now vetoes the phantom verdict regardless of the durable-agent signals. */ private isPhantomExecutorBinding(task: Task, options: { executionAgeMs: number | null; @@ -1115,6 +1119,14 @@ export class SelfHealingManager { const checkedOutBy = typeof task.checkedOutBy === "string" && task.checkedOutBy.trim().length > 0 ? task.checkedOutBy : null; const worktreeExists = Boolean(task.worktree && existsSync(task.worktree)); const hasRecentRunAudit = options.lastActivityMs !== null && options.lastActivityMs <= RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS; + // FN-7566: in-process liveness that survives for ephemeral executors. The registered + // session path is the faithful proxy for the live session surfaces + // (`activeSessions`/`activeStepExecutors`/`activeWorkflowStepSessions`) that + // `clearPhantomExecutorBinding` itself refuses to detach. + const livePaths = activeSessionRegistry.pathsForTask(task.id).filter((path) => activeSessionRegistry.isPathActive(path)); + const hasLiveInProcessSession = livePaths.length > 0 + || executingTaskLock.has(task.id) + || this.options.isTaskActive?.(task.id) === true; const safeAgeMs = options.graceMs * PHANTOM_EXECUTOR_BINDING_AGE_MULTIPLIER; const metadata = { taskId: task.id, @@ -1125,6 +1137,8 @@ export class SelfHealingManager { agentPresent, lastActivityMs: options.lastActivityMs, hasRecentRunAudit, + hasLiveInProcessSession, + liveSessionPaths: livePaths, worktree: task.worktree ?? null, branch: task.branch ?? null, worktreeExists, @@ -1137,7 +1151,8 @@ export class SelfHealingManager { && options.executionAgeMs > safeAgeMs && !checkedOutBy && !agentPresent - && !hasRecentRunAudit, + && !hasRecentRunAudit + && !hasLiveInProcessSession, metadata, }; } @@ -3123,7 +3138,22 @@ export class SelfHealingManager { // FNXC:SelfHealingReclaim 2026-06-30-00:00: preserveWorktrees keeps the held worktree's // session-registry entry so the moveTask(preserveWorktree:true) re-dispatch reattaches to // the same worktree instead of orphaning it and acquiring a new one (FN-7249 regression). - this.options.clearPhantomExecutorBinding?.(task.id, { preserveWorktrees: true }); + // FNXC:SelfHealingReclaim 2026-07-05-08:15: FN-7566 — honor clearPhantomExecutorBinding's + // live-session refusal (returns false when any session surface is still registered) as the + // last line of defense before the destructive moveTask(→todo), matching reapLeakedConcurrencySlots. + // Even if a future liveness signal slips past isPhantomExecutorBinding, a refused clear must NOT + // be followed by a hard-cancel of a live executor: fall through to the no-action audit instead. + const released = this.options.clearPhantomExecutorBinding?.(task.id, { preserveWorktrees: true }); + if (released === false) { + await this.emitFalsePositiveRequeueNoAction( + task, + "reclaim-self-owned-branch-conflict", + "task:reclaim-self-owned-branch-conflict-no-action", + "phantom-clear-refused-live-session", + { ...phantomBinding.metadata, signalReason: liveExecutionSignal.reason }, + ); + continue; + } await createRunAuditor(this.store, { runId: generateSyntheticRunId("self-healing-phantom-executor-binding", task.id), agentId: "self-healing",