feat(FN-5219): detect and recover in-progress limbo tasks

The merge adds a new self-healing recovery path for tasks stuck in `in-progress` limbo (no pending step updates but not marked done), hardening `resetTask` and `recoverInProgressLimboTasks` with proper worker binding cleanup, audit event coverage, and integration tests validating the invariant acros

Fusion-Task-Id: FN-5219

packages/engine/src/__tests__/reliability-interactions/in-progress-limbo-recovery.test.ts

@@ -0,0 +1,171 @@
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { execSync } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { logger } = vi.hoisted(() => ({ logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
+vi.mock("../../logger.js", () => ({ createLogger: vi.fn(() => logger) }));
+
+import { TaskStore } from "@fusion/core";
+import { SelfHealingManager } from "../../self-healing.js";
+
+function git(cwd: string, command: string): string {
+  return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
+}
+
+describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
+  let rootDir = "";
+  let store: TaskStore;
+
+  beforeEach(async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-05-20T12:00:00.000Z"));
+    rootDir = mkdtempSync(join(tmpdir(), "fn-5219-reliability-"));
+    git(rootDir, "init -b main");
+    git(rootDir, "config user.name 'Fusion'");
+    git(rootDir, "config user.email 'hi@runfusion.ai'");
+    writeFileSync(join(rootDir, "README.md"), "root\n");
+    git(rootDir, "add README.md");
+    git(rootDir, "commit -m 'init'");
+    mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
+    store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
+  });
+
+  afterEach(() => {
+    try { store?.close(); } catch {}
+    if (rootDir) rmSync(rootDir, { recursive: true, force: true });
+    vi.useRealTimers();
+    vi.clearAllMocks();
+  });
+
+  async function createInProgressTask(title: string) {
+    const task = await store.createTask({ title, description: title });
+    await store.moveTask(task.id, "todo");
+    await store.moveTask(task.id, "in-progress");
+    return task.id;
+  }
+
+  it("FN-5149: reset twice → stranded in-progress with missing worktree → recovered to todo", async () => {
+    const mockStore = Object.assign(new EventEmitter(), {
+      getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false }),
+      listTasks: vi.fn()
+        .mockResolvedValueOnce([
+          {
+            id: "FN-5149",
+            column: "in-progress",
+            paused: false,
+            branch: null,
+            worktree: join(rootDir, ".worktrees", "fn-5149-missing"),
+            updatedAt: "2026-05-20T12:00:00.000Z",
+            steps: [{ status: "pending" }],
+            log: [],
+          },
+        ])
+        .mockResolvedValueOnce([]),
+      updateTask: vi.fn().mockResolvedValue({}),
+      logEntry: vi.fn().mockResolvedValue(undefined),
+      moveTask: vi.fn().mockResolvedValue(undefined),
+      recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
+      getRootDir: vi.fn().mockReturnValue(rootDir),
+    }) as unknown as TaskStore;
+    vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
+
+    const manager = new SelfHealingManager(mockStore, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const first = await manager.recoverInProgressLimbo();
+    const second = await manager.recoverOrphanedExecutions();
+
+    expect(first).toBe(1);
+    expect(second).toBe(0);
+    expect(mockStore.moveTask).toHaveBeenCalledWith("FN-5149", "todo", { preserveProgress: true });
+    expect(mockStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
+      mutationType: "task:auto-recover-in-progress-limbo",
+      target: "FN-5149",
+    }));
+  });
+
+  it("reconcile-task-worktree-metadata runs first so a live rebindable worktree wins", async () => {
+    const id = await createInProgressTask("metadata rebind wins");
+    const liveWorktree = join(rootDir, ".worktrees", `${id.toLowerCase()}-live`);
+    const branch = `fusion/${id.toLowerCase()}`;
+    git(rootDir, `worktree add -b ${branch} ${liveWorktree}`);
+    writeFileSync(join(liveWorktree, `${id}.txt`), `${id}\n`);
+    git(liveWorktree, `add ${id}.txt`);
+    git(liveWorktree, `commit -m 'task work'`);
+    git(rootDir, "checkout main");
+
+    await store.updateTask(id, {
+      branch: null,
+      worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
+      steps: [{ name: "step", status: "pending" }],
+    });
+    vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
+
+    const manager = new SelfHealingManager(store, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const repaired = await manager.reconcileTaskWorktreeMetadata({ includeTaskIds: new Set([id]) });
+    const recovered = await manager.recoverInProgressLimbo();
+    const updated = await store.getTask(id);
+
+    expect(repaired).toBe(1);
+    expect(recovered).toBe(0);
+    expect(updated?.column).toBe("in-progress");
+    expect(updated?.branch).toBe(branch);
+    expect(updated?.worktree?.endsWith(`${id.toLowerCase()}-live`)).toBe(true);
+  });
+
+  it("keeps in-review missing-worktree failures on the review-specific recovery path", async () => {
+    const id = await createInProgressTask("review failure disjoint");
+    await store.moveTask(id, "in-review");
+    await store.updateTask(id, {
+      status: "failed",
+      error: `Refusing to start coding agent in missing worktree: ${join(rootDir, ".worktrees", "missing-review")}`,
+      branch: `fusion/${id.toLowerCase()}`,
+      worktree: join(rootDir, ".worktrees", "missing-review-stale"),
+      steps: [{ name: "step", status: "done" }, { name: "next", status: "pending" }],
+    });
+
+    const manager = new SelfHealingManager(store, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const limboRecovered = await manager.recoverInProgressLimbo();
+    const reviewRecovered = await manager.recoverMissingWorktreeReviewFailures();
+    const updated = await store.getTask(id);
+
+    expect(limboRecovered).toBe(0);
+    expect(reviewRecovered).toBe(1);
+    expect(updated?.column).toBe("todo");
+  });
+
+  it("skips limbo recovery while the executor still claims the task id", async () => {
+    const id = await createInProgressTask("executor claim wins");
+    await store.updateTask(id, {
+      branch: null,
+      worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
+      steps: [{ name: "step", status: "pending" }],
+    });
+    vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
+
+    const manager = new SelfHealingManager(store, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>([id]),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+    const updated = await store.getTask(id);
+
+    expect(recovered).toBe(0);
+    expect(updated?.column).toBe("in-progress");
+    expect(updated?.worktree).toContain("missing");
+  });
+});

packages/engine/src/__tests__/self-healing-in-progress-limbo.test.ts

@@ -0,0 +1,212 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { EventEmitter } from "node:events";
+
+vi.mock("node:fs", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("node:fs")>();
+  return {
+    ...actual,
+    existsSync: vi.fn(actual.existsSync),
+  };
+});
+
+const { logger } = vi.hoisted(() => ({ logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
+vi.mock("../logger.js", () => ({
+  createLogger: vi.fn(() => logger),
+  schedulerLog: { log: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+vi.mock("../worktree-pool.js", () => ({
+  WorktreePool: vi.fn(),
+  RemovalReason: {
+    HardCancel: "hard-cancel",
+    ExecutorTransientRetry: "executor-transient-retry",
+    ExecutorStuckKilled: "executor-stuck-killed",
+    ExecutorDispose: "executor-dispose",
+    StepSessionCleanup: "step-session-cleanup",
+    MergerPostMerge: "merger-post-merge",
+    MergerCleanup: "merger-cleanup",
+    SelfHealingReclaim: "self-healing-reclaim",
+    SelfHealingStaleActiveBranch: "self-healing-stale-active-branch",
+    SelfHealingBranchConflict: "self-healing-branch-conflict",
+    SelfHealingOrphanRescue: "self-healing-orphan-rescue",
+    SelfHealingIdleSweep: "self-healing-idle-sweep",
+    PoolPrune: "pool-prune",
+  },
+  scanIdleWorktrees: vi.fn().mockResolvedValue([]),
+  cleanupOrphanedWorktrees: vi.fn().mockResolvedValue(0),
+  scanOrphanedBranches: vi.fn().mockResolvedValue([]),
+  isUsableTaskWorktree: vi.fn().mockResolvedValue(true),
+  removeWorktree: vi.fn().mockResolvedValue(undefined),
+  resolveWorktreeBackend: vi.fn(),
+}));
+
+vi.mock("../merger.js", () => ({ classifyOwnedLandedEvidence: vi.fn() }));
+
+import { existsSync } from "node:fs";
+import { SelfHealingManager } from "../self-healing.js";
+import type { Settings, Task, TaskStore } from "@fusion/core";
+
+function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
+  const emitter = new EventEmitter();
+  return Object.assign(emitter, {
+    getSettings: vi.fn().mockResolvedValue({
+      autoMerge: true,
+      globalPause: false,
+      enginePaused: false,
+      maintenanceIntervalMs: 0,
+    } as unknown as Settings),
+    listTasks: vi.fn().mockResolvedValue([]),
+    updateTask: vi.fn().mockResolvedValue({} as Task),
+    logEntry: vi.fn().mockResolvedValue(undefined),
+    moveTask: vi.fn().mockResolvedValue(undefined),
+    recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
+    getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
+    ...overrides,
+  }) as unknown as TaskStore & EventEmitter;
+}
+
+describe("recoverInProgressLimbo", () => {
+  let store: TaskStore & EventEmitter;
+
+  const baseTask = {
+    id: "FN-5149",
+    column: "in-progress",
+    paused: false,
+    branch: null,
+    worktree: "/tmp/test-project/.worktrees/missing-fn-5149",
+    checkedOutBy: "agent-1",
+    executionStartedAt: "2026-05-20T12:00:00.000Z",
+    updatedAt: "2026-05-20T12:00:00.000Z",
+    steps: [{ status: "pending" }, { status: "pending" }],
+  };
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-05-20T12:05:00.000Z"));
+    vi.mocked(existsSync).mockImplementation(() => false);
+    store = createMockStore();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.clearAllMocks();
+  });
+
+  it("requeues FN-5149-signature limbo tasks to todo with audit telemetry", async () => {
+    const reconcileLeaseRow = vi.fn().mockResolvedValue(undefined);
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
+
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(),
+      leaseManager: {
+        recoverAbandonedLease: vi.fn().mockResolvedValue(false),
+        reconcileLeaseRow,
+      } as any,
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(1);
+    expect(reconcileLeaseRow).toHaveBeenCalledWith("FN-5149");
+    expect(store.updateTask).toHaveBeenCalledWith("FN-5149", expect.objectContaining({
+      worktree: null,
+      branch: null,
+      status: null,
+      error: null,
+      checkedOutBy: null,
+      executionStartedAt: null,
+      worktreeSessionRetryCount: null,
+      taskDoneRetryCount: null,
+      sessionFile: null,
+    }));
+    expect(store.moveTask).toHaveBeenCalledWith("FN-5149", "todo", { preserveProgress: true });
+    expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
+      domain: "database",
+      mutationType: "task:auto-recover-in-progress-limbo",
+      target: "FN-5149",
+    }));
+  });
+
+  it("skips tasks whose worktree still exists on disk", async () => {
+    vi.mocked(existsSync).mockImplementation((path) => path === baseTask.worktree);
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(0);
+    expect(store.updateTask).not.toHaveBeenCalled();
+  });
+
+  it("skips tasks whose branch is still set", async () => {
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([{ ...baseTask, branch: "fusion/fn-5149" }]);
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(0);
+    expect(store.moveTask).not.toHaveBeenCalled();
+  });
+
+  it("skips tasks currently claimed by the executor", async () => {
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(["FN-5149"]),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(0);
+    expect(store.moveTask).not.toHaveBeenCalled();
+  });
+
+  it("skips paused tasks", async () => {
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([{ ...baseTask, paused: true }]);
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(0);
+    expect(store.moveTask).not.toHaveBeenCalled();
+  });
+
+  it("skips tasks still within the grace window", async () => {
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([{ ...baseTask, updatedAt: "2026-05-20T12:04:30.000Z" }]);
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(0);
+    expect(store.moveTask).not.toHaveBeenCalled();
+  });
+
+  it("skips entirely when the engine is globally paused", async () => {
+    store = createMockStore({
+      getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: true, enginePaused: false } as unknown as Settings),
+    });
+    (store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
+    const manager = new SelfHealingManager(store, {
+      rootDir: "/tmp/test-project",
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+
+    expect(recovered).toBe(0);
+    expect(store.listTasks).not.toHaveBeenCalled();
+  });
+});

packages/engine/src/run-audit.ts

@@ -235,6 +235,7 @@ export type DatabaseMutationType =
   | "task:auto-recover-completion-handoff-limbo"
   | "task:auto-recover-completion-handoff-limbo-exhausted"
   | "task:auto-recover-worktree-session-exhausted"
+  | "task:auto-recover-in-progress-limbo"
   /** Metadata: { taskId: string; ignoredStepUpdateCount: number; stuckKillStreak: number; lastReason: "no-progress-churn" } */
   | "task:stuck-no-progress-churn-terminalized"
   | "task:auto-recover-starved-refinement"

packages/engine/src/self-healing.ts

@@ -678,6 +678,7 @@ export class SelfHealingManager {
       { name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts().then(() => undefined) },
       // FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
       { name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata().then(() => undefined) },
+      { name: "recover-in-progress-limbo", fn: () => this.recoverInProgressLimbo().then(() => undefined) },
       { name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
       { name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches().then(() => undefined) },
       { name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
@@ -1314,6 +1315,7 @@ export class SelfHealingManager {
           { name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
           // FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
           { name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata() },
+          { name: "recover-in-progress-limbo", fn: () => this.recoverInProgressLimbo() },
           { name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
           { name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches() },
           { name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
@@ -5683,6 +5685,112 @@ export class SelfHealingManager {
    * established, typically when the scheduler reserved a worktree path but the
    * executor never materialized it or crashed before tracking the run.
    */
+  async recoverInProgressLimbo(): Promise<number> {
+    const settings = await this.store.getSettings();
+    if (settings.globalPause || settings.enginePaused) {
+      return 0;
+    }
+
+    try {
+      const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
+      const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
+      const now = Date.now();
+
+      const stranded = tasks.filter((task) => {
+        if (task.column !== "in-progress" || task.paused || executingIds.has(task.id)) {
+          return false;
+        }
+        const hasMissingWorktreePath = typeof task.worktree === "string" && task.worktree.length > 0 && !existsSync(task.worktree);
+        const hasNoWorktreePath = !task.worktree;
+        if (!hasMissingWorktreePath && !hasNoWorktreePath) {
+          return false;
+        }
+        if (typeof task.branch === "string" && task.branch.trim().length > 0) {
+          return false;
+        }
+        if (task.steps.some((step) => step.status !== "pending")) {
+          return false;
+        }
+        const staleness = now - new Date(task.updatedAt).getTime();
+        return staleness >= ORPHANED_EXECUTION_RECOVERY_GRACE_MS;
+      });
+
+      const describeWorktreeState = (task: Task): string => task.worktree ? "missing worktree path" : "cleared worktree metadata";
+
+      if (stranded.length === 0) return 0;
+
+      log.warn(`Found ${stranded.length} in-progress limbo task(s) with missing/cleared worktree + null branch`);
+
+      let recovered = 0;
+      for (const task of stranded) {
+        try {
+          if (this.options.leaseManager && task.checkedOutBy) {
+            await this.options.leaseManager.recoverAbandonedLease(
+              task.id,
+              `in-progress limbo: ${describeWorktreeState(task)} + null branch`,
+              { preserveProgress: true },
+            );
+            await this.options.leaseManager.reconcileLeaseRow(task.id);
+          }
+
+          const stepStatuses = task.steps.map((step) => step.status);
+          const ageMs = Math.max(0, now - new Date(task.updatedAt).getTime());
+          await this.store.updateTask(task.id, {
+            status: null,
+            error: null,
+            worktree: null,
+            branch: null,
+            checkedOutBy: null,
+            executionStartedAt: null,
+            worktreeSessionRetryCount: null,
+            taskDoneRetryCount: null,
+            sessionFile: null,
+          });
+          await this.store.logEntry(
+            task.id,
+            `Auto-recovered in-progress limbo — ${describeWorktreeState(task)}/null branch with no step progress, moved back to todo`,
+            JSON.stringify({
+              priorWorktree: task.worktree ?? null,
+              priorBranch: task.branch ?? null,
+              ageMs,
+              stepStatuses,
+            }),
+          );
+          await createRunAuditor(this.store, {
+            runId: generateSyntheticRunId("self-healing-in-progress-limbo", task.id),
+            agentId: "self-healing",
+            taskId: task.id,
+            taskLineageId: task.lineageId,
+            phase: "recover-in-progress-limbo",
+          }).database({
+            type: "task:auto-recover-in-progress-limbo",
+            target: task.id,
+            metadata: {
+              priorWorktree: task.worktree ?? null,
+              priorBranch: task.branch ?? null,
+              ageMs,
+              stepStatuses,
+            },
+          });
+          await this.store.moveTask(task.id, "todo", { preserveProgress: true });
+          recovered++;
+        } catch (err: unknown) {
+          const errorMessage = err instanceof Error ? err.message : String(err);
+          log.error(`Failed to recover in-progress limbo task ${task.id}: ${errorMessage}`);
+        }
+      }
+
+      if (recovered > 0) {
+        log.log(`Recovered ${recovered} in-progress limbo task(s) → todo`);
+      }
+      return recovered;
+    } catch (err: unknown) {
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      log.error(`In-progress limbo recovery failed: ${errorMessage}`);
+      return 0;
+    }
+  }
+
   async recoverOrphanedExecutions(): Promise<number> {
     try {
       const tasks = await this.store.listTasks({ column: "in-progress", slim: true });