feat(FN-5219): detect and recover in-progress limbo tasks

The merge adds a new self-healing recovery path for tasks stuck in `in-progress` limbo (no pending step updates but not marked done), hardening `resetTask` and `recoverInProgressLimboTasks` with proper worker binding cleanup, audit event coverage, and integration tests validating the invariant acros

Fusion-Task-Id: FN-5219

packages/engine/src/__tests__/reliability-interactions/in-progress-limbo-recovery.test.ts

@@ -0,0 +1,171 @@
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { execSync } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { logger } = vi.hoisted(() => ({ logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
+vi.mock("../../logger.js", () => ({ createLogger: vi.fn(() => logger) }));
+
+import { TaskStore } from "@fusion/core";
+import { SelfHealingManager } from "../../self-healing.js";
+
+function git(cwd: string, command: string): string {
+  return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
+}
+
+describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
+  let rootDir = "";
+  let store: TaskStore;
+
+  beforeEach(async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-05-20T12:00:00.000Z"));
+    rootDir = mkdtempSync(join(tmpdir(), "fn-5219-reliability-"));
+    git(rootDir, "init -b main");
+    git(rootDir, "config user.name 'Fusion'");
+    git(rootDir, "config user.email 'hi@runfusion.ai'");
+    writeFileSync(join(rootDir, "README.md"), "root\n");
+    git(rootDir, "add README.md");
+    git(rootDir, "commit -m 'init'");
+    mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
+    store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
+  });
+
+  afterEach(() => {
+    try { store?.close(); } catch {}
+    if (rootDir) rmSync(rootDir, { recursive: true, force: true });
+    vi.useRealTimers();
+    vi.clearAllMocks();
+  });
+
+  async function createInProgressTask(title: string) {
+    const task = await store.createTask({ title, description: title });
+    await store.moveTask(task.id, "todo");
+    await store.moveTask(task.id, "in-progress");
+    return task.id;
+  }
+
+  it("FN-5149: reset twice → stranded in-progress with missing worktree → recovered to todo", async () => {
+    const mockStore = Object.assign(new EventEmitter(), {
+      getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false }),
+      listTasks: vi.fn()
+        .mockResolvedValueOnce([
+          {
+            id: "FN-5149",
+            column: "in-progress",
+            paused: false,
+            branch: null,
+            worktree: join(rootDir, ".worktrees", "fn-5149-missing"),
+            updatedAt: "2026-05-20T12:00:00.000Z",
+            steps: [{ status: "pending" }],
+            log: [],
+          },
+        ])
+        .mockResolvedValueOnce([]),
+      updateTask: vi.fn().mockResolvedValue({}),
+      logEntry: vi.fn().mockResolvedValue(undefined),
+      moveTask: vi.fn().mockResolvedValue(undefined),
+      recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
+      getRootDir: vi.fn().mockReturnValue(rootDir),
+    }) as unknown as TaskStore;
+    vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
+
+    const manager = new SelfHealingManager(mockStore, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const first = await manager.recoverInProgressLimbo();
+    const second = await manager.recoverOrphanedExecutions();
+
+    expect(first).toBe(1);
+    expect(second).toBe(0);
+    expect(mockStore.moveTask).toHaveBeenCalledWith("FN-5149", "todo", { preserveProgress: true });
+    expect(mockStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
+      mutationType: "task:auto-recover-in-progress-limbo",
+      target: "FN-5149",
+    }));
+  });
+
+  it("reconcile-task-worktree-metadata runs first so a live rebindable worktree wins", async () => {
+    const id = await createInProgressTask("metadata rebind wins");
+    const liveWorktree = join(rootDir, ".worktrees", `${id.toLowerCase()}-live`);
+    const branch = `fusion/${id.toLowerCase()}`;
+    git(rootDir, `worktree add -b ${branch} ${liveWorktree}`);
+    writeFileSync(join(liveWorktree, `${id}.txt`), `${id}\n`);
+    git(liveWorktree, `add ${id}.txt`);
+    git(liveWorktree, `commit -m 'task work'`);
+    git(rootDir, "checkout main");
+
+    await store.updateTask(id, {
+      branch: null,
+      worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
+      steps: [{ name: "step", status: "pending" }],
+    });
+    vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
+
+    const manager = new SelfHealingManager(store, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const repaired = await manager.reconcileTaskWorktreeMetadata({ includeTaskIds: new Set([id]) });
+    const recovered = await manager.recoverInProgressLimbo();
+    const updated = await store.getTask(id);
+
+    expect(repaired).toBe(1);
+    expect(recovered).toBe(0);
+    expect(updated?.column).toBe("in-progress");
+    expect(updated?.branch).toBe(branch);
+    expect(updated?.worktree?.endsWith(`${id.toLowerCase()}-live`)).toBe(true);
+  });
+
+  it("keeps in-review missing-worktree failures on the review-specific recovery path", async () => {
+    const id = await createInProgressTask("review failure disjoint");
+    await store.moveTask(id, "in-review");
+    await store.updateTask(id, {
+      status: "failed",
+      error: `Refusing to start coding agent in missing worktree: ${join(rootDir, ".worktrees", "missing-review")}`,
+      branch: `fusion/${id.toLowerCase()}`,
+      worktree: join(rootDir, ".worktrees", "missing-review-stale"),
+      steps: [{ name: "step", status: "done" }, { name: "next", status: "pending" }],
+    });
+
+    const manager = new SelfHealingManager(store, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>(),
+    });
+
+    const limboRecovered = await manager.recoverInProgressLimbo();
+    const reviewRecovered = await manager.recoverMissingWorktreeReviewFailures();
+    const updated = await store.getTask(id);
+
+    expect(limboRecovered).toBe(0);
+    expect(reviewRecovered).toBe(1);
+    expect(updated?.column).toBe("todo");
+  });
+
+  it("skips limbo recovery while the executor still claims the task id", async () => {
+    const id = await createInProgressTask("executor claim wins");
+    await store.updateTask(id, {
+      branch: null,
+      worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
+      steps: [{ name: "step", status: "pending" }],
+    });
+    vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
+
+    const manager = new SelfHealingManager(store, {
+      rootDir,
+      getExecutingTaskIds: () => new Set<string>([id]),
+    });
+
+    const recovered = await manager.recoverInProgressLimbo();
+    const updated = await store.getTask(id);
+
+    expect(recovered).toBe(0);
+    expect(updated?.column).toBe("in-progress");
+    expect(updated?.worktree).toContain("missing");
+  });
+});