fix: recover orphaned in-progress tasks even when worktree exists on disk

Previously, recoverOrphanedExecutions() skipped tasks whose worktree
directory still existed, assuming an active session. After engine crashes
where resumeOrphaned() failed, these tasks were stuck forever. Now uses
a tiered grace period: 60s for missing worktrees, 5min for existing ones
to avoid racing with startup recovery.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-09 07:21:14 -07:00
parent 93eec2a882
commit a0606884cb
2 changed files with 113 additions and 9 deletions

View File

@@ -4,6 +4,16 @@ vi.mock("node:child_process", () => ({
execSync: vi.fn(),
}));
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
return {
...actual,
existsSync: vi.fn(actual.existsSync),
readdirSync: vi.fn(actual.readdirSync),
statSync: vi.fn(actual.statSync),
};
});
vi.mock("./worktree-pool.js", () => ({
WorktreePool: vi.fn(),
scanIdleWorktrees: vi.fn().mockResolvedValue([]),
@@ -15,6 +25,7 @@ import { SelfHealingManager } from "./self-healing.js";
import type { TaskStore, Settings, Task } from "@fusion/core";
import { EventEmitter } from "node:events";
import { execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { scanOrphanedBranches } from "./worktree-pool.js";
const mockedExecSync = vi.mocked(execSync);
@@ -797,6 +808,84 @@ describe("SelfHealingManager", () => {
managerWithRecovery.stop();
});
it("recovers tasks with existing worktree but no active session after grace period", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const mockedExistsSync = vi.mocked(existsSync);
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-210",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/active-tree",
steps: [{ status: "done" }, { status: "in-progress" }, { status: "pending" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
// Worktree directory exists on disk
mockedExistsSync.mockImplementation((p) =>
p === "/tmp/test-project/.worktrees/active-tree" ? true : false,
);
// 10 minutes past — well beyond the 5-minute grace period
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(1);
expect(store.updateTask).toHaveBeenCalledWith("FN-210", {
status: "stuck-killed",
worktree: null,
branch: null,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-210",
expect.stringContaining("worktree exists but no active session"),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-210", "todo");
managerWithRecovery.stop();
});
it("skips tasks with existing worktree within the extended grace period", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const mockedExistsSync = vi.mocked(existsSync);
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-211",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/active-tree",
steps: [{ status: "in-progress" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
mockedExistsSync.mockImplementation((p) =>
p === "/tmp/test-project/.worktrees/active-tree" ? true : false,
);
// Only 2 minutes past — within the 5-minute grace period for existing worktrees
vi.setSystemTime(new Date("2026-01-01T00:02:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
});
describe("recoverApprovedTriageTasks", () => {

View File

@@ -54,6 +54,13 @@ export interface SelfHealingOptions {
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
const ORPHANED_EXECUTION_RECOVERY_GRACE_MS = 60_000;
/**
* Longer grace period for tasks that still have a worktree on disk.
* This avoids racing with `executor.resumeOrphaned()` which runs on
* engine startup and may legitimately re-execute these tasks.
* 5 minutes is well past any startup window.
*/
const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
export class SelfHealingManager {
// ── Auto-unpause state ──────────────────────────────────────────────
@@ -392,14 +399,17 @@ export class SelfHealingManager {
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
const now = Date.now();
const orphaned = tasks.filter((t) =>
t.column === "in-progress" &&
!t.paused &&
!executingIds.has(t.id) &&
!isTaskWorkComplete(t) &&
(!t.worktree || !existsSync(t.worktree)) &&
now - new Date(t.updatedAt).getTime() >= ORPHANED_EXECUTION_RECOVERY_GRACE_MS,
);
const orphaned = tasks.filter((t) => {
if (t.column !== "in-progress" || t.paused || executingIds.has(t.id) || isTaskWorkComplete(t)) {
return false;
}
const staleness = now - new Date(t.updatedAt).getTime();
// Tasks with an existing worktree get a longer grace period to avoid
// racing with executor.resumeOrphaned() on engine startup.
const hasWorktree = t.worktree && existsSync(t.worktree);
const graceMs = hasWorktree ? ORPHANED_WITH_WORKTREE_GRACE_MS : ORPHANED_EXECUTION_RECOVERY_GRACE_MS;
return staleness >= graceMs;
});
if (orphaned.length === 0) return 0;
@@ -408,6 +418,11 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of orphaned) {
try {
const hadWorktree = task.worktree && existsSync(task.worktree);
const reason = hadWorktree
? "worktree exists but no active session"
: "missing worktree/session";
await this.store.updateTask(task.id, {
status: "stuck-killed",
worktree: null,
@@ -415,7 +430,7 @@ export class SelfHealingManager {
});
await this.store.logEntry(
task.id,
"Auto-recovered orphaned executor task — missing worktree/session, moved back to todo",
`Auto-recovered orphaned executor task — ${reason}, moved back to todo`,
);
await this.store.moveTask(task.id, "todo");
recovered++;