fix: recover orphaned in-progress tasks even when worktree exists on disk
Previously, recoverOrphanedExecutions() skipped tasks whose worktree directory still existed, assuming an active session. After engine crashes where resumeOrphaned() failed, these tasks were stuck forever. Now uses a tiered grace period: 60s for missing worktrees, 5min for existing ones to avoid racing with startup recovery. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,16 @@ vi.mock("node:child_process", () => ({
|
||||
execSync: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
return {
|
||||
...actual,
|
||||
existsSync: vi.fn(actual.existsSync),
|
||||
readdirSync: vi.fn(actual.readdirSync),
|
||||
statSync: vi.fn(actual.statSync),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./worktree-pool.js", () => ({
|
||||
WorktreePool: vi.fn(),
|
||||
scanIdleWorktrees: vi.fn().mockResolvedValue([]),
|
||||
@@ -15,6 +25,7 @@ import { SelfHealingManager } from "./self-healing.js";
|
||||
import type { TaskStore, Settings, Task } from "@fusion/core";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { execSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { scanOrphanedBranches } from "./worktree-pool.js";
|
||||
|
||||
const mockedExecSync = vi.mocked(execSync);
|
||||
@@ -797,6 +808,84 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
|
||||
it("recovers tasks with existing worktree but no active session after grace period", async () => {
|
||||
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
|
||||
const mockedExistsSync = vi.mocked(existsSync);
|
||||
const managerWithRecovery = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: getExecuting,
|
||||
});
|
||||
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||
{
|
||||
id: "FN-210",
|
||||
column: "in-progress",
|
||||
paused: false,
|
||||
worktree: "/tmp/test-project/.worktrees/active-tree",
|
||||
steps: [{ status: "done" }, { status: "in-progress" }, { status: "pending" }],
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
},
|
||||
]);
|
||||
|
||||
// Worktree directory exists on disk
|
||||
mockedExistsSync.mockImplementation((p) =>
|
||||
p === "/tmp/test-project/.worktrees/active-tree" ? true : false,
|
||||
);
|
||||
|
||||
// 10 minutes past — well beyond the 5-minute grace period
|
||||
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
|
||||
|
||||
const result = await managerWithRecovery.recoverOrphanedExecutions();
|
||||
|
||||
expect(result).toBe(1);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-210", {
|
||||
status: "stuck-killed",
|
||||
worktree: null,
|
||||
branch: null,
|
||||
});
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-210",
|
||||
expect.stringContaining("worktree exists but no active session"),
|
||||
);
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-210", "todo");
|
||||
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
|
||||
it("skips tasks with existing worktree within the extended grace period", async () => {
|
||||
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
|
||||
const mockedExistsSync = vi.mocked(existsSync);
|
||||
const managerWithRecovery = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: getExecuting,
|
||||
});
|
||||
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||
{
|
||||
id: "FN-211",
|
||||
column: "in-progress",
|
||||
paused: false,
|
||||
worktree: "/tmp/test-project/.worktrees/active-tree",
|
||||
steps: [{ status: "in-progress" }],
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
},
|
||||
]);
|
||||
|
||||
mockedExistsSync.mockImplementation((p) =>
|
||||
p === "/tmp/test-project/.worktrees/active-tree" ? true : false,
|
||||
);
|
||||
|
||||
// Only 2 minutes past — within the 5-minute grace period for existing worktrees
|
||||
vi.setSystemTime(new Date("2026-01-01T00:02:00.000Z"));
|
||||
|
||||
const result = await managerWithRecovery.recoverOrphanedExecutions();
|
||||
|
||||
expect(result).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe("recoverApprovedTriageTasks", () => {
|
||||
|
||||
@@ -54,6 +54,13 @@ export interface SelfHealingOptions {
|
||||
|
||||
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
|
||||
const ORPHANED_EXECUTION_RECOVERY_GRACE_MS = 60_000;
|
||||
/**
|
||||
* Longer grace period for tasks that still have a worktree on disk.
|
||||
* This avoids racing with `executor.resumeOrphaned()` which runs on
|
||||
* engine startup and may legitimately re-execute these tasks.
|
||||
* 5 minutes is well past any startup window.
|
||||
*/
|
||||
const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
|
||||
|
||||
export class SelfHealingManager {
|
||||
// ── Auto-unpause state ──────────────────────────────────────────────
|
||||
@@ -392,14 +399,17 @@ export class SelfHealingManager {
|
||||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||||
const now = Date.now();
|
||||
|
||||
const orphaned = tasks.filter((t) =>
|
||||
t.column === "in-progress" &&
|
||||
!t.paused &&
|
||||
!executingIds.has(t.id) &&
|
||||
!isTaskWorkComplete(t) &&
|
||||
(!t.worktree || !existsSync(t.worktree)) &&
|
||||
now - new Date(t.updatedAt).getTime() >= ORPHANED_EXECUTION_RECOVERY_GRACE_MS,
|
||||
);
|
||||
const orphaned = tasks.filter((t) => {
|
||||
if (t.column !== "in-progress" || t.paused || executingIds.has(t.id) || isTaskWorkComplete(t)) {
|
||||
return false;
|
||||
}
|
||||
const staleness = now - new Date(t.updatedAt).getTime();
|
||||
// Tasks with an existing worktree get a longer grace period to avoid
|
||||
// racing with executor.resumeOrphaned() on engine startup.
|
||||
const hasWorktree = t.worktree && existsSync(t.worktree);
|
||||
const graceMs = hasWorktree ? ORPHANED_WITH_WORKTREE_GRACE_MS : ORPHANED_EXECUTION_RECOVERY_GRACE_MS;
|
||||
return staleness >= graceMs;
|
||||
});
|
||||
|
||||
if (orphaned.length === 0) return 0;
|
||||
|
||||
@@ -408,6 +418,11 @@ export class SelfHealingManager {
|
||||
let recovered = 0;
|
||||
for (const task of orphaned) {
|
||||
try {
|
||||
const hadWorktree = task.worktree && existsSync(task.worktree);
|
||||
const reason = hadWorktree
|
||||
? "worktree exists but no active session"
|
||||
: "missing worktree/session";
|
||||
|
||||
await this.store.updateTask(task.id, {
|
||||
status: "stuck-killed",
|
||||
worktree: null,
|
||||
@@ -415,7 +430,7 @@ export class SelfHealingManager {
|
||||
});
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
"Auto-recovered orphaned executor task — missing worktree/session, moved back to todo",
|
||||
`Auto-recovered orphaned executor task — ${reason}, moved back to todo`,
|
||||
);
|
||||
await this.store.moveTask(task.id, "todo");
|
||||
recovered++;
|
||||
|
||||
Reference in New Issue
Block a user