root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-5337): remove speculative orphan requeue from self-healing sweep

Browse Source 
Removes the speculative orphan requeue mutation path from self-healing, replacing it with an observation-only sweep that no longer attempts to re-enqueue orphaned tasks — a conservative regression that eliminates noisy false-positive recovery attempts. The change includes rewritten unit coverage, a

Fusion-Task-Id: FN-5337
This commit is contained in:
Fusion (runfusion.ai)
2026-05-20 18:46:33 -07:00
committed by gsxdsm
parent dbccdb1275
commit 1a5aff9c44
8 changed files with 309 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
packages/engine/src/__tests__/reliability-interactions/lease-recovery-central-claim.test.ts
View File

@@ -107,29 +107,30 @@ describe("reliability interactions: lease recovery central claim", () => {
expect(reconcileLeaseRow).toHaveBeenCalledWith("FN-X");
});
it("self-healing orphan recovery invokes reconcile once when recovery returns false", async () => {
it("self-healing orphan sweep is observation-only and does not mutate lease state", async () => {
const task = makeTask({ column: "in-progress", worktree: undefined, updatedAt: "2026-01-01T00:00:00.000Z" });
const store = {
listTasks: vi.fn().mockResolvedValue([task]),
updateTask: vi.fn().mockResolvedValue(task),
logEntry: vi.fn().mockResolvedValue(undefined),
moveTask: vi.fn().mockResolvedValue(task),
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
} as unknown as TaskStore;
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const recoverAbandonedLease = vi.fn().mockResolvedValue(false);
const reconcileLeaseRow = vi.fn().mockResolvedValue(true);
const manager = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: () => new Set<string>(),
leaseManager: {
recoverAbandonedLease: vi.fn().mockResolvedValue(false),
reconcileLeaseRow,
} as any,
// FN-5337: recoverOrphanedExecutions no longer touches lease manager.
leaseManager: { recoverAbandonedLease, reconcileLeaseRow } as any,
});
const recovered = await manager.recoverOrphanedExecutions();
expect(recovered).toBe(1);
expect(reconcileLeaseRow).toHaveBeenCalledWith("FN-X");
expect(recovered).toBe(0);
expect(recoverAbandonedLease).not.toHaveBeenCalled();
expect(reconcileLeaseRow).not.toHaveBeenCalled();
manager.stop();
vi.useRealTimers();
});

191
packages/engine/src/__tests__/reliability-interactions/orphan-detected-no-requeue.test.ts Normal file
View File

@@ -0,0 +1,191 @@
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { execSync } from "node:child_process";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { TaskStore } from "@fusion/core";
import { SelfHealingManager } from "../../self-healing.js";
import { activeSessionRegistry, executingTaskLock } from "../../active-session-registry.js";
function git(cwd: string, command: string): string {
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
}
describe("FN-5337 reliability interactions: orphan detected no requeue", () => {
let rootDir = "";
let store: TaskStore;
beforeEach(async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-20T12:00:00.000Z"));
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
rootDir = mkdtempSync(join(tmpdir(), "fn-5337-reliability-"));
git(rootDir, "init -b main");
git(rootDir, "config user.name 'Fusion'");
git(rootDir, "config user.email 'hi@runfusion.ai'");
writeFileSync(join(rootDir, "README.md"), "root\n");
git(rootDir, "add README.md");
git(rootDir, "commit -m 'init'");
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
});
afterEach(() => {
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
try { store?.close(); } catch {}
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
vi.useRealTimers();
vi.clearAllMocks();
});
async function createInProgressTask(title: string) {
const task = await store.createTask({ title, description: title });
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
return task.id;
}
async function orphanEvents(taskId: string) {
return store.getRunAuditEvents({ taskId, mutationType: "task:orphan-detected-no-action" });
}
it("Scenario A: FN-5279 repro shape emits no-action event without lifecycle mutation", async () => {
const id = await createInProgressTask("fn-5279 repro");
const liveWorktree = join(rootDir, ".worktrees", `${id.toLowerCase()}-live`);
const branch = `fusion/${id.toLowerCase()}`;
git(rootDir, `worktree add -b ${branch} ${liveWorktree}`);
activeSessionRegistry.registerPath(liveWorktree, { taskId: id, kind: "executor", ownerKey: "run-1" });
await store.updateTask(id, { branch: null, worktree: null });
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
const recovered = await manager.recoverOrphanedExecutions();
const task = await store.getTask(id);
const events = await orphanEvents(id);
expect(recovered).toBe(0);
expect(task?.column).toBe("in-progress");
expect(task?.branch ?? null).toBeNull();
expect(task?.worktree ?? null).toBeNull();
expect(events).toHaveLength(1);
expect(events[0]?.mutationType).toBe("task:orphan-detected-no-action");
manager.stop();
});
it("Scenario B: worktree exists with no active session emits no-action reason and no move", async () => {
const id = await createInProgressTask("existing worktree no session");
const worktree = join(rootDir, ".worktrees", `${id.toLowerCase()}-stale`);
const branch = `fusion/${id.toLowerCase()}`;
git(rootDir, `worktree add -b ${branch} ${worktree}`);
await store.updateTask(id, { branch, worktree });
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
await manager.recoverOrphanedExecutions();
const task = await store.getTask(id);
const events = await orphanEvents(id);
expect(task?.column).toBe("in-progress");
expect(task?.branch).toBe(branch);
expect(task?.worktree).toBe(worktree);
expect(events).toHaveLength(1);
expect(events[0]?.metadata).toEqual(expect.objectContaining({ reason: "worktree-exists-no-active-session" }));
manager.stop();
});
it("Scenario C: missing worktree emits no-action then limbo recovery handles proof-based case", async () => {
const id = await createInProgressTask("missing worktree");
await store.updateTask(id, {
branch: null,
worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
steps: [{ name: "step", status: "pending" }],
});
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
const orphanRecovered = await manager.recoverOrphanedExecutions();
const limboRecovered = await manager.recoverInProgressLimbo();
const task = await store.getTask(id);
expect(orphanRecovered).toBe(0);
expect(limboRecovered).toBe(1);
expect(task?.column).toBe("todo");
expect((await orphanEvents(id)).length).toBe(1);
manager.stop();
});
it("Scenario D: ordering with FN-5219 remains proof-path owned", async () => {
const id = await createInProgressTask("ordering");
await store.updateTask(id, {
branch: null,
worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
steps: [{ name: "step", status: "pending" }],
});
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
const orphanFirst = await manager.recoverOrphanedExecutions();
const limboSecond = await manager.recoverInProgressLimbo();
const task = await store.getTask(id);
expect(orphanFirst).toBe(0);
expect(limboSecond).toBe(1);
expect(task?.column).toBe("todo");
expect(await orphanEvents(id)).toHaveLength(1);
manager.stop();
});
it("Scenario E: in-review tasks are ignored", async () => {
const id = await createInProgressTask("in-review ignore");
await store.moveTask(id, "in-review");
await store.updateTask(id, {});
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
await manager.recoverOrphanedExecutions();
expect(await orphanEvents(id)).toHaveLength(0);
manager.stop();
});
it("Scenario F: branch-cleared task with live branch remains unchanged except annotation", async () => {
const id = await createInProgressTask("branch cleared");
const worktree = join(rootDir, ".worktrees", `${id.toLowerCase()}-branch`);
git(rootDir, `worktree add -b fusion/${id.toLowerCase()} ${worktree}`);
await store.updateTask(id, { branch: null, worktree: null });
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
await manager.recoverOrphanedExecutions();
const task = await store.getTask(id);
expect(task?.branch ?? null).toBeNull();
expect(await orphanEvents(id)).toHaveLength(1);
manager.stop();
});
it("Scenario G: lease manager is untouched", async () => {
const id = await createInProgressTask("lease untouched");
await store.updateTask(id, { worktree: null, checkedOutBy: "agent-x" });
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const recoverAbandonedLease = vi.fn();
const reconcileLeaseRow = vi.fn();
const manager = new SelfHealingManager(store, {
rootDir,
getExecutingTaskIds: () => new Set<string>(),
leaseManager: { recoverAbandonedLease, reconcileLeaseRow } as any,
});
await manager.recoverOrphanedExecutions();
expect(recoverAbandonedLease).not.toHaveBeenCalled();
expect(reconcileLeaseRow).not.toHaveBeenCalled();
manager.stop();
});
it("Scenario H: re-sweep emits one event per candidate per sweep", async () => {
const id = await createInProgressTask("idempotent re-sweep");
await store.updateTask(id, { worktree: null });
vi.setSystemTime(new Date("2026-05-20T12:07:00.000Z"));
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>() });
await manager.recoverOrphanedExecutions();
await manager.recoverOrphanedExecutions();
expect(await orphanEvents(id)).toHaveLength(2);
manager.stop();
});
});

236
packages/engine/src/__tests__/self-healing.test.ts
View File

@@ -5379,236 +5379,124 @@ describe("SelfHealingManager", () => {
});
describe("recoverOrphanedExecutions", () => {
it("requeues in-progress tasks whose reserved worktree is missing", async () => {
const expectNoMutation = () => {
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.logEntry).not.toHaveBeenCalled();
};
it("emits no-action audit for missing worktree candidates past grace", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const recoverAbandonedLease = vi.fn();
const reconcileLeaseRow = vi.fn();
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
leaseManager: { recoverAbandonedLease, reconcileLeaseRow } as any,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-200",
column: "in-progress",
paused: false,
worktree: undefined,
branch: undefined,
steps: [{ status: "in-progress" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(1);
expect(store.updateTask).toHaveBeenCalledWith("FN-200", {
status: "stuck-killed",
worktree: null,
branch: null,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-200",
"Auto-recovered orphaned executor task — missing worktree/session, moved back to todo",
);
expect(store.moveTask).toHaveBeenCalledWith("FN-200", "todo", { preserveProgress: true });
managerWithRecovery.stop();
});
it("skips orphan recovery for actively executing tasks", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set(["FN-201"]));
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-201",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/missing-tree",
steps: [{ status: "in-progress" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expectNoMutation();
expect(recoverAbandonedLease).not.toHaveBeenCalled();
expect(reconcileLeaseRow).not.toHaveBeenCalled();
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
mutationType: "task:orphan-detected-no-action",
target: "FN-200",
metadata: expect.objectContaining({ reason: "missing-worktree-or-session" }),
}));
managerWithRecovery.stop();
});
it("skips tasks that are already complete", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-202",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/missing-tree",
steps: [{ status: "done" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("skips tasks still within the grace window", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-203",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/missing-tree",
steps: [{ status: "in-progress" }],
updatedAt: "2026-01-01T00:04:30.000Z",
},
]);
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("recovers tasks with existing worktree but no active session after grace period", async () => {
it("emits no-action audit for existing worktree candidates past grace", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const mockedExistsSync = vi.mocked(existsSync);
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-210",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/active-tree",
steps: [{ status: "done" }, { status: "in-progress" }, { status: "pending" }],
steps: [{ status: "done" }, { status: "in-progress" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
// Worktree directory exists on disk
mockedExistsSync.mockImplementation((p) =>
p === "/tmp/test-project/.worktrees/active-tree" ? true : false,
);
// 10 minutes past — well beyond the 5-minute grace period
mockedExistsSync.mockImplementation((p) => p === "/tmp/test-project/.worktrees/active-tree");
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(1);
expect(store.updateTask).toHaveBeenCalledWith("FN-210", {
status: "stuck-killed",
worktree: null,
branch: null,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-210",
expect.stringContaining("worktree exists but no active session"),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-210", "todo", { preserveProgress: true });
managerWithRecovery.stop();
});
it("skips tasks with existing worktree within the extended grace period", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const mockedExistsSync = vi.mocked(existsSync);
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-211",
column: "in-progress",
paused: false,
worktree: "/tmp/test-project/.worktrees/active-tree",
steps: [{ status: "in-progress" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
]);
mockedExistsSync.mockImplementation((p) =>
p === "/tmp/test-project/.worktrees/active-tree" ? true : false,
);
// Only 2 minutes past — within the 5-minute grace period for existing worktrees
vi.setSystemTime(new Date("2026-01-01T00:02:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expectNoMutation();
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
mutationType: "task:orphan-detected-no-action",
target: "FN-210",
metadata: expect.objectContaining({ reason: "worktree-exists-no-active-session" }),
}));
managerWithRecovery.stop();
});
it("reconciles lease state once when abandoned-lease recovery returns false", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const reconcileLeaseRow = vi.fn().mockResolvedValue(false);
it("skips within grace, executing, paused, and complete candidates", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set(["FN-201"]));
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
leaseManager: {
recoverAbandonedLease: vi.fn().mockResolvedValue(false),
reconcileLeaseRow,
} as any,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-212",
column: "in-progress",
paused: false,
checkedOutBy: "agent-1",
worktree: undefined,
steps: [{ status: "in-progress" }],
updatedAt: "2026-01-01T00:00:00.000Z",
},
{ id: "FN-201", column: "in-progress", paused: false, worktree: undefined, steps: [{ status: "in-progress" }], updatedAt: "2026-01-01T00:00:00.000Z" },
{ id: "FN-202", column: "in-progress", paused: true, worktree: undefined, steps: [{ status: "in-progress" }], updatedAt: "2026-01-01T00:00:00.000Z" },
{ id: "FN-203", column: "in-progress", paused: false, worktree: undefined, steps: [{ status: "done" }], updatedAt: "2026-01-01T00:00:00.000Z" },
{ id: "FN-204", column: "in-progress", paused: false, worktree: undefined, steps: [{ status: "in-progress" }], updatedAt: "2026-01-01T00:04:30.000Z" },
]);
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(1);
expect(reconcileLeaseRow).toHaveBeenCalledTimes(1);
expect(reconcileLeaseRow).toHaveBeenCalledWith("FN-212");
expect(result).toBe(0);
expectNoMutation();
expect(store.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({
mutationType: "task:orphan-detected-no-action",
}));
managerWithRecovery.stop();
});
it("emits one audit event per candidate per sweep", async () => {
const getExecuting = vi.fn().mockReturnValue(new Set<string>());
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: getExecuting,
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{ id: "FN-220", column: "in-progress", paused: false, worktree: undefined, steps: [{ status: "in-progress" }], updatedAt: "2026-01-01T00:00:00.000Z" },
{ id: "FN-221", column: "in-progress", paused: false, worktree: undefined, steps: [{ status: "in-progress" }], updatedAt: "2026-01-01T00:00:00.000Z" },
]);
vi.setSystemTime(new Date("2026-01-01T00:05:00.000Z"));
const result = await managerWithRecovery.recoverOrphanedExecutions();
expect(result).toBe(0);
const orphanAudits = (store.recordRunAuditEvent as ReturnType<typeof vi.fn>).mock.calls.filter(
([arg]) => arg?.mutationType === "task:orphan-detected-no-action",
);
expect(orphanAudits).toHaveLength(2);
expectNoMutation();
managerWithRecovery.stop();
});
});

1
packages/engine/src/run-audit.ts
View File

@@ -245,6 +245,7 @@ export type DatabaseMutationType =
| "task:auto-recover-completion-handoff-limbo-exhausted"
| "task:auto-recover-worktree-session-exhausted"
| "task:auto-recover-in-progress-limbo"
| "task:orphan-detected-no-action"
/** Metadata: { taskId: string; ignoredStepUpdateCount: number; stuckKillStreak: number; lastReason: "no-progress-churn" } */
| "task:stuck-no-progress-churn-terminalized"
| "task:auto-recover-starved-refinement"

75
packages/engine/src/self-healing.ts
View File

@@ -5813,6 +5813,14 @@ export class SelfHealingManager {
}
}
/**
* FN-5337 contract: observation-only. Emits `task:orphan-detected-no-action`
* for row-metadata orphan candidates but never moves tasks backward.
* Proof-based recovery belongs to recoverInProgressLimbo (FN-5219),
* RestartRecoveryCoordinator, and recoverMissingWorktreeReviewFailures.
* Do not reintroduce lifecycle mutation here without hard git/session proof
* and explicit CEO+CTO+PM sign-off.
*/
async recoverOrphanedExecutions(): Promise<number> {
try {
const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
@@ -5833,53 +5841,44 @@ export class SelfHealingManager {
if (orphaned.length === 0) return 0;
log.warn(`Found ${orphaned.length} orphaned executor task(s) stuck in in-progress`);
log.warn(`[orphan-detected] observed ${orphaned.length} candidate(s) — no lifecycle action taken`);
let recovered = 0;
for (const task of orphaned) {
try {
const hadWorktree = task.worktree && existsSync(task.worktree);
const hadWorktree = Boolean(task.worktree && existsSync(task.worktree));
const stalenessMs = now - new Date(task.updatedAt).getTime();
const reason = hadWorktree
? "worktree exists but no active session"
: "missing worktree/session";
? "worktree-exists-no-active-session"
: "missing-worktree-or-session";
if (this.options.leaseManager && task.checkedOutBy) {
const leaseRecovered = await this.options.leaseManager.recoverAbandonedLease(
task.id,
`orphaned execution: ${reason}`,
{ preserveProgress: true },
);
if (leaseRecovered) {
recovered++;
continue;
}
await this.options.leaseManager.reconcileLeaseRow(task.id);
}
// Reset steps whose work was never committed before clearing the worktree
await this.resetStepsIfWorkLost(task);
await this.store.updateTask(task.id, {
status: "stuck-killed",
worktree: null,
branch: null,
await createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-healing-orphan-detected", task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "recover-orphaned-executions",
}).database({
type: "task:orphan-detected-no-action",
target: task.id,
metadata: {
priorWorktree: task.worktree ?? null,
priorBranch: task.branch ?? null,
hadWorktree,
stalenessMs,
reason,
},
});
await this.store.logEntry(
task.id,
`Auto-recovered orphaned executor task — ${reason}, moved back to todo`,
);
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
recovered++;
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Failed to recover orphaned executor task ${task.id}: ${errorMessage}`);
log.log(`[orphan-detected] ${task.id}: ${reason} — no action (operator-decides)`);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Failed to annotate orphaned executor candidate ${task.id}: ${errorMessage}`);
}
}
if (recovered > 0) {
log.log(`Recovered ${recovered} orphaned executor task(s) → todo`);
}
return recovered;
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
return 0;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Orphaned executor recovery failed: ${errorMessage}`);
return 0;
}