fix(FN-1290): recover orphaned stuck tasks

This commit is contained in:
gsxdsm
2026-04-08 20:22:58 -07:00
parent a9bbb25fd0
commit f2829c7634
5 changed files with 229 additions and 4 deletions

View File

@@ -53,6 +53,7 @@ export interface SelfHealingOptions {
}
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
const ORPHANED_EXECUTION_RECOVERY_GRACE_MS = 60_000;
export class SelfHealingManager {
// ── Auto-unpause state ──────────────────────────────────────────────
@@ -264,6 +265,7 @@ export class SelfHealingManager {
this.checkpointWal();
await this.enforceWorktreeCap();
await this.recoverCompletedTasks();
await this.recoverOrphanedExecutions();
await this.recoverApprovedTriageTasks();
const elapsedMs = Date.now() - startMs;
@@ -322,6 +324,59 @@ export class SelfHealingManager {
}
}
/**
* Recover executor tasks stranded in `in-progress` before a real session was
* established, typically when the scheduler reserved a worktree path but the
* executor never materialized it or crashed before tracking the run.
*/
async recoverOrphanedExecutions(): Promise<number> {
try {
const tasks = await this.store.listTasks();
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
const now = Date.now();
const orphaned = tasks.filter((t) =>
t.column === "in-progress" &&
!t.paused &&
!executingIds.has(t.id) &&
!isTaskWorkComplete(t) &&
(!t.worktree || !existsSync(t.worktree)) &&
now - new Date(t.updatedAt).getTime() >= ORPHANED_EXECUTION_RECOVERY_GRACE_MS,
);
if (orphaned.length === 0) return 0;
log.warn(`Found ${orphaned.length} orphaned executor task(s) stuck in in-progress`);
let recovered = 0;
for (const task of orphaned) {
try {
await this.store.updateTask(task.id, {
status: "stuck-killed",
worktree: null,
branch: null,
});
await this.store.logEntry(
task.id,
"Auto-recovered orphaned executor task — missing worktree/session, moved back to todo",
);
await this.store.moveTask(task.id, "todo");
recovered++;
} catch (err: any) {
log.error(`Failed to recover orphaned executor task ${task.id}: ${err.message}`);
}
}
if (recovered > 0) {
log.log(`Recovered ${recovered} orphaned executor task(s) → todo`);
}
return recovered;
} catch (err: any) {
log.error(`Orphaned executor recovery failed: ${err.message}`);
return 0;
}
}
/**
* Recover triage tasks that already have an approved specification but were
* left stuck in `status: "specifying"` without an active triage session.
@@ -549,3 +604,8 @@ function hasLatestSpecReviewApproval(task: Task): boolean {
}
return false;
}
function isTaskWorkComplete(task: Task): boolean {
if (task.steps.length === 0) return false;
return task.steps.every((step) => step.status === "done" || step.status === "skipped");
}