feat(FN-3962): align fn_task_done recovery detection in self-healing
Refined self-healing recovery logic for tasks that lack a `fn_task_done` signal, aligned detection in the engine with matching test coverage and architecture documentation for the recovery behavior. Fusion-Task-Id: FN-3962
This commit is contained in:
@@ -587,6 +587,7 @@ Runtime action-gate flow (v1):
|
||||
- `recoverMissingWorktreeReviewFailures()` is a narrow failed-review recovery: only `status: "failed"` `in-review` tasks with the explicit session-start signature `Refusing to start coding agent in missing worktree:` (from `assertValidWorktreeSession()`) are requeued. Recovery clears stale session metadata (`worktree`, `branch`, `sessionFile`, transient failure state), preserves valid step progress/retry counters, logs the auto-recovery reason, and moves the task back to `todo` for a clean retry.
|
||||
- `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`.
|
||||
- `recoverAlreadyMergedReviewTasks()` auto-finalizes retry-exhausted `in-review` tasks when self-healing can prove their work already landed on the merge target, preventing deadlocked cards from remaining indefinitely in failed review state.
|
||||
- No-`fn_task_done` recovery classification is normalized across executor, restart recovery, and self-healing: detection keys on executor-emitted `"without calling fn_task_done"` strings (while still tolerating legacy `task_done` wording), then applies the bounded ladder deterministically (in-session retries → bounded todo requeues with preserved progress when appropriate → terminal surfaced failure when budget is exhausted).
|
||||
- `clearStaleBlockedBy()` clears `blockedBy` (and transient `status`) on todo tasks when their blocker is missing, done, archived, paused in-review, or failed in-review with merge retries exhausted. FN-3924 extends this with a dependency-integrity guard: if a task has explicit dependencies and `blockedBy` is not one of the currently unresolved deps, the stale marker is cleared. This repairs rows corrupted by historical overlap re-stamping and lets scheduler re-evaluate from live dependency state.
|
||||
- Together, `recoverAlreadyMergedReviewTasks()`, `clearStaleBlockedBy()`, and paused-aware in-review scheduling prevent merge-deadlock loops by finalizing already-landed work, clearing stale dependency blockers, and avoiding paused review cards re-blocking overlap dispatch.
|
||||
- Merge commit attribution is ownership-aware: a `mergeDetails.commitSha` is trusted only when reachable from `HEAD` **and** attributable to the task via `Fusion-Task-Id` trailer or task-ID-bearing subject. Reachable-but-unowned SHAs are rejected to prevent sibling done tasks from sharing misleading merge metadata.
|
||||
|
||||
@@ -789,7 +789,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-1473",
|
||||
column: "in-progress",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
steps: [],
|
||||
},
|
||||
@@ -825,7 +825,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-1473",
|
||||
column: "in-progress",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
steps: [{ status: "done" }, { status: "pending" }],
|
||||
},
|
||||
@@ -852,7 +852,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-1473",
|
||||
column: "in-progress",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
steps: [{ status: "pending" }],
|
||||
},
|
||||
@@ -1415,7 +1415,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-300",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
steps: [{ status: "done" }, { status: "done" }, { status: "skipped" }],
|
||||
log: [],
|
||||
},
|
||||
@@ -1446,7 +1446,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-301",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
steps: [{ status: "done" }, { status: "in-progress" }],
|
||||
log: [],
|
||||
},
|
||||
@@ -1494,7 +1494,7 @@ describe("SelfHealingManager", () => {
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
paused: true,
|
||||
error: "Agent finished without calling task_done",
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
steps: [{ status: "done" }, { status: "done" }],
|
||||
log: [],
|
||||
},
|
||||
@@ -1520,7 +1520,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-2164",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
steps: [{ status: "done" }, { status: "pending" }, { status: "pending" }],
|
||||
log: [],
|
||||
@@ -1556,7 +1556,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-2164",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
taskDoneRetryCount: 3,
|
||||
steps: [{ status: "done" }, { status: "pending" }],
|
||||
@@ -1583,7 +1583,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-2164",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
steps: [{ status: "done" }, { status: "done" }, { status: "skipped" }],
|
||||
log: [],
|
||||
@@ -1608,7 +1608,7 @@ describe("SelfHealingManager", () => {
|
||||
id: "FN-2164",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "Agent finished without calling task_done (after retry)",
|
||||
error: "Agent finished without calling fn_task_done (after retry)",
|
||||
paused: false,
|
||||
steps: [{ status: "pending" }, { status: "pending" }],
|
||||
log: [],
|
||||
|
||||
@@ -117,7 +117,7 @@ const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
|
||||
|
||||
/**
|
||||
* Maximum times a task can be auto-requeued after the agent exits without
|
||||
* calling `task_done`. Bounded so a persistently-broken task cannot loop
|
||||
* calling `fn_task_done`. Bounded so a persistently-broken task cannot loop
|
||||
* forever; when exhausted the task stays in `in-review` for human inspection.
|
||||
*/
|
||||
const MAX_TASK_DONE_RETRIES = 3;
|
||||
@@ -1901,7 +1901,7 @@ export class SelfHealingManager {
|
||||
/**
|
||||
* Recover tasks in `in-review` marked as `failed` where all steps are
|
||||
* actually done. This catches the case where an agent completed all work
|
||||
* but the session ended without calling `task_done` (e.g., context
|
||||
* but the session ended without calling `fn_task_done` (e.g., context
|
||||
* overflow, compaction losing tool awareness). The executor marks these
|
||||
* as failed, but the work is complete — clear the error so the normal
|
||||
* review flow can proceed.
|
||||
@@ -1916,7 +1916,7 @@ export class SelfHealingManager {
|
||||
t.column === "in-review" &&
|
||||
!t.paused &&
|
||||
t.status === "failed" &&
|
||||
t.error?.includes("without calling task_done") &&
|
||||
isNoTaskDoneFailure(t) &&
|
||||
t.steps.length > 0 &&
|
||||
t.steps.every((s) => s.status === "done" || s.status === "skipped"),
|
||||
);
|
||||
@@ -1934,7 +1934,7 @@ export class SelfHealingManager {
|
||||
});
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
"Auto-recovered: all steps complete despite 'no task_done' failure — cleared error for normal review",
|
||||
"Auto-recovered: all steps complete despite 'no fn_task_done' failure — cleared error for normal review",
|
||||
);
|
||||
log.log(`Recovered misclassified failure ${task.id}: ${task.title || task.description?.slice(0, 60) || "(untitled)"}`);
|
||||
recovered++;
|
||||
@@ -2199,7 +2199,7 @@ export class SelfHealingManager {
|
||||
|
||||
/**
|
||||
* Recover `in-progress` tasks that failed only because the agent exited
|
||||
* without calling task_done, and where there is no sign of work to preserve.
|
||||
* without calling fn_task_done, and where there is no sign of work to preserve.
|
||||
*
|
||||
* These are safe to requeue automatically when no steps progressed and git
|
||||
* has neither worktree changes nor branch commits. Cases with any evidence
|
||||
@@ -2312,7 +2312,7 @@ export class SelfHealingManager {
|
||||
|
||||
/**
|
||||
* Recover `in-review` tasks marked as `failed` because the agent exited
|
||||
* without calling `task_done` *with partial step progress* (some steps done,
|
||||
* without calling `fn_task_done` *with partial step progress* (some steps done,
|
||||
* some still pending). The work-in-progress is valuable but incomplete —
|
||||
* the existing worktree and branch are preserved and the task is moved back
|
||||
* to `todo` so the scheduler re-dispatches it for a fresh execution that
|
||||
@@ -2362,7 +2362,7 @@ export class SelfHealingManager {
|
||||
});
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Auto-retry ${nextCount}/${MAX_TASK_DONE_RETRIES}: agent finished without task_done — requeuing to todo to resume partial work`,
|
||||
`Auto-retry ${nextCount}/${MAX_TASK_DONE_RETRIES}: agent finished without fn_task_done — requeuing to todo to resume partial work`,
|
||||
);
|
||||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||||
recovered++;
|
||||
@@ -2810,7 +2810,8 @@ function isTaskWorkComplete(task: Task): boolean {
|
||||
}
|
||||
|
||||
function isNoTaskDoneFailure(task: Task): boolean {
|
||||
return task.error?.includes("without calling task_done") === true;
|
||||
const error = task.error?.toLowerCase() ?? "";
|
||||
return error.includes("without calling fn_task_done") || error.includes("without calling task_done");
|
||||
}
|
||||
|
||||
function hasStepProgress(task: Task): boolean {
|
||||
|
||||
Reference in New Issue
Block a user