fix(FN-4806): silently recover when worktree/branch reclaimed mid-retry
The executor's no-fn_task_done retry loop has two reclaim signals (retryAbortedDueToReclaim=true): (1) pre-retry liveness recheck where the task DB shows worktree/branch was cleared, and (2) session-start failure because the worktree path no longer exists. Both are engine self-heal situations triggered by FN-4546 stale-active-branch reclaim, FN-4742 self-healing removals, or related housekeeping paths — the agent never got a fair retry attempt. Previously these surfaced as task status=failed with error 'Worktree/branch reclaimed during no-fn_task_done retry — requeueing', fired onError, and burned the taskDoneRetryCount budget. Three legitimate problems followed: tasks accumulated spurious failures in the UI, the exhausted-budget branch escalated reclaimed tasks to in-review instead of retrying, and the noise masked the underlying worktree-removal regression (FN-4811). Now the reclaim branch silently: - clears stale worktree/branch metadata so the next pickup creates a fresh worktree - requeues to todo with preserveProgress - logs an informational 'engine self-heal, no failure' line - does NOT set status=failed, does NOT bump taskDoneRetryCount, does NOT call onError The genuine 'agent finished without calling fn_task_done after N retries' exhaustion path (retryAbortedDueToReclaim=false) is unchanged. Tests updated in packages/engine/src/__tests__/reliability-interactions/executor-no-task-done-vs-worktree-reclaim.test.ts to assert the new silent-recovery contract on all three reclaim paths. Fusion-Task-Id: FN-4806
This commit is contained in:
7
.changeset/FN-4806-silent-reclaim-recovery.md
Normal file
7
.changeset/FN-4806-silent-reclaim-recovery.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
fix(FN-4806): silently recover when worktree/branch reclaimed mid-retry
|
||||||
|
|
||||||
|
When the executor's no-`fn_task_done` retry loop detects that a task's worktree or branch was reclaimed by an engine-side housekeeping path (FN-4546 stale-active-branch reclaim, FN-4742 self-healing removals, session-start unusable-worktree), it now requeues the task to `todo` silently with preserved progress. The task is no longer marked `failed`, `taskDoneRetryCount` is no longer burned, and `onError` is no longer surfaced — this is engine self-heal, not an agent failure. The genuine "agent finished without calling fn_task_done after N retries" exhaustion path is unchanged.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"id": "fusion-plugin-dependency-graph",
|
||||||
|
"name": "Dependency Graph",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"description": "Top-level dependency graph dashboard view",
|
||||||
|
"dashboardViews": [
|
||||||
|
{
|
||||||
|
"viewId": "graph",
|
||||||
|
"label": "Graph",
|
||||||
|
"componentPath": "./dashboard-view",
|
||||||
|
"icon": "Network",
|
||||||
|
"placement": "more",
|
||||||
|
"order": 40
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -39,7 +39,7 @@ describe("reliability interactions: executor no-fn_task_done vs worktree reclaim
|
|||||||
resetExecutorMocks();
|
resetExecutorMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("pre-retry liveness recheck aborts retry and requeues", async () => {
|
it("pre-retry liveness recheck aborts retry and silently requeues (FN-4806)", async () => {
|
||||||
const store = createMockStore();
|
const store = createMockStore();
|
||||||
const state = makeTask();
|
const state = makeTask();
|
||||||
let getTaskCalls = 0;
|
let getTaskCalls = 0;
|
||||||
@@ -58,14 +58,29 @@ describe("reliability interactions: executor no-fn_task_done vs worktree reclaim
|
|||||||
await executor.execute(state);
|
await executor.execute(state);
|
||||||
|
|
||||||
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
|
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
|
||||||
|
// FN-4806: silent requeue — task goes to todo with preserveProgress, no failed status,
|
||||||
|
// no taskDoneRetryCount burn, no onError surface.
|
||||||
expect(store.moveTask).toHaveBeenCalledWith("FN-4601", "todo", { preserveProgress: true });
|
expect(store.moveTask).toHaveBeenCalledWith("FN-4601", "todo", { preserveProgress: true });
|
||||||
expect(store.logEntry).toHaveBeenCalledWith(
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
"FN-4601",
|
"FN-4601",
|
||||||
expect.stringContaining("worktree/branch reclaimed during no-fn_task_done retry"),
|
expect.stringContaining("engine self-heal, no failure"),
|
||||||
undefined,
|
undefined,
|
||||||
expect.any(Object),
|
expect.any(Object),
|
||||||
);
|
);
|
||||||
expect(store.updateTask).toHaveBeenCalledWith("FN-4601", expect.objectContaining({ taskDoneRetryCount: 1 }));
|
// Reclaim path must NOT mark task failed and must NOT burn taskDoneRetryCount budget.
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||||
|
"FN-4601",
|
||||||
|
expect.objectContaining({ status: "failed" }),
|
||||||
|
);
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||||
|
"FN-4601",
|
||||||
|
expect.objectContaining({ taskDoneRetryCount: expect.any(Number) }),
|
||||||
|
);
|
||||||
|
// Stale binding must be cleared so the next pickup creates a fresh worktree.
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith(
|
||||||
|
"FN-4601",
|
||||||
|
expect.objectContaining({ worktree: null, branch: null }),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("missing-worktree session-start error during retry clears metadata and requeues", async () => {
|
it("missing-worktree session-start error during retry clears metadata and requeues", async () => {
|
||||||
@@ -87,7 +102,16 @@ describe("reliability interactions: executor no-fn_task_done vs worktree reclaim
|
|||||||
baseCommitSha: null,
|
baseCommitSha: null,
|
||||||
});
|
});
|
||||||
expect(store.moveTask).toHaveBeenCalledWith("FN-4601", "todo", { preserveProgress: true });
|
expect(store.moveTask).toHaveBeenCalledWith("FN-4601", "todo", { preserveProgress: true });
|
||||||
expect(store.updateTask).toHaveBeenCalledWith("FN-4601", expect.objectContaining({ taskDoneRetryCount: 1 }));
|
// FN-4806: session-start missing-worktree is engine self-heal, must not burn retry budget
|
||||||
|
// and must not mark the task failed.
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||||
|
"FN-4601",
|
||||||
|
expect.objectContaining({ status: "failed" }),
|
||||||
|
);
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||||
|
"FN-4601",
|
||||||
|
expect.objectContaining({ taskDoneRetryCount: expect.any(Number) }),
|
||||||
|
);
|
||||||
expect(store.moveTask).not.toHaveBeenCalledWith("FN-4601", "in-review");
|
expect(store.moveTask).not.toHaveBeenCalledWith("FN-4601", "in-review");
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -107,7 +131,10 @@ describe("reliability interactions: executor no-fn_task_done vs worktree reclaim
|
|||||||
expect(store.updateTask).not.toHaveBeenCalledWith("FN-4601", expect.objectContaining({ baseCommitSha: null, worktree: null, branch: null }));
|
expect(store.updateTask).not.toHaveBeenCalledWith("FN-4601", expect.objectContaining({ baseCommitSha: null, worktree: null, branch: null }));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("exhausted requeue budget keeps reclaim path in in-review", async () => {
|
it("reclaim path ignores requeue budget and always silently requeues (FN-4806)", async () => {
|
||||||
|
// FN-4806: reclaim is engine self-heal, not an agent failure, so it must not be subject to
|
||||||
|
// the no-fn_task_done requeue cap. Even at the previously-exhausted budget the task must
|
||||||
|
// still go silently to todo, not in-review.
|
||||||
const store = createMockStore();
|
const store = createMockStore();
|
||||||
const state = makeTask({ taskDoneRetryCount: 3 });
|
const state = makeTask({ taskDoneRetryCount: 3 });
|
||||||
let getTaskCalls = 0;
|
let getTaskCalls = 0;
|
||||||
@@ -125,7 +152,11 @@ describe("reliability interactions: executor no-fn_task_done vs worktree reclaim
|
|||||||
const executor = new TaskExecutor(store as any, "/tmp/test");
|
const executor = new TaskExecutor(store as any, "/tmp/test");
|
||||||
await executor.execute(state);
|
await executor.execute(state);
|
||||||
|
|
||||||
expect(store.moveTask).toHaveBeenCalledWith("FN-4601", "in-review");
|
expect(store.moveTask).toHaveBeenCalledWith("FN-4601", "todo", { preserveProgress: true });
|
||||||
expect(store.moveTask).not.toHaveBeenCalledWith("FN-4601", "todo", { preserveProgress: true });
|
expect(store.moveTask).not.toHaveBeenCalledWith("FN-4601", "in-review");
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||||
|
"FN-4601",
|
||||||
|
expect.objectContaining({ status: "failed" }),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -4032,12 +4032,28 @@ export class TaskExecutor {
|
|||||||
this.clearCompletedTaskWatchdog(task.id);
|
this.clearCompletedTaskWatchdog(task.id);
|
||||||
executorLog.log(`✓ ${task.id} completed on retry → in-review`);
|
executorLog.log(`✓ ${task.id} completed on retry → in-review`);
|
||||||
this.options.onComplete?.(task);
|
this.options.onComplete?.(task);
|
||||||
|
} else if (retryAbortedDueToReclaim) {
|
||||||
|
// FN-4806: Worktree/branch was reclaimed mid-retry by an engine-side housekeeping path
|
||||||
|
// (e.g. FN-4546 stale-active-branch reclaim, FN-4742 self-healing removals). This is NOT
|
||||||
|
// an agent failure — the agent never got a fair retry attempt. Silently requeue to todo
|
||||||
|
// with preserved progress so a fresh worktree is created on next pickup. Do not mark
|
||||||
|
// status=failed, do not surface onError, do not burn taskDoneRetryCount budget.
|
||||||
|
const silentMessage = `${task.id}: worktree/branch reclaimed mid-retry — requeued to todo (engine self-heal, no failure)`;
|
||||||
|
await this.store.logEntry(
|
||||||
|
task.id,
|
||||||
|
"Worktree/branch reclaimed mid-retry — requeued to todo (engine self-heal, no failure)",
|
||||||
|
undefined,
|
||||||
|
this.currentRunContext,
|
||||||
|
);
|
||||||
|
// Clear any stale binding so the next pickup creates a fresh worktree.
|
||||||
|
await this.store.updateTask(task.id, { worktree: null, branch: null });
|
||||||
|
await this.persistTokenUsage(task.id);
|
||||||
|
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||||||
|
executorLog.log(silentMessage);
|
||||||
} else {
|
} else {
|
||||||
const priorRequeues = task.taskDoneRetryCount ?? 0;
|
const priorRequeues = task.taskDoneRetryCount ?? 0;
|
||||||
const nextRequeueCount = priorRequeues + 1;
|
const nextRequeueCount = priorRequeues + 1;
|
||||||
const errorMessage = retryAbortedDueToReclaim
|
const errorMessage = `Agent finished without calling fn_task_done (after ${MAX_TASK_DONE_SESSION_RETRIES} retries)`;
|
||||||
? "Worktree/branch reclaimed during no-fn_task_done retry — requeueing"
|
|
||||||
: `Agent finished without calling fn_task_done (after ${MAX_TASK_DONE_SESSION_RETRIES} retries)`;
|
|
||||||
|
|
||||||
if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) {
|
if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) {
|
||||||
await this.store.updateTask(task.id, {
|
await this.store.updateTask(task.id, {
|
||||||
|
|||||||
Reference in New Issue
Block a user