diff --git a/.changeset/park-incomplete-stuck-loop.md b/.changeset/park-incomplete-stuck-loop.md new file mode 100644 index 0000000000..800723f870 --- /dev/null +++ b/.changeset/park-incomplete-stuck-loop.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Park incomplete tasks that exhaust stuck-loop recovery instead of making them scheduler-runnable again. diff --git a/packages/engine/src/__tests__/reliability-interactions/non-progress-churn.test.ts b/packages/engine/src/__tests__/reliability-interactions/non-progress-churn.test.ts index bc3c2d84ce..17d29c4dcc 100644 --- a/packages/engine/src/__tests__/reliability-interactions/non-progress-churn.test.ts +++ b/packages/engine/src/__tests__/reliability-interactions/non-progress-churn.test.ts @@ -190,7 +190,7 @@ describe("reliability interactions: non-progress churn", () => { manager.stop(); }); - it("re-queues incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => { + it("parks incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => { const task = baseTask({ id: "FN-5168-LOOP", stuckKillCount: 6 }); const store = createStore(task); const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" }); @@ -210,19 +210,19 @@ describe("reliability interactions: non-progress churn", () => { await detector.killAndRetry(task.id, 60_000); - expect(task.error).toBeNull(); - expect(task.status).toBe("queued"); + expect(task.error).toContain("STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget"); + expect(task.status).toBe("failed"); expect(task.column).toBe("todo"); - expect(task.paused).toBe(false); + expect(task.paused).toBe(true); // FN-6252 / Move-Task contract: engine rebounds do not write userPaused, // so a never-user-paused task remains undefined while still not user-paused. expect(task.userPaused).not.toBe(true); - expect(task.pausedReason).toBeNull(); + expect(task.pausedReason).toBe("stuck-loop-exhausted-manual-intervention-required"); expect(task.stuckKillCount).toBe(7); expect(task.steps).toEqual([{ name: "Implement", status: "in-progress" }]); - expect(task.log?.some((entry) => entry.action.includes("incomplete task exhausted stuck kill budget"))).toBe(true); + expect(task.log?.some((entry) => entry.action.includes("Parked in todo with progress preserved"))).toBe(true); expect(store.handoffToReview).not.toHaveBeenCalled(); - expect(isRunnableQueuedOverlapCandidate(task, [task])).toBe(true); + expect(isRunnableQueuedOverlapCandidate(task, [task])).toBe(false); manager.stop(); }); diff --git a/packages/engine/src/__tests__/reliability-interactions/todo-inprogress-flapping.test.ts b/packages/engine/src/__tests__/reliability-interactions/todo-inprogress-flapping.test.ts index 445d1ac490..46e821479a 100644 --- a/packages/engine/src/__tests__/reliability-interactions/todo-inprogress-flapping.test.ts +++ b/packages/engine/src/__tests__/reliability-interactions/todo-inprogress-flapping.test.ts @@ -290,7 +290,7 @@ describe("FN-5941 reliability interactions: todo/in-progress flapping", () => { manager.stop(); }); - it("still requeues a genuinely dead task when stuck-kill budget is exhausted", async () => { + it("parks a genuinely dead incomplete task when stuck-kill budget is exhausted", async () => { const task = makeTask(rootDir, { id: "FN-5941-DEAD", stuckKillCount: 6, @@ -314,7 +314,11 @@ describe("FN-5941 reliability interactions: todo/in-progress flapping", () => { })); expect(task.column).toBe("todo"); expect(task.stuckKillCount).toBe(7); - expect(task.status).toBe("queued"); + expect(task.status).toBe("failed"); + expect(task.paused).toBe(true); + expect(task.pausedReason).toBe("stuck-loop-exhausted-manual-intervention-required"); + expect(task.error).toContain("STUCK_LOOP_EXHAUSTED"); + expect(task.userPaused).not.toBe(true); manager.stop(); }); diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index a0e82fd0a1..248a9665b4 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -441,11 +441,12 @@ describe("SelfHealingManager", () => { ); }); - it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => { + it("parks incomplete stuck-loop exhaustion in todo without review handoff or automatic retry", async () => { (store.getTask as ReturnType).mockResolvedValue({ id: "FN-001", column: "in-progress", stuckKillCount: 6, + assignedAgentId: "agent-1", steps: [ { name: "Preflight", status: "done" }, { name: "Delivery", status: "in-progress" }, @@ -457,23 +458,32 @@ describe("SelfHealingManager", () => { const result = await manager.checkStuckBudget("FN-001", "loop"); expect(result).toBe(false); - expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 }); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({ + stuckKillCount: 7, + status: "failed", + error: expect.stringContaining("STUCK_LOOP_EXHAUSTED"), + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + pausedByAgentId: "self-healing", + assignedAgentId: null, + checkedOutBy: null, + checkedOutAt: null, + checkoutNodeId: null, + checkoutRunId: null, + checkoutLeaseRenewedAt: null, + checkoutLeaseEpoch: 0, + nextRecoveryAt: null, + })); expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true, preserveStatus: true, moveSource: "engine", recoveryRehome: true, }); - expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({ - stuckKillCount: 7, - paused: false, - pausedReason: null, - status: "queued", - })); expect(store.handoffToReview).not.toHaveBeenCalled(); expect(store.logEntry).toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.", ); }); @@ -509,7 +519,7 @@ describe("SelfHealingManager", () => { })); }); - it("falls back to executor requeue when todo parking fails", async () => { + it("does not fall back to executor requeue when todo parking fails", async () => { (store.getTask as ReturnType).mockResolvedValue({ id: "FN-001", column: "in-progress", @@ -525,8 +535,13 @@ describe("SelfHealingManager", () => { const result = await manager.checkStuckBudget("FN-001", "loop"); - expect(result).toBe(true); - expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 }); + expect(result).toBe(false); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({ + stuckKillCount: 7, + status: "failed", + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + })); expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true, preserveStatus: true, @@ -542,11 +557,74 @@ describe("SelfHealingManager", () => { expect(store.handoffToReview).not.toHaveBeenCalled(); expect(store.logEntry).toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); falling back to executor stuck-kill requeue.", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.", ); }); - it("logs post-move requeue patch failures without executor fallback", async () => { + it("does not requeue when in-place park succeeds but success logging fails", async () => { + (store.getTask as ReturnType).mockResolvedValue({ + id: "FN-001", + column: "in-progress", + stuckKillCount: 6, + steps: [ + { name: "Preflight", status: "done" }, + { name: "Delivery", status: "in-progress" }, + ], + } as unknown as Task); + (store.moveTask as ReturnType).mockRejectedValueOnce(new Error("database is busy")); + (store.logEntry as ReturnType).mockRejectedValueOnce(new Error("log unavailable")); + + manager.start(); + + const result = await manager.checkStuckBudget("FN-001", "loop"); + + expect(result).toBe(false); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({ + stuckKillCount: 7, + status: "failed", + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + })); + expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", expect.objectContaining({ + paused: false, + status: "queued", + })); + expect(store.handoffToReview).not.toHaveBeenCalled(); + }); + + it("logs in-place park patch failures after todo move failure without executor fallback", async () => { + (store.getTask as ReturnType).mockResolvedValue({ + id: "FN-001", + column: "in-progress", + stuckKillCount: 6, + steps: [ + { name: "Preflight", status: "done" }, + { name: "Delivery", status: "in-progress" }, + ], + } as unknown as Task); + (store.updateTask as ReturnType) + .mockResolvedValueOnce({} as Task) + .mockRejectedValueOnce(new Error("write conflict")); + (store.moveTask as ReturnType).mockRejectedValueOnce(new Error("database is busy")); + + manager.start(); + + const result = await manager.checkStuckBudget("FN-001", "loop"); + + expect(result).toBe(false); + expect(store.updateTask).toHaveBeenCalledTimes(2); + expect(store.handoffToReview).not.toHaveBeenCalled(); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-001", + "STUCK_LOOP_EXHAUSTED: incomplete task failed to move to todo (database is busy), and the in-place park patch also failed (write conflict); pre-move park metadata was already applied, but operator verification is required before retry.", + ); + expect(store.logEntry).not.toHaveBeenCalledWith( + "FN-001", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.", + ); + }); + + it("logs post-move park patch failures without executor fallback", async () => { (store.getTask as ReturnType).mockResolvedValue({ id: "FN-001", column: "in-progress", @@ -573,11 +651,11 @@ describe("SelfHealingManager", () => { }); expect(store.logEntry).toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (write conflict); scheduler retry may wait for the next state repair pass.", + "STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (write conflict); operator repair is required before retry.", ); - expect(store.logEntry).toHaveBeenCalledWith( + expect(store.logEntry).not.toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.", ); expect(store.handoffToReview).not.toHaveBeenCalled(); }); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 3c359e0322..4efffa2e1c 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -1211,9 +1211,13 @@ export class SelfHealingManager { * Terminal contract for stuck-loop exhaustion and no-progress churn: * - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted. Once * exhausted, tasks with incomplete steps are moved back to `todo` with - * progress preserved and pause metadata reapplied for manual resume or + * progress preserved, marked failed, and paused for manual resume or * decomposition; tasks with only terminal steps keep the legacy failed * `in-review` handoff path. + * + * FNXC:SelfHealing 2026-06-14-10:51: + * Incomplete stuck-loop exhaustion must park work in a failed/paused state before moving columns, because a post-move patch failure must not leave the task scheduler-runnable. + * Engine-owned recovery must not mutate `userPaused`; user intent stays authoritative across races. * - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on * the first trigger with operator guidance to decompose or rescope. * @@ -1315,8 +1319,26 @@ export class SelfHealingManager { return false; } - log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — re-queueing in todo with progress preserved`); - await this.store.updateTask(taskId, { stuckKillCount: newCount }); + log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — parking in todo with progress preserved`); + const exhaustedError = `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}) after last reason=${reason}. Progress was preserved; manually retry, decompose, or rescope before execution resumes.`; + const parkUpdate = { + stuckKillCount: newCount, + status: "failed", + error: exhaustedError, + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + pausedByAgentId: "self-healing", + assignedAgentId: null, + checkedOutBy: null, + checkedOutAt: null, + checkoutNodeId: null, + checkoutRunId: null, + checkoutLeaseRenewedAt: null, + checkoutLeaseEpoch: 0, + nextRecoveryAt: null, + } satisfies Parameters[1]; + + await this.store.updateTask(taskId, parkUpdate); try { await this.store.moveTask(taskId, "todo", { preserveProgress: true, @@ -1327,34 +1349,44 @@ export class SelfHealingManager { }); } catch (moveErr: unknown) { const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr); - log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — falling back to executor stuck-kill requeue`); - await this.store.logEntry( - taskId, - `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); falling back to executor stuck-kill requeue.`, - ); - return true; + log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — marking failed/paused in place`); + try { + await this.store.updateTask(taskId, parkUpdate); + } catch (patchErr: unknown) { + const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr); + log.warn(`${taskId} in-place park patch failed after moveTask(todo) failure during incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`); + await this.store.logEntry( + taskId, + `STUCK_LOOP_EXHAUSTED: incomplete task failed to move to todo (${moveErrMessage}), and the in-place park patch also failed (${patchErrMessage}); pre-move park metadata was already applied, but operator verification is required before retry.`, + ); + return false; + } + try { + await this.store.logEntry( + taskId, + `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task was marked failed/paused in place and will not be automatically retried.`, + ); + } catch (logErr: unknown) { + const logErrMessage = logErr instanceof Error ? logErr.message : String(logErr); + log.warn(`${taskId} failed to log in-place stuck-loop park success after moveTask(todo) failure: ${logErrMessage}`); + } + return false; } - const requeueUpdate = { - stuckKillCount: newCount, - paused: false, - pausedReason: null, - status: "queued", - } satisfies Parameters[1]; try { - await this.store.updateTask(taskId, requeueUpdate); - } catch (patchErr: unknown) { - const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr); - log.warn(`${taskId} post-move requeue patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`); + await this.store.updateTask(taskId, parkUpdate); await this.store.logEntry( taskId, - `STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (${patchErrMessage}); scheduler retry may wait for the next state repair pass.`, + `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.`, + ); + } catch (patchErr: unknown) { + const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr); + log.warn(`${taskId} post-move park patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`); + await this.store.logEntry( + taskId, + `STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (${patchErrMessage}); operator repair is required before retry.`, ); } - await this.store.logEntry( - taskId, - `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.`, - ); return false; }