From 19eca3d74b7067f4ceccf62568f099bb800d0b78 Mon Sep 17 00:00:00 2001 From: Phil Larson Date: Sun, 14 Jun 2026 10:25:16 -0700 Subject: [PATCH] fix(engine): park incomplete stuck-loop exhaustion Preserve incomplete task progress after stuck-kill budget exhaustion while marking the task failed and paused for manual intervention instead of making it scheduler-runnable again. --- .changeset/park-incomplete-stuck-loop.md | 5 ++ .../engine/src/__tests__/self-healing.test.ts | 49 ++++++++++------ packages/engine/src/self-healing.ts | 57 ++++++++++++------- 3 files changed, 73 insertions(+), 38 deletions(-) create mode 100644 .changeset/park-incomplete-stuck-loop.md diff --git a/.changeset/park-incomplete-stuck-loop.md b/.changeset/park-incomplete-stuck-loop.md new file mode 100644 index 0000000000..800723f870 --- /dev/null +++ b/.changeset/park-incomplete-stuck-loop.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Park incomplete tasks that exhaust stuck-loop recovery instead of making them scheduler-runnable again. diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index a0e82fd0a1..6c88f45ee8 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -441,11 +441,12 @@ describe("SelfHealingManager", () => { ); }); - it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => { + it("parks incomplete stuck-loop exhaustion in todo without review handoff or automatic retry", async () => { (store.getTask as ReturnType).mockResolvedValue({ id: "FN-001", column: "in-progress", stuckKillCount: 6, + assignedAgentId: "agent-1", steps: [ { name: "Preflight", status: "done" }, { name: "Delivery", status: "in-progress" }, @@ -457,23 +458,32 @@ describe("SelfHealingManager", () => { const result = await manager.checkStuckBudget("FN-001", "loop"); expect(result).toBe(false); - expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 }); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({ + stuckKillCount: 7, + status: "failed", + error: expect.stringContaining("STUCK_LOOP_EXHAUSTED"), + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + pausedByAgentId: "self-healing", + assignedAgentId: null, + checkedOutBy: null, + checkedOutAt: null, + checkoutNodeId: null, + checkoutRunId: null, + checkoutLeaseRenewedAt: null, + checkoutLeaseEpoch: 0, + nextRecoveryAt: null, + })); expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true, preserveStatus: true, moveSource: "engine", recoveryRehome: true, }); - expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({ - stuckKillCount: 7, - paused: false, - pausedReason: null, - status: "queued", - })); expect(store.handoffToReview).not.toHaveBeenCalled(); expect(store.logEntry).toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.", ); }); @@ -509,7 +519,7 @@ describe("SelfHealingManager", () => { })); }); - it("falls back to executor requeue when todo parking fails", async () => { + it("does not fall back to executor requeue when todo parking fails", async () => { (store.getTask as ReturnType).mockResolvedValue({ id: "FN-001", column: "in-progress", @@ -525,8 +535,13 @@ describe("SelfHealingManager", () => { const result = await manager.checkStuckBudget("FN-001", "loop"); - expect(result).toBe(true); - expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 }); + expect(result).toBe(false); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({ + stuckKillCount: 7, + status: "failed", + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + })); expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true, preserveStatus: true, @@ -542,11 +557,11 @@ describe("SelfHealingManager", () => { expect(store.handoffToReview).not.toHaveBeenCalled(); expect(store.logEntry).toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); falling back to executor stuck-kill requeue.", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.", ); }); - it("logs post-move requeue patch failures without executor fallback", async () => { + it("logs post-move park patch failures without executor fallback", async () => { (store.getTask as ReturnType).mockResolvedValue({ id: "FN-001", column: "in-progress", @@ -573,11 +588,11 @@ describe("SelfHealingManager", () => { }); expect(store.logEntry).toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (write conflict); scheduler retry may wait for the next state repair pass.", + "STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (write conflict); operator repair is required before retry.", ); - expect(store.logEntry).toHaveBeenCalledWith( + expect(store.logEntry).not.toHaveBeenCalledWith( "FN-001", - "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.", + "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.", ); expect(store.handoffToReview).not.toHaveBeenCalled(); }); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 3c359e0322..1a408873b6 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -1211,7 +1211,7 @@ export class SelfHealingManager { * Terminal contract for stuck-loop exhaustion and no-progress churn: * - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted. Once * exhausted, tasks with incomplete steps are moved back to `todo` with - * progress preserved and pause metadata reapplied for manual resume or + * progress preserved, marked failed, and paused for manual resume or * decomposition; tasks with only terminal steps keep the legacy failed * `in-review` handoff path. * - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on @@ -1315,8 +1315,28 @@ export class SelfHealingManager { return false; } - log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — re-queueing in todo with progress preserved`); - await this.store.updateTask(taskId, { stuckKillCount: newCount }); + log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — parking in todo with progress preserved`); + const exhaustedError = + `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}) after last reason=${reason}. ` + + "Progress was preserved; manually retry, decompose, or rescope before execution resumes."; + const parkUpdate = { + stuckKillCount: newCount, + status: "failed", + error: exhaustedError, + paused: true, + pausedReason: "stuck-loop-exhausted-manual-intervention-required", + pausedByAgentId: "self-healing", + assignedAgentId: null, + checkedOutBy: null, + checkedOutAt: null, + checkoutNodeId: null, + checkoutRunId: null, + checkoutLeaseRenewedAt: null, + checkoutLeaseEpoch: 0, + nextRecoveryAt: null, + } satisfies Parameters[1]; + + await this.store.updateTask(taskId, parkUpdate); try { await this.store.moveTask(taskId, "todo", { preserveProgress: true, @@ -1327,34 +1347,29 @@ export class SelfHealingManager { }); } catch (moveErr: unknown) { const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr); - log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — falling back to executor stuck-kill requeue`); + log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — marking failed/paused in place`); + await this.store.updateTask(taskId, parkUpdate); await this.store.logEntry( taskId, - `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); falling back to executor stuck-kill requeue.`, + `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task was marked failed/paused in place and will not be automatically retried.`, ); - return true; + return false; } - const requeueUpdate = { - stuckKillCount: newCount, - paused: false, - pausedReason: null, - status: "queued", - } satisfies Parameters[1]; try { - await this.store.updateTask(taskId, requeueUpdate); - } catch (patchErr: unknown) { - const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr); - log.warn(`${taskId} post-move requeue patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`); + await this.store.updateTask(taskId, parkUpdate); await this.store.logEntry( taskId, - `STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (${patchErrMessage}); scheduler retry may wait for the next state repair pass.`, + `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.`, + ); + } catch (patchErr: unknown) { + const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr); + log.warn(`${taskId} post-move park patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`); + await this.store.logEntry( + taskId, + `STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (${patchErrMessage}); operator repair is required before retry.`, ); } - await this.store.logEntry( - taskId, - `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.`, - ); return false; }