fix(engine): park incomplete stuck-loop exhaustion

Preserve incomplete task progress after stuck-kill budget exhaustion while marking the task failed and paused for manual intervention instead of making it scheduler-runnable again.

.changeset/park-incomplete-stuck-loop.md

@@ -0,0 +1,5 @@
+---
+"@runfusion/fusion": patch
+---
+
+Park incomplete tasks that exhaust stuck-loop recovery instead of making them scheduler-runnable again.

packages/engine/src/__tests__/self-healing.test.ts

@@ -441,11 +441,12 @@ describe("SelfHealingManager", () => {
       );
     });
 
-    it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => {
+    it("parks incomplete stuck-loop exhaustion in todo without review handoff or automatic retry", async () => {
       (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
         id: "FN-001",
         column: "in-progress",
         stuckKillCount: 6,
+        assignedAgentId: "agent-1",
         steps: [
           { name: "Preflight", status: "done" },
           { name: "Delivery", status: "in-progress" },
@@ -457,23 +458,32 @@ describe("SelfHealingManager", () => {
       const result = await manager.checkStuckBudget("FN-001", "loop");
 
       expect(result).toBe(false);
-      expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
+        stuckKillCount: 7,
+        status: "failed",
+        error: expect.stringContaining("STUCK_LOOP_EXHAUSTED"),
+        paused: true,
+        pausedReason: "stuck-loop-exhausted-manual-intervention-required",
+        pausedByAgentId: "self-healing",
+        assignedAgentId: null,
+        checkedOutBy: null,
+        checkedOutAt: null,
+        checkoutNodeId: null,
+        checkoutRunId: null,
+        checkoutLeaseRenewedAt: null,
+        checkoutLeaseEpoch: 0,
+        nextRecoveryAt: null,
+      }));
       expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
         preserveProgress: true,
         preserveStatus: true,
         moveSource: "engine",
         recoveryRehome: true,
       });
-      expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({
-        stuckKillCount: 7,
-        paused: false,
-        pausedReason: null,
-        status: "queued",
-      }));
       expect(store.handoffToReview).not.toHaveBeenCalled();
       expect(store.logEntry).toHaveBeenCalledWith(
         "FN-001",
-        "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
+        "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.",
       );
     });
 
@@ -509,7 +519,7 @@ describe("SelfHealingManager", () => {
       }));
     });
 
-    it("falls back to executor requeue when todo parking fails", async () => {
+    it("does not fall back to executor requeue when todo parking fails", async () => {
       (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
         id: "FN-001",
         column: "in-progress",
@@ -525,8 +535,13 @@ describe("SelfHealingManager", () => {
 
       const result = await manager.checkStuckBudget("FN-001", "loop");
 
-      expect(result).toBe(true);
+      expect(result).toBe(false);
-      expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
+        stuckKillCount: 7,
+        status: "failed",
+        paused: true,
+        pausedReason: "stuck-loop-exhausted-manual-intervention-required",
+      }));
       expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
         preserveProgress: true,
         preserveStatus: true,
@@ -542,11 +557,11 @@ describe("SelfHealingManager", () => {
       expect(store.handoffToReview).not.toHaveBeenCalled();
       expect(store.logEntry).toHaveBeenCalledWith(
         "FN-001",
-        "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); falling back to executor stuck-kill requeue.",
+        "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.",
       );
     });
 
-    it("logs post-move requeue patch failures without executor fallback", async () => {
+    it("logs post-move park patch failures without executor fallback", async () => {
       (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
         id: "FN-001",
         column: "in-progress",
@@ -573,11 +588,11 @@ describe("SelfHealingManager", () => {
       });
       expect(store.logEntry).toHaveBeenCalledWith(
         "FN-001",
-        "STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (write conflict); scheduler retry may wait for the next state repair pass.",
+        "STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (write conflict); operator repair is required before retry.",
       );
-      expect(store.logEntry).toHaveBeenCalledWith(
+      expect(store.logEntry).not.toHaveBeenCalledWith(
         "FN-001",
-        "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
+        "STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.",
       );
       expect(store.handoffToReview).not.toHaveBeenCalled();
     });

packages/engine/src/self-healing.ts

@@ -1211,7 +1211,7 @@ export class SelfHealingManager {
    * Terminal contract for stuck-loop exhaustion and no-progress churn:
    * - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted. Once
    *   exhausted, tasks with incomplete steps are moved back to `todo` with
-   *   progress preserved and pause metadata reapplied for manual resume or
+   *   progress preserved, marked failed, and paused for manual resume or
    *   decomposition; tasks with only terminal steps keep the legacy failed
    *   `in-review` handoff path.
    * - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on
@@ -1315,8 +1315,28 @@ export class SelfHealingManager {
             return false;
           }
 
-          log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — re-queueing in todo with progress preserved`);
+          log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — parking in todo with progress preserved`);
-          await this.store.updateTask(taskId, { stuckKillCount: newCount });
+          const exhaustedError =
+            `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}) after last reason=${reason}. ` +
+            "Progress was preserved; manually retry, decompose, or rescope before execution resumes.";
+          const parkUpdate = {
+            stuckKillCount: newCount,
+            status: "failed",
+            error: exhaustedError,
+            paused: true,
+            pausedReason: "stuck-loop-exhausted-manual-intervention-required",
+            pausedByAgentId: "self-healing",
+            assignedAgentId: null,
+            checkedOutBy: null,
+            checkedOutAt: null,
+            checkoutNodeId: null,
+            checkoutRunId: null,
+            checkoutLeaseRenewedAt: null,
+            checkoutLeaseEpoch: 0,
+            nextRecoveryAt: null,
+          } satisfies Parameters<typeof this.store.updateTask>[1];
+
+          await this.store.updateTask(taskId, parkUpdate);
           try {
             await this.store.moveTask(taskId, "todo", {
               preserveProgress: true,
@@ -1327,34 +1347,29 @@ export class SelfHealingManager {
             });
           } catch (moveErr: unknown) {
             const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
-            log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — falling back to executor stuck-kill requeue`);
+            log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — marking failed/paused in place`);
+            await this.store.updateTask(taskId, parkUpdate);
             await this.store.logEntry(
               taskId,
-              `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); falling back to executor stuck-kill requeue.`,
+              `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task was marked failed/paused in place and will not be automatically retried.`,
             );
-            return true;
+            return false;
           }
 
-          const requeueUpdate = {
-            stuckKillCount: newCount,
-            paused: false,
-            pausedReason: null,
-            status: "queued",
-          } satisfies Parameters<typeof this.store.updateTask>[1];
           try {
-            await this.store.updateTask(taskId, requeueUpdate);
+            await this.store.updateTask(taskId, parkUpdate);
-          } catch (patchErr: unknown) {
-            const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
-            log.warn(`${taskId} post-move requeue patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
             await this.store.logEntry(
               taskId,
-              `STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (${patchErrMessage}); scheduler retry may wait for the next state repair pass.`,
+              `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.`,
+            );
+          } catch (patchErr: unknown) {
+            const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
+            log.warn(`${taskId} post-move park patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
+            await this.store.logEntry(
+              taskId,
+              `STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (${patchErrMessage}); operator repair is required before retry.`,
             );
           }
-          await this.store.logEntry(
-            taskId,
-            `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.`,
-          );
           return false;
         }