root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1676 from plarson/fix/incomplete-stuck-loop-parking

Browse Source 
fix: park incomplete stuck-loop exhaustion
This commit is contained in:
gsxdsm
2026-06-14 12:18:06 -07:00
committed by GitHub
parent 4482425d4b 93fd220aaf
commit 23c2bc935a
5 changed files with 169 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.changeset/park-incomplete-stuck-loop.md Normal file
View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": patch
---
Park incomplete tasks that exhaust stuck-loop recovery instead of making them scheduler-runnable again.

14
packages/engine/src/__tests__/reliability-interactions/non-progress-churn.test.ts
View File

@@ -190,7 +190,7 @@ describe("reliability interactions: non-progress churn", () => {
manager.stop();
});
it("re-queues incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => {
it("parks incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => {
const task = baseTask({ id: "FN-5168-LOOP", stuckKillCount: 6 });
const store = createStore(task);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
@@ -210,19 +210,19 @@ describe("reliability interactions: non-progress churn", () => {
await detector.killAndRetry(task.id, 60_000);
expect(task.error).toBeNull();
expect(task.status).toBe("queued");
expect(task.error).toContain("STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget");
expect(task.status).toBe("failed");
expect(task.column).toBe("todo");
expect(task.paused).toBe(false);
expect(task.paused).toBe(true);
// FN-6252 / Move-Task contract: engine rebounds do not write userPaused,
// so a never-user-paused task remains undefined while still not user-paused.
expect(task.userPaused).not.toBe(true);
expect(task.pausedReason).toBeNull();
expect(task.pausedReason).toBe("stuck-loop-exhausted-manual-intervention-required");
expect(task.stuckKillCount).toBe(7);
expect(task.steps).toEqual([{ name: "Implement", status: "in-progress" }]);
expect(task.log?.some((entry) => entry.action.includes("incomplete task exhausted stuck kill budget"))).toBe(true);
expect(task.log?.some((entry) => entry.action.includes("Parked in todo with progress preserved"))).toBe(true);
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(isRunnableQueuedOverlapCandidate(task, [task])).toBe(true);
expect(isRunnableQueuedOverlapCandidate(task, [task])).toBe(false);
manager.stop();
});

8
packages/engine/src/__tests__/reliability-interactions/todo-inprogress-flapping.test.ts
View File

@@ -290,7 +290,7 @@ describe("FN-5941 reliability interactions: todo/in-progress flapping", () => {
manager.stop();
});
it("still requeues a genuinely dead task when stuck-kill budget is exhausted", async () => {
it("parks a genuinely dead incomplete task when stuck-kill budget is exhausted", async () => {
const task = makeTask(rootDir, {
id: "FN-5941-DEAD",
stuckKillCount: 6,
@@ -314,7 +314,11 @@ describe("FN-5941 reliability interactions: todo/in-progress flapping", () => {
}));
expect(task.column).toBe("todo");
expect(task.stuckKillCount).toBe(7);
expect(task.status).toBe("queued");
expect(task.status).toBe("failed");
expect(task.paused).toBe(true);
expect(task.pausedReason).toBe("stuck-loop-exhausted-manual-intervention-required");
expect(task.error).toContain("STUCK_LOOP_EXHAUSTED");
expect(task.userPaused).not.toBe(true);
manager.stop();
});

112
packages/engine/src/__tests__/self-healing.test.ts
View File

@@ -441,11 +441,12 @@ describe("SelfHealingManager", () => {
);
});
it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => {
it("parks incomplete stuck-loop exhaustion in todo without review handoff or automatic retry", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
stuckKillCount: 6,
assignedAgentId: "agent-1",
steps: [
{ name: "Preflight", status: "done" },
{ name: "Delivery", status: "in-progress" },
@@ -457,23 +458,32 @@ describe("SelfHealingManager", () => {
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
status: "failed",
error: expect.stringContaining("STUCK_LOOP_EXHAUSTED"),
paused: true,
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
pausedByAgentId: "self-healing",
assignedAgentId: null,
checkedOutBy: null,
checkedOutAt: null,
checkoutNodeId: null,
checkoutRunId: null,
checkoutLeaseRenewedAt: null,
checkoutLeaseEpoch: 0,
nextRecoveryAt: null,
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
preserveProgress: true,
preserveStatus: true,
moveSource: "engine",
recoveryRehome: true,
});
expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: false,
pausedReason: null,
status: "queued",
}));
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.",
);
});
@@ -509,7 +519,7 @@ describe("SelfHealingManager", () => {
}));
});
it("falls back to executor requeue when todo parking fails", async () => {
it("does not fall back to executor requeue when todo parking fails", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
@@ -525,8 +535,13 @@ describe("SelfHealingManager", () => {
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(true);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
status: "failed",
paused: true,
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
preserveProgress: true,
preserveStatus: true,
@@ -542,11 +557,74 @@ describe("SelfHealingManager", () => {
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); falling back to executor stuck-kill requeue.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.",
);
});
it("logs post-move requeue patch failures without executor fallback", async () => {
it("does not requeue when in-place park succeeds but success logging fails", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
stuckKillCount: 6,
steps: [
{ name: "Preflight", status: "done" },
{ name: "Delivery", status: "in-progress" },
],
} as unknown as Task);
(store.moveTask as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("database is busy"));
(store.logEntry as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("log unavailable"));
manager.start();
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
status: "failed",
paused: true,
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
}));
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", expect.objectContaining({
paused: false,
status: "queued",
}));
expect(store.handoffToReview).not.toHaveBeenCalled();
});
it("logs in-place park patch failures after todo move failure without executor fallback", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
stuckKillCount: 6,
steps: [
{ name: "Preflight", status: "done" },
{ name: "Delivery", status: "in-progress" },
],
} as unknown as Task);
(store.updateTask as ReturnType<typeof vi.fn>)
.mockResolvedValueOnce({} as Task)
.mockRejectedValueOnce(new Error("write conflict"));
(store.moveTask as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("database is busy"));
manager.start();
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledTimes(2);
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task failed to move to todo (database is busy), and the in-place park patch also failed (write conflict); pre-move park metadata was already applied, but operator verification is required before retry.",
);
expect(store.logEntry).not.toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.",
);
});
it("logs post-move park patch failures without executor fallback", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
@@ -573,11 +651,11 @@ describe("SelfHealingManager", () => {
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (write conflict); scheduler retry may wait for the next state repair pass.",
"STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (write conflict); operator repair is required before retry.",
);
expect(store.logEntry).toHaveBeenCalledWith(
expect(store.logEntry).not.toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.",
);
expect(store.handoffToReview).not.toHaveBeenCalled();
});

80
packages/engine/src/self-healing.ts
View File

@@ -1211,9 +1211,13 @@ export class SelfHealingManager {
* Terminal contract for stuck-loop exhaustion and no-progress churn:
* - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted. Once
* exhausted, tasks with incomplete steps are moved back to `todo` with
* progress preserved and pause metadata reapplied for manual resume or
* progress preserved, marked failed, and paused for manual resume or
* decomposition; tasks with only terminal steps keep the legacy failed
* `in-review` handoff path.
*
* FNXC:SelfHealing 2026-06-14-10:51:
* Incomplete stuck-loop exhaustion must park work in a failed/paused state before moving columns, because a post-move patch failure must not leave the task scheduler-runnable.
* Engine-owned recovery must not mutate `userPaused`; user intent stays authoritative across races.
* - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on
* the first trigger with operator guidance to decompose or rescope.
*
@@ -1315,8 +1319,26 @@ export class SelfHealingManager {
return false;
}
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — re-queueing in todo with progress preserved`);
await this.store.updateTask(taskId, { stuckKillCount: newCount });
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — parking in todo with progress preserved`);
const exhaustedError = `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}) after last reason=${reason}. Progress was preserved; manually retry, decompose, or rescope before execution resumes.`;
const parkUpdate = {
stuckKillCount: newCount,
status: "failed",
error: exhaustedError,
paused: true,
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
pausedByAgentId: "self-healing",
assignedAgentId: null,
checkedOutBy: null,
checkedOutAt: null,
checkoutNodeId: null,
checkoutRunId: null,
checkoutLeaseRenewedAt: null,
checkoutLeaseEpoch: 0,
nextRecoveryAt: null,
} satisfies Parameters<typeof this.store.updateTask>[1];
await this.store.updateTask(taskId, parkUpdate);
try {
await this.store.moveTask(taskId, "todo", {
preserveProgress: true,
@@ -1327,34 +1349,44 @@ export class SelfHealingManager {
});
} catch (moveErr: unknown) {
const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — falling back to executor stuck-kill requeue`);
await this.store.logEntry(
taskId,
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); falling back to executor stuck-kill requeue.`,
);
return true;
log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — marking failed/paused in place`);
try {
await this.store.updateTask(taskId, parkUpdate);
} catch (patchErr: unknown) {
const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
log.warn(`${taskId} in-place park patch failed after moveTask(todo) failure during incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
await this.store.logEntry(
taskId,
`STUCK_LOOP_EXHAUSTED: incomplete task failed to move to todo (${moveErrMessage}), and the in-place park patch also failed (${patchErrMessage}); pre-move park metadata was already applied, but operator verification is required before retry.`,
);
return false;
}
try {
await this.store.logEntry(
taskId,
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task was marked failed/paused in place and will not be automatically retried.`,
);
} catch (logErr: unknown) {
const logErrMessage = logErr instanceof Error ? logErr.message : String(logErr);
log.warn(`${taskId} failed to log in-place stuck-loop park success after moveTask(todo) failure: ${logErrMessage}`);
}
return false;
}
const requeueUpdate = {
stuckKillCount: newCount,
paused: false,
pausedReason: null,
status: "queued",
} satisfies Parameters<typeof this.store.updateTask>[1];
try {
await this.store.updateTask(taskId, requeueUpdate);
} catch (patchErr: unknown) {
const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
log.warn(`${taskId} post-move requeue patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
await this.store.updateTask(taskId, parkUpdate);
await this.store.logEntry(
taskId,
`STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (${patchErrMessage}); scheduler retry may wait for the next state repair pass.`,
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.`,
);
} catch (patchErr: unknown) {
const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
log.warn(`${taskId} post-move park patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
await this.store.logEntry(
taskId,
`STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (${patchErrMessage}); operator repair is required before retry.`,
);
}
await this.store.logEntry(
taskId,
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.`,
);
return false;
}