root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1221 from plarson/fix/incomplete-stuck-loop-parking

Browse Source 
fix(engine): requeue incomplete stuck-loop exhaustion
This commit is contained in:
gsxdsm
2026-05-31 11:54:47 -07:00
committed by GitHub
parent 93e8a5f78a a7bd44ca96
commit 69f22c0140
4 changed files with 98 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/engine/src/__tests__/reliability-interactions/non-progress-churn.test.ts
View File

@@ -4,6 +4,7 @@ import "../executor-test-helpers.js";
import type { Task, TaskStore } from "@fusion/core";
import { TaskExecutor } from "../../executor.js";
import { SelfHealingManager } from "../../self-healing.js";
import { isRunnableQueuedOverlapCandidate } from "../../scheduler.js";
import { StuckTaskDetector } from "../../stuck-task-detector.js";
type MockTaskStore = TaskStore & EventEmitter & {
@@ -189,7 +190,7 @@ describe("reliability interactions: non-progress churn", () => {
manager.stop();
});
it("parks incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => {
it("re-queues incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => {
const task = baseTask({ id: "FN-5168-LOOP", stuckKillCount: 6 });
const store = createStore(task);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
@@ -210,15 +211,16 @@ describe("reliability interactions: non-progress churn", () => {
await detector.killAndRetry(task.id, 60_000);
expect(task.error).toBeNull();
expect(task.status).toBeNull();
expect(task.status).toBe("queued");
expect(task.column).toBe("todo");
expect(task.paused).toBe(true);
expect(task.userPaused).toBe(true);
expect(task.pausedReason).toBe("stuck-loop-exhausted-incomplete-steps");
expect(task.paused).toBe(false);
expect(task.userPaused).toBe(false);
expect(task.pausedReason).toBeNull();
expect(task.stuckKillCount).toBe(7);
expect(task.steps).toEqual([{ name: "Implement", status: "in-progress" }]);
expect(task.log?.some((entry) => entry.action.includes("incomplete task exhausted stuck kill budget"))).toBe(true);
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(isRunnableQueuedOverlapCandidate(task, [task])).toBe(true);
manager.stop();
});

79
packages/engine/src/__tests__/self-healing.test.ts
View File

@@ -424,7 +424,7 @@ describe("SelfHealingManager", () => {
);
});
it("parks incomplete stuck-loop exhaustion in todo without review handoff", async () => {
it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
@@ -440,27 +440,26 @@ describe("SelfHealingManager", () => {
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true });
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
preserveProgress: true,
preserveStatus: true,
});
expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
}));
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; manual review/resume required before retry.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
);
});
it("leaves incomplete stuck-loop exhaustion paused when todo parking fails", async () => {
it("falls back to executor requeue when todo parking fails", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
@@ -476,21 +475,59 @@ describe("SelfHealingManager", () => {
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
expect(result).toBe(true);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
preserveProgress: true,
preserveStatus: true,
});
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", expect.objectContaining({
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true });
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task remains paused for manual intervention.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); falling back to executor stuck-kill requeue.",
);
});
it("logs post-move requeue patch failures without executor fallback", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
stuckKillCount: 6,
steps: [
{ name: "Preflight", status: "done" },
{ name: "Delivery", status: "in-progress" },
],
} as unknown as Task);
(store.updateTask as ReturnType<typeof vi.fn>)
.mockResolvedValueOnce({} as Task)
.mockRejectedValueOnce(new Error("write conflict"));
manager.start();
const result = await manager.checkStuckBudget("FN-001", "loop");
expect(result).toBe(false);
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
preserveProgress: true,
preserveStatus: true,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (write conflict); scheduler retry may wait for the next state repair pass.",
);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
);
expect(store.handoffToReview).not.toHaveBeenCalled();
});
it("terminalizes no-progress churn without incrementing stuck kill budget", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",