fix(engine): requeue incomplete stuck-loop exhaustion

This commit is contained in:
Phil Larson
2026-05-31 07:58:11 -07:00
parent b154844286
commit 5861884456
4 changed files with 36 additions and 31 deletions

View File

@@ -189,7 +189,7 @@ describe("reliability interactions: non-progress churn", () => {
manager.stop();
});
it("parks incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => {
it("re-queues incomplete STUCK_LOOP_EXHAUSTED tasks in todo when the churn signal does not fire", async () => {
const task = baseTask({ id: "FN-5168-LOOP", stuckKillCount: 6 });
const store = createStore(task);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
@@ -210,11 +210,11 @@ describe("reliability interactions: non-progress churn", () => {
await detector.killAndRetry(task.id, 60_000);
expect(task.error).toBeNull();
expect(task.status).toBeNull();
expect(task.status).toBe("queued");
expect(task.column).toBe("todo");
expect(task.paused).toBe(true);
expect(task.userPaused).toBe(true);
expect(task.pausedReason).toBe("stuck-loop-exhausted-incomplete-steps");
expect(task.paused).toBe(false);
expect(task.userPaused).toBe(false);
expect(task.pausedReason).toBeNull();
expect(task.stuckKillCount).toBe(7);
expect(task.steps).toEqual([{ name: "Implement", status: "in-progress" }]);
expect(task.log?.some((entry) => entry.action.includes("incomplete task exhausted stuck kill budget"))).toBe(true);

View File

@@ -424,7 +424,7 @@ describe("SelfHealingManager", () => {
);
});
it("parks incomplete stuck-loop exhaustion in todo without review handoff", async () => {
it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
@@ -442,25 +442,27 @@ describe("SelfHealingManager", () => {
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true });
expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
}));
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; manual review/resume required before retry.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
);
});
it("leaves incomplete stuck-loop exhaustion paused when todo parking fails", async () => {
it("leaves incomplete stuck-loop exhaustion unpaused when todo parking fails", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
column: "in-progress",
@@ -479,15 +481,16 @@ describe("SelfHealingManager", () => {
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
stuckKillCount: 7,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveProgress: true });
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task remains paused for manual intervention.",
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task remains unpaused for scheduler retry.",
);
});

View File

@@ -1072,33 +1072,35 @@ export class SelfHealingManager {
const hasIncompleteSteps = !!task.steps?.some((step) => NON_TERMINAL_STEP_STATUSES.has(step.status));
if (hasIncompleteSteps) {
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — parking in todo`);
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — re-queueing in todo with progress preserved`);
await this.store.updateTask(taskId, {
stuckKillCount: newCount,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
} as Partial<Task> & { userPaused: boolean });
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
} as unknown as Partial<Task> & { userPaused: boolean });
let parkedInTodo = true;
let moveErrMessage = "";
try {
await this.store.moveTask(taskId, "todo", { preserveProgress: true });
await this.store.updateTask(taskId, {
stuckKillCount: newCount,
paused: true,
userPaused: true,
pausedReason: "stuck-loop-exhausted-incomplete-steps",
} as Partial<Task> & { userPaused: boolean });
paused: false,
userPaused: false,
pausedReason: null,
status: "queued",
} as unknown as Partial<Task> & { userPaused: boolean });
} catch (moveErr: unknown) {
parkedInTodo = false;
moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — task remains paused for manual intervention`);
log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — task remains queued for scheduler retry`);
}
await this.store.logEntry(
taskId,
parkedInTodo
? `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Parked in todo with progress preserved; manual review/resume required before retry.`
: `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task remains paused for manual intervention.`,
? `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.`
: `STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task remains unpaused for scheduler retry.`,
);
return false;
}