fix(engine): park incomplete stuck-loop exhaustion
Preserve incomplete task progress after stuck-kill budget exhaustion while marking the task failed and paused for manual intervention instead of making it scheduler-runnable again.
This commit is contained in:
Phil Larson
5
.changeset/park-incomplete-stuck-loop.md
Normal file
5
.changeset/park-incomplete-stuck-loop.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
Park incomplete tasks that exhaust stuck-loop recovery instead of making them scheduler-runnable again.
|
||||
@@ -441,11 +441,12 @@ describe("SelfHealingManager", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("re-queues incomplete stuck-loop exhaustion in todo without review handoff", async () => {
|
||||
it("parks incomplete stuck-loop exhaustion in todo without review handoff or automatic retry", async () => {
|
||||
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "FN-001",
|
||||
column: "in-progress",
|
||||
stuckKillCount: 6,
|
||||
assignedAgentId: "agent-1",
|
||||
steps: [
|
||||
{ name: "Preflight", status: "done" },
|
||||
{ name: "Delivery", status: "in-progress" },
|
||||
@@ -457,23 +458,32 @@ describe("SelfHealingManager", () => {
|
||||
const result = await manager.checkStuckBudget("FN-001", "loop");
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
|
||||
stuckKillCount: 7,
|
||||
status: "failed",
|
||||
error: expect.stringContaining("STUCK_LOOP_EXHAUSTED"),
|
||||
paused: true,
|
||||
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
|
||||
pausedByAgentId: "self-healing",
|
||||
assignedAgentId: null,
|
||||
checkedOutBy: null,
|
||||
checkedOutAt: null,
|
||||
checkoutNodeId: null,
|
||||
checkoutRunId: null,
|
||||
checkoutLeaseRenewedAt: null,
|
||||
checkoutLeaseEpoch: 0,
|
||||
nextRecoveryAt: null,
|
||||
}));
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
|
||||
preserveProgress: true,
|
||||
preserveStatus: true,
|
||||
moveSource: "engine",
|
||||
recoveryRehome: true,
|
||||
});
|
||||
expect(store.updateTask).toHaveBeenLastCalledWith("FN-001", expect.objectContaining({
|
||||
stuckKillCount: 7,
|
||||
paused: false,
|
||||
pausedReason: null,
|
||||
status: "queued",
|
||||
}));
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -509,7 +519,7 @@ describe("SelfHealingManager", () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it("falls back to executor requeue when todo parking fails", async () => {
|
||||
it("does not fall back to executor requeue when todo parking fails", async () => {
|
||||
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "FN-001",
|
||||
column: "in-progress",
|
||||
@@ -525,8 +535,13 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
const result = await manager.checkStuckBudget("FN-001", "loop");
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 7 });
|
||||
expect(result).toBe(false);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
|
||||
stuckKillCount: 7,
|
||||
status: "failed",
|
||||
paused: true,
|
||||
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
|
||||
}));
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", {
|
||||
preserveProgress: true,
|
||||
preserveStatus: true,
|
||||
@@ -542,11 +557,11 @@ describe("SelfHealingManager", () => {
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); falling back to executor stuck-kill requeue.",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Failed to move task to todo (database is busy); task was marked failed/paused in place and will not be automatically retried.",
|
||||
);
|
||||
});
|
||||
|
||||
it("logs post-move requeue patch failures without executor fallback", async () => {
|
||||
it("logs post-move park patch failures without executor fallback", async () => {
|
||||
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "FN-001",
|
||||
column: "in-progress",
|
||||
@@ -573,11 +588,11 @@ describe("SelfHealingManager", () => {
|
||||
});
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (write conflict); scheduler retry may wait for the next state repair pass.",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (write conflict); operator repair is required before retry.",
|
||||
);
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
expect(store.logEntry).not.toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.",
|
||||
"STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (7/6), last reason=loop. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.",
|
||||
);
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1211,7 +1211,7 @@ export class SelfHealingManager {
|
||||
* Terminal contract for stuck-loop exhaustion and no-progress churn:
|
||||
* - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted. Once
|
||||
* exhausted, tasks with incomplete steps are moved back to `todo` with
|
||||
* progress preserved and pause metadata reapplied for manual resume or
|
||||
* progress preserved, marked failed, and paused for manual resume or
|
||||
* decomposition; tasks with only terminal steps keep the legacy failed
|
||||
* `in-review` handoff path.
|
||||
* - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on
|
||||
@@ -1315,8 +1315,28 @@ export class SelfHealingManager {
|
||||
return false;
|
||||
}
|
||||
|
||||
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — re-queueing in todo with progress preserved`);
|
||||
await this.store.updateTask(taskId, { stuckKillCount: newCount });
|
||||
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) with incomplete steps — parking in todo with progress preserved`);
|
||||
const exhaustedError =
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}) after last reason=${reason}. ` +
|
||||
"Progress was preserved; manually retry, decompose, or rescope before execution resumes.";
|
||||
const parkUpdate = {
|
||||
stuckKillCount: newCount,
|
||||
status: "failed",
|
||||
error: exhaustedError,
|
||||
paused: true,
|
||||
pausedReason: "stuck-loop-exhausted-manual-intervention-required",
|
||||
pausedByAgentId: "self-healing",
|
||||
assignedAgentId: null,
|
||||
checkedOutBy: null,
|
||||
checkedOutAt: null,
|
||||
checkoutNodeId: null,
|
||||
checkoutRunId: null,
|
||||
checkoutLeaseRenewedAt: null,
|
||||
checkoutLeaseEpoch: 0,
|
||||
nextRecoveryAt: null,
|
||||
} satisfies Parameters<typeof this.store.updateTask>[1];
|
||||
|
||||
await this.store.updateTask(taskId, parkUpdate);
|
||||
try {
|
||||
await this.store.moveTask(taskId, "todo", {
|
||||
preserveProgress: true,
|
||||
@@ -1327,34 +1347,29 @@ export class SelfHealingManager {
|
||||
});
|
||||
} catch (moveErr: unknown) {
|
||||
const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
|
||||
log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — falling back to executor stuck-kill requeue`);
|
||||
log.warn(`${taskId} moveTask(todo) failed (${moveErrMessage}) after incomplete STUCK_LOOP_EXHAUSTED terminalization — marking failed/paused in place`);
|
||||
await this.store.updateTask(taskId, parkUpdate);
|
||||
await this.store.logEntry(
|
||||
taskId,
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); falling back to executor stuck-kill requeue.`,
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Failed to move task to todo (${moveErrMessage}); task was marked failed/paused in place and will not be automatically retried.`,
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const requeueUpdate = {
|
||||
stuckKillCount: newCount,
|
||||
paused: false,
|
||||
pausedReason: null,
|
||||
status: "queued",
|
||||
} satisfies Parameters<typeof this.store.updateTask>[1];
|
||||
try {
|
||||
await this.store.updateTask(taskId, requeueUpdate);
|
||||
} catch (patchErr: unknown) {
|
||||
const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
|
||||
log.warn(`${taskId} post-move requeue patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
|
||||
await this.store.updateTask(taskId, parkUpdate);
|
||||
await this.store.logEntry(
|
||||
taskId,
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move requeue patch failed (${patchErrMessage}); scheduler retry may wait for the next state repair pass.`,
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Parked in todo with progress preserved; no further automatic retries will run until an operator manually retries, decomposes, or rescopes the task.`,
|
||||
);
|
||||
} catch (patchErr: unknown) {
|
||||
const patchErrMessage = patchErr instanceof Error ? patchErr.message : String(patchErr);
|
||||
log.warn(`${taskId} post-move park patch failed after incomplete STUCK_LOOP_EXHAUSTED terminalization: ${patchErrMessage}`);
|
||||
await this.store.logEntry(
|
||||
taskId,
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task moved to todo with progress preserved, but post-move park patch failed (${patchErrMessage}); operator repair is required before retry.`,
|
||||
);
|
||||
}
|
||||
await this.store.logEntry(
|
||||
taskId,
|
||||
`STUCK_LOOP_EXHAUSTED: incomplete task exhausted stuck kill budget (${newCount}/${maxKills}), last reason=${reason}. Re-queued in todo with progress preserved; scheduler may retry without manual unpause.`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user