feat(FN-5168): complete Step 3 — no-progress churn terminalization

Fusion-Task-Id: FN-5168
Fusion-Task-Lineage: 0c865d3e-0886-4e51-a7ff-cb4c713dcc54
This commit is contained in:
Fusion (runfusion.ai)
2026-05-19 14:00:15 -07:00
committed by gsxdsm
parent af830a74cc
commit 368c4c37fe
4 changed files with 96 additions and 10 deletions

View File

@@ -404,6 +404,42 @@ describe("SelfHealingManager", () => {
);
});
it("terminalizes no-progress churn without incrementing stuck kill budget", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
lineageId: "lin-001",
stuckKillCount: 3,
} as unknown as Task);
manager.start();
const result = await manager.checkStuckBudget("FN-001", "no-progress-churn", {
ignoredStepUpdateCount: 25,
});
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", {
status: "failed",
error: "STUCK_NO_PROGRESS_CHURN: detected 25 ignored step-update rebuffs after compact-and-resume failed to recover progress. Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.",
});
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-review");
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
"STUCK_NO_PROGRESS_CHURN: detected 25 ignored step-update rebuffs after compact-and-resume failed to recover progress. No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.",
);
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
domain: "database",
mutationType: "task:stuck-no-progress-churn-terminalized",
target: "FN-001",
metadata: expect.objectContaining({
taskId: "FN-001",
ignoredStepUpdateCount: 25,
stuckKillStreak: 3,
lastReason: "no-progress-churn",
}),
}));
});
it("respects custom maxStuckKills setting", async () => {
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
maxStuckKills: 1,

View File

@@ -228,6 +228,8 @@ export type DatabaseMutationType =
| "task:auto-recover-completion-handoff-limbo"
| "task:auto-recover-completion-handoff-limbo-exhausted"
| "task:auto-recover-worktree-session-exhausted"
/** Metadata: { taskId: string; ignoredStepUpdateCount: number; stuckKillStreak: number; lastReason: "no-progress-churn" } */
| "task:stuck-no-progress-churn-terminalized"
| "task:auto-recover-starved-refinement"
/** Metadata: { rawDiffFileCount: number; attributedFileCount: number; foreignCommitCount: number; foreignCommitShas: string[]; source: string } */
| "task:worktree-contamination-detected"

View File

@@ -360,7 +360,7 @@ export class InProcessRuntime
// 5b. Initialize TaskExecutor
this.stuckTaskDetector = new StuckTaskDetector(this.taskStore, {
beforeRequeue: (taskId, reason) => this.selfHealingManager?.checkStuckBudget(taskId, reason) ?? Promise.resolve(true),
beforeRequeue: (taskId, reason, event) => this.selfHealingManager?.checkStuckBudget(taskId, reason, event) ?? Promise.resolve(true),
onLoopDetected: (event) => this.executor?.handleLoopDetected(event) ?? Promise.resolve(false),
onStuck: (event) => {
this.triageProcessor?.markStuckAborted(event.taskId);

View File

@@ -806,22 +806,70 @@ export class SelfHealingManager {
* Check whether a stuck-killed task should be re-queued or marked as failed.
* Called by StuckTaskDetector's `beforeRequeue` callback.
*
* Terminal contract for stuck-loop exhaustion:
* - Task is marked `status: "failed"` with `error` starting with
* `STUCK_LOOP_EXHAUSTED: ` and including kill count, max, and last reason.
* - Task is moved to `in-review` (best-effort if move fails).
* - Task log gets a final `STUCK_LOOP_EXHAUSTED` entry with operator guidance
* to manually retry, pause, or move to triage.
* Terminal contract for stuck-loop exhaustion and no-progress churn:
* - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted, then
* marks the task failed and parks it in `in-review`.
* - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on
* the first trigger with operator guidance to decompose or rescope.
*
* @returns `true` if the task should be re-queued, `false` if budget exhausted
* (task has been marked as permanently failed).
* @returns `true` if the task should be re-queued, `false` if terminalized.
*/
async checkStuckBudget(taskId: string, reason: "loop" | "inactivity" = "inactivity"): Promise<boolean> {
async checkStuckBudget(
taskId: string,
reason: "loop" | "inactivity" | "no-progress-churn" = "inactivity",
event?: { ignoredStepUpdateCount?: number },
): Promise<boolean> {
try {
const settings = await this.store.getSettings();
const maxKills = settings.maxStuckKills ?? 6;
const task = await this.store.getTask(taskId);
if (reason === "no-progress-churn") {
const ignoredStepUpdateCount = event?.ignoredStepUpdateCount ?? 0;
const stuckKillStreak = task.stuckKillCount ?? 0;
log.warn(
`${taskId} no-progress churn detected ` +
`(ignoredStepUpdates=${ignoredStepUpdateCount}, stuckKillStreak=${stuckKillStreak}) — marking failed`,
);
const churnError =
`STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` +
`Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.`;
await this.store.updateTask(taskId, {
status: "failed",
error: churnError,
});
try {
await this.store.moveTask(taskId, "in-review");
} catch (moveErr: unknown) {
const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
log.warn(`${taskId} moveTask("in-review") failed (${moveErrMessage}) after STUCK_NO_PROGRESS_CHURN terminalization — task already marked failed, not re-queuing`);
}
await this.store.logEntry(
taskId,
`STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` +
`No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.`,
);
const churnAudit = createRunAuditor(this.store, {
runId: generateSyntheticRunId("fn5168-stuck-churn", taskId),
agentId: "self-healing",
taskId,
taskLineageId: task.lineageId,
phase: "stuck-no-progress-churn-terminalized",
});
await churnAudit.database({
type: "task:stuck-no-progress-churn-terminalized",
target: taskId,
metadata: {
taskId,
ignoredStepUpdateCount,
stuckKillStreak,
lastReason: reason,
},
});
return false;
}
const newCount = (task.stuckKillCount ?? 0) + 1;
if (newCount > maxKills) {