From 368c4c37fe0681f14225c6b0a2556e0f08a1798c Mon Sep 17 00:00:00 2001 From: "Fusion (runfusion.ai)" Date: Tue, 19 May 2026 14:00:15 -0700 Subject: [PATCH] =?UTF-8?q?feat(FN-5168):=20complete=20Step=203=20?= =?UTF-8?q?=E2=80=94=20no-progress=20churn=20terminalization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fusion-Task-Id: FN-5168 Fusion-Task-Lineage: 0c865d3e-0886-4e51-a7ff-cb4c713dcc54 --- .../engine/src/__tests__/self-healing.test.ts | 36 ++++++++++ packages/engine/src/run-audit.ts | 2 + .../engine/src/runtimes/in-process-runtime.ts | 2 +- packages/engine/src/self-healing.ts | 66 ++++++++++++++++--- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 70a8957cd..9fb2d9678 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -404,6 +404,42 @@ describe("SelfHealingManager", () => { ); }); + it("terminalizes no-progress churn without incrementing stuck kill budget", async () => { + (store.getTask as ReturnType).mockResolvedValue({ + id: "FN-001", + lineageId: "lin-001", + stuckKillCount: 3, + } as unknown as Task); + + manager.start(); + + const result = await manager.checkStuckBudget("FN-001", "no-progress-churn", { + ignoredStepUpdateCount: 25, + }); + + expect(result).toBe(false); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", { + status: "failed", + error: "STUCK_NO_PROGRESS_CHURN: detected 25 ignored step-update rebuffs after compact-and-resume failed to recover progress. Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.", + }); + expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-review"); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-001", + "STUCK_NO_PROGRESS_CHURN: detected 25 ignored step-update rebuffs after compact-and-resume failed to recover progress. No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.", + ); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + domain: "database", + mutationType: "task:stuck-no-progress-churn-terminalized", + target: "FN-001", + metadata: expect.objectContaining({ + taskId: "FN-001", + ignoredStepUpdateCount: 25, + stuckKillStreak: 3, + lastReason: "no-progress-churn", + }), + })); + }); + it("respects custom maxStuckKills setting", async () => { (store.getSettings as ReturnType).mockResolvedValue({ maxStuckKills: 1, diff --git a/packages/engine/src/run-audit.ts b/packages/engine/src/run-audit.ts index ad4c98453..2d7342d7e 100644 --- a/packages/engine/src/run-audit.ts +++ b/packages/engine/src/run-audit.ts @@ -228,6 +228,8 @@ export type DatabaseMutationType = | "task:auto-recover-completion-handoff-limbo" | "task:auto-recover-completion-handoff-limbo-exhausted" | "task:auto-recover-worktree-session-exhausted" + /** Metadata: { taskId: string; ignoredStepUpdateCount: number; stuckKillStreak: number; lastReason: "no-progress-churn" } */ + | "task:stuck-no-progress-churn-terminalized" | "task:auto-recover-starved-refinement" /** Metadata: { rawDiffFileCount: number; attributedFileCount: number; foreignCommitCount: number; foreignCommitShas: string[]; source: string } */ | "task:worktree-contamination-detected" diff --git a/packages/engine/src/runtimes/in-process-runtime.ts b/packages/engine/src/runtimes/in-process-runtime.ts index 631e76756..9d784b835 100644 --- a/packages/engine/src/runtimes/in-process-runtime.ts +++ b/packages/engine/src/runtimes/in-process-runtime.ts @@ -360,7 +360,7 @@ export class InProcessRuntime // 5b. Initialize TaskExecutor this.stuckTaskDetector = new StuckTaskDetector(this.taskStore, { - beforeRequeue: (taskId, reason) => this.selfHealingManager?.checkStuckBudget(taskId, reason) ?? Promise.resolve(true), + beforeRequeue: (taskId, reason, event) => this.selfHealingManager?.checkStuckBudget(taskId, reason, event) ?? Promise.resolve(true), onLoopDetected: (event) => this.executor?.handleLoopDetected(event) ?? Promise.resolve(false), onStuck: (event) => { this.triageProcessor?.markStuckAborted(event.taskId); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index eb07af402..1ab9cfa39 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -806,22 +806,70 @@ export class SelfHealingManager { * Check whether a stuck-killed task should be re-queued or marked as failed. * Called by StuckTaskDetector's `beforeRequeue` callback. * - * Terminal contract for stuck-loop exhaustion: - * - Task is marked `status: "failed"` with `error` starting with - * `STUCK_LOOP_EXHAUSTED: ` and including kill count, max, and last reason. - * - Task is moved to `in-review` (best-effort if move fails). - * - Task log gets a final `STUCK_LOOP_EXHAUSTED` entry with operator guidance - * to manually retry, pause, or move to triage. + * Terminal contract for stuck-loop exhaustion and no-progress churn: + * - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted, then + * marks the task failed and parks it in `in-review`. + * - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on + * the first trigger with operator guidance to decompose or rescope. * - * @returns `true` if the task should be re-queued, `false` if budget exhausted - * (task has been marked as permanently failed). + * @returns `true` if the task should be re-queued, `false` if terminalized. */ - async checkStuckBudget(taskId: string, reason: "loop" | "inactivity" = "inactivity"): Promise { + async checkStuckBudget( + taskId: string, + reason: "loop" | "inactivity" | "no-progress-churn" = "inactivity", + event?: { ignoredStepUpdateCount?: number }, + ): Promise { try { const settings = await this.store.getSettings(); const maxKills = settings.maxStuckKills ?? 6; const task = await this.store.getTask(taskId); + + if (reason === "no-progress-churn") { + const ignoredStepUpdateCount = event?.ignoredStepUpdateCount ?? 0; + const stuckKillStreak = task.stuckKillCount ?? 0; + log.warn( + `${taskId} no-progress churn detected ` + + `(ignoredStepUpdates=${ignoredStepUpdateCount}, stuckKillStreak=${stuckKillStreak}) — marking failed`, + ); + const churnError = + `STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` + + `Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.`; + await this.store.updateTask(taskId, { + status: "failed", + error: churnError, + }); + try { + await this.store.moveTask(taskId, "in-review"); + } catch (moveErr: unknown) { + const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr); + log.warn(`${taskId} moveTask("in-review") failed (${moveErrMessage}) after STUCK_NO_PROGRESS_CHURN terminalization — task already marked failed, not re-queuing`); + } + await this.store.logEntry( + taskId, + `STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` + + `No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.`, + ); + const churnAudit = createRunAuditor(this.store, { + runId: generateSyntheticRunId("fn5168-stuck-churn", taskId), + agentId: "self-healing", + taskId, + taskLineageId: task.lineageId, + phase: "stuck-no-progress-churn-terminalized", + }); + await churnAudit.database({ + type: "task:stuck-no-progress-churn-terminalized", + target: taskId, + metadata: { + taskId, + ignoredStepUpdateCount, + stuckKillStreak, + lastReason: reason, + }, + }); + return false; + } + const newCount = (task.stuckKillCount ?? 0) + 1; if (newCount > maxKills) {