feat(FN-5168): complete Step 3 — no-progress churn terminalization
Fusion-Task-Id: FN-5168 Fusion-Task-Lineage: 0c865d3e-0886-4e51-a7ff-cb4c713dcc54
This commit is contained in:
committed by
gsxdsm
parent
af830a74cc
commit
368c4c37fe
@@ -404,6 +404,42 @@ describe("SelfHealingManager", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("terminalizes no-progress churn without incrementing stuck kill budget", async () => {
|
||||
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "FN-001",
|
||||
lineageId: "lin-001",
|
||||
stuckKillCount: 3,
|
||||
} as unknown as Task);
|
||||
|
||||
manager.start();
|
||||
|
||||
const result = await manager.checkStuckBudget("FN-001", "no-progress-churn", {
|
||||
ignoredStepUpdateCount: 25,
|
||||
});
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", {
|
||||
status: "failed",
|
||||
error: "STUCK_NO_PROGRESS_CHURN: detected 25 ignored step-update rebuffs after compact-and-resume failed to recover progress. Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.",
|
||||
});
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-review");
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"STUCK_NO_PROGRESS_CHURN: detected 25 ignored step-update rebuffs after compact-and-resume failed to recover progress. No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.",
|
||||
);
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: "task:stuck-no-progress-churn-terminalized",
|
||||
target: "FN-001",
|
||||
metadata: expect.objectContaining({
|
||||
taskId: "FN-001",
|
||||
ignoredStepUpdateCount: 25,
|
||||
stuckKillStreak: 3,
|
||||
lastReason: "no-progress-churn",
|
||||
}),
|
||||
}));
|
||||
});
|
||||
|
||||
it("respects custom maxStuckKills setting", async () => {
|
||||
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
maxStuckKills: 1,
|
||||
|
||||
@@ -228,6 +228,8 @@ export type DatabaseMutationType =
|
||||
| "task:auto-recover-completion-handoff-limbo"
|
||||
| "task:auto-recover-completion-handoff-limbo-exhausted"
|
||||
| "task:auto-recover-worktree-session-exhausted"
|
||||
/** Metadata: { taskId: string; ignoredStepUpdateCount: number; stuckKillStreak: number; lastReason: "no-progress-churn" } */
|
||||
| "task:stuck-no-progress-churn-terminalized"
|
||||
| "task:auto-recover-starved-refinement"
|
||||
/** Metadata: { rawDiffFileCount: number; attributedFileCount: number; foreignCommitCount: number; foreignCommitShas: string[]; source: string } */
|
||||
| "task:worktree-contamination-detected"
|
||||
|
||||
@@ -360,7 +360,7 @@ export class InProcessRuntime
|
||||
|
||||
// 5b. Initialize TaskExecutor
|
||||
this.stuckTaskDetector = new StuckTaskDetector(this.taskStore, {
|
||||
beforeRequeue: (taskId, reason) => this.selfHealingManager?.checkStuckBudget(taskId, reason) ?? Promise.resolve(true),
|
||||
beforeRequeue: (taskId, reason, event) => this.selfHealingManager?.checkStuckBudget(taskId, reason, event) ?? Promise.resolve(true),
|
||||
onLoopDetected: (event) => this.executor?.handleLoopDetected(event) ?? Promise.resolve(false),
|
||||
onStuck: (event) => {
|
||||
this.triageProcessor?.markStuckAborted(event.taskId);
|
||||
|
||||
@@ -806,22 +806,70 @@ export class SelfHealingManager {
|
||||
* Check whether a stuck-killed task should be re-queued or marked as failed.
|
||||
* Called by StuckTaskDetector's `beforeRequeue` callback.
|
||||
*
|
||||
* Terminal contract for stuck-loop exhaustion:
|
||||
* - Task is marked `status: "failed"` with `error` starting with
|
||||
* `STUCK_LOOP_EXHAUSTED: ` and including kill count, max, and last reason.
|
||||
* - Task is moved to `in-review` (best-effort if move fails).
|
||||
* - Task log gets a final `STUCK_LOOP_EXHAUSTED` entry with operator guidance
|
||||
* to manually retry, pause, or move to triage.
|
||||
* Terminal contract for stuck-loop exhaustion and no-progress churn:
|
||||
* - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted, then
|
||||
* marks the task failed and parks it in `in-review`.
|
||||
* - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on
|
||||
* the first trigger with operator guidance to decompose or rescope.
|
||||
*
|
||||
* @returns `true` if the task should be re-queued, `false` if budget exhausted
|
||||
* (task has been marked as permanently failed).
|
||||
* @returns `true` if the task should be re-queued, `false` if terminalized.
|
||||
*/
|
||||
async checkStuckBudget(taskId: string, reason: "loop" | "inactivity" = "inactivity"): Promise<boolean> {
|
||||
async checkStuckBudget(
|
||||
taskId: string,
|
||||
reason: "loop" | "inactivity" | "no-progress-churn" = "inactivity",
|
||||
event?: { ignoredStepUpdateCount?: number },
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
const maxKills = settings.maxStuckKills ?? 6;
|
||||
|
||||
const task = await this.store.getTask(taskId);
|
||||
|
||||
if (reason === "no-progress-churn") {
|
||||
const ignoredStepUpdateCount = event?.ignoredStepUpdateCount ?? 0;
|
||||
const stuckKillStreak = task.stuckKillCount ?? 0;
|
||||
log.warn(
|
||||
`${taskId} no-progress churn detected ` +
|
||||
`(ignoredStepUpdates=${ignoredStepUpdateCount}, stuckKillStreak=${stuckKillStreak}) — marking failed`,
|
||||
);
|
||||
const churnError =
|
||||
`STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` +
|
||||
`Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.`;
|
||||
await this.store.updateTask(taskId, {
|
||||
status: "failed",
|
||||
error: churnError,
|
||||
});
|
||||
try {
|
||||
await this.store.moveTask(taskId, "in-review");
|
||||
} catch (moveErr: unknown) {
|
||||
const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
|
||||
log.warn(`${taskId} moveTask("in-review") failed (${moveErrMessage}) after STUCK_NO_PROGRESS_CHURN terminalization — task already marked failed, not re-queuing`);
|
||||
}
|
||||
await this.store.logEntry(
|
||||
taskId,
|
||||
`STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` +
|
||||
`No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.`,
|
||||
);
|
||||
const churnAudit = createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("fn5168-stuck-churn", taskId),
|
||||
agentId: "self-healing",
|
||||
taskId,
|
||||
taskLineageId: task.lineageId,
|
||||
phase: "stuck-no-progress-churn-terminalized",
|
||||
});
|
||||
await churnAudit.database({
|
||||
type: "task:stuck-no-progress-churn-terminalized",
|
||||
target: taskId,
|
||||
metadata: {
|
||||
taskId,
|
||||
ignoredStepUpdateCount,
|
||||
stuckKillStreak,
|
||||
lastReason: reason,
|
||||
},
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
const newCount = (task.stuckKillCount ?? 0) + 1;
|
||||
|
||||
if (newCount > maxKills) {
|
||||
|
||||
Reference in New Issue
Block a user