From 988420325bc70629f233905062855b6ace497966 Mon Sep 17 00:00:00 2001 From: Fusion Date: Wed, 13 May 2026 02:17:02 -0700 Subject: [PATCH] =?UTF-8?q?feat(FN-4217):=20complete=20Step=204=20?= =?UTF-8?q?=E2=80=94=20surface=20in-review=20stalls=20in=20self-healing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fusion-Task-Id: FN-4217 Fusion-Task-Lineage: f5434e90-ad0c-41c5-bee9-1244cb69622b --- .../engine/src/__tests__/self-healing.test.ts | 112 ++++++++++++++++++ packages/engine/src/self-healing.ts | 56 ++++++++- 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 9bc2cfa66..889a46865 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -442,6 +442,7 @@ describe("SelfHealingManager", () => { const recoverOrphanedAgents = vi.spyOn(manager, "recoverOrphanedAgents").mockResolvedValue(1); const recoverAgentsRunningOnInactiveTasks = vi.spyOn(manager, "recoverAgentsRunningOnInactiveTasks").mockResolvedValue(1); const clearStaleBlockedBy = vi.spyOn(manager, "clearStaleBlockedBy").mockResolvedValue(1); + const surfaceInReviewStalls = vi.spyOn(manager, "surfaceInReviewStalls").mockResolvedValue(1); await manager.runStartupRecovery(); @@ -455,6 +456,7 @@ describe("SelfHealingManager", () => { expect(recoverOrphanedAgents).toHaveBeenCalledTimes(1); expect(recoverAgentsRunningOnInactiveTasks).toHaveBeenCalledTimes(1); expect(clearStaleBlockedBy).toHaveBeenCalledTimes(1); + expect(surfaceInReviewStalls).toHaveBeenCalledTimes(1); }); it("runStartupRecovery clears stale blockedBy rows", async () => { @@ -3805,6 +3807,116 @@ describe("SelfHealingManager", () => { }); }); + describe("surfaceInReviewStalls", () => { + function staleMergingTask(overrides: Record = {}) { + return { + id: "FN-4110", + column: "in-review", + paused: false, + status: "merging", + mergeRetries: 0, + mergeDetails: {}, + worktree: "/tmp/FN-4110", + updatedAt: "2026-01-01T00:00:00.000Z", + steps: [{ name: "step", status: "done" }], + workflowStepResults: [], + log: [], + ...overrides, + }; + } + + it("logs FN-4110 stale transient merge status once without moving task", async () => { + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.getSettings as ReturnType).mockResolvedValue({ taskStuckTimeoutMs: 60_000 }); + (store.listTasks as ReturnType).mockResolvedValue([staleMergingTask()]); + + const result = await managerWithRecovery.surfaceInReviewStalls(); + + expect(result).toBe(1); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-4110", + expect.stringContaining("In-review stall surfaced [transient-merge-status-no-owner]:"), + ); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.moveTask).not.toHaveBeenCalled(); + managerWithRecovery.stop(); + }); + + it("deduplicates same code inside stuck-timeout window", async () => { + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.getSettings as ReturnType).mockResolvedValue({ taskStuckTimeoutMs: 60_000 }); + (store.listTasks as ReturnType).mockResolvedValue([ + staleMergingTask({ + log: [{ + timestamp: "2026-01-01T00:09:30.000Z", + action: "In-review stall surfaced [transient-merge-status-no-owner]: already surfaced", + }], + }), + ]); + + expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(0); + expect(store.logEntry).not.toHaveBeenCalled(); + managerWithRecovery.stop(); + }); + + it("re-logs after window expiry and on code transitions", async () => { + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.getSettings as ReturnType).mockResolvedValue({ taskStuckTimeoutMs: 60_000 }); + (store.listTasks as ReturnType) + .mockResolvedValueOnce([ + staleMergingTask({ + log: [{ + timestamp: "2026-01-01T00:08:00.000Z", + action: "In-review stall surfaced [transient-merge-status-no-owner]: old", + }], + }), + ]) + .mockResolvedValueOnce([ + staleMergingTask({ + status: undefined, + mergeRetries: 3, + log: [{ + timestamp: "2026-01-01T00:09:30.000Z", + action: "In-review stall surfaced [transient-merge-status-no-owner]: recent", + }], + }), + ]); + + expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(1); + expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(1); + expect(store.logEntry).toHaveBeenLastCalledWith( + "FN-4110", + expect.stringContaining("In-review stall surfaced [merge-retries-exhausted]:"), + ); + managerWithRecovery.stop(); + }); + + it("skips per-cycle dedup, paused, active merge owner, executing, awaiting-user-review, and mergeConfirmed", async () => { + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + const managerWithRecovery = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + getActiveMergeTaskId: () => "FN-ACTIVE", + getExecutingTaskIds: () => new Set(["FN-EXEC"]), + }); + (store.getSettings as ReturnType).mockResolvedValue({ taskStuckTimeoutMs: 60_000 }); + (store.listTasks as ReturnType).mockResolvedValue([ + staleMergingTask({ id: "FN-CYCLE", updatedAt: "2026-01-01T00:10:00.000Z" }), + staleMergingTask({ id: "FN-PAUSED", paused: true }), + staleMergingTask({ id: "FN-ACTIVE" }), + staleMergingTask({ id: "FN-EXEC" }), + staleMergingTask({ id: "FN-AWAIT", status: "awaiting-user-review" }), + staleMergingTask({ id: "FN-MERGED", mergeDetails: { mergeConfirmed: true } }), + ]); + + expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(0); + expect(store.logEntry).not.toHaveBeenCalled(); + managerWithRecovery.stop(); + }); + }); + describe("recoverGhostReviewTasks", () => { it("preserves failed in-review tasks so actionable merge failures are not ghost-retried", async () => { const managerWithRecovery = new SelfHealingManager(store, { diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 8132bc050..063f32e96 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -17,7 +17,7 @@ import { exec, execSync } from "node:child_process"; import { promisify } from "node:util"; import { existsSync, readdirSync, rmSync, statSync } from "node:fs"; import { isAbsolute, join, relative, resolve } from "node:path"; -import { getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core"; +import { getInReviewStallReason, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core"; import type { MeshLeaseManager } from "./mesh-lease-manager.js"; import { createLogger } from "./logger.js"; import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js"; @@ -325,6 +325,7 @@ export class SelfHealingManager { { name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns().then(() => undefined) }, { name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks().then(() => undefined) }, { name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy().then(() => undefined) }, + { name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) }, ]; for (const step of steps) { @@ -968,6 +969,7 @@ export class SelfHealingManager { { name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns() }, { name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks() }, { name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy() }, + { name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() }, ]; for (const fn of batch2Fns) { try { @@ -1696,6 +1698,58 @@ export class SelfHealingManager { * * @returns Number of tasks kicked back to todo */ + async surfaceInReviewStalls(): Promise { + try { + const settings = await this.store.getSettings(); + if (settings.globalPause || settings.enginePaused) return 0; + + const cycleStartMs = Date.now(); + const timeoutMs = settings.taskStuckTimeoutMs; + if (!timeoutMs || timeoutMs <= 0) return 0; + + const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null; + const executingTaskIds = this.options.getExecutingTaskIds?.() ?? new Set(); + const tasks = await this.store.listTasks({ column: "in-review", slim: false }); + let surfaced = 0; + + for (const task of tasks) { + const signal = getInReviewStallReason(task, { + now: cycleStartMs, + activeMergeTaskId, + executingTaskIds, + staleMergingMinAgeMs: this.options.staleMergingStatusMinAgeMs ?? DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS, + maxAutoMergeRetries: MAX_AUTO_MERGE_RETRIES, + }); + if (!signal) continue; + + if (Date.parse(task.updatedAt) >= cycleStartMs) { + continue; + } + + const previous = [...(task.log ?? [])] + .reverse() + .find((entry) => entry.action.startsWith("In-review stall surfaced [")); + if (previous) { + const parsed = /^In-review stall surfaced \[([^\]]+)\]/.exec(previous.action); + const previousCode = parsed?.[1]; + const previousAt = Date.parse(previous.timestamp); + if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - timeoutMs && previousCode === signal.code) { + continue; + } + } + + await this.store.logEntry(task.id, `In-review stall surfaced [${signal.code}]: ${signal.reason}`); + surfaced += 1; + } + + return surfaced; + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.error(`In-review stall surfacing failed: ${errorMessage}`); + return 0; + } + } + async recoverGhostReviewTasks(): Promise { try { const settings = await this.store.getSettings();