root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-5093): complete Step 4 — add self-healing in-review-stalled surfacing

Browse Source 
Fusion-Task-Id: FN-5093
Fusion-Task-Lineage: fe96617a-88f8-4a3c-b96c-c516d2206331
This commit is contained in:
Fusion (runfusion.ai)
2026-05-19 03:38:57 -07:00
committed by gsxdsm
parent 85bc219279
commit 275cee9c78
2 changed files with 167 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
packages/engine/src/__tests__/self-healing.test.ts
View File

@@ -522,6 +522,7 @@ describe("SelfHealingManager", () => {
const recoverAgentsRunningOnInactiveTasks = vi.spyOn(manager, "recoverAgentsRunningOnInactiveTasks").mockResolvedValue(1);
const clearStaleBlockedBy = vi.spyOn(manager, "clearStaleBlockedBy").mockResolvedValue(1);
const surfaceInReviewStalls = vi.spyOn(manager, "surfaceInReviewStalls").mockResolvedValue(1);
const surfaceInReviewStalled = vi.spyOn(manager, "surfaceInReviewStalled").mockResolvedValue(1);
const surfaceStalePausedReviews = vi.spyOn(manager, "surfaceStalePausedReviews").mockResolvedValue(1);
const surfaceStalePausedTodos = vi.spyOn(manager, "surfaceStalePausedTodos").mockResolvedValue(1);
@@ -538,6 +539,7 @@ describe("SelfHealingManager", () => {
expect(recoverAgentsRunningOnInactiveTasks).toHaveBeenCalledTimes(1);
expect(clearStaleBlockedBy).toHaveBeenCalledTimes(1);
expect(surfaceInReviewStalls).toHaveBeenCalledTimes(1);
expect(surfaceInReviewStalled).toHaveBeenCalledTimes(1);
expect(surfaceStalePausedReviews).toHaveBeenCalledTimes(1);
expect(surfaceStalePausedTodos).toHaveBeenCalledTimes(1);
});
@@ -4909,6 +4911,101 @@ describe("SelfHealingManager", () => {
});
});
describe("surfaceInReviewStalled", () => {
function inReviewTask(overrides: Record<string, unknown> = {}) {
return {
id: "FN-5093",
column: "in-review",
paused: false,
status: "in-review",
mergeDetails: {},
columnMovedAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
log: [],
...overrides,
};
}
it("logs for quiet in-review tasks beyond threshold", async () => {
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: true });
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([inReviewTask()]);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(1);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-5093",
expect.stringContaining("In-review stalled surfaced [in-review-stalled]: quiet"),
);
expect(store.logEntry).toHaveBeenCalledWith("FN-5093", expect.stringContaining("lastActivitySource=column-moved"));
managerWithRecovery.stop();
});
it("skips for recent activity, paused, global pause, engine pause, autoMerge off, threshold off, executing, and active merge", async () => {
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getExecutingTaskIds: () => new Set(["FN-EXEC"]),
getActiveMergeTaskId: () => "FN-MERGE",
});
(store.getSettings as ReturnType<typeof vi.fn>)
.mockResolvedValueOnce({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: true })
.mockResolvedValueOnce({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: true, globalPause: true })
.mockResolvedValueOnce({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: true, enginePaused: true })
.mockResolvedValueOnce({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: false })
.mockResolvedValueOnce({ inReviewStalledThresholdMs: 0, autoMerge: true });
(store.listTasks as ReturnType<typeof vi.fn>)
.mockResolvedValueOnce([
inReviewTask({ id: "FN-RECENT", log: [{ timestamp: "2026-01-02T00:59:59.000Z", action: "recent" }] }),
inReviewTask({ id: "FN-PAUSED", paused: true }),
inReviewTask({ id: "FN-EXEC" }),
inReviewTask({ id: "FN-MERGE" }),
]);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(store.logEntry).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("dedupes within threshold window and re-emits after window", async () => {
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: true });
(store.listTasks as ReturnType<typeof vi.fn>)
.mockResolvedValueOnce([
inReviewTask({
log: [{ timestamp: "2026-01-01T12:00:00.000Z", action: "In-review stalled surfaced [in-review-stalled]: recent" }],
}),
])
.mockResolvedValueOnce([
inReviewTask({
log: [{ timestamp: "2025-12-29T00:00:00.000Z", action: "In-review stalled surfaced [in-review-stalled]: old" }],
}),
]);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(1);
managerWithRecovery.stop();
});
it("suppresses while recent reason-driven in-review stall exists", async () => {
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ inReviewStalledThresholdMs: 24 * 60 * 60_000, autoMerge: true });
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
inReviewTask({ log: [{ timestamp: "2026-01-02T00:10:00.000Z", action: "In-review stall surfaced [merge-blocker]: blocked" }] }),
]);
expect(await managerWithRecovery.surfaceInReviewStalled()).toBe(0);
expect(store.logEntry).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
});
describe("surfaceStalePausedReviews", () => {
function pausedReviewTask(overrides: Record<string, unknown> = {}) {
return {

71
packages/engine/src/self-healing.ts
View File

@@ -28,7 +28,7 @@ import { exec, execSync } from "node:child_process";
import { promisify } from "node:util";
import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
import { isAbsolute, join, relative, resolve } from "node:path";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js";
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
@@ -665,6 +665,7 @@ export class SelfHealingManager {
{ name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches().then(() => undefined) },
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
{ name: "surface-in-review-stalled", fn: () => this.surfaceInReviewStalled().then(() => undefined) },
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews().then(() => undefined) },
{ name: "surface-stale-paused-todos", fn: () => this.surfaceStalePausedTodos().then(() => undefined) },
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates().then(() => undefined) },
@@ -1252,6 +1253,7 @@ export class SelfHealingManager {
{ name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches() },
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
{ name: "surface-in-review-stalled", fn: () => this.surfaceInReviewStalled() },
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
{ name: "surface-stale-paused-todos", fn: () => this.surfaceStalePausedTodos() },
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates() },
@@ -4078,6 +4080,73 @@ export class SelfHealingManager {
}
}
/**
* Surface quiet-window backlog-health diagnostics for unpaused in-review tasks.
*
* Non-overlap contract:
* - `surfaceStalePausedReviews()` owns paused in-review tasks.
* - `surfaceInReviewStalls()` owns reason-driven in-review stalls.
*/
async surfaceInReviewStalled(): Promise<number> {
try {
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) return 0;
if (!settings.autoMerge) return 0;
const cycleStartMs = Date.now();
const thresholdMs = settings.inReviewStalledThresholdMs;
if (!thresholdMs || thresholdMs <= 0) return 0;
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null;
const executingTaskIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
let surfaced = 0;
for (const task of tasks) {
if (task.paused === true) continue;
if (task.id === activeMergeTaskId || executingTaskIds.has(task.id)) continue;
const signal = getInReviewStalledSignal(task, {
now: cycleStartMs,
thresholdMs,
autoMerge: true,
activeMergeTaskId,
executingTaskIds,
});
if (!signal) continue;
if (Date.parse(task.updatedAt) >= cycleStartMs) {
continue;
}
const previous = [...(task.log ?? [])]
.reverse()
.find((entry) => entry.action.startsWith("In-review stalled surfaced ["));
if (previous) {
const parsed = /^In-review stalled surfaced \[([^\]]+)\]/.exec(previous.action);
const previousCode = parsed?.[1];
const previousAt = Date.parse(previous.timestamp);
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - thresholdMs && previousCode === signal.code) {
continue;
}
}
const hours = (signal.quietMs / 3_600_000).toFixed(1);
await this.store.logEntry(
task.id,
`In-review stalled surfaced [${signal.code}]: quiet ${hours}h beyond ${(thresholdMs / 3_600_000).toFixed(1)}h threshold; disposition options — nudge review, retry, archive, or create follow-up task. lastActivitySource=${signal.lastActivitySource}`,
);
surfaced += 1;
}
return surfaced;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`In-review stalled surfacing failed: ${errorMessage}`);
return 0;
}
}
async surfaceStalePausedReviews(): Promise<number> {
try {
const settings = await this.store.getSettings();