root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-4217): complete Step 4 — surface in-review stalls in self-healing

Browse Source 
Fusion-Task-Id: FN-4217
Fusion-Task-Lineage: f5434e90-ad0c-41c5-bee9-1244cb69622b
This commit is contained in:
Fusion
2026-05-13 02:17:02 -07:00
committed by gsxdsm
parent 426d4fb67d
commit 988420325b
2 changed files with 167 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
packages/engine/src/__tests__/self-healing.test.ts
View File

@@ -442,6 +442,7 @@ describe("SelfHealingManager", () => {
const recoverOrphanedAgents = vi.spyOn(manager, "recoverOrphanedAgents").mockResolvedValue(1);
const recoverAgentsRunningOnInactiveTasks = vi.spyOn(manager, "recoverAgentsRunningOnInactiveTasks").mockResolvedValue(1);
const clearStaleBlockedBy = vi.spyOn(manager, "clearStaleBlockedBy").mockResolvedValue(1);
const surfaceInReviewStalls = vi.spyOn(manager, "surfaceInReviewStalls").mockResolvedValue(1);
await manager.runStartupRecovery();
@@ -455,6 +456,7 @@ describe("SelfHealingManager", () => {
expect(recoverOrphanedAgents).toHaveBeenCalledTimes(1);
expect(recoverAgentsRunningOnInactiveTasks).toHaveBeenCalledTimes(1);
expect(clearStaleBlockedBy).toHaveBeenCalledTimes(1);
expect(surfaceInReviewStalls).toHaveBeenCalledTimes(1);
});
it("runStartupRecovery clears stale blockedBy rows", async () => {
@@ -3805,6 +3807,116 @@ describe("SelfHealingManager", () => {
});
});
describe("surfaceInReviewStalls", () => {
function staleMergingTask(overrides: Record<string, unknown> = {}) {
return {
id: "FN-4110",
column: "in-review",
paused: false,
status: "merging",
mergeRetries: 0,
mergeDetails: {},
worktree: "/tmp/FN-4110",
updatedAt: "2026-01-01T00:00:00.000Z",
steps: [{ name: "step", status: "done" }],
workflowStepResults: [],
log: [],
...overrides,
};
}
it("logs FN-4110 stale transient merge status once without moving task", async () => {
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ taskStuckTimeoutMs: 60_000 });
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([staleMergingTask()]);
const result = await managerWithRecovery.surfaceInReviewStalls();
expect(result).toBe(1);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-4110",
expect.stringContaining("In-review stall surfaced [transient-merge-status-no-owner]:"),
);
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("deduplicates same code inside stuck-timeout window", async () => {
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ taskStuckTimeoutMs: 60_000 });
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
staleMergingTask({
log: [{
timestamp: "2026-01-01T00:09:30.000Z",
action: "In-review stall surfaced [transient-merge-status-no-owner]: already surfaced",
}],
}),
]);
expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(0);
expect(store.logEntry).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("re-logs after window expiry and on code transitions", async () => {
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ taskStuckTimeoutMs: 60_000 });
(store.listTasks as ReturnType<typeof vi.fn>)
.mockResolvedValueOnce([
staleMergingTask({
log: [{
timestamp: "2026-01-01T00:08:00.000Z",
action: "In-review stall surfaced [transient-merge-status-no-owner]: old",
}],
}),
])
.mockResolvedValueOnce([
staleMergingTask({
status: undefined,
mergeRetries: 3,
log: [{
timestamp: "2026-01-01T00:09:30.000Z",
action: "In-review stall surfaced [transient-merge-status-no-owner]: recent",
}],
}),
]);
expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(1);
expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(1);
expect(store.logEntry).toHaveBeenLastCalledWith(
"FN-4110",
expect.stringContaining("In-review stall surfaced [merge-retries-exhausted]:"),
);
managerWithRecovery.stop();
});
it("skips per-cycle dedup, paused, active merge owner, executing, awaiting-user-review, and mergeConfirmed", async () => {
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
getActiveMergeTaskId: () => "FN-ACTIVE",
getExecutingTaskIds: () => new Set(["FN-EXEC"]),
});
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ taskStuckTimeoutMs: 60_000 });
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
staleMergingTask({ id: "FN-CYCLE", updatedAt: "2026-01-01T00:10:00.000Z" }),
staleMergingTask({ id: "FN-PAUSED", paused: true }),
staleMergingTask({ id: "FN-ACTIVE" }),
staleMergingTask({ id: "FN-EXEC" }),
staleMergingTask({ id: "FN-AWAIT", status: "awaiting-user-review" }),
staleMergingTask({ id: "FN-MERGED", mergeDetails: { mergeConfirmed: true } }),
]);
expect(await managerWithRecovery.surfaceInReviewStalls()).toBe(0);
expect(store.logEntry).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
});
describe("recoverGhostReviewTasks", () => {
it("preserves failed in-review tasks so actionable merge failures are not ghost-retried", async () => {
const managerWithRecovery = new SelfHealingManager(store, {

56
packages/engine/src/self-healing.ts
View File

@@ -17,7 +17,7 @@ import { exec, execSync } from "node:child_process";
import { promisify } from "node:util";
import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
import { isAbsolute, join, relative, resolve } from "node:path";
import { getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
import { getInReviewStallReason, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js";
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
@@ -325,6 +325,7 @@ export class SelfHealingManager {
{ name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns().then(() => undefined) },
{ name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks().then(() => undefined) },
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy().then(() => undefined) },
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
];
for (const step of steps) {
@@ -968,6 +969,7 @@ export class SelfHealingManager {
{ name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns() },
{ name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks() },
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy() },
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
];
for (const fn of batch2Fns) {
try {
@@ -1696,6 +1698,58 @@ export class SelfHealingManager {
*
* @returns Number of tasks kicked back to todo
*/
async surfaceInReviewStalls(): Promise<number> {
try {
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) return 0;
const cycleStartMs = Date.now();
const timeoutMs = settings.taskStuckTimeoutMs;
if (!timeoutMs || timeoutMs <= 0) return 0;
const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null;
const executingTaskIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
let surfaced = 0;
for (const task of tasks) {
const signal = getInReviewStallReason(task, {
now: cycleStartMs,
activeMergeTaskId,
executingTaskIds,
staleMergingMinAgeMs: this.options.staleMergingStatusMinAgeMs ?? DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS,
maxAutoMergeRetries: MAX_AUTO_MERGE_RETRIES,
});
if (!signal) continue;
if (Date.parse(task.updatedAt) >= cycleStartMs) {
continue;
}
const previous = [...(task.log ?? [])]
.reverse()
.find((entry) => entry.action.startsWith("In-review stall surfaced ["));
if (previous) {
const parsed = /^In-review stall surfaced \[([^\]]+)\]/.exec(previous.action);
const previousCode = parsed?.[1];
const previousAt = Date.parse(previous.timestamp);
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - timeoutMs && previousCode === signal.code) {
continue;
}
}
await this.store.logEntry(task.id, `In-review stall surfaced [${signal.code}]: ${signal.reason}`);
surfaced += 1;
}
return surfaced;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`In-review stall surfacing failed: ${errorMessage}`);
return 0;
}
}
async recoverGhostReviewTasks(): Promise<number> {
try {
const settings = await this.store.getSettings();