feat(FN-4452): complete Step 3 — surface stale paused reviews in self-healing
Fusion-Task-Id: FN-4452 Fusion-Task-Lineage: 7d8b1c13-883b-4815-94ee-fdf18ad24e5f
This commit is contained in:
@@ -488,6 +488,7 @@ describe("SelfHealingManager", () => {
|
||||
const recoverAgentsRunningOnInactiveTasks = vi.spyOn(manager, "recoverAgentsRunningOnInactiveTasks").mockResolvedValue(1);
|
||||
const clearStaleBlockedBy = vi.spyOn(manager, "clearStaleBlockedBy").mockResolvedValue(1);
|
||||
const surfaceInReviewStalls = vi.spyOn(manager, "surfaceInReviewStalls").mockResolvedValue(1);
|
||||
const surfaceStalePausedReviews = vi.spyOn(manager, "surfaceStalePausedReviews").mockResolvedValue(1);
|
||||
|
||||
await manager.runStartupRecovery();
|
||||
|
||||
@@ -502,6 +503,7 @@ describe("SelfHealingManager", () => {
|
||||
expect(recoverAgentsRunningOnInactiveTasks).toHaveBeenCalledTimes(1);
|
||||
expect(clearStaleBlockedBy).toHaveBeenCalledTimes(1);
|
||||
expect(surfaceInReviewStalls).toHaveBeenCalledTimes(1);
|
||||
expect(surfaceStalePausedReviews).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("runStartupRecovery clears stale blockedBy rows", async () => {
|
||||
@@ -4265,6 +4267,98 @@ describe("SelfHealingManager", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("surfaceStalePausedReviews", () => {
|
||||
function pausedReviewTask(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: "FN-4233",
|
||||
column: "in-review",
|
||||
paused: true,
|
||||
pausedReason: "manual-hold",
|
||||
mergeDetails: {},
|
||||
columnMovedAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
log: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it("no-ops under threshold", async () => {
|
||||
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
|
||||
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ stalePausedReviewThresholdMs: 24 * 60 * 60_000 });
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([pausedReviewTask()]);
|
||||
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(0);
|
||||
expect(store.logEntry).not.toHaveBeenCalled();
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
|
||||
it("logs disposition recommendation when threshold met", async () => {
|
||||
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
|
||||
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ stalePausedReviewThresholdMs: 24 * 60 * 60_000 });
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([pausedReviewTask()]);
|
||||
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(1);
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-4233",
|
||||
expect.stringContaining("Stale paused review surfaced [stale-paused-review]: paused"),
|
||||
);
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-4233",
|
||||
expect.stringContaining("disposition options — unpause, retry, archive, or create follow-up task"),
|
||||
);
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
|
||||
it("skips merge-confirmed, non-paused, recently-updated, and paused/global short-circuit", async () => {
|
||||
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
|
||||
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
(store.getSettings as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce({ stalePausedReviewThresholdMs: 24 * 60 * 60_000 })
|
||||
.mockResolvedValueOnce({ stalePausedReviewThresholdMs: 24 * 60 * 60_000, globalPause: true })
|
||||
.mockResolvedValueOnce({ stalePausedReviewThresholdMs: 24 * 60 * 60_000, enginePaused: true });
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||
pausedReviewTask({ id: "FN-MERGED", mergeDetails: { mergeConfirmed: true } }),
|
||||
pausedReviewTask({ id: "FN-RUN", paused: false }),
|
||||
pausedReviewTask({ id: "FN-UPD", updatedAt: "2026-01-02T01:00:00.000Z" }),
|
||||
]);
|
||||
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(0);
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(0);
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(0);
|
||||
expect(store.logEntry).not.toHaveBeenCalled();
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
|
||||
it("rate-limits within window and re-emits after threshold window", async () => {
|
||||
vi.setSystemTime(new Date("2026-01-02T01:00:00.000Z"));
|
||||
const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ stalePausedReviewThresholdMs: 24 * 60 * 60_000 });
|
||||
(store.listTasks as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce([
|
||||
pausedReviewTask({
|
||||
log: [{
|
||||
timestamp: "2026-01-01T12:00:00.000Z",
|
||||
action: "Stale paused review surfaced [stale-paused-review]: recent",
|
||||
}],
|
||||
}),
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
pausedReviewTask({
|
||||
log: [{
|
||||
timestamp: "2025-12-30T00:00:00.000Z",
|
||||
action: "Stale paused review surfaced [stale-paused-review]: old",
|
||||
}],
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(0);
|
||||
expect(await managerWithRecovery.surfaceStalePausedReviews()).toBe(1);
|
||||
managerWithRecovery.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe("recoverGhostReviewTasks", () => {
|
||||
it("preserves failed in-review tasks so actionable merge failures are not ghost-retried", async () => {
|
||||
const managerWithRecovery = new SelfHealingManager(store, {
|
||||
|
||||
@@ -17,7 +17,7 @@ import { exec, execSync } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { getInReviewStallReason, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
|
||||
import { getInReviewStallReason, getStalePausedReviewSignal, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
|
||||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { getRegisteredWorktreePaths, isUsableTaskWorktree, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
@@ -439,6 +439,7 @@ export class SelfHealingManager {
|
||||
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy().then(() => undefined) },
|
||||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts().then(() => undefined) },
|
||||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
|
||||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews().then(() => undefined) },
|
||||
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates().then(() => undefined) },
|
||||
];
|
||||
|
||||
@@ -1152,6 +1153,7 @@ export class SelfHealingManager {
|
||||
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy() },
|
||||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
|
||||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
|
||||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
|
||||
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates() },
|
||||
];
|
||||
for (const fn of batch2Fns) {
|
||||
@@ -2517,6 +2519,52 @@ export class SelfHealingManager {
|
||||
}
|
||||
}
|
||||
|
||||
async surfaceStalePausedReviews(): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) return 0;
|
||||
|
||||
const cycleStartMs = Date.now();
|
||||
const thresholdMs = settings.stalePausedReviewThresholdMs;
|
||||
if (!thresholdMs || thresholdMs <= 0) return 0;
|
||||
|
||||
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
|
||||
let surfaced = 0;
|
||||
|
||||
for (const task of tasks) {
|
||||
if (task.paused !== true) continue;
|
||||
const signal = getStalePausedReviewSignal(task, { now: cycleStartMs, thresholdMs });
|
||||
if (!signal) continue;
|
||||
if (Date.parse(task.updatedAt) >= cycleStartMs) continue;
|
||||
|
||||
const previous = [...(task.log ?? [])]
|
||||
.reverse()
|
||||
.find((entry) => entry.action.startsWith("Stale paused review surfaced ["));
|
||||
if (previous) {
|
||||
const parsed = /^Stale paused review surfaced \[([^\]]+)\]/.exec(previous.action);
|
||||
const previousCode = parsed?.[1];
|
||||
const previousAt = Date.parse(previous.timestamp);
|
||||
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - thresholdMs && previousCode === signal.code) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const hours = (signal.ageMs / 3_600_000).toFixed(1);
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Stale paused review surfaced [${signal.code}]: paused ${hours}h; disposition options — unpause, retry, archive, or create follow-up task. pausedReason=${signal.pausedReason ?? "none"}`,
|
||||
);
|
||||
surfaced += 1;
|
||||
}
|
||||
|
||||
return surfaced;
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.error(`Stale paused review surfacing failed: ${errorMessage}`);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async recoverGhostReviewTasks(): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
|
||||
Reference in New Issue
Block a user