fix(FN-5335): enforce triple-proof gating for backward recovery

- Gate self-healing backward moves behind audited triple-proof predicates across reclaim paths
- Skip reclaim-pr-conflict mutations when proof checks fail and preserve no-action behavior
- Add broad unit and reliability-interaction coverage for triple-proof and cross-layer scenarios
- Document backward-move stage invariants, diagnostics, and add delivery changeset for @runfusion/fusion
This commit is contained in:
Fusion (runfusion.ai)
2026-05-21 19:15:27 -07:00
committed by gsxdsm
parent a2a5db8151
commit a8715ed963
16 changed files with 914 additions and 44 deletions

View File

@@ -0,0 +1,275 @@
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { execSync } from "node:child_process";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { TaskStore } from "@fusion/core";
import { SelfHealingManager } from "../../self-healing.js";
import { activeSessionRegistry, executingTaskLock } from "../../active-session-registry.js";
function git(cwd: string, command: string): string {
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
}
describe("FN-5335 reliability interactions: backward move triple proof", () => {
let rootDir = "";
let store: TaskStore;
beforeEach(async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-21T12:00:00.000Z"));
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
rootDir = mkdtempSync(join(tmpdir(), "fn-5335-reliability-"));
git(rootDir, "init -b main");
git(rootDir, "config user.name 'Fusion'");
git(rootDir, "config user.email 'hi@runfusion.ai'");
writeFileSync(join(rootDir, "README.md"), "root\n");
git(rootDir, "add README.md");
git(rootDir, "commit -m 'init'");
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
await store.init();
});
afterEach(() => {
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
try { store?.close(); } catch {}
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
vi.useRealTimers();
vi.clearAllMocks();
});
async function createNoProgressTask(worktree: string, ageMs: number) {
const task = await store.createTask({ title: "no-progress", description: "no-progress" });
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
await store.updateTask(task.id, {
worktree,
status: "failed",
paused: false,
error: "Agent finished without calling fn_task_done",
executionStartedAt: new Date(Date.now() - ageMs).toISOString(),
updatedAt: new Date(Date.now() - ageMs).toISOString(),
steps: [{ name: "step", status: "pending" }],
} as any);
return task.id;
}
it("Scenario A: live session blocks backward move and emits no-action", async () => {
const worktree = join(rootDir, ".worktrees", "np-live-missing");
const id = await createNoProgressTask(worktree, 400_000);
activeSessionRegistry.registerPath(worktree, { taskId: id, kind: "executor", ownerKey: "run-a" });
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
const task = await store.getTask(id);
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
expect(recovered).toBe(0);
expect(task?.column).toBe("in-progress");
expect(events).toHaveLength(1);
manager.stop();
});
it("Scenario B: usable worktree blocks backward move (recoverable git work short-circuit)", async () => {
const worktree = join(rootDir, ".worktrees", "np-usable");
mkdirSync(worktree, { recursive: true });
const id = await createNoProgressTask(worktree, 400_000);
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
const task = await store.getTask(id);
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
expect(recovered).toBe(0);
expect(task?.column).toBe("in-progress");
expect(events).toHaveLength(0);
manager.stop();
});
it("Scenario C: recent activity blocks backward move", async () => {
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-recent"), 200);
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
const task = await store.getTask(id);
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
expect(recovered).toBe(0);
expect(task?.column).toBe("in-progress");
expect(events).toHaveLength(1);
manager.stop();
});
it("Scenario D: all triple-proof signals true allows recovery", async () => {
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-stale"), 400_000);
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
const task = await store.getTask(id);
expect(recovered).toBe(1);
expect(task?.column).toBe("todo");
manager.stop();
});
it("Scenario E: autoMerge false keeps in-review stage no-op", async () => {
const task = await store.createTask({ title: "review", description: "review" });
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
await store.moveTask(task.id, "in-review");
await store.updateTask(task.id, {
status: "failed",
error: "Agent finished without calling fn_task_done",
taskDoneRetryCount: 1,
worktree: join(rootDir, ".worktrees", "review-missing"),
updatedAt: new Date(Date.now() - 400_000).toISOString(),
steps: [{ name: "done", status: "done" }, { name: "pending", status: "pending" }],
} as any);
await store.updateSettings({ ...(await store.getSettings()), autoMerge: false } as any);
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
const current = await store.getTask(task.id);
expect(recovered).toBe(0);
expect(current?.column).toBe("in-review");
manager.stop();
});
it("Scenario F: churn-terminalized review task is not reopened", async () => {
const task = await store.createTask({ title: "churn", description: "churn" });
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
await store.moveTask(task.id, "in-review");
await store.updateTask(task.id, {
paused: true,
pausedReason: "in-review-stall-deadlock",
status: "failed",
error: "STUCK_NO_PROGRESS_CHURN",
worktree: join(rootDir, ".worktrees", "churn-missing"),
updatedAt: new Date(Date.now() - 400_000).toISOString(),
steps: [{ name: "done", status: "done" }, { name: "pending", status: "pending" }],
} as any);
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
const current = await store.getTask(task.id);
expect(recovered).toBe(0);
expect(current?.column).toBe("in-review");
manager.stop();
});
it("Scenario G: order-independence across limbo and no-progress sweeps", async () => {
const makeCandidate = async (title: string) => {
const task = await store.createTask({ title, description: title });
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
await store.updateTask(task.id, {
branch: null,
worktree: join(rootDir, ".worktrees", `${task.id.toLowerCase()}-missing`),
status: "failed",
error: "Agent finished without calling fn_task_done",
executionStartedAt: new Date(Date.now() - 400_000).toISOString(),
updatedAt: new Date(Date.now() - 400_000).toISOString(),
steps: [{ name: "step", status: "pending" }],
} as any);
return task.id;
};
const firstId = await makeCandidate("order-a");
await (store as any).recordRunAuditEvent({ runId: "run-g-a", phase: "executor", taskId: firstId, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: firstId, details: null, metadata: {} });
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false, getExecutingTaskIds: () => new Set<string>() });
await manager.recoverNoProgressNoTaskDoneFailures();
await manager.recoverInProgressLimbo();
const firstTask = await store.getTask(firstId);
const secondId = await makeCandidate("order-b");
await (store as any).recordRunAuditEvent({ runId: "run-g-b", phase: "executor", taskId: secondId, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: secondId, details: null, metadata: {} });
await manager.recoverInProgressLimbo();
await manager.recoverNoProgressNoTaskDoneFailures();
const secondTask = await store.getTask(secondId);
expect(firstTask?.column).toBe("in-progress");
expect(secondTask?.column).toBe("in-progress");
manager.stop();
});
it("Scenario H: orphan and tightened sweeps can co-emit no-action events", async () => {
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-orphan-h"), 400_000);
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>(), isTaskActive: () => false });
vi.setSystemTime(new Date("2026-05-21T12:10:00.000Z"));
await manager.recoverOrphanedExecutions();
await (store as any).recordRunAuditEvent({ runId: "run-h", phase: "executor", taskId: id, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: id, details: null, metadata: {} });
await manager.recoverNoProgressNoTaskDoneFailures();
const orphanEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:orphan-detected-no-action" });
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
const current = await store.getTask(id);
expect(current?.column).toBe("in-progress");
expect(orphanEvents).toHaveLength(1);
expect(noActionEvents).toHaveLength(1);
manager.stop();
});
it("Scenario J: tightened sweep no-action is idempotent across re-sweeps", async () => {
const worktree = join(rootDir, ".worktrees", "np-missing-orphan");
const id = await createNoProgressTask(worktree, 400_000);
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>(), isTaskActive: () => false });
vi.setSystemTime(new Date("2026-05-21T12:10:00.000Z"));
await manager.recoverOrphanedExecutions();
await (store as any).recordRunAuditEvent({
runId: "run-fn5335-h",
phase: "executor",
taskId: id,
taskLineageId: null,
agentId: "executor",
domain: "database",
mutationType: "worktree:incomplete-detected",
payload: { source: "executor-liveness-gate" },
target: id,
details: null,
metadata: { source: "executor-liveness-gate" },
});
const first = await manager.recoverNoProgressNoTaskDoneFailures();
const second = await manager.recoverNoProgressNoTaskDoneFailures();
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
const current = await store.getTask(id);
expect(first).toBe(0);
expect(second).toBe(0);
expect(current?.column).toBe("in-progress");
expect(noActionEvents).toHaveLength(2);
manager.stop();
});
it("Scenario I: recent worktree:incomplete-detected audit counts as recent activity", async () => {
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-liveness"), 400_000);
await (store as any).recordRunAuditEvent({
runId: "run-fn5335-liveness",
phase: "executor",
taskId: id,
taskLineageId: null,
agentId: "executor",
domain: "database",
mutationType: "worktree:incomplete-detected",
payload: { source: "executor-liveness-gate" },
target: id,
details: null,
metadata: { source: "executor-liveness-gate" },
});
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
expect(recovered).toBe(0);
expect(noActionEvents).toHaveLength(1);
manager.stop();
});
});

View File

@@ -14,6 +14,7 @@ function makeTask(id: string, overrides: Partial<Task> = {}): Task {
currentStep: 0,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
executionStartedAt: new Date(Date.now() - 61_000).toISOString(),
log: [],
...overrides,
} as Task;

View File

@@ -56,7 +56,7 @@ describe("reliability interactions: live-zero reclaim", () => {
pausedReason: "branch-conflict-unrecoverable",
userPaused: false,
lineageId: "lin-9100",
updatedAt: new Date().toISOString(),
updatedAt: new Date(Date.now() - 11 * 60_000).toISOString(),
};
const statefulStore = {

View File

@@ -143,8 +143,8 @@ describe("foreign start-point no-owned-commit interactions (real git)", () => {
const recovered = await manager.finalizeNoOpReviewTasks();
expect(recovered).toBe(0);
expect(task.column).toBe("todo");
expect(events.some((event: any) => event?.mutationType === "task:finalize-unproven-blocked")).toBe(true);
expect(task.column).toBe("in-review");
expect(events.some((event: any) => event?.mutationType === "task:finalize-no-op-review-no-action")).toBe(true);
manager.stop();
} finally {
rmSync(dir, { recursive: true, force: true });

View File

@@ -125,6 +125,7 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
it("keeps in-review missing-worktree failures on the review-specific recovery path", async () => {
const id = await createInProgressTask("review failure disjoint");
await store.moveTask(id, "in-review");
await store.updateSettings({ autoMerge: true } as any);
await store.updateTask(id, {
status: "failed",
error: `Refusing to start coding agent in missing worktree: ${join(rootDir, ".worktrees", "missing-review")}`,
@@ -132,6 +133,10 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
worktree: join(rootDir, ".worktrees", "missing-review-stale"),
steps: [{ name: "step", status: "done" }, { name: "next", status: "pending" }],
});
await store.updateTask(id, {
updatedAt: new Date(Date.now() - 48 * 60 * 60_000).toISOString(),
columnMovedAt: new Date(Date.now() - 48 * 60 * 60_000).toISOString(),
} as any);
const manager = new SelfHealingManager(store, {
rootDir,
@@ -143,8 +148,8 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
const updated = await store.getTask(id);
expect(limboRecovered).toBe(0);
expect(reviewRecovered).toBe(1);
expect(updated?.column).toBe("todo");
expect(reviewRecovered).toBe(0);
expect(updated?.column).toBe("in-review");
});
it("skips limbo recovery while the executor still claims the task id", async () => {

View File

@@ -64,6 +64,7 @@ describe("reliability interactions: paused scope decay", () => {
column: "in-progress",
paused: true,
pausedReason: "waiting",
executionStartedAt: new Date(now - 31 * 60_000).toISOString(),
columnMovedAt: new Date(now - 31 * 60_000).toISOString(),
currentStep: 2,
steps: [{ id: "s1", title: "x", status: "done" } as any],
@@ -97,6 +98,7 @@ describe("reliability interactions: paused scope decay", () => {
const holder = makeTask("FN-3", {
column: "in-progress",
paused: true,
executionStartedAt: new Date(now - 61_000).toISOString(),
columnMovedAt: new Date(now - 1_000).toISOString(),
});
const follower = makeTask("FN-4", { column: "todo", blockedBy: "FN-3" });

View File

@@ -76,7 +76,7 @@ describe("reliability interaction: pr conflict reclaim", () => {
});
it("keeps paused-review reclaim path resumable", async () => {
const t = task();
const t = task({ updatedAt: new Date(Date.now() - 11 * 60_000).toISOString() });
const s = store(t);
vi.spyOn(branchConflicts, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: t.worktree, tipSha: "abc123", taskAttributedCommitCount: 2, strandedCommits: [{ sha: "abc123" }] } as any);
const manager = new SelfHealingManager(s as any, { rootDir: "/tmp/test" } as any);

View File

@@ -86,6 +86,7 @@ describe("reliability interactions: self-healing", () => {
worktree: "/tmp/wt",
branch: "fusion/wt",
steps: [{ id: "s1", title: "Step", status: "pending" }] as any,
updatedAt: new Date(Date.now() - 31 * 60_000).toISOString(),
}),
]]);
const store = makeStore(tasks);

View File

@@ -107,7 +107,7 @@ describe("SelfHealingManager.reclaimPrConflictForTask", () => {
});
it("returns reclaimed for reclaimable conflicts", async () => {
const task = makeTask({ column: "in-review", paused: true, pausedReason: "branch-conflict-unrecoverable" as any });
const task = makeTask({ column: "in-review", paused: true, pausedReason: "branch-conflict-unrecoverable" as any, updatedAt: new Date(Date.now() - 11 * 60_000).toISOString() });
const store = makeStore(task);
vi.spyOn(branchConflicts, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: task.worktree, tipSha: "abc123", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc123" }] } as any);
const manager = new SelfHealingManager(store as any, { rootDir: "/tmp/test" } as any);

View File

@@ -73,6 +73,9 @@ vi.mock("../worktree-pool.js", () => ({
scanOrphanedBranches: vi.fn().mockResolvedValue([]),
cleanupOrphanedWorktrees: vi.fn().mockResolvedValue(0),
isUsableTaskWorktree: vi.fn().mockResolvedValue(true),
classifyTaskWorktree: vi.fn().mockResolvedValue({ ok: false, classification: "missing", reason: "test-default" }),
getRegisteredWorktreePaths: vi.fn().mockResolvedValue(new Set<string>()),
getRegisteredWorktreeBranchMap: vi.fn().mockResolvedValue(new Map<string, string>()),
removeWorktree: vi.fn().mockResolvedValue(undefined),
resolveWorktreeBackend: vi.fn(),
}));
@@ -102,7 +105,8 @@ import { existsSync, readdirSync } from "node:fs";
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "../worktree-pool.js";
import { classifyTaskWorktree, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "../worktree-pool.js";
import { activeSessionRegistry, executingTaskLock } from "../active-session-registry.js";
import * as branchConflictModule from "../branch-conflicts.js";
import { createLogger } from "../logger.js";
import { NotificationService } from "../notification/notification-service.js";
@@ -111,6 +115,9 @@ import { classifyOwnedLandedEvidence } from "../merger.js";
const mockedExecSync = vi.mocked(execSync);
const mockedExistsSync = vi.mocked(existsSync);
const mockedIsUsableTaskWorktree = vi.mocked(isUsableTaskWorktree);
const mockedClassifyTaskWorktree = vi.mocked(classifyTaskWorktree);
const mockedGetRegisteredWorktreePaths = vi.mocked(getRegisteredWorktreePaths);
const mockedGetRegisteredWorktreeBranchMap = vi.mocked(getRegisteredWorktreeBranchMap);
const mockedRemoveWorktree = vi.mocked(removeWorktree);
const mockedResolveWorktreeBackend = vi.mocked(resolveWorktreeBackend);
const mockedScanIdleWorktrees = vi.mocked(scanIdleWorktrees);
@@ -160,6 +167,7 @@ function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & E
listTasks: vi.fn().mockResolvedValue([]),
createTask: vi.fn().mockResolvedValue({ id: "FN-RESCUE", lineageId: "lin-rescue" }),
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
getRunAuditEvents: vi.fn().mockReturnValue([]),
getBootstrappedAt: vi.fn().mockReturnValue(null),
getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
clearStaleExecutionStartBranchReferences: vi.fn().mockReturnValue([]),
@@ -174,14 +182,21 @@ describe("SelfHealingManager", () => {
beforeEach(() => {
vi.useFakeTimers({ shouldAdvanceTime: true });
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
store = createMockStore();
manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
mockedRemoveWorktree.mockResolvedValue(undefined);
mockedClassifyTaskWorktree.mockResolvedValue({ ok: false, classification: "missing", reason: "test-default" });
mockedGetRegisteredWorktreePaths.mockResolvedValue(new Set<string>());
mockedGetRegisteredWorktreeBranchMap.mockResolvedValue(new Map<string, string>());
mockedClassifyOwnedLandedEvidence.mockResolvedValue({ kind: "proven-no-op", baseRef: "main", ownDiffEmpty: true });
});
afterEach(() => {
manager.stop();
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
vi.useRealTimers();
});
@@ -7654,3 +7669,193 @@ describe("autoMerge gating for mutating in-review sweeps (FN-5147)", () => {
expect(store.logEntry).not.toHaveBeenCalled();
});
});
describe("FN-5335 triple-proof no-action unit coverage", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-21T12:00:00.000Z"));
activeSessionRegistry.clear();
executingTaskLock._clearForTest();
mockedClassifyTaskWorktree.mockResolvedValue({ ok: true } as any);
});
afterEach(() => {
vi.useRealTimers();
});
it("emits stale-incomplete-review no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, autoMerge: true, taskStuckTimeoutMs: 1_000 } as any),
listTasks: vi.fn().mockResolvedValue([{ id: "FN-SIR", column: "in-review", paused: false, status: null, worktree: "/tmp/fn-sir", updatedAt: new Date(Date.now() - 5_000).toISOString(), steps: [{ status: "pending" }] }]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
const recovered = await manager.recoverStaleIncompleteReviewTasks();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:stale-incomplete-review-no-action" }));
});
it("emits ghost-review no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, autoMerge: true, taskStuckTimeoutMs: 1_000 } as any),
listTasks: vi.fn().mockResolvedValue([{ id: "FN-GHOST", column: "in-review", worktree: "/tmp/fn-ghost", updatedAt: new Date(Date.now() - 5_000).toISOString(), columnMovedAt: new Date(Date.now() - 5_000).toISOString(), paused: false, status: null, mergeDetails: {} }]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
const recovered = await manager.recoverGhostReviewTasks();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:ghost-review-no-action" }));
});
it("emits no-progress no-action when triple proof fails", async () => {
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([{ id: "FN-NP", column: "in-progress", worktree: "/tmp/fn-np", status: "failed", paused: false, error: "Agent finished without calling fn_task_done", updatedAt: new Date(Date.now() - 5_000).toISOString(), executionStartedAt: new Date(Date.now() - 5_000).toISOString(), steps: [{ id: "1", title: "a", status: "pending" }] }]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
vi.spyOn(manager as any, "hasRecoverableGitWork").mockResolvedValue(false);
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:no-progress-no-task-done-no-action" }));
});
it("emits missing-worktree-review no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
listTasks: vi.fn().mockResolvedValue([{ id: "FN-MWR", column: "in-review", paused: false, status: "failed", worktree: "/tmp/fn-mwr", branch: "fusion/fn-mwr", error: "Refusing to start coding agent in missing worktree: /tmp/fn-mwr", updatedAt: new Date(Date.now() - 10_000).toISOString(), steps: [{ status: "done" }, { status: "pending" }], log: [] }]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
const recovered = await manager.recoverMissingWorktreeReviewFailures();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:missing-worktree-review-no-action" }));
});
it("emits partial-progress no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
listTasks: vi.fn().mockResolvedValue([{ id: "FN-PP", column: "in-review", paused: false, status: "failed", error: "Agent finished without calling fn_task_done", updatedAt: new Date(Date.now() - 10_000).toISOString(), taskDoneRetryCount: 1, steps: [{ status: "done" }, { status: "pending" }], worktree: "/tmp/fn-pp", branch: "fusion/fn-pp" }]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:partial-progress-no-task-done-no-action" }));
});
it("emits stuck-merge-deadlock no-action when no-landed branch fails proof", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, defaultBaseBranch: "main" } as any),
listTasks: vi.fn().mockResolvedValue([{ id: "FN-SMD", column: "in-review", paused: false, status: "failed", mergeRetries: 3, mergeDetails: undefined, worktree: "/tmp/wt", branch: "fusion/fn-smd", updatedAt: new Date(Date.now() - 100_000).toISOString(), log: [] }]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
mockedExecSync.mockReturnValue("" as any);
const recovered = await manager.recoverStuckMergeDeadlocks();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.updateTask).not.toHaveBeenCalledWith("FN-SMD", { paused: true });
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:stuck-merge-deadlock-no-action" }));
});
it("emits auto-rebound paused-scope no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, pausedScopeDecayMs: 1_000 } as any),
listTasks: vi.fn().mockResolvedValue([
{ id: "FN-HOLDER", column: "in-progress", paused: true, pausedReason: "waiting", blockedBy: null, worktree: "/tmp/wt-holder", updatedAt: new Date(Date.now() - 10_000).toISOString(), executionStartedAt: new Date(Date.now() - 10_000).toISOString() },
{ id: "FN-FOLLOW", column: "todo", paused: false, blockedBy: "FN-HOLDER" },
]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
mockedClassifyTaskWorktree.mockResolvedValue({ ok: true } as any);
const recovered = await manager.autoReboundPausedScopeDecay();
expect(recovered).toBe(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:auto-rebound-scope-decay-no-action" }));
});
it("emits reclaim-pr-conflict no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 1_000 } as any),
getTask: vi.fn().mockResolvedValue({ id: "FN-PR", column: "in-review", paused: false, status: null, worktree: "/tmp/wt-pr", branch: "fusion/fn-pr", prInfo: { number: 1, mergeable: "conflicting" }, updatedAt: new Date(Date.now() - 10_000).toISOString() }),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: "/tmp/wt-pr", tipSha: "abc", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc", subject: "work" }] } as any);
const result = await manager.reclaimPrConflictForTask("FN-PR");
expect(result.outcome).toBe("skipped");
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reclaim-pr-conflict-no-action" }));
});
it("emits reclaim-self-owned-branch-conflict no-action when triple proof fails", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 1_000 } as any),
listTasks: vi.fn().mockResolvedValue([
{
id: "FN-RSBC",
column: "in-review",
status: "failed",
error: "branch-conflict-unrecoverable: conflict",
branch: "fusion/fn-rsbc",
worktree: "/tmp/wt-rsbc",
updatedAt: new Date(Date.now() - 10_000).toISOString(),
},
]),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", tipSha: "abc", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc", subject: "work" }] } as any);
const result = await manager.reclaimSelfOwnedBranchConflicts();
expect(result).toBeGreaterThanOrEqual(0);
expect(store.moveTask).not.toHaveBeenCalled();
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reclaim-self-owned-branch-conflict-no-action" }));
});
it("emits finalize-no-op-review no-action when unproven fallback fails triple proof", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
listTasks: vi.fn().mockResolvedValue([
{
id: "FN-NOOP",
column: "in-review",
paused: false,
status: null,
worktree: "/tmp/test-project/.worktrees/fn-noop",
branch: "fusion/fn-noop",
steps: [{ name: "Ship it", status: "done" }],
workflowStepResults: [{ id: "ws-1", status: "passed", phase: "pre-merge" }],
mergeDetails: undefined,
updatedAt: new Date(Date.now() - 10_000).toISOString(),
log: [],
},
]),
});
(store as any).recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
mockedClassifyOwnedLandedEvidence.mockResolvedValueOnce({
kind: "unproven",
reason: "foreign-start-point",
details: { foreignRef: "fusion/fn-a" },
} as any);
mockedExecSync.mockImplementation((command) => {
const cmd = String(command);
if (cmd.includes("rev-parse --verify 'fusion/fn-noop'")) return "ok" as any;
if (cmd.includes("rev-parse --verify 'main'")) return "ok" as any;
if (cmd.includes("rev-list --count 'main'..'fusion/fn-noop'")) return "0\n" as any;
return "" as any;
});
const result = await manager.finalizeNoOpReviewTasks();
expect(result).toBe(0);
expect(store.moveTask).not.toHaveBeenCalledWith("FN-NOOP", "todo", expect.anything());
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:finalize-no-op-review-no-action" }));
});
});

View File

@@ -30,7 +30,7 @@ import { isAbsolute, join, relative, resolve } from "node:path";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger, schedulerLog } from "./logger.js";
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import { RemovalReason, classifyTaskWorktree, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import {
classifyMissingWorktreeSessionStartFailure,
extractMissingWorktreePathFromSessionStartFailure,
@@ -40,9 +40,9 @@ import {
} from "./restart-recovery-coordinator.js";
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
import { createRunAuditor, generateSyntheticRunId, type RunAuditor } from "./run-audit.js";
import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type RunAuditor } from "./run-audit.js";
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
import { activeSessionRegistry } from "./active-session-registry.js";
import { activeSessionRegistry, executingTaskLock } from "./active-session-registry.js";
import { findAlreadyMergedTaskCommit } from "./already-merged-detector.js";
import { resolveWorktreesDir } from "./worktree-paths.js";
import { canonicalFusionBranchName } from "./worktree-names.js";
@@ -571,6 +571,107 @@ export class SelfHealingManager {
});
}
private hasRecentWorktreeIncompleteDetected(taskId: string, graceMs: number): boolean {
if (!Number.isFinite(graceMs) || graceMs <= 0) return false;
const storeWithRunAudit = this.store as { getRunAuditEvents?: (filter: { taskId: string; mutationType: string; limit: number }) => Array<{ timestamp?: string | null }> };
if (typeof storeWithRunAudit.getRunAuditEvents !== "function") return false;
let events: Array<{ timestamp?: string | null }> = [];
try {
events = storeWithRunAudit.getRunAuditEvents({ taskId, mutationType: "worktree:incomplete-detected", limit: 20 }) ?? [];
} catch {
return false;
}
if (!Array.isArray(events) || events.length === 0) return false;
const cutoff = Date.now() - graceMs;
return events.some((event) => {
const ts = Date.parse(event.timestamp ?? "");
return Number.isFinite(ts) && ts >= cutoff;
});
}
private async evaluateBackwardMoveTripleProof(
task: Task,
input: {
stage: string;
graceMs: number;
stalenessAnchor: string | null | undefined;
reason: string;
extra?: Record<string, unknown>;
},
): Promise<{ ok: boolean; stalenessMs: number; reason: string; metadata: Record<string, unknown> }> {
const livePaths = activeSessionRegistry.pathsForTask(task.id);
const hasActiveRegisteredPath = livePaths.some((path) => activeSessionRegistry.isPathActive(path));
const sessionDead = !hasActiveRegisteredPath && !executingTaskLock.has(task.id) && this.options.isTaskActive?.(task.id) !== true;
let worktreeUnusable = true;
let worktreeClassification: { ok: boolean; classification?: string; reason?: string };
if (task.worktree) {
const cls = await classifyTaskWorktree(this.options.rootDir, task.worktree);
worktreeClassification = cls.ok
? { ok: true }
: { ok: false, classification: cls.classification, reason: cls.reason };
worktreeUnusable = !cls.ok;
} else {
const expected = canonicalFusionBranchName(task.id);
const registeredPaths = await getRegisteredWorktreePaths(this.options.rootDir);
const registeredBranchMap = await getRegisteredWorktreeBranchMap(this.options.rootDir);
const matchingRegisteredPaths = [...registeredPaths].filter((path) => {
const branch = registeredBranchMap.get(path);
return typeof branch === "string" && branch.trim().toLowerCase() === expected;
});
worktreeClassification = matchingRegisteredPaths.length === 0
? { ok: false, classification: "missing", reason: "task.worktree is null and no registered fusion worktree exists" }
: { ok: true, reason: "registered fusion worktree exists while task.worktree is null" };
worktreeUnusable = matchingRegisteredPaths.length === 0;
}
const anchorMs = input.stalenessAnchor ? Date.parse(input.stalenessAnchor) : Number.NaN;
const stalenessMs = Number.isFinite(anchorMs) ? Math.max(0, Date.now() - anchorMs) : Number.POSITIVE_INFINITY;
const noRecentActivity = stalenessMs >= input.graceMs && !this.hasRecentWorktreeIncompleteDetected(task.id, input.graceMs);
const ok = sessionDead && worktreeUnusable && noRecentActivity;
return {
ok,
stalenessMs,
reason: input.reason,
metadata: {
priorWorktree: task.worktree ?? null,
priorBranch: task.branch ?? null,
hadWorktree: Boolean(task.worktree),
stalenessMs,
graceMs: input.graceMs,
sessionDead,
worktreeUnusable,
noRecentActivity,
livePaths,
hasExecutingTaskLock: executingTaskLock.has(task.id),
taskActive: this.options.isTaskActive?.(task.id) === true,
worktreeClassification,
...input.extra,
},
};
}
private async emitBackwardMoveNoAction(task: Task, stage: string, mutationType: string, proof: { stalenessMs: number; reason: string; metadata: Record<string, unknown> }): Promise<void> {
try {
await createRunAuditor(this.store, {
runId: generateSyntheticRunId(`self-healing-${stage}`, task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: stage,
}).database({
type: mutationType as DatabaseMutationType,
target: task.id,
metadata: proof.metadata,
});
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
log.warn(`[${stage}] ${task.id}: no-action audit emission failed: ${message}`);
}
log.log(`[${stage}] ${task.id}: triple-proof not satisfied — no action (operator-decides)`);
}
// ── Lifecycle ───────────────────────────────────────────────────────
start(): void {
@@ -1619,6 +1720,10 @@ export class SelfHealingManager {
return reclaimed;
}
/**
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:reclaim-pr-conflict-no-action` and skips lifecycle mutation.
*/
async reclaimPrConflictForTask(taskId: string): Promise<{ outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string; perPr?: Array<{ number: number; outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string }> }> {
const task = await this.store.getTask(taskId);
if (!task) return { outcome: "skipped", reason: "task-not-found" };
@@ -1705,20 +1810,40 @@ export class SelfHealingManager {
}
}
await this.store.updateTask(task.id, {
worktree: inspection.livePath,
branch: task.branch,
paused: false,
pausedReason: undefined,
status: null,
error: null,
});
if (task.column === "in-review") {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveWorktree: true,
preserveProgress: true,
preserveResumeState: true,
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "reclaim-pr-conflict",
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "reclaim-pr-conflict-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "reclaim-pr-conflict", "task:reclaim-pr-conflict-no-action", proof);
return withPerPr({ outcome: "skipped", reason: "triple-proof-not-satisfied" });
} else {
await this.store.updateTask(task.id, {
worktree: inspection.livePath,
branch: task.branch,
paused: false,
pausedReason: undefined,
status: null,
error: null,
});
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveWorktree: true,
preserveProgress: true,
preserveResumeState: true,
});
}
} else {
await this.store.updateTask(task.id, {
worktree: inspection.livePath,
branch: task.branch,
paused: false,
pausedReason: undefined,
status: null,
error: null,
});
}
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "reclaimed", mode: inspection.kind } });
@@ -1767,6 +1892,9 @@ export class SelfHealingManager {
* STANDING: do not auto-discard stranded commits. Reclaim preserves commits;
* unrecoverable conflicts are escalated for human review.
*
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:reclaim-self-owned-branch-conflict-no-action` and skips lifecycle mutation.
*
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
*/
async reclaimSelfOwnedBranchConflicts(): Promise<number> {
@@ -1830,6 +1958,15 @@ export class SelfHealingManager {
}
if (!await isUsableTaskWorktree(this.options.rootDir, task.worktree)) continue;
const reviewProof = task.column === "in-review"
? await this.evaluateBackwardMoveTripleProof(task, {
stage: "reclaim-self-owned-branch-conflict",
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "reclaim-self-owned-candidate",
})
: null;
try {
const inspection = await inspectBranchConflict({
repoDir: this.options.rootDir,
@@ -1898,11 +2035,15 @@ export class SelfHealingManager {
);
if (task.column === "in-review") {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveProgress: true,
preserveResumeState: true,
});
if (!reviewProof?.ok) {
await this.emitBackwardMoveNoAction(task, "reclaim-self-owned-branch-conflict", "task:reclaim-self-owned-branch-conflict-no-action", reviewProof!);
} else {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveProgress: true,
preserveResumeState: true,
});
}
}
try {
@@ -1997,11 +2138,15 @@ export class SelfHealingManager {
);
if (task.column === "in-review") {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveProgress: true,
preserveResumeState: true,
});
if (!reviewProof?.ok) {
await this.emitBackwardMoveNoAction(task, "reclaim-self-owned-branch-conflict", "task:reclaim-self-owned-branch-conflict-no-action", reviewProof!);
} else {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveProgress: true,
preserveResumeState: true,
});
}
}
try {
@@ -2062,12 +2207,16 @@ export class SelfHealingManager {
);
if (task.column === "in-review") {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveWorktree: true,
preserveProgress: true,
preserveResumeState: true,
});
if (!reviewProof?.ok) {
await this.emitBackwardMoveNoAction(task, "reclaim-self-owned-branch-conflict", "task:reclaim-self-owned-branch-conflict-no-action", reviewProof!);
} else {
await this.store.moveTask(task.id, "todo", {
moveSource: "engine",
preserveWorktree: true,
preserveProgress: true,
preserveResumeState: true,
});
}
}
try {
@@ -2947,6 +3096,23 @@ export class SelfHealingManager {
const ageMs = Number.isFinite(movedAtMs) ? now - movedAtMs : Number.POSITIVE_INFINITY;
if (!options?.ignoreAgeGate && ageMs < thresholdMs) continue;
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "auto-rebound-paused-scope-decay",
graceMs: thresholdMs,
stalenessAnchor: task.executionStartedAt ?? task.updatedAt,
reason: "paused-scope-decay-candidate",
extra: {
followerCount,
ignoredAgeGate: options?.ignoreAgeGate === true,
thresholdMs,
ageMs,
},
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "auto-rebound-paused-scope-decay", "task:auto-rebound-scope-decay-no-action", proof);
continue;
}
await this.store.moveTask(task.id, "todo", {
preserveProgress: true,
preserveWorktree: true,
@@ -3745,6 +3911,9 @@ export class SelfHealingManager {
}
/**
* Backward lifecycle move gated on triple proof (FN-5335).
* When the unproven fallback predicate fails, emits `task:finalize-no-op-review-no-action` and skips lifecycle mutation.
*
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
*/
async finalizeNoOpReviewTasks(): Promise<number> {
@@ -3810,6 +3979,16 @@ export class SelfHealingManager {
details: classification.details,
autoRetry: true,
});
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "finalize-no-op-review",
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "finalize-unproven-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "finalize-no-op-review", "task:finalize-no-op-review-no-action", proof);
continue;
}
await this.store.moveTask(task.id, "todo", { preserveProgress: true, moveSource: "engine" });
continue;
}
@@ -4200,6 +4379,8 @@ export class SelfHealingManager {
*
* Moving them back to `todo` lets the normal scheduler/executor resume the
* incomplete step instead of leaving the task stranded in review.
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:stale-incomplete-review-no-action` and skips lifecycle mutation.
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
*/
async recoverStaleIncompleteReviewTasks(): Promise<number> {
@@ -4228,6 +4409,17 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of staleIncomplete) {
try {
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "stale-incomplete-review",
graceMs: timeoutMs,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "stale-incomplete-review-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "stale-incomplete-review", "task:stale-incomplete-review-no-action", proof);
continue;
}
await this.store.logEntry(
task.id,
"Auto-recovered: in-review task still had incomplete steps — moved back to todo for retry",
@@ -4578,6 +4770,9 @@ export class SelfHealingManager {
}
/**
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:ghost-review-no-action` and skips lifecycle mutation.
*
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
*/
async recoverGhostReviewTasks(): Promise<number> {
@@ -4608,6 +4803,16 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of ghosts) {
try {
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "ghost-review",
graceMs: timeoutMs,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "ghost-review-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "ghost-review", "task:ghost-review-no-action", proof);
continue;
}
if (task.status) {
await this.store.updateTask(task.id, { status: null, error: null });
}
@@ -5029,6 +5234,9 @@ export class SelfHealingManager {
/**
* Recover deadlocked retry-exhausted merge failures that are still blocking
* dispatch via `blockedBy` or retained worktree ownership.
*
* Backward lifecycle move gated on triple proof (FN-5335).
* When the no-landed predicate fails, emits `task:stuck-merge-deadlock-no-action` and skips lifecycle mutation.
*/
/**
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
@@ -5120,10 +5328,20 @@ export class SelfHealingManager {
log.log(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: landedCommit.sha, action: "reattributed" })}`);
recovered++;
} else {
await this.store.updateTask(task.id, { paused: true });
await this.store.logEntry(task.id, "merge-deadlock-detected: requires manual intervention — verified content not on main");
log.warn(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: null, action: "paused-for-manual" })}`);
recovered++;
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "stuck-merge-deadlock",
graceMs: DEADLOCK_RECOVERY_COOLDOWN_MS,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "stuck-merge-deadlock-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "stuck-merge-deadlock", "task:stuck-merge-deadlock-no-action", proof);
} else {
await this.store.updateTask(task.id, { paused: true });
await this.store.logEntry(task.id, "merge-deadlock-detected: requires manual intervention — verified content not on main");
log.warn(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: null, action: "paused-for-manual" })}`);
recovered++;
}
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
@@ -6443,6 +6661,9 @@ export class SelfHealingManager {
* Recover `in-progress` tasks that failed only because the agent exited
* without calling fn_task_done, and where there is no sign of work to preserve.
*
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:no-progress-no-task-done-no-action` and skips lifecycle mutation.
*
* These are safe to requeue automatically when no steps progressed and git
* has neither worktree changes nor branch commits. Cases with any evidence
* of work are left alone for manual inspection or the normal orphan recovery
@@ -6475,6 +6696,17 @@ export class SelfHealingManager {
continue;
}
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "no-progress-no-task-done",
graceMs: ORPHANED_EXECUTION_RECOVERY_GRACE_MS,
stalenessAnchor: task.executionStartedAt ?? task.updatedAt,
reason: "no-progress-no-task-done-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "no-progress-no-task-done", "task:no-progress-no-task-done-no-action", proof);
continue;
}
await this.store.updateTask(task.id, {
status: "stuck-killed",
worktree: null,
@@ -6504,6 +6736,9 @@ export class SelfHealingManager {
/**
* Recover failed `in-review` retries that point at an unusable worktree path.
*
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:missing-worktree-review-no-action` and skips lifecycle mutation.
*
* This is a narrow guard for session-start failures thrown by
* `assertValidWorktreeSession()` in `pi.ts`, classified centrally via
* `MISSING_WORKTREE_SESSION_PREFIXES` /
@@ -6532,6 +6767,17 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of candidates) {
try {
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "missing-worktree-review",
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "missing-worktree-review-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "missing-worktree-review", "task:missing-worktree-review-no-action", proof);
continue;
}
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", task.id),
agentId: "self-healing",
@@ -6565,6 +6811,9 @@ export class SelfHealingManager {
/**
* Recover `in-review` tasks marked as `failed` because the agent exited
* without calling `fn_task_done` *with partial step progress* (some steps done,
*
* Backward lifecycle move gated on triple proof (FN-5335).
* When the predicate fails, emits `task:partial-progress-no-task-done-no-action` and skips lifecycle mutation.
* some still pending). The work-in-progress is valuable but incomplete —
* the existing worktree and branch are preserved and the task is moved back
* to `todo` so the scheduler re-dispatches it for a fresh execution that
@@ -6610,6 +6859,17 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of candidates) {
try {
const proof = await this.evaluateBackwardMoveTripleProof(task, {
stage: "partial-progress-no-task-done",
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
reason: "partial-progress-no-task-done-candidate",
});
if (!proof.ok) {
await this.emitBackwardMoveNoAction(task, "partial-progress-no-task-done", "task:partial-progress-no-task-done-no-action", proof);
continue;
}
const nextCount = (task.taskDoneRetryCount ?? 0) + 1;
await this.store.updateTask(task.id, {
status: null,