fix(FN-5335): enforce triple-proof gating for backward recovery
- Gate self-healing backward moves behind audited triple-proof predicates across reclaim paths - Skip reclaim-pr-conflict mutations when proof checks fail and preserve no-action behavior - Add broad unit and reliability-interaction coverage for triple-proof and cross-layer scenarios - Document backward-move stage invariants, diagnostics, and add delivery changeset for @runfusion/fusion
This commit is contained in:
committed by
gsxdsm
parent
a2a5db8151
commit
a8715ed963
@@ -0,0 +1,275 @@
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { TaskStore } from "@fusion/core";
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
import { activeSessionRegistry, executingTaskLock } from "../../active-session-registry.js";
|
||||
|
||||
function git(cwd: string, command: string): string {
|
||||
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
describe("FN-5335 reliability interactions: backward move triple proof", () => {
|
||||
let rootDir = "";
|
||||
let store: TaskStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-21T12:00:00.000Z"));
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
rootDir = mkdtempSync(join(tmpdir(), "fn-5335-reliability-"));
|
||||
git(rootDir, "init -b main");
|
||||
git(rootDir, "config user.name 'Fusion'");
|
||||
git(rootDir, "config user.email 'hi@runfusion.ai'");
|
||||
writeFileSync(join(rootDir, "README.md"), "root\n");
|
||||
git(rootDir, "add README.md");
|
||||
git(rootDir, "commit -m 'init'");
|
||||
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
|
||||
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
|
||||
await store.init();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
try { store?.close(); } catch {}
|
||||
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
async function createNoProgressTask(worktree: string, ageMs: number) {
|
||||
const task = await store.createTask({ title: "no-progress", description: "no-progress" });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.updateTask(task.id, {
|
||||
worktree,
|
||||
status: "failed",
|
||||
paused: false,
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
executionStartedAt: new Date(Date.now() - ageMs).toISOString(),
|
||||
updatedAt: new Date(Date.now() - ageMs).toISOString(),
|
||||
steps: [{ name: "step", status: "pending" }],
|
||||
} as any);
|
||||
return task.id;
|
||||
}
|
||||
|
||||
it("Scenario A: live session blocks backward move and emits no-action", async () => {
|
||||
const worktree = join(rootDir, ".worktrees", "np-live-missing");
|
||||
const id = await createNoProgressTask(worktree, 400_000);
|
||||
activeSessionRegistry.registerPath(worktree, { taskId: id, kind: "executor", ownerKey: "run-a" });
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task?.column).toBe("in-progress");
|
||||
expect(events).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario B: usable worktree blocks backward move (recoverable git work short-circuit)", async () => {
|
||||
const worktree = join(rootDir, ".worktrees", "np-usable");
|
||||
mkdirSync(worktree, { recursive: true });
|
||||
const id = await createNoProgressTask(worktree, 400_000);
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task?.column).toBe("in-progress");
|
||||
expect(events).toHaveLength(0);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario C: recent activity blocks backward move", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-recent"), 200);
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task?.column).toBe("in-progress");
|
||||
expect(events).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario D: all triple-proof signals true allows recovery", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-stale"), 400_000);
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
|
||||
expect(recovered).toBe(1);
|
||||
expect(task?.column).toBe("todo");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario E: autoMerge false keeps in-review stage no-op", async () => {
|
||||
const task = await store.createTask({ title: "review", description: "review" });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.moveTask(task.id, "in-review");
|
||||
await store.updateTask(task.id, {
|
||||
status: "failed",
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
taskDoneRetryCount: 1,
|
||||
worktree: join(rootDir, ".worktrees", "review-missing"),
|
||||
updatedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
steps: [{ name: "done", status: "done" }, { name: "pending", status: "pending" }],
|
||||
} as any);
|
||||
await store.updateSettings({ ...(await store.getSettings()), autoMerge: false } as any);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
|
||||
const current = await store.getTask(task.id);
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(current?.column).toBe("in-review");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario F: churn-terminalized review task is not reopened", async () => {
|
||||
const task = await store.createTask({ title: "churn", description: "churn" });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.moveTask(task.id, "in-review");
|
||||
await store.updateTask(task.id, {
|
||||
paused: true,
|
||||
pausedReason: "in-review-stall-deadlock",
|
||||
status: "failed",
|
||||
error: "STUCK_NO_PROGRESS_CHURN",
|
||||
worktree: join(rootDir, ".worktrees", "churn-missing"),
|
||||
updatedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
steps: [{ name: "done", status: "done" }, { name: "pending", status: "pending" }],
|
||||
} as any);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
|
||||
const current = await store.getTask(task.id);
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(current?.column).toBe("in-review");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario G: order-independence across limbo and no-progress sweeps", async () => {
|
||||
const makeCandidate = async (title: string) => {
|
||||
const task = await store.createTask({ title, description: title });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.updateTask(task.id, {
|
||||
branch: null,
|
||||
worktree: join(rootDir, ".worktrees", `${task.id.toLowerCase()}-missing`),
|
||||
status: "failed",
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
executionStartedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
updatedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
steps: [{ name: "step", status: "pending" }],
|
||||
} as any);
|
||||
return task.id;
|
||||
};
|
||||
|
||||
const firstId = await makeCandidate("order-a");
|
||||
await (store as any).recordRunAuditEvent({ runId: "run-g-a", phase: "executor", taskId: firstId, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: firstId, details: null, metadata: {} });
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false, getExecutingTaskIds: () => new Set<string>() });
|
||||
await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
await manager.recoverInProgressLimbo();
|
||||
const firstTask = await store.getTask(firstId);
|
||||
|
||||
const secondId = await makeCandidate("order-b");
|
||||
await (store as any).recordRunAuditEvent({ runId: "run-g-b", phase: "executor", taskId: secondId, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: secondId, details: null, metadata: {} });
|
||||
await manager.recoverInProgressLimbo();
|
||||
await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const secondTask = await store.getTask(secondId);
|
||||
|
||||
expect(firstTask?.column).toBe("in-progress");
|
||||
expect(secondTask?.column).toBe("in-progress");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario H: orphan and tightened sweeps can co-emit no-action events", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-orphan-h"), 400_000);
|
||||
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>(), isTaskActive: () => false });
|
||||
vi.setSystemTime(new Date("2026-05-21T12:10:00.000Z"));
|
||||
await manager.recoverOrphanedExecutions();
|
||||
await (store as any).recordRunAuditEvent({ runId: "run-h", phase: "executor", taskId: id, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: id, details: null, metadata: {} });
|
||||
await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
|
||||
const orphanEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:orphan-detected-no-action" });
|
||||
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
const current = await store.getTask(id);
|
||||
|
||||
expect(current?.column).toBe("in-progress");
|
||||
expect(orphanEvents).toHaveLength(1);
|
||||
expect(noActionEvents).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario J: tightened sweep no-action is idempotent across re-sweeps", async () => {
|
||||
const worktree = join(rootDir, ".worktrees", "np-missing-orphan");
|
||||
const id = await createNoProgressTask(worktree, 400_000);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>(), isTaskActive: () => false });
|
||||
vi.setSystemTime(new Date("2026-05-21T12:10:00.000Z"));
|
||||
await manager.recoverOrphanedExecutions();
|
||||
await (store as any).recordRunAuditEvent({
|
||||
runId: "run-fn5335-h",
|
||||
phase: "executor",
|
||||
taskId: id,
|
||||
taskLineageId: null,
|
||||
agentId: "executor",
|
||||
domain: "database",
|
||||
mutationType: "worktree:incomplete-detected",
|
||||
payload: { source: "executor-liveness-gate" },
|
||||
target: id,
|
||||
details: null,
|
||||
metadata: { source: "executor-liveness-gate" },
|
||||
});
|
||||
const first = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const second = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
const current = await store.getTask(id);
|
||||
|
||||
expect(first).toBe(0);
|
||||
expect(second).toBe(0);
|
||||
expect(current?.column).toBe("in-progress");
|
||||
expect(noActionEvents).toHaveLength(2);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario I: recent worktree:incomplete-detected audit counts as recent activity", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-liveness"), 400_000);
|
||||
await (store as any).recordRunAuditEvent({
|
||||
runId: "run-fn5335-liveness",
|
||||
phase: "executor",
|
||||
taskId: id,
|
||||
taskLineageId: null,
|
||||
agentId: "executor",
|
||||
domain: "database",
|
||||
mutationType: "worktree:incomplete-detected",
|
||||
payload: { source: "executor-liveness-gate" },
|
||||
target: id,
|
||||
details: null,
|
||||
metadata: { source: "executor-liveness-gate" },
|
||||
});
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(noActionEvents).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@ function makeTask(id: string, overrides: Partial<Task> = {}): Task {
|
||||
currentStep: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
executionStartedAt: new Date(Date.now() - 61_000).toISOString(),
|
||||
log: [],
|
||||
...overrides,
|
||||
} as Task;
|
||||
|
||||
@@ -56,7 +56,7 @@ describe("reliability interactions: live-zero reclaim", () => {
|
||||
pausedReason: "branch-conflict-unrecoverable",
|
||||
userPaused: false,
|
||||
lineageId: "lin-9100",
|
||||
updatedAt: new Date().toISOString(),
|
||||
updatedAt: new Date(Date.now() - 11 * 60_000).toISOString(),
|
||||
};
|
||||
|
||||
const statefulStore = {
|
||||
|
||||
@@ -143,8 +143,8 @@ describe("foreign start-point no-owned-commit interactions (real git)", () => {
|
||||
const recovered = await manager.finalizeNoOpReviewTasks();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task.column).toBe("todo");
|
||||
expect(events.some((event: any) => event?.mutationType === "task:finalize-unproven-blocked")).toBe(true);
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(events.some((event: any) => event?.mutationType === "task:finalize-no-op-review-no-action")).toBe(true);
|
||||
manager.stop();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
|
||||
@@ -125,6 +125,7 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
it("keeps in-review missing-worktree failures on the review-specific recovery path", async () => {
|
||||
const id = await createInProgressTask("review failure disjoint");
|
||||
await store.moveTask(id, "in-review");
|
||||
await store.updateSettings({ autoMerge: true } as any);
|
||||
await store.updateTask(id, {
|
||||
status: "failed",
|
||||
error: `Refusing to start coding agent in missing worktree: ${join(rootDir, ".worktrees", "missing-review")}`,
|
||||
@@ -132,6 +133,10 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
worktree: join(rootDir, ".worktrees", "missing-review-stale"),
|
||||
steps: [{ name: "step", status: "done" }, { name: "next", status: "pending" }],
|
||||
});
|
||||
await store.updateTask(id, {
|
||||
updatedAt: new Date(Date.now() - 48 * 60 * 60_000).toISOString(),
|
||||
columnMovedAt: new Date(Date.now() - 48 * 60 * 60_000).toISOString(),
|
||||
} as any);
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir,
|
||||
@@ -143,8 +148,8 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
const updated = await store.getTask(id);
|
||||
|
||||
expect(limboRecovered).toBe(0);
|
||||
expect(reviewRecovered).toBe(1);
|
||||
expect(updated?.column).toBe("todo");
|
||||
expect(reviewRecovered).toBe(0);
|
||||
expect(updated?.column).toBe("in-review");
|
||||
});
|
||||
|
||||
it("skips limbo recovery while the executor still claims the task id", async () => {
|
||||
|
||||
@@ -64,6 +64,7 @@ describe("reliability interactions: paused scope decay", () => {
|
||||
column: "in-progress",
|
||||
paused: true,
|
||||
pausedReason: "waiting",
|
||||
executionStartedAt: new Date(now - 31 * 60_000).toISOString(),
|
||||
columnMovedAt: new Date(now - 31 * 60_000).toISOString(),
|
||||
currentStep: 2,
|
||||
steps: [{ id: "s1", title: "x", status: "done" } as any],
|
||||
@@ -97,6 +98,7 @@ describe("reliability interactions: paused scope decay", () => {
|
||||
const holder = makeTask("FN-3", {
|
||||
column: "in-progress",
|
||||
paused: true,
|
||||
executionStartedAt: new Date(now - 61_000).toISOString(),
|
||||
columnMovedAt: new Date(now - 1_000).toISOString(),
|
||||
});
|
||||
const follower = makeTask("FN-4", { column: "todo", blockedBy: "FN-3" });
|
||||
|
||||
@@ -76,7 +76,7 @@ describe("reliability interaction: pr conflict reclaim", () => {
|
||||
});
|
||||
|
||||
it("keeps paused-review reclaim path resumable", async () => {
|
||||
const t = task();
|
||||
const t = task({ updatedAt: new Date(Date.now() - 11 * 60_000).toISOString() });
|
||||
const s = store(t);
|
||||
vi.spyOn(branchConflicts, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: t.worktree, tipSha: "abc123", taskAttributedCommitCount: 2, strandedCommits: [{ sha: "abc123" }] } as any);
|
||||
const manager = new SelfHealingManager(s as any, { rootDir: "/tmp/test" } as any);
|
||||
|
||||
@@ -86,6 +86,7 @@ describe("reliability interactions: self-healing", () => {
|
||||
worktree: "/tmp/wt",
|
||||
branch: "fusion/wt",
|
||||
steps: [{ id: "s1", title: "Step", status: "pending" }] as any,
|
||||
updatedAt: new Date(Date.now() - 31 * 60_000).toISOString(),
|
||||
}),
|
||||
]]);
|
||||
const store = makeStore(tasks);
|
||||
|
||||
@@ -107,7 +107,7 @@ describe("SelfHealingManager.reclaimPrConflictForTask", () => {
|
||||
});
|
||||
|
||||
it("returns reclaimed for reclaimable conflicts", async () => {
|
||||
const task = makeTask({ column: "in-review", paused: true, pausedReason: "branch-conflict-unrecoverable" as any });
|
||||
const task = makeTask({ column: "in-review", paused: true, pausedReason: "branch-conflict-unrecoverable" as any, updatedAt: new Date(Date.now() - 11 * 60_000).toISOString() });
|
||||
const store = makeStore(task);
|
||||
vi.spyOn(branchConflicts, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: task.worktree, tipSha: "abc123", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc123" }] } as any);
|
||||
const manager = new SelfHealingManager(store as any, { rootDir: "/tmp/test" } as any);
|
||||
|
||||
@@ -73,6 +73,9 @@ vi.mock("../worktree-pool.js", () => ({
|
||||
scanOrphanedBranches: vi.fn().mockResolvedValue([]),
|
||||
cleanupOrphanedWorktrees: vi.fn().mockResolvedValue(0),
|
||||
isUsableTaskWorktree: vi.fn().mockResolvedValue(true),
|
||||
classifyTaskWorktree: vi.fn().mockResolvedValue({ ok: false, classification: "missing", reason: "test-default" }),
|
||||
getRegisteredWorktreePaths: vi.fn().mockResolvedValue(new Set<string>()),
|
||||
getRegisteredWorktreeBranchMap: vi.fn().mockResolvedValue(new Map<string, string>()),
|
||||
removeWorktree: vi.fn().mockResolvedValue(undefined),
|
||||
resolveWorktreeBackend: vi.fn(),
|
||||
}));
|
||||
@@ -102,7 +105,8 @@ import { existsSync, readdirSync } from "node:fs";
|
||||
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "../worktree-pool.js";
|
||||
import { classifyTaskWorktree, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "../worktree-pool.js";
|
||||
import { activeSessionRegistry, executingTaskLock } from "../active-session-registry.js";
|
||||
import * as branchConflictModule from "../branch-conflicts.js";
|
||||
import { createLogger } from "../logger.js";
|
||||
import { NotificationService } from "../notification/notification-service.js";
|
||||
@@ -111,6 +115,9 @@ import { classifyOwnedLandedEvidence } from "../merger.js";
|
||||
const mockedExecSync = vi.mocked(execSync);
|
||||
const mockedExistsSync = vi.mocked(existsSync);
|
||||
const mockedIsUsableTaskWorktree = vi.mocked(isUsableTaskWorktree);
|
||||
const mockedClassifyTaskWorktree = vi.mocked(classifyTaskWorktree);
|
||||
const mockedGetRegisteredWorktreePaths = vi.mocked(getRegisteredWorktreePaths);
|
||||
const mockedGetRegisteredWorktreeBranchMap = vi.mocked(getRegisteredWorktreeBranchMap);
|
||||
const mockedRemoveWorktree = vi.mocked(removeWorktree);
|
||||
const mockedResolveWorktreeBackend = vi.mocked(resolveWorktreeBackend);
|
||||
const mockedScanIdleWorktrees = vi.mocked(scanIdleWorktrees);
|
||||
@@ -160,6 +167,7 @@ function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & E
|
||||
listTasks: vi.fn().mockResolvedValue([]),
|
||||
createTask: vi.fn().mockResolvedValue({ id: "FN-RESCUE", lineageId: "lin-rescue" }),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
getRunAuditEvents: vi.fn().mockReturnValue([]),
|
||||
getBootstrappedAt: vi.fn().mockReturnValue(null),
|
||||
getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
|
||||
clearStaleExecutionStartBranchReferences: vi.fn().mockReturnValue([]),
|
||||
@@ -174,14 +182,21 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
store = createMockStore();
|
||||
manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
mockedRemoveWorktree.mockResolvedValue(undefined);
|
||||
mockedClassifyTaskWorktree.mockResolvedValue({ ok: false, classification: "missing", reason: "test-default" });
|
||||
mockedGetRegisteredWorktreePaths.mockResolvedValue(new Set<string>());
|
||||
mockedGetRegisteredWorktreeBranchMap.mockResolvedValue(new Map<string, string>());
|
||||
mockedClassifyOwnedLandedEvidence.mockResolvedValue({ kind: "proven-no-op", baseRef: "main", ownDiffEmpty: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
manager.stop();
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
@@ -7654,3 +7669,193 @@ describe("autoMerge gating for mutating in-review sweeps (FN-5147)", () => {
|
||||
expect(store.logEntry).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("FN-5335 triple-proof no-action unit coverage", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-21T12:00:00.000Z"));
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
mockedClassifyTaskWorktree.mockResolvedValue({ ok: true } as any);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("emits stale-incomplete-review no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, autoMerge: true, taskStuckTimeoutMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-SIR", column: "in-review", paused: false, status: null, worktree: "/tmp/fn-sir", updatedAt: new Date(Date.now() - 5_000).toISOString(), steps: [{ status: "pending" }] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverStaleIncompleteReviewTasks();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:stale-incomplete-review-no-action" }));
|
||||
});
|
||||
|
||||
it("emits ghost-review no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, autoMerge: true, taskStuckTimeoutMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-GHOST", column: "in-review", worktree: "/tmp/fn-ghost", updatedAt: new Date(Date.now() - 5_000).toISOString(), columnMovedAt: new Date(Date.now() - 5_000).toISOString(), paused: false, status: null, mergeDetails: {} }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverGhostReviewTasks();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:ghost-review-no-action" }));
|
||||
});
|
||||
|
||||
it("emits no-progress no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-NP", column: "in-progress", worktree: "/tmp/fn-np", status: "failed", paused: false, error: "Agent finished without calling fn_task_done", updatedAt: new Date(Date.now() - 5_000).toISOString(), executionStartedAt: new Date(Date.now() - 5_000).toISOString(), steps: [{ id: "1", title: "a", status: "pending" }] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "hasRecoverableGitWork").mockResolvedValue(false);
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:no-progress-no-task-done-no-action" }));
|
||||
});
|
||||
|
||||
it("emits missing-worktree-review no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-MWR", column: "in-review", paused: false, status: "failed", worktree: "/tmp/fn-mwr", branch: "fusion/fn-mwr", error: "Refusing to start coding agent in missing worktree: /tmp/fn-mwr", updatedAt: new Date(Date.now() - 10_000).toISOString(), steps: [{ status: "done" }, { status: "pending" }], log: [] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverMissingWorktreeReviewFailures();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:missing-worktree-review-no-action" }));
|
||||
});
|
||||
|
||||
it("emits partial-progress no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-PP", column: "in-review", paused: false, status: "failed", error: "Agent finished without calling fn_task_done", updatedAt: new Date(Date.now() - 10_000).toISOString(), taskDoneRetryCount: 1, steps: [{ status: "done" }, { status: "pending" }], worktree: "/tmp/fn-pp", branch: "fusion/fn-pp" }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:partial-progress-no-task-done-no-action" }));
|
||||
});
|
||||
|
||||
it("emits stuck-merge-deadlock no-action when no-landed branch fails proof", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, defaultBaseBranch: "main" } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-SMD", column: "in-review", paused: false, status: "failed", mergeRetries: 3, mergeDetails: undefined, worktree: "/tmp/wt", branch: "fusion/fn-smd", updatedAt: new Date(Date.now() - 100_000).toISOString(), log: [] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
mockedExecSync.mockReturnValue("" as any);
|
||||
|
||||
const recovered = await manager.recoverStuckMergeDeadlocks();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith("FN-SMD", { paused: true });
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:stuck-merge-deadlock-no-action" }));
|
||||
});
|
||||
|
||||
it("emits auto-rebound paused-scope no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, pausedScopeDecayMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([
|
||||
{ id: "FN-HOLDER", column: "in-progress", paused: true, pausedReason: "waiting", blockedBy: null, worktree: "/tmp/wt-holder", updatedAt: new Date(Date.now() - 10_000).toISOString(), executionStartedAt: new Date(Date.now() - 10_000).toISOString() },
|
||||
{ id: "FN-FOLLOW", column: "todo", paused: false, blockedBy: "FN-HOLDER" },
|
||||
]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
mockedClassifyTaskWorktree.mockResolvedValue({ ok: true } as any);
|
||||
|
||||
const recovered = await manager.autoReboundPausedScopeDecay();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:auto-rebound-scope-decay-no-action" }));
|
||||
});
|
||||
|
||||
it("emits reclaim-pr-conflict no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 1_000 } as any),
|
||||
getTask: vi.fn().mockResolvedValue({ id: "FN-PR", column: "in-review", paused: false, status: null, worktree: "/tmp/wt-pr", branch: "fusion/fn-pr", prInfo: { number: 1, mergeable: "conflicting" }, updatedAt: new Date(Date.now() - 10_000).toISOString() }),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: "/tmp/wt-pr", tipSha: "abc", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc", subject: "work" }] } as any);
|
||||
|
||||
const result = await manager.reclaimPrConflictForTask("FN-PR");
|
||||
expect(result.outcome).toBe("skipped");
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reclaim-pr-conflict-no-action" }));
|
||||
});
|
||||
|
||||
it("emits reclaim-self-owned-branch-conflict no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: "FN-RSBC",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "branch-conflict-unrecoverable: conflict",
|
||||
branch: "fusion/fn-rsbc",
|
||||
worktree: "/tmp/wt-rsbc",
|
||||
updatedAt: new Date(Date.now() - 10_000).toISOString(),
|
||||
},
|
||||
]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", tipSha: "abc", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc", subject: "work" }] } as any);
|
||||
|
||||
const result = await manager.reclaimSelfOwnedBranchConflicts();
|
||||
expect(result).toBeGreaterThanOrEqual(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reclaim-self-owned-branch-conflict-no-action" }));
|
||||
});
|
||||
|
||||
it("emits finalize-no-op-review no-action when unproven fallback fails triple proof", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: "FN-NOOP",
|
||||
column: "in-review",
|
||||
paused: false,
|
||||
status: null,
|
||||
worktree: "/tmp/test-project/.worktrees/fn-noop",
|
||||
branch: "fusion/fn-noop",
|
||||
steps: [{ name: "Ship it", status: "done" }],
|
||||
workflowStepResults: [{ id: "ws-1", status: "passed", phase: "pre-merge" }],
|
||||
mergeDetails: undefined,
|
||||
updatedAt: new Date(Date.now() - 10_000).toISOString(),
|
||||
log: [],
|
||||
},
|
||||
]),
|
||||
});
|
||||
(store as any).recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
mockedClassifyOwnedLandedEvidence.mockResolvedValueOnce({
|
||||
kind: "unproven",
|
||||
reason: "foreign-start-point",
|
||||
details: { foreignRef: "fusion/fn-a" },
|
||||
} as any);
|
||||
mockedExecSync.mockImplementation((command) => {
|
||||
const cmd = String(command);
|
||||
if (cmd.includes("rev-parse --verify 'fusion/fn-noop'")) return "ok" as any;
|
||||
if (cmd.includes("rev-parse --verify 'main'")) return "ok" as any;
|
||||
if (cmd.includes("rev-list --count 'main'..'fusion/fn-noop'")) return "0\n" as any;
|
||||
return "" as any;
|
||||
});
|
||||
|
||||
const result = await manager.finalizeNoOpReviewTasks();
|
||||
expect(result).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalledWith("FN-NOOP", "todo", expect.anything());
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:finalize-no-op-review-no-action" }));
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
@@ -30,7 +30,7 @@ import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
|
||||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||||
import { createLogger, schedulerLog } from "./logger.js";
|
||||
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
import { RemovalReason, classifyTaskWorktree, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
import {
|
||||
classifyMissingWorktreeSessionStartFailure,
|
||||
extractMissingWorktreePathFromSessionStartFailure,
|
||||
@@ -40,9 +40,9 @@ import {
|
||||
} from "./restart-recovery-coordinator.js";
|
||||
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
|
||||
import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
|
||||
import { createRunAuditor, generateSyntheticRunId, type RunAuditor } from "./run-audit.js";
|
||||
import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type RunAuditor } from "./run-audit.js";
|
||||
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
|
||||
import { activeSessionRegistry } from "./active-session-registry.js";
|
||||
import { activeSessionRegistry, executingTaskLock } from "./active-session-registry.js";
|
||||
import { findAlreadyMergedTaskCommit } from "./already-merged-detector.js";
|
||||
import { resolveWorktreesDir } from "./worktree-paths.js";
|
||||
import { canonicalFusionBranchName } from "./worktree-names.js";
|
||||
@@ -571,6 +571,107 @@ export class SelfHealingManager {
|
||||
});
|
||||
}
|
||||
|
||||
private hasRecentWorktreeIncompleteDetected(taskId: string, graceMs: number): boolean {
|
||||
if (!Number.isFinite(graceMs) || graceMs <= 0) return false;
|
||||
const storeWithRunAudit = this.store as { getRunAuditEvents?: (filter: { taskId: string; mutationType: string; limit: number }) => Array<{ timestamp?: string | null }> };
|
||||
if (typeof storeWithRunAudit.getRunAuditEvents !== "function") return false;
|
||||
let events: Array<{ timestamp?: string | null }> = [];
|
||||
try {
|
||||
events = storeWithRunAudit.getRunAuditEvents({ taskId, mutationType: "worktree:incomplete-detected", limit: 20 }) ?? [];
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (!Array.isArray(events) || events.length === 0) return false;
|
||||
const cutoff = Date.now() - graceMs;
|
||||
return events.some((event) => {
|
||||
const ts = Date.parse(event.timestamp ?? "");
|
||||
return Number.isFinite(ts) && ts >= cutoff;
|
||||
});
|
||||
}
|
||||
|
||||
private async evaluateBackwardMoveTripleProof(
|
||||
task: Task,
|
||||
input: {
|
||||
stage: string;
|
||||
graceMs: number;
|
||||
stalenessAnchor: string | null | undefined;
|
||||
reason: string;
|
||||
extra?: Record<string, unknown>;
|
||||
},
|
||||
): Promise<{ ok: boolean; stalenessMs: number; reason: string; metadata: Record<string, unknown> }> {
|
||||
const livePaths = activeSessionRegistry.pathsForTask(task.id);
|
||||
const hasActiveRegisteredPath = livePaths.some((path) => activeSessionRegistry.isPathActive(path));
|
||||
const sessionDead = !hasActiveRegisteredPath && !executingTaskLock.has(task.id) && this.options.isTaskActive?.(task.id) !== true;
|
||||
|
||||
let worktreeUnusable = true;
|
||||
let worktreeClassification: { ok: boolean; classification?: string; reason?: string };
|
||||
if (task.worktree) {
|
||||
const cls = await classifyTaskWorktree(this.options.rootDir, task.worktree);
|
||||
worktreeClassification = cls.ok
|
||||
? { ok: true }
|
||||
: { ok: false, classification: cls.classification, reason: cls.reason };
|
||||
worktreeUnusable = !cls.ok;
|
||||
} else {
|
||||
const expected = canonicalFusionBranchName(task.id);
|
||||
const registeredPaths = await getRegisteredWorktreePaths(this.options.rootDir);
|
||||
const registeredBranchMap = await getRegisteredWorktreeBranchMap(this.options.rootDir);
|
||||
const matchingRegisteredPaths = [...registeredPaths].filter((path) => {
|
||||
const branch = registeredBranchMap.get(path);
|
||||
return typeof branch === "string" && branch.trim().toLowerCase() === expected;
|
||||
});
|
||||
worktreeClassification = matchingRegisteredPaths.length === 0
|
||||
? { ok: false, classification: "missing", reason: "task.worktree is null and no registered fusion worktree exists" }
|
||||
: { ok: true, reason: "registered fusion worktree exists while task.worktree is null" };
|
||||
worktreeUnusable = matchingRegisteredPaths.length === 0;
|
||||
}
|
||||
|
||||
const anchorMs = input.stalenessAnchor ? Date.parse(input.stalenessAnchor) : Number.NaN;
|
||||
const stalenessMs = Number.isFinite(anchorMs) ? Math.max(0, Date.now() - anchorMs) : Number.POSITIVE_INFINITY;
|
||||
const noRecentActivity = stalenessMs >= input.graceMs && !this.hasRecentWorktreeIncompleteDetected(task.id, input.graceMs);
|
||||
|
||||
const ok = sessionDead && worktreeUnusable && noRecentActivity;
|
||||
return {
|
||||
ok,
|
||||
stalenessMs,
|
||||
reason: input.reason,
|
||||
metadata: {
|
||||
priorWorktree: task.worktree ?? null,
|
||||
priorBranch: task.branch ?? null,
|
||||
hadWorktree: Boolean(task.worktree),
|
||||
stalenessMs,
|
||||
graceMs: input.graceMs,
|
||||
sessionDead,
|
||||
worktreeUnusable,
|
||||
noRecentActivity,
|
||||
livePaths,
|
||||
hasExecutingTaskLock: executingTaskLock.has(task.id),
|
||||
taskActive: this.options.isTaskActive?.(task.id) === true,
|
||||
worktreeClassification,
|
||||
...input.extra,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async emitBackwardMoveNoAction(task: Task, stage: string, mutationType: string, proof: { stalenessMs: number; reason: string; metadata: Record<string, unknown> }): Promise<void> {
|
||||
try {
|
||||
await createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId(`self-healing-${stage}`, task.id),
|
||||
agentId: "self-healing",
|
||||
taskId: task.id,
|
||||
taskLineageId: task.lineageId,
|
||||
phase: stage,
|
||||
}).database({
|
||||
type: mutationType as DatabaseMutationType,
|
||||
target: task.id,
|
||||
metadata: proof.metadata,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
log.warn(`[${stage}] ${task.id}: no-action audit emission failed: ${message}`);
|
||||
}
|
||||
log.log(`[${stage}] ${task.id}: triple-proof not satisfied — no action (operator-decides)`);
|
||||
}
|
||||
|
||||
// ── Lifecycle ───────────────────────────────────────────────────────
|
||||
|
||||
start(): void {
|
||||
@@ -1619,6 +1720,10 @@ export class SelfHealingManager {
|
||||
return reclaimed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:reclaim-pr-conflict-no-action` and skips lifecycle mutation.
|
||||
*/
|
||||
async reclaimPrConflictForTask(taskId: string): Promise<{ outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string; perPr?: Array<{ number: number; outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string }> }> {
|
||||
const task = await this.store.getTask(taskId);
|
||||
if (!task) return { outcome: "skipped", reason: "task-not-found" };
|
||||
@@ -1705,20 +1810,40 @@ export class SelfHealingManager {
|
||||
}
|
||||
}
|
||||
|
||||
await this.store.updateTask(task.id, {
|
||||
worktree: inspection.livePath,
|
||||
branch: task.branch,
|
||||
paused: false,
|
||||
pausedReason: undefined,
|
||||
status: null,
|
||||
error: null,
|
||||
});
|
||||
if (task.column === "in-review") {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveWorktree: true,
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "reclaim-pr-conflict",
|
||||
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "reclaim-pr-conflict-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "reclaim-pr-conflict", "task:reclaim-pr-conflict-no-action", proof);
|
||||
return withPerPr({ outcome: "skipped", reason: "triple-proof-not-satisfied" });
|
||||
} else {
|
||||
await this.store.updateTask(task.id, {
|
||||
worktree: inspection.livePath,
|
||||
branch: task.branch,
|
||||
paused: false,
|
||||
pausedReason: undefined,
|
||||
status: null,
|
||||
error: null,
|
||||
});
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveWorktree: true,
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
await this.store.updateTask(task.id, {
|
||||
worktree: inspection.livePath,
|
||||
branch: task.branch,
|
||||
paused: false,
|
||||
pausedReason: undefined,
|
||||
status: null,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "reclaimed", mode: inspection.kind } });
|
||||
@@ -1767,6 +1892,9 @@ export class SelfHealingManager {
|
||||
* STANDING: do not auto-discard stranded commits. Reclaim preserves commits;
|
||||
* unrecoverable conflicts are escalated for human review.
|
||||
*
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:reclaim-self-owned-branch-conflict-no-action` and skips lifecycle mutation.
|
||||
*
|
||||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||||
*/
|
||||
async reclaimSelfOwnedBranchConflicts(): Promise<number> {
|
||||
@@ -1830,6 +1958,15 @@ export class SelfHealingManager {
|
||||
}
|
||||
if (!await isUsableTaskWorktree(this.options.rootDir, task.worktree)) continue;
|
||||
|
||||
const reviewProof = task.column === "in-review"
|
||||
? await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "reclaim-self-owned-branch-conflict",
|
||||
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "reclaim-self-owned-candidate",
|
||||
})
|
||||
: null;
|
||||
|
||||
try {
|
||||
const inspection = await inspectBranchConflict({
|
||||
repoDir: this.options.rootDir,
|
||||
@@ -1898,11 +2035,15 @@ export class SelfHealingManager {
|
||||
);
|
||||
|
||||
if (task.column === "in-review") {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
if (!reviewProof?.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "reclaim-self-owned-branch-conflict", "task:reclaim-self-owned-branch-conflict-no-action", reviewProof!);
|
||||
} else {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -1997,11 +2138,15 @@ export class SelfHealingManager {
|
||||
);
|
||||
|
||||
if (task.column === "in-review") {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
if (!reviewProof?.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "reclaim-self-owned-branch-conflict", "task:reclaim-self-owned-branch-conflict-no-action", reviewProof!);
|
||||
} else {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -2062,12 +2207,16 @@ export class SelfHealingManager {
|
||||
);
|
||||
|
||||
if (task.column === "in-review") {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveWorktree: true,
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
if (!reviewProof?.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "reclaim-self-owned-branch-conflict", "task:reclaim-self-owned-branch-conflict-no-action", reviewProof!);
|
||||
} else {
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveWorktree: true,
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -2947,6 +3096,23 @@ export class SelfHealingManager {
|
||||
const ageMs = Number.isFinite(movedAtMs) ? now - movedAtMs : Number.POSITIVE_INFINITY;
|
||||
if (!options?.ignoreAgeGate && ageMs < thresholdMs) continue;
|
||||
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "auto-rebound-paused-scope-decay",
|
||||
graceMs: thresholdMs,
|
||||
stalenessAnchor: task.executionStartedAt ?? task.updatedAt,
|
||||
reason: "paused-scope-decay-candidate",
|
||||
extra: {
|
||||
followerCount,
|
||||
ignoredAgeGate: options?.ignoreAgeGate === true,
|
||||
thresholdMs,
|
||||
ageMs,
|
||||
},
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "auto-rebound-paused-scope-decay", "task:auto-rebound-scope-decay-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
preserveProgress: true,
|
||||
preserveWorktree: true,
|
||||
@@ -3745,6 +3911,9 @@ export class SelfHealingManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the unproven fallback predicate fails, emits `task:finalize-no-op-review-no-action` and skips lifecycle mutation.
|
||||
*
|
||||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||||
*/
|
||||
async finalizeNoOpReviewTasks(): Promise<number> {
|
||||
@@ -3810,6 +3979,16 @@ export class SelfHealingManager {
|
||||
details: classification.details,
|
||||
autoRetry: true,
|
||||
});
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "finalize-no-op-review",
|
||||
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "finalize-unproven-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "finalize-no-op-review", "task:finalize-no-op-review-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
await this.store.moveTask(task.id, "todo", { preserveProgress: true, moveSource: "engine" });
|
||||
continue;
|
||||
}
|
||||
@@ -4200,6 +4379,8 @@ export class SelfHealingManager {
|
||||
*
|
||||
* Moving them back to `todo` lets the normal scheduler/executor resume the
|
||||
* incomplete step instead of leaving the task stranded in review.
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:stale-incomplete-review-no-action` and skips lifecycle mutation.
|
||||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||||
*/
|
||||
async recoverStaleIncompleteReviewTasks(): Promise<number> {
|
||||
@@ -4228,6 +4409,17 @@ export class SelfHealingManager {
|
||||
let recovered = 0;
|
||||
for (const task of staleIncomplete) {
|
||||
try {
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "stale-incomplete-review",
|
||||
graceMs: timeoutMs,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "stale-incomplete-review-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "stale-incomplete-review", "task:stale-incomplete-review-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
"Auto-recovered: in-review task still had incomplete steps — moved back to todo for retry",
|
||||
@@ -4578,6 +4770,9 @@ export class SelfHealingManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:ghost-review-no-action` and skips lifecycle mutation.
|
||||
*
|
||||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||||
*/
|
||||
async recoverGhostReviewTasks(): Promise<number> {
|
||||
@@ -4608,6 +4803,16 @@ export class SelfHealingManager {
|
||||
let recovered = 0;
|
||||
for (const task of ghosts) {
|
||||
try {
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "ghost-review",
|
||||
graceMs: timeoutMs,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "ghost-review-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "ghost-review", "task:ghost-review-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
if (task.status) {
|
||||
await this.store.updateTask(task.id, { status: null, error: null });
|
||||
}
|
||||
@@ -5029,6 +5234,9 @@ export class SelfHealingManager {
|
||||
/**
|
||||
* Recover deadlocked retry-exhausted merge failures that are still blocking
|
||||
* dispatch via `blockedBy` or retained worktree ownership.
|
||||
*
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the no-landed predicate fails, emits `task:stuck-merge-deadlock-no-action` and skips lifecycle mutation.
|
||||
*/
|
||||
/**
|
||||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||||
@@ -5120,10 +5328,20 @@ export class SelfHealingManager {
|
||||
log.log(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: landedCommit.sha, action: "reattributed" })}`);
|
||||
recovered++;
|
||||
} else {
|
||||
await this.store.updateTask(task.id, { paused: true });
|
||||
await this.store.logEntry(task.id, "merge-deadlock-detected: requires manual intervention — verified content not on main");
|
||||
log.warn(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: null, action: "paused-for-manual" })}`);
|
||||
recovered++;
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "stuck-merge-deadlock",
|
||||
graceMs: DEADLOCK_RECOVERY_COOLDOWN_MS,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "stuck-merge-deadlock-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "stuck-merge-deadlock", "task:stuck-merge-deadlock-no-action", proof);
|
||||
} else {
|
||||
await this.store.updateTask(task.id, { paused: true });
|
||||
await this.store.logEntry(task.id, "merge-deadlock-detected: requires manual intervention — verified content not on main");
|
||||
log.warn(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: null, action: "paused-for-manual" })}`);
|
||||
recovered++;
|
||||
}
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
@@ -6443,6 +6661,9 @@ export class SelfHealingManager {
|
||||
* Recover `in-progress` tasks that failed only because the agent exited
|
||||
* without calling fn_task_done, and where there is no sign of work to preserve.
|
||||
*
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:no-progress-no-task-done-no-action` and skips lifecycle mutation.
|
||||
*
|
||||
* These are safe to requeue automatically when no steps progressed and git
|
||||
* has neither worktree changes nor branch commits. Cases with any evidence
|
||||
* of work are left alone for manual inspection or the normal orphan recovery
|
||||
@@ -6475,6 +6696,17 @@ export class SelfHealingManager {
|
||||
continue;
|
||||
}
|
||||
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "no-progress-no-task-done",
|
||||
graceMs: ORPHANED_EXECUTION_RECOVERY_GRACE_MS,
|
||||
stalenessAnchor: task.executionStartedAt ?? task.updatedAt,
|
||||
reason: "no-progress-no-task-done-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "no-progress-no-task-done", "task:no-progress-no-task-done-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.store.updateTask(task.id, {
|
||||
status: "stuck-killed",
|
||||
worktree: null,
|
||||
@@ -6504,6 +6736,9 @@ export class SelfHealingManager {
|
||||
/**
|
||||
* Recover failed `in-review` retries that point at an unusable worktree path.
|
||||
*
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:missing-worktree-review-no-action` and skips lifecycle mutation.
|
||||
*
|
||||
* This is a narrow guard for session-start failures thrown by
|
||||
* `assertValidWorktreeSession()` in `pi.ts`, classified centrally via
|
||||
* `MISSING_WORKTREE_SESSION_PREFIXES` /
|
||||
@@ -6532,6 +6767,17 @@ export class SelfHealingManager {
|
||||
let recovered = 0;
|
||||
for (const task of candidates) {
|
||||
try {
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "missing-worktree-review",
|
||||
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "missing-worktree-review-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "missing-worktree-review", "task:missing-worktree-review-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
|
||||
const auditor = createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("self-heal", task.id),
|
||||
agentId: "self-healing",
|
||||
@@ -6565,6 +6811,9 @@ export class SelfHealingManager {
|
||||
/**
|
||||
* Recover `in-review` tasks marked as `failed` because the agent exited
|
||||
* without calling `fn_task_done` *with partial step progress* (some steps done,
|
||||
*
|
||||
* Backward lifecycle move gated on triple proof (FN-5335).
|
||||
* When the predicate fails, emits `task:partial-progress-no-task-done-no-action` and skips lifecycle mutation.
|
||||
* some still pending). The work-in-progress is valuable but incomplete —
|
||||
* the existing worktree and branch are preserved and the task is moved back
|
||||
* to `todo` so the scheduler re-dispatches it for a fresh execution that
|
||||
@@ -6610,6 +6859,17 @@ export class SelfHealingManager {
|
||||
let recovered = 0;
|
||||
for (const task of candidates) {
|
||||
try {
|
||||
const proof = await this.evaluateBackwardMoveTripleProof(task, {
|
||||
stage: "partial-progress-no-task-done",
|
||||
graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS,
|
||||
stalenessAnchor: task.columnMovedAt ?? task.updatedAt,
|
||||
reason: "partial-progress-no-task-done-candidate",
|
||||
});
|
||||
if (!proof.ok) {
|
||||
await this.emitBackwardMoveNoAction(task, "partial-progress-no-task-done", "task:partial-progress-no-task-done-no-action", proof);
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextCount = (task.taskDoneRetryCount ?? 0) + 1;
|
||||
await this.store.updateTask(task.id, {
|
||||
status: null,
|
||||
|
||||
Reference in New Issue
Block a user