fix(FN-5335): enforce triple-proof gating for backward recovery
- Gate self-healing backward moves behind audited triple-proof predicates across reclaim paths - Skip reclaim-pr-conflict mutations when proof checks fail and preserve no-action behavior - Add broad unit and reliability-interaction coverage for triple-proof and cross-layer scenarios - Document backward-move stage invariants, diagnostics, and add delivery changeset for @runfusion/fusion
This commit is contained in:
committed by
gsxdsm
parent
a2a5db8151
commit
a8715ed963
@@ -0,0 +1,275 @@
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { TaskStore } from "@fusion/core";
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
import { activeSessionRegistry, executingTaskLock } from "../../active-session-registry.js";
|
||||
|
||||
function git(cwd: string, command: string): string {
|
||||
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
describe("FN-5335 reliability interactions: backward move triple proof", () => {
|
||||
let rootDir = "";
|
||||
let store: TaskStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-21T12:00:00.000Z"));
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
rootDir = mkdtempSync(join(tmpdir(), "fn-5335-reliability-"));
|
||||
git(rootDir, "init -b main");
|
||||
git(rootDir, "config user.name 'Fusion'");
|
||||
git(rootDir, "config user.email 'hi@runfusion.ai'");
|
||||
writeFileSync(join(rootDir, "README.md"), "root\n");
|
||||
git(rootDir, "add README.md");
|
||||
git(rootDir, "commit -m 'init'");
|
||||
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
|
||||
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
|
||||
await store.init();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
try { store?.close(); } catch {}
|
||||
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
async function createNoProgressTask(worktree: string, ageMs: number) {
|
||||
const task = await store.createTask({ title: "no-progress", description: "no-progress" });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.updateTask(task.id, {
|
||||
worktree,
|
||||
status: "failed",
|
||||
paused: false,
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
executionStartedAt: new Date(Date.now() - ageMs).toISOString(),
|
||||
updatedAt: new Date(Date.now() - ageMs).toISOString(),
|
||||
steps: [{ name: "step", status: "pending" }],
|
||||
} as any);
|
||||
return task.id;
|
||||
}
|
||||
|
||||
it("Scenario A: live session blocks backward move and emits no-action", async () => {
|
||||
const worktree = join(rootDir, ".worktrees", "np-live-missing");
|
||||
const id = await createNoProgressTask(worktree, 400_000);
|
||||
activeSessionRegistry.registerPath(worktree, { taskId: id, kind: "executor", ownerKey: "run-a" });
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task?.column).toBe("in-progress");
|
||||
expect(events).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario B: usable worktree blocks backward move (recoverable git work short-circuit)", async () => {
|
||||
const worktree = join(rootDir, ".worktrees", "np-usable");
|
||||
mkdirSync(worktree, { recursive: true });
|
||||
const id = await createNoProgressTask(worktree, 400_000);
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task?.column).toBe("in-progress");
|
||||
expect(events).toHaveLength(0);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario C: recent activity blocks backward move", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-recent"), 200);
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
const events = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task?.column).toBe("in-progress");
|
||||
expect(events).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario D: all triple-proof signals true allows recovery", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-stale"), 400_000);
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const task = await store.getTask(id);
|
||||
|
||||
expect(recovered).toBe(1);
|
||||
expect(task?.column).toBe("todo");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario E: autoMerge false keeps in-review stage no-op", async () => {
|
||||
const task = await store.createTask({ title: "review", description: "review" });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.moveTask(task.id, "in-review");
|
||||
await store.updateTask(task.id, {
|
||||
status: "failed",
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
taskDoneRetryCount: 1,
|
||||
worktree: join(rootDir, ".worktrees", "review-missing"),
|
||||
updatedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
steps: [{ name: "done", status: "done" }, { name: "pending", status: "pending" }],
|
||||
} as any);
|
||||
await store.updateSettings({ ...(await store.getSettings()), autoMerge: false } as any);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
|
||||
const current = await store.getTask(task.id);
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(current?.column).toBe("in-review");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario F: churn-terminalized review task is not reopened", async () => {
|
||||
const task = await store.createTask({ title: "churn", description: "churn" });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.moveTask(task.id, "in-review");
|
||||
await store.updateTask(task.id, {
|
||||
paused: true,
|
||||
pausedReason: "in-review-stall-deadlock",
|
||||
status: "failed",
|
||||
error: "STUCK_NO_PROGRESS_CHURN",
|
||||
worktree: join(rootDir, ".worktrees", "churn-missing"),
|
||||
updatedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
steps: [{ name: "done", status: "done" }, { name: "pending", status: "pending" }],
|
||||
} as any);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
|
||||
const current = await store.getTask(task.id);
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(current?.column).toBe("in-review");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario G: order-independence across limbo and no-progress sweeps", async () => {
|
||||
const makeCandidate = async (title: string) => {
|
||||
const task = await store.createTask({ title, description: title });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
await store.updateTask(task.id, {
|
||||
branch: null,
|
||||
worktree: join(rootDir, ".worktrees", `${task.id.toLowerCase()}-missing`),
|
||||
status: "failed",
|
||||
error: "Agent finished without calling fn_task_done",
|
||||
executionStartedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
updatedAt: new Date(Date.now() - 400_000).toISOString(),
|
||||
steps: [{ name: "step", status: "pending" }],
|
||||
} as any);
|
||||
return task.id;
|
||||
};
|
||||
|
||||
const firstId = await makeCandidate("order-a");
|
||||
await (store as any).recordRunAuditEvent({ runId: "run-g-a", phase: "executor", taskId: firstId, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: firstId, details: null, metadata: {} });
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false, getExecutingTaskIds: () => new Set<string>() });
|
||||
await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
await manager.recoverInProgressLimbo();
|
||||
const firstTask = await store.getTask(firstId);
|
||||
|
||||
const secondId = await makeCandidate("order-b");
|
||||
await (store as any).recordRunAuditEvent({ runId: "run-g-b", phase: "executor", taskId: secondId, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: secondId, details: null, metadata: {} });
|
||||
await manager.recoverInProgressLimbo();
|
||||
await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const secondTask = await store.getTask(secondId);
|
||||
|
||||
expect(firstTask?.column).toBe("in-progress");
|
||||
expect(secondTask?.column).toBe("in-progress");
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario H: orphan and tightened sweeps can co-emit no-action events", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-orphan-h"), 400_000);
|
||||
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>(), isTaskActive: () => false });
|
||||
vi.setSystemTime(new Date("2026-05-21T12:10:00.000Z"));
|
||||
await manager.recoverOrphanedExecutions();
|
||||
await (store as any).recordRunAuditEvent({ runId: "run-h", phase: "executor", taskId: id, taskLineageId: null, agentId: "executor", domain: "database", mutationType: "worktree:incomplete-detected", payload: {}, target: id, details: null, metadata: {} });
|
||||
await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
|
||||
const orphanEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:orphan-detected-no-action" });
|
||||
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
const current = await store.getTask(id);
|
||||
|
||||
expect(current?.column).toBe("in-progress");
|
||||
expect(orphanEvents).toHaveLength(1);
|
||||
expect(noActionEvents).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario J: tightened sweep no-action is idempotent across re-sweeps", async () => {
|
||||
const worktree = join(rootDir, ".worktrees", "np-missing-orphan");
|
||||
const id = await createNoProgressTask(worktree, 400_000);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set<string>(), isTaskActive: () => false });
|
||||
vi.setSystemTime(new Date("2026-05-21T12:10:00.000Z"));
|
||||
await manager.recoverOrphanedExecutions();
|
||||
await (store as any).recordRunAuditEvent({
|
||||
runId: "run-fn5335-h",
|
||||
phase: "executor",
|
||||
taskId: id,
|
||||
taskLineageId: null,
|
||||
agentId: "executor",
|
||||
domain: "database",
|
||||
mutationType: "worktree:incomplete-detected",
|
||||
payload: { source: "executor-liveness-gate" },
|
||||
target: id,
|
||||
details: null,
|
||||
metadata: { source: "executor-liveness-gate" },
|
||||
});
|
||||
const first = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const second = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
const current = await store.getTask(id);
|
||||
|
||||
expect(first).toBe(0);
|
||||
expect(second).toBe(0);
|
||||
expect(current?.column).toBe("in-progress");
|
||||
expect(noActionEvents).toHaveLength(2);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("Scenario I: recent worktree:incomplete-detected audit counts as recent activity", async () => {
|
||||
const id = await createNoProgressTask(join(rootDir, ".worktrees", "np-missing-liveness"), 400_000);
|
||||
await (store as any).recordRunAuditEvent({
|
||||
runId: "run-fn5335-liveness",
|
||||
phase: "executor",
|
||||
taskId: id,
|
||||
taskLineageId: null,
|
||||
agentId: "executor",
|
||||
domain: "database",
|
||||
mutationType: "worktree:incomplete-detected",
|
||||
payload: { source: "executor-liveness-gate" },
|
||||
target: id,
|
||||
details: null,
|
||||
metadata: { source: "executor-liveness-gate" },
|
||||
});
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir, isTaskActive: () => false });
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
const noActionEvents = await store.getRunAuditEvents({ taskId: id, mutationType: "task:no-progress-no-task-done-no-action" });
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(noActionEvents).toHaveLength(1);
|
||||
manager.stop();
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@ function makeTask(id: string, overrides: Partial<Task> = {}): Task {
|
||||
currentStep: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
executionStartedAt: new Date(Date.now() - 61_000).toISOString(),
|
||||
log: [],
|
||||
...overrides,
|
||||
} as Task;
|
||||
|
||||
@@ -56,7 +56,7 @@ describe("reliability interactions: live-zero reclaim", () => {
|
||||
pausedReason: "branch-conflict-unrecoverable",
|
||||
userPaused: false,
|
||||
lineageId: "lin-9100",
|
||||
updatedAt: new Date().toISOString(),
|
||||
updatedAt: new Date(Date.now() - 11 * 60_000).toISOString(),
|
||||
};
|
||||
|
||||
const statefulStore = {
|
||||
|
||||
@@ -143,8 +143,8 @@ describe("foreign start-point no-owned-commit interactions (real git)", () => {
|
||||
const recovered = await manager.finalizeNoOpReviewTasks();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(task.column).toBe("todo");
|
||||
expect(events.some((event: any) => event?.mutationType === "task:finalize-unproven-blocked")).toBe(true);
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(events.some((event: any) => event?.mutationType === "task:finalize-no-op-review-no-action")).toBe(true);
|
||||
manager.stop();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
|
||||
@@ -125,6 +125,7 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
it("keeps in-review missing-worktree failures on the review-specific recovery path", async () => {
|
||||
const id = await createInProgressTask("review failure disjoint");
|
||||
await store.moveTask(id, "in-review");
|
||||
await store.updateSettings({ autoMerge: true } as any);
|
||||
await store.updateTask(id, {
|
||||
status: "failed",
|
||||
error: `Refusing to start coding agent in missing worktree: ${join(rootDir, ".worktrees", "missing-review")}`,
|
||||
@@ -132,6 +133,10 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
worktree: join(rootDir, ".worktrees", "missing-review-stale"),
|
||||
steps: [{ name: "step", status: "done" }, { name: "next", status: "pending" }],
|
||||
});
|
||||
await store.updateTask(id, {
|
||||
updatedAt: new Date(Date.now() - 48 * 60 * 60_000).toISOString(),
|
||||
columnMovedAt: new Date(Date.now() - 48 * 60 * 60_000).toISOString(),
|
||||
} as any);
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir,
|
||||
@@ -143,8 +148,8 @@ describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
const updated = await store.getTask(id);
|
||||
|
||||
expect(limboRecovered).toBe(0);
|
||||
expect(reviewRecovered).toBe(1);
|
||||
expect(updated?.column).toBe("todo");
|
||||
expect(reviewRecovered).toBe(0);
|
||||
expect(updated?.column).toBe("in-review");
|
||||
});
|
||||
|
||||
it("skips limbo recovery while the executor still claims the task id", async () => {
|
||||
|
||||
@@ -64,6 +64,7 @@ describe("reliability interactions: paused scope decay", () => {
|
||||
column: "in-progress",
|
||||
paused: true,
|
||||
pausedReason: "waiting",
|
||||
executionStartedAt: new Date(now - 31 * 60_000).toISOString(),
|
||||
columnMovedAt: new Date(now - 31 * 60_000).toISOString(),
|
||||
currentStep: 2,
|
||||
steps: [{ id: "s1", title: "x", status: "done" } as any],
|
||||
@@ -97,6 +98,7 @@ describe("reliability interactions: paused scope decay", () => {
|
||||
const holder = makeTask("FN-3", {
|
||||
column: "in-progress",
|
||||
paused: true,
|
||||
executionStartedAt: new Date(now - 61_000).toISOString(),
|
||||
columnMovedAt: new Date(now - 1_000).toISOString(),
|
||||
});
|
||||
const follower = makeTask("FN-4", { column: "todo", blockedBy: "FN-3" });
|
||||
|
||||
@@ -76,7 +76,7 @@ describe("reliability interaction: pr conflict reclaim", () => {
|
||||
});
|
||||
|
||||
it("keeps paused-review reclaim path resumable", async () => {
|
||||
const t = task();
|
||||
const t = task({ updatedAt: new Date(Date.now() - 11 * 60_000).toISOString() });
|
||||
const s = store(t);
|
||||
vi.spyOn(branchConflicts, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: t.worktree, tipSha: "abc123", taskAttributedCommitCount: 2, strandedCommits: [{ sha: "abc123" }] } as any);
|
||||
const manager = new SelfHealingManager(s as any, { rootDir: "/tmp/test" } as any);
|
||||
|
||||
@@ -86,6 +86,7 @@ describe("reliability interactions: self-healing", () => {
|
||||
worktree: "/tmp/wt",
|
||||
branch: "fusion/wt",
|
||||
steps: [{ id: "s1", title: "Step", status: "pending" }] as any,
|
||||
updatedAt: new Date(Date.now() - 31 * 60_000).toISOString(),
|
||||
}),
|
||||
]]);
|
||||
const store = makeStore(tasks);
|
||||
|
||||
@@ -107,7 +107,7 @@ describe("SelfHealingManager.reclaimPrConflictForTask", () => {
|
||||
});
|
||||
|
||||
it("returns reclaimed for reclaimable conflicts", async () => {
|
||||
const task = makeTask({ column: "in-review", paused: true, pausedReason: "branch-conflict-unrecoverable" as any });
|
||||
const task = makeTask({ column: "in-review", paused: true, pausedReason: "branch-conflict-unrecoverable" as any, updatedAt: new Date(Date.now() - 11 * 60_000).toISOString() });
|
||||
const store = makeStore(task);
|
||||
vi.spyOn(branchConflicts, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: task.worktree, tipSha: "abc123", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc123" }] } as any);
|
||||
const manager = new SelfHealingManager(store as any, { rootDir: "/tmp/test" } as any);
|
||||
|
||||
@@ -73,6 +73,9 @@ vi.mock("../worktree-pool.js", () => ({
|
||||
scanOrphanedBranches: vi.fn().mockResolvedValue([]),
|
||||
cleanupOrphanedWorktrees: vi.fn().mockResolvedValue(0),
|
||||
isUsableTaskWorktree: vi.fn().mockResolvedValue(true),
|
||||
classifyTaskWorktree: vi.fn().mockResolvedValue({ ok: false, classification: "missing", reason: "test-default" }),
|
||||
getRegisteredWorktreePaths: vi.fn().mockResolvedValue(new Set<string>()),
|
||||
getRegisteredWorktreeBranchMap: vi.fn().mockResolvedValue(new Map<string, string>()),
|
||||
removeWorktree: vi.fn().mockResolvedValue(undefined),
|
||||
resolveWorktreeBackend: vi.fn(),
|
||||
}));
|
||||
@@ -102,7 +105,8 @@ import { existsSync, readdirSync } from "node:fs";
|
||||
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "../worktree-pool.js";
|
||||
import { classifyTaskWorktree, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "../worktree-pool.js";
|
||||
import { activeSessionRegistry, executingTaskLock } from "../active-session-registry.js";
|
||||
import * as branchConflictModule from "../branch-conflicts.js";
|
||||
import { createLogger } from "../logger.js";
|
||||
import { NotificationService } from "../notification/notification-service.js";
|
||||
@@ -111,6 +115,9 @@ import { classifyOwnedLandedEvidence } from "../merger.js";
|
||||
const mockedExecSync = vi.mocked(execSync);
|
||||
const mockedExistsSync = vi.mocked(existsSync);
|
||||
const mockedIsUsableTaskWorktree = vi.mocked(isUsableTaskWorktree);
|
||||
const mockedClassifyTaskWorktree = vi.mocked(classifyTaskWorktree);
|
||||
const mockedGetRegisteredWorktreePaths = vi.mocked(getRegisteredWorktreePaths);
|
||||
const mockedGetRegisteredWorktreeBranchMap = vi.mocked(getRegisteredWorktreeBranchMap);
|
||||
const mockedRemoveWorktree = vi.mocked(removeWorktree);
|
||||
const mockedResolveWorktreeBackend = vi.mocked(resolveWorktreeBackend);
|
||||
const mockedScanIdleWorktrees = vi.mocked(scanIdleWorktrees);
|
||||
@@ -160,6 +167,7 @@ function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & E
|
||||
listTasks: vi.fn().mockResolvedValue([]),
|
||||
createTask: vi.fn().mockResolvedValue({ id: "FN-RESCUE", lineageId: "lin-rescue" }),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
getRunAuditEvents: vi.fn().mockReturnValue([]),
|
||||
getBootstrappedAt: vi.fn().mockReturnValue(null),
|
||||
getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
|
||||
clearStaleExecutionStartBranchReferences: vi.fn().mockReturnValue([]),
|
||||
@@ -174,14 +182,21 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
store = createMockStore();
|
||||
manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
mockedRemoveWorktree.mockResolvedValue(undefined);
|
||||
mockedClassifyTaskWorktree.mockResolvedValue({ ok: false, classification: "missing", reason: "test-default" });
|
||||
mockedGetRegisteredWorktreePaths.mockResolvedValue(new Set<string>());
|
||||
mockedGetRegisteredWorktreeBranchMap.mockResolvedValue(new Map<string, string>());
|
||||
mockedClassifyOwnedLandedEvidence.mockResolvedValue({ kind: "proven-no-op", baseRef: "main", ownDiffEmpty: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
manager.stop();
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
@@ -7654,3 +7669,193 @@ describe("autoMerge gating for mutating in-review sweeps (FN-5147)", () => {
|
||||
expect(store.logEntry).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("FN-5335 triple-proof no-action unit coverage", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-21T12:00:00.000Z"));
|
||||
activeSessionRegistry.clear();
|
||||
executingTaskLock._clearForTest();
|
||||
mockedClassifyTaskWorktree.mockResolvedValue({ ok: true } as any);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("emits stale-incomplete-review no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, autoMerge: true, taskStuckTimeoutMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-SIR", column: "in-review", paused: false, status: null, worktree: "/tmp/fn-sir", updatedAt: new Date(Date.now() - 5_000).toISOString(), steps: [{ status: "pending" }] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverStaleIncompleteReviewTasks();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:stale-incomplete-review-no-action" }));
|
||||
});
|
||||
|
||||
it("emits ghost-review no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, autoMerge: true, taskStuckTimeoutMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-GHOST", column: "in-review", worktree: "/tmp/fn-ghost", updatedAt: new Date(Date.now() - 5_000).toISOString(), columnMovedAt: new Date(Date.now() - 5_000).toISOString(), paused: false, status: null, mergeDetails: {} }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverGhostReviewTasks();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:ghost-review-no-action" }));
|
||||
});
|
||||
|
||||
it("emits no-progress no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-NP", column: "in-progress", worktree: "/tmp/fn-np", status: "failed", paused: false, error: "Agent finished without calling fn_task_done", updatedAt: new Date(Date.now() - 5_000).toISOString(), executionStartedAt: new Date(Date.now() - 5_000).toISOString(), steps: [{ id: "1", title: "a", status: "pending" }] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "hasRecoverableGitWork").mockResolvedValue(false);
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
const recovered = await manager.recoverNoProgressNoTaskDoneFailures();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:no-progress-no-task-done-no-action" }));
|
||||
});
|
||||
|
||||
it("emits missing-worktree-review no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-MWR", column: "in-review", paused: false, status: "failed", worktree: "/tmp/fn-mwr", branch: "fusion/fn-mwr", error: "Refusing to start coding agent in missing worktree: /tmp/fn-mwr", updatedAt: new Date(Date.now() - 10_000).toISOString(), steps: [{ status: "done" }, { status: "pending" }], log: [] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverMissingWorktreeReviewFailures();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:missing-worktree-review-no-action" }));
|
||||
});
|
||||
|
||||
it("emits partial-progress no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-PP", column: "in-review", paused: false, status: "failed", error: "Agent finished without calling fn_task_done", updatedAt: new Date(Date.now() - 10_000).toISOString(), taskDoneRetryCount: 1, steps: [{ status: "done" }, { status: "pending" }], worktree: "/tmp/fn-pp", branch: "fusion/fn-pp" }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
const recovered = await manager.recoverPartialProgressNoTaskDoneFailures();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:partial-progress-no-task-done-no-action" }));
|
||||
});
|
||||
|
||||
it("emits stuck-merge-deadlock no-action when no-landed branch fails proof", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, defaultBaseBranch: "main" } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([{ id: "FN-SMD", column: "in-review", paused: false, status: "failed", mergeRetries: 3, mergeDetails: undefined, worktree: "/tmp/wt", branch: "fusion/fn-smd", updatedAt: new Date(Date.now() - 100_000).toISOString(), log: [] }]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
mockedExecSync.mockReturnValue("" as any);
|
||||
|
||||
const recovered = await manager.recoverStuckMergeDeadlocks();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith("FN-SMD", { paused: true });
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:stuck-merge-deadlock-no-action" }));
|
||||
});
|
||||
|
||||
it("emits auto-rebound paused-scope no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false, pausedScopeDecayMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([
|
||||
{ id: "FN-HOLDER", column: "in-progress", paused: true, pausedReason: "waiting", blockedBy: null, worktree: "/tmp/wt-holder", updatedAt: new Date(Date.now() - 10_000).toISOString(), executionStartedAt: new Date(Date.now() - 10_000).toISOString() },
|
||||
{ id: "FN-FOLLOW", column: "todo", paused: false, blockedBy: "FN-HOLDER" },
|
||||
]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
mockedClassifyTaskWorktree.mockResolvedValue({ ok: true } as any);
|
||||
|
||||
const recovered = await manager.autoReboundPausedScopeDecay();
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:auto-rebound-scope-decay-no-action" }));
|
||||
});
|
||||
|
||||
it("emits reclaim-pr-conflict no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 1_000 } as any),
|
||||
getTask: vi.fn().mockResolvedValue({ id: "FN-PR", column: "in-review", paused: false, status: null, worktree: "/tmp/wt-pr", branch: "fusion/fn-pr", prInfo: { number: 1, mergeable: "conflicting" }, updatedAt: new Date(Date.now() - 10_000).toISOString() }),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", livePath: "/tmp/wt-pr", tipSha: "abc", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc", subject: "work" }] } as any);
|
||||
|
||||
const result = await manager.reclaimPrConflictForTask("FN-PR");
|
||||
expect(result.outcome).toBe("skipped");
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reclaim-pr-conflict-no-action" }));
|
||||
});
|
||||
|
||||
it("emits reclaim-self-owned-branch-conflict no-action when triple proof fails", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 1_000 } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: "FN-RSBC",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
error: "branch-conflict-unrecoverable: conflict",
|
||||
branch: "fusion/fn-rsbc",
|
||||
worktree: "/tmp/wt-rsbc",
|
||||
updatedAt: new Date(Date.now() - 10_000).toISOString(),
|
||||
},
|
||||
]),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
vi.spyOn(branchConflictModule, "inspectBranchConflict").mockResolvedValue({ kind: "reclaimable", tipSha: "abc", taskAttributedCommitCount: 1, strandedCommits: [{ sha: "abc", subject: "work" }] } as any);
|
||||
|
||||
const result = await manager.reclaimSelfOwnedBranchConflicts();
|
||||
expect(result).toBeGreaterThanOrEqual(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reclaim-self-owned-branch-conflict-no-action" }));
|
||||
});
|
||||
|
||||
it("emits finalize-no-op-review no-action when unproven fallback fails triple proof", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false } as any),
|
||||
listTasks: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: "FN-NOOP",
|
||||
column: "in-review",
|
||||
paused: false,
|
||||
status: null,
|
||||
worktree: "/tmp/test-project/.worktrees/fn-noop",
|
||||
branch: "fusion/fn-noop",
|
||||
steps: [{ name: "Ship it", status: "done" }],
|
||||
workflowStepResults: [{ id: "ws-1", status: "passed", phase: "pre-merge" }],
|
||||
mergeDetails: undefined,
|
||||
updatedAt: new Date(Date.now() - 10_000).toISOString(),
|
||||
log: [],
|
||||
},
|
||||
]),
|
||||
});
|
||||
(store as any).recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false, reason: "test" });
|
||||
mockedClassifyOwnedLandedEvidence.mockResolvedValueOnce({
|
||||
kind: "unproven",
|
||||
reason: "foreign-start-point",
|
||||
details: { foreignRef: "fusion/fn-a" },
|
||||
} as any);
|
||||
mockedExecSync.mockImplementation((command) => {
|
||||
const cmd = String(command);
|
||||
if (cmd.includes("rev-parse --verify 'fusion/fn-noop'")) return "ok" as any;
|
||||
if (cmd.includes("rev-parse --verify 'main'")) return "ok" as any;
|
||||
if (cmd.includes("rev-list --count 'main'..'fusion/fn-noop'")) return "0\n" as any;
|
||||
return "" as any;
|
||||
});
|
||||
|
||||
const result = await manager.finalizeNoOpReviewTasks();
|
||||
expect(result).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalledWith("FN-NOOP", "todo", expect.anything());
|
||||
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:finalize-no-op-review-no-action" }));
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user