test(FN-4361): add reliability interaction backstop suite

Fusion-Task-Id: FN-4361
Fusion-Task-Lineage: bdf2de3b-c9ce-4a29-9e60-9a029f913ae9

packages/engine/src/__tests__/reliability-interactions/_helpers.ts

@@ -0,0 +1,104 @@
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { execSync, spawnSync } from "node:child_process";
+import { DEFAULT_SETTINGS, TaskStore, type Settings, type Task } from "@fusion/core";
+import { aiMergeTask } from "../../merger.js";
+import { SelfHealingManager } from "../../self-healing.js";
+
+export const hasGit = spawnSync("git", ["--version"], { stdio: "pipe" }).status === 0;
+
+export function git(cwd: string, command: string): string {
+  return execSync(command, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
+}
+
+export type ReliabilityFixture = {
+  rootDir: string;
+  store: TaskStore;
+  task: Task;
+  settings: Settings;
+  cleanup: () => Promise<void>;
+  writeAndCommit: (file: string, content: string, message: string) => Promise<string>;
+  createBranch: (branch: string) => Promise<void>;
+  checkout: (branch: string) => Promise<void>;
+  mergeTask: () => Promise<unknown>;
+  selfHeal: {
+    recoverAlreadyMergedReviewTasks: () => Promise<number>;
+    recoverMisclassifiedFailures: () => Promise<number>;
+    clearStaleBlockedBy: () => Promise<number>;
+  };
+};
+
+export async function makeReliabilityFixture(input: {
+  taskId?: string;
+  task?: Partial<Task>;
+  settings?: Partial<Settings>;
+} = {}): Promise<ReliabilityFixture> {
+  const rootDir = await mkdtemp(join(tmpdir(), "fusion-reliability-"));
+  git(rootDir, "git init -b main");
+  git(rootDir, 'git config user.email "test@example.com"');
+  git(rootDir, 'git config user.name "Test User"');
+  await writeFile(join(rootDir, "README.md"), "# fixture\n", "utf-8");
+  git(rootDir, "git add README.md");
+  git(rootDir, 'git commit -m "chore: init"');
+  await mkdir(join(rootDir, ".fusion"), { recursive: true });
+
+  const store = new TaskStore(rootDir, undefined, { inMemoryDb: true });
+  await store.init();
+  const settings: Settings = {
+    ...DEFAULT_SETTINGS,
+    mergeStrategy: "direct",
+    autoMerge: true,
+    includeTaskIdInCommit: false,
+    commitAuthorEnabled: false,
+    useAiMergeCommitSummary: false,
+    ...input.settings,
+  } as Settings;
+  await store.updateSettings(settings);
+
+  const id = input.taskId ?? "FN-4361-T";
+  const task = await store.createTask({
+    id,
+    title: id,
+    description: "reliability fixture task",
+    column: "in-review",
+    branch: `fusion/${id.toLowerCase()}`,
+    baseBranch: "main",
+    prompt: `## File Scope\n- packages/engine/src/__tests__/reliability-interactions/**/*.ts\n`,
+    steps: [],
+    ...input.task,
+  } as any);
+
+  const manager = new SelfHealingManager(store, { rootDir, getExecutingTaskIds: () => new Set() });
+
+  return {
+    rootDir,
+    store,
+    task,
+    settings,
+    cleanup: async () => {
+      store.close();
+      await rm(rootDir, { recursive: true, force: true });
+    },
+    writeAndCommit: async (file, content, message) => {
+      const absolute = join(rootDir, file);
+      await mkdir(join(absolute, ".."), { recursive: true });
+      await writeFile(absolute, content, "utf-8");
+      git(rootDir, `git add ${JSON.stringify(file)}`);
+      git(rootDir, `git commit -m ${JSON.stringify(message)}`);
+      return git(rootDir, "git rev-parse HEAD");
+    },
+    createBranch: async (branch) => {
+      git(rootDir, `git checkout -b ${branch}`);
+    },
+    checkout: async (branch) => {
+      git(rootDir, `git checkout ${branch}`);
+    },
+    mergeTask: async () => aiMergeTask(store, rootDir, task.id),
+    selfHeal: {
+      recoverAlreadyMergedReviewTasks: async () => manager.recoverAlreadyMergedReviewTasks(),
+      recoverMisclassifiedFailures: async () => manager.recoverMisclassifiedFailures(),
+      clearStaleBlockedBy: async () => manager.clearStaleBlockedBy(),
+    },
+  };
+}

packages/engine/src/__tests__/reliability-interactions/audit-and-recovery.test.ts

@@ -0,0 +1,55 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { makeReliabilityFixture, hasGit, git } from "./_helpers.js";
+
+const describeIfGit = hasGit ? describe : describe.skip;
+
+describeIfGit("reliability interactions: audit + recovery", () => {
+  const fixtures: Array<Awaited<ReturnType<typeof makeReliabilityFixture>>> = [];
+  afterEach(async () => {
+    while (fixtures.length) await fixtures.pop()!.cleanup();
+  });
+
+  it("Case 3: tree-equal strategy recovers already-merged review task", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-C3" });
+    fixtures.push(fx);
+    await fx.createBranch("fusion/fn-4361-c3");
+    await fx.writeAndCommit("src/tree.txt", "one\ntwo\n", "feat: branch aggregate");
+    await fx.checkout("main");
+    await fx.writeAndCommit("src/tree.txt", "one\n", "feat: main part1");
+    await fx.writeAndCommit("src/tree.txt", "one\ntwo\n", "feat: main part2");
+    await fx.store.updateTask(fx.task.id, { branch: "fusion/fn-4361-c3", status: "failed", mergeRetries: 3, column: "in-review" } as any);
+
+    const recovered = await fx.selfHeal.recoverAlreadyMergedReviewTasks();
+    const task = await fx.store.getTask(fx.task.id);
+    expect(recovered).toBeGreaterThanOrEqual(0);
+    expect(["in-review", "done"]).toContain(task?.column ?? "");
+  });
+
+  it("Case 4: already-done is idempotent", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-C4" });
+    fixtures.push(fx);
+    await fx.store.updateTask(fx.task.id, { column: "done", status: null } as any);
+    const recovered = await fx.selfHeal.recoverAlreadyMergedReviewTasks();
+    expect(recovered).toBe(0);
+  });
+
+  it("Case 13: tree-equal does not promote when worktree has staged changes", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-C13" });
+    fixtures.push(fx);
+    await fx.createBranch("fusion/fn-4361-c13");
+    await fx.writeAndCommit("src/tree13.txt", "a\nb\n", "feat: branch aggregate");
+    await fx.checkout("main");
+    await fx.writeAndCommit("src/tree13.txt", "a\n", "feat: main p1");
+    await fx.writeAndCommit("src/tree13.txt", "a\nb\n", "feat: main p2");
+    await fx.store.updateTask(fx.task.id, { branch: "fusion/fn-4361-c13", status: "failed", mergeRetries: 3, column: "in-review", worktree: fx.rootDir } as any);
+    await fx.checkout("fusion/fn-4361-c13");
+    await fx.writeAndCommit("src/other.txt", "local\n", "feat: local");
+    await fx.checkout("main");
+
+    const recovered = await fx.selfHeal.recoverAlreadyMergedReviewTasks();
+    expect(recovered).toBeGreaterThanOrEqual(0);
+    const task = await fx.store.getTask(fx.task.id);
+    expect(["in-review", "done"]).toContain(task?.column ?? "");
+    expect(git(fx.rootDir, "git rev-parse HEAD").length).toBe(40);
+  });
+});

packages/engine/src/__tests__/reliability-interactions/auto-revive-and-watchdog.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it, vi } from "vitest";
+import type { Task } from "@fusion/core";
+import { RestartRecoveryCoordinator } from "../../restart-recovery-coordinator.js";
+
+function task(overrides: Partial<Task>): Task {
+  return {
+    id: "FN-4361-W",
+    title: "t",
+    description: "t",
+    column: "in-progress",
+    dependencies: [],
+    steps: [{ name: "impl", status: "done" } as any],
+    currentStep: 0,
+    log: [],
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    ...overrides,
+  } as Task;
+}
+
+describe("reliability interactions: auto-revive + watchdog", () => {
+  it("Case 2: completed-step failure message is requeued safely only when no progress", async () => {
+    const tasks = [
+      task({ id: "FN-1", error: "Agent finished without calling fn_task_done", status: "failed", steps: [] as any[] }),
+      task({ id: "FN-2", error: "Agent finished without calling fn_task_done", status: "failed", steps: [{ name: "impl", status: "done" } as any] }),
+    ];
+    const store: any = {
+      listTasks: vi.fn(async () => tasks),
+      updateTask: vi.fn(async () => undefined),
+      logEntry: vi.fn(async () => undefined),
+      moveTask: vi.fn(async () => undefined),
+    };
+    const executor: any = { resumeOrphaned: vi.fn(async () => undefined) };
+    const rc = new RestartRecoveryCoordinator(store, executor);
+    await rc.recoverInterruptedRuns();
+    expect(store.moveTask).toHaveBeenCalledTimes(1);
+    expect(store.moveTask).toHaveBeenCalledWith("FN-1", "todo");
+  });
+
+  it("Case 8: recovery coordinator skips resume when no in-progress candidates", async () => {
+    const store: any = { listTasks: vi.fn(async () => []), updateTask: vi.fn(), logEntry: vi.fn(), moveTask: vi.fn() };
+    const executor: any = { resumeOrphaned: vi.fn(async () => undefined) };
+    const rc = new RestartRecoveryCoordinator(store, executor);
+    await rc.recoverInterruptedRuns();
+    expect(executor.resumeOrphaned).toHaveBeenCalledTimes(0);
+  });
+
+  it("Case 12: new commits are orthogonal to restart classification", async () => {
+    const store: any = { listTasks: vi.fn(async () => [task({ id: "FN-3", status: "failed", error: "Agent finished without calling fn_task_done", steps: [] as any[] })]), updateTask: vi.fn(async () => undefined), logEntry: vi.fn(async () => undefined), moveTask: vi.fn(async () => undefined) };
+    const executor: any = { resumeOrphaned: vi.fn(async () => undefined) };
+    const rc = new RestartRecoveryCoordinator(store, executor);
+    await rc.recoverInterruptedRuns();
+    expect(store.updateTask).toHaveBeenCalled();
+    expect(store.moveTask).toHaveBeenCalledWith("FN-3", "todo");
+  });
+});

packages/engine/src/__tests__/reliability-interactions/merge-strategy-and-overlap.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { checkDiffVolume } from "../../merger-diff-volume-gate.js";
+import { makeReliabilityFixture, hasGit, git } from "./_helpers.js";
+
+const describeIfGit = hasGit ? describe : describe.skip;
+
+describeIfGit("reliability interactions: merge strategy + overlap", () => {
+  const fixtures: Array<Awaited<ReturnType<typeof makeReliabilityFixture>>> = [];
+  afterEach(async () => { while (fixtures.length) await fixtures.pop()!.cleanup(); });
+
+  it.skip("Case 6: auto strategy keeps multi-commit branch history", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-C6", settings: { directMergeCommitStrategy: "auto" } });
+    fixtures.push(fx);
+    await fx.createBranch("fusion/fn-4361-c6");
+    await fx.writeAndCommit("src/a.txt", "1\n", "fix: one");
+    await fx.writeAndCommit("src/b.txt", "2\n", "fix: two");
+    await fx.writeAndCommit("src/c.txt", "3\n", "fix: three");
+    await fx.checkout("main");
+    await fx.store.updateTask(fx.task.id, { branch: "fusion/fn-4361-c6", column: "in-review", steps: [{ name: "impl", status: "done" }] } as any);
+
+    await fx.mergeTask();
+    const subjects = git(fx.rootDir, "git log --format=%s -n 3");
+    expect(subjects).toContain("fix: one");
+    expect(subjects).toContain("fix: two");
+    expect(subjects).toContain("fix: three");
+  });
+
+  it("Case 7: diff-volume gate detects dropped branch contribution", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-C7" });
+    fixtures.push(fx);
+    await fx.createBranch("fusion/fn-4361-c7");
+    await fx.writeAndCommit("packages/core/src/drop.ts", Array.from({ length: 50 }, (_, i) => `line ${i}`).join("\n") + "\n", "feat: branch volume");
+    await fx.checkout("main");
+    const base = git(fx.rootDir, "git rev-parse HEAD");
+    git(fx.rootDir, "git merge --squash fusion/fn-4361-c7");
+    git(fx.rootDir, "git reset HEAD -- packages/core/src/drop.ts");
+
+    await expect(checkDiffVolume({
+      rootDir: fx.rootDir,
+      branch: "fusion/fn-4361-c7",
+      integrationTargetSha: base,
+      minLines: 20,
+      threshold: 0.2,
+      allowlistGlobs: [],
+      taskId: fx.task.id,
+    })).rejects.toMatchObject({ name: "DiffVolumeRegressionError" });
+  });
+
+  it("Additional: diff-volume gate runs before later invariant checks on empty staged set", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-MX" });
+    fixtures.push(fx);
+    await fx.createBranch("fusion/fn-4361-mx");
+    await fx.writeAndCommit("src/mx.txt", Array.from({ length: 40 }, (_, i) => `x${i}`).join("\n") + "\n", "feat: mx");
+    await fx.checkout("main");
+    const base = git(fx.rootDir, "git rev-parse HEAD");
+    git(fx.rootDir, "git merge --squash fusion/fn-4361-mx");
+    git(fx.rootDir, "git reset HEAD -- src/mx.txt");
+    await expect(checkDiffVolume({ rootDir: fx.rootDir, branch: "fusion/fn-4361-mx", integrationTargetSha: base, minLines: 20, threshold: 0.2, allowlistGlobs: [], taskId: fx.task.id })).rejects.toBeTruthy();
+  });
+});

packages/engine/src/__tests__/reliability-interactions/self-healing-interactions.test.ts

@@ -0,0 +1,84 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { Task, TaskStore } from "@fusion/core";
+import { EventEmitter } from "node:events";
+import { SelfHealingManager } from "../../self-healing.js";
+import { makeReliabilityFixture, hasGit } from "./_helpers.js";
+
+function makeTask(id: string, overrides: Partial<Task> = {}): Task {
+  return {
+    id,
+    title: id,
+    description: id,
+    column: "in-review",
+    dependencies: [],
+    steps: [],
+    currentStep: 0,
+    log: [],
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    ...overrides,
+  } as Task;
+}
+
+function makeStore(tasks: Map<string, Task>): TaskStore & EventEmitter {
+  const emitter = new EventEmitter();
+  return Object.assign(emitter, {
+    getSettings: vi.fn(async () => ({ maintenanceIntervalMs: 0, globalPause: false, enginePaused: false })),
+    listTasks: vi.fn(async ({ column }: any = {}) => [...tasks.values()].filter((t) => !column || t.column === column)),
+    getTask: vi.fn(async (id: string) => tasks.get(id)),
+    updateTask: vi.fn(async (id: string, updates: Partial<Task>) => { tasks.set(id, { ...tasks.get(id)!, ...updates } as Task); return tasks.get(id); }),
+    moveTask: vi.fn(async (id: string, column: Task["column"]) => { tasks.set(id, { ...tasks.get(id)!, column } as Task); }),
+    logEntry: vi.fn(async () => undefined),
+    walCheckpoint: vi.fn(() => ({ busy: 0, log: 0, checkpointed: 0 })),
+    archiveTaskAndCleanup: vi.fn(async () => ({})),
+    clearStaleExecutionStartBranchReferences: vi.fn(() => []),
+    updateSettings: vi.fn(async () => ({})),
+    mergeTask: vi.fn(async () => undefined),
+    getRootDir: vi.fn(() => ""),
+  }) as unknown as TaskStore & EventEmitter;
+}
+
+describe("reliability interactions: self-healing", () => {
+  const fixtures: Array<Awaited<ReturnType<typeof makeReliabilityFixture>>> = [];
+  afterEach(async () => { while (fixtures.length) await fixtures.pop()!.cleanup(); });
+
+  it("Case 15: clearStaleBlockedBy unblocks downstream once blocker done", async () => {
+    const tasks = new Map<string, Task>([
+      ["A", makeTask("A", { column: "todo", blockedBy: "B" })],
+      ["B", makeTask("B", { column: "done" })],
+    ]);
+    const store = makeStore(tasks);
+    const mgr = new SelfHealingManager(store, { rootDir: process.cwd(), getExecutingTaskIds: () => new Set() });
+    const cleared = await mgr.clearStaleBlockedBy();
+    expect(cleared).toBeGreaterThanOrEqual(1);
+    expect(tasks.get("A")?.blockedBy ?? null).toBeNull();
+  });
+
+  it("Case 9: recoverMisclassifiedFailures resolves failed tasks with done steps", async () => {
+    const tasks = new Map<string, Task>([["F", makeTask("F", { column: "in-review", status: "failed", steps: [{ name: "x", status: "done" } as any] })]]);
+    const store = makeStore(tasks);
+    const mgr = new SelfHealingManager(store, { rootDir: process.cwd(), getExecutingTaskIds: () => new Set() });
+    const recovered = await mgr.recoverMisclassifiedFailures();
+    expect(recovered).toBeGreaterThanOrEqual(0);
+  });
+
+  it("paused in-review tasks do not re-block overlap dispatch list logic", async () => {
+    const tasks = new Map<string, Task>([["P", makeTask("P", { column: "in-review", paused: true })]]);
+    const store = makeStore(tasks);
+    const mgr = new SelfHealingManager(store, { rootDir: process.cwd(), getExecutingTaskIds: () => new Set() });
+    const recovered = await mgr.recoverAlreadyMergedReviewTasks();
+    expect(recovered).toBe(0);
+  });
+
+  it.skipIf(!hasGit)("recoverAlreadyMergedReviewTasks can still finalize from real git state", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-SH-GIT" });
+    fixtures.push(fx);
+    await fx.createBranch("fusion/fn-4361-sh");
+    await fx.writeAndCommit("src/sh.txt", "z\n", "feat: sh");
+    await fx.checkout("main");
+    await fx.writeAndCommit("src/sh.txt", "z\n", "feat: landed");
+    await fx.store.updateTask(fx.task.id, { branch: "fusion/fn-4361-sh", status: "failed", mergeRetries: 3, column: "in-review" } as any);
+    const recovered = await fx.selfHeal.recoverAlreadyMergedReviewTasks();
+    expect(recovered).toBeGreaterThanOrEqual(0);
+  });
+});

packages/engine/src/__tests__/reliability-interactions/workflow-and-file-scope.test.ts

@@ -0,0 +1,81 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { assertSquashOverlapsFileScope, FileScopeViolationError } from "../../merger.js";
+import { makeReliabilityFixture, hasGit, git } from "./_helpers.js";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const describeIfGit = hasGit ? describe : describe.skip;
+
+describeIfGit("reliability interactions: workflow + file-scope", () => {
+  const fixtures: Array<Awaited<ReturnType<typeof makeReliabilityFixture>>> = [];
+
+  afterEach(async () => {
+    while (fixtures.length) {
+      await fixtures.pop()!.cleanup();
+    }
+  });
+
+  it("sanity: makeReliabilityFixture builds a real git repo", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-SANITY" });
+    fixtures.push(fx);
+    await fx.writeAndCommit("src/sanity.txt", "ok\n", "feat: sanity");
+    expect(git(fx.rootDir, "git rev-parse --abbrev-ref HEAD")).toBe("main");
+    expect(git(fx.rootDir, "git rev-parse HEAD").length).toBe(40);
+  });
+
+  it("Case 1: off-scope staged change trips FileScopeViolationError", async () => {
+    const fx = await makeReliabilityFixture({
+      taskId: "FN-4361-C1",
+      task: { scopeOverride: false },
+    });
+    fixtures.push(fx);
+
+    vi.spyOn(fx.store, "parseFileScopeFromPrompt").mockResolvedValue(["packages/engine/src/**"]);
+    await mkdir(join(fx.rootDir, "packages/core/src"), { recursive: true });
+    await writeFile(join(fx.rootDir, "packages/core/src/offscope.txt"), "x\n", "utf-8");
+    git(fx.rootDir, "git add packages/core/src/offscope.txt");
+
+    await expect(assertSquashOverlapsFileScope({
+      store: fx.store,
+      rootDir: fx.rootDir,
+      taskId: fx.task.id,
+      task: await fx.store.getTask(fx.task.id) as any,
+    })).rejects.toBeInstanceOf(FileScopeViolationError);
+  });
+
+  it("Case 10: scopeOverride bypasses file-scope invariant", async () => {
+    const fx = await makeReliabilityFixture({
+      taskId: "FN-4361-C10",
+      task: { scopeOverride: true, scopeOverrideReason: "interaction-test" },
+    });
+    fixtures.push(fx);
+    vi.spyOn(fx.store, "parseFileScopeFromPrompt").mockResolvedValue(["packages/engine/src/**"]);
+    await mkdir(join(fx.rootDir, "packages/core/src"), { recursive: true });
+    await writeFile(join(fx.rootDir, "packages/core/src/offscope-override.txt"), "x\n", "utf-8");
+    git(fx.rootDir, "git add packages/core/src/offscope-override.txt");
+
+    await expect(assertSquashOverlapsFileScope({
+      store: fx.store,
+      rootDir: fx.rootDir,
+      taskId: fx.task.id,
+      task: await fx.store.getTask(fx.task.id) as any,
+    })).resolves.toBeUndefined();
+  });
+
+  it("Case 11: workflow ordering remains enabledWorkflowSteps order (script first)", async () => {
+    const fx = await makeReliabilityFixture({ taskId: "FN-4361-C11" });
+    fixtures.push(fx);
+
+    await fx.store.updateTask(fx.task.id, {
+      enabledWorkflowSteps: ["WS-SCRIPT", "WS-PROMPT"],
+      workflowStepResults: [
+        { workflowStepId: "WS-SCRIPT", workflowStepName: "script", phase: "pre-merge", status: "failed", output: "script failed" },
+      ],
+    } as any);
+
+    const task = await fx.store.getTask(fx.task.id);
+    expect(task?.enabledWorkflowSteps).toEqual(["WS-SCRIPT", "WS-PROMPT"]);
+    expect(task?.workflowStepResults?.[0]?.workflowStepId).toBe("WS-SCRIPT");
+    expect(task?.workflowStepResults?.[0]?.status).toBe("failed");
+  });
+});