feat(FN-4559): complete remaining recovery, tests, and docs

Fusion-Task-Id: FN-4559 Fusion-Task-Lineage: 9276f683-657c-4604-aaef-143da488b287
2026-05-15 00:35:19 -07:00
parent 941213ec14
commit 38267f479f
7 changed files with 283 additions and 3 deletions
--- a/packages/engine/src/tests/reliability-interactions/verification-fix-already-on-main.real-git.test.ts
+++ b/packages/engine/src/tests/reliability-interactions/verification-fix-already-on-main.real-git.test.ts
@@ -0,0 +1,95 @@
+import { describe, it, expect } from "vitest";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { execSync } from "node:child_process";
+import { EventEmitter } from "node:events";
+import type { Settings, Task, TaskStore } from "@fusion/core";
+import { commitOrAmendMergeWithFixes } from "../../merger.js";
+import { SelfHealingManager } from "../../self-healing.js";
+
+function git(dir: string, cmd: string): string {
+  return execSync(cmd, { cwd: dir, stdio: "pipe" }).toString().trim();
+}
+
+function makeStore(task: Task, settings: Partial<Settings> = {}): TaskStore & EventEmitter {
+  const emitter = new EventEmitter();
+  const allSettings = { globalPause: false, enginePaused: false, ...settings } as Settings;
+  return Object.assign(emitter, {
+    getSettings: async () => allSettings,
+    listTasks: async ({ column }: { column?: string } = {}) => (column ? [task].filter((t) => t.column === column) : [task]),
+    updateTask: async (_id: string, updates: Partial<Task>) => Object.assign(task, updates),
+    moveTask: async (_id: string, column: Task["column"]) => { task.column = column; },
+    logEntry: async () => undefined,
+    getTask: async () => task,
+    walCheckpoint: () => ({ busy: 0, log: 0, checkpointed: 0 }),
+    archiveTaskAndCleanup: async () => ({}),
+    clearStaleExecutionStartBranchReferences: () => [],
+    updateSettings: async () => allSettings,
+    mergeTask: async () => undefined,
+    getRootDir: () => "",
+    recordRunAuditEvent: async () => undefined,
+  }) as unknown as TaskStore & EventEmitter;
+}
+
+describe("verification-fix already-on-main reliability interactions (real git)", () => {
+  it("recovers no-content finalize and allows self-healing done transition", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "fn-4559-ri-"));
+    try {
+      git(dir, "git init -b main");
+      git(dir, 'git config user.email "test@example.com"');
+      git(dir, 'git config user.name "Test"');
+      git(dir, "git commit --allow-empty -m init");
+
+      git(dir, "git commit --allow-empty -m 'feat(FN-4545): unrelated'");
+      const unrelatedSha = git(dir, "git rev-parse HEAD");
+      writeFileSync(join(dir, "file.txt"), "task\n");
+      git(dir, "git add file.txt");
+      git(dir, "git commit -m 'feat(FN-4553): landed' -m 'Fusion-Task-Id: FN-4553'");
+      const landedSha = git(dir, "git rev-parse HEAD");
+      git(dir, "git commit --allow-empty -m 'chore: post'");
+      const preAttemptHeadSha = git(dir, "git rev-parse HEAD");
+
+      git(dir, `git branch fusion/fn-4553 ${unrelatedSha}`);
+
+      const finalized = await commitOrAmendMergeWithFixes(
+        dir,
+        "FN-4553",
+        "fusion/fn-4553",
+        "feat(FN-4553): merge",
+        true,
+        preAttemptHeadSha,
+        "",
+      );
+      expect(finalized.ok && finalized.reason === "branch-already-merged-on-main").toBe(true);
+      if (finalized.ok && finalized.reason === "branch-already-merged-on-main") {
+        expect(finalized.mergeSha).toBe(landedSha);
+      }
+
+      const task = {
+        id: "FN-4553",
+        title: "t",
+        description: "d",
+        column: "in-review",
+        dependencies: [],
+        steps: [],
+        currentStep: 0,
+        log: [],
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+        branch: "fusion/fn-4553",
+        baseBranch: "main",
+      } as Task;
+      const store = makeStore(task);
+      const manager = new SelfHealingManager(store, { rootDir: dir, getExecutingTaskIds: () => new Set() });
+      await (manager as any).recoverBranchMisboundInReviewTasks();
+      expect(task.column).toBe("done");
+
+      const pausedStore = makeStore({ ...task, id: "FN-4553B", column: "in-review", branch: "fusion/fn-4553" } as Task, { globalPause: true });
+      const pausedMgr = new SelfHealingManager(pausedStore, { rootDir: dir, getExecutingTaskIds: () => new Set() });
+      await expect((pausedMgr as any).recoverBranchMisboundInReviewTasks()).resolves.toBe(0);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});
--- a/packages/engine/src/tests/self-healing-already-merged.real-git.test.ts
+++ b/packages/engine/src/tests/self-healing-already-merged.real-git.test.ts
@@ -235,6 +235,40 @@ describeIfGit("SelfHealingManager recoverAlreadyMergedReviewTasks (real git)", (
    expect(existsSync(worktreePath)).toBe(true);
  });

+  it("recovers misbound in-review branch when main already carries task trailer", async () => {
+    const repo = setupRepo();
+    mkdirSync(path.join(repo, "src"), { recursive: true });
+    writeFileSync(path.join(repo, "src", "other.txt"), "other\n", "utf-8");
+    git(repo, "git add src/other.txt && git commit -m 'feat: unrelated' -m 'Fusion-Task-Id: FN-OTHER'");
+    const unrelatedSha = git(repo, "git rev-parse HEAD");
+
+    writeFileSync(path.join(repo, "src", "misbound.txt"), "landed\n", "utf-8");
+    git(repo, "git add src/misbound.txt && git commit -m 'feat: landed' -m 'Fusion-Task-Id: FN-TEST-MISBOUND'");
+    const landedSha = git(repo, "git rev-parse HEAD");
+
+    const worktreePath = path.join(repo, ".worktrees", "fn-test-misbound");
+    mkdirSync(path.dirname(worktreePath), { recursive: true });
+    git(repo, `git branch fusion/fn-test-misbound ${unrelatedSha}`);
+    git(repo, `git worktree add ${JSON.stringify(worktreePath)} fusion/fn-test-misbound`);
+
+    const tasks: TaskMap = new Map([
+      ["FN-TEST-MISBOUND", makeTask({ id: "FN-TEST-MISBOUND", column: "in-review", paused: false, baseBranch: "main", branch: "fusion/fn-test-misbound", worktree: worktreePath })],
+    ]);
+    const store = createStore(tasks);
+    const manager = new SelfHealingManager(store, { rootDir: repo, getExecutingTaskIds: () => new Set() });
+
+    await (manager as any).runMaintenance();
+
+    const task = tasks.get("FN-TEST-MISBOUND")!;
+    expect(task.column).toBe("done");
+    expect(task.branch).toBeNull();
+    expect(task.worktree).toBeNull();
+    expect(task.mergeDetails?.commitSha).toBe(landedSha);
+    expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ mutationType: "task:auto-recover-branch-misbound", target: "FN-TEST-MISBOUND" }),
+    );
+  });
+
  it("is idempotent across two maintenance passes", async () => {
    const repo = setupRepo();
    mkdirSync(path.join(repo, "src"), { recursive: true });
--- a/packages/engine/src/merger.ts
+++ b/packages/engine/src/merger.ts
@@ -3494,7 +3494,7 @@ export async function commitOrAmendMergeWithFixes(
            );
            await auditor?.database({
              type: "task:auto-recover-finalize-already-on-main",
-              taskId,
+              target: taskId,
              metadata: {
                mergeSha: landed.sha,
                mergeStrategy: landed.strategy,
--- a/packages/engine/src/run-audit.ts
+++ b/packages/engine/src/run-audit.ts
@@ -100,6 +100,8 @@ export type DatabaseMutationType =
  | "task:unpause"
  | "task:dependency:add"
  | "task:auto-recover-already-merged"
+  | "task:auto-recover-finalize-already-on-main"
+  | "task:auto-recover-branch-misbound"
  | "task:auto-recover-completion-fanout"
  | "auto-recovery:classify-decision"
  | "auto-recovery:retry-issued"
--- a/packages/engine/src/self-healing.ts
+++ b/packages/engine/src/self-healing.ts
@@ -409,6 +409,7 @@ export class SelfHealingManager {
      { name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) },
      { name: "done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata().then(() => undefined) },
      { name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) },
+      { name: "recover-branch-misbound-in-review", fn: () => this.recoverBranchMisboundInReviewTasks().then(() => undefined) },
      { name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations().then(() => undefined) },
      { name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) },
      { name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
@@ -830,6 +831,17 @@ export class SelfHealingManager {
    return commit;
  }

+  private async findAlreadyMergedTaskCommit(input: {
+    taskId: string;
+    lineageId?: string;
+    repoDir: string;
+    baseBranch: string;
+    taskBranch?: string;
+    baseCommitSha?: string;
+  }) {
+    return findAlreadyMergedTaskCommit(input);
+  }
+
  private async cleanupWorktreeOnly(task: Task): Promise<void> {
    if (task.worktree && existsSync(task.worktree)) {
      try {
@@ -925,6 +937,7 @@ export class SelfHealingManager {
          { name: "recover-mergeable-review", fn: () => this.recoverMergeableReviewTasks() },
          { name: "recover-merged-review", fn: () => this.recoverMergedReviewTasks() },
          { name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() },
+          { name: "recover-branch-misbound-in-review", fn: () => this.recoverBranchMisboundInReviewTasks() },
          { name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations() },
          { name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() },
          { name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
@@ -3063,7 +3076,7 @@ export class SelfHealingManager {
          if (hasDeclaredOverlap) continue;

          const baseBranch = task.baseBranch || task.executionStartBranch || "main";
-          const landed = await findAlreadyMergedTaskCommit({
+          const landed = await this.findAlreadyMergedTaskCommit({
            taskId: task.id,
            lineageId: task.lineageId,
            repoDir: this.options.rootDir,
@@ -3173,7 +3186,7 @@ export class SelfHealingManager {
          const baseBranch = task.baseBranch || task.executionStartBranch || "main";
          if (!baseBranch) continue;

-          const landed = await findAlreadyMergedTaskCommit({
+          const landed = await this.findAlreadyMergedTaskCommit({
            taskId: task.id,
            lineageId: task.lineageId,
            repoDir: this.options.rootDir,
@@ -3263,6 +3276,135 @@ export class SelfHealingManager {
    }
  }

+  private async isBranchTipMisboundToTask(input: {
+    branch: string;
+    taskId: string;
+    lineageId?: string;
+    baseBranch: string;
+  }): Promise<{ misbound: boolean; branchTip: string; landed: Awaited<ReturnType<typeof findAlreadyMergedTaskCommit>> }> {
+    const { branch, taskId, lineageId, baseBranch } = input;
+    const { stdout: bodyOut } = await execAsync(`git log -1 --format=%B ${shellQuote(branch)}`, {
+      cwd: this.options.rootDir,
+      timeout: 30_000,
+      maxBuffer: 1024 * 1024,
+    });
+    const body = bodyOut;
+    const hasTaskId = body.includes(`Fusion-Task-Id: ${taskId}`);
+    const hasLineage = lineageId ? body.includes(`Fusion-Task-Lineage: ${lineageId}`) : false;
+    const { stdout: tipOut } = await execAsync(`git rev-parse ${shellQuote(branch)}`, {
+      cwd: this.options.rootDir,
+      timeout: 30_000,
+      maxBuffer: 1024 * 1024,
+    });
+    const branchTip = tipOut.trim();
+    const landed = await this.findAlreadyMergedTaskCommit({
+      taskId,
+      lineageId,
+      repoDir: this.options.rootDir,
+      baseBranch,
+      taskBranch: branch,
+    });
+    return { misbound: !hasTaskId && !hasLineage, branchTip, landed };
+  }
+
+  async recoverBranchMisboundInReviewTasks(): Promise<number> {
+    try {
+      const settings = await this.store.getSettings();
+      if (settings.globalPause || settings.enginePaused) return 0;
+
+      const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
+      const tasks = await this.store.listTasks({ column: "in-review", slim: true });
+      const candidates = tasks.filter((task) =>
+        task.column === "in-review" &&
+        Boolean(task.branch) &&
+        task.mergeDetails?.mergeConfirmed !== true &&
+        !executingIds.has(task.id),
+      );
+
+      let recovered = 0;
+      for (const task of candidates) {
+        try {
+          const branch = task.branch;
+          if (!branch) continue;
+          const baseBranch = task.baseBranch || task.executionStartBranch || "main";
+          const check = await this.isBranchTipMisboundToTask({
+            branch,
+            taskId: task.id,
+            lineageId: task.lineageId,
+            baseBranch,
+          });
+          if (!check.misbound || !check.landed) continue;
+
+          const mergeDetails: MergeDetails = {
+            commitSha: check.landed.sha,
+            mergedAt: new Date().toISOString(),
+            mergeConfirmed: true,
+            prNumber: task.prInfo?.number,
+          };
+
+          await this.store.updateTask(task.id, {
+            mergeDetails,
+            branch: null,
+            worktree: null,
+            status: null,
+            error: null,
+          });
+
+          if (task.worktree && existsSync(task.worktree)) {
+            await execAsync(`git worktree remove --force ${shellQuote(task.worktree)}`, {
+              cwd: this.options.rootDir,
+              timeout: 30_000,
+            }).catch(() => undefined);
+          }
+
+          await this.clearCompletionBranchIfSubsumed(task, branch).catch(() => false);
+
+          await this.store.moveTask(task.id, "done");
+          await this.store.logEntry(
+            task.id,
+            `Auto-recovered: branch tip misbound but content found on ${baseBranch} at ${check.landed.sha.slice(0, 8)} via ${check.landed.strategy}`,
+          );
+          await this.reconcileCompletedTask(task.id, { worktreeHint: task.worktree ?? undefined });
+
+          try {
+            const auditor = createRunAuditor(this.store, {
+              runId: generateSyntheticRunId("self-heal", task.id),
+              agentId: "self-healing",
+              taskId: task.id,
+              taskLineageId: task.lineageId,
+              phase: "recover-branch-misbound-in-review",
+            });
+            await auditor.database({
+              type: "task:auto-recover-branch-misbound",
+              target: task.id,
+              metadata: {
+                branch,
+                branchTip: check.branchTip,
+                mergeSha: check.landed.sha,
+                mergeStrategy: check.landed.strategy,
+                lineageId: task.lineageId,
+                baseBranch,
+              },
+            });
+          } catch (err: unknown) {
+            const errorMessage = err instanceof Error ? err.message : String(err);
+            log.warn(`recoverBranchMisboundInReviewTasks: failed to record run-audit event for ${task.id}: ${errorMessage}`);
+          }
+          recovered++;
+        } catch (err: unknown) {
+          const errorMessage = err instanceof Error ? err.message : String(err);
+          log.warn(`recoverBranchMisboundInReviewTasks: failed for task ${task.id}: ${errorMessage}`);
+        }
+      }
+
+      return recovered;
+    } catch (err: unknown) {
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      log.error(`Branch-misbound in-review recovery failed: ${errorMessage}`);
+      return 0;
+    }
+  }
+
  /**
   * Recover tasks in `in-review` marked as `failed` where all steps are
   * actually done. This catches the case where an agent completed all work