feat(FN-4536): complete auto-recovery branch/worktree handler

Fusion-Task-Id: FN-4536
Fusion-Task-Lineage: 8d5615e4-e75d-413d-b3ab-0c4874b21fc5
This commit is contained in:
Fusion
2026-05-14 19:03:14 -07:00
committed by gsxdsm
parent 180c65ec70
commit 5e1a5a55bb
7 changed files with 465 additions and 12 deletions

View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": patch
---
Add branch/worktree auto-recovery handler that resolves FN-4519-class incidents (ghost worktrees, branch misbinding, stale branch-conflict-unrecoverable parking) by re-running deterministic classification (FN-4499 bootstrap re-anchor, FN-4500 zero-unique-commit reclaim via inspectBranchConflict kinds `stale-resolved` / `fully-subsumed`, FN-4499 `reclaimable`-with-zero-own-commits re-anchor) against live evidence and requeueing through the FN-4534 dispatcher. Adds run-audit events branch-worktree:auto-requeue, branch-worktree:ai-session-spawned, branch-worktree:irreducible-pause. Genuine live-foreign (FN-3936-class) cases continue to pause; userPaused (FN-4429) is preserved; autoRecovery.mode === "off" behavior is byte-identical to legacy parking.

View File

@@ -0,0 +1,109 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { AutoRecoveryContext, AutoRecoveryDecision, AutoRecoveryFailure } from "../auto-recovery.js";
import { BranchWorktreeAutoRecoveryHandler } from "../auto-recovery-handlers/branch-worktree.js";
const branchConflictMocks = vi.hoisted(() => ({
inspectBranchConflict: vi.fn(),
classifyBootstrapMisbinding: vi.fn(),
reanchorBranchToBase: vi.fn(),
}));
vi.mock("../branch-conflicts.js", () => ({
inspectBranchConflict: branchConflictMocks.inspectBranchConflict,
classifyBootstrapMisbinding: branchConflictMocks.classifyBootstrapMisbinding,
reanchorBranchToBase: branchConflictMocks.reanchorBranchToBase,
}));
function createTask(overrides: Record<string, unknown> = {}) {
return {
id: "FN-4536",
column: "in-progress",
branch: "fusion/fn-4536",
worktree: "/tmp/wt",
baseCommitSha: "main",
pausedReason: null,
userPaused: false,
...overrides,
} as any;
}
function createFixtures(taskOverrides: Record<string, unknown> = {}, mode = "programmatic") {
const task = createTask(taskOverrides);
const taskStore = {
updateTask: vi.fn(async () => undefined),
moveTask: vi.fn(async () => undefined),
} as any;
const runAudit = { database: vi.fn(async () => undefined), git: vi.fn(), filesystem: vi.fn() } as any;
const logger = { warn: vi.fn(), log: vi.fn(), error: vi.fn() } as any;
const spawnAiRecoverySession = vi.fn(async () => ({ outcome: "exhausted" as const }));
const handler = new BranchWorktreeAutoRecoveryHandler({ taskStore, runAudit, logger, spawnAiRecoverySession });
const failure: AutoRecoveryFailure = { class: "branch-conflict-unrecoverable", taskId: task.id, pausedReason: "branch-conflict-unrecoverable", evidence: {} };
const decision: AutoRecoveryDecision = { action: "retry", rationale: "mode", legacyPausedReason: "branch-conflict-unrecoverable", auditMetadata: { mode } };
const ctx: AutoRecoveryContext = { task, retryCount: 0, settings: { mode: "programmatic", maxRetries: 3 } as any };
return { taskStore, runAudit, logger, spawnAiRecoverySession, handler, failure, decision, ctx };
}
describe("BranchWorktreeAutoRecoveryHandler", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("requeues on fully-subsumed", async () => {
const f = createFixtures();
branchConflictMocks.inspectBranchConflict.mockResolvedValue({ kind: "fully-subsumed", livePath: "/tmp/wt", tipSha: "abc" });
await f.handler.issueRetry(f.failure, f.decision, f.ctx);
expect(f.taskStore.updateTask).toHaveBeenCalledWith("FN-4536", { branch: null, baseCommitSha: null });
expect(f.taskStore.moveTask).toHaveBeenCalledWith("FN-4536", "todo", expect.objectContaining({ moveSource: "engine", preserveWorktree: false }));
expect(f.runAudit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:auto-requeue" }));
});
it("reanchors bootstrap misbinding then requeues", async () => {
const f = createFixtures();
branchConflictMocks.inspectBranchConflict.mockResolvedValue({ kind: "reclaimable", livePath: "/tmp/wt", tipSha: "abc", taskAttributedCommitCount: 0, strandedCommits: [] });
branchConflictMocks.classifyBootstrapMisbinding.mockResolvedValue({ isBootstrapMisbinding: true, ownCommitCount: 0, nonAttributedCount: 0 });
branchConflictMocks.reanchorBranchToBase.mockResolvedValue({});
await f.handler.issueRetry(f.failure, f.decision, f.ctx);
expect(branchConflictMocks.reanchorBranchToBase).toHaveBeenCalledTimes(1);
expect(f.taskStore.moveTask).toHaveBeenCalledWith("FN-4536", "todo", expect.objectContaining({ moveSource: "engine" }));
expect(f.runAudit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:auto-requeue", metadata: expect.objectContaining({ rationale: "bootstrap-misbinding-reanchor" }) }));
});
it("unparks stale paused conflict", async () => {
const f = createFixtures({ paused: true, pausedReason: "branch-conflict-unrecoverable" });
branchConflictMocks.inspectBranchConflict.mockResolvedValue({ kind: "stale-resolved" });
await f.handler.issueRetry(f.failure, f.decision, f.ctx);
expect(f.taskStore.moveTask).toHaveBeenCalled();
expect(f.runAudit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:auto-requeue", metadata: expect.objectContaining({ prevPausedReason: "branch-conflict-unrecoverable" }) }));
});
it("live-foreign emits irreducible pause without mutation", async () => {
const f = createFixtures();
branchConflictMocks.inspectBranchConflict.mockResolvedValue({ kind: "live-foreign", livePath: "/tmp/wt", error: new Error("foreign") });
await f.handler.issueRetry(f.failure, f.decision, f.ctx);
expect(f.taskStore.moveTask).not.toHaveBeenCalled();
expect(f.runAudit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:irreducible-pause", metadata: expect.objectContaining({ reason: "live-foreign" }) }));
});
it("ai-assisted exhaustion logs spawned and irreducible", async () => {
const f = createFixtures({}, "ai-assisted");
await f.handler.spawnAiRecovery(f.failure, { ...f.decision, auditMetadata: { mode: "ai-assisted" } }, f.ctx);
expect(f.spawnAiRecoverySession).toHaveBeenCalledTimes(1);
expect(f.runAudit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:ai-session-spawned", metadata: expect.objectContaining({ outcome: "exhausted" }) }));
expect(f.runAudit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:irreducible-pause", metadata: expect.objectContaining({ reason: "ai-session-unresolved" }) }));
});
it("mode off is no-op", async () => {
const f = createFixtures({}, "off");
await f.handler.issueRetry(f.failure, { ...f.decision, auditMetadata: { mode: "off" } }, f.ctx);
expect(f.taskStore.moveTask).not.toHaveBeenCalled();
expect(f.runAudit.database).not.toHaveBeenCalled();
});
it("userPaused skips", async () => {
const f = createFixtures({ userPaused: true, pausedReason: "branch-conflict-unrecoverable", paused: true });
await f.handler.issueRetry(f.failure, f.decision, f.ctx);
expect(f.taskStore.moveTask).not.toHaveBeenCalled();
expect(f.runAudit.database).not.toHaveBeenCalled();
expect(f.logger.warn).toHaveBeenCalledWith(expect.stringContaining("skipped (userPaused)"));
});
});

View File

@@ -0,0 +1,82 @@
import { describe, expect, it, vi } from "vitest";
import { AutoRecoveryDispatcher } from "../../auto-recovery.js";
import { BranchWorktreeAutoRecoveryHandler } from "../../auto-recovery-handlers/branch-worktree.js";
const branchConflictMocks = vi.hoisted(() => ({
inspectBranchConflict: vi.fn(),
classifyBootstrapMisbinding: vi.fn(),
reanchorBranchToBase: vi.fn(),
}));
vi.mock("../../branch-conflicts.js", () => ({
inspectBranchConflict: branchConflictMocks.inspectBranchConflict,
classifyBootstrapMisbinding: branchConflictMocks.classifyBootstrapMisbinding,
reanchorBranchToBase: branchConflictMocks.reanchorBranchToBase,
}));
function task(overrides: Record<string, unknown> = {}) {
return {
id: "FN-4519",
lineageId: "L1",
column: "in-progress",
branch: "fusion/fn-4519",
worktree: "/tmp/wt",
baseCommitSha: "main",
paused: true,
pausedReason: "branch-conflict-unrecoverable",
userPaused: false,
recoveryRetryCount: 0,
...overrides,
} as any;
}
describe("reliability interaction: branch/worktree auto-recovery", () => {
it("dispatcher delegates to handler for FN-4519 path and requeues", async () => {
const t = task();
const taskStore = {
updateTask: vi.fn(async () => undefined),
moveTask: vi.fn(async () => undefined),
} as any;
const audit = { database: vi.fn(async () => undefined), git: vi.fn(), filesystem: vi.fn() } as any;
const handler = new BranchWorktreeAutoRecoveryHandler({ taskStore, runAudit: audit });
const dispatcher = new AutoRecoveryDispatcher({
taskStore,
auditEmitter: audit,
handlers: { issueRetry: (f, d, c) => handler.issueRetry(f, d, c) },
});
branchConflictMocks.inspectBranchConflict.mockResolvedValue({ kind: "fully-subsumed", livePath: "/tmp/wt", tipSha: "abc" });
const decision = await dispatcher.dispatch({
class: "branch-conflict-unrecoverable",
taskId: t.id,
pausedReason: "branch-conflict-unrecoverable",
evidence: { branchName: t.branch, conflictingWorktreePath: t.worktree },
}, {
task: t,
retryCount: 0,
settings: { mode: "programmatic", maxRetries: 3 },
});
expect(decision.action).toBe("retry");
expect(taskStore.updateTask).toHaveBeenCalledWith(t.id, { branch: null, baseCommitSha: null });
expect(taskStore.moveTask).toHaveBeenCalledWith(t.id, "todo", expect.objectContaining({ moveSource: "engine" }));
expect(audit.database).toHaveBeenCalledWith(expect.objectContaining({ type: "branch-worktree:auto-requeue" }));
});
it("preserves userPaused contract", async () => {
const t = task({ userPaused: true });
const taskStore = { updateTask: vi.fn(async () => undefined), moveTask: vi.fn(async () => undefined) } as any;
const audit = { database: vi.fn(async () => undefined), git: vi.fn(), filesystem: vi.fn() } as any;
const handler = new BranchWorktreeAutoRecoveryHandler({ taskStore, runAudit: audit, logger: { warn: vi.fn(), log: vi.fn(), error: vi.fn() } as any });
await handler.issueRetry(
{ class: "branch-conflict-unrecoverable", taskId: t.id, pausedReason: "branch-conflict-unrecoverable", evidence: { branchName: t.branch, conflictingWorktreePath: t.worktree } },
{ action: "retry", rationale: "mode-programmatic", legacyPausedReason: "branch-conflict-unrecoverable", auditMetadata: { mode: "programmatic" } },
{ task: t, retryCount: 0, settings: { mode: "programmatic", maxRetries: 3 } as any },
);
expect(taskStore.moveTask).not.toHaveBeenCalled();
expect(audit.database).not.toHaveBeenCalled();
});
});

View File

@@ -1,11 +1,20 @@
import { exec } from "node:child_process";
import { existsSync } from "node:fs";
import { promisify } from "node:util";
import type { TaskStore } from "@fusion/core";
import type { Task, TaskStore } from "@fusion/core";
import {
classifyBootstrapMisbinding,
inspectBranchConflict,
reanchorBranchToBase,
} from "../branch-conflicts.js";
import type { AutoRecoveryContext, AutoRecoveryDecision, AutoRecoveryFailure } from "../auto-recovery.js";
import type { Logger } from "../logger.js";
import { createLogger, type Logger } from "../logger.js";
import type { RunAuditor } from "../run-audit.js";
const execAsync = promisify(exec);
const baseLog = createLogger("auto-recovery:branch-worktree");
const GIT_TIMEOUT_MS = 30_000;
const GIT_MAX_BUFFER = 10 * 1024 * 1024;
export interface BranchWorktreeRecoveryDeps {
taskStore: TaskStore;
@@ -19,17 +28,246 @@ export interface BranchWorktreeRecoveryDeps {
now?: () => Date;
}
interface RecoveryEvidence {
branchExists: boolean;
worktreePresent: boolean;
tipSha?: string;
inspectionKind?: string;
}
export class BranchWorktreeAutoRecoveryHandler {
constructor(private readonly deps: BranchWorktreeRecoveryDeps) {
void this.deps;
void execAsync;
constructor(private readonly deps: BranchWorktreeRecoveryDeps) {}
private get logger(): Logger {
return this.deps.logger ?? baseLog;
}
async issueRetry(_failure: AutoRecoveryFailure, _decision: AutoRecoveryDecision, _ctx: AutoRecoveryContext): Promise<void> {
// Implemented in FN-4536 Step 2.
private async runGit(repoDir: string, command: string): Promise<string> {
const { stdout } = await execAsync(command, {
cwd: repoDir,
timeout: GIT_TIMEOUT_MS,
maxBuffer: GIT_MAX_BUFFER,
encoding: "utf-8",
});
return stdout.trim();
}
async spawnAiRecovery(_failure: AutoRecoveryFailure, _decision: AutoRecoveryDecision, _ctx: AutoRecoveryContext): Promise<void> {
// Implemented in FN-4536 Step 4.
private quote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
private async hasBranchRef(repoDir: string, branchName: string): Promise<boolean> {
try {
await this.runGit(repoDir, `git rev-parse --verify ${this.quote(`refs/heads/${branchName}`)}`);
return true;
} catch {
return false;
}
}
private async getTipSha(repoDir: string, branchName: string): Promise<string | undefined> {
try {
return await this.runGit(repoDir, `git rev-parse --verify ${this.quote(`refs/heads/${branchName}`)}`);
} catch {
return undefined;
}
}
private async getWorktreeBranchMap(repoDir: string): Promise<Map<string, string>> {
const output = await this.runGit(repoDir, "git worktree list --porcelain").catch(() => "");
const map = new Map<string, string>();
let path: string | null = null;
for (const line of output.split("\n")) {
if (line.startsWith("worktree ")) path = line.slice("worktree ".length).trim();
if (line.startsWith("branch refs/heads/") && path) {
map.set(line.slice("branch refs/heads/".length).trim(), path);
}
if (!line.trim()) path = null;
}
return map;
}
private async resolveRepoDir(ctx: AutoRecoveryContext, failure: AutoRecoveryFailure): Promise<string> {
const repoFromFailure = typeof failure.evidence?.repoDir === "string" ? failure.evidence.repoDir : undefined;
if (repoFromFailure) return repoFromFailure;
if (ctx.task.worktree) {
const top = await this.runGit(ctx.task.worktree, "git rev-parse --show-toplevel").catch(() => "");
if (top) return top;
}
return process.cwd();
}
private async requeueAfterRecovery(task: Task, failure: AutoRecoveryFailure, rationale: string, evidence: RecoveryEvidence): Promise<void> {
if (task.userPaused) return;
if (task.column === "in-progress") {
await this.deps.taskStore.updateTask(task.id, { branch: null, baseCommitSha: null });
}
await this.deps.taskStore.moveTask(task.id, "todo", {
moveSource: "engine",
preserveResumeState: true,
preserveProgress: true,
preserveWorktree: false,
});
await this.deps.runAudit.database({
type: "branch-worktree:auto-requeue",
target: task.id,
metadata: {
class: failure.class,
rationale,
prevPausedReason: task.pausedReason ?? null,
evidence,
},
});
}
private async emitIrreduciblePause(task: Task, failure: AutoRecoveryFailure, reason: string, evidence: Record<string, unknown>): Promise<void> {
await this.deps.runAudit.database({
type: "branch-worktree:irreducible-pause",
target: task.id,
metadata: {
class: failure.class,
reason,
evidence,
},
});
}
async issueRetry(failure: AutoRecoveryFailure, decision: AutoRecoveryDecision, ctx: AutoRecoveryContext): Promise<void> {
if (ctx.task.userPaused) {
this.logger.warn(`auto-recovery: skipped (userPaused) class=${failure.class} task=${ctx.task.id}`);
return;
}
if (decision.auditMetadata.mode === "off") return;
const repoDir = await this.resolveRepoDir(ctx, failure);
const branchName = (ctx.task.branch ?? (typeof failure.evidence?.branchName === "string" ? failure.evidence.branchName : "")).trim();
const conflictingWorktreePath = (ctx.task.worktree ?? (typeof failure.evidence?.conflictingWorktreePath === "string"
? failure.evidence.conflictingWorktreePath
: typeof failure.evidence?.worktreePath === "string"
? failure.evidence.worktreePath
: "")).trim();
if (!branchName || !conflictingWorktreePath) return;
const inspection = await inspectBranchConflict({
repoDir,
branchName,
conflictingWorktreePath,
requestingTaskId: ctx.task.id,
ownerTaskId: ctx.task.id,
startPoint: ctx.task.baseCommitSha ?? "main",
});
if (inspection.kind === "stale-resolved" || inspection.kind === "fully-subsumed" || inspection.kind === "tip-already-merged") {
await this.requeueAfterRecovery(ctx.task, failure, inspection.kind, {
branchExists: await this.hasBranchRef(repoDir, branchName),
worktreePresent: existsSync(conflictingWorktreePath),
tipSha: "tipSha" in inspection ? inspection.tipSha : undefined,
inspectionKind: inspection.kind,
});
return;
}
if (inspection.kind === "stale") {
await this.runGit(repoDir, "git worktree prune").catch(() => undefined);
const [branchExists, map] = await Promise.all([
this.hasBranchRef(repoDir, branchName),
this.getWorktreeBranchMap(repoDir),
]);
if (!branchExists) {
await this.requeueAfterRecovery(ctx.task, failure, "stale-branch-deleted", {
branchExists,
worktreePresent: false,
inspectionKind: inspection.kind,
});
return;
}
if (!map.has(branchName)) {
await this.requeueAfterRecovery(ctx.task, failure, "stale-resolved-after-prune", {
branchExists,
worktreePresent: false,
inspectionKind: inspection.kind,
});
return;
}
}
if (inspection.kind === "reclaimable" && inspection.taskAttributedCommitCount === 0) {
const bootstrap = await classifyBootstrapMisbinding({
repoDir,
branchName,
baseSha: ctx.task.baseCommitSha ?? "main",
taskId: ctx.task.id,
foreignCommits: [],
}).catch(() => ({ isBootstrapMisbinding: false, ownCommitCount: 0, nonAttributedCount: 0 }));
if (bootstrap.isBootstrapMisbinding) {
const reanchor = await reanchorBranchToBase({
repoDir,
worktreePath: inspection.livePath,
branchName,
baseSha: ctx.task.baseCommitSha ?? "main",
taskId: ctx.task.id,
}).catch(() => null);
if (reanchor) {
await this.requeueAfterRecovery(ctx.task, failure, "bootstrap-misbinding-reanchor", {
branchExists: true,
worktreePresent: true,
tipSha: inspection.tipSha,
inspectionKind: inspection.kind,
});
return;
}
}
}
if (inspection.kind === "live-foreign") {
await this.emitIrreduciblePause(ctx.task, failure, "live-foreign", {
branchName,
conflictingWorktreePath,
inspectionKind: inspection.kind,
});
return;
}
const branchExists = await this.hasBranchRef(repoDir, branchName);
const tipSha = await this.getTipSha(repoDir, branchName);
await this.emitIrreduciblePause(ctx.task, failure, "deterministic-unresolved", {
branchName,
conflictingWorktreePath,
inspectionKind: inspection.kind,
branchExists,
worktreePresent: existsSync(conflictingWorktreePath),
tipSha,
});
}
async spawnAiRecovery(failure: AutoRecoveryFailure, decision: AutoRecoveryDecision, ctx: AutoRecoveryContext): Promise<void> {
if (!this.deps.spawnAiRecoverySession) return;
if (decision.auditMetadata.mode !== "ai-assisted") return;
const result = await this.deps.spawnAiRecoverySession(failure, decision, ctx);
await this.deps.runAudit.database({
type: "branch-worktree:ai-session-spawned",
target: ctx.task.id,
metadata: {
class: failure.class,
outcome: result.outcome,
evidence: {
...(failure.evidence ?? {}),
allowedTools: ["bash:git-log", "bash:git-rev-parse", "bash:git-diff", "fn_task_create"],
},
},
});
if (result.outcome === "resolved") {
await this.issueRetry(failure, decision, ctx);
return;
}
await this.emitIrreduciblePause(ctx.task, failure, "ai-session-unresolved", {
outcome: result.outcome,
...(result.metadata ?? {}),
});
}
}

View File

@@ -1,6 +1,6 @@
import { exec } from "node:child_process";
import { promisify } from "node:util";
import type { ProjectSettings, Task, TaskStore } from "@fusion/core";
import type { ProjectSettings, TaskStore } from "@fusion/core";
import type { AutoRecoveryContext, AutoRecoveryDecision, AutoRecoveryFailure } from "../auto-recovery.js";
import type { Logger } from "../logger.js";
import type { RunAuditor } from "../run-audit.js";

View File

@@ -72,6 +72,7 @@ import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import type { AgentReflectionService } from "./agent-reflection.js";
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
import { BranchWorktreeAutoRecoveryHandler } from "./auto-recovery-handlers/branch-worktree.js";
import { createFileScopeAutoRecoveryHandler } from "./auto-recovery-handlers/file-scope.js";
import { ReadonlyViolationError, filterCustomToolsForReadonly } from "./workflow-step-tool-policy.js";
import { evaluateSpecStaleness, getPromptPath } from "./spec-staleness.js";
@@ -842,12 +843,27 @@ export class TaskExecutor {
classifyPatchIds: async () => ({ unique: [], alreadyUpstream: [] }),
settings: () => ({ autoRecovery: { mode: "deterministic-only", maxRetries: 3 } } as ProjectSettings),
});
const branchWorktreeHandler = new BranchWorktreeAutoRecoveryHandler({
taskStore: this.store,
runAudit: audit,
logger: executorLog,
});
return new AutoRecoveryDispatcher({
taskStore: this.store,
auditEmitter: audit,
handlers: {
issueRetry: fileScopeHandler.issueRetry.bind(fileScopeHandler),
spawnAiRecovery: fileScopeHandler.spawnAiRecovery.bind(fileScopeHandler),
issueRetry: async (failure, decision, ctx) => {
if (failure.class === "branch-conflict-unrecoverable") {
return branchWorktreeHandler.issueRetry(failure, decision, ctx);
}
return fileScopeHandler.issueRetry(failure, decision, ctx);
},
spawnAiRecovery: async (failure, decision, ctx) => {
if (failure.class === "branch-conflict-unrecoverable") {
return branchWorktreeHandler.spawnAiRecovery(failure, decision, ctx);
}
return fileScopeHandler.spawnAiRecovery(failure, decision, ctx);
},
},
});
}

View File

@@ -104,6 +104,9 @@ export type DatabaseMutationType =
| "auto-recovery:retry-issued"
| "auto-recovery:ai-session-spawned"
| "auto-recovery:pause-because-destructive-ambiguity"
| "branch-worktree:auto-requeue"
| "branch-worktree:ai-session-spawned"
| "branch-worktree:irreducible-pause"
| "document:write"
| "workflow-step:result"
| "agent:create:requested"