feat(FN-4485): complete Step 4-5 orphan rescue and audit paths

Fusion-Task-Id: FN-4485
Fusion-Task-Lineage: 034088dc-ebc4-4e12-8314-39419d41b23f
This commit is contained in:
Fusion
2026-05-14 11:03:02 -07:00
committed by gsxdsm
parent bc6c1cb1a8
commit b72bd0575d
5 changed files with 375 additions and 71 deletions

View File

@@ -0,0 +1,63 @@
import { EventEmitter } from "node:events";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { TaskStore } from "@fusion/core";
import { SelfHealingManager } from "../../self-healing.js";
import * as branchConflicts from "../../branch-conflicts.js";
import * as worktreePool from "../../worktree-pool.js";
function createStore(): TaskStore & EventEmitter {
const emitter = new EventEmitter() as TaskStore & EventEmitter;
(emitter as any).getSettings = vi.fn().mockResolvedValue({ globalPause: false, enginePaused: false });
(emitter as any).listTasks = vi.fn();
(emitter as any).updateTask = vi.fn().mockResolvedValue(undefined);
(emitter as any).moveTask = vi.fn().mockResolvedValue(undefined);
(emitter as any).logEntry = vi.fn().mockResolvedValue(undefined);
(emitter as any).recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
(emitter as any).createTask = vi.fn();
(emitter as any).clearStaleExecutionStartBranchReferences = vi.fn().mockReturnValue([]);
return emitter;
}
describe("reliability interactions: branch recovery + orphan rescue", () => {
let store: TaskStore & EventEmitter;
let manager: SelfHealingManager;
beforeEach(() => {
store = createStore();
manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
vi.spyOn(worktreePool, "isUsableTaskWorktree").mockResolvedValue(true);
});
it("keeps userPaused tasks unswept even if reclaimable", async () => {
(store.listTasks as any)
.mockResolvedValueOnce([{ id: "FN-4429", column: "todo", checkedOutBy: null, branch: "fusion/fn-4429", worktree: "/tmp/fn-4429", paused: true, userPaused: true, pausedReason: "branch-conflict-unrecoverable" }])
.mockResolvedValueOnce([])
.mockResolvedValueOnce([]);
const inspectSpy = vi.spyOn(branchConflicts, "inspectBranchConflict");
const recovered = await manager.reclaimSelfOwnedBranchConflicts();
expect(recovered).toBe(0);
expect(inspectSpy).not.toHaveBeenCalled();
});
it("orphan-rescue sweep is idempotent across consecutive runs", async () => {
const branch = "fusion/fn-4470";
vi.spyOn(worktreePool, "scanOrphanedBranches")
.mockResolvedValueOnce([branch])
.mockResolvedValueOnce([branch]);
vi.spyOn(manager as any, "inspectOrphanedBranch")
.mockResolvedValueOnce({ branch, tipSha: "abc123", uniqueCommitCount: 2, uniqueCommitSubjects: ["feat: keep work"], derivedTaskId: "FN-4470", registeredWorktreePath: null })
.mockResolvedValueOnce({ branch, tipSha: "abc123", uniqueCommitCount: 2, uniqueCommitSubjects: ["feat: keep work"], derivedTaskId: "FN-4470", registeredWorktreePath: null });
(store.listTasks as any)
.mockResolvedValueOnce([])
.mockResolvedValueOnce([{ id: "FN-5001", column: "triage", branch }]);
(store.createTask as any).mockResolvedValueOnce({ id: "FN-5001", lineageId: "lin-5001" });
await manager.cleanupOrphanedBranches();
await manager.cleanupOrphanedBranches();
expect(store.createTask).toHaveBeenCalledTimes(1);
});
});

View File

@@ -0,0 +1,104 @@
import { EventEmitter } from "node:events";
import { beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("node:child_process", async () => {
const { promisify } = await import("node:util");
const execSyncFn = vi.fn(() => Buffer.from(""));
const execFn: any = vi.fn((cmd: string, opts: any, cb: any) => {
const callback = typeof opts === "function" ? opts : cb;
if (typeof callback === "function") callback(null, "", "");
});
execFn[promisify.custom] = () => Promise.resolve({ stdout: "", stderr: "" });
return { exec: execFn, execSync: execSyncFn };
});
import type { TaskStore } from "@fusion/core";
import { SelfHealingManager } from "../self-healing.js";
import * as worktreePool from "../worktree-pool.js";
function createStore(): TaskStore & EventEmitter {
const emitter = new EventEmitter() as TaskStore & EventEmitter;
(emitter as any).listTasks = vi.fn();
(emitter as any).createTask = vi.fn();
(emitter as any).updateTask = vi.fn().mockResolvedValue(undefined);
(emitter as any).logEntry = vi.fn().mockResolvedValue(undefined);
(emitter as any).clearStaleExecutionStartBranchReferences = vi.fn().mockReturnValue([]);
(emitter as any).recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
return emitter;
}
describe("self-healing orphan branch rescue", () => {
let store: TaskStore & EventEmitter;
let manager: SelfHealingManager;
beforeEach(() => {
store = createStore();
manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
});
it("prunes subsumed orphan branches and emits branch:orphan-prune", async () => {
vi.spyOn(worktreePool, "scanOrphanedBranches").mockResolvedValueOnce(["fusion/fn-4470"]);
vi.spyOn(manager as any, "inspectOrphanedBranch").mockResolvedValueOnce({
branch: "fusion/fn-4470",
tipSha: "abc123",
uniqueCommitCount: 0,
uniqueCommitSubjects: [],
derivedTaskId: "FN-4470",
registeredWorktreePath: null,
});
vi.spyOn(store, "listTasks" as any).mockResolvedValueOnce([]);
const result = await manager.cleanupOrphanedBranches();
expect(result).toBe(1);
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "branch:orphan-prune" }));
});
it("creates a rescue triage task when unique commits exist and no task row exists", async () => {
vi.spyOn(worktreePool, "scanOrphanedBranches").mockResolvedValueOnce(["fusion/fn-4470"]);
vi.spyOn(manager as any, "inspectOrphanedBranch").mockResolvedValueOnce({
branch: "fusion/fn-4470",
tipSha: "deadbeef",
uniqueCommitCount: 2,
uniqueCommitSubjects: ["feat: preserve orphan"],
derivedTaskId: "FN-4470",
registeredWorktreePath: "/tmp/wt-fn-4470",
});
vi.spyOn(store, "listTasks" as any).mockResolvedValueOnce([]);
(store.createTask as any).mockResolvedValueOnce({ id: "FN-5000", lineageId: "lin-5000" });
const result = await manager.cleanupOrphanedBranches();
expect(result).toBe(0);
expect(store.createTask).toHaveBeenCalledWith(expect.objectContaining({
title: "Recover orphaned branch fusion/fn-4470",
column: "triage",
branch: "fusion/fn-4470",
}));
expect(store.updateTask).toHaveBeenCalledWith("FN-5000", { worktree: "/tmp/wt-fn-4470" });
expect(store.logEntry).toHaveBeenCalledWith("FN-5000", expect.stringContaining("[recovery] orphan-rescue-created"));
expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "branch:orphan-rescued" }));
});
it("leaves archived matching tasks untouched and only acknowledges once", async () => {
const archivedTask = { id: "FN-4470", column: "archived", metadata: {} };
vi.spyOn(worktreePool, "scanOrphanedBranches").mockResolvedValueOnce(["fusion/fn-4470"]);
vi.spyOn(manager as any, "inspectOrphanedBranch").mockResolvedValueOnce({
branch: "fusion/fn-4470",
tipSha: "deadbeef",
uniqueCommitCount: 1,
uniqueCommitSubjects: ["feat: preserve orphan"],
derivedTaskId: "FN-4470",
registeredWorktreePath: null,
});
vi.spyOn(store, "listTasks" as any).mockResolvedValueOnce([archivedTask]);
const result = await manager.cleanupOrphanedBranches();
expect(result).toBe(0);
expect(store.updateTask).toHaveBeenCalledWith("FN-4470", expect.objectContaining({
metadata: expect.objectContaining({ orphanRescueAcknowledged: true }),
}));
expect(store.createTask).not.toHaveBeenCalled();
});
});

View File

@@ -1294,8 +1294,11 @@ describe("SelfHealingManager", () => {
expect(mockedExecSync).not.toHaveBeenCalled();
});
it("deletes orphaned branches with safe delete (-d)", async () => {
it("deletes only subsumed orphaned branches with safe delete (-d)", async () => {
mockedScanOrphanedBranches.mockResolvedValueOnce(["fusion/fn-001", "fusion/fn-002"]);
vi.spyOn(manager as any, "inspectOrphanedBranch")
.mockResolvedValueOnce({ branch: "fusion/fn-001", tipSha: "abc", uniqueCommitCount: 0, uniqueCommitSubjects: [], derivedTaskId: "FN-001", registeredWorktreePath: null })
.mockResolvedValueOnce({ branch: "fusion/fn-002", tipSha: "def", uniqueCommitCount: 0, uniqueCommitSubjects: [], derivedTaskId: "FN-002", registeredWorktreePath: null });
const result = await manager.cleanupOrphanedBranches();
@@ -1310,45 +1313,31 @@ describe("SelfHealingManager", () => {
);
});
it("falls back to force delete (-D) when safe delete fails", async () => {
it("does not force-delete unique-commit orphaned branches", async () => {
mockedScanOrphanedBranches.mockResolvedValueOnce(["fusion/fn-003"]);
// Safe delete fails
mockedExecSync.mockImplementationOnce(() => {
throw new Error("not fully merged");
});
// Force delete succeeds
mockedExecSync.mockImplementationOnce(() => Buffer.from(""));
vi.spyOn(manager as any, "inspectOrphanedBranch")
.mockResolvedValueOnce({ branch: "fusion/fn-003", tipSha: "abc", uniqueCommitCount: 2, uniqueCommitSubjects: ["feat: keep"], derivedTaskId: "FN-003", registeredWorktreePath: null });
const result = await manager.cleanupOrphanedBranches();
expect(result).toBe(1);
expect(mockedExecSync).toHaveBeenCalledWith(
expect.stringContaining('git branch -d "fusion/fn-003"'),
expect.any(Object),
);
expect(mockedExecSync).toHaveBeenCalledWith(
expect(result).toBe(0);
expect(mockedExecSync).not.toHaveBeenCalledWith(
expect.stringContaining('git branch -D "fusion/fn-003"'),
expect.any(Object),
);
});
it("counts only successfully deleted branches", async () => {
it("counts only successfully pruned subsumed branches", async () => {
mockedScanOrphanedBranches.mockResolvedValueOnce(["fusion/fn-004", "fusion/fn-005"]);
// First branch: safe delete succeeds
mockedExecSync.mockImplementationOnce(() => Buffer.from(""));
// Second branch: both safe and force delete fail
mockedExecSync.mockImplementationOnce(() => {
throw new Error("not fully merged");
});
mockedExecSync.mockImplementationOnce(() => {
throw new Error("branch not found");
});
vi.spyOn(manager as any, "inspectOrphanedBranch")
.mockResolvedValueOnce({ branch: "fusion/fn-004", tipSha: "abc", uniqueCommitCount: 0, uniqueCommitSubjects: [], derivedTaskId: "FN-004", registeredWorktreePath: null })
.mockResolvedValueOnce({ branch: "fusion/fn-005", tipSha: "def", uniqueCommitCount: 1, uniqueCommitSubjects: ["feat"], derivedTaskId: "FN-005", registeredWorktreePath: null });
mockedExecSync.mockReset();
mockedExecSync.mockImplementation(() => Buffer.from(""));
const result = await manager.cleanupOrphanedBranches();
expect(result).toBe(1);
expect(result).toBe(0);
});
it("returns 0 when scanOrphanedBranches throws", async () => {
@@ -2099,7 +2088,7 @@ describe("SelfHealingManager", () => {
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
taskStuckTimeoutMs: 60_000,
});
const staleUpdatedAt = new Date(Date.now() - 61_000).toISOString();
const staleUpdatedAt = new Date(Date.now() - 6 * 60_000).toISOString();
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{

View File

@@ -77,6 +77,8 @@ export type GitMutationType =
| "merge:file-scope-violation"
| "merge:audit-failure"
| "branch:auto-reclaim"
| "branch:orphan-prune"
| "branch:orphan-rescued"
| "stash:push"
| "stash:pop";

View File

@@ -221,6 +221,16 @@ const DURABLE_ERROR_RECOVERY_MAX_RETRIES = 5;
const DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS = 30_000;
const DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS = 15 * 60_000;
const RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS = 5 * 60_000;
const ORPHAN_RESCUE_SUBJECT_CAP = 10;
interface OrphanBranchInspection {
branch: string;
tipSha: string;
uniqueCommitCount: number;
uniqueCommitSubjects: string[];
derivedTaskId: string | null;
registeredWorktreePath: string | null;
}
interface LandedTaskCommit {
sha: string;
@@ -3818,18 +3828,84 @@ export class SelfHealingManager {
return cleaned;
}
private deriveTaskIdFromFusionBranch(branch: string): string | null {
const match = /^fusion\/(fn|kb)-(\d+)$/i.exec(branch.trim());
if (!match) return null;
return `${match[1].toUpperCase()}-${match[2]}`;
}
private async getRegisteredWorktreePathForBranch(branch: string): Promise<string | null> {
try {
const stdout = execSync("git worktree list --porcelain", {
cwd: this.options.rootDir,
encoding: "utf-8",
stdio: ["pipe", "pipe", "pipe"],
}) || "";
const lines = stdout.split("\n");
let currentPath: string | null = null;
for (const line of lines) {
if (line.startsWith("worktree ")) {
currentPath = line.slice("worktree ".length).trim();
continue;
}
if (line.startsWith("branch ")) {
const fullRef = line.slice("branch ".length).trim();
const branchName = fullRef.startsWith("refs/heads/") ? fullRef.slice("refs/heads/".length) : fullRef;
if (branchName === branch && currentPath) {
return currentPath;
}
}
}
} catch (err: unknown) {
log.warn(`Failed to inspect registered worktree for ${branch}: ${err instanceof Error ? err.message : String(err)}`);
}
return null;
}
private async inspectOrphanedBranch(branch: string): Promise<OrphanBranchInspection | null> {
try {
const tipSha = String(execSync(`git rev-parse --verify "${branch}"`, {
cwd: this.options.rootDir,
encoding: "utf-8",
stdio: ["pipe", "pipe", "pipe"],
})).trim();
if (!tipSha) return null;
const uniqueCommitCount = Number.parseInt(String(execSync(`git rev-list --count "${branch}" --not main`, {
cwd: this.options.rootDir,
encoding: "utf-8",
stdio: ["pipe", "pipe", "pipe"],
})).trim(), 10) || 0;
let uniqueCommitSubjects: string[] = [];
if (uniqueCommitCount > 0) {
const subjectOutput = String(execSync(`git log --format=%s --max-count=${ORPHAN_RESCUE_SUBJECT_CAP} "${branch}" --not main`, {
cwd: this.options.rootDir,
encoding: "utf-8",
stdio: ["pipe", "pipe", "pipe"],
}));
uniqueCommitSubjects = subjectOutput.split("\n").map((line) => line.trim()).filter(Boolean);
}
return {
branch,
tipSha,
uniqueCommitCount,
uniqueCommitSubjects,
derivedTaskId: this.deriveTaskIdFromFusionBranch(branch),
registeredWorktreePath: await this.getRegisteredWorktreePathForBranch(branch),
};
} catch (err: unknown) {
log.warn(`Failed to inspect orphaned branch ${branch}: ${err instanceof Error ? err.message : String(err)}`);
return null;
}
}
/**
* Remove orphaned `fusion/*` branches that are not associated with any
* active (non-archived, non-merger-managed) task.
*
* For each orphaned branch:
* 1. Try `git branch -d` (safe delete — only works if branch is fully merged)
* 2. Fall back to `git branch -D` (force delete) if safe delete fails
* 3. Log each cleanup action
*
* Individual branch deletion failures are non-fatal.
*
* @returns Number of branches successfully deleted
* Resolve orphaned `fusion/*` branches.
* - Subsumed branches are pruned (`git branch -d`).
* - Unique-commit branches with missing task rows are rescued into triage tasks.
* - Archived matching tasks are left untouched with one-time acknowledgement logging.
*/
async cleanupOrphanedBranches(): Promise<number> {
try {
@@ -3837,52 +3913,122 @@ export class SelfHealingManager {
if (orphaned.length === 0) return 0;
let cleaned = 0;
const deletedBranches: string[] = [];
const prunedBranches: string[] = [];
const allTasks = await this.store.listTasks({ slim: true, includeArchived: true });
const taskById = new Map(allTasks.map((task) => [task.id.toUpperCase(), task]));
for (const branch of orphaned) {
try {
// Try safe delete first (-d requires branch to be merged)
await execAsync(`git branch -d "${branch}"`, {
cwd: this.options.rootDir,
timeout: 30_000,
});
log.log(`Deleted branch: ${branch}`);
cleaned++;
deletedBranches.push(branch);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.warn(
`Safe delete failed for orphaned branch ${branch}: ${errorMessage} — attempting force delete`,
);
// Safe delete failed (not merged) — force delete
const inspection = await this.inspectOrphanedBranch(branch);
if (!inspection) continue;
if (inspection.uniqueCommitCount <= 0) {
try {
await execAsync(`git branch -D "${branch}"`, {
execSync(`git branch -d "${branch}"`, {
cwd: this.options.rootDir,
timeout: 30_000,
stdio: ["pipe", "pipe", "pipe"],
});
log.log(`Force-deleted branch: ${branch}`);
prunedBranches.push(branch);
cleaned++;
deletedBranches.push(branch);
} catch (forceErr: unknown) {
const forceErrorMessage = forceErr instanceof Error ? forceErr.message : String(forceErr);
log.warn(`Failed to force-delete orphaned branch ${branch}: ${forceErrorMessage} — non-fatal`);
// Individual failure is non-fatal
try {
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", "orphan-branch"),
agentId: "self-healing",
phase: "orphan-branch-rescue",
});
await auditor.git({
type: "branch:orphan-prune",
target: branch,
metadata: {
phase: "orphan-branch-rescue",
tipSha: inspection.tipSha,
uniqueCommitCount: inspection.uniqueCommitCount,
},
});
} catch (auditErr: unknown) {
log.warn(`Failed to write branch:orphan-prune run-audit event for ${branch}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
}
} catch (err: unknown) {
log.warn(`Failed to prune subsumed orphaned branch ${branch}: ${err instanceof Error ? err.message : String(err)} — non-fatal`);
}
continue;
}
const derivedTaskId = inspection.derivedTaskId;
const matchedTask = derivedTaskId ? taskById.get(derivedTaskId.toUpperCase()) : undefined;
const existingBranchTask = allTasks.find((task) => task.branch === branch);
if (matchedTask?.column === "archived") {
const metadata = (matchedTask.metadata && typeof matchedTask.metadata === "object")
? matchedTask.metadata as Record<string, unknown>
: {};
if (metadata.orphanRescueAcknowledged !== true) {
await this.store.updateTask(matchedTask.id, {
metadata: {
...metadata,
orphanRescueAcknowledged: true,
},
});
log.warn(`[recovery] orphan-rescue-archived-skip ${matchedTask.id} branch=${branch} tip=${inspection.tipSha.slice(0, 12)} unique=${inspection.uniqueCommitCount}`);
}
continue;
}
if (!matchedTask && !existingBranchTask) {
const summaryLines = [
`Recovered orphaned branch: ${branch}`,
`Tip: ${inspection.tipSha}`,
`Unique commits vs main: ${inspection.uniqueCommitCount}`,
];
if (inspection.uniqueCommitSubjects.length > 0) {
summaryLines.push("Recent commit subjects:");
for (const subject of inspection.uniqueCommitSubjects) {
summaryLines.push(`- ${subject}`);
}
}
const rescueTask = await this.store.createTask({
title: `Recover orphaned branch ${branch}`,
description: summaryLines.join("\n"),
branch,
column: "triage",
});
allTasks.push({ ...rescueTask, branch, column: "triage" } as Task);
if (inspection.registeredWorktreePath) {
await this.store.updateTask(rescueTask.id, { worktree: inspection.registeredWorktreePath });
}
await this.store.logEntry(rescueTask.id, `[recovery] orphan-rescue-created ${rescueTask.id} from ${branch} (${inspection.uniqueCommitCount} unique commits)`);
try {
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", rescueTask.id),
agentId: "self-healing",
taskId: rescueTask.id,
taskLineageId: rescueTask.lineageId,
phase: "orphan-branch-rescue",
});
await auditor.git({
type: "branch:orphan-rescued",
target: branch,
metadata: {
phase: "orphan-branch-rescue",
rescueTaskId: rescueTask.id,
tipSha: inspection.tipSha,
uniqueCommitCount: inspection.uniqueCommitCount,
derivedTaskId: derivedTaskId ?? null,
},
});
} catch (auditErr: unknown) {
log.warn(`Failed to write branch:orphan-rescued run-audit event for ${branch}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
}
}
}
if (deletedBranches.length > 0) {
// FN-2165 regression guard: if any dependent task stored one of these
// now-gone branches as its baseBranch, null it so the task doesn't
// hard-fail at worktree creation time.
const cleared = this.store.clearStaleExecutionStartBranchReferences(deletedBranches);
if (prunedBranches.length > 0) {
const cleared = this.store.clearStaleExecutionStartBranchReferences(prunedBranches);
if (cleared.length > 0) {
log.log(`Cleared stale baseBranch on ${cleared.length} task(s): ${cleared.join(", ")}`);
}
}
if (cleaned > 0) {
log.log(`Cleaned ${cleaned} orphaned branch(es)`);
}
return cleaned;
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Orphaned branch cleanup failed: ${errorMessage}`);