feat(FN-4962): complete Step 3 — reconcile stale worktree metadata

Fusion-Task-Id: FN-4962
Fusion-Task-Lineage: de4ce54b-c5e4-40f9-a048-677adac3e1a0
This commit is contained in:
Fusion (runfusion.ai)
2026-05-17 15:23:49 -07:00
committed by gsxdsm
parent 235754a766
commit 40f9aa99e6
4 changed files with 336 additions and 1 deletions

View File

@@ -0,0 +1,167 @@
import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { execSync } from "node:child_process";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { EventEmitter } from "node:events";
const { logger } = vi.hoisted(() => ({
logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
vi.mock("../logger.js", () => ({
createLogger: vi.fn(() => logger),
}));
import type { Task } from "@fusion/core";
import { TaskStore } from "@fusion/core";
import { SelfHealingManager } from "../self-healing.js";
function git(cwd: string, command: string): string {
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
}
function makeSlimTask(id: string, overrides: Partial<Task> = {}): Task {
return {
id,
title: id,
description: id,
column: "todo",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
...overrides,
} as Task;
}
describe("self-healing worktree metadata reconcile", () => {
let rootDir = "";
let store: TaskStore;
beforeEach(async () => {
rootDir = mkdtempSync(join(tmpdir(), "fn-4962-"));
git(rootDir, "init -b main");
git(rootDir, "config user.name 'Fusion'");
git(rootDir, "config user.email 'hi@runfusion.ai'");
writeFileSync(join(rootDir, "README.md"), "root\n");
git(rootDir, "add README.md");
git(rootDir, "commit -m 'init'");
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
await store.createTask({
title: "FN-4913 reproduction",
description: "repro",
});
});
afterEach(() => {
try {
store?.close();
} catch {
// noop
}
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
vi.clearAllMocks();
});
it("keeps reconcile-task-worktree-metadata ordered before reclaim-stale-active-branches in startup and maintenance", () => {
const selfHealingPath = fileURLToPath(new URL("../self-healing.ts", import.meta.url));
const source = readFileSync(selfHealingPath, "utf8");
const startupSlice = source.slice(
source.indexOf("const steps:"),
source.indexOf("for (const step of steps)"),
);
expect(startupSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeGreaterThan(-1);
expect(startupSlice.indexOf('"reclaim-stale-active-branches"')).toBeGreaterThan(-1);
expect(startupSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeLessThan(
startupSlice.indexOf('"reclaim-stale-active-branches"'),
);
const maintenanceSlice = source.slice(
source.indexOf("const batch2Fns:"),
source.indexOf("for (const fn of batch2Fns)"),
);
expect(maintenanceSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeGreaterThan(-1);
expect(maintenanceSlice.indexOf('"reclaim-stale-active-branches"')).toBeGreaterThan(-1);
expect(maintenanceSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeLessThan(
maintenanceSlice.indexOf('"reclaim-stale-active-branches"'),
);
});
it("rebinds stale task.worktree + null branch to live fusion/<id> worktree", async () => {
const [task] = await store.listTasks();
expect(task).toBeTruthy();
const stalePath = join(rootDir, ".worktrees", "misty-grove");
const livePath = join(rootDir, ".worktrees", "sleek-stone");
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
const branch = `fusion/${task.id.toLowerCase()}`;
git(rootDir, `branch ${branch}`);
git(rootDir, `worktree add ${livePath} ${branch}`);
writeFileSync(join(livePath, "feature.txt"), "changed\n");
git(livePath, "add feature.txt");
git(livePath, "commit -m 'feature commit'");
await store.updateTask(task.id, {
column: "in-review",
worktree: stalePath,
branch: null,
});
const auditSpy = vi.spyOn(store, "recordRunAuditEvent");
const manager = new SelfHealingManager(store, {
rootDir,
getExecutingTaskIds: () => new Set<string>(),
});
await (manager as any).reconcileTaskWorktreeMetadata();
const canonicalLivePath = realpathSync(livePath);
const updated = await store.getTask(task.id);
expect(updated?.worktree).toBe(canonicalLivePath);
expect(updated?.branch).toBe(branch);
const taskJson = JSON.parse(
readFileSync(join(rootDir, ".fusion", "tasks", task.id, "task.json"), "utf8"),
) as { worktree?: string | null; branch?: string | null };
expect(taskJson.worktree).toBe(canonicalLivePath);
expect(taskJson.branch).toBe(branch);
expect(logger.log).toHaveBeenCalled();
expect(auditSpy).toHaveBeenCalledWith(
expect.objectContaining({ mutationType: "task:auto-recover-worktree-metadata-rebound" }),
);
});
});
describe("reconcileTaskWorktreeMetadata matrix", () => {
it("skips done/archived and executing tasks", async () => {
const tasks = [
makeSlimTask("FN-100", { column: "done", worktree: "/missing", branch: null }),
makeSlimTask("FN-101", { column: "archived", worktree: "/missing", branch: null }),
makeSlimTask("FN-102", { column: "todo", worktree: "/missing", branch: null }),
];
const store = Object.assign(new EventEmitter(), {
getSettings: vi.fn(async () => ({ globalPause: false, enginePaused: false })),
listTasks: vi.fn(async () => tasks),
updateTask: vi.fn(async () => undefined),
recordRunAuditEvent: vi.fn(async () => undefined),
}) as unknown as TaskStore;
const manager = new SelfHealingManager(store, {
rootDir: process.cwd(),
getExecutingTaskIds: () => new Set(["FN-102"]),
});
const repaired = await manager.reconcileTaskWorktreeMetadata();
expect(repaired).toBe(0);
expect((store as any).updateTask).not.toHaveBeenCalled();
});
});

View File

@@ -47,6 +47,7 @@ vi.mock("node:fs", () => ({
import {
WorktreePool,
getRegisteredWorktreeBranchMap,
getRegisteredWorktreePaths,
isGitRepository,
scanIdleWorktrees,
@@ -613,6 +614,39 @@ describe("getRegisteredWorktreePaths", () => {
});
});
describe("getRegisteredWorktreeBranchMap", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("returns a branch→worktree map from porcelain output", async () => {
mockedExecSync.mockImplementation((cmd: any) => {
if (String(cmd) === "git worktree list --porcelain") {
return [
"worktree /root",
"HEAD abc",
"branch refs/heads/main",
"",
"worktree /root/.worktrees/sleek-stone",
"HEAD def",
"branch refs/heads/fusion/fn-4913",
"",
"worktree /root/.worktrees/detached",
"HEAD 123",
"detached",
"",
].join("\n") as any;
}
return Buffer.from("");
});
const map = await getRegisteredWorktreeBranchMap("/root");
expect(map.get("main")).toBe("/root");
expect(map.get("fusion/fn-4913")).toBe("/root/.worktrees/sleek-stone");
expect(map.has("detached")).toBe(false);
});
});
// ── Helper for mock store ─────────────────────────────────────────────
function makeTask(id: string, column: Column, worktree?: string): Task {

View File

@@ -29,7 +29,7 @@ import { isAbsolute, join, relative, resolve } from "node:path";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStallReason, getStalePausedReviewSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js";
import { RemovalReason, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import {
extractMissingWorktreePathFromSessionStartFailure,
isMissingWorktreeSessionStartFailure,
@@ -46,6 +46,7 @@ import { resolveWorktreesDir } from "./worktree-paths.js";
import type { OwnedLandedClassification } from "./merger.js";
const log = createLogger("self-healing");
const worktreeMetadataReconcileLog = createLogger("worktree-metadata-reconcile");
const execAsync = promisify(exec);
const DONE_TASK_INTEGRITY_SWEEP_LIMIT = 50;
@@ -556,6 +557,8 @@ export class SelfHealingManager {
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies().then(() => undefined) },
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts().then(() => undefined) },
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts().then(() => undefined) },
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata().then(() => undefined) },
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches().then(() => undefined) },
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews().then(() => undefined) },
@@ -1131,6 +1134,8 @@ export class SelfHealingManager {
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies() },
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts() },
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata() },
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches() },
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
@@ -2165,6 +2170,7 @@ export class SelfHealingManager {
if (settings.globalPause || settings.enginePaused) return result;
const task = await this.store.getTask(taskId);
await this.reconcileTaskWorktreeMetadata({ includeTaskIds: new Set([taskId]) });
const allTasks = await this.store.listTasks({ slim: true, includeArchived: true });
const taskById = new Map(allTasks.map((t) => [t.id, t]));
const todoTasks = await this.store.listTasks({ column: "todo", slim: true });
@@ -2299,6 +2305,109 @@ export class SelfHealingManager {
}
}
private async emitWorktreeMetadataAuditEvent(input: {
taskId: string;
mutationType: "task:auto-recover-worktree-metadata-rebound" | "task:auto-recover-worktree-metadata-cleared";
previousWorktree: string | null;
newWorktree: string | null;
previousBranch: string | null;
newBranch: string | null;
}): Promise<void> {
try {
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", input.taskId),
agentId: "self-healing",
taskId: input.taskId,
phase: "worktree-metadata-reconcile",
});
await auditor.database({
type: input.mutationType as never,
target: input.taskId,
metadata: {
taskId: input.taskId,
previousWorktree: input.previousWorktree,
newWorktree: input.newWorktree,
previousBranch: input.previousBranch,
newBranch: input.newBranch,
},
});
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
worktreeMetadataReconcileLog.warn(
`Failed to record ${input.mutationType} for ${input.taskId}: ${errorMessage}`,
);
}
}
async reconcileTaskWorktreeMetadata(options?: { includeTaskIds?: Set<string> }): Promise<number> {
try {
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) return 0;
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
const branchMap = await getRegisteredWorktreeBranchMap(this.options.rootDir);
const registeredPaths = new Set(branchMap.values());
let repaired = 0;
for (const task of allTasks) {
if (!task.worktree) continue;
if (!options?.includeTaskIds?.has(task.id) && (task.column === "done" || task.column === "archived")) {
continue;
}
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
if (executingIds.has(task.id)) continue;
if (activeSessionRegistry.isPathActive(task.worktree)) continue;
const normalizedBranch = `fusion/${task.id.toLowerCase()}`;
const canonicalTaskWorktree = resolve(task.worktree);
const stale = !existsSync(task.worktree) || !registeredPaths.has(canonicalTaskWorktree);
if (!stale) continue;
const previousWorktree = task.worktree;
const previousBranch = task.branch ?? null;
const liveWorktree = branchMap.get(normalizedBranch);
if (liveWorktree) {
await this.store.updateTask(task.id, { worktree: liveWorktree, branch: normalizedBranch });
await this.emitWorktreeMetadataAuditEvent({
taskId: task.id,
mutationType: "task:auto-recover-worktree-metadata-rebound",
previousWorktree,
newWorktree: liveWorktree,
previousBranch,
newBranch: normalizedBranch,
});
worktreeMetadataReconcileLog.log(
`[worktree-metadata-reconcile] rebound ${task.id}: ${previousWorktree} -> ${liveWorktree} (${previousBranch ?? "<none>"} -> ${normalizedBranch})`,
);
repaired++;
continue;
}
await this.store.updateTask(task.id, { worktree: null, branch: null });
await this.emitWorktreeMetadataAuditEvent({
taskId: task.id,
mutationType: "task:auto-recover-worktree-metadata-cleared",
previousWorktree,
newWorktree: null,
previousBranch,
newBranch: null,
});
worktreeMetadataReconcileLog.log(
`[worktree-metadata-reconcile] cleared ${task.id}: ${previousWorktree} (${previousBranch ?? "<none>"})`,
);
repaired++;
}
return repaired;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
worktreeMetadataReconcileLog.error(`reconcileTaskWorktreeMetadata failed: ${errorMessage}`);
return 0;
}
}
async clearStaleBlockedBy(): Promise<number> {
try {
const settings = await this.store.getSettings();

View File

@@ -121,6 +121,31 @@ export async function getRegisteredWorktreePaths(rootDir: string): Promise<Set<s
return new Set(canonicalized);
}
export async function getRegisteredWorktreeBranchMap(rootDir: string): Promise<Map<string, string>> {
const { rawOutput } = await describeRegisteredWorktrees(rootDir);
const branchMap = new Map<string, string>();
let currentWorktree: string | null = null;
for (const line of rawOutput.split("\n")) {
if (line.startsWith("worktree ")) {
currentWorktree = canonicalizePath(line.slice("worktree ".length));
continue;
}
if (line.startsWith("branch ") && currentWorktree) {
const branchRef = line.slice("branch ".length).trim();
const branchName = branchRef.startsWith("refs/heads/")
? branchRef.slice("refs/heads/".length)
: branchRef;
if (branchName) {
branchMap.set(branchName, currentWorktree);
}
}
}
return branchMap;
}
export async function isRegisteredGitWorktree(rootDir: string, worktreePath: string): Promise<boolean> {
return (await getRegisteredWorktreePaths(rootDir)).has(canonicalizePath(worktreePath));
}