feat(FN-4962): complete Step 3 — reconcile stale worktree metadata
Fusion-Task-Id: FN-4962 Fusion-Task-Lineage: de4ce54b-c5e4-40f9-a048-677adac3e1a0
This commit is contained in:
committed by
gsxdsm
parent
235754a766
commit
40f9aa99e6
@@ -0,0 +1,167 @@
|
||||
import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
const { logger } = vi.hoisted(() => ({
|
||||
logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
vi.mock("../logger.js", () => ({
|
||||
createLogger: vi.fn(() => logger),
|
||||
}));
|
||||
|
||||
import type { Task } from "@fusion/core";
|
||||
import { TaskStore } from "@fusion/core";
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
|
||||
function git(cwd: string, command: string): string {
|
||||
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
function makeSlimTask(id: string, overrides: Partial<Task> = {}): Task {
|
||||
return {
|
||||
id,
|
||||
title: id,
|
||||
description: id,
|
||||
column: "todo",
|
||||
dependencies: [],
|
||||
steps: [],
|
||||
currentStep: 0,
|
||||
log: [],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
} as Task;
|
||||
}
|
||||
|
||||
describe("self-healing worktree metadata reconcile", () => {
|
||||
let rootDir = "";
|
||||
let store: TaskStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
rootDir = mkdtempSync(join(tmpdir(), "fn-4962-"));
|
||||
git(rootDir, "init -b main");
|
||||
git(rootDir, "config user.name 'Fusion'");
|
||||
git(rootDir, "config user.email 'hi@runfusion.ai'");
|
||||
writeFileSync(join(rootDir, "README.md"), "root\n");
|
||||
git(rootDir, "add README.md");
|
||||
git(rootDir, "commit -m 'init'");
|
||||
|
||||
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
|
||||
await store.createTask({
|
||||
title: "FN-4913 reproduction",
|
||||
description: "repro",
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
store?.close();
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("keeps reconcile-task-worktree-metadata ordered before reclaim-stale-active-branches in startup and maintenance", () => {
|
||||
const selfHealingPath = fileURLToPath(new URL("../self-healing.ts", import.meta.url));
|
||||
const source = readFileSync(selfHealingPath, "utf8");
|
||||
|
||||
const startupSlice = source.slice(
|
||||
source.indexOf("const steps:"),
|
||||
source.indexOf("for (const step of steps)"),
|
||||
);
|
||||
expect(startupSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeGreaterThan(-1);
|
||||
expect(startupSlice.indexOf('"reclaim-stale-active-branches"')).toBeGreaterThan(-1);
|
||||
expect(startupSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeLessThan(
|
||||
startupSlice.indexOf('"reclaim-stale-active-branches"'),
|
||||
);
|
||||
|
||||
const maintenanceSlice = source.slice(
|
||||
source.indexOf("const batch2Fns:"),
|
||||
source.indexOf("for (const fn of batch2Fns)"),
|
||||
);
|
||||
expect(maintenanceSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeGreaterThan(-1);
|
||||
expect(maintenanceSlice.indexOf('"reclaim-stale-active-branches"')).toBeGreaterThan(-1);
|
||||
expect(maintenanceSlice.indexOf('"reconcile-task-worktree-metadata"')).toBeLessThan(
|
||||
maintenanceSlice.indexOf('"reclaim-stale-active-branches"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("rebinds stale task.worktree + null branch to live fusion/<id> worktree", async () => {
|
||||
const [task] = await store.listTasks();
|
||||
expect(task).toBeTruthy();
|
||||
|
||||
const stalePath = join(rootDir, ".worktrees", "misty-grove");
|
||||
const livePath = join(rootDir, ".worktrees", "sleek-stone");
|
||||
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
|
||||
|
||||
const branch = `fusion/${task.id.toLowerCase()}`;
|
||||
git(rootDir, `branch ${branch}`);
|
||||
git(rootDir, `worktree add ${livePath} ${branch}`);
|
||||
writeFileSync(join(livePath, "feature.txt"), "changed\n");
|
||||
git(livePath, "add feature.txt");
|
||||
git(livePath, "commit -m 'feature commit'");
|
||||
|
||||
await store.updateTask(task.id, {
|
||||
column: "in-review",
|
||||
worktree: stalePath,
|
||||
branch: null,
|
||||
});
|
||||
|
||||
const auditSpy = vi.spyOn(store, "recordRunAuditEvent");
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir,
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
await (manager as any).reconcileTaskWorktreeMetadata();
|
||||
|
||||
const canonicalLivePath = realpathSync(livePath);
|
||||
const updated = await store.getTask(task.id);
|
||||
expect(updated?.worktree).toBe(canonicalLivePath);
|
||||
expect(updated?.branch).toBe(branch);
|
||||
|
||||
const taskJson = JSON.parse(
|
||||
readFileSync(join(rootDir, ".fusion", "tasks", task.id, "task.json"), "utf8"),
|
||||
) as { worktree?: string | null; branch?: string | null };
|
||||
expect(taskJson.worktree).toBe(canonicalLivePath);
|
||||
expect(taskJson.branch).toBe(branch);
|
||||
|
||||
expect(logger.log).toHaveBeenCalled();
|
||||
expect(auditSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ mutationType: "task:auto-recover-worktree-metadata-rebound" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reconcileTaskWorktreeMetadata matrix", () => {
|
||||
it("skips done/archived and executing tasks", async () => {
|
||||
const tasks = [
|
||||
makeSlimTask("FN-100", { column: "done", worktree: "/missing", branch: null }),
|
||||
makeSlimTask("FN-101", { column: "archived", worktree: "/missing", branch: null }),
|
||||
makeSlimTask("FN-102", { column: "todo", worktree: "/missing", branch: null }),
|
||||
];
|
||||
|
||||
const store = Object.assign(new EventEmitter(), {
|
||||
getSettings: vi.fn(async () => ({ globalPause: false, enginePaused: false })),
|
||||
listTasks: vi.fn(async () => tasks),
|
||||
updateTask: vi.fn(async () => undefined),
|
||||
recordRunAuditEvent: vi.fn(async () => undefined),
|
||||
}) as unknown as TaskStore;
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: process.cwd(),
|
||||
getExecutingTaskIds: () => new Set(["FN-102"]),
|
||||
});
|
||||
|
||||
const repaired = await manager.reconcileTaskWorktreeMetadata();
|
||||
expect(repaired).toBe(0);
|
||||
expect((store as any).updateTask).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -47,6 +47,7 @@ vi.mock("node:fs", () => ({
|
||||
|
||||
import {
|
||||
WorktreePool,
|
||||
getRegisteredWorktreeBranchMap,
|
||||
getRegisteredWorktreePaths,
|
||||
isGitRepository,
|
||||
scanIdleWorktrees,
|
||||
@@ -613,6 +614,39 @@ describe("getRegisteredWorktreePaths", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("getRegisteredWorktreeBranchMap", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("returns a branch→worktree map from porcelain output", async () => {
|
||||
mockedExecSync.mockImplementation((cmd: any) => {
|
||||
if (String(cmd) === "git worktree list --porcelain") {
|
||||
return [
|
||||
"worktree /root",
|
||||
"HEAD abc",
|
||||
"branch refs/heads/main",
|
||||
"",
|
||||
"worktree /root/.worktrees/sleek-stone",
|
||||
"HEAD def",
|
||||
"branch refs/heads/fusion/fn-4913",
|
||||
"",
|
||||
"worktree /root/.worktrees/detached",
|
||||
"HEAD 123",
|
||||
"detached",
|
||||
"",
|
||||
].join("\n") as any;
|
||||
}
|
||||
return Buffer.from("");
|
||||
});
|
||||
|
||||
const map = await getRegisteredWorktreeBranchMap("/root");
|
||||
expect(map.get("main")).toBe("/root");
|
||||
expect(map.get("fusion/fn-4913")).toBe("/root/.worktrees/sleek-stone");
|
||||
expect(map.has("detached")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Helper for mock store ─────────────────────────────────────────────
|
||||
|
||||
function makeTask(id: string, column: Column, worktree?: string): Task {
|
||||
|
||||
@@ -29,7 +29,7 @@ import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStallReason, getStalePausedReviewSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority } from "@fusion/core";
|
||||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { RemovalReason, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
import {
|
||||
extractMissingWorktreePathFromSessionStartFailure,
|
||||
isMissingWorktreeSessionStartFailure,
|
||||
@@ -46,6 +46,7 @@ import { resolveWorktreesDir } from "./worktree-paths.js";
|
||||
import type { OwnedLandedClassification } from "./merger.js";
|
||||
|
||||
const log = createLogger("self-healing");
|
||||
const worktreeMetadataReconcileLog = createLogger("worktree-metadata-reconcile");
|
||||
const execAsync = promisify(exec);
|
||||
const DONE_TASK_INTEGRITY_SWEEP_LIMIT = 50;
|
||||
|
||||
@@ -556,6 +557,8 @@ export class SelfHealingManager {
|
||||
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies().then(() => undefined) },
|
||||
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts().then(() => undefined) },
|
||||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts().then(() => undefined) },
|
||||
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
|
||||
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata().then(() => undefined) },
|
||||
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches().then(() => undefined) },
|
||||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
|
||||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews().then(() => undefined) },
|
||||
@@ -1131,6 +1134,8 @@ export class SelfHealingManager {
|
||||
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies() },
|
||||
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts() },
|
||||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
|
||||
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
|
||||
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata() },
|
||||
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches() },
|
||||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
|
||||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
|
||||
@@ -2165,6 +2170,7 @@ export class SelfHealingManager {
|
||||
if (settings.globalPause || settings.enginePaused) return result;
|
||||
|
||||
const task = await this.store.getTask(taskId);
|
||||
await this.reconcileTaskWorktreeMetadata({ includeTaskIds: new Set([taskId]) });
|
||||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: true });
|
||||
const taskById = new Map(allTasks.map((t) => [t.id, t]));
|
||||
const todoTasks = await this.store.listTasks({ column: "todo", slim: true });
|
||||
@@ -2299,6 +2305,109 @@ export class SelfHealingManager {
|
||||
}
|
||||
}
|
||||
|
||||
private async emitWorktreeMetadataAuditEvent(input: {
|
||||
taskId: string;
|
||||
mutationType: "task:auto-recover-worktree-metadata-rebound" | "task:auto-recover-worktree-metadata-cleared";
|
||||
previousWorktree: string | null;
|
||||
newWorktree: string | null;
|
||||
previousBranch: string | null;
|
||||
newBranch: string | null;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const auditor = createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("self-heal", input.taskId),
|
||||
agentId: "self-healing",
|
||||
taskId: input.taskId,
|
||||
phase: "worktree-metadata-reconcile",
|
||||
});
|
||||
await auditor.database({
|
||||
type: input.mutationType as never,
|
||||
target: input.taskId,
|
||||
metadata: {
|
||||
taskId: input.taskId,
|
||||
previousWorktree: input.previousWorktree,
|
||||
newWorktree: input.newWorktree,
|
||||
previousBranch: input.previousBranch,
|
||||
newBranch: input.newBranch,
|
||||
},
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
worktreeMetadataReconcileLog.warn(
|
||||
`Failed to record ${input.mutationType} for ${input.taskId}: ${errorMessage}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async reconcileTaskWorktreeMetadata(options?: { includeTaskIds?: Set<string> }): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) return 0;
|
||||
|
||||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||||
const branchMap = await getRegisteredWorktreeBranchMap(this.options.rootDir);
|
||||
const registeredPaths = new Set(branchMap.values());
|
||||
let repaired = 0;
|
||||
|
||||
for (const task of allTasks) {
|
||||
if (!task.worktree) continue;
|
||||
if (!options?.includeTaskIds?.has(task.id) && (task.column === "done" || task.column === "archived")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||||
if (executingIds.has(task.id)) continue;
|
||||
if (activeSessionRegistry.isPathActive(task.worktree)) continue;
|
||||
|
||||
const normalizedBranch = `fusion/${task.id.toLowerCase()}`;
|
||||
const canonicalTaskWorktree = resolve(task.worktree);
|
||||
const stale = !existsSync(task.worktree) || !registeredPaths.has(canonicalTaskWorktree);
|
||||
if (!stale) continue;
|
||||
|
||||
const previousWorktree = task.worktree;
|
||||
const previousBranch = task.branch ?? null;
|
||||
const liveWorktree = branchMap.get(normalizedBranch);
|
||||
|
||||
if (liveWorktree) {
|
||||
await this.store.updateTask(task.id, { worktree: liveWorktree, branch: normalizedBranch });
|
||||
await this.emitWorktreeMetadataAuditEvent({
|
||||
taskId: task.id,
|
||||
mutationType: "task:auto-recover-worktree-metadata-rebound",
|
||||
previousWorktree,
|
||||
newWorktree: liveWorktree,
|
||||
previousBranch,
|
||||
newBranch: normalizedBranch,
|
||||
});
|
||||
worktreeMetadataReconcileLog.log(
|
||||
`[worktree-metadata-reconcile] rebound ${task.id}: ${previousWorktree} -> ${liveWorktree} (${previousBranch ?? "<none>"} -> ${normalizedBranch})`,
|
||||
);
|
||||
repaired++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.store.updateTask(task.id, { worktree: null, branch: null });
|
||||
await this.emitWorktreeMetadataAuditEvent({
|
||||
taskId: task.id,
|
||||
mutationType: "task:auto-recover-worktree-metadata-cleared",
|
||||
previousWorktree,
|
||||
newWorktree: null,
|
||||
previousBranch,
|
||||
newBranch: null,
|
||||
});
|
||||
worktreeMetadataReconcileLog.log(
|
||||
`[worktree-metadata-reconcile] cleared ${task.id}: ${previousWorktree} (${previousBranch ?? "<none>"})`,
|
||||
);
|
||||
repaired++;
|
||||
}
|
||||
|
||||
return repaired;
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
worktreeMetadataReconcileLog.error(`reconcileTaskWorktreeMetadata failed: ${errorMessage}`);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async clearStaleBlockedBy(): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
|
||||
@@ -121,6 +121,31 @@ export async function getRegisteredWorktreePaths(rootDir: string): Promise<Set<s
|
||||
return new Set(canonicalized);
|
||||
}
|
||||
|
||||
export async function getRegisteredWorktreeBranchMap(rootDir: string): Promise<Map<string, string>> {
|
||||
const { rawOutput } = await describeRegisteredWorktrees(rootDir);
|
||||
const branchMap = new Map<string, string>();
|
||||
let currentWorktree: string | null = null;
|
||||
|
||||
for (const line of rawOutput.split("\n")) {
|
||||
if (line.startsWith("worktree ")) {
|
||||
currentWorktree = canonicalizePath(line.slice("worktree ".length));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.startsWith("branch ") && currentWorktree) {
|
||||
const branchRef = line.slice("branch ".length).trim();
|
||||
const branchName = branchRef.startsWith("refs/heads/")
|
||||
? branchRef.slice("refs/heads/".length)
|
||||
: branchRef;
|
||||
if (branchName) {
|
||||
branchMap.set(branchName, currentWorktree);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return branchMap;
|
||||
}
|
||||
|
||||
export async function isRegisteredGitWorktree(rootDir: string, worktreePath: string): Promise<boolean> {
|
||||
return (await getRegisteredWorktreePaths(rootDir)).has(canonicalizePath(worktreePath));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user