From ac342f501289c6175b4e6e2b90e76f968f606b45 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Thu, 11 Jun 2026 20:02:21 -0700 Subject: [PATCH] FN-6220: harden AI merge worktree cleanup Harden AI merge cleanup so temporary clean-room worktrees are removed and audited reliably. - Canonicalize AI merge cleanup paths and prune stale git worktree metadata after removals. - Treat already-absent or de-registered cleanup paths as idempotent success while still auditing genuine failures. - Add regression coverage for successful, no-op, retry, thrown, orphaned, and dangling-registration cleanup paths. - Document the inline and self-healing cleanup lifecycle for AI merge temp worktrees. Files changed: docs/architecture.md | 4 +- .../engine/src/__tests__/merger-ai-cleanup.test.ts | 28 +- .../ai-merge-worktree-cleanup.test.ts | 284 +++++++++++++++++++++ packages/engine/src/merger-ai.ts | 108 +++++--- 4 files changed, 392 insertions(+), 32 deletions(-) Fusion-Task-Id: FN-6220 Fusion-Task-Lineage: b002313c-4349-49c2-8d0f-cd1853369d84 --- docs/architecture.md | 4 +- .../src/__tests__/merger-ai-cleanup.test.ts | 28 +- .../ai-merge-worktree-cleanup.test.ts | 284 ++++++++++++++++++ packages/engine/src/merger-ai.ts | 112 +++++-- 4 files changed, 394 insertions(+), 34 deletions(-) create mode 100644 packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 1940321e04..1bad8ee65d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -670,7 +670,9 @@ Runtime action-gate flow (v1): - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - Batch 1 maintenance now includes one `fts-maintenance` step for both search indexes. The live `tasks_fts` branch still runs `merge` every tick, `optimize` every 4th tick, and `rebuild` above `32 MiB` or `1 MiB × live task count`. The archive `archived_tasks_fts` branch is lighter because archive writes are mostly append-only: `merge` every 8th tick, `optimize` every 24th tick, and `rebuild` above `64 MiB` or `512 KiB × archived row count`. Each branch is independently guarded by `fts5Available` and emits `task:fts-maintenance` run-audit telemetry with distinct `target` values (`tasks_fts` vs `archived_tasks_fts`). - - Batch 1 also sweeps stale AI merge clean-room worktrees under `tmpdir()` whose names start with `fusion-ai-merge-`. `runAiMerge` registers each live clean-room worktree in `activeSessionRegistry` with kind `ai-merge` as soon as the temp directory exists and keeps both raw and canonical paths registered for the duration of the merge, so both the periodic sweep and pre-merge prune defer when either path is active. The default age gate is 2 hours; task-aware cleanup uses a 10-minute grace period for `done`/`archived` tasks and for genuinely missing/deleted task rows, and every removal path is clamped by the same 10-minute minimum-age floor so a freshly created worktree is never reaped. Transient `getTask` lookup failures (for example SQLite busy/parse errors) are not treated as deletion evidence; they log a warning, emit `lookup-error` only if eventually removed, and retain the conservative 2-hour gate. The sweep canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force ` before filesystem removal, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. Standalone AI-merge cleanup is idempotent for an already-removed/de-registered temp worktree (`ENOENT`, `spawn git ENOENT`, `No such file or directory`, or `is not a working tree`): it emits successful `merge:ai-worktree-cleanup` telemetry with `alreadyAbsent`/`idempotent` markers and must not turn a confirmed land into a failed merge, while genuine removal failures still emit `success:false`. It is intentionally native even when `worktrunk.enabled` because these temp-dir worktrees are outside the worktrunk-managed project layout. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle. + - AI merge clean-room worktrees are created under `tmpdir()` as `fusion-ai-merge-fn--` detached worktrees. Inline cleanup runs from `runAiMerge`'s clean-room `finally` for successful lands, empty/no-op finalization, concurrent-advance retries, and thrown/aborted merges. Cleanup canonicalizes the temp path, attempts `git worktree remove --force`, always falls back to filesystem removal, then runs `git worktree prune` so stale or partial registrations (including `git worktree add` failures) do not dangle. Cleanup emits `merge:ai-worktree-cleanup` audit events for git-remove, fs-rm, and prune phases; benign already-absent/de-registered paths are treated as idempotent success, while genuine filesystem-removal failures are logged/audited with `success: false` rather than silently swallowed. + - Batch 1 also sweeps stale AI merge clean-room worktrees under `tmpdir()` whose names start with `fusion-ai-merge-`. `runAiMerge` registers each live clean-room worktree in `activeSessionRegistry` with kind `ai-merge` as soon as the temp directory exists and keeps both raw and canonical paths registered for the duration of the merge, so both the periodic sweep and pre-merge prune defer when either path is active. The default age gate is 2 hours; task-aware cleanup uses a 10-minute grace period for `done`/`archived` tasks and for genuinely missing/deleted task rows, and every removal path is clamped by the same 10-minute minimum-age floor so a freshly created worktree is never reaped. Transient `getTask` lookup failures (for example SQLite busy/parse errors) are not treated as deletion evidence; they log a warning, emit `lookup-error` only if eventually removed, and retain the conservative 2-hour gate. The sweep canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force ` before filesystem removal, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. It is intentionally native even when `worktrunk.enabled` because these temp-dir worktrees are outside the worktrunk-managed project layout. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle. + - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - Mission validation has a dedicated stale-run reaper: startup recovery and Batch 2 maintenance call `reapStaleMissionValidatorRuns()` when wired by the runtime, using `VALIDATOR_RUN_STALE_MAX_AGE_MS` (currently 6 hours). The sweep terminates ownerless `mission_validator_runs.status='running'` rows as `error`, writes the reap reason into `summary`, leaves `lastValidatorRunId` pointing at the now-terminal run, and emits run-audit telemetry with `mutationType: "mission:validator-run-reaped"` plus `runId`/`featureId`/`missionId`/`triggerType`/`elapsedMs` metadata. Active mission features move to `loopState="needs_fix"` + `lastValidatorStatus="error"` unless their parent mission is already `complete`/`archived`. diff --git a/packages/engine/src/__tests__/merger-ai-cleanup.test.ts b/packages/engine/src/__tests__/merger-ai-cleanup.test.ts index 5cd21ec756..65fb75e15d 100644 --- a/packages/engine/src/__tests__/merger-ai-cleanup.test.ts +++ b/packages/engine/src/__tests__/merger-ai-cleanup.test.ts @@ -171,6 +171,7 @@ describe("AI merge temp worktree cleanup", () => { it("treats spawn git ENOENT during cleanup as idempotent already-absent success", async () => { const mergeRoot = mkdtempSync(join(tmpdir(), "fusion-ai-merge-fn-1-enoent-cleanup-test-")); tracked.add(mergeRoot); + const canonicalMergeRoot = realpathSync(mergeRoot); const err = Object.assign(new Error("spawn git ENOENT"), { code: "ENOENT" }); const gitRunner = vi.fn(async () => { throw err; }); @@ -179,7 +180,7 @@ describe("AI merge temp worktree cleanup", () => { gitRunner, }); - expect(gitRunner).toHaveBeenCalledWith(["worktree", "remove", "--force", mergeRoot], process.cwd()); + expect(gitRunner).toHaveBeenCalledWith(["worktree", "remove", "--force", canonicalMergeRoot], process.cwd()); expect(events).toEqual(expect.arrayContaining([ expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: true, alreadyAbsent: true, idempotent: true, code: "ENOENT" }) }), @@ -219,10 +220,33 @@ describe("AI merge temp worktree cleanup", () => { const { events } = await cleanup({ worktreeAdded: false, gitRunner }); - expect(gitRunner).not.toHaveBeenCalled(); + expect(gitRunner).toHaveBeenCalledTimes(1); + expect(gitRunner).toHaveBeenCalledWith(["worktree", "prune"], expect.any(String), { timeout: 30_000 }); expect(events.some((event) => event.metadata.phase === "git-remove")).toBe(false); expect(events).toEqual(expect.arrayContaining([ expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }), + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-prune", success: true }) }), + ])); + }); + + it("prunes stale worktree metadata after git removal failure", async () => { + const err = new Error("git remove failed") as Error & { stderr?: string; code?: string }; + err.stderr = "fatal: not a working tree registered in git metadata"; + err.code = "1"; + const gitRunner = vi.fn(async (args: string[]) => { + if (args[0] === "worktree" && args[1] === "remove" && args[2] === "--force") throw err; + return ""; + }); + + const { mergeRoot, events, logs } = await cleanup({ gitRunner }); + + expect(existsSync(mergeRoot)).toBe(false); + expect(gitRunner).toHaveBeenCalledWith(["worktree", "prune"], expect.any(String), { timeout: 30_000 }); + expect(logs.join("\n")).toContain("not a working tree registered"); + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: false }) }), + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }), + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-prune", success: true }) }), ])); }); diff --git a/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts b/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts new file mode 100644 index 0000000000..69423cfd1f --- /dev/null +++ b/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts @@ -0,0 +1,284 @@ +import { afterAll, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { execSync } from "node:child_process"; +import { DEFAULT_SETTINGS, TaskStore, type Settings } from "@fusion/core"; +import { cleanupAiMergeWorktree, runAiMerge } from "../../merger-ai.js"; +import { hasGit } from "./_helpers.js"; +import type { RunAuditor } from "../../run-audit.js"; + +const tracked = new Set(); +const taskIds = new Set(); +const RM = { recursive: true, force: true, maxRetries: 5, retryDelay: 50 } as const; + +afterAll(() => { + for (const taskId of taskIds) removeTmpAiMergeDirs(taskId); + for (const dir of tracked) { + try { + rmSync(dir, RM); + } catch { + // best effort cleanup + } + } +}); + +function git(cwd: string, args: string): string { + return execSync(`git ${args}`, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim(); +} + +function aiMergePrefix(taskId: string): string { + return `fusion-ai-merge-${taskId.toLowerCase()}-`; +} + +function tmpAiMergeDirs(taskId: string): string[] { + const prefix = aiMergePrefix(taskId); + return readdirSync(tmpdir()) + .filter((entry) => entry.startsWith(prefix)) + .map((entry) => join(tmpdir(), entry)); +} + +function removeTmpAiMergeDirs(taskId: string): void { + for (const dir of tmpAiMergeDirs(taskId)) { + try { + rmSync(dir, RM); + } catch { + // best effort cleanup + } + } +} + +function expectNoAiMergeWorktrees(rootDir: string, taskId: string): void { + expect(tmpAiMergeDirs(taskId), `tmpdir entries for ${taskId}`).toEqual([]); + const worktrees = git(rootDir, "worktree list --porcelain"); + expect(worktrees).not.toContain(aiMergePrefix(taskId)); +} + +function realMergeAgent(branch: string) { + return vi.fn(async (cwd: string) => { + execSync(`git merge --squash ${branch}`, { cwd, stdio: "pipe" }); + execSync("git add -A", { cwd, stdio: "pipe" }); + execSync('git commit -q -m "squash: feature"', { cwd, stdio: "pipe" }); + }); +} + +async function createFixture(label: string) { + const rootDir = mkdtempSync(join(tmpdir(), `fusion-ai-merge-cleanup-${label.toLowerCase()}-`)); + tracked.add(rootDir); + git(rootDir, "init -q -b main"); + git(rootDir, 'config user.email "test@example.com"'); + git(rootDir, 'config user.name "Test User"'); + writeFileSync(join(rootDir, "README.md"), `# ${label}\n`); + git(rootDir, "add README.md"); + git(rootDir, 'commit -q -m "chore: init"'); + + const store = new TaskStore(rootDir, undefined, { inMemoryDb: true }); + await store.init(); + const settings: Settings = { + ...DEFAULT_SETTINGS, + autoMerge: true, + includeTaskIdInCommit: true, + commitAuthorEnabled: false, + merger: { ...(DEFAULT_SETTINGS.merger ?? {}), mode: "ai", maxReviewPasses: 1 }, + } as Settings; + await store.updateSettings(settings); + + const created = await store.createTask({ + title: label, + description: "AI merge worktree cleanup fixture", + column: "in-review", + baseBranch: "main", + prompt: "## File Scope\n- packages/engine/src/**\n", + } as any); + const branch = `fusion/${created.id.toLowerCase()}`; + await store.updateTask(created.id, { + column: "in-review", + branch, + baseBranch: "main", + steps: [{ title: "ready", status: "done" }], + status: null, + } as any); + taskIds.add(created.id); + removeTmpAiMergeDirs(created.id); + + return { + rootDir, + store, + taskId: created.id, + branch, + cleanup: async () => { + removeTmpAiMergeDirs(created.id); + store.close(); + rmSync(rootDir, RM); + tracked.delete(rootDir); + }, + }; +} + +function commitTaskBranch(rootDir: string, branch: string, filename: string, contents: string): void { + git(rootDir, `checkout -q -b ${branch}`); + writeFileSync(join(rootDir, filename), contents); + git(rootDir, `add ${filename}`); + git(rootDir, `commit -q -m "feat: ${filename}"`); + git(rootDir, "checkout -q main"); +} + +function makeAudit() { + const events: any[] = []; + const audit: RunAuditor = { + git: vi.fn(async (event: any) => { events.push(event); }), + database: vi.fn(async () => undefined), + filesystem: vi.fn(async () => undefined), + sandbox: vi.fn(async () => undefined), + }; + return { audit, events }; +} + +describe("FN-6220 AI-merge worktree cleanup lifecycle (real git)", () => { + it.skipIf(!hasGit)("removes temp worktree after a successful AI land", async () => { + const fixture = await createFixture("success"); + const { rootDir, store, taskId, branch, cleanup } = fixture; + + try { + commitTaskBranch(rootDir, branch, "feature.txt", "feature work\n"); + + const result = await runAiMerge(store, rootDir, taskId, { manual: true, allowDirtyLocalCheckoutSync: true }, { + mergeAgent: realMergeAgent(branch), + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + }); + + expect(result).toMatchObject({ ok: true, merged: true }); + expectNoAiMergeWorktrees(rootDir, taskId); + } finally { + await cleanup(); + } + }, 20_000); + + it.skipIf(!hasGit)("removes temp worktree after an empty no-op AI merge", async () => { + const fixture = await createFixture("noop"); + const { rootDir, store, taskId, branch, cleanup } = fixture; + + try { + git(rootDir, `checkout -q -b ${branch}`); + git(rootDir, "checkout -q main"); + + const result = await runAiMerge(store, rootDir, taskId, { manual: true, allowDirtyLocalCheckoutSync: true }, { + mergeAgent: vi.fn(async () => { + // Leave HEAD at the integration tip so mergeAndReview returns null. + }), + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + }); + + expect(result).toMatchObject({ ok: true, noOp: true, merged: false }); + expectNoAiMergeWorktrees(rootDir, taskId); + } finally { + await cleanup(); + } + }, 20_000); + + it.skipIf(!hasGit)("cleans each temp worktree before retrying after a concurrent advance", async () => { + const fixture = await createFixture("concurrent"); + const { rootDir, store, taskId, branch, cleanup } = fixture; + + try { + commitTaskBranch(rootDir, branch, "feature.txt", "feature work\n"); + git(rootDir, "checkout -q -b parking main"); + let attempts = 0; + const mergeRoots: string[] = []; + const mergeAgent = vi.fn(async (cwd: string) => { + attempts++; + mergeRoots.push(cwd); + execSync(`git merge --squash ${branch}`, { cwd, stdio: "pipe" }); + execSync("git add -A", { cwd, stdio: "pipe" }); + execSync(`git commit -q -m "squash: feature attempt ${attempts}"`, { cwd, stdio: "pipe" }); + if (attempts === 1) { + const mainBefore = git(rootDir, "rev-parse refs/heads/main"); + const concurrentSha = git(rootDir, 'commit-tree refs/heads/main^{tree} -p refs/heads/main -m "chore: concurrent advance"'); + git(rootDir, `update-ref refs/heads/main ${concurrentSha} ${mainBefore}`); + } + }); + + const result = await runAiMerge(store, rootDir, taskId, { manual: true, allowDirtyLocalCheckoutSync: true }, { + mergeAgent, + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + }); + + expect(result).toMatchObject({ ok: true, merged: true }); + expect(attempts).toBe(2); + expect(mergeRoots).toHaveLength(2); + expect(mergeRoots.every((dir) => !existsSync(dir))).toBe(true); + expectNoAiMergeWorktrees(rootDir, taskId); + } finally { + await cleanup(); + } + }, 20_000); + + it.skipIf(!hasGit)("removes temp worktree when the merge agent throws", async () => { + const fixture = await createFixture("throws"); + const { rootDir, store, taskId, branch, cleanup } = fixture; + + try { + commitTaskBranch(rootDir, branch, "feature.txt", "feature work\n"); + + await expect(runAiMerge(store, rootDir, taskId, { manual: true, allowDirtyLocalCheckoutSync: true }, { + mergeAgent: vi.fn(async () => { throw new Error("simulated merge failure"); }), + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + })).rejects.toThrow("simulated merge failure"); + + expectNoAiMergeWorktrees(rootDir, taskId); + } finally { + await cleanup(); + } + }, 20_000); + + it.skipIf(!hasGit)("pre-merge prune removes an FN-6207-style directory whose git registration is already gone", async () => { + const fixture = await createFixture("orphan-dir"); + const { rootDir, store, taskId, branch, cleanup } = fixture; + + try { + commitTaskBranch(rootDir, branch, "feature.txt", "feature work\n"); + const orphanDir = mkdtempSync(join(tmpdir(), aiMergePrefix(taskId))); + expect(existsSync(orphanDir)).toBe(true); + + await runAiMerge(store, rootDir, taskId, { manual: true, allowDirtyLocalCheckoutSync: true }, { + mergeAgent: realMergeAgent(branch), + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + }); + + expect(existsSync(orphanDir)).toBe(false); + expectNoAiMergeWorktrees(rootDir, taskId); + } finally { + await cleanup(); + } + }, 20_000); + + it.skipIf(!hasGit)("cleanup prunes a dangling git registration whose directory is already gone", async () => { + const fixture = await createFixture("dangling-registration"); + const { rootDir, taskId, cleanup } = fixture; + const { audit } = makeAudit(); + const logs: string[] = []; + + try { + const staleRoot = mkdtempSync(join(tmpdir(), aiMergePrefix(taskId))); + rmSync(staleRoot, RM); + git(rootDir, `worktree add --detach ${staleRoot} main`); + rmSync(staleRoot, RM); + expect(git(rootDir, "worktree list --porcelain")).toContain(staleRoot); + + await cleanupAiMergeWorktree({ + taskId, + mergeRoot: staleRoot, + projectRootDir: rootDir, + worktreeAdded: true, + audit, + log: vi.fn(async (message: string) => { logs.push(message); }), + }); + + expect(existsSync(staleRoot)).toBe(false); + expectNoAiMergeWorktrees(rootDir, taskId); + expect(logs.join("\n")).not.toContain("filesystem rm failed"); + } finally { + await cleanup(); + } + }, 20_000); +}); diff --git a/packages/engine/src/merger-ai.ts b/packages/engine/src/merger-ai.ts index 1cc387acc8..7d369dd014 100644 --- a/packages/engine/src/merger-ai.ts +++ b/packages/engine/src/merger-ai.ts @@ -113,6 +113,21 @@ export function isBenignAbsentWorktreeError(err: unknown): boolean { return /is not a working tree|No such file or directory|spawn\s+.*\bENOENT\b/i.test(description); } +function getAiMergeTempSearchRoots(): string[] { + const roots = [tmpdir()]; + const testWorkerRoot = process.env.FUSION_TEST_WORKER_ROOT; + if (testWorkerRoot) { + try { + for (const entry of readdirSync(testWorkerRoot)) { + if (entry.startsWith("redir-")) roots.push(join(testWorkerRoot, entry)); + } + } catch { + // Best effort for the test harness' bounded temp-dir redirection root. + } + } + return Array.from(new Set(roots)); +} + export async function pruneExistingAiMergeWorktrees( taskId: string, projectRootDir: string, @@ -120,18 +135,21 @@ export async function pruneExistingAiMergeWorktrees( log: (message: string) => Promise, ): Promise { const prefix = `fusion-ai-merge-${taskId.toLowerCase()}-`; - const tempRoot = tmpdir(); - let entries: string[]; - try { - entries = readdirSync(tempRoot).filter((entry) => entry.startsWith(prefix)); - } catch (err: unknown) { - await log(`AI merge pre-merge prune: failed to read ${tempRoot}: ${getErrorMessage(err)}`); - throw err; - } + const tempRoots = getAiMergeTempSearchRoots(); let pruned = 0; - for (const entry of entries) { - const candidatePath = join(tempRoot, entry); + for (const tempRoot of tempRoots) { + let entries: string[]; + try { + entries = readdirSync(tempRoot).filter((entry) => entry.startsWith(prefix)); + } catch (err: unknown) { + await log(`AI merge pre-merge prune: failed to read ${tempRoot}: ${getErrorMessage(err)}`); + if (tempRoot === tmpdir()) throw err; + continue; + } + + for (const entry of entries) { + const candidatePath = join(tempRoot, entry); let canonicalPath = candidatePath; try { canonicalPath = realpathSync(candidatePath); @@ -188,6 +206,7 @@ export async function pruneExistingAiMergeWorktrees( await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: false, error, ...(code ? { code } : {}) } }); } } + } return pruned; } @@ -203,44 +222,75 @@ export async function cleanupAiMergeWorktree(input: { rmRunner?: typeof rm; }): Promise { const { taskId, mergeRoot, projectRootDir, worktreeAdded, audit, log, gitRunner = git, rmRunner = rm } = input; + let canonicalRoot = mergeRoot; + try { + canonicalRoot = realpathSync(mergeRoot); + } catch { + canonicalRoot = mergeRoot; + } + const removalTargets = canonicalRoot === mergeRoot ? [mergeRoot] : [canonicalRoot, mergeRoot]; + const cleanupMetadata = { taskId, mergeRoot: canonicalRoot, requestedMergeRoot: mergeRoot }; let alreadyAbsent = false; + if (worktreeAdded) { - if (!existsSync(mergeRoot)) { + if (!existsSync(canonicalRoot) && !existsSync(mergeRoot)) { alreadyAbsent = true; - await log(`AI merge cleanup: worktree ${mergeRoot} was already absent before git removal; treating cleanup as idempotent`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: true, alreadyAbsent: true, idempotent: true, code: "ENOENT" } }); + await log(`AI merge cleanup: worktree ${canonicalRoot} was already absent before git removal; treating cleanup as idempotent`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalRoot, metadata: { ...cleanupMetadata, phase: "git-remove", success: true, alreadyAbsent: true, idempotent: true, code: "ENOENT" } }); } else { try { - await gitRunner(["worktree", "remove", "--force", mergeRoot], projectRootDir); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: true } }); + await gitRunner(["worktree", "remove", "--force", canonicalRoot], projectRootDir); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalRoot, metadata: { ...cleanupMetadata, phase: "git-remove", success: true } }); } catch (err: unknown) { const error = describeCleanupError(err); const code = getErrorStringProperty(err, "code"); if (isBenignAbsentWorktreeError(err)) { alreadyAbsent = true; - await log(`AI merge cleanup: worktree ${mergeRoot} was already absent/de-registered during git removal; treating cleanup as idempotent`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: true, alreadyAbsent: true, idempotent: true, error, ...(code ? { code } : {}) } }); + await log(`AI merge cleanup: worktree ${canonicalRoot} was already absent/de-registered during git removal; treating cleanup as idempotent`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalRoot, metadata: { ...cleanupMetadata, phase: "git-remove", success: true, alreadyAbsent: true, idempotent: true, error, ...(code ? { code } : {}) } }); } else { - await log(`AI merge cleanup: git worktree remove failed for ${mergeRoot}${code ? ` (${code})` : ""}: ${error}`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: false, error, ...(code ? { code } : {}) } }); + await log(`AI merge cleanup: git worktree remove failed for ${canonicalRoot}${code ? ` (${code})` : ""}: ${error}`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalRoot, metadata: { ...cleanupMetadata, phase: "git-remove", success: false, error, ...(code ? { code } : {}) } }); } } } } - try { - await rmRunner(mergeRoot, { recursive: true, force: true }); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: true, ...(alreadyAbsent ? { alreadyAbsent: true, idempotent: true } : {}) } }); - } catch (err: unknown) { - const error = getErrorMessage(err); - const code = getErrorStringProperty(err, "code"); - if (isBenignAbsentWorktreeError(err)) { - await log(`AI merge cleanup: worktree ${mergeRoot} was already absent during filesystem cleanup; treating cleanup as idempotent`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: true, alreadyAbsent: true, idempotent: true, error, ...(code ? { code } : {}) } }); - return; + + let removedFromFilesystem = false; + for (const target of removalTargets) { + try { + await rmRunner(target, { recursive: true, force: true }); + await audit.git({ type: "merge:ai-worktree-cleanup", target, metadata: { ...cleanupMetadata, phase: "fs-rm", path: target, success: true, ...(alreadyAbsent ? { alreadyAbsent: true, idempotent: true } : {}) } }); + removedFromFilesystem = true; + break; + } catch (err: unknown) { + const error = getErrorMessage(err); + const code = getErrorStringProperty(err, "code"); + if (isBenignAbsentWorktreeError(err)) { + await log(`AI merge cleanup: worktree ${target} was already absent during filesystem cleanup; treating cleanup as idempotent`); + await audit.git({ type: "merge:ai-worktree-cleanup", target, metadata: { ...cleanupMetadata, phase: "fs-rm", path: target, success: true, alreadyAbsent: true, idempotent: true, error, ...(code ? { code } : {}) } }); + removedFromFilesystem = true; + break; + } + await log(`AI merge cleanup: filesystem rm failed for ${target}${code ? ` (${code})` : ""}: ${error}`); + await audit.git({ type: "merge:ai-worktree-cleanup", target, metadata: { ...cleanupMetadata, phase: "fs-rm", path: target, success: false, error, ...(code ? { code } : {}) } }); } - await log(`AI merge cleanup: filesystem rm failed for ${mergeRoot}${code ? ` (${code})` : ""}: ${error}`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: false, error, ...(code ? { code } : {}) } }); } + + if (!removedFromFilesystem) { + await log(`AI merge cleanup: filesystem cleanup did not remove ${canonicalRoot}; continuing to prune worktree metadata`); + } + + try { + await gitRunner(["worktree", "prune"], projectRootDir, { timeout: 30_000 }); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalRoot, metadata: { ...cleanupMetadata, phase: "git-prune", success: true } }); + } catch (err: unknown) { + const error = describeCleanupError(err); + const code = getErrorStringProperty(err, "code"); + await log(`AI merge cleanup: git worktree prune failed after removing ${canonicalRoot}${code ? ` (${code})` : ""}: ${error}`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalRoot, metadata: { ...cleanupMetadata, phase: "git-prune", success: false, error, ...(code ? { code } : {}) } }); + } + } const FUSION_TASK_ID_TRAILER_KEY = "Fusion-Task-Id";