From 4032a35a2f12a027248d2f337caa1980d9f9da19 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 10 Jun 2026 05:13:40 -0700 Subject: [PATCH] FN-6188: harden AI merge temp worktree cleanup Improve AI merge temp worktree cleanup and add recovery coverage. - audit AI merge clean-room cleanup success and failure paths, including stderr/code details - sweep stale fusion-ai-merge temp directories from tmpdir() during self-healing with active-session safeguards - document the native temp-dir sweep behavior and add merge cleanup/self-healing regression tests Files changed: docs/architecture.md | 1 + .../engine/src/__tests__/merger-ai-cleanup.test.ts | 169 +++++++++++++++++ .../__tests__/self-healing-tempdir-sweep.test.ts | 200 +++++++++++++++++++++ packages/engine/src/merger-ai.ts | 54 +++++- packages/engine/src/run-audit.ts | 26 +++ packages/engine/src/self-healing.ts | 105 +++++++++++ 6 files changed, 551 insertions(+), 4 deletions(-) Fusion-Task-Id: FN-6188 Fusion-Task-Lineage: eae9bcdc-9765-4104-85dc-81bb50445e99 --- docs/architecture.md | 1 + .../src/__tests__/merger-ai-cleanup.test.ts | 169 +++++++++++++++ .../self-healing-tempdir-sweep.test.ts | 200 ++++++++++++++++++ packages/engine/src/merger-ai.ts | 54 ++++- packages/engine/src/run-audit.ts | 26 +++ packages/engine/src/self-healing.ts | 105 +++++++++ 6 files changed, 551 insertions(+), 4 deletions(-) create mode 100644 packages/engine/src/__tests__/merger-ai-cleanup.test.ts create mode 100644 packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 0bba41d2e2..8b2e2e8f17 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -670,6 +670,7 @@ Runtime action-gate flow (v1): - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - Batch 1 maintenance now includes one `fts-maintenance` step for both search indexes. The live `tasks_fts` branch still runs `merge` every tick, `optimize` every 4th tick, and `rebuild` above `32 MiB` or `1 MiB × live task count`. The archive `archived_tasks_fts` branch is lighter because archive writes are mostly append-only: `merge` every 8th tick, `optimize` every 24th tick, and `rebuild` above `64 MiB` or `512 KiB × archived row count`. Each branch is independently guarded by `fts5Available` and emits `task:fts-maintenance` run-audit telemetry with distinct `target` values (`tasks_fts` vs `archived_tasks_fts`). + - Batch 1 also sweeps stale AI merge clean-room worktrees under `tmpdir()` whose names start with `fusion-ai-merge-`. The sweep only considers directories older than 2 hours, canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force ` before filesystem removal, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. It is intentionally native even when `worktrunk.enabled` because these temp-dir worktrees are outside the worktrunk-managed project layout. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle. - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - Mission validation has a dedicated stale-run reaper: startup recovery and Batch 2 maintenance call `reapStaleMissionValidatorRuns()` when wired by the runtime, using `VALIDATOR_RUN_STALE_MAX_AGE_MS` (currently 6 hours). The sweep terminates ownerless `mission_validator_runs.status='running'` rows as `error`, writes the reap reason into `summary`, leaves `lastValidatorRunId` pointing at the now-terminal run, and emits run-audit telemetry with `mutationType: "mission:validator-run-reaped"` plus `runId`/`featureId`/`missionId`/`triggerType`/`elapsedMs` metadata. Active mission features move to `loopState="needs_fix"` + `lastValidatorStatus="error"` unless their parent mission is already `complete`/`archived`. diff --git a/packages/engine/src/__tests__/merger-ai-cleanup.test.ts b/packages/engine/src/__tests__/merger-ai-cleanup.test.ts new file mode 100644 index 0000000000..b81f7d74ae --- /dev/null +++ b/packages/engine/src/__tests__/merger-ai-cleanup.test.ts @@ -0,0 +1,169 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execSync } from "node:child_process"; +import { cleanupAiMergeWorktree, runAiMerge } from "../merger-ai.js"; +import type { RunAuditor } from "../run-audit.js"; + +const tracked = new Set(); +const RM = { recursive: true, force: true, maxRetries: 5, retryDelay: 50 } as const; + +afterEach(() => { + vi.restoreAllMocks(); + for (const dir of tracked) { + try { rmSync(dir, RM); } catch { /* best effort */ } + } + tracked.clear(); +}); + +function git(cwd: string, args: string): string { + return execSync(`git ${args}`, { cwd, encoding: "utf-8" }).trim(); +} + +function makeAudit() { + const events: any[] = []; + const audit: RunAuditor = { + git: vi.fn(async (event: any) => { events.push(event); }), + database: vi.fn(async () => undefined), + filesystem: vi.fn(async () => undefined), + sandbox: vi.fn(async () => undefined), + }; + return { audit, events }; +} + +async function cleanup(input: Partial[0]> = {}) { + const mergeRoot = input.mergeRoot ?? mkdtempSync(join(tmpdir(), "fusion-ai-merge-fn-1-cleanup-test-")); + tracked.add(mergeRoot); + const { audit, events } = makeAudit(); + const logs: string[] = []; + await cleanupAiMergeWorktree({ + taskId: "FN-1", + mergeRoot, + projectRootDir: input.projectRootDir ?? process.cwd(), + worktreeAdded: input.worktreeAdded ?? true, + audit: input.audit ?? audit, + log: input.log ?? vi.fn(async (message: string) => { logs.push(message); }), + gitRunner: input.gitRunner ?? vi.fn(async () => ""), + rmRunner: input.rmRunner ?? rm, + }); + return { mergeRoot, events, logs }; +} + +function initRepoWithBranch(): { dir: string } { + const dir = mkdtempSync(join(tmpdir(), "fusion-ai-merge-cleanup-test-")); + tracked.add(dir); + git(dir, "init -q -b main"); + git(dir, "config user.email t@t.t"); + git(dir, "config user.name t"); + writeFileSync(join(dir, "base.txt"), "base\n"); + git(dir, "add -A"); + git(dir, "commit -q -m base"); + git(dir, "checkout -q -b fusion/fn-1"); + writeFileSync(join(dir, "feature.txt"), "feature work\n"); + git(dir, "add -A"); + git(dir, "commit -q -m 'feat: work'"); + git(dir, "checkout -q main"); + return { dir }; +} + +function makeStore() { + const task: any = { + id: "FN-1", + column: "in-review", + status: null, + branch: "fusion/fn-1", + worktree: null, + title: "do the thing", + steps: [], + }; + const audits: any[] = []; + const logs: string[] = []; + const store: any = { + getTask: vi.fn(async () => task), + getSettings: vi.fn(async () => ({ merger: { mode: "ai", maxReviewPasses: 1 } })), + updateTask: vi.fn(async (_id: string, patch: Record) => { Object.assign(task, patch); return task; }), + moveTask: vi.fn(async (_id: string, column: string) => { task.column = column; return task; }), + emit: vi.fn(), + logEntry: vi.fn(async (_id: string, message: string) => { logs.push(message); }), + appendAgentLog: vi.fn(async (_id: string, message: string) => { logs.push(message); }), + recordRunAuditEvent: vi.fn(async (event: any) => { audits.push(event); }), + }; + return { store, audits, logs }; +} + +function realMergeAgent() { + return vi.fn(async (cwd: string) => { + execSync("git merge --squash fusion/fn-1", { cwd, stdio: "pipe" }); + execSync("git add -A", { cwd, stdio: "pipe" }); + execSync('git commit -q -m "squash: feature"', { cwd, stdio: "pipe" }); + }); +} + +describe("AI merge temp worktree cleanup", () => { + it("emits audit event and logs stderr on git worktree removal failure", async () => { + const err = new Error("git remove failed") as Error & { stderr?: string; code?: string }; + err.stderr = "fatal: simulated worktree remove failure"; + err.code = "1"; + + const { mergeRoot, events, logs } = await cleanup({ gitRunner: vi.fn(async () => { throw err; }) }); + + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: false, error: expect.stringContaining("simulated worktree remove failure"), code: "1" }) }), + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }), + ])); + expect(logs.join("\n")).toContain("simulated worktree remove failure"); + expect(existsSync(mergeRoot)).toBe(false); + }); + + it("emits audit event and logs errno details on filesystem rm failure", async () => { + const err = new Error("simulated filesystem cleanup denial") as NodeJS.ErrnoException; + err.code = "EACCES"; + + const { events, logs } = await cleanup({ rmRunner: vi.fn(async () => { throw err; }) as typeof rm }); + + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: true }) }), + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: false, code: "EACCES", error: expect.stringContaining("simulated filesystem cleanup denial") }) }), + ])); + expect(logs.join("\n")).toContain("EACCES"); + }); + + it("emits success audit events on happy-path cleanup", async () => { + const { events } = await cleanup(); + + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: true }) }), + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }), + ])); + }); + + it("skips git removal but still audits filesystem cleanup when worktree was not added", async () => { + const gitRunner = vi.fn(async () => ""); + + const { events } = await cleanup({ worktreeAdded: false, gitRunner }); + + expect(gitRunner).not.toHaveBeenCalled(); + expect(events.some((event) => event.metadata.phase === "git-remove")).toBe(false); + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }), + ])); + }); + + it("runAiMerge emits success cleanup audit events", async () => { + const { dir } = initRepoWithBranch(); + const { store, audits } = makeStore(); + + await runAiMerge(store, dir, "FN-1", { manual: true }, { + mergeAgent: realMergeAgent(), + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + }); + + const cleanupEvents = audits.filter((event) => event.mutationType === "merge:ai-worktree-cleanup"); + expect(cleanupEvents).toEqual(expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ phase: "git-remove", success: true }) }), + expect.objectContaining({ metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }), + ])); + }); +}); diff --git a/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts b/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts new file mode 100644 index 0000000000..37416f188a --- /dev/null +++ b/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts @@ -0,0 +1,200 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, utimesSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const osState = vi.hoisted(() => ({ tempRoot: "" })); +const fsState = vi.hoisted(() => ({ failRmPath: "", rmCalls: [] as string[] })); +const childState = vi.hoisted(() => ({ execCalls: [] as string[] })); + +vi.mock("node:os", async () => { + const actual = await vi.importActual("node:os"); + return { ...actual, tmpdir: vi.fn(() => osState.tempRoot || actual.tmpdir()) }; +}); + +vi.mock("node:fs", async () => { + const actual = await vi.importActual("node:fs"); + return { + ...actual, + rmSync: vi.fn((path: Parameters[0], options?: Parameters[1]) => { + const pathString = String(path); + fsState.rmCalls.push(pathString); + if (fsState.failRmPath && pathString === fsState.failRmPath) { + const err = new Error("simulated tempdir rm failure") as NodeJS.ErrnoException; + err.code = "EACCES"; + throw err; + } + return actual.rmSync(path, options); + }), + }; +}); + +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { + ...actual, + exec: vi.fn((command: string, optionsOrCallback: unknown, maybeCallback?: unknown) => { + childState.execCalls.push(command); + const callback = typeof optionsOrCallback === "function" ? optionsOrCallback : maybeCallback; + queueMicrotask(() => { + if (typeof callback === "function") callback(null, "", ""); + }); + return {} as ReturnType; + }), + }; +}); + +import { activeSessionRegistry } from "../active-session-registry.js"; +import { SelfHealingManager } from "../self-healing.js"; + +const RM = { recursive: true, force: true, maxRetries: 5, retryDelay: 50 } as const; +let sandboxRoot = ""; +let projectRoot = ""; + +beforeEach(() => { + sandboxRoot = mkdtempSync(join(tmpdir(), "fusion-tempdir-sweep-sandbox-")); + projectRoot = mkdtempSync(join(tmpdir(), "fusion-tempdir-sweep-project-")); + osState.tempRoot = sandboxRoot; + fsState.failRmPath = ""; + fsState.rmCalls = []; + childState.execCalls = []; + activeSessionRegistry.clear(); +}); + +afterEach(() => { + activeSessionRegistry.clear(); + osState.tempRoot = ""; + fsState.failRmPath = ""; + fsState.rmCalls = []; + childState.execCalls = []; + for (const dir of [sandboxRoot, projectRoot]) { + try { rmSync(dir, RM); } catch { /* best effort */ } + } +}); + +function makeStore(settings: Record = {}) { + const audits: any[] = []; + const store: any = { + getSettings: vi.fn(async () => ({ ...settings })), + recordRunAuditEvent: vi.fn(async (event: any) => { audits.push(event); }), + }; + return { store, audits }; +} + +function makeManager(settings: Record = {}) { + const { store, audits } = makeStore(settings); + const manager = new SelfHealingManager(store, { rootDir: projectRoot }); + return { manager, audits }; +} + +function tempMergeDir(name = `fusion-ai-merge-fn-1-${Math.random().toString(36).slice(2)}`): string { + const dir = join(sandboxRoot, name); + mkdirSync(dir, { recursive: true }); + return dir; +} + +function makeStale(path: string): void { + const old = new Date(Date.now() - 3 * 60 * 60 * 1000); + utimesSync(path, old, old); +} + +async function sweep(manager: SelfHealingManager): Promise { + return await (manager as any).cleanupStaleTempMergeWorktrees(); +} + +function sweepAudits(audits: any[]) { + return audits.filter((event) => event.mutationType === "worktree:tempdir-sweep"); +} + +describe("SelfHealingManager temp-dir AI merge worktree sweep", () => { + it("removes stale fusion-ai-merge directories and emits success audits", async () => { + const stale = tempMergeDir(); + makeStale(stale); + const { manager, audits } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(1); + + expect(existsSync(stale)).toBe(false); + expect(sweepAudits(audits)).toEqual(expect.arrayContaining([ + expect.objectContaining({ mutationType: "worktree:tempdir-sweep", metadata: expect.objectContaining({ path: realpathSync(sandboxRoot) + "/" + stale.split("/").pop(), success: true, reason: "stale" }) }), + ])); + }); + + it("skips directories younger than the staleness threshold", async () => { + const fresh = tempMergeDir(); + const { manager } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(0); + + expect(existsSync(fresh)).toBe(true); + }); + + it("skips active session paths and removes them after unregister", async () => { + const stale = tempMergeDir(); + makeStale(stale); + const canonical = realpathSync(stale); + activeSessionRegistry.registerPath(canonical, { taskId: "FN-1", kind: "executor", ownerKey: "FN-1" }); + const { manager, audits } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(0); + expect(existsSync(stale)).toBe(true); + expect(sweepAudits(audits)).toEqual(expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ path: canonical, success: false, reason: "active-session" }) }), + ])); + + activeSessionRegistry.unregisterPath(canonical); + await expect(sweep(manager)).resolves.toBe(1); + expect(existsSync(stale)).toBe(false); + }); + + it("leaves non-fusion-ai-merge directories untouched", async () => { + const other = join(sandboxRoot, "other-temp-dir"); + mkdirSync(other, { recursive: true }); + makeStale(other); + const { manager } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(0); + + expect(existsSync(other)).toBe(true); + }); + + it("attempts git worktree removal before filesystem removal", async () => { + const stale = tempMergeDir(); + makeStale(stale); + const canonical = realpathSync(stale); + const { manager } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(1); + + expect(childState.execCalls[0]).toContain(`git worktree remove --force '${canonical.replace(/'/g, `'"'"'`)}'`); + expect(fsState.rmCalls[0]).toBe(canonical); + }); + + it("continues when one stale directory fails filesystem removal", async () => { + const failing = tempMergeDir("fusion-ai-merge-fn-1-failing"); + const succeeding = tempMergeDir("fusion-ai-merge-fn-1-succeeding"); + makeStale(failing); + makeStale(succeeding); + fsState.failRmPath = realpathSync(failing); + const { manager, audits } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(1); + + expect(existsSync(failing)).toBe(true); + expect(existsSync(succeeding)).toBe(false); + expect(sweepAudits(audits)).toEqual(expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ path: realpathSync(failing), success: false, reason: "fs-rm-failed", error: expect.stringContaining("simulated tempdir rm failure") }) }), + expect.objectContaining({ metadata: expect.objectContaining({ path: expect.stringContaining("succeeding"), success: true, reason: "stale" }) }), + ])); + }); + + it("proceeds when worktrunk is enabled", async () => { + const stale = tempMergeDir(); + makeStale(stale); + const { manager } = makeManager({ worktrunk: { enabled: true } }); + + await expect(sweep(manager)).resolves.toBe(1); + + expect(existsSync(stale)).toBe(false); + }); +}); diff --git a/packages/engine/src/merger-ai.ts b/packages/engine/src/merger-ai.ts index add49dbe2c..f59d6c26a5 100644 --- a/packages/engine/src/merger-ai.ts +++ b/packages/engine/src/merger-ai.ts @@ -87,6 +87,55 @@ async function gitOk(args: string[], cwd: string): Promise { } } +function getErrorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function getErrorStringProperty(err: unknown, key: "stderr" | "code"): string | undefined { + if (!err || typeof err !== "object" || !(key in err)) return undefined; + const value = (err as Record)[key]; + return typeof value === "string" && value.trim() ? value : undefined; +} + +function describeCleanupError(err: unknown): string { + const stderr = getErrorStringProperty(err, "stderr"); + const message = getErrorMessage(err); + return stderr ? `${message}: ${stderr.trim()}` : message; +} + +export async function cleanupAiMergeWorktree(input: { + taskId: string; + mergeRoot: string; + projectRootDir: string; + worktreeAdded: boolean; + audit: RunAuditor; + log: (message: string) => Promise; + gitRunner?: typeof git; + rmRunner?: typeof rm; +}): Promise { + const { taskId, mergeRoot, projectRootDir, worktreeAdded, audit, log, gitRunner = git, rmRunner = rm } = input; + if (worktreeAdded) { + try { + await gitRunner(["worktree", "remove", "--force", mergeRoot], projectRootDir); + await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: true } }); + } catch (err: unknown) { + const error = describeCleanupError(err); + const code = getErrorStringProperty(err, "code"); + await log(`AI merge cleanup: git worktree remove failed for ${mergeRoot}${code ? ` (${code})` : ""}: ${error}`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: false, error, ...(code ? { code } : {}) } }); + } + } + try { + await rmRunner(mergeRoot, { recursive: true, force: true }); + await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: true } }); + } catch (err: unknown) { + const error = getErrorMessage(err); + const code = getErrorStringProperty(err, "code"); + await log(`AI merge cleanup: filesystem rm failed for ${mergeRoot}${code ? ` (${code})` : ""}: ${error}`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: false, error, ...(code ? { code } : {}) } }); + } +} + const FUSION_TASK_ID_TRAILER_KEY = "Fusion-Task-Id"; /** Trailers that associate the squash commit with its board task: the @@ -836,10 +885,7 @@ export async function runAiMerge( await log(`AI merge: advanced ${integrationBranch} → ${short(squashSha)} (local checkout: ${landed.localSync})`); return await finalizeMerged(store, projectRootDir, taskId, task, branch, integrationBranch, squashSha, audit, log, { empty: false }); } finally { - if (worktreeAdded) { - await gitOk(["worktree", "remove", "--force", mergeRoot], projectRootDir); - } - await rm(mergeRoot, { recursive: true, force: true }).catch(() => undefined); + await cleanupAiMergeWorktree({ taskId, mergeRoot, projectRootDir, worktreeAdded, audit, log }); } } } diff --git a/packages/engine/src/run-audit.ts b/packages/engine/src/run-audit.ts index a8cec71a20..21465a436f 100644 --- a/packages/engine/src/run-audit.ts +++ b/packages/engine/src/run-audit.ts @@ -167,6 +167,32 @@ export type GitMutationType = | "merge:ai-review-landed-with-concerns" | "merge:ai-local-sync" | "merge:ai-landed" + /** + * Metadata shape: + * ```ts + * { + * taskId: string; + * mergeRoot: string; + * phase: "git-remove" | "fs-rm"; + * success: boolean; + * error?: string; + * code?: string; + * } + * ``` + */ + | "merge:ai-worktree-cleanup" + /** + * Metadata shape: + * ```ts + * { + * path: string; + * success: boolean; + * reason?: "stale" | "active-session" | "git-remove-failed" | "fs-rm-failed" | "not-directory" | "stat-failed"; + * error?: string; + * } + * ``` + */ + | "worktree:tempdir-sweep" | "merge:reuse-handoff-acquired" | "merge:reuse-handoff-refused" | "merge:reuse-handoff-released" diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 53878b3044..5e8b1be703 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -18,6 +18,7 @@ * - `pruneWorktrees`: defer to backend prune * - `cleanupOrphans`: defer to backend prune/remove semantics * - `reapUnregisteredOrphans`: defer to backend prune/remove semantics + * - `cleanupStaleTempMergeWorktrees`: remains native (temp-dir scope, outside worktrunk layout) * - `enforceWorktreeCap`: defer to backend prune/remove semantics * - `reclaimSelfOwnedBranchConflicts`: remains native (branch-level) * - `reclaimStaleActiveBranches`: remains native (branch-level) @@ -27,6 +28,7 @@ import { exec, execSync } from "node:child_process"; import { promisify } from "node:util"; import { setImmediate as setImmediateCb } from "node:timers"; import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { isAbsolute, join, relative, resolve } from "node:path"; import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core"; import type { MeshLeaseManager } from "./mesh-lease-manager.js"; @@ -73,6 +75,7 @@ const BOARD_STALL_NOTIFICATION_COOLDOWN_MS = 60 * 60_000; const DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS = 60 * 60 * 1000; const FTS_MAINTENANCE_MERGE_CADENCE_TICKS = 1; const FTS_MAINTENANCE_OPTIMIZE_CADENCE_TICKS = 4; +const STALE_TEMP_MERGE_WORKTREE_MS = 2 * 60 * 60 * 1000; // Live pathology peaked around 775 KB/task (~96 MB for ~120 tasks), while a // rebuilt healthy index was ~0.1 MB. Keep the steady-state budget generous but // bounded so sustained text churn heals before segment growth becomes material. @@ -1808,6 +1811,16 @@ export class SelfHealingManager { const batch1Fns: Array<{ name: string; fn: () => Promise }> = [ { name: "prune-worktrees", fn: () => this.pruneWorktrees() }, { name: "cleanup-orphans", fn: () => this.cleanupOrphans() }, + { + name: "cleanup-stale-temp-merge-worktrees", + fn: async () => { + const cleaned = await this.cleanupStaleTempMergeWorktrees(); + if (cleaned > 0) { + log.log(`Cleaned ${cleaned} stale AI merge temp worktree(s)`); + } + return cleaned; + }, + }, { name: "cleanup-orphaned-branches", fn: () => this.cleanupOrphanedBranches() }, { name: "cleanup-old-chats", @@ -8838,6 +8851,98 @@ export class SelfHealingManager { return cleaned; } + /** + * Sweep stale AI merge clean-room worktrees from `tmpdir()`. + * + * These worktrees are intentionally outside the project/worktrunk-managed + * `.worktrees/` layout, so this native sweep proceeds even when worktrunk is + * enabled. Safety is bounded by a two-hour age gate plus active-session checks. + */ + private async cleanupStaleTempMergeWorktrees(): Promise { + try { + const settings = await this.store.getSettings(); + if (settings.worktrunk?.enabled === true) { + log.log("[self-healing] temp-dir sweep: worktrunk enabled — AI merge temp worktrees are outside worktrunk's managed layout, proceeding with native sweep"); + } + + const tempRoot = tmpdir(); + let entries: string[]; + try { + entries = readdirSync(tempRoot).filter((entry) => entry.startsWith("fusion-ai-merge-")); + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: failed to read ${tempRoot}: ${errorMessage}`); + return 0; + } + if (entries.length === 0) return 0; + + const auditor = createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-heal", "tempdir-sweep"), + agentId: "self-healing", + phase: "tempdir-sweep", + }); + const now = Date.now(); + let cleaned = 0; + + for (const entry of entries) { + const path = join(tempRoot, entry); + let canonicalPath = path; + try { + const stat = statSync(path); + if (!stat.isDirectory()) { + await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "not-directory" } }); + continue; + } + const ageMs = now - stat.mtimeMs; + if (ageMs < STALE_TEMP_MERGE_WORKTREE_MS) continue; + try { + canonicalPath = realpathSync(path); + } catch { + canonicalPath = path; + } + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: failed to stat ${path}: ${errorMessage}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "stat-failed", error: errorMessage } }); + continue; + } + + if (activeSessionRegistry.isPathActive(canonicalPath) || activeSessionRegistry.isPathActive(path)) { + log.log(`[self-healing] temp-dir sweep: deferring ${canonicalPath}: active session present`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "active-session" } }); + continue; + } + + try { + await execAsync(`git worktree remove --force ${shellQuote(canonicalPath)}`, { + cwd: this.options.rootDir, + timeout: 120_000, + }); + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: git worktree remove failed for ${canonicalPath}: ${errorMessage} — falling back to filesystem removal`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "git-remove-failed", error: errorMessage } }); + } + + try { + rmSync(canonicalPath, { recursive: true, force: true }); + log.log(`[self-healing] temp-dir sweep: cleaned stale AI merge worktree ${canonicalPath}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: true, reason: "stale" } }); + cleaned++; + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: failed to remove ${canonicalPath}: ${errorMessage}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "fs-rm-failed", error: errorMessage } }); + } + } + + return cleaned; + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.error(`[self-healing] temp-dir sweep failed: ${errorMessage}`); + return 0; + } + } /** * Resolve orphaned `fusion/*` branches.