From 2085610e9e60250eb7f7727f2db25ac37798a353 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 12 Jun 2026 06:49:29 -0700 Subject: [PATCH] FN-6246: move AI merge clean rooms into repo storage Move AI merge clean-room worktrees into a repo-local ignored root while preserving legacy temp cleanup. - Add repo-local `.fusion/ai-merge` resolution and ignore handling for AI merge worktrees. - Sweep both repo-local clean rooms and legacy tempdir worktrees with active-session and age safeguards. - Prune stale git worktree metadata after cleanup and classify missing clean-room errors as transient. - Update cleanup tests, architecture docs, and changeset coverage for the relocation. Files changed: .changeset/fn-6246-ai-merge-cleanroom-relocation.md | 5 + AGENTS.md | 2 +- docs/architecture.md | 6 +- .../merger-ai-cleanup-active-session.test.ts | 22 ++- .../engine/src/__tests__/merger-ai-cleanup.test.ts | 48 +++++- .../ai-merge-worktree-cleanup.test.ts | 26 ++- .../__tests__/self-healing-tempdir-sweep.test.ts | 34 ++++ .../transient-merge-error-classifier.test.ts | 2 + packages/engine/src/merger-ai.ts | 144 ++++++++++------- packages/engine/src/self-healing.ts | 180 ++++++++++++--------- .../engine/src/transient-merge-error-classifier.ts | 14 +- 11 files changed, 325 insertions(+), 158 deletions(-) Fusion-Task-Id: FN-6246 Fusion-Task-Lineage: f6fe8ef2-aa7b-4cc3-903f-b7a5cc165487 --- .../fn-6246-ai-merge-cleanroom-relocation.md | 5 + AGENTS.md | 2 +- docs/architecture.md | 6 +- .../merger-ai-cleanup-active-session.test.ts | 22 ++- .../src/__tests__/merger-ai-cleanup.test.ts | 48 ++++- .../ai-merge-worktree-cleanup.test.ts | 26 ++- .../self-healing-tempdir-sweep.test.ts | 34 ++++ .../transient-merge-error-classifier.test.ts | 2 + packages/engine/src/merger-ai.ts | 146 ++++++++------ packages/engine/src/self-healing.ts | 184 ++++++++++-------- .../src/transient-merge-error-classifier.ts | 14 +- 11 files changed, 328 insertions(+), 161 deletions(-) create mode 100644 .changeset/fn-6246-ai-merge-cleanroom-relocation.md diff --git a/.changeset/fn-6246-ai-merge-cleanroom-relocation.md b/.changeset/fn-6246-ai-merge-cleanroom-relocation.md new file mode 100644 index 0000000000..1451f6a61d --- /dev/null +++ b/.changeset/fn-6246-ai-merge-cleanroom-relocation.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Move AI-merge clean-room worktrees into a repo-local cleanup-exempt root, guard cleanup sweeps by active merge ownership, and classify missing clean-room worktree failures as transient so merges can retry cleanly. diff --git a/AGENTS.md b/AGENTS.md index 94606de4a7..f48195fda7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ Never kill processes on port 4040 and never start test servers on 4040. Use `--p Do not issue a recursive `find` (or any unbounded recursive directory walk) rooted at the OS temp directory — `$TMPDIR`, `/tmp`, or macOS `/var/folders/...` (canonical `/private/var/...`). The temp root can hold an enormous number of entries on CI and long-lived dev hosts, so a broad scan can hang for minutes and pin I/O. -When you need a Fusion temp artifact, target the known prefix directly and list a single level with a prefix filter — never walk the whole temp tree. The canonical bounded pattern is the engine's own sweep: a non-recursive `readdirSync(tmpdir())` filtered by a known prefix such as `fusion-ai-merge-` (`SelfHealingManager.cleanupStaleTempMergeWorktrees()` in `packages/engine/src/self-healing.ts`). Scoped `find` calls under a project worktree or `.fusion/` are fine; only the broad temp-root scan is forbidden. +When you need a Fusion temp artifact, target the known prefix directly and list a single level with a prefix filter — never walk the whole temp tree. The canonical bounded pattern is the engine's own sweep: non-recursive `readdirSync(...)` passes over the repo-local `.fusion/ai-merge/` root plus legacy `tmpdir()` leftovers, filtered by a known prefix such as `fusion-ai-merge-` (`SelfHealingManager.cleanupStaleTempMergeWorktrees()` in `packages/engine/src/self-healing.ts`). Scoped `find` calls under a project worktree or `.fusion/` are fine; only the broad temp-root scan is forbidden. ### Engine Process Rules diff --git a/docs/architecture.md b/docs/architecture.md index 357e47027a..59d4f77fa1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -670,8 +670,8 @@ Runtime action-gate flow (v1): - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - Batch 1 maintenance now includes one `fts-maintenance` step for both search indexes. The live `tasks_fts` branch still runs `merge` every tick, `optimize` every 4th tick, and `rebuild` above `32 MiB` or `1 MiB × live task count`. The archive `archived_tasks_fts` branch is lighter because archive writes are mostly append-only: `merge` every 8th tick, `optimize` every 24th tick, and `rebuild` above `64 MiB` or `512 KiB × archived row count`. Each branch is independently guarded by `fts5Available` and emits `task:fts-maintenance` run-audit telemetry with distinct `target` values (`tasks_fts` vs `archived_tasks_fts`). - - AI merge clean-room worktrees are created under `tmpdir()` as `fusion-ai-merge-fn--` detached worktrees. Inline cleanup runs from `runAiMerge`'s clean-room `finally` for successful lands, empty/no-op finalization, concurrent-advance retries, and thrown/aborted merges. Cleanup canonicalizes the temp path, attempts `git worktree remove --force`, always falls back to filesystem removal, then runs `git worktree prune` so stale or partial registrations (including `git worktree add` failures) do not dangle. Cleanup emits `merge:ai-worktree-cleanup` audit events for git-remove, fs-rm, and prune phases; benign already-absent/de-registered paths are treated as idempotent success, while genuine filesystem-removal failures are logged/audited with `success: false` rather than silently swallowed. - - Batch 1 also sweeps stale AI merge clean-room worktrees under `tmpdir()` whose names start with `fusion-ai-merge-`. `runAiMerge` registers each live clean-room worktree in `activeSessionRegistry` with kind `ai-merge` as soon as the temp directory exists and keeps both raw and canonical paths registered for the duration of the merge, so both the periodic sweep and pre-merge prune defer when either path is active. The default age gate is 2 hours; task-aware cleanup uses a 10-minute grace period for `done`/`archived` tasks and for genuinely missing/deleted task rows, and every removal path is clamped by the same 10-minute minimum-age floor so a freshly created worktree is never reaped. Transient `getTask` lookup failures (for example SQLite busy/parse errors) are not treated as deletion evidence; they log a warning, emit `lookup-error` only if eventually removed, and retain the conservative 2-hour gate. The sweep canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force ` before filesystem removal, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. It is intentionally native even when `worktrunk.enabled` because these temp-dir worktrees are outside the worktrunk-managed project layout. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle. + - AI merge clean-room worktrees are created under the repo-local cleanup-exempt root `.fusion/ai-merge/` as `fusion-ai-merge-fn--` detached worktrees, with `.fusion/ai-merge/` added to the repo's local git exclude when possible so an in-flight clean room does not dirty the integration checkout. Inline cleanup runs from `runAiMerge`'s clean-room `finally` for successful lands, empty/no-op finalization, concurrent-advance retries, and thrown/aborted merges. Cleanup canonicalizes the path, attempts `git worktree remove --force`, always falls back to filesystem removal, then runs `git worktree prune` so stale or partial registrations (including `git worktree add` failures) do not dangle. Cleanup emits `merge:ai-worktree-cleanup` audit events for git-remove, fs-rm, and prune phases; benign already-absent/de-registered paths are treated as idempotent success, while genuine filesystem-removal failures are logged/audited with `success: false` rather than silently swallowed. + - Batch 1 sweeps stale AI merge clean-room worktrees both under the repo-local `.fusion/ai-merge/` root and the legacy `tmpdir()` location for pre-relocation leftovers; candidates are bounded to names starting with `fusion-ai-merge-`. `runAiMerge` registers each live clean-room worktree in `activeSessionRegistry` with kind `ai-merge` as soon as the directory exists and keeps both raw and canonical paths registered for the duration of the merge, so both the periodic sweep and pre-merge prune defer when either path is active (including concurrent same-task merge attempts). The default age gate is 2 hours; task-aware cleanup uses a 10-minute grace period for `done`/`archived` tasks and for genuinely missing/deleted task rows, and every removal path is clamped by the same 10-minute minimum-age floor so a freshly created worktree is never reaped. Transient `getTask` lookup failures (for example SQLite busy/parse errors) are not treated as deletion evidence; they log a warning, emit `lookup-error` only if eventually removed, and retain the conservative 2-hour gate. The sweep canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force ` before filesystem removal, runs `git worktree prune` after cleanup attempts, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. It is intentionally native even when `worktrunk.enabled` because these clean-room worktrees are outside the worktrunk-managed project layout. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle. - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - Mission validation has a dedicated stale-run reaper: startup recovery and Batch 2 maintenance call `reapStaleMissionValidatorRuns()` when wired by the runtime, using `VALIDATOR_RUN_STALE_MAX_AGE_MS` (currently 6 hours). The sweep terminates ownerless `mission_validator_runs.status='running'` rows as `error`, writes the reap reason into `summary`, leaves `lastValidatorRunId` pointing at the now-terminal run, and emits run-audit telemetry with `mutationType: "mission:validator-run-reaped"` plus `runId`/`featureId`/`missionId`/`triggerType`/`elapsedMs` metadata. Active mission features move to `loopState="needs_fix"` + `lastValidatorStatus="error"` unless their parent mission is already `complete`/`archived`. @@ -681,7 +681,7 @@ When stuck-kill retries are exhausted, `checkStuckBudget()` marks the task `stat - `recoverMissingWorktreeReviewFailures()` is a narrow failed-review recovery: only `status: "failed"` `in-review` tasks with the explicit session-start signature `Refusing to start coding agent in missing worktree:` (from `assertValidWorktreeSession()`) are requeued. Recovery clears stale session metadata (`worktree`, `branch`, `sessionFile`, transient failure state), preserves valid step progress/retry counters, logs the auto-recovery reason, and moves the task back to `todo` for a clean retry. - `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`. - `recoverAlreadyMergedReviewTasks()` auto-finalizes retry-exhausted `in-review` tasks when self-healing can prove their work already landed on the merge target. On this landed-content path it clears soft blockers (`paused`, stale `status: "failed"`, and residual `error`) before moving to `done`; true hard blockers (for example incomplete steps, awaiting-user-review, or failed pre-merge workflow steps) still park the task in stable `in-review/failed` state with a blocker error instead of entering an auto-finalize loop. - - `recoverTransientMergeFailures()` handles retry-exhausted `in-review` merge failures only when `classifyTransientMergeError()` returns a bounded transient class: `lease-handoff-target-not-queued`, `spurious-concurrent-advance-same-sha`, or `process-spawn-failure` (`spawn ENOTDIR` / `spawn … ENOENT`). Recovery resets `mergeRetries`, clears transient `status`/`error`, increments `mergeDetails.transientRecoveryCount`, and requeues auto-merge. The budget stays capped by `MAX_TRANSIENT_MERGE_RECOVERIES`; exhausted tasks remain parked with the `merger:transient-failure-budget-exhausted` audit path so real structural failures cannot loop forever. FN-6278 makes this recovery mostly after-the-fact insurance for cwd spawn faults: the merge runner now preflights reuse integration roots and repairs/reacquires missing or de-registered task worktrees before the first git spawn, so a stale `task.worktree` should not consume the transient recovery budget by repeatedly producing `spawn git ENOENT`. + - `recoverTransientMergeFailures()` handles retry-exhausted `in-review` merge failures only when `classifyTransientMergeError()` returns a bounded transient class: `lease-handoff-target-not-queued`, `spurious-concurrent-advance-same-sha`, or `process-spawn-failure` (`spawn ENOTDIR`, `spawn … ENOENT`, or a clean-room path reported as `is not a working tree`). Recovery resets `mergeRetries`, clears transient `status`/`error`, increments `mergeDetails.transientRecoveryCount`, and requeues auto-merge so the next attempt recreates the AI-merge clean room. The budget stays capped by `MAX_TRANSIENT_MERGE_RECOVERIES`; exhausted tasks remain parked with the `merger:transient-failure-budget-exhausted` audit path so real structural failures cannot loop forever. FN-6278 makes this recovery mostly after-the-fact insurance for cwd spawn faults: the merge runner now preflights reuse integration roots and repairs/reacquires missing or de-registered task worktrees before the first git spawn, so a stale `task.worktree` should not consume the transient recovery budget by repeatedly producing `spawn git ENOENT`. - `reconcileTaskWorktreeMetadata()` (FN-4962) reconciles stale `task.worktree`/`task.branch` rows against authoritative `git worktree list --porcelain` branch mappings during startup recovery, periodic maintenance, and completion fan-out. The stage must run before `reclaim-stale-active-branches`: stale rows rebound to live `fusion/` worktrees emit `task:auto-recover-worktree-metadata-rebound`; stale rows with no live branch mapping are nulled (`worktree=null`, `branch=null`, `baseCommitSha` unchanged) and emit `task:auto-recover-worktree-metadata-cleared`. - `recoverInProgressLimbo()` (FN-5219) is the safety net for stranded executor rows: reset/requeue paths must never leave a task in `in-progress` without a runnable execution context. After metadata reconcile, stale `in-progress` tasks with null branch, missing/cleared worktree metadata, no live executor claim, and all-pending steps are audited and moved back to `todo`. diff --git a/packages/engine/src/__tests__/merger-ai-cleanup-active-session.test.ts b/packages/engine/src/__tests__/merger-ai-cleanup-active-session.test.ts index 89458f2169..440cd94166 100644 --- a/packages/engine/src/__tests__/merger-ai-cleanup-active-session.test.ts +++ b/packages/engine/src/__tests__/merger-ai-cleanup-active-session.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { existsSync, mkdirSync, realpathSync, rmSync, utimesSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { pruneExistingAiMergeWorktrees } from "../merger-ai.js"; +import { pruneExistingAiMergeWorktrees, resolveAiMergeRoot } from "../merger-ai.js"; import { activeSessionRegistry } from "../active-session-registry.js"; import { MIN_TEMP_WORKTREE_REAP_AGE_MS } from "../self-healing.js"; import type { RunAuditor } from "../run-audit.js"; @@ -29,8 +29,15 @@ function makeAudit() { return { audit, events }; } -function tempAiMergeDir(name: string): string { - const dir = join(tmpdir(), name); +function tempProjectRoot(): string { + const dir = join(tmpdir(), `fusion-ai-merge-active-session-project-${Math.random().toString(36).slice(2)}-`); + mkdirSync(dir, { recursive: true }); + tracked.add(dir); + return dir; +} + +function tempAiMergeDir(rootDir: string, name: string): string { + const dir = join(resolveAiMergeRoot(rootDir), name); mkdirSync(dir, { recursive: true }); tracked.add(dir); return dir; @@ -43,18 +50,19 @@ function makeAge(path: string, ageMs: number): void { describe("AI merge active-session pruning", () => { it("pruneExistingAiMergeWorktrees skips active-session paths", async () => { - const stale = tempAiMergeDir("fusion-ai-merge-fn-777-active"); + const projectRoot = tempProjectRoot(); + const stale = tempAiMergeDir(projectRoot, "fusion-ai-merge-fn-777-active"); const canonical = realpathSync(stale); - activeSessionRegistry.registerPath(canonical, { taskId: "FN-777", kind: "executor", ownerKey: "FN-777" }); + activeSessionRegistry.registerPath(canonical, { taskId: "FN-777", kind: "ai-merge", ownerKey: "ai-merge:FN-777:attempt-1" }); const { audit, events } = makeAudit(); - await expect(pruneExistingAiMergeWorktrees("FN-777", process.cwd(), audit, vi.fn(async () => undefined))).resolves.toBe(0); + await expect(pruneExistingAiMergeWorktrees("FN-777", projectRoot, audit, vi.fn(async () => undefined))).resolves.toBe(0); expect(existsSync(stale)).toBe(true); expect(events).toEqual([]); activeSessionRegistry.unregisterPath(canonical); makeAge(stale, MIN_TEMP_WORKTREE_REAP_AGE_MS + 1_000); - await expect(pruneExistingAiMergeWorktrees("FN-777", process.cwd(), audit, vi.fn(async () => undefined))).resolves.toBe(1); + await expect(pruneExistingAiMergeWorktrees("FN-777", projectRoot, audit, vi.fn(async () => undefined))).resolves.toBe(1); expect(existsSync(stale)).toBe(false); }); }); diff --git a/packages/engine/src/__tests__/merger-ai-cleanup.test.ts b/packages/engine/src/__tests__/merger-ai-cleanup.test.ts index 65fb75e15d..0a7cbfd47c 100644 --- a/packages/engine/src/__tests__/merger-ai-cleanup.test.ts +++ b/packages/engine/src/__tests__/merger-ai-cleanup.test.ts @@ -4,9 +4,10 @@ import { rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { execSync } from "node:child_process"; -import { cleanupAiMergeWorktree, pruneExistingAiMergeWorktrees, runAiMerge } from "../merger-ai.js"; +import { cleanupAiMergeWorktree, pruneExistingAiMergeWorktrees, resolveAiMergeRoot, runAiMerge } from "../merger-ai.js"; import { activeSessionRegistry } from "../active-session-registry.js"; import { MIN_TEMP_WORKTREE_REAP_AGE_MS } from "../self-healing.js"; +import { classifyTransientMergeError } from "../transient-merge-error-classifier.js"; import type { RunAuditor } from "../run-audit.js"; const fsState = vi.hoisted(() => ({ failReaddirPath: "" })); @@ -117,6 +118,12 @@ function tempAiMergeDir(name: string): string { return dir; } +function tempProjectRoot(): string { + const dir = mkdtempSync(join(tmpdir(), "fusion-ai-merge-project-")); + tracked.add(dir); + return dir; +} + function makeAge(path: string, ageMs: number): void { const old = new Date(Date.now() - ageMs); utimesSync(path, old, old); @@ -257,7 +264,9 @@ describe("AI merge temp worktree cleanup", () => { const { audit, events } = makeAudit(); const logs: string[] = []; - await expect(pruneExistingAiMergeWorktrees("FN-777", process.cwd(), audit, vi.fn(async (message: string) => { logs.push(message); }))).resolves.toBe(1); + const projectRoot = tempProjectRoot(); + + await expect(pruneExistingAiMergeWorktrees("FN-777", projectRoot, audit, vi.fn(async (message: string) => { logs.push(message); }))).resolves.toBe(1); expect(existsSync(stale)).toBe(false); expect(events).toEqual(expect.arrayContaining([ @@ -270,7 +279,9 @@ describe("AI merge temp worktree cleanup", () => { const { audit, events } = makeAudit(); const logs: string[] = []; - await expect(pruneExistingAiMergeWorktrees("FN-777", process.cwd(), audit, vi.fn(async (message: string) => { logs.push(message); }))).resolves.toBe(0); + const projectRoot = tempProjectRoot(); + + await expect(pruneExistingAiMergeWorktrees("FN-777", projectRoot, audit, vi.fn(async (message: string) => { logs.push(message); }))).resolves.toBe(0); expect(existsSync(fresh)).toBe(true); expect(events).toEqual([]); @@ -281,7 +292,9 @@ describe("AI merge temp worktree cleanup", () => { const other = tempAiMergeDir("fusion-ai-merge-fn-778-stale"); const { audit, events } = makeAudit(); - await expect(pruneExistingAiMergeWorktrees("FN-777", process.cwd(), audit, vi.fn(async () => undefined))).resolves.toBe(0); + const projectRoot = tempProjectRoot(); + + await expect(pruneExistingAiMergeWorktrees("FN-777", projectRoot, audit, vi.fn(async () => undefined))).resolves.toBe(0); expect(existsSync(other)).toBe(true); expect(events).toEqual([]); @@ -304,6 +317,9 @@ describe("AI merge temp worktree cleanup", () => { }); expect(observedMergeRoot).toContain("fusion-ai-merge-fn-1-"); + expect(observedMergeRoot).toContain(join(dir, ".fusion", "ai-merge")); + expect(observedMergeRoot.startsWith(join(tmpdir(), "fusion-ai-merge-fn-1-"))).toBe(false); + expect(observedMergeRoot.startsWith(resolveAiMergeRoot(dir))).toBe(true); expect(activeSessionRegistry.pathsForTask("FN-1")).toEqual([]); const cleanupEvents = audits.filter((event) => event.mutationType === "merge:ai-worktree-cleanup"); expect(cleanupEvents).toEqual(expect.arrayContaining([ @@ -330,6 +346,30 @@ describe("AI merge temp worktree cleanup", () => { ])); }); + it("classifies a clean-room deleted mid-merge as transient", async () => { + const { dir } = initRepoWithBranch(); + const { store } = makeStore(); + let observedMergeRoot = ""; + + let thrown: unknown; + try { + await runAiMerge(store, dir, "FN-1", { manual: true }, { + mergeAgent: vi.fn(async (cwd: string) => { + observedMergeRoot = cwd; + rmSync(cwd, { recursive: true, force: true }); + throw Object.assign(new Error("spawn git ENOTDIR"), { code: "ENOTDIR" }); + }), + reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"), + }); + } catch (err: unknown) { + thrown = err; + } + + expect(observedMergeRoot).toContain(join(dir, ".fusion", "ai-merge")); + expect(String(thrown)).toMatch(/ENOENT|ENOTDIR|not a working tree/i); + expect(classifyTransientMergeError(String(thrown))).toBe("process-spawn-failure"); + }); + it("pre-merge prune failure does not abort merge", async () => { const { dir } = initRepoWithBranch(); const { store, logs } = makeStore(); diff --git a/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts b/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts index 69423cfd1f..29161305d0 100644 --- a/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts +++ b/packages/engine/src/__tests__/reliability-interactions/ai-merge-worktree-cleanup.test.ts @@ -1,10 +1,10 @@ import { afterAll, describe, expect, it, vi } from "vitest"; -import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, utimesSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { execSync } from "node:child_process"; import { DEFAULT_SETTINGS, TaskStore, type Settings } from "@fusion/core"; -import { cleanupAiMergeWorktree, runAiMerge } from "../../merger-ai.js"; +import { cleanupAiMergeWorktree, resolveAiMergeRoot, runAiMerge } from "../../merger-ai.js"; import { hasGit } from "./_helpers.js"; import type { RunAuditor } from "../../run-audit.js"; @@ -38,6 +38,14 @@ function tmpAiMergeDirs(taskId: string): string[] { .map((entry) => join(tmpdir(), entry)); } +function localAiMergeDirs(rootDir: string, taskId: string): string[] { + const root = resolveAiMergeRoot(rootDir); + const prefix = aiMergePrefix(taskId); + return readdirSync(root) + .filter((entry) => entry.startsWith(prefix)) + .map((entry) => join(root, entry)); +} + function removeTmpAiMergeDirs(taskId: string): void { for (const dir of tmpAiMergeDirs(taskId)) { try { @@ -49,11 +57,17 @@ function removeTmpAiMergeDirs(taskId: string): void { } function expectNoAiMergeWorktrees(rootDir: string, taskId: string): void { - expect(tmpAiMergeDirs(taskId), `tmpdir entries for ${taskId}`).toEqual([]); + expect(tmpAiMergeDirs(taskId), `legacy tmpdir entries for ${taskId}`).toEqual([]); + expect(localAiMergeDirs(rootDir, taskId), `repo-local AI merge entries for ${taskId}`).toEqual([]); const worktrees = git(rootDir, "worktree list --porcelain"); expect(worktrees).not.toContain(aiMergePrefix(taskId)); } +function makeAge(path: string, ageMs: number): void { + const old = new Date(Date.now() - ageMs); + utimesSync(path, old, old); +} + function realMergeAgent(branch: string) { return vi.fn(async (cwd: string) => { execSync(`git merge --squash ${branch}`, { cwd, stdio: "pipe" }); @@ -108,6 +122,7 @@ async function createFixture(label: string) { branch, cleanup: async () => { removeTmpAiMergeDirs(created.id); + for (const dir of localAiMergeDirs(rootDir, created.id)) rmSync(dir, RM); store.close(); rmSync(rootDir, RM); tracked.delete(rootDir); @@ -237,7 +252,10 @@ describe("FN-6220 AI-merge worktree cleanup lifecycle (real git)", () => { try { commitTaskBranch(rootDir, branch, "feature.txt", "feature work\n"); - const orphanDir = mkdtempSync(join(tmpdir(), aiMergePrefix(taskId))); + const orphanRoot = resolveAiMergeRoot(rootDir); + mkdirSync(orphanRoot, { recursive: true }); + const orphanDir = mkdtempSync(join(orphanRoot, aiMergePrefix(taskId))); + makeAge(orphanDir, 11 * 60_000); expect(existsSync(orphanDir)).toBe(true); await runAiMerge(store, rootDir, taskId, { manual: true, allowDirtyLocalCheckoutSync: true }, { diff --git a/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts b/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts index a03e674737..aef562d042 100644 --- a/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts +++ b/packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts @@ -94,6 +94,12 @@ function tempMergeDir(name = `fusion-ai-merge-fn-1-${Math.random().toString(36). return dir; } +function localMergeDir(name = `fusion-ai-merge-fn-1-${Math.random().toString(36).slice(2)}`): string { + const dir = join(projectRoot, ".fusion", "ai-merge", name); + mkdirSync(dir, { recursive: true }); + return dir; +} + function makeAge(path: string, ageMs: number): void { const old = new Date(Date.now() - ageMs); utimesSync(path, old, old); @@ -141,6 +147,34 @@ describe("SelfHealingManager temp-dir AI merge worktree sweep", () => { ])); }); + it("removes stale repo-local AI merge directories", async () => { + const stale = localMergeDir("fusion-ai-merge-fn-1-localstale"); + makeStale(stale); + const { manager, audits } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(1); + + expect(existsSync(stale)).toBe(false); + expect(sweepAudits(audits)).toEqual(expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ path: realpathSync(join(projectRoot, ".fusion", "ai-merge")) + "/fusion-ai-merge-fn-1-localstale", success: true, reason: "stale" }) }), + ])); + }); + + it("defers active repo-local AI merge directories", async () => { + const stale = localMergeDir("fusion-ai-merge-fn-1-localactive"); + makeStale(stale); + const canonical = realpathSync(stale); + activeSessionRegistry.registerPath(canonical, { taskId: "FN-1", kind: "ai-merge", ownerKey: "ai-merge:FN-1" }); + const { manager, audits } = makeManager(); + + await expect(sweep(manager)).resolves.toBe(0); + + expect(existsSync(stale)).toBe(true); + expect(sweepAudits(audits)).toEqual(expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ path: canonical, success: false, reason: "active-session" }) }), + ])); + }); + it("skips directories younger than the staleness threshold", async () => { const fresh = tempMergeDir(); const { manager } = makeManager(); diff --git a/packages/engine/src/__tests__/transient-merge-error-classifier.test.ts b/packages/engine/src/__tests__/transient-merge-error-classifier.test.ts index 2404cd2063..1b5001c67d 100644 --- a/packages/engine/src/__tests__/transient-merge-error-classifier.test.ts +++ b/packages/engine/src/__tests__/transient-merge-error-classifier.test.ts @@ -14,6 +14,8 @@ describe("classifyTransientMergeError", () => { expect(classifyTransientMergeError("spawn ENOENT")).toBe("process-spawn-failure"); expect(classifyTransientMergeError("Bash tool failed: spawn node ENOTDIR while starting merge verification")) .toBe("process-spawn-failure"); + expect(classifyTransientMergeError("fatal: '/var/folders/x/fusion-ai-merge-fn-1-abc' is not a working tree")) + .toBe("process-spawn-failure"); expect(classifyTransientMergeError("ENOTDIR while reading packages/cli/package.json")) .toBeNull(); diff --git a/packages/engine/src/merger-ai.ts b/packages/engine/src/merger-ai.ts index 7d369dd014..f4485ef3a8 100644 --- a/packages/engine/src/merger-ai.ts +++ b/packages/engine/src/merger-ai.ts @@ -32,10 +32,10 @@ */ import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { existsSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; +import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { buildTaskLineageTrailer, getPrimaryPrInfo, @@ -113,8 +113,29 @@ export function isBenignAbsentWorktreeError(err: unknown): boolean { return /is not a working tree|No such file or directory|spawn\s+.*\bENOENT\b/i.test(description); } -function getAiMergeTempSearchRoots(): string[] { - const roots = [tmpdir()]; +function ensureAiMergeRootIgnored(projectRootDir: string): void { + const excludePath = join(projectRootDir, ".git", "info", "exclude"); + if (!existsSync(excludePath)) return; + try { + const current = readFileSync(excludePath, "utf-8"); + if (!/(?:^|\n)\.fusion\/ai-merge\/(?:\n|$)/.test(current)) { + appendFileSync(excludePath, `${current.endsWith("\n") ? "" : "\n"}.fusion/ai-merge/\n`); + } + } catch { + // Best effort only: cleanup still removes the root contents, and existing + // projects generally ignore .fusion already. + } +} + +export function resolveAiMergeRoot(projectRootDir: string, _settings?: Settings): string { + const root = resolve(projectRootDir, ".fusion", "ai-merge"); + mkdirSync(root, { recursive: true }); + ensureAiMergeRootIgnored(projectRootDir); + return root; +} + +function getAiMergeTempSearchRoots(projectRootDir: string, settings?: Settings): string[] { + const roots = [resolveAiMergeRoot(projectRootDir, settings), tmpdir()]; const testWorkerRoot = process.env.FUSION_TEST_WORKER_ROOT; if (testWorkerRoot) { try { @@ -133,11 +154,13 @@ export async function pruneExistingAiMergeWorktrees( projectRootDir: string, audit: RunAuditor, log: (message: string) => Promise, + settings?: Settings, ): Promise { const prefix = `fusion-ai-merge-${taskId.toLowerCase()}-`; - const tempRoots = getAiMergeTempSearchRoots(); + const tempRoots = getAiMergeTempSearchRoots(projectRootDir, settings); let pruned = 0; + let cleanupAttempted = false; for (const tempRoot of tempRoots) { let entries: string[]; try { @@ -150,62 +173,72 @@ export async function pruneExistingAiMergeWorktrees( for (const entry of entries) { const candidatePath = join(tempRoot, entry); - let canonicalPath = candidatePath; - try { - canonicalPath = realpathSync(candidatePath); - } catch { - canonicalPath = candidatePath; - } + let canonicalPath = candidatePath; + try { + canonicalPath = realpathSync(candidatePath); + } catch { + canonicalPath = candidatePath; + } - if (activeSessionRegistry.isPathActive(canonicalPath) || activeSessionRegistry.isPathActive(candidatePath)) { - await log(`AI merge pre-merge prune: skipping active worktree ${canonicalPath}`); - continue; - } - - try { - const stat = statSync(canonicalPath); - const ageMs = Date.now() - stat.mtimeMs; - if (ageMs < MIN_TEMP_WORKTREE_REAP_AGE_MS) { - await log(`AI merge pre-merge prune: skipping too-new worktree ${canonicalPath} (age ${Math.max(0, Math.round(ageMs))}ms)`); + if (activeSessionRegistry.isPathActive(canonicalPath) || activeSessionRegistry.isPathActive(candidatePath)) { + await log(`AI merge pre-merge prune: skipping active worktree ${canonicalPath}`); continue; } - } catch (err: unknown) { - await log(`AI merge pre-merge prune: failed to stat ${canonicalPath}: ${getErrorMessage(err)} — skipping candidate`); - continue; - } - let alreadyAbsent = false; - try { - await execFileAsync("git", ["worktree", "remove", "--force", canonicalPath], { - cwd: projectRootDir, - timeout: 30_000, - }); - } catch (err: unknown) { - if (isBenignAbsentWorktreeError(err)) { - alreadyAbsent = true; - await log(`AI merge pre-merge prune: worktree ${canonicalPath} was already absent/de-registered; treating cleanup as idempotent`); - } else { - await log(`AI merge pre-merge prune: git worktree remove failed for ${canonicalPath}: ${describeCleanupError(err)} — falling back to filesystem removal`); + try { + const stat = statSync(canonicalPath); + const ageMs = Date.now() - stat.mtimeMs; + if (ageMs < MIN_TEMP_WORKTREE_REAP_AGE_MS) { + await log(`AI merge pre-merge prune: skipping too-new worktree ${canonicalPath} (age ${Math.max(0, Math.round(ageMs))}ms)`); + continue; + } + } catch (err: unknown) { + await log(`AI merge pre-merge prune: failed to stat ${canonicalPath}: ${getErrorMessage(err)} — skipping candidate`); + continue; } - } - try { - rmSync(canonicalPath, { recursive: true, force: true }); - await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: true, ...(alreadyAbsent ? { alreadyAbsent: true, idempotent: true } : {}) } }); - pruned++; - } catch (err: unknown) { - if (isBenignAbsentWorktreeError(err)) { - await log(`AI merge pre-merge prune: worktree ${canonicalPath} was already absent during filesystem cleanup; treating cleanup as idempotent`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: true, alreadyAbsent: true, idempotent: true } }); + let alreadyAbsent = false; + try { + cleanupAttempted = true; + await execFileAsync("git", ["worktree", "remove", "--force", canonicalPath], { + cwd: projectRootDir, + timeout: 30_000, + }); + } catch (err: unknown) { + if (isBenignAbsentWorktreeError(err)) { + alreadyAbsent = true; + await log(`AI merge pre-merge prune: worktree ${canonicalPath} was already absent/de-registered; treating cleanup as idempotent`); + } else { + await log(`AI merge pre-merge prune: git worktree remove failed for ${canonicalPath}: ${describeCleanupError(err)} — falling back to filesystem removal`); + } + } + + try { + cleanupAttempted = true; + rmSync(canonicalPath, { recursive: true, force: true }); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: true, ...(alreadyAbsent ? { alreadyAbsent: true, idempotent: true } : {}) } }); pruned++; - continue; + } catch (err: unknown) { + if (isBenignAbsentWorktreeError(err)) { + await log(`AI merge pre-merge prune: worktree ${canonicalPath} was already absent during filesystem cleanup; treating cleanup as idempotent`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: true, alreadyAbsent: true, idempotent: true } }); + pruned++; + continue; + } + const error = getErrorMessage(err); + const code = getErrorStringProperty(err, "code"); + await log(`AI merge pre-merge prune: filesystem rm failed for ${canonicalPath}${code ? ` (${code})` : ""}: ${error}`); + await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: false, error, ...(code ? { code } : {}) } }); } - const error = getErrorMessage(err); - const code = getErrorStringProperty(err, "code"); - await log(`AI merge pre-merge prune: filesystem rm failed for ${canonicalPath}${code ? ` (${code})` : ""}: ${error}`); - await audit.git({ type: "merge:ai-worktree-cleanup", target: canonicalPath, metadata: { taskId, mergeRoot: canonicalPath, phase: "pre-merge-prune", success: false, error, ...(code ? { code } : {}) } }); } } + + if (cleanupAttempted) { + try { + await execFileAsync("git", ["worktree", "prune"], { cwd: projectRootDir, timeout: 30_000 }); + } catch (err: unknown) { + await log(`AI merge pre-merge prune: git worktree prune failed: ${describeCleanupError(err)}`); + } } return pruned; @@ -997,7 +1030,7 @@ export async function runAiMerge( await setStatus("merging"); try { - const pruned = await pruneExistingAiMergeWorktrees(taskId, projectRootDir, audit, log); + const pruned = await pruneExistingAiMergeWorktrees(taskId, projectRootDir, audit, log, settings); if (pruned > 0) await log(`AI merge: pruned ${pruned} pre-existing worktree(s) for ${taskId}`); } catch (err: unknown) { await log(`AI merge: pre-merge prune failed: ${getErrorMessage(err)}`); @@ -1008,7 +1041,7 @@ export async function runAiMerge( const tipSha = await git(["rev-parse", "--verify", `refs/heads/${integrationBranch}`], projectRootDir); // 1. Clean-room worktree at the integration tip. - const mergeRoot = await mkdtemp(join(tmpdir(), `fusion-ai-merge-${taskId.toLowerCase()}-`)); + const mergeRoot = await mkdtemp(join(resolveAiMergeRoot(projectRootDir, settings), `fusion-ai-merge-${taskId.toLowerCase()}-`)); let worktreeAdded = false; const registeredMergePaths = new Set(); const registerMergeRoot = (pathToRegister: string): void => { @@ -1016,9 +1049,10 @@ export async function runAiMerge( activeSessionRegistry.registerPath(pathToRegister, { taskId, kind: "ai-merge", ownerKey: `ai-merge:${taskId}` }); registeredMergePaths.add(pathToRegister); }; - // Register the tmpdir path as soon as it exists, before `git worktree add`, - // so the self-healing tmpdir sweep cannot reap a just-created clean room in - // the small window before canonical registration is available. + // Register the repo-local clean-room path as soon as it exists, before + // `git worktree add`, so self-healing/pre-merge sweeps cannot reap a + // just-created clean room in the small window before canonical registration + // is available. registerMergeRoot(mergeRoot); try { await git(["worktree", "add", "--detach", mergeRoot, tipSha], projectRootDir); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index e7fef7342b..409b4198ed 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -18,7 +18,7 @@ * - `pruneWorktrees`: defer to backend prune * - `cleanupOrphans`: defer to backend prune/remove semantics * - `reapUnregisteredOrphans`: defer to backend prune/remove semantics - * - `cleanupStaleTempMergeWorktrees`: remains native (temp-dir scope, outside worktrunk layout) + * - `cleanupStaleTempMergeWorktrees`: remains native (repo-local AI-merge root + legacy temp-dir scope, outside worktrunk layout) * - `enforceWorktreeCap`: defer to backend prune/remove semantics * - `reclaimSelfOwnedBranchConflicts`: remains native (branch-level) * - `reclaimStaleActiveBranches`: remains native (branch-level) @@ -101,6 +101,10 @@ function extractTaskIdFromTempMergeDir(dirname: string): string | null { return match?.[1]?.toUpperCase() ?? null; } +function resolveRepoLocalAiMergeRoot(rootDir: string): string { + return resolve(rootDir, ".fusion", "ai-merge"); +} + function getErrorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } @@ -8922,30 +8926,21 @@ export class SelfHealingManager { } /** - * Sweep stale AI merge clean-room worktrees from `tmpdir()`. + * Sweep stale AI merge clean-room worktrees from the repo-local clean-room + * root plus the legacy `tmpdir()` location used by older engine versions. * * These worktrees are intentionally outside the project/worktrunk-managed * `.worktrees/` layout, so this native sweep proceeds even when worktrunk is - * enabled. Safety is bounded by a two-hour age gate plus active-session checks. + * enabled. Safety is bounded by age gates plus active-session checks. */ private async cleanupStaleTempMergeWorktrees(): Promise { try { const settings = await this.store.getSettings(); if (settings.worktrunk?.enabled === true) { - log.log("[self-healing] temp-dir sweep: worktrunk enabled — AI merge temp worktrees are outside worktrunk's managed layout, proceeding with native sweep"); + log.log("[self-healing] temp-dir sweep: worktrunk enabled — AI merge clean-room worktrees are outside worktrunk's managed layout, proceeding with native sweep"); } - const tempRoot = tmpdir(); - let entries: string[]; - try { - entries = readdirSync(tempRoot).filter((entry) => entry.startsWith("fusion-ai-merge-")); - } catch (err: unknown) { - const errorMessage = err instanceof Error ? err.message : String(err); - log.warn(`[self-healing] temp-dir sweep: failed to read ${tempRoot}: ${errorMessage}`); - return 0; - } - if (entries.length === 0) return 0; - + const roots = Array.from(new Set([resolveRepoLocalAiMergeRoot(this.options.rootDir), tmpdir()])); const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("self-heal", "tempdir-sweep"), agentId: "self-healing", @@ -8954,78 +8949,105 @@ export class SelfHealingManager { const now = Date.now(); let cleaned = 0; - for (const entry of entries) { - const path = join(tempRoot, entry); - let canonicalPath = path; - let cleanupReason = "stale"; + for (const tempRoot of roots) { + let entries: string[]; try { - const stat = statSync(path); - if (!stat.isDirectory()) { - await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "not-directory" } }); + entries = readdirSync(tempRoot).filter((entry) => entry.startsWith("fusion-ai-merge-")); + } catch (err: unknown) { + if (!existsSync(tempRoot)) continue; + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: failed to read ${tempRoot}: ${errorMessage}`); + if (tempRoot === tmpdir()) return cleaned; + continue; + } + if (entries.length === 0) continue; + + for (const entry of entries) { + const path = join(tempRoot, entry); + let canonicalPath = path; + let cleanupReason = "stale"; + try { + const stat = statSync(path); + if (!stat.isDirectory()) { + await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "not-directory" } }); + continue; + } + const ageMs = now - stat.mtimeMs; + let ageGateMs = STALE_TEMP_MERGE_WORKTREE_MS; + cleanupReason = "stale"; + const taskId = extractTaskIdFromTempMergeDir(entry); + if (taskId) { + try { + const task = await this.store.getTask(taskId); + if (task.column === "done" || task.column === "archived") { + ageGateMs = DONE_TASK_TEMP_WORKTREE_GRACE_MS; + cleanupReason = "done-task-stale"; + } + } catch (err: unknown) { + if (isTaskNotFoundError(err)) { + ageGateMs = MIN_TEMP_WORKTREE_REAP_AGE_MS; + cleanupReason = "deleted-task"; + } else { + const errorMessage = getErrorMessage(err); + cleanupReason = "lookup-error"; + log.warn(`[self-healing] temp-dir sweep: task lookup failed for ${taskId}: ${errorMessage}; using conservative age gate`); + } + } + } + ageGateMs = Math.max(ageGateMs, MIN_TEMP_WORKTREE_REAP_AGE_MS); + if (ageMs < ageGateMs) continue; + try { + canonicalPath = realpathSync(path); + } catch { + canonicalPath = path; + } + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: failed to stat ${path}: ${errorMessage}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "stat-failed", error: errorMessage } }); continue; } - const ageMs = now - stat.mtimeMs; - let ageGateMs = STALE_TEMP_MERGE_WORKTREE_MS; - cleanupReason = "stale"; - const taskId = extractTaskIdFromTempMergeDir(entry); - if (taskId) { - try { - const task = await this.store.getTask(taskId); - if (task.column === "done" || task.column === "archived") { - ageGateMs = DONE_TASK_TEMP_WORKTREE_GRACE_MS; - cleanupReason = "done-task-stale"; - } - } catch (err: unknown) { - if (isTaskNotFoundError(err)) { - ageGateMs = MIN_TEMP_WORKTREE_REAP_AGE_MS; - cleanupReason = "deleted-task"; - } else { - const errorMessage = getErrorMessage(err); - cleanupReason = "lookup-error"; - log.warn(`[self-healing] temp-dir sweep: task lookup failed for ${taskId}: ${errorMessage}; using conservative age gate`); + + if (activeSessionRegistry.isPathActive(canonicalPath) || activeSessionRegistry.isPathActive(path)) { + log.log(`[self-healing] temp-dir sweep: deferring ${canonicalPath}: active session present`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "active-session" } }); + continue; + } + + let cleanupAttempted = false; + try { + cleanupAttempted = true; + await execAsync(`git worktree remove --force ${shellQuote(canonicalPath)}`, { + cwd: this.options.rootDir, + timeout: 120_000, + }); + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: git worktree remove failed for ${canonicalPath}: ${errorMessage} — falling back to filesystem removal`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "git-remove-failed", error: errorMessage } }); + } + + try { + cleanupAttempted = true; + rmSync(canonicalPath, { recursive: true, force: true }); + log.log(`[self-healing] temp-dir sweep: cleaned stale AI merge worktree ${canonicalPath}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: true, reason: cleanupReason } }); + cleaned++; + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: failed to remove ${canonicalPath}: ${errorMessage}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "fs-rm-failed", error: errorMessage } }); + } finally { + if (cleanupAttempted) { + try { + await execAsync("git worktree prune", { cwd: this.options.rootDir, timeout: 30_000 }); + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] temp-dir sweep: git worktree prune failed after cleaning ${canonicalPath}: ${errorMessage}`); + await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "git-prune-failed", error: errorMessage } }); } } } - ageGateMs = Math.max(ageGateMs, MIN_TEMP_WORKTREE_REAP_AGE_MS); - if (ageMs < ageGateMs) continue; - try { - canonicalPath = realpathSync(path); - } catch { - canonicalPath = path; - } - } catch (err: unknown) { - const errorMessage = err instanceof Error ? err.message : String(err); - log.warn(`[self-healing] temp-dir sweep: failed to stat ${path}: ${errorMessage}`); - await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "stat-failed", error: errorMessage } }); - continue; - } - - if (activeSessionRegistry.isPathActive(canonicalPath) || activeSessionRegistry.isPathActive(path)) { - log.log(`[self-healing] temp-dir sweep: deferring ${canonicalPath}: active session present`); - await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "active-session" } }); - continue; - } - - try { - await execAsync(`git worktree remove --force ${shellQuote(canonicalPath)}`, { - cwd: this.options.rootDir, - timeout: 120_000, - }); - } catch (err: unknown) { - const errorMessage = err instanceof Error ? err.message : String(err); - log.warn(`[self-healing] temp-dir sweep: git worktree remove failed for ${canonicalPath}: ${errorMessage} — falling back to filesystem removal`); - await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "git-remove-failed", error: errorMessage } }); - } - - try { - rmSync(canonicalPath, { recursive: true, force: true }); - log.log(`[self-healing] temp-dir sweep: cleaned stale AI merge worktree ${canonicalPath}`); - await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: true, reason: cleanupReason } }); - cleaned++; - } catch (err: unknown) { - const errorMessage = err instanceof Error ? err.message : String(err); - log.warn(`[self-healing] temp-dir sweep: failed to remove ${canonicalPath}: ${errorMessage}`); - await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "fs-rm-failed", error: errorMessage } }); } } diff --git a/packages/engine/src/transient-merge-error-classifier.ts b/packages/engine/src/transient-merge-error-classifier.ts index 2a933edf2c..3d66546cc9 100644 --- a/packages/engine/src/transient-merge-error-classifier.ts +++ b/packages/engine/src/transient-merge-error-classifier.ts @@ -36,11 +36,12 @@ * * - `process-spawn-failure`: Node/OS process launch failed while the merger * was operating from an integration cwd (`spawn ENOTDIR`, `spawn git ENOENT`, - * `spawn ENOENT`). These indicate the command could not even start because - * the cwd/entrypoint was missing or file-shadowed (for example a stale temp - * merge checkout), not that the task branch's code failed. A fresh merge - * attempt gets a fresh/revalidated worktree, so the self-healing sweep can - * recover these within its bounded retry budget. + * `spawn ENOENT`) or git reported that the AI-merge clean-room path `is not + * a working tree`. These indicate the command could not even start because + * the cwd/entrypoint/worktree was missing or file-shadowed (for example a + * stale temp merge checkout), not that the task branch's code failed. A + * fresh merge attempt gets a fresh/revalidated worktree, so the self-healing + * sweep can recover these within its bounded retry budget. */ export function classifyTransientMergeError(error: string | null | undefined): string | null { if (!error) return null; @@ -50,6 +51,9 @@ export function classifyTransientMergeError(error: string | null | undefined): s if (/\bspawn(?:\s+\S+)?\s+ENO(?:TDIR|ENT)\b/i.test(error)) { return "process-spawn-failure"; } + if (/\bis not a working tree\b/i.test(error)) { + return "process-spawn-failure"; + } const sameSha = error.match(/advanced concurrently \(expected ([0-9a-f]{7,40}),\s+observed ([0-9a-f]{7,40})\)/i); if (sameSha && sameSha[1].toLowerCase() === sameSha[2].toLowerCase()) { return "spurious-concurrent-advance-same-sha";