From db8e715452719d87d2cfbebdd2bb8ce79b1d9f25 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 14 Aug 2026 23:14:19 -0700 Subject: [PATCH] FN-9056: reclaim safe terminal workspace worktrees Reclaim safely abandoned workspace worktrees and canonical task branches without disrupting live or recoverable tasks. - Tear down eligible complete, idle failed, and soft-deleted per-repository worktrees with bounded retries. - Veto live, paused, scheduled-recovery, ambiguous, and unsafe-path cleanup candidates. - Verify failed-task landing evidence against each repository integration branch before deleting canonical branches. - Cover liveness and stale landed-SHA safety regressions. Files changed: .changeset/fn-9056-workspace-terminal-teardown.md | 7 + AGENTS.md | 2 +- docs/architecture.md | 1 + .../src/__tests__/self-healing-workspace.test.ts | 319 ++++++++++++++++++++- .../engine/src/executor/cleanup-task-worktree.ts | 8 +- packages/engine/src/self-healing.ts | 268 ++++++++++++----- packages/engine/src/util/run-audit.ts | 7 +- 7 files changed, 525 insertions(+), 87 deletions(-) Fusion-Task-Id: FN-9056 Fusion-Task-Lineage: 4353f7d9-063b-442b-88d3-6f3da1c9aae8 Co-authored-by: Fusion (runfusion.ai) --- .../fn-9056-workspace-terminal-teardown.md | 7 + AGENTS.md | 2 +- docs/architecture.md | 1 + .../__tests__/self-healing-workspace.test.ts | 319 +++++++++++++++++- .../src/executor/cleanup-task-worktree.ts | 8 +- packages/engine/src/self-healing.ts | 278 ++++++++++----- packages/engine/src/util/run-audit.ts | 7 +- 7 files changed, 530 insertions(+), 92 deletions(-) create mode 100644 .changeset/fn-9056-workspace-terminal-teardown.md diff --git a/.changeset/fn-9056-workspace-terminal-teardown.md b/.changeset/fn-9056-workspace-terminal-teardown.md new file mode 100644 index 0000000000..72fe4f5280 --- /dev/null +++ b/.changeset/fn-9056-workspace-terminal-teardown.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Reclaim stale workspace worktrees and safe task branches after terminal tasks. +category: fix +dev: reconcileOrphanedWorkspaceWorktrees now bounds prune-only retries and skips duplicate claims. diff --git a/AGENTS.md b/AGENTS.md index 0bc544840b..81e073e56a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -305,7 +305,7 @@ Scoped exception (FN-5819/FN-8823): while project auto-merge is On, shared-branc - FN-6793/FN-6797: self-healing emits `task:reconcile-in-review-unmet-dependencies` when it rebounds an `in-review` task whose declared dependencies are still unmet, and `task:reconcile-in-review-unmet-dependencies-no-action` when pause/user-pause, `autoMerge:false`, live execution/checkout proof, or a failed rebound mutation blocks that backward move. - Workspace (Phase D U1): self-healing emits `task:reconcile-workspace-partial-land` when it re-enqueues a partial/zero-landed workspace task's per-repo land (or parks it `failed` for proven branch absence or exhausted `evidence-unavailable` branch reads), and `task:reconcile-workspace-partial-land-no-action` when `autoMerge:false`, user-pause, a live sub-repo worktree (workspace-aware liveness), or `evidence-unavailable` blocks that backward move. The bounded evidence-exhaustion reason is `evidence-unavailable-exhausted`; audit metadata remains ids/counts/outcomes-only. - Workspace (Phase D U1): self-healing emits `task:reclaim-phantom-workspace-land-lease` when it clears a leaked `workspace-repo-land` lease whose owning task is terminal/dead and older than the FN-6736 staleness floor. Archived-role and soft-deleted owners are terminal; live merging, executing, or merge-pending owners are untouched. -- Workspace (Phase D U1): self-healing emits `task:reconcile-orphaned-workspace-worktree` when it removes a done/dead workspace task's recorded per-repo worktree from its stored `worktreePath` (guarded by `isPathActive`; no temp-root walk). +- FN-9056: self-healing emits `task:reconcile-orphaned-workspace-worktree` when it reclaims a complete-lane or conservatively-idle failed/soft-deleted workspace entry. It vetoes raw/canonical active paths, task-session/executor/merge liveness, pauses and scheduled recovery; archived rows remain archive-lifecycle-owned. It runs `git worktree prune` even for already-gone paths and deletes only safely-discardable canonical `fusion/` branches. Duplicate, foreign, unowned, or outside-root claims are skipped without git work; one entry-scoped `MAX_STARVATION_DROPS` budget plus settlement bounds retries. Metadata is ids/counts/fixed outcomes: task/repo/path, success/reason/lane, worktree/prune/branch outcomes, and attempt. - FN-8144: archive emits `archive-workspace-worktree-disposer-missing` when a workspace archive has no store-scoped backend disposer; per-repository archive removal is awaited under canonical-path reservations, with failed paths quarantined for successor reconciliation. - FN-7514: the planner overseer's per-task oversight loop (`PlannerRecoveryController.tick`) emits `overseer:oversight-withheld-human-control` when the pure `evaluateOverseerHumanControl` guard withholds ALL oversight action (no steering, retry, targeted-fix, or pending confirmation) for a task that is user-paused (`task.userPaused===true`, or `task.paused===true` with no `pausedReason`) or ineligible for auto-merge processing per `allowsAutoMergeProcessing` (`autoMerge:false`/PR-based human-review terminal contract). The guard runs BEFORE FN-7513's confirmation classification, so a withheld task never records a pending confirmation. Metadata: `{ taskId, reason: "user-paused" | "auto-merge-off-human-review", stage, oversightLevel }`; deduped per (taskId, withheld reason) so it is not re-emitted every poll while the reason is unchanged. - FN-7720: `TaskStore.bypassFailedPreMergeReviewStep` emits `task:bypass-review` when a privileged operator bypasses the latest failed pre-merge review step of an `in-review` task; metadata includes `workflowStepId`, `workflowStepName`, `bypassedFromStatus`, `bypassedFromVerdict`, and the mandatory `reason`. The bypass rewrites the step's `status` to `"skipped"` with `bypassedBy`/`bypassedAt`/`bypassReason`/`bypassedFromStatus` fields; it never fabricates a reviewer `verdict` and clears only the failed-pre-merge-step `getTaskMergeBlocker` reason. Reachable via `fn_task_bypass_review` (CLI/pi-extension operator tool surface only — not executor/reviewer/triage) and `POST /tasks/:id/bypass-review`. diff --git a/docs/architecture.md b/docs/architecture.md index e7d38995f5..a59f238bf9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -712,6 +712,7 @@ Runtime action-gate flow (v1): - Durable agent error recovery (FN-7835/FN-7844/FN-7859/FN-7878/FN-7884): a heartbeat-managed, runtime-enabled non-ephemeral agent that lands in `state:"error"` remains timer-eligible and clears `lastError` by transitioning `error → active` at the next heartbeat run entry when `lastError` is recoverable. Generic/unknown errors are recoverable by default; immediate `error-unrecoverable` parking is reserved for operator-actionable auth/model/billing/quota failures, while stale worktree/module-resolution errors stay on their dedicated self-healing suppression/rebuild path. Recovery is bounded by one shared `heartbeatErrorRecovery` attempt budget (`MAX_HEARTBEAT_ERROR_RECOVERY_ATTEMPTS`, settings-overridable through the engine's optional cast-based knob) across both the timer path and `SelfHealingManager.recoverOrphanedAgents()`. Self-healing is the stale-agent backstop and still stores `durableErrorRecovery` cooldown/stale-module metadata, but it writes/reads the shared heartbeat counter and emits the same `agent:auto-recover-error-state` / `agent:error-retry-exhausted` audit surface with `source:"self-healing"`. The sweep flips `error → active` before `restartDurableAgentHeartbeat()` calls `executeHeartbeat()`, preventing run-entry recovery from re-counting or double-emitting for the same recovery. Success resets the shared counter and clears legacy sweep retry state; budget exhaustion parks the agent `paused` with `pauseReason:"error-retry-exhausted"`. On engine startup, `SelfHealingManager.resetDurableAgentErrorStateOnStartup()` runs before the steady-state sweep and treats restart as an explicit operator retry: eligible `error` and `error-retry-exhausted` durable agents have shared/legacy retry metadata reset, `lastError` and the exhaustion pause cleared, state set to `active`, heartbeat re-armed, and `agent:reset-error-state-on-startup` emitted without applying the sweep's staleness/cooldown/exhaustion gates. Non-recoverable durable heartbeat errors are not restarted; timer, startup, and sweep paths preserve exclusions for disabled runtime agents, ephemeral agents, active executions, user pauses, `error-unrecoverable` parks, operator-actionable errors, and stale worktree/module-resolution suppression. - Reports Health Check (FN-8569): engine-side `classifyReportHealth` treats any non-empty `pauseReason` as authoritative over `state`, so a live-looking row with an `error-unrecoverable`, retry-exhausted, or model-unavailable park marker is rendered operator-actionable rather than healthy. The classifier deliberately excludes `lastError`, which remains diagnostic history; `@fusion/core` must not import this engine helper. `AgentStore.updateAgentState()` performs best-effort `pauseReason` cleanup only when resuming from `paused`/`error` into a live state, preserves `lastError`, and does not make state/marker writes atomic because independent marker writers can still create a desynced row. - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions + - **Workspace terminal worktree reclamation (FN-9056):** `reconcileOrphanedWorkspaceWorktrees()` removes addressable per-repo worktrees for complete-lane rows and only conservatively-idle failed or soft-deleted rows; archived rows remain owned by archive lifecycle. It vetoes raw and canonical active paths, task/executor/merge liveness, operator pauses, and scheduled recovery. A forensic claim index rejects duplicate, foreign, outside-root, and unowned repo paths before invoking git. Successful removal (or an already-gone path) always runs `git worktree prune`; canonical `fusion/` branches are deleted only with landed/operator-delete/zero-ahead proof. One entry-scoped `MAX_STARVATION_DROPS` budget covers remove, prune, evidence, and branch operations; completed or exhausted entries settle so periodic maintenance cannot repeat git work indefinitely. - Batch 1 maintenance includes `reconcile-orphaned-task-dirs` (FN-6783), a paused-safe housekeeping step that calls `TaskStore.reconcileOrphanedTaskDirs()` so valid live `.fusion/tasks/{ID}/task.json` records missing from PostgreSQL become visible without waiting for process restart. The guard skips any ID already present in active, soft-deleted, archived, or tombstoned storage and emits `task:reconcile-orphaned-task-dir` only for recovered rows. - Batch 1 `prune-github-check-states` removes expired project-scoped event-driven CI state on a six-hour per-project cadence. It is scheduled rather than delivery-driven so the 14-day retention still applies after webhooks stop; failures only log and never interrupt maintenance. - Batch 1 maintenance also includes `reconcile-phantom-committed-reservations` (FN-7069), which calls `TaskStore.reconcilePhantomCommittedReservations()` for committed task-ID reservations that have no live/soft-deleted/archived task row and no `.fusion/tasks/{ID}/task.json`. The sweep prunes orphaned `activityLog` rows and `agents`/cascaded `agentRuns`, preserves `runAuditEvents`, and keeps the reservation `committed` per FN-5105 so the ID is permanently reserved rather than resurrected or handed out again. diff --git a/packages/engine/src/__tests__/self-healing-workspace.test.ts b/packages/engine/src/__tests__/self-healing-workspace.test.ts index 22188fc8a2..578b345eb2 100644 --- a/packages/engine/src/__tests__/self-healing-workspace.test.ts +++ b/packages/engine/src/__tests__/self-healing-workspace.test.ts @@ -20,12 +20,12 @@ Surfaces (FN-5893): import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { EventEmitter } from "node:events"; import { execSync } from "node:child_process"; -import { existsSync, writeFileSync } from "node:fs"; +import { existsSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { Settings, Task, TaskStore } from "@fusion/core"; import { SelfHealingManager } from "../self-healing.js"; import { classifyBranchProbeError } from "../self-healing-git-evidence.js"; -import { activeSessionRegistry } from "../agents/active-session-registry.js"; +import { activeSessionRegistry, executingTaskLock } from "../agents/active-session-registry.js"; import { landWorkspaceTask } from "../merge/merger-ai.js"; import { createWorkspaceFixture, hasGit, type WorkspaceFixture } from "./_workspace-fixture.js"; @@ -58,8 +58,8 @@ function createStore(rows: Task[], settings: Partial = {}): TaskStore emitted, enqueued, getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 60_000, ...settings } as unknown as Settings), - listTasks: vi.fn(async (opts?: { column?: string }) => { - const all = [...tasks.values()]; + listTasks: vi.fn(async (opts?: { column?: string; includeDeleted?: boolean }) => { + const all = [...tasks.values()].filter((task) => opts?.includeDeleted || !task.deletedAt); return opts?.column ? all.filter((t) => t.column === opts.column) : all; }), getTask: vi.fn(async (id: string) => tasks.get(id) ?? null), @@ -121,6 +121,18 @@ class UnavailableBranchProbeManager extends SelfHealingManager { } } +class PruneFailureWorkspaceTeardownManager extends SelfHealingManager { + pruneCalls = 0; + + protected override async execWorkspaceTeardownGit(command: string, options: { cwd: string; timeout: number }): Promise<{ stdout: string }> { + if (command === "git worktree prune") { + this.pruneCalls++; + throw new Error("injected prune failure"); + } + return super.execWorkspaceTeardownGit(command, options); + } +} + /** Add a real `fusion/` branch in a sub-repo with one non-conflicting own commit. */ function addRepoBranch(fx: WorkspaceFixture, repoRel: string, content: string): void { const repoDir = fx.repoPath(repoRel); @@ -168,6 +180,7 @@ describeIfGit("workspace-aware self-healing (Phase D U1)", () => { }); afterEach(() => { activeSessionRegistry.clear(); + executingTaskLock.release(TASK_ID); vi.useRealTimers(); vi.clearAllMocks(); fx?.cleanup(); @@ -750,7 +763,7 @@ describeIfGit("workspace-aware self-healing (Phase D U1)", () => { { column: "done" }, ); // Mark repo-b's worktree as active → it must be SKIPPED. - activeSessionRegistry.registerPath(wtB, { taskId: TASK_ID, kind: "executor", ownerKey: "x" }); + activeSessionRegistry.registerPath(wtB, { taskId: "FN-other", kind: "executor", ownerKey: "x" }); const store = createStore([task]); const manager = makeManager(store, fx.rootDir); @@ -762,6 +775,302 @@ describeIfGit("workspace-aware self-healing (Phase D U1)", () => { expect(existsSync(wtB)).toBe(true); // active → skipped }); + it("keeps a complete-lane workspace task's worktree while its executor is live", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-complete-live"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH } }, { column: "done" }); + activeSessionRegistry.registerPath(worktreePath, { taskId: TASK_ID, kind: "executor", ownerKey: "live" }); + + expect(await makeManager(createStore([task]), fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + }); + + it("tears down an idle failed workspace worktree and its safely landed branch", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-terminal"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const landedSha = fx.git("repo-a", "git rev-parse HEAD").trim(); + const task = workspaceTask( + { "repo-a": { worktreePath, branch: BRANCH, landedSha } }, + { status: "failed", updatedAt: old, columnMovedAt: old }, + ); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(existsSync(worktreePath)).toBe(false); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toBe(""); + expect(store.updateTask).toHaveBeenCalled(); + }); + + /* + FNXC:Workspace 2026-08-15-05:33: + Failed and soft-deleted workspace rows are destructive candidates only after their one-day floor. + These real-git cases lock the worktree/prune/branch policy so terminal cleanup cannot regress into + either leaking abandoned repositories or destroying an unlanded failed-task branch. + */ + it("tears down a soft-deleted workspace worktree and branch as operator-discarded", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-deleted"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH } }, { deletedAt: old, updatedAt: old, columnMovedAt: old }); + const manager = makeManager(createStore([task]), fx.rootDir); + + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(existsSync(worktreePath)).toBe(false); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toBe(""); + }); + + it("retains a failed-task branch when its recorded landed SHA is not reachable from integration", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-stale-landed-sha"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "not-a-real-commit" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + + expect(await makeManager(createStore([task]), fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(existsSync(worktreePath)).toBe(false); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toContain(BRANCH); + }); + + it("retains an unlanded failed-task branch while retiring only its worktree path", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const repo = fx.repoPath("repo-a"); + const baseCommitSha = fx.git("repo-a", "git rev-parse HEAD").trim(); + const worktreePath = path.join(repo, ".wt-unlanded"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + configureIdentity(worktreePath); + writeFileSync(path.join(worktreePath, "unlanded.txt"), "keep\n"); + execSync("git add unlanded.txt && git commit -m unlanded", { cwd: worktreePath, stdio: "pipe" }); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, baseCommitSha } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([task]); + + expect(await makeManager(store, fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(existsSync(worktreePath)).toBe(false); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toContain(BRANCH); + expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, expect.objectContaining({ + workspaceWorktrees: expect.objectContaining({ "repo-a": expect.objectContaining({ branch: BRANCH, baseCommitSha, worktreePath: "" }) }), + })); + }); + + it("prunes an already-gone recorded worktree and settles its absent branch", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-prune-only"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + rmSync(worktreePath, { recursive: true, force: true }); + expect(fx.git("repo-a", "git worktree list --porcelain")).toContain(worktreePath); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const landedSha = fx.git("repo-a", "git rev-parse HEAD").trim(); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(fx.git("repo-a", "git worktree list --porcelain")).not.toContain(worktreePath); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toBe(""); + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(0); + }); + + it("settles an already-absent safe branch without spending the retry budget", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-absent-branch"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + fx.git("repo-a", `git worktree remove --force ${worktreePath}`); + fx.git("repo-a", `git branch -D ${BRANCH}`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, expect.objectContaining({ + workspaceWorktrees: expect.objectContaining({ "repo-a": expect.objectContaining({ branch: BRANCH, worktreePath: "" }) }), + })); + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(0); + }); + + it("skips a terminal path claimed by a live workspace row without settling it", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-shared"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const terminal = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const live = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH } }, { id: "FN-7002", column: "in-progress", status: null }); + const store = createStore([terminal, live]); + + expect(await makeManager(store, fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("skips duplicate repo-entry claims from the same terminal task before git work", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-duplicate"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ + "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" }, + "repo-b": { worktreePath, branch: BRANCH, landedSha: "landed" }, + }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([task]); + + expect(await makeManager(store, fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + /* + FNXC:Workspace 2026-08-15-05:39: + Each liveness veto below starts from a real failed worktree that the positive terminal cases prove + removable. Keeping these cases isolated means deleting one guard makes its own destructive-path + regression fail instead of being hidden by another veto. + */ + it.each([ + "raw-path-session", "resolved-path-session", "task-session-path", "executing-lock", "task-active", "merge-pending", "active-merge", "paused", "user-paused", "recovery-scheduled", + ])("keeps a proven terminal worktree when the %s veto alone is present", async (veto) => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-veto"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const recordedPath = veto === "resolved-path-session" + ? path.join(fx.repoPath("repo-a"), ".wt-veto-alias") + : worktreePath; + if (recordedPath !== worktreePath) symlinkSync(worktreePath, recordedPath, "dir"); + const extra: Partial = { status: "failed", updatedAt: old, columnMovedAt: old, landedSha: undefined }; + if (veto === "paused") extra.paused = true; + if (veto === "user-paused") extra.userPaused = true; + if (veto === "recovery-scheduled") extra.nextRecoveryAt = new Date(Date.now() + 60_000).toISOString(); + const task = workspaceTask({ "repo-a": { worktreePath: recordedPath, branch: BRANCH, landedSha: "landed" } }, extra); + const companionPath = path.join(fx.repoPath("repo-a"), ".wt-veto-companion"); + const companionBranch = "fusion/fn-7002"; + fx.git("repo-a", `git worktree add -b ${companionBranch} ${companionPath} HEAD`); + const companionLandedSha = fx.git("repo-a", "git rev-parse HEAD").trim(); + const companion = workspaceTask({ "repo-a": { worktreePath: companionPath, branch: companionBranch, landedSha: companionLandedSha } }, { id: "FN-7002", status: "failed", updatedAt: old, columnMovedAt: old }); + const options: Record = {}; + if (veto === "raw-path-session") activeSessionRegistry.registerPath(recordedPath, { taskId: "FN-elsewhere", kind: "executor", ownerKey: "raw" }); + // Register the physical path while the row records a symlink alias: only canonical lookup can veto. + if (veto === "resolved-path-session") activeSessionRegistry.registerPath(realpathSync(worktreePath), { taskId: "FN-elsewhere", kind: "executor", ownerKey: "resolved" }); + if (veto === "task-session-path") activeSessionRegistry.registerPath(fx.repoPath("repo-a"), { taskId: TASK_ID, kind: "executor", ownerKey: "task" }); + if (veto === "executing-lock") executingTaskLock.tryClaim(TASK_ID); + if (veto === "task-active") options.isTaskActive = (id: string) => id === TASK_ID; + if (veto === "merge-pending") options.isMergePending = (id: string) => id === TASK_ID; + if (veto === "active-merge") options.getActiveMergeTaskId = () => TASK_ID; + const store = createStore([task, companion]); + + expect(await makeManager(store, fx.rootDir, options).reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(existsSync(worktreePath)).toBe(true); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toContain(BRANCH); + // Negative scope: one primary veto cannot silently disable teardown of another terminal row. + expect(existsSync(companionPath)).toBe(false); + expect(fx.git("repo-a", `git branch --list ${companionBranch}`).trim()).toBe(""); + expect(store.updateTask).toHaveBeenCalledTimes(1); + }); + + it("settles the prune phase while retaining a duplicate branch claim without re-pruning", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-duplicate-branch"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const first = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + // This stale forensic claim cannot own a second checkout of the same branch, but it must veto + // branch deletion until its task row is gone. + const second = workspaceTask({ "repo-a": { worktreePath: path.join(fx.repoPath("repo-a"), ".missing-claim"), branch: BRANCH } }, { id: "FN-7002", status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([first, second]); + const manager = makeManager(store, fx.rootDir); + + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(existsSync(worktreePath)).toBe(false); + expect(fx.git("repo-a", `git branch --list ${BRANCH}`).trim()).toContain(BRANCH); + const phase = (manager as unknown as { prunedWorkspaceWorktreeTeardowns: Set }).prunedWorkspaceWorktreeTeardowns; + expect(phase.size).toBeGreaterThan(0); + // A second tick sees the same ambiguity but performs no git work for the completed first entry. + expect(await manager.reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(phase.size).toBeGreaterThan(0); + }); + + it("skips terminal paths claimed by two terminal task rows", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-two-terminal"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const one = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const two = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH } }, { id: "FN-7002", status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([one, two]); + expect(await makeManager(store, fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("skips a path attributed to the wrong repo and a path outside the workspace root", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const ownedByA = path.join(fx.repoPath("repo-a"), ".wt-wrong-owner"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${ownedByA} HEAD`); + const outside = path.join(path.dirname(fx.rootDir), ".outside-worktree"); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ + "repo-b": { worktreePath: ownedByA, branch: BRANCH, landedSha: "landed" }, + "repo-a": { worktreePath: outside, branch: "fusion/fn-7001-outside", landedSha: "landed" }, + }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([task]); + expect(await makeManager(store, fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(ownedByA)).toBe(true); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("canonicalizes symlinked duplicate claims before destructive teardown", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-symlink"); + const alias = path.join(fx.repoPath("repo-a"), ".wt-symlink-alias"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + symlinkSync(worktreePath, alias, "dir"); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const first = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const second = workspaceTask({ "repo-a": { worktreePath: alias, branch: BRANCH } }, { id: "FN-7002", status: "failed", updatedAt: old, columnMovedAt: old }); + expect(await makeManager(createStore([first, second]), fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + }); + + it("bounds a prune-only git failure and keeps soft-delete settlement in memory when persistence rejects", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const missingPath = path.join(fx.repoPath("repo-a"), ".gone"); + const broken = workspaceTask({ "repo-a": { worktreePath: missingPath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const brokenStore = createStore([broken]); + const manager = new PruneFailureWorkspaceTeardownManager(brokenStore, managerOptions(brokenStore, fx.rootDir) as never); + // Use the production reconciliation loop with only the git runner narrowed to a failing prune. + for (let attempt = 0; attempt < 4; attempt++) await manager.reconcileOrphanedWorkspaceWorktrees(); + const failures = (manager as unknown as { orphanWorktreeRemovalFailures: Map }).orphanWorktreeRemovalFailures; + expect([...failures.values()]).toEqual([3]); + expect(manager.pruneCalls).toBe(3); + + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-reject-settle"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + rmSync(worktreePath, { recursive: true, force: true }); + const rejected = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { id: "FN-7003", deletedAt: old, updatedAt: old, columnMovedAt: old }); + const store = createStore([rejected]); + store.updateTask.mockRejectedValue(new Error("soft-deleted")); + const settled = makeManager(store, fx.rootDir); + expect(await settled.reconcileOrphanedWorkspaceWorktrees()).toBe(1); + expect(await settled.reconcileOrphanedWorkspaceWorktrees()).toBe(0); + }); + + it.each(["globalPause", "enginePaused"])("short-circuits all workspace teardown for %s", async (pauseFlag) => { + fx = await createWorkspaceFixture(["repo-a"]); + const worktreePath = path.join(fx.repoPath("repo-a"), ".wt-pause"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${worktreePath} HEAD`); + const old = new Date(Date.now() - 25 * 60 * 60_000).toISOString(); + const task = workspaceTask({ "repo-a": { worktreePath, branch: BRANCH, landedSha: "landed" } }, { status: "failed", updatedAt: old, columnMovedAt: old }); + const store = createStore([task], { [pauseFlag]: true }); + + expect(await makeManager(store, fx.rootDir).reconcileOrphanedWorkspaceWorktrees()).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + // ── regression: single-repo task untouched by workspace reconcilers ──────── it("single-repo (non-workspace) task is ignored by the workspace reconcilers", async () => { fx = await createWorkspaceFixture(["repo-a"]); diff --git a/packages/engine/src/executor/cleanup-task-worktree.ts b/packages/engine/src/executor/cleanup-task-worktree.ts index 0e87e613cf..dbf0967a39 100644 --- a/packages/engine/src/executor/cleanup-task-worktree.ts +++ b/packages/engine/src/executor/cleanup-task-worktree.ts @@ -34,7 +34,13 @@ export async function cleanupTaskWorktree( deps.activeWorktrees.delete(taskId); - // FNXC:Workspace 2026-06-21-12:00: KTD1 — in workspace mode the tracked path is the non-git workspace root (browse-only), never a removable worktree. Drop the in-memory tracking above but never remove the root. Per-repo worktree teardown returns in Phase B. + /* + FNXC:Workspace 2026-08-15-05:13: + In workspace mode the tracked path is the browse-only non-git root, never a removable worktree. + Per-repo teardown belongs to SelfHealingManager.reconcileOrphanedWorkspaceWorktrees for complete + and terminal lanes, while archive-lifecycle owns archived rows. A failed task may be retried, so + executor cleanup only drops in-memory tracking and must not discard sub-repo work at this boundary. + */ if (workspaceConfig) { return; } diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 0ef83f4e54..bce7d44c51 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -29,7 +29,7 @@ import { setImmediate as setImmediateCb } from "node:timers"; import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { isAbsolute, join, relative, resolve } from "node:path"; +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; import { type TaskMoveLanes, resolveColumnFlags, IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, hasSharedBranchMemberAutoMergeHold, resolveEffectiveAutoMerge, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, evaluateNoCommitsNoOpFinalize, evaluateCompletedPromotionFailureProvenance, evaluateSkipBypassTaint, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkspaceTask, isSharedBranchGroupMemberIntegration, isLiveSharedBranchGroupMemberIntegration, isNearDuplicateCanonicalInactive, parseExplicitDuplicateMarker, flagTriageDuplicate, isTriageDuplicateKeepAcknowledged, resolveMaxAutoMergeRetries, resolveOptionalStepRevisionBudget, resolveOptionalReviewRevisionBudget, getBuiltinWorkflow, isBuiltinWorkflowId, resolveWorkflowIrForTask, resolveWorkflowIrForTaskWithProvenance, resolveReboundTarget, resolveReboundTargetForTask, columnsWithFlag, resolveLifecycleColumns, resolveTaskLifecycleColumns, workflowHasColumn, planLegacyAdoption, resolveOrphanedPendingStepResults, classifyReviewLease, PLAN_REVIEW_LEASE_STALENESS_MS, DEFAULT_MAX_POST_REVIEW_FIXES, ACTIVE_WORKFLOW_WORK_ITEM_STATES, AWAITING_APPROVAL_PAUSE_REASON, type Agent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult, type WorkflowStepResult, type WorkflowIr, resolveNearDuplicateCanonicalFlags, LEGACY_COLUMN_IDS_BY_ROLE, @@ -571,6 +571,12 @@ const RECONCILE_SCOPE_OVERRIDE_MERGE_ACTIVE_STATUS_SET = new Set(MERGE_A import { classifyTransientMergeError } from "./errors/transient-merge-error-classifier.js"; export { classifyTransientMergeError } from "./errors/transient-merge-error-classifier.js"; const MAX_STARVATION_DROPS = 3; +/* +FNXC:Workspace 2026-08-15-05:13: +Failed workspace tasks are routinely retried with their progress preserved. Terminal teardown therefore +waits a full day, unlike short lease recovery floors, so a transient park cannot discard repo worktrees. +*/ +const TERMINAL_WORKSPACE_WORKTREE_TEARDOWN_MIN_IDLE_MS = 24 * 60 * 60 * 1000; const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000; // DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS now lives in ./merge-active-status.js (imported above) // so the manual Retry gate and this sweep cannot drift apart (FN-8004 follow-up). @@ -704,7 +710,21 @@ export class SelfHealingManager extends SelfHealingGitEvidence { */ private workspacePartialLandDrops: Map = new Map(); private workspacePartialLandEvidenceDefers: Map = new Map(); + /* + FNXC:Workspace 2026-08-15-05:13: + A prune-only entry performs git work even when its recorded directory is permanently absent. The same + MAX_STARVATION_DROPS budget bounds remove, prune, evidence, and branch work; settlement retires a + completed entry when a durable row update is unavailable. + */ private orphanWorktreeRemovalFailures: Map = new Map(); + private settledWorkspaceWorktreeTeardowns = new Set(); + /* + FNXC:Workspace 2026-08-15-05:39: + A duplicate branch claimant must preserve future branch-deletion eligibility, but a completed + worktree/prune phase must not rerun `git worktree prune` every maintenance tick. This phase marker + records that safe half of teardown until the claim index becomes unambiguous or the entry settles. + */ + private prunedWorkspaceWorktreeTeardowns = new Set(); private finalizeUnprovenWarned = new Set(); /* * FNXC:Lifecycle 2026-07-16-10:30: @@ -10448,96 +10468,186 @@ const movedTask = await this.store.moveTask(task.id, completeLane); active, mirroring the temp-dir sweep at the AI-merge worktree guard) so a still-live path is never yanked. Emit `task:reconcile-orphaned-workspace-worktree` per removed path. */ + /* + FNXC:Workspace 2026-08-15-05:39: + Keep terminal teardown on the production timed async-git path, while exposing only this narrow seam + for a deterministic prune-failure regression. Real fixtures continue to prove normal git behavior. + */ + protected async execWorkspaceTeardownGit(command: string, options: { cwd: string; timeout: number }): Promise<{ stdout: string }> { + return execAsync(command, options); + } + async reconcileOrphanedWorkspaceWorktrees(): Promise { try { const settings = await this.store.getSettings(); if (settings.globalPause || settings.enginePaused) return 0; - - // Done workspace tasks are the canonical "safe to clean" set (their lands are finalized). - /* - FNXC:WorkflowResolvedColumns 2026-07-30-21:40 (the query-filter class, thirty-fourth sweep): - Removes the per-repo worktrees a finished workspace task left behind. The literal read meant that - on a renamed board they were never removed — disk held by tasks that finished, growing quietly. - - `complete` only, NOT the terminal union: the comment above calls DONE tasks "the canonical safe to - clean set" precisely because their lands are finalized, and an archived row is a different claim. - No per-card verdict: the filter is `isWorkspaceTask`, not a lane test. - */ - const wsDoneColumns = await resolveProjectColumnsForRoles(this.store, ["complete"]); - const wsDoneById = new Map(); - for (const column of wsDoneColumns) { - for (const entry of await this.store.listTasks({ column, slim: true })) wsDoneById.set(entry.id, entry); - } - const candidates = [...wsDoneById.values()].filter((task) => isWorkspaceTask(task)); - if (candidates.length === 0) return 0; - - let cleaned = 0; - for (const task of candidates) { - const workspaceWorktrees = task.workspaceWorktrees ?? {}; - for (const repoRel of Object.keys(workspaceWorktrees)) { - const worktreePath = workspaceWorktrees[repoRel]?.worktreePath; - if (!worktreePath) continue; - // GUARD: skip an active path (mirror self-healing temp-dir sweep isPathActive guard). - if (activeSessionRegistry.isPathActive(worktreePath)) continue; - // Nothing on disk → nothing to remove (already cleaned). Skip silently; clear any prior - // failure count so a re-created path starts fresh. - if (!existsSync(worktreePath)) { - this.orphanWorktreeRemovalFailures.delete(worktreePath); - continue; - } - /* - FNXC:Workspace 2026-06-22-14:10 (Phase D review E — bounded + observable orphan removal): - A `git worktree remove --force` failure was caught + audit-logged but NOT engine-logged, - and retried EVERY tick FOREVER (a genuinely stuck path pins this sweep indefinitely). Bound - the retry per-path: after MAX_STARVATION_DROPS consecutive failures stop attempting (leave - the path for manual cleanup) and `log.warn` each failure for observability. - */ - if ((this.orphanWorktreeRemovalFailures.get(worktreePath) ?? 0) >= MAX_STARVATION_DROPS) { - continue; // exhausted retries — stop hammering a stuck path. - } - - const repoRootDir = join(this.options.rootDir, repoRel); - let success = false; - let reason = "removed"; - try { - await execAsync(`git worktree remove --force ${shellQuote(worktreePath)}`, { - cwd: repoRootDir, - timeout: 120_000, - }); - success = true; - } catch (err: unknown) { - reason = `git-remove-failed: ${err instanceof Error ? err.message : String(err)}`; - } - try { - await createRunAuditor(this.store, { - runId: generateSyntheticRunId("self-healing-orphaned-workspace-worktree", task.id), - agentId: "self-healing", - taskId: task.id, - taskLineageId: task.lineageId, - phase: "reconcile-orphaned-workspace-worktree", - }).database({ - type: "task:reconcile-orphaned-workspace-worktree", - target: task.id, - metadata: { taskId: task.id, repo: repoRel, worktreePath, success, reason }, - }); - } catch { /* audit best-effort */ } - if (success) { - this.orphanWorktreeRemovalFailures.delete(worktreePath); - log.log(`reconcileOrphanedWorkspaceWorktrees: removed ${worktreePath} (task ${task.id}, repo ${repoRel})`); - cleaned++; - } else { - const failures = (this.orphanWorktreeRemovalFailures.get(worktreePath) ?? 0) + 1; - this.orphanWorktreeRemovalFailures.set(worktreePath, failures); - log.warn(`reconcileOrphanedWorkspaceWorktrees: ${reason} for ${worktreePath} (task ${task.id}, repo ${repoRel}) [${failures}/${MAX_STARVATION_DROPS}]${failures >= MAX_STARVATION_DROPS ? " — giving up; manual cleanup required" : ""}`); - } + const now = Date.now(); + const archivedColumns = new Set(await resolveProjectColumnsForRoles(this.store, ["archived"])); + // One forensic read includes deleted and live rows: live claimants must veto destructive cleanup. + const allRows = await this.store.listTasks({ slim: true, includeDeleted: true }); + const completeColumns = new Set(await resolveProjectColumnsForRoles(this.store, ["complete"])); + // Preserve resolved-column semantics for stores whose forensic list is filter-blind or omits lanes. + for (const column of completeColumns) { + for (const row of await this.store.listTasks({ column, slim: true })) { + if (!allRows.some((known) => known.id === row.id)) allRows.push(row); } } - if (cleaned > 0) log.log(`reconcileOrphanedWorkspaceWorktrees: removed ${cleaned} orphaned per-repo worktree(s)`); + type Lane = "complete" | "failed" | "soft-deleted"; + type Candidate = { task: Task; lane: Lane }; + const candidates: Candidate[] = []; + for (const task of allRows) { + if (!isWorkspaceTask(task)) continue; + if (archivedColumns.has(task.column)) continue; + /* + FNXC:Workspace 2026-08-15-06:11: + Complete-lane placement proves no retry floor is needed, not that ownership has ended. Every + destructive terminal teardown must still yield to pauses, scheduled recovery, executor/task + liveness, and queued or active merge work before considering its lane-specific eligibility. + */ + if (task.paused || task.userPaused || task.nextRecoveryAt + || this.isWorkspaceTaskLive(task).live + || this.options.isMergePending?.(task.id) === true + || this.options.getActiveMergeTaskId?.() === task.id) continue; + if (completeColumns.has(task.column) || task.column === "done") { candidates.push({ task, lane: "complete" }); continue; } + const lane: Lane | null = task.deletedAt ? "soft-deleted" : task.status === "failed" ? "failed" : null; + if (!lane) continue; + const touched = Math.max(Date.parse(task.columnMovedAt ?? "") || 0, Date.parse(task.updatedAt ?? "") || 0, Date.parse(task.deletedAt ?? "") || 0); + if (!touched || now - touched < TERMINAL_WORKSPACE_WORKTREE_TEARDOWN_MIN_IDLE_MS) continue; + if (this.isWorkspaceTaskLive(task).live || this.options.isMergePending?.(task.id) === true || this.options.getActiveMergeTaskId?.() === task.id) continue; + candidates.push({ task, lane }); + } + if (!candidates.length) return 0; + + const canonicalPath = (value: string): string => { + const absolute = resolve(value); + try { return realpathSync(absolute); } catch { + /* + FNXC:Workspace 2026-08-15-05:33: + A manually removed worktree has no leaf to realpath. Canonicalize its existing parent so + /var and /private/var aliases still share one destructive claim and one retry budget. + */ + try { return join(realpathSync(dirname(absolute)), basename(absolute)); } catch { return absolute; } + } + }; + type Claim = { taskId: string; repoRel: string; candidate: boolean }; + const pathClaims = new Map(); + const branchClaims = new Map(); + const candidateIds = new Set(candidates.map(({ task }) => task.id)); + /* + FNXC:Workspace 2026-08-15-05:33: + Terminal cleanup is destructive, so claims are indexed across every forensic row before any + git call. A shared path or branch may still belong to a live row; ambiguity is always skipped. + */ + for (const task of allRows) for (const [repoRel, entry] of Object.entries(task.workspaceWorktrees ?? {})) { + const claim = { taskId: task.id, repoRel, candidate: candidateIds.has(task.id) }; + if (entry?.worktreePath) { + const key = canonicalPath(entry.worktreePath); + pathClaims.set(key, [...(pathClaims.get(key) ?? []), claim]); + } + if (entry?.branch) { + const branchKey = `${repoRel}::${entry.branch}`; + branchClaims.set(branchKey, [...(branchClaims.get(branchKey) ?? []), claim]); + } + } + let cleaned = 0; + for (const { task, lane } of candidates) for (const [repoRel, entry] of Object.entries(task.workspaceWorktrees ?? {})) { + const worktreePath = entry?.worktreePath; + if (!worktreePath) continue; + const pathKey = canonicalPath(worktreePath); + const entryKey = `${task.id}::${repoRel}::${pathKey}`; + if (this.settledWorkspaceWorktreeTeardowns.has(entryKey) || (this.orphanWorktreeRemovalFailures.get(entryKey) ?? 0) >= MAX_STARVATION_DROPS) continue; + const repoRootDir = join(this.options.rootDir, repoRel); + const canonicalRootDir = canonicalPath(this.options.rootDir); + const claims = pathClaims.get(pathKey) ?? []; + const uniqueClaims = new Set(claims.map((claim) => `${claim.taskId}::${claim.repoRel}`)); + // FNXC:Workspace 2026-08-15-05:13: destructive terminal cleanup treats shared, foreign, and + // misattributed paths as ambiguous. Skipping is safer than deleting another row's worktree. + if (uniqueClaims.size !== 1 || claims.some((claim) => !claim.candidate) + || !relative(canonicalRootDir, pathKey) || relative(canonicalRootDir, pathKey).startsWith("..") + || pathKey === canonicalRootDir || pathKey === canonicalPath(repoRootDir) || pathKey === canonicalPath(join(repoRootDir, ".git"))) continue; + const resolvedPath = resolve(worktreePath); + if (activeSessionRegistry.isPathActive(worktreePath) || activeSessionRegistry.isPathActive(resolvedPath) || activeSessionRegistry.isPathActive(pathKey)) continue; + const branch = entry.branch; + const branchKey = branch ? `${repoRel}::${branch}` : ""; + const branchClaimCount = branch ? new Set((branchClaims.get(branchKey) ?? []).map((claim) => claim.taskId)).size : 0; + const pruneCompleted = this.prunedWorkspaceWorktreeTeardowns.has(entryKey); + // While a duplicate claim remains, do not spend maintenance cycles repeatedly pruning an + // already removed worktree. Re-evaluate the cheap in-memory claim index next tick instead. + if (pruneCompleted && branchClaimCount > 1) continue; + let failed = false; + let worktreeGone = pruneCompleted || !existsSync(worktreePath); + let pruned = pruneCompleted; + let branchOutcome = "absent"; + try { + if (!worktreeGone) { + const listing = await this.execWorkspaceTeardownGit("git worktree list --porcelain", { cwd: repoRootDir, timeout: 120_000 }); + const owned = listing.stdout.split("\n").some((line) => line.startsWith("worktree ") && canonicalPath(line.slice(9)) === pathKey); + /* FNXC:Workspace 2026-08-15-05:33: Only the attributed sub-repo may prove a directory removable. */ + if (!owned) continue; + await this.execWorkspaceTeardownGit(`git worktree remove --force ${shellQuote(worktreePath)}`, { cwd: repoRootDir, timeout: 120_000 }); + worktreeGone = !existsSync(worktreePath); + } + // Prune is required even for an already-gone path; git otherwise retains .git/worktrees metadata. + if (!pruneCompleted) { + await this.execWorkspaceTeardownGit("git worktree prune", { cwd: repoRootDir, timeout: 120_000 }); + pruned = true; + this.prunedWorkspaceWorktreeTeardowns.add(entryKey); + } + if (branch && branch === canonicalFusionBranchName(task.id) && worktreeGone) { + if (branchClaimCount > 1) branchOutcome = "retained-duplicate-claim"; + else { + /* + FNXC:Workspace 2026-08-15-06:11: + A failed row's recorded landed SHA is evidence to verify, never an operator discard + instruction. Use the canonical landed predicate against this sub-repo's integration + branch; only soft deletion authorizes discard without land proof. + */ + let safe = Boolean(task.deletedAt); + if (!safe && entry.landedSha) { + const integrationBranch = await resolveIntegrationBranch( + repoRootDir, + { ...settings, integrationBranch: undefined, baseBranch: undefined }, + ); + safe = await isRepoLanded(repoRootDir, integrationBranch, entry.landedSha, task.id, branch); + } + if (!safe && entry.baseCommitSha) { + const count = await this.execWorkspaceTeardownGit(`git rev-list --count ${shellQuote(entry.baseCommitSha)}..${shellQuote(branch)}`, { cwd: repoRootDir, timeout: 120_000 }); + safe = count.stdout.trim() === "0"; + } + if (safe) { + /* FNXC:Workspace 2026-08-15-05:33: An absent ref is already a completed teardown, not retryable failure. */ + const listed = await this.execWorkspaceTeardownGit(`git branch --list ${shellQuote(branch)}`, { cwd: repoRootDir, timeout: 120_000 }); + if (!listed.stdout.trim()) branchOutcome = "absent"; + else { + await this.execWorkspaceTeardownGit(`git branch -D ${shellQuote(branch)}`, { cwd: repoRootDir, timeout: 120_000 }); + branchOutcome = "deleted"; + } + } else branchOutcome = "retained-unlanded"; + } + } else if (branch) branchOutcome = "retained-non-canonical"; + } catch (err: unknown) { failed = true; log.warn(`reconcileOrphanedWorkspaceWorktrees: git teardown failed for ${worktreePath}: ${err instanceof Error ? err.message : String(err)}`); } + const attempt = failed ? (this.orphanWorktreeRemovalFailures.get(entryKey) ?? 0) + 1 : this.orphanWorktreeRemovalFailures.get(entryKey) ?? 0; + if (failed) this.orphanWorktreeRemovalFailures.set(entryKey, attempt); + /* FNXC:Workspace 2026-08-15-05:33: A duplicate branch claim remains eligible after ambiguity clears. */ + const settled = !failed && worktreeGone && pruned && ["deleted", "absent", "retained-unlanded", "retained-non-canonical"].includes(branchOutcome); + if (settled) { + this.orphanWorktreeRemovalFailures.delete(entryKey); this.settledWorkspaceWorktreeTeardowns.add(entryKey); cleaned++; + try { + /* + FNXC:Workspace 2026-08-15-05:33: + Settlement retires only the disposable path. Retained branches preserve their branch/base/ + landed evidence for operator recovery and later safe deletion; deleting the whole entry + would turn a safe retain into a permanent leak. + */ + const worktrees = { ...(task.workspaceWorktrees ?? {}) }; + if (worktrees[repoRel]) worktrees[repoRel] = { ...worktrees[repoRel], worktreePath: "" }; + await this.store.updateTask(task.id, { workspaceWorktrees: worktrees }); + } catch { /* soft-deleted rows may reject best-effort settlement */ } + } + try { await createRunAuditor(this.store, { runId: generateSyntheticRunId("self-healing-orphaned-workspace-worktree", task.id), agentId: "self-healing", taskId: task.id, taskLineageId: task.lineageId, phase: "reconcile-orphaned-workspace-worktree" }).database({ type: "task:reconcile-orphaned-workspace-worktree", target: task.id, metadata: { taskId: task.id, repo: repoRel, worktreePath, success: settled, reason: failed ? "git-teardown-failed" : "settled", lane, worktreeOutcome: worktreeGone ? "gone" : "present", pruned, branch: entry.branch, branchOutcome, attempt } }); } catch { /* audit best-effort */ } + } return cleaned; - } catch (err: unknown) { - log.error(`reconcileOrphanedWorkspaceWorktrees sweep failed: ${err instanceof Error ? err.message : String(err)}`); - return 0; - } + } catch (err: unknown) { log.error(`reconcileOrphanedWorkspaceWorktrees sweep failed: ${err instanceof Error ? err.message : String(err)}`); return 0; } } diff --git a/packages/engine/src/util/run-audit.ts b/packages/engine/src/util/run-audit.ts index 79b77c261b..252e6b4d5c 100644 --- a/packages/engine/src/util/run-audit.ts +++ b/packages/engine/src/util/run-audit.ts @@ -626,7 +626,12 @@ export type DatabaseMutationType = | "task:reconcile-workspace-partial-land-no-action" /** Metadata: { taskId, path, kind: "workspace-repo-land", registeredAt, ageMs, staleBindingAgeFloorMs, ownerColumn, ownerTerminalReason: "missing" | "complete" | "archived" | "deleted" | "failed" } */ | "task:reclaim-phantom-workspace-land-lease" - /** Metadata: { taskId, repo, worktreePath, success, reason } */ + /* + FNXC:Workspace 2026-08-15-05:13: + Metadata: { taskId, repo, worktreePath, success, reason, lane, worktreeOutcome, pruned, branch, + branchOutcome, attempt }. Values are ids/counts/fixed outcomes only; branch cleanup is auditable + without recording repository prose. + */ | "task:reconcile-orphaned-workspace-worktree" /** * FNXC:AgentTaskStateDrift 2026-06-23-08:50: