diff --git a/.changeset/workspace-phase-d-self-healing.md b/.changeset/workspace-phase-d-self-healing.md new file mode 100644 index 0000000000..6d412dd404 --- /dev/null +++ b/.changeset/workspace-phase-d-self-healing.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": minor +--- + +Workspace mode Phase D (U1): workspace-aware self-healing. The existing merging-status reconcilers no longer mis-finalize a partial-landed workspace task (recoverInterruptedMergingTasks now clears the transient `merging` status and re-enqueues the idempotent per-repo land instead of running the single-commit finalize over the non-git workspace root), and recoverMergeableReviewTasks now admits workspace tasks (task.worktree is null). Adds three reconcilers: partial-land recovery (re-enqueue via enqueueMerge, FORK-A unrecoverable → park failed; guarded by autoMerge:false + user-pause + workspace-aware liveness), phantom `workspace-repo-land` lease reclaim (new `entriesByKind` registry seam), and per-repo worktree cleanup from stored paths (no temp walk). New run-audit events: `task:reconcile-workspace-partial-land`(`-no-action`), `task:reclaim-phantom-workspace-land-lease`, `task:reconcile-orphaned-workspace-worktree`. diff --git a/AGENTS.md b/AGENTS.md index f68a9512d4..1e45475137 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -195,6 +195,9 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme - FN-6783: task-store open and self-healing housekeeping emit `task:reconcile-orphaned-task-dir` when they non-destructively re-import a valid live `.fusion/tasks/{ID}/task.json` directory that has no task row anywhere, preserving soft-deleted/archived/tombstoned IDs. - FN-6782/FN-6796: self-healing emits `task:auto-recover-paused-abort-park` when it clears a benign pause-abort operator park, requeueing safe `todo`/`in-progress` rows or preserving a clean auto-merge-eligible `in-review` row for review progression. - FN-6793/FN-6797: self-healing emits `task:reconcile-in-review-unmet-dependencies` when it rebounds an `in-review` task whose declared dependencies are still unmet, and `task:reconcile-in-review-unmet-dependencies-no-action` when pause/user-pause, `autoMerge:false`, live execution/checkout proof, or a failed rebound mutation blocks that backward move. +- Workspace (Phase D U1): self-healing emits `task:reconcile-workspace-partial-land` when it re-enqueues a partial/zero-landed workspace task's per-repo land (or parks it `failed` when a sub-repo's `fusion/` branch is gone with no `landedSha`), and `task:reconcile-workspace-partial-land-no-action` when `autoMerge:false`, user-pause, or a live sub-repo worktree (workspace-aware liveness) blocks that backward move. +- Workspace (Phase D U1): self-healing emits `task:reclaim-phantom-workspace-land-lease` when it clears a leaked `workspace-repo-land` lease whose owning task is terminal/dead and older than the FN-6736 staleness floor (a live merging owner is left untouched). +- Workspace (Phase D U1): self-healing emits `task:reconcile-orphaned-workspace-worktree` when it removes a done/dead workspace task's recorded per-repo worktree from its stored `worktreePath` (guarded by `isPathActive`; no temp-root walk). ## Reference docs (deeper detail) diff --git a/packages/engine/src/__tests__/self-healing-workspace.test.ts b/packages/engine/src/__tests__/self-healing-workspace.test.ts new file mode 100644 index 0000000000..ecd32a7892 --- /dev/null +++ b/packages/engine/src/__tests__/self-healing-workspace.test.ts @@ -0,0 +1,417 @@ +/* +FNXC:Workspace 2026-06-22-09:30 (Phase D U1 — workspace-aware self-healing): +Exercises the workspace-aware self-healing reconcilers against a REAL two-repo git fixture under +a NON-git workspace root (createWorkspaceFixture), so a leaked rootDir git preflight or a +single-commit finalize over the non-git root would actually fail. Real git is used only where the +invariant requires it (per-repo landedSha ancestor check, FORK-A branch-gone check, per-repo +worktree removal); fake timers drive the FN-6736 phantom-lease staleness floor. No mock-the-world +child_process, no unbounded temp walk, never touches port 4040. + +Surfaces (FN-5893): +- P0: a PARTIAL-landed workspace task stuck "merging" with no live holder → recoverInterruptedMergingTasks + does NOT finalize it done (no single-commit finalize); the partial-land reconciler re-enqueues. +- P1: a zero-landed mergeable workspace task → recoverMergeableReviewTasks re-enqueues (not skipped by worktree gate). +- guards: autoMerge:false / user-paused / a live sub-repo worktree → -no-action, not moved backward. +- phantom: a workspace-repo-land lease with a terminal owner older than the floor → reclaimed; live owner → untouched. +- cleanup: a done task's recorded per-repo worktrees → removed (isPathActive-guarded); no temp walk. +- FORK-A: branch-gone + landedSha-unset → parked failed; branch-gone + landedSha-set → skipped as landed. +- regression: a single-repo (non-workspace) task → reconcilers behave identically. +*/ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EventEmitter } from "node:events"; +import { execSync } from "node:child_process"; +import { existsSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import type { Settings, Task, TaskStore } from "@fusion/core"; +import { SelfHealingManager } from "../self-healing.js"; +import { activeSessionRegistry } from "../active-session-registry.js"; +import { landWorkspaceTask } from "../merger-ai.js"; +import { createWorkspaceFixture, hasGit, type WorkspaceFixture } from "./_workspace-fixture.js"; + +const describeIfGit = hasGit ? describe : describe.skip; + +const TASK_ID = "FN-7001"; +const BRANCH = "fusion/fn-7001"; + +function configureIdentity(dir: string): void { + execSync('git config user.email "test@example.com"', { cwd: dir, stdio: "pipe" }); + execSync('git config user.name "Test"', { cwd: dir, stdio: "pipe" }); +} + +interface RecordingStore extends EventEmitter { + tasks: Map; + emitted: Array<{ event: string; payload: unknown }>; + enqueued: string[]; + updateTask: ReturnType; + moveTask: ReturnType; +} + +function createStore(rows: Task[], settings: Partial = {}): TaskStore & RecordingStore { + const emitter = new EventEmitter(); + const tasks = new Map(rows.map((t) => [t.id, t])); + const emitted: Array<{ event: string; payload: unknown }> = []; + const enqueued: string[] = []; + const realEmit = emitter.emit.bind(emitter); + const store = Object.assign(emitter, { + tasks, + emitted, + enqueued, + getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false, taskStuckTimeoutMs: 60_000, ...settings } as unknown as Settings), + listTasks: vi.fn(async (opts?: { column?: string }) => { + const all = [...tasks.values()]; + return opts?.column ? all.filter((t) => t.column === opts.column) : all; + }), + getTask: vi.fn(async (id: string) => tasks.get(id) ?? null), + updateTask: vi.fn(async (id: string, patch: Partial) => { + const cur = tasks.get(id); + if (cur) tasks.set(id, { ...cur, ...patch } as Task); + return tasks.get(id) as Task; + }), + moveTask: vi.fn(async (id: string, column: string) => { + const cur = tasks.get(id); + const next = { ...(cur ?? { id }), column } as Task; + tasks.set(id, next); + return next; + }), + logEntry: vi.fn().mockResolvedValue(undefined), + appendAgentLog: vi.fn().mockResolvedValue(undefined), + recordRunAuditEvent: vi.fn().mockResolvedValue(undefined), + peekMergeQueue: vi.fn().mockReturnValue([]), + getRootDir: vi.fn().mockReturnValue("/tmp/test"), + emit: (event: string, payload?: unknown) => { + emitted.push({ event, payload }); + return realEmit(event, payload); + }, + }) as unknown as TaskStore & RecordingStore; + return store; +} + +function makeManager(store: TaskStore, rootDir: string, opts: Record = {}): SelfHealingManager { + const enqueueMerge = (taskId: string) => { + (store as unknown as RecordingStore).enqueued.push(taskId); + return true; + }; + return new SelfHealingManager(store, { + rootDir, + enqueueMerge, + clearMergeActive: vi.fn(), + ...opts, + } as never); +} + +/** Add a real `fusion/` branch in a sub-repo with one non-conflicting own commit. */ +function addRepoBranch(fx: WorkspaceFixture, repoRel: string, content: string): void { + const repoDir = fx.repoPath(repoRel); + const wt = path.join(repoDir, ".wt-branch"); + fx.git(repoRel, `git worktree add -b ${BRANCH} ${wt} HEAD`); + configureIdentity(wt); + writeFileSync(path.join(wt, "feature.txt"), content, "utf-8"); + execSync("git add feature.txt", { cwd: wt, stdio: "pipe" }); + execSync(`git commit -m "feat(${TASK_ID}): add"`, { cwd: wt, stdio: "pipe" }); + fx.git(repoRel, `git worktree remove --force ${wt}`); +} + +/** Land one sub-repo for real (squash onto main) and return its landedSha. */ +function landRepoForReal(fx: WorkspaceFixture, repoRel: string): string { + const repoDir = fx.repoPath(repoRel); + configureIdentity(repoDir); + execSync(`git merge --squash ${BRANCH}`, { cwd: repoDir, stdio: "pipe" }); + execSync(`git commit -m "feat(${TASK_ID}): landed\n\nFusion-Task-Id: ${TASK_ID}"`, { cwd: repoDir, stdio: "pipe" }); + return fx.git(repoRel, "git rev-parse refs/heads/main"); +} + +function workspaceTask(workspaceWorktrees: Task["workspaceWorktrees"], extra: Partial = {}): Task { + return { + id: TASK_ID, + title: "Workspace task", + column: "in-review", + branch: BRANCH, + worktree: null, + dependencies: [], + steps: [], + currentStep: 0, + log: [], + paused: false, + workspaceWorktrees, + createdAt: new Date().toISOString(), + updatedAt: new Date(Date.now() - 10 * 60_000).toISOString(), + ...extra, + } as unknown as Task; +} + +describeIfGit("workspace-aware self-healing (Phase D U1)", () => { + let fx: WorkspaceFixture; + beforeEach(() => { + activeSessionRegistry.clear(); + }); + afterEach(() => { + activeSessionRegistry.clear(); + vi.useRealTimers(); + vi.clearAllMocks(); + fx?.cleanup(); + }); + + // ── KTD1 P0: partial-landed "merging" task must NOT be finalized done ────── + it("recoverInterruptedMergingTasks does NOT finalize a partial-landed workspace task (P0)", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + addRepoBranch(fx, "repo-a", "a\n"); + addRepoBranch(fx, "repo-b", "b\n"); + const landedA = landRepoForReal(fx, "repo-a"); // repo A landed; repo B NOT. + + const task = workspaceTask( + { + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH, landedSha: landedA }, + "repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH }, + }, + { status: "merging", updatedAt: new Date(Date.now() - 30 * 60_000).toISOString() }, + ); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + await manager.recoverInterruptedMergingTasks(); + + // NOT finalized done; status cleared; never emitted task:merged on a single repo. + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.emitted.some((e) => e.event === "task:merged")).toBe(false); + expect(store.tasks.get(TASK_ID)?.status).toBeNull(); + expect(store.tasks.get(TASK_ID)?.column).toBe("in-review"); + // It re-enqueued the per-repo land for idempotent completion. + expect(store.enqueued).toContain(TASK_ID); + }); + + it("partial-land reconciler re-enqueues a partial-landed workspace task", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + addRepoBranch(fx, "repo-a", "a\n"); + addRepoBranch(fx, "repo-b", "b\n"); + const landedA = landRepoForReal(fx, "repo-a"); + + const task = workspaceTask({ + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH, landedSha: landedA }, + "repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH }, + }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + const n = await manager.reconcileWorkspacePartialLands(); + + expect(n).toBe(1); + expect(store.enqueued).toContain(TASK_ID); + // Not moved backward / not parked failed (repo B branch still exists → retryable). + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + }); + + // ── KTD1 P1: zero-landed mergeable workspace task admitted ───────────────── + it("recoverMergeableReviewTasks re-enqueues a zero-landed mergeable workspace task (P1)", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const task = workspaceTask({ + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH }, + "repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH }, + }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + await manager.recoverMergeableReviewTasks(); + + expect(store.enqueued).toContain(TASK_ID); + }); + + // ── KTD2 guards: never move backward when human-gated / live ─────────────── + it("partial-land reconciler emits -no-action for autoMerge:false (not moved backward)", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const task = workspaceTask({ + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH }, + "repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH }, + }); + const store = createStore([task], { autoMerge: false }); + const manager = makeManager(store, fx.rootDir); + + const n = await manager.reconcileWorkspacePartialLands(); + + expect(n).toBe(0); + expect(store.enqueued).not.toContain(TASK_ID); + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + }); + + it("partial-land reconciler emits -no-action for a user-paused task", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const task = workspaceTask( + { "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH } }, + { userPaused: true }, + ); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + const n = await manager.reconcileWorkspacePartialLands(); + expect(n).toBe(0); + expect(store.enqueued).not.toContain(TASK_ID); + }); + + it("partial-land reconciler emits -no-action when a sub-repo worktree is live", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const wtPath = fx.repoPath("repo-a"); + const task = workspaceTask({ + "repo-a": { worktreePath: wtPath, branch: BRANCH }, + "repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH }, + }); + // A live sub-repo session (workspace-aware liveness via pathsForTask ∩ isPathActive). + activeSessionRegistry.registerPath(wtPath, { taskId: TASK_ID, kind: "executor", ownerKey: "x" }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + const n = await manager.reconcileWorkspacePartialLands(); + expect(n).toBe(0); + expect(store.enqueued).not.toContain(TASK_ID); + }); + + // ── KTD2 FORK-A: branch-gone classification ──────────────────────────────── + it("FORK-A: branch gone + landedSha unset → parked failed", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + // No fusion branch created in repo-a, and no landedSha → unrecoverable. + const task = workspaceTask({ + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH }, + }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + const n = await manager.reconcileWorkspacePartialLands(); + expect(n).toBe(1); + expect(store.tasks.get(TASK_ID)?.status).toBe("failed"); + expect(store.enqueued).not.toContain(TASK_ID); + }); + + it("FORK-A: branch gone + landedSha set → skipped as landed (re-enqueue finalize)", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + addRepoBranch(fx, "repo-a", "a\n"); + const landedA = landRepoForReal(fx, "repo-a"); + fx.git("repo-a", `git branch -D ${BRANCH}`); // branch gone, but landedSha is an ancestor. + + const task = workspaceTask({ + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH, landedSha: landedA }, + }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + const n = await manager.reconcileWorkspacePartialLands(); + // All landed → not parked failed; re-enqueued for finalize-once. + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + expect(store.enqueued).toContain(TASK_ID); + expect(n).toBe(1); + }); + + // ── KTD3 phantom lease reclaim ───────────────────────────────────────────── + it("reclaims a workspace-repo-land lease whose owner is terminal and older than the floor", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const leasePath = fx.repoPath("repo-a"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-06-22T00:00:00.000Z")); + activeSessionRegistry.registerPath(leasePath, { taskId: TASK_ID, kind: "workspace-repo-land", ownerKey: "land" }); + + // Owner is done (terminal). Floor = taskStuckTimeoutMs(60s) * 3 = 180s. Advance well past it. + const task = workspaceTask({ "repo-a": { worktreePath: leasePath, branch: BRANCH } }, { column: "done" }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + vi.setSystemTime(new Date("2026-06-22T00:10:00.000Z")); + const n = await manager.reclaimPhantomWorkspaceLandLeases(); + + expect(n).toBe(1); + expect(activeSessionRegistry.isPathActive(leasePath)).toBe(false); + }); + + it("does NOT reclaim a land lease owned by a live merging task", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const leasePath = fx.repoPath("repo-a"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-06-22T00:00:00.000Z")); + activeSessionRegistry.registerPath(leasePath, { taskId: TASK_ID, kind: "workspace-repo-land", ownerKey: "land" }); + + // Owner is in-review with an active "merging" status → live; lease must be left alone. + const task = workspaceTask({ "repo-a": { worktreePath: leasePath, branch: BRANCH } }, { status: "merging" }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + vi.setSystemTime(new Date("2026-06-22T00:10:00.000Z")); + const n = await manager.reclaimPhantomWorkspaceLandLeases(); + + expect(n).toBe(0); + expect(activeSessionRegistry.isPathActive(leasePath)).toBe(true); + }); + + it("does NOT reclaim a land lease younger than the staleness floor", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const leasePath = fx.repoPath("repo-a"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-06-22T00:00:00.000Z")); + activeSessionRegistry.registerPath(leasePath, { taskId: TASK_ID, kind: "workspace-repo-land", ownerKey: "land" }); + + const task = workspaceTask({ "repo-a": { worktreePath: leasePath, branch: BRANCH } }, { column: "done" }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + vi.setSystemTime(new Date("2026-06-22T00:01:00.000Z")); // 60s < 180s floor. + const n = await manager.reclaimPhantomWorkspaceLandLeases(); + + expect(n).toBe(0); + expect(activeSessionRegistry.isPathActive(leasePath)).toBe(true); + }); + + // ── KTD4 per-repo worktree cleanup ───────────────────────────────────────── + it("removes a done workspace task's recorded per-repo worktrees (isPathActive-guarded)", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + // Create a real per-repo worktree for each sub-repo (the recorded worktreePath). + const wtA = path.join(fx.repoPath("repo-a"), ".wt-task"); + const wtB = path.join(fx.repoPath("repo-b"), ".wt-task"); + fx.git("repo-a", `git worktree add -b ${BRANCH} ${wtA} HEAD`); + fx.git("repo-b", `git worktree add -b ${BRANCH} ${wtB} HEAD`); + expect(existsSync(wtA)).toBe(true); + expect(existsSync(wtB)).toBe(true); + + const task = workspaceTask( + { + "repo-a": { worktreePath: wtA, branch: BRANCH }, + "repo-b": { worktreePath: wtB, branch: BRANCH }, + }, + { column: "done" }, + ); + // Mark repo-b's worktree as active → it must be SKIPPED. + activeSessionRegistry.registerPath(wtB, { taskId: TASK_ID, kind: "executor", ownerKey: "x" }); + + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + const cleaned = await manager.reconcileOrphanedWorkspaceWorktrees(); + + expect(cleaned).toBe(1); + expect(existsSync(wtA)).toBe(false); // removed + expect(existsSync(wtB)).toBe(true); // active → skipped + }); + + // ── regression: single-repo task untouched by workspace reconcilers ──────── + it("single-repo (non-workspace) task is ignored by the workspace reconcilers", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + const single = { + id: "FN-9001", + column: "in-review", + branch: "fusion/fn-9001", + worktree: "/tmp/wt/fn-9001", + status: "merging", + paused: false, + dependencies: [], + steps: [], + currentStep: 0, + updatedAt: new Date(Date.now() - 30 * 60_000).toISOString(), + } as unknown as Task; + const store = createStore([single]); + const manager = makeManager(store, fx.rootDir); + + const partial = await manager.reconcileWorkspacePartialLands(); + const leases = await manager.reclaimPhantomWorkspaceLandLeases(); + const orphans = await manager.reconcileOrphanedWorkspaceWorktrees(); + + expect(partial).toBe(0); + expect(leases).toBe(0); + expect(orphans).toBe(0); + expect(store.enqueued).not.toContain("FN-9001"); + expect(store.tasks.get("FN-9001")?.status).toBe("merging"); // untouched + }); +}); diff --git a/packages/engine/src/active-session-registry.ts b/packages/engine/src/active-session-registry.ts index f560e25388..4a454fd531 100644 --- a/packages/engine/src/active-session-registry.ts +++ b/packages/engine/src/active-session-registry.ts @@ -125,6 +125,27 @@ export class ActiveSessionRegistry { return paths; } + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD3 — enumeration seam for phantom-lease reclaim): + The existing accessors are path-first (lookupByPath / isPathActive) or task-first + (pathsForTask). Phantom-lease reclaim needs the inverse: enumerate every live entry of a + given KIND so self-healing can find a leaked "workspace-repo-land" lease whose owning task is + already terminal/dead. A dead task is gone from the in-progress lists, so FN-6736's + iterate-tasks approach cannot surface the lease — it must be discovered from the registry + itself. Returns shallow copies (path + the full record fields incl. `registeredAt`, already + tracked) so callers can age-gate against the FN-6736 staleness floor without holding a + reference into the internal map. + */ + entriesByKind(kind: ActiveSessionKind): Array<{ path: string; taskId: string; kind: ActiveSessionKind; registeredAt: number }> { + const out: Array<{ path: string; taskId: string; kind: ActiveSessionKind; registeredAt: number }> = []; + for (const [path, record] of this.records.entries()) { + if (record.kind === kind) { + out.push({ path, taskId: record.taskId, kind: record.kind, registeredAt: record.registeredAt }); + } + } + return out; + } + reconcileStaleSelfOwned(worktreePath: string, expectedTaskId: string): ReconcileStaleSelfOwnedResult { const record = this.lookupByPath(worktreePath); if (!record) { diff --git a/packages/engine/src/run-audit.ts b/packages/engine/src/run-audit.ts index e92d23656d..dc071041d5 100644 --- a/packages/engine/src/run-audit.ts +++ b/packages/engine/src/run-audit.ts @@ -516,6 +516,15 @@ export type DatabaseMutationType = | "task:resume-limbo-escalated" /** Metadata: { taskId, executionAgeMs, graceMs, staleBindingAgeFloorMs, checkedOutBy, agentPresent, lastActivityMs, hasRecentRunAudit, worktree, branch, worktreeExists, signalReason } */ | "task:reclaim-phantom-executor-binding" + /* FNXC:Workspace 2026-06-22-09:30 (Phase D U1) — workspace-mode self-healing run-audit events. */ + /** Metadata: { taskId, landedRepos: string[], unlandedRepos: string[], failedRepos: string[], action: "re-enqueue" | "park-failed", reason } */ + | "task:reconcile-workspace-partial-land" + /** Metadata: { taskId, reason: "auto-merge-off" | "user-paused" | "live-worktree", livePaths: string[] } */ + | "task:reconcile-workspace-partial-land-no-action" + /** Metadata: { taskId, path, kind: "workspace-repo-land", registeredAt, ageMs, staleBindingAgeFloorMs, ownerColumn } */ + | "task:reclaim-phantom-workspace-land-lease" + /** Metadata: { taskId, repo, worktreePath, success, reason } */ + | "task:reconcile-orphaned-workspace-worktree" /** Metadata: { taskId, branch, worktree, checkedOutBy, executionStartedAt, executionAgeMs, graceMs, liveWorktreeBoundBranch, reason } */ | "task:reclaim-self-owned-branch-conflict-no-action" | "task:orphan-detected-no-action" diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index d65c4ac997..7aae65f4ee 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -30,7 +30,7 @@ import { setImmediate as setImmediateCb } from "node:timers"; import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { isAbsolute, join, relative, resolve } from "node:path"; -import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, evaluateNoCommitsNoOpFinalize, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, resolveMaxAutoMergeRetries, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core"; +import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, evaluateNoCommitsNoOpFinalize, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isWorkspaceTask, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, resolveMaxAutoMergeRetries, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core"; import type { MeshLeaseManager } from "./mesh-lease-manager.js"; import { createLogger, schedulerLog } from "./logger.js"; import { mergeEffectiveSettings } from "./effective-settings.js"; @@ -46,7 +46,15 @@ import { classifyError, extractMissingModulePath, isNonContinuableSessionError, import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js"; import { createRunAuditor, generateSyntheticRunId, type DatabaseMutationType, type RunAuditor } from "./run-audit.js"; import { AutoRecoveryDispatcher } from "./auto-recovery.js"; -import { activeSessionRegistry, executingTaskLock } from "./active-session-registry.js"; +import { activeSessionRegistry, executingTaskLock, type ActiveSessionKind } from "./active-session-registry.js"; +/* +FNXC:Workspace 2026-06-22-09:30 (Phase D U1): +`isRepoLanded` is the CANONICAL per-repo landed predicate (Phase C, exported A6). Self-healing +reuses it rather than reimplementing the ancestor/trailer check. merger-ai also imports a const +from self-healing (MIN_TEMP_WORKTREE_REAP_AGE_MS), so this is a static cycle — safe because +`isRepoLanded` is only referenced at call time, never at module-eval time. +*/ +import { isRepoLanded } from "./merger-ai.js"; import { findAlreadyMergedTaskCommit } from "./already-merged-detector.js"; import { isAiMergeContainerDir, resolveAiMergeRootPath, resolveLegacyAiMergeRootPath, resolveWorktreesDir } from "./worktree-paths.js"; import { canonicalFusionBranchName, resolveTaskWorkingBranch } from "./worktree-names.js"; @@ -817,6 +825,24 @@ export class SelfHealingManager { }); } + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD2 — workspace-aware liveness predicate): + `evaluateBackwardMoveTripleProof` is NOT workspace-aware: it keys liveness off the SINGULAR + `task.worktree` / `canonicalFusionBranchName(task.id)`, but a workspace task's liveness lives + across N sub-repo worktrees (task.worktree is null). A workspace task is LIVE iff ANY of its + sub-repo paths is still registered as active in the in-memory session registry + (`pathsForTask` ∩ `isPathActive`) OR a process-wide executing/active signal is held. Used by + the partial-land reconciler as the "safe to move backward / re-enqueue" gate so a live merging + task is never moved backward. + */ + private isWorkspaceTaskLive(task: Task): { live: boolean; livePaths: string[] } { + const livePaths = activeSessionRegistry.pathsForTask(task.id).filter((path) => activeSessionRegistry.isPathActive(path)); + const live = livePaths.length > 0 + || executingTaskLock.has(task.id) + || this.options.isTaskActive?.(task.id) === true; + return { live, livePaths }; + } + private async evaluateBackwardMoveTripleProof( task: Task, input: { @@ -2142,6 +2168,10 @@ export class SelfHealingManager { { name: "reconcile-done-task-integrity", fn: () => this.reconcileDoneTaskIntegrity() }, { name: "reconcile-stale-merger-status", fn: () => this.reconcileStaleMergerStatus() }, { name: "recover-mergeable-review", fn: () => this.recoverMergeableReviewTasks() }, + // FNXC:Workspace 2026-06-22-09:30 (Phase D U1) — workspace-mode reconcilers. + { name: "reconcile-workspace-partial-lands", fn: () => this.reconcileWorkspacePartialLands() }, + { name: "reclaim-phantom-workspace-land-leases", fn: () => this.reclaimPhantomWorkspaceLandLeases() }, + { name: "reconcile-orphaned-workspace-worktrees", fn: () => this.reconcileOrphanedWorkspaceWorktrees() }, { name: "recover-merged-review", fn: () => this.recoverMergedReviewTasks() }, { name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() }, { name: "recover-post-done-noncontinuable-wedge", fn: () => this.recoverPostDoneNonContinuableWedge() }, @@ -2470,6 +2500,15 @@ export class SelfHealingManager { for (const task of stale) { const previousStatus = task.status; try { + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD1 — workspace-safe by construction): + This reconciler makes NO single-commit assumption: it only clears the transient + `merging`/`merging-pr` status (status:null) + clearMergeActive and never calls + findLandedTaskCommit or moves the task. That is exactly the correct workspace action + (clear the stale status so a re-land can be re-enqueued; the partial-land reconciler / + recover-interrupted-merging owns the actual re-enqueue). So a workspace task is handled + identically and safely here — no workspace-specific branch is needed. + */ log.warn(`Clearing stale merge status for ${task.id}: ${previousStatus}`); await this.store.updateTask(task.id, { status: null }); this.options.clearMergeActive?.(task.id); @@ -5775,7 +5814,12 @@ export class SelfHealingManager { // stale ones are handled by recoverStaleMergingStatus(). t.status !== "merging" && t.status !== "merging-pr" && - Boolean(t.worktree) && + // FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD1 — admit workspace tasks): + // A workspace task has task.worktree===null (its worktrees live per-repo in + // workspaceWorktrees), so the old `Boolean(t.worktree)` gate skipped a zero-landed + // mergeable workspace task FOREVER. Admit `isWorkspaceTask(t)` so a workspace task whose + // merge enqueue was dropped is re-enqueued via enqueueMerge → idempotent landWorkspaceTask. + (Boolean(t.worktree) || isWorkspaceTask(t)) && t.mergeDetails?.mergeConfirmed !== true && t.mergeDetails?.noOpMerge !== true && !hasTerminalInvalidDoneTransition(t) && @@ -6690,6 +6734,38 @@ export class SelfHealingManager { let recovered = 0; for (const task of candidates) { try { + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD1 — P0 workspace gate): + A workspace task lands PER-REPO and `landWorkspaceTask` sets status:"merging". The + singular `findLandedTaskCommit` runs git over `this.options.rootDir` (the NON-git + workspace root) → wrong/empty, and a one-repo hit would finalize the WHOLE task done + + emit task:merged on a single repo's commit — a P0 data bug that marks a PARTIAL-landed + workspace task fully merged. So for a workspace task we MUST NOT call findLandedTaskCommit + / the single-commit finalize. Instead clear the transient "merging" status and re-enqueue + via `enqueueMerge`, which routes to the idempotent `landWorkspaceTask`: it skips repos + whose `landedSha` is already an ancestor (isRepoLanded) and finalizes to done EXACTLY ONCE + only when EVERY acquired repo is landed; a partial/none state simply re-lands the missing + repos. The partial-land reconciler (KTD2) is the standing recovery for a re-enqueue drop. + */ + if (isWorkspaceTask(task)) { + await this.store.updateTask(task.id, { status: null, error: null }); + this.options.clearMergeActive?.(task.id); + await this.store.logEntry( + task.id, + "Auto-recovered (workspace): cleared stale 'merging' status; per-repo land will be re-enqueued (no single-commit finalize)", + ); + try { + this.options.enqueueMerge?.(task.id); + } catch (enqueueErr: unknown) { + log.warn( + `Failed to re-enqueue workspace ${task.id} after stale-merge recovery (will rely on partial-land reconciler/polling sweep): ${enqueueErr instanceof Error ? enqueueErr.message : String(enqueueErr)}`, + ); + } + log.log(`Recovered interrupted workspace merge ${task.id}: cleared stale status, re-enqueued per-repo land`); + recovered++; + continue; + } + const mergeTarget = await this.resolveSelfHealingMergeTarget(task, settings, "recover-interrupted-merging"); const landedCommit = await this.findLandedTaskCommit(task); @@ -6779,6 +6855,335 @@ export class SelfHealingManager { } } + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD2 — partial-land reconciler): + Recovers non-done workspace tasks whose per-repo land is incomplete (some/none landed) and + whose binding is stale — re-enqueuing the merge via `enqueueMerge` (which routes to the + idempotent `landWorkspaceTask`; already-landed repos are skipped via `isRepoLanded`). We do NOT + call `landWorkspaceTask` directly. GUARDS (reuse, never reinvent): `allowsAutoMergeProcessing` + (FN-5147 autoMerge:false), user-pause, and the WORKSPACE-AWARE liveness predicate + (`isWorkspaceTaskLive`) — triple-proof is NOT workspace-aware so it is deliberately NOT used + here. A live / paused / autoMerge-off task emits `task:reconcile-workspace-partial-land-no-action` + and is NEVER moved backward. + + FORK-A (unrecoverable): a sub-repo is unrecoverable iff its `fusion/` branch is GONE AND its + `landedSha` is UNSET (nothing landed, nothing to land) → park the task `status:"failed"`. Branch + gone but `landedSha` set → already landed (isRepoLanded ancestor/trailer) → that repo is skipped. + Otherwise the task is retryable (re-enqueue). + */ + async reconcileWorkspacePartialLands(): Promise { + try { + const settings = await this.store.getSettings(); + if (settings.globalPause || settings.enginePaused) return 0; + + const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null; + // Workspace tasks live in in-review (post-capture/review, pre/partial land). A task already + // done is finished; todo/in-progress are owned by execution-stage reconcilers. + const tasks = await this.store.listTasks({ column: "in-review", slim: true }); + const candidates = tasks.filter((task) => + task.column === "in-review" && + isWorkspaceTask(task) && + task.mergeDetails?.mergeConfirmed !== true && + // Active transient merge statuses are owned by the live merger; recover-interrupted / + // recover-stale-merging clear STALE ones. A non-transient status (or null) is our domain. + !(task.status && ACTIVE_MERGE_STATUSES.has(task.status)), + ); + if (candidates.length === 0) return 0; + + let recovered = 0; + for (const task of candidates) { + try { + // GUARD 1 — FN-5147 autoMerge:false: in-review is human-gated; never move it backward. + if (!allowsAutoMergeProcessing(task, settings)) { + await this.emitWorkspacePartialLandNoAction(task, "auto-merge-off", []); + continue; + } + // GUARD 2 — user-pause: a hard operator stop. + if (task.userPaused || task.paused) { + await this.emitWorkspacePartialLandNoAction(task, "user-paused", []); + continue; + } + // GUARD 3 — workspace-aware liveness: ANY active sub-repo path / process signal. + const liveness = this.isWorkspaceTaskLive(task); + if (liveness.live) { + await this.emitWorkspacePartialLandNoAction(task, "live-worktree", liveness.livePaths); + continue; + } + // GUARD 4 — a live merge lane owns this exact task right now. + if (activeMergeTaskId && activeMergeTaskId === task.id) { + await this.emitWorkspacePartialLandNoAction(task, "live-worktree", liveness.livePaths); + continue; + } + + // Classify each acquired sub-repo: landed / retryable / unrecoverable (FORK-A). + const workspaceWorktrees = task.workspaceWorktrees ?? {}; + const repoKeys = Object.keys(workspaceWorktrees); + const landedRepos: string[] = []; + const unlandedRepos: string[] = []; + const unrecoverableRepos: string[] = []; + for (const repoRel of repoKeys) { + const entry = workspaceWorktrees[repoRel]; + const repoRootDir = join(this.options.rootDir, repoRel); + let integrationBranch: string; + try { + integrationBranch = await resolveIntegrationBranch( + repoRootDir, + { ...settings, integrationBranch: undefined, baseBranch: undefined }, + ); + } catch { + // Cannot resolve the sub-repo's integration branch → treat as retryable (re-enqueue + // re-runs the same resolution and surfaces the real error there). + unlandedRepos.push(repoRel); + continue; + } + if (await isRepoLanded(repoRootDir, integrationBranch, entry.landedSha, task.id, entry.branch)) { + landedRepos.push(repoRel); + continue; + } + // Not landed. FORK-A unrecoverable iff the task branch is GONE and nothing landed. + const branchPresent = entry.branch + ? await this.repoBranchExists(repoRootDir, entry.branch) + : false; + if (!branchPresent && !entry.landedSha) { + unrecoverableRepos.push(repoRel); + } else { + unlandedRepos.push(repoRel); + } + } + + const auditor = createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-healing-workspace-partial-land", task.id), + agentId: "self-healing", + taskId: task.id, + taskLineageId: task.lineageId, + phase: "reconcile-workspace-partial-land", + }); + + if (unrecoverableRepos.length > 0) { + // FORK-A: at least one repo can never land (branch gone, nothing landed) → park failed. + const error = `Workspace partial-land unrecoverable: sub-repo(s) ${unrecoverableRepos.join(", ")} have no fusion/${task.id.toLowerCase()} branch and no landedSha — manual intervention required.`; + await this.store.updateTask(task.id, { status: "failed", error }); + await this.store.logEntry(task.id, error); + await auditor.database({ + type: "task:reconcile-workspace-partial-land", + target: task.id, + metadata: { taskId: task.id, landedRepos, unlandedRepos, failedRepos: unrecoverableRepos, action: "park-failed", reason: "branch-gone-and-unlanded" }, + }).catch(() => undefined); + log.warn(`reconcileWorkspacePartialLands: parked ${task.id} failed (unrecoverable repos: ${unrecoverableRepos.join(", ")})`); + recovered++; + continue; + } + + if (unlandedRepos.length === 0) { + // Every acquired repo is already landed but the task was never finalized (the finalize + // enqueue was dropped). Re-enqueue: landWorkspaceTask skips all repos and finalizes once. + this.options.enqueueMerge?.(task.id); + await this.store.logEntry(task.id, "Auto-recovered (workspace): all sub-repos landed but task not finalized — re-enqueued finalize-once"); + await auditor.database({ + type: "task:reconcile-workspace-partial-land", + target: task.id, + metadata: { taskId: task.id, landedRepos, unlandedRepos: [], failedRepos: [], action: "re-enqueue", reason: "all-landed-not-finalized" }, + }).catch(() => undefined); + recovered++; + continue; + } + + // Partial / none landed, all unlanded repos retryable → re-enqueue the per-repo land. + this.options.enqueueMerge?.(task.id); + await this.store.logEntry(task.id, `Auto-recovered (workspace): re-enqueued partial land (${landedRepos.length} landed, ${unlandedRepos.length} pending)`); + await auditor.database({ + type: "task:reconcile-workspace-partial-land", + target: task.id, + metadata: { taskId: task.id, landedRepos, unlandedRepos, failedRepos: [], action: "re-enqueue", reason: landedRepos.length > 0 ? "partial-land" : "zero-land" }, + }).catch(() => undefined); + recovered++; + } catch (err: unknown) { + log.error(`reconcileWorkspacePartialLands: failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`); + } + } + if (recovered > 0) log.log(`reconcileWorkspacePartialLands: recovered ${recovered} workspace task(s)`); + return recovered; + } catch (err: unknown) { + log.error(`reconcileWorkspacePartialLands sweep failed: ${err instanceof Error ? err.message : String(err)}`); + return 0; + } + } + + private async emitWorkspacePartialLandNoAction( + task: Task, + reason: "auto-merge-off" | "user-paused" | "live-worktree", + livePaths: string[], + ): Promise { + try { + await createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-healing-workspace-partial-land-no-action", task.id), + agentId: "self-healing", + taskId: task.id, + taskLineageId: task.lineageId, + phase: "reconcile-workspace-partial-land", + }).database({ + type: "task:reconcile-workspace-partial-land-no-action", + target: task.id, + metadata: { taskId: task.id, reason, livePaths }, + }); + } catch (err: unknown) { + log.warn(`reconcileWorkspacePartialLands: audit emit failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`); + } + } + + /** True iff `branch` exists as a local ref in the sub-repo at `repoRootDir`. */ + private async repoBranchExists(repoRootDir: string, branch: string): Promise { + try { + await execAsync(`git rev-parse --verify ${shellQuote(`refs/heads/${branch}`)}`, { + cwd: repoRootDir, + timeout: 30_000, + }); + return true; + } catch { + return false; + } + } + + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD3 — phantom workspace-repo-land lease reclaim): + A `workspace-repo-land` lease is registered on a sub-repo's ABSOLUTE path while a workspace task + lands it, and released in a finally. If the holder dies between register and release, the lease + leaks; because the owner is terminal/dead it is gone from the in-progress lists, so FN-6736's + iterate-tasks reclaim cannot surface it. We enumerate `workspace-repo-land` entries via the new + registry seam and, for each whose owning task is terminal/dead AND whose `registeredAt` is older + than the FN-6736 staleness floor (graceMs * PHANTOM_EXECUTOR_BINDING_AGE_MULTIPLIER), clear the + lease (unregister the path) + emit `task:reclaim-phantom-workspace-land-lease`. A lease owned by a + LIVE merging task (still in-review with a transient merge status, or the active merge task) is + UNTOUCHED — only a demonstrably dead owner is reclaimed. + */ + async reclaimPhantomWorkspaceLandLeases(): Promise { + try { + const settings = await this.store.getSettings(); + if (settings.globalPause || settings.enginePaused) return 0; + + const entries = activeSessionRegistry.entriesByKind("workspace-repo-land" as ActiveSessionKind); + if (entries.length === 0) return 0; + + const graceMs = settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS; + const staleFloorMs = graceMs * PHANTOM_EXECUTOR_BINDING_AGE_MULTIPLIER; + const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null; + const now = Date.now(); + + let reclaimed = 0; + for (const entry of entries) { + try { + const ageMs = now - entry.registeredAt; + if (ageMs < staleFloorMs) continue; // too recent — a live land is still warming. + + // A live merge lane / executing owner keeps the lease. + if (activeMergeTaskId && activeMergeTaskId === entry.taskId) continue; + if (executingTaskLock.has(entry.taskId) || this.options.isTaskActive?.(entry.taskId) === true) continue; + + const owner = await this.store.getTask(entry.taskId).catch(() => null); + // Owner is dead/terminal iff: not found, archived/done/failed, OR in-review with NO active + // transient merge status (a merging owner is live; a clean in-review is finished landing). + const ownerColumn = owner?.column ?? "deleted"; + const ownerHasActiveMergeStatus = Boolean(owner?.status && ACTIVE_MERGE_STATUSES.has(owner.status)); + const ownerLive = Boolean(owner) + && owner!.column !== "done" + && owner!.status !== "failed" + && ownerHasActiveMergeStatus; + if (ownerLive) continue; // live merging owner → leave its lease alone. + + activeSessionRegistry.unregisterPath(entry.path); + await createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-healing-phantom-workspace-land-lease", entry.taskId), + agentId: "self-healing", + taskId: entry.taskId, + phase: "reclaim-phantom-workspace-land-lease", + }).database({ + type: "task:reclaim-phantom-workspace-land-lease", + target: entry.taskId, + metadata: { taskId: entry.taskId, path: entry.path, kind: entry.kind, registeredAt: entry.registeredAt, ageMs, staleBindingAgeFloorMs: staleFloorMs, ownerColumn }, + }).catch(() => undefined); + log.warn(`reclaimPhantomWorkspaceLandLeases: reclaimed leaked land lease on ${entry.path} (owner ${entry.taskId}, age ${ageMs}ms)`); + reclaimed++; + } catch (err: unknown) { + log.error(`reclaimPhantomWorkspaceLandLeases: failed for ${entry.path}: ${err instanceof Error ? err.message : String(err)}`); + } + } + if (reclaimed > 0) log.log(`reclaimPhantomWorkspaceLandLeases: reclaimed ${reclaimed} leaked lease(s)`); + return reclaimed; + } catch (err: unknown) { + log.error(`reclaimPhantomWorkspaceLandLeases sweep failed: ${err instanceof Error ? err.message : String(err)}`); + return 0; + } + } + + /* + FNXC:Workspace 2026-06-22-09:30 (Phase D U1, KTD4 — per-repo worktree cleanup from STORED paths): + For done/dead workspace tasks, remove each recorded per-repo worktree. The paths are ADDRESSABLE + from the task row (`workspaceWorktrees[repo].worktreePath`, persisted) so we NEVER walk the temp + root / readdir the temp tree (AGENTS.md forbids unbounded temp walks) — the sweep is bounded by + construction. Each removal is GUARDED by `activeSessionRegistry.isPathActive(path)` (skip if + active, mirroring the temp-dir sweep at the AI-merge worktree guard) so a still-live path is never + yanked. Emit `task:reconcile-orphaned-workspace-worktree` per removed path. + */ + async reconcileOrphanedWorkspaceWorktrees(): Promise { + try { + const settings = await this.store.getSettings(); + if (settings.globalPause || settings.enginePaused) return 0; + + // Done workspace tasks are the canonical "safe to clean" set (their lands are finalized). + const doneTasks = await this.store.listTasks({ column: "done", slim: true }); + const candidates = doneTasks.filter((task) => isWorkspaceTask(task)); + if (candidates.length === 0) return 0; + + let cleaned = 0; + for (const task of candidates) { + const workspaceWorktrees = task.workspaceWorktrees ?? {}; + for (const repoRel of Object.keys(workspaceWorktrees)) { + const worktreePath = workspaceWorktrees[repoRel]?.worktreePath; + if (!worktreePath) continue; + // GUARD: skip an active path (mirror self-healing temp-dir sweep isPathActive guard). + if (activeSessionRegistry.isPathActive(worktreePath)) continue; + // Nothing on disk → nothing to remove (already cleaned). Skip silently. + if (!existsSync(worktreePath)) continue; + + const repoRootDir = join(this.options.rootDir, repoRel); + let success = false; + let reason = "removed"; + try { + await execAsync(`git worktree remove --force ${shellQuote(worktreePath)}`, { + cwd: repoRootDir, + timeout: 120_000, + }); + success = true; + } catch (err: unknown) { + reason = `git-remove-failed: ${err instanceof Error ? err.message : String(err)}`; + } + try { + await createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-healing-orphaned-workspace-worktree", task.id), + agentId: "self-healing", + taskId: task.id, + taskLineageId: task.lineageId, + phase: "reconcile-orphaned-workspace-worktree", + }).database({ + type: "task:reconcile-orphaned-workspace-worktree", + target: task.id, + metadata: { taskId: task.id, repo: repoRel, worktreePath, success, reason }, + }); + } catch { /* audit best-effort */ } + if (success) { + log.log(`reconcileOrphanedWorkspaceWorktrees: removed ${worktreePath} (task ${task.id}, repo ${repoRel})`); + cleaned++; + } + } + } + if (cleaned > 0) log.log(`reconcileOrphanedWorkspaceWorktrees: removed ${cleaned} orphaned per-repo worktree(s)`); + return cleaned; + } catch (err: unknown) { + log.error(`reconcileOrphanedWorkspaceWorktrees sweep failed: ${err instanceof Error ? err.message : String(err)}`); + return 0; + } + } + private async readShortstatForSha( sha: string, rebaseBaseSha?: string,