diff --git a/.changeset/task-pinned-orphan-recovery.md b/.changeset/task-pinned-orphan-recovery.md new file mode 100644 index 0000000000..2dce381755 --- /dev/null +++ b/.changeset/task-pinned-orphan-recovery.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Recover task-pinned worktrees when an incomplete directory occupies the pinned path. +category: fix +dev: Preserve inactive incomplete or unregistered directories under .fusion/recovery before recreating the task worktree. diff --git a/packages/engine/src/__tests__/worktree-acquisition.test.ts b/packages/engine/src/__tests__/worktree-acquisition.test.ts index d0968e934e..1d97a905f7 100644 --- a/packages/engine/src/__tests__/worktree-acquisition.test.ts +++ b/packages/engine/src/__tests__/worktree-acquisition.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { execSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; @@ -8,8 +8,10 @@ import { acquireTaskWorktree, RepoRootWorktreeError, WorktreeBaseRefreshError } import { classifyTaskWorktree, PoolDoubleLeaseError } from "../worktree/worktree-pool.js"; import * as desktopArtifacts from "../worktree/worktree-desktop-artifacts.js"; import * as branchConflicts from "../execution/branch-conflicts.js"; +import { activeSessionRegistry } from "../agents/active-session-registry.js"; import { NativeWorktreeBackend } from "../worktree/worktree-backend.js"; + vi.mock("../worktree/worktree-pool.js", async () => { const actual = await vi.importActual("../worktree/worktree-pool.js"); return { @@ -77,6 +79,7 @@ function makeRepo(): string { } afterEach(() => { + vi.restoreAllMocks(); for (const path of cleanupPaths.splice(0)) { rmSync(path, { recursive: true, force: true }); } @@ -345,6 +348,202 @@ describe("acquireTaskWorktree", () => { expect(store.updateTask).toHaveBeenCalledWith("FN-1", { worktree: null, branch: null, sessionFile: null }); }); + it.each([ + { classification: "incomplete" as const, reason: "missing .git metadata" }, + { classification: "unregistered" as const, reason: "not registered in git worktree list" }, + ])("reclaims a $classification task-pinned directory before recreating the worktree", async ({ classification, reason }) => { + const rootDir = makeRepo(); + const pinnedPath = join(rootDir, ".worktrees", "fn-1"); + mkdirSync(join(pinnedPath, ".build"), { recursive: true }); + mkdirSync(join(pinnedPath, ".swiftpm"), { recursive: true }); + writeFileSync(join(pinnedPath, ".build", "cache"), "stale\n", "utf-8"); + vi.mocked(classifyTaskWorktree).mockResolvedValueOnce({ + ok: false, + classification, + reason, + }); + + const result = await acquireTaskWorktree({ + task: { ...task, worktree: pinnedPath, branch: "fusion/fn-1" }, + rootDir, + store, + settings: { worktreeNaming: "task-id", recycleWorktrees: false }, + }); + + expect(result).toMatchObject({ + worktreePath: pinnedPath, + branch: "fusion/fn-1", + source: "fresh", + isResume: false, + }); + expect(existsSync(join(pinnedPath, ".git"))).toBe(true); + expect(git(rootDir, "git worktree list --porcelain")).toContain(pinnedPath); + const recoveryRoot = join(rootDir, ".fusion", "recovery", "worktrees"); + const preserved = readdirSync(recoveryRoot); + expect(preserved).toHaveLength(1); + expect(readFileSync(join(recoveryRoot, preserved[0], ".build", "cache"), "utf-8")).toBe("stale\n"); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-1", + expect.stringContaining("Preserved orphaned task-pinned directory"), + expect.stringContaining(join(".fusion", "recovery", "worktrees")), + undefined, + ); + }); + + it("preserves an orphan beside an external worktree root when project recovery is cross-device", async () => { + const rootDir = makeRepo(); + const externalWorktrees = track(mkdtempSync(join(tmpdir(), "fn-external-worktrees-"))); + const pinnedPath = join(externalWorktrees, "fn-1"); + mkdirSync(join(pinnedPath, ".build"), { recursive: true }); + writeFileSync(join(pinnedPath, ".build", "cache"), "stale\n", "utf-8"); + vi.mocked(classifyTaskWorktree).mockResolvedValueOnce({ + ok: false, + classification: "incomplete", + reason: "missing .git metadata", + }); + const actualFs = await vi.importActual("node:fs/promises"); + const renameWorktreeDirectory = vi.fn() + .mockRejectedValueOnce(Object.assign(new Error("cross-device link"), { code: "EXDEV" })) + .mockImplementationOnce(actualFs.rename); + + const result = await acquireTaskWorktree({ + task: { ...task, worktree: pinnedPath, branch: "fusion/fn-1" }, + rootDir, + store, + settings: { worktreeNaming: "task-id", worktreesDir: externalWorktrees, recycleWorktrees: false }, + renameWorktreeDirectory, + }); + + const recoveryRoot = join(externalWorktrees, ".fusion-recovery", "worktrees"); + const preservedPath = join(recoveryRoot, readdirSync(recoveryRoot)[0]); + expect(result.worktreePath).toBe(pinnedPath); + expect(readFileSync(join(preservedPath, ".build", "cache"), "utf-8")).toBe("stale\n"); + expect(renameWorktreeDirectory).toHaveBeenCalledTimes(2); + }); + + it("preserves the orphan in place when its path becomes active during recovery", async () => { + const rootDir = makeRepo(); + const pinnedPath = join(rootDir, ".worktrees", "fn-1"); + mkdirSync(join(pinnedPath, ".build"), { recursive: true }); + writeFileSync(join(pinnedPath, ".build", "cache"), "stale\n", "utf-8"); + vi.mocked(classifyTaskWorktree).mockResolvedValueOnce({ + ok: false, + classification: "incomplete", + reason: "missing .git metadata", + }); + let becameActive = false; + vi.spyOn(activeSessionRegistry, "isPathActive").mockImplementation((path: string) => { + if (path !== pinnedPath) return false; + if (!becameActive) { + becameActive = true; + return false; + } + return true; + }); + + await expect(acquireTaskWorktree({ + task: { ...task, worktree: pinnedPath, branch: "fusion/fn-1" }, + rootDir, + store, + settings: { worktreeNaming: "task-id", recycleWorktrees: false }, + })).rejects.toThrow(/became active/); + + expect(readFileSync(join(pinnedPath, ".build", "cache"), "utf-8")).toBe("stale\n"); + expect(existsSync(join(rootDir, ".fusion", "recovery", "worktrees"))).toBe(true); + expect(readdirSync(join(rootDir, ".fusion", "recovery", "worktrees"))).toHaveLength(0); + }); + + it("refuses to preserve an orphan through a recovery-directory symlink", async () => { + const rootDir = makeRepo(); + const pinnedPath = join(rootDir, ".worktrees", "fn-1"); + const outside = track(mkdtempSync(join(tmpdir(), "fn-orphan-recovery-outside-"))); + mkdirSync(join(pinnedPath, ".build"), { recursive: true }); + writeFileSync(join(pinnedPath, ".build", "cache"), "stale\n", "utf-8"); + mkdirSync(join(rootDir, ".fusion", "recovery"), { recursive: true }); + symlinkSync(outside, join(rootDir, ".fusion", "recovery", "worktrees")); + vi.mocked(classifyTaskWorktree).mockResolvedValueOnce({ + ok: false, + classification: "incomplete", + reason: "missing .git metadata", + }); + + await expect(acquireTaskWorktree({ + task: { ...task, worktree: pinnedPath, branch: "fusion/fn-1" }, + rootDir, + store, + settings: { worktreeNaming: "task-id", recycleWorktrees: false }, + })).rejects.toThrow(/outside the project root/); + + expect(readFileSync(join(pinnedPath, ".build", "cache"), "utf-8")).toBe("stale\n"); + expect(readdirSync(outside)).toHaveLength(0); + }); + + it("refuses to create recovery contents through an ancestor symlink", async () => { + const rootDir = makeRepo(); + const pinnedPath = join(rootDir, ".worktrees", "fn-1"); + const outside = track(mkdtempSync(join(tmpdir(), "fn-orphan-recovery-ancestor-outside-"))); + mkdirSync(join(pinnedPath, ".build"), { recursive: true }); + writeFileSync(join(pinnedPath, ".build", "cache"), "stale\n", "utf-8"); + mkdirSync(join(rootDir, ".fusion"), { recursive: true }); + symlinkSync(outside, join(rootDir, ".fusion", "recovery")); + vi.mocked(classifyTaskWorktree).mockResolvedValueOnce({ + ok: false, + classification: "incomplete", + reason: "missing .git metadata", + }); + + await expect(acquireTaskWorktree({ + task: { ...task, worktree: pinnedPath, branch: "fusion/fn-1" }, + rootDir, + store, + settings: { worktreeNaming: "task-id", recycleWorktrees: false }, + })).rejects.toThrow(/outside the project root/); + + expect(readFileSync(join(pinnedPath, ".build", "cache"), "utf-8")).toBe("stale\n"); + expect(readdirSync(outside)).toHaveLength(0); + }); + + it("serializes concurrent orphan recovery through fresh recreation", async () => { + const rootDir = makeRepo(); + const pinnedPath = join(rootDir, ".worktrees", "fn-1"); + mkdirSync(join(pinnedPath, ".build"), { recursive: true }); + writeFileSync(join(pinnedPath, ".build", "cache"), "stale\n", "utf-8"); + const actualPool = await vi.importActual("../worktree/worktree-pool.js"); + vi.mocked(classifyTaskWorktree).mockResolvedValueOnce({ + ok: false, + classification: "incomplete", + reason: "missing .git metadata", + }).mockImplementation(actualPool.classifyTaskWorktree); + let signalCreateStarted!: () => void; + const createStarted = new Promise((resolve) => { signalCreateStarted = resolve; }); + let allowCreate!: () => void; + const createGate = new Promise((resolve) => { allowCreate = resolve; }); + const createWorktree = vi.fn(async (branch: string, path: string) => { + signalCreateStarted(); + await createGate; + git(rootDir, `git worktree add -b ${JSON.stringify(branch)} ${JSON.stringify(path)} main`); + return { path, branch }; + }); + const input = { + task: { ...task, worktree: pinnedPath, branch: "fusion/fn-1" }, + rootDir, + store, + settings: { worktreeNaming: "task-id", recycleWorktrees: false }, + createWorktree, + } as const; + + const first = acquireTaskWorktree(input); + await createStarted; + const second = acquireTaskWorktree(input); + allowCreate(); + + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(firstResult.worktreePath).toBe(pinnedPath); + expect(secondResult.worktreePath).toBe(pinnedPath); + expect(createWorktree).toHaveBeenCalledTimes(1); + expect(readFileSync(join(rootDir, ".fusion", "recovery", "worktrees", readdirSync(join(rootDir, ".fusion", "recovery", "worktrees"))[0], ".build", "cache"), "utf-8")).toBe("stale\n"); + }); + it("refreshes a recreated existing task branch after its dependency branch is deleted", async () => { const rootDir = makeRepo(); const staleBase = git(rootDir, "git rev-parse HEAD"); diff --git a/packages/engine/src/worktree/worktree-acquisition.ts b/packages/engine/src/worktree/worktree-acquisition.ts index 90ae17c427..fb1a12c0ab 100644 --- a/packages/engine/src/worktree/worktree-acquisition.ts +++ b/packages/engine/src/worktree/worktree-acquisition.ts @@ -1,7 +1,8 @@ import { existsSync } from "node:fs"; -import { readFile, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, realpath, rename, stat, writeFile } from "node:fs/promises"; import { exec } from "node:child_process"; -import { isAbsolute, resolve } from "node:path"; +import { isAbsolute, join, relative, resolve } from "node:path"; import { promisify } from "node:util"; import {acquireWorktreePathReservation, canonicalizeWorktreePath, type RunMutationContext, type Settings, type Task, type TaskStore, type SecretsStore} from "@fusion/core"; import { generateWorktreeName, resolveTaskWorkingBranch, slugify } from "./worktree-names.js"; @@ -109,6 +110,8 @@ export interface AcquireTaskWorktreeOptions { }>; taskEnv?: NodeJS.ProcessEnv; backend?: WorktreeBackend; + /** Test seam for filesystem-device recovery behavior. */ + renameWorktreeDirectory?: typeof rename; /** Execution callers opt in; planning, review, and merge reuse remain unchanged. */ refreshStaleBase?: boolean; } @@ -145,6 +148,24 @@ export class RepoRootWorktreeError extends Error { const INIT_OUTCOME_MAX_CHARS = 2_000; +async function ensureContainedDirectory(parentCanonicalPath: string, name: string): Promise { + const candidate = join(parentCanonicalPath, name); + try { + await mkdir(candidate); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + const canonicalCandidate = await realpath(candidate); + const candidateRelative = relative(parentCanonicalPath, canonicalCandidate); + if (candidateRelative === "" || candidateRelative.startsWith("..") || isAbsolute(candidateRelative)) { + throw new Error(`Refusing to use recovery directory outside the project root: ${canonicalCandidate}`); + } + if (!(await stat(canonicalCandidate)).isDirectory()) { + throw new Error(`Refusing to use non-directory recovery path: ${canonicalCandidate}`); + } + return canonicalCandidate; +} + function configuredCommandErrorMessage(result: { spawnError?: string | Error; timedOut?: boolean; exitCode?: number | null }): string { if (result.spawnError) return `Failed to start command: ${result.spawnError}`; if (result.timedOut) return "Command timed out"; @@ -240,6 +261,7 @@ async function pinnedWorktreeBranchMatches(rootDir: string, worktreePath: string export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Promise { const { task, rootDir, store, settings, pool, logger, audit, runContext, createWorktree, runConfiguredCommand, runInitCommand, taskEnv, secretsStore } = opts; + const renameWorktreeDirectory = opts.renameWorktreeDirectory ?? rename; const refreshExistingWorktree = async ( path: string, backendKind: WorktreeBackend["kind"], @@ -422,77 +444,89 @@ export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Pro Acquisition delegates branch creation to the isolated-worktree primitive. The project root remains on its current branch; task branch selection must never use a root-checkout `git checkout` or `git switch`. */ - const createWorktreeImpl = async ( + const createWorktreeWithoutReservation = async ( createBranch: string, createPath: string, createTaskId: string, startPoint?: string, allowRename?: boolean, ): Promise<{ path: string; branch: string; backendKind: WorktreeBackend["kind"] }> => { - if (createWorktree) { - const created = await createWorktree(createBranch, createPath, createTaskId, startPoint, allowRename); - return { ...created, backendKind: opts.createWorktreeBackendKind ?? backend.kind }; - } - const reservation = await acquireWorktreePathReservation({ - canonicalPath: await canonicalizeWorktreePath(createPath), - worktreesDir: resolveWorktreesDir(rootDir, settings), + try { + const created = await backend.create({ rootDir, - /* - FNXC:WorkflowLifecycle 2026-07-16-10:00: - A failed archive removal leaves a durable quarantine record. The next - owner must reconcile that old pinned path while it exclusively holds - the reservation, rather than colliding with it during creation. - */ - reconcileQuarantined: async () => { - await removeWorktree({ - worktreePath: createPath, - rootDir, - settings, - taskId: createTaskId, - reason: RemovalReason.ExecutorDispose, - force: true, - }); - }, + branch: createBranch, + worktreePath: createPath, + startPoint, + taskId: createTaskId, + allowSiblingBranchRename: allowRename, }); - try { - const created = await backend.create({ - rootDir, - branch: createBranch, - worktreePath: createPath, - startPoint, - taskId: createTaskId, - allowSiblingBranchRename: allowRename, + if (backend.kind === "worktrunk") { + await audit?.git({ + type: "worktree:worktrunk-create", + target: created.path, + metadata: { branch: created.branch }, }); - if (backend.kind === "worktrunk") { - await audit?.git({ - type: "worktree:worktrunk-create", - target: created.path, - metadata: { branch: created.branch }, - }); - } - await persistWorktreeBackendKind(created.path, backend.kind); - return { ...created, backendKind: backend.kind }; - } catch (error) { - if (backend.kind === "worktrunk" && error instanceof WorktrunkOperationError) { - // FNXC:WorktreeAcquisition 2026-07-16-00:00: FN-8132 requires native fallback collision dispositions to be audited just like direct native acquisition. - const nativeBackend = new NativeWorktreeBackend({ logger: logger ?? undefined, audit }); - const fallback = () => nativeBackend.create({ - rootDir, - branch: createBranch, - worktreePath: createPath, - startPoint, - taskId: createTaskId, - allowSiblingBranchRename: allowRename, - }); - const created = await handleWorktrunkFailure("create", error, fallback) as { path: string; branch: string }; - await persistWorktreeBackendKind(created.path, "native"); - return { ...created, backendKind: "native" }; - } - throw error; - } finally { - if (reservation.state === "held") await reservation.release(); } - }; + await persistWorktreeBackendKind(created.path, backend.kind); + return { ...created, backendKind: backend.kind }; + } catch (error) { + if (backend.kind !== "worktrunk" || !(error instanceof WorktrunkOperationError)) throw error; + // FNXC:WorktreeAcquisition 2026-07-16-00:00: FN-8132 requires native fallback collision dispositions to be audited just like direct native acquisition. + const nativeBackend = new NativeWorktreeBackend({ logger: logger ?? undefined, audit }); + const fallback = () => nativeBackend.create({ + rootDir, + branch: createBranch, + worktreePath: createPath, + startPoint, + taskId: createTaskId, + allowSiblingBranchRename: allowRename, + }); + const created = await handleWorktrunkFailure("create", error, fallback) as { path: string; branch: string }; + await persistWorktreeBackendKind(created.path, "native"); + return { ...created, backendKind: "native" }; + } + }; + + const createWorktreeImpl = async ( + createBranch: string, + createPath: string, + createTaskId: string, + startPoint?: string, + allowRename?: boolean, + reservationHeld = false, + ): Promise<{ path: string; branch: string; backendKind: WorktreeBackend["kind"] }> => { + if (createWorktree) { + const created = await createWorktree(createBranch, createPath, createTaskId, startPoint, allowRename); + return { ...created, backendKind: opts.createWorktreeBackendKind ?? backend.kind }; + } + if (reservationHeld) return createWorktreeWithoutReservation(createBranch, createPath, createTaskId, startPoint, allowRename); + const reservation = await acquireWorktreePathReservation({ + canonicalPath: await canonicalizeWorktreePath(createPath), + worktreesDir: resolveWorktreesDir(rootDir, settings), + rootDir, + /* + FNXC:WorkflowLifecycle 2026-07-16-10:00: + A failed archive removal leaves a durable quarantine record. The next + owner must reconcile that old pinned path while it exclusively holds + the reservation, rather than colliding with it during creation. + */ + reconcileQuarantined: async () => { + await removeWorktree({ + worktreePath: createPath, + rootDir, + settings, + taskId: createTaskId, + reason: RemovalReason.ExecutorDispose, + force: true, + }); + }, + }); + try { + return await createWorktreeWithoutReservation(createBranch, createPath, createTaskId, startPoint, allowRename); + } finally { + if (reservation.state === "held") await reservation.release(); + } + }; const logConfiguredCopyFileResults = async (results: WorktreeCopyFileResult[], source: "fresh" | "pool") => { if (results.length === 0) return; @@ -685,8 +719,18 @@ export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Pro await store.updateTask(task.id, { worktree: pinnedPath }); } - worktreePath = pinnedPath; - branch = branchName; + const reservation = await acquireWorktreePathReservation({ + canonicalPath: await canonicalizeWorktreePath(pinnedPath), + worktreesDir: resolveWorktreesDir(rootDir, settings), + rootDir, + isLiveWorktree: async () => { + if (activeSessionRegistry.isPathActive(pinnedPath)) return true; + return (await classifyTaskWorktree(rootDir, pinnedPath)).ok; + }, + }); + try { + worktreePath = pinnedPath; + branch = branchName; if (existsSync(pinnedPath)) { const classification = await classifyTaskWorktree(rootDir, pinnedPath); @@ -729,14 +773,62 @@ export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Pro ); if (isInsideWorktreesDir(rootDir, pinnedPath, settings)) { try { - await removeWorktree({ - rootDir, - worktreePath: pinnedPath, - settings, - reason: RemovalReason.PoolPrune, - taskId: task.id, - audit: undefined, - }); + const preserveAsOrphanDirectory = !classification.ok + && (classification.classification === "incomplete" || classification.classification === "unregistered") + && !activeSessionRegistry.isPathActive(pinnedPath); + if (preserveAsOrphanDirectory) { + const canonicalRoot = await realpath(rootDir); + const fusionRoot = await ensureContainedDirectory(canonicalRoot, ".fusion"); + const recoveryRoot = await ensureContainedDirectory(fusionRoot, "recovery"); + const canonicalRecoveryRoot = await ensureContainedDirectory(recoveryRoot, "worktrees"); + // The path reservation serializes classification, preservation, and + // recreation across processes; this final probe also protects against + // an in-process owner that registered before the reservation was held. + if (activeSessionRegistry.isPathActive(pinnedPath)) { + throw new Error(`Task-pinned worktree ${pinnedPath} became active during orphan recovery`); + } + let preservedPath = join(canonicalRecoveryRoot, `${task.id.toLowerCase()}-${randomUUID()}`); + try { + await renameWorktreeDirectory(pinnedPath, preservedPath); + } catch (renameError) { + if ((renameError as NodeJS.ErrnoException).code !== "EXDEV") throw renameError; + /* + * FNXC:TaskPinnedWorktrees 2026-08-09-03:20: + * Configured worktrees may live on another filesystem. Preserve atomically beside the + * configured worktree root instead of weakening recovery to recursive copy-and-delete. + */ + const canonicalWorktreesRoot = await realpath(resolveWorktreesDir(rootDir, settings)); + const localRecoveryRoot = await ensureContainedDirectory(canonicalWorktreesRoot, ".fusion-recovery"); + const localRecoveryWorktrees = await ensureContainedDirectory(localRecoveryRoot, "worktrees"); + preservedPath = join(localRecoveryWorktrees, `${task.id.toLowerCase()}-${randomUUID()}`); + await renameWorktreeDirectory(pinnedPath, preservedPath); + } + await audit?.filesystem({ + type: "file:write", + target: preservedPath, + metadata: { + taskId: task.id, + classification: classification.classification, + reason: "task-pinned-orphan-preserved", + sourcePath: pinnedPath, + }, + }); + await store.logEntry( + task.id, + `Preserved orphaned task-pinned directory ${pinnedPath} before recreation`, + preservedPath, + runContext, + ); + } else { + await removeWorktree({ + rootDir, + worktreePath: pinnedPath, + settings, + reason: RemovalReason.PoolPrune, + taskId: task.id, + audit: undefined, + }); + } } catch (removeErr) { /* * FNXC:TaskPinnedWorktrees 2026-07-16-12:30: @@ -756,8 +848,11 @@ export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Pro await store.updateTask(task.id, { sessionFile: null }); } - const created = await createWorktreeImpl(branchName, pinnedPath, task.id, freshStartPoint, allowSiblingBranchRename); - return finalizeCreatedWorktree(created, "fresh", "normal"); + const created = await createWorktreeImpl(branchName, pinnedPath, task.id, freshStartPoint, allowSiblingBranchRename, true); + return await finalizeCreatedWorktree(created, "fresh", "normal"); + } finally { + if (reservation.state === "held") await reservation.release(); + } }; if (pinned) {