diff --git a/.changeset/fn-9049-workspace-branch-evidence.md b/.changeset/fn-9049-workspace-branch-evidence.md new file mode 100644 index 0000000000..f2e6ee6e37 --- /dev/null +++ b/.changeset/fn-9049-workspace-branch-evidence.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Prevent transient Git evidence failures from failing workspace tasks. +category: fix +dev: Replaces repoBranchExists with tri-state probeRepoBranch and an execBranchProbe seam, switches to show-ref, adds evidence-unavailable audit handling, and bounds deferred evidence retries. diff --git a/AGENTS.md b/AGENTS.md index 6229c7b963..0bc544840b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -303,7 +303,7 @@ Scoped exception (FN-5819/FN-8823): while project auto-merge is On, shared-branc - FN-6782/FN-6796: self-healing emits `task:auto-recover-paused-abort-park` when it clears a benign pause-abort operator park, requeueing safe `todo`/`in-progress` rows or preserving a clean auto-merge-eligible `in-review` row for review progression. - FN-8908: self-healing reserves `task:auto-recover-terminal-failure` and `task:auto-recover-terminal-failure-exhausted` for generic terminal-failure budget recovery. Metadata must remain ids/counts/outcomes-only and never include failure prose or the rotating `wedgeNotification.autoRecovery.applyToken`; that durable budget is the backoff source, and its apply fence—not the grace heuristic—authorizes the single clear/requeue transition. - FN-6793/FN-6797: self-healing emits `task:reconcile-in-review-unmet-dependencies` when it rebounds an `in-review` task whose declared dependencies are still unmet, and `task:reconcile-in-review-unmet-dependencies-no-action` when pause/user-pause, `autoMerge:false`, live execution/checkout proof, or a failed rebound mutation blocks that backward move. -- Workspace (Phase D U1): self-healing emits `task:reconcile-workspace-partial-land` when it re-enqueues a partial/zero-landed workspace task's per-repo land (or parks it `failed` when a sub-repo's `fusion/` branch is gone with no `landedSha`), and `task:reconcile-workspace-partial-land-no-action` when `autoMerge:false`, user-pause, or a live sub-repo worktree (workspace-aware liveness) blocks that backward move. +- Workspace (Phase D U1): self-healing emits `task:reconcile-workspace-partial-land` when it re-enqueues a partial/zero-landed workspace task's per-repo land (or parks it `failed` for proven branch absence or exhausted `evidence-unavailable` branch reads), and `task:reconcile-workspace-partial-land-no-action` when `autoMerge:false`, user-pause, a live sub-repo worktree (workspace-aware liveness), or `evidence-unavailable` blocks that backward move. The bounded evidence-exhaustion reason is `evidence-unavailable-exhausted`; audit metadata remains ids/counts/outcomes-only. - Workspace (Phase D U1): self-healing emits `task:reclaim-phantom-workspace-land-lease` when it clears a leaked `workspace-repo-land` lease whose owning task is terminal/dead and older than the FN-6736 staleness floor. Archived-role and soft-deleted owners are terminal; live merging, executing, or merge-pending owners are untouched. - Workspace (Phase D U1): self-healing emits `task:reconcile-orphaned-workspace-worktree` when it removes a done/dead workspace task's recorded per-repo worktree from its stored `worktreePath` (guarded by `isPathActive`; no temp-root walk). - FN-8144: archive emits `archive-workspace-worktree-disposer-missing` when a workspace archive has no store-scoped backend disposer; per-repository archive removal is awaited under canonical-path reservations, with failed paths quarantined for successor reconciliation. diff --git a/packages/engine/src/__tests__/self-healing-workspace.test.ts b/packages/engine/src/__tests__/self-healing-workspace.test.ts index 2edf9bafbf..22188fc8a2 100644 --- a/packages/engine/src/__tests__/self-healing-workspace.test.ts +++ b/packages/engine/src/__tests__/self-healing-workspace.test.ts @@ -24,6 +24,7 @@ import { existsSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { Settings, Task, TaskStore } from "@fusion/core"; import { SelfHealingManager } from "../self-healing.js"; +import { classifyBranchProbeError } from "../self-healing-git-evidence.js"; import { activeSessionRegistry } from "../agents/active-session-registry.js"; import { landWorkspaceTask } from "../merge/merger-ai.js"; import { createWorkspaceFixture, hasGit, type WorkspaceFixture } from "./_workspace-fixture.js"; @@ -86,17 +87,38 @@ function createStore(rows: Task[], settings: Partial = {}): TaskStore return store; } -function makeManager(store: TaskStore, rootDir: string, opts: Record = {}): SelfHealingManager { +function managerOptions(store: TaskStore, rootDir: string, opts: Record = {}): Record { const enqueueMerge = (taskId: string) => { (store as unknown as RecordingStore).enqueued.push(taskId); return true; }; - return new SelfHealingManager(store, { - rootDir, - enqueueMerge, - clearMergeActive: vi.fn(), - ...opts, - } as never); + return { rootDir, enqueueMerge, clearMergeActive: vi.fn(), ...opts }; +} + +function makeManager(store: TaskStore, rootDir: string, opts: Record = {}): SelfHealingManager { + return new SelfHealingManager(store, managerOptions(store, rootDir, opts) as never); +} + +/* +FNXC:Workspace 2026-08-15-04:42: +These harnesses inject only the exec boundary so timeout tests execute the production probe try/catch +and classifier. Returning an outcome from an overridden probe would make the regression tautological. +*/ +class BranchProbeHarness extends SelfHealingManager { + async readBranchEvidence(repoRootDir: string, branch: string): Promise<"present" | "absent" | "unknown"> { + return this.probeRepoBranch(repoRootDir, branch); + } +} + +class UnavailableBranchProbeManager extends SelfHealingManager { + unavailable = true; + + protected override async execBranchProbe(repoRootDir: string, branch: string): Promise { + if (this.unavailable) { + throw Object.assign(new Error("Command failed: git show-ref"), { killed: true, signal: "SIGTERM", code: null }); + } + return super.execBranchProbe(repoRootDir, branch); + } } /** Add a real `fusion/` branch in a sub-repo with one non-conflicting own commit. */ @@ -319,6 +341,104 @@ describeIfGit("workspace-aware self-healing (Phase D U1)", () => { expect(store.enqueued).not.toContain(TASK_ID); }); + it("classifies only a clean exit-1 branch probe error as absent", () => { + expect(classifyBranchProbeError({ code: 1 })).toBe("absent"); + for (const error of [ + { code: 1, killed: true, signal: "SIGTERM" }, + { code: 128 }, + { code: "ENOENT" }, + { code: "EACCES" }, + { killed: true, signal: "SIGTERM", code: null }, + {}, + "non-error throw", + ]) { + expect(classifyBranchProbeError(error)).toBe("unknown"); + } + }); + + it("probes real existing and deleted branches as present and absent", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + addRepoBranch(fx, "repo-a", "a\n"); + const store = createStore([]); + const manager = new BranchProbeHarness(store, managerOptions(store, fx.rootDir) as never); + + expect(await manager.readBranchEvidence(fx.repoPath("repo-a"), BRANCH)).toBe("present"); + fx.git("repo-a", `git branch -D ${BRANCH}`); + expect(await manager.readBranchEvidence(fx.repoPath("repo-a"), BRANCH)).toBe("absent"); + }); + + it("defers missing sub-repo evidence without failing, moving, or enqueuing", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const task = workspaceTask({ + "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH }, + "missing-repo": { worktreePath: path.join(fx.rootDir, "missing-repo"), branch: BRANCH }, + }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + expect(await manager.reconcileWorkspacePartialLands()).toBe(0); + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + expect(store.tasks.get(TASK_ID)?.column).toBe("in-review"); + expect(store.enqueued).not.toContain(TASK_ID); + expect((store.recordRunAuditEvent as ReturnType).mock.calls.some( + ([event]) => (event as { metadata?: { reason?: string } }).metadata?.reason === "evidence-unavailable", + )).toBe(true); + }); + + it("defers timeout-shaped evidence through the real probe classifier", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + addRepoBranch(fx, "repo-a", "a\n"); + const task = workspaceTask({ "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH } }); + const store = createStore([task]); + const manager = new UnavailableBranchProbeManager(store, managerOptions(store, fx.rootDir) as never); + + expect(await manager.reconcileWorkspacePartialLands()).toBe(0); + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + expect(store.tasks.get(TASK_ID)?.column).toBe("in-review"); + expect(store.enqueued).not.toContain(TASK_ID); + expect((store.recordRunAuditEvent as ReturnType).mock.calls.some( + ([event]) => (event as { metadata?: { reason?: string } }).metadata?.reason === "evidence-unavailable", + )).toBe(true); + }); + + it("parks only after bounded unavailable-evidence deferrals and resets after recovery", async () => { + fx = await createWorkspaceFixture(["repo-a"]); + addRepoBranch(fx, "repo-a", "a\n"); + const task = workspaceTask({ "repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH } }); + const store = createStore([task]); + const manager = new UnavailableBranchProbeManager(store, managerOptions(store, fx.rootDir) as never); + + await manager.reconcileWorkspacePartialLands(); + await manager.reconcileWorkspacePartialLands(); + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + manager.unavailable = false; + expect(await manager.reconcileWorkspacePartialLands()).toBe(1); + expect(store.enqueued).toContain(TASK_ID); + + store.enqueued.length = 0; + manager.unavailable = true; + await manager.reconcileWorkspacePartialLands(); + await manager.reconcileWorkspacePartialLands(); + await manager.reconcileWorkspacePartialLands(); + expect(store.tasks.get(TASK_ID)?.status).toBe("failed"); + expect(store.tasks.get(TASK_ID)?.error).toContain("evidence unavailable"); + expect(store.tasks.get(TASK_ID)?.error).not.toContain("no fusion/"); + }); + + it("defers when unknown evidence is mixed with a genuinely absent branch", async () => { + fx = await createWorkspaceFixture(["repo-a", "repo-b"]); + const task = workspaceTask({ + "missing-repo": { worktreePath: path.join(fx.rootDir, "missing-repo"), branch: BRANCH }, + "repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH }, + }); + const store = createStore([task]); + const manager = makeManager(store, fx.rootDir); + + expect(await manager.reconcileWorkspacePartialLands()).toBe(0); + expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed"); + expect(store.enqueued).not.toContain(TASK_ID); + }); + it("FORK-A: branch gone + landedSha set → skipped as landed (re-enqueue finalize)", async () => { fx = await createWorkspaceFixture(["repo-a"]); addRepoBranch(fx, "repo-a", "a\n"); diff --git a/packages/engine/src/self-healing-git-evidence.ts b/packages/engine/src/self-healing-git-evidence.ts index 2e2bdcbc1f..a0ceea7cc5 100644 --- a/packages/engine/src/self-healing-git-evidence.ts +++ b/packages/engine/src/self-healing-git-evidence.ts @@ -42,6 +42,27 @@ import type { SelfHealingOptions } from "./self-healing.js"; export const execAsync = promisify(exec); +/** + * FNXC:Workspace 2026-08-15-04:42: + * Evidence unavailable must never be laundered into proof that a branch is absent: the sole + * consumer turns absence into an irreversible `status:"failed"` park. `git show-ref --verify + * --quiet` exits 0 for present, 1 for absent, and 128 for a non-repository/ref-read failure; + * only its clean numeric exit 1 is absence evidence. Spawn, timeout, signal, and malformed + * errors are unknown so later sweeps can retry. + */ +export function classifyBranchProbeError(error: unknown): "absent" | "unknown" { + if ( + typeof error === "object" && + error !== null && + "code" in error && + (error as { code?: unknown }).code === 1 && + !(error as { killed?: unknown }).killed && + ((error as { signal?: unknown }).signal === null || (error as { signal?: unknown }).signal === undefined) + ) { + return "absent"; + } + return "unknown"; +} export interface LandedTaskCommit { sha: string; @@ -434,18 +455,40 @@ export abstract class SelfHealingGitEvidence { } } - protected async repoBranchExists(repoRootDir: string, branch: string): Promise { + /** + * FNXC:Workspace 2026-08-15-04:42: + * `show-ref --verify --quiet` distinguishes present (0), absent (clean 1), and unreadable + * repository evidence (128/error), unlike `rev-parse --verify`, which returns 128 for both a + * missing ref and a non-repository cwd. The irreversible partial-land park may consume only + * absence proof; all other errors remain unknown for a later sweep. + */ + protected async probeRepoBranch(repoRootDir: string, branch: string): Promise<"present" | "absent" | "unknown"> { try { - await execAsync(`git rev-parse --verify ${shellQuote(`refs/heads/${branch}`)}`, { - cwd: repoRootDir, - timeout: 30_000, - }); - return true; - } catch { - return false; + await this.execBranchProbe(repoRootDir, branch); + return "present"; + } catch (err: unknown) { + const outcome = classifyBranchProbeError(err); + if (outcome === "unknown") { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`[self-healing] branch evidence unavailable for ${repoRootDir} (${branch}): ${errorMessage}`); + } + return outcome; } } + /** + * FNXC:Workspace 2026-08-15-04:42: + * This protected seam sits below the real probe classifier so fixture tests can inject timeout + * and spawn-error shapes without mocking module-local `execAsync`. It deliberately has no + * catch: `probeRepoBranch` maps only a clean show-ref exit 1 to absence evidence. + */ + protected async execBranchProbe(repoRootDir: string, branch: string): Promise { + await execAsync(`git show-ref --verify --quiet ${shellQuote(`refs/heads/${branch}`)}`, { + cwd: repoRootDir, + timeout: 30_000, + }); + } + protected async readShortstatForSha( sha: string, rebaseBaseSha?: string, diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index d396c6750b..0ef83f4e54 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -695,14 +695,15 @@ export class SelfHealingManager extends SelfHealingGitEvidence { private deadlockRecoveryCooldown: Map = new Map(); private mergeStarvationDrops: Map = new Map(); /* - FNXC:Workspace 2026-06-22-14:10 (Phase D review B/E — bounded workspace re-enqueue / orphan-remove): - Per-task drop counter for the workspace partial-land re-enqueue (mirror of `mergeStarvationDrops`): - `enqueueMerge` returns false when the merge queue rejects (full). Without bounding, a perpetually - rejected workspace task is re-enqueued FOREVER. After MAX_STARVATION_DROPS consecutive drops we - park it `status:"failed"`. `orphanWorktreeRemovalFailures` likewise bounds the per-path - `git worktree remove --force` retry in reconcileOrphanedWorkspaceWorktrees. + FNXC:Workspace 2026-08-15-04:42: + The partial-land reconciler separately bounds rejected merge enqueues and unavailable branch + evidence. A clean `show-ref` exit 1 proves a branch is absent; timeout, missing directories, and + ref-read failures prove nothing, so they defer rather than falsely terminalize intact work. Both + counters are candidate-scoped and exhaustion parks visibly: infinite silent deferral is also a + failure mode requiring operator intervention. */ private workspacePartialLandDrops: Map = new Map(); + private workspacePartialLandEvidenceDefers: Map = new Map(); private orphanWorktreeRemovalFailures: Map = new Map(); private finalizeUnprovenWarned = new Set(); /* @@ -10020,6 +10021,9 @@ const movedTask = await this.store.moveTask(task.id, completeLane); for (const taskId of [...this.workspacePartialLandDrops.keys()]) { if (!candidateIds.has(taskId)) this.workspacePartialLandDrops.delete(taskId); } + for (const taskId of [...this.workspacePartialLandEvidenceDefers.keys()]) { + if (!candidateIds.has(taskId)) this.workspacePartialLandEvidenceDefers.delete(taskId); + } if (candidates.length === 0) return 0; @@ -10063,12 +10067,13 @@ const movedTask = await this.store.moveTask(task.id, completeLane); continue; } - // Classify each acquired sub-repo: landed / retryable / unrecoverable (FORK-A). + // Classify each acquired sub-repo: landed / retryable / unrecoverable / unreadable (FORK-A). const workspaceWorktrees = task.workspaceWorktrees ?? {}; const repoKeys = Object.keys(workspaceWorktrees); const landedRepos: string[] = []; const unlandedRepos: string[] = []; const unrecoverableRepos: string[] = []; + const evidenceUnavailableRepos: string[] = []; for (const repoRel of repoKeys) { const entry = workspaceWorktrees[repoRel]; const repoRootDir = join(this.options.rootDir, repoRel); @@ -10089,22 +10094,19 @@ const movedTask = await this.store.moveTask(task.id, completeLane); continue; } /* - FNXC:Workspace 2026-06-22-14:10 (Phase D review D — FORK-A: branch-gone-and-not-landed - is unrecoverable, regardless of a STALE landedSha): - We are here because `isRepoLanded` returned FALSE — the recorded `landedSha` (if any) is - NOT reachable from the integration tip (branch was force-reset / rolled back / never - actually landed) AND no task-trailer commit is on the ref. The old test was - `!branchPresent && !entry.landedSha`, which let a repo with a STALE landedSha set but - UNREACHABLE, and its `fusion/` branch GONE, fall to `unlandedRepos` → re-enqueued → - `landWorkspaceTask` has NO branch to land → loops forever. Since the repo is provably - NOT landed, the correct test is: branch GONE ⇒ unrecoverable, whether or not a (stale) - landedSha is present. Only a branch that still EXISTS is retryable. + FNXC:Workspace 2026-08-15-04:42: + FORK-A may park only on proof, never absence of evidence. `probeRepoBranch` uses + `show-ref`: present (0) retries, absent (clean 1) is unrecoverable, and timeout/spawn/ + ref-read failures are unknown. Deferral wins over any sibling absent repo because a task + is not proven unrecoverable while even one sub-repo's branch state cannot be read. */ - const branchPresent = entry.branch - ? await this.repoBranchExists(repoRootDir, entry.branch) - : false; - if (!branchPresent) { + const branchEvidence = entry.branch + ? await this.probeRepoBranch(repoRootDir, entry.branch) + : "absent"; + if (branchEvidence === "absent") { unrecoverableRepos.push(repoRel); + } else if (branchEvidence === "unknown") { + evidenceUnavailableRepos.push(repoRel); } else { unlandedRepos.push(repoRel); } @@ -10118,8 +10120,31 @@ const movedTask = await this.store.moveTask(task.id, completeLane); phase: "reconcile-workspace-partial-land", }); + if (evidenceUnavailableRepos.length > 0) { + const defers = (this.workspacePartialLandEvidenceDefers.get(task.id) ?? 0) + 1; + this.workspacePartialLandEvidenceDefers.set(task.id, defers); + if (defers >= MAX_STARVATION_DROPS) { + const error = `Workspace partial-land evidence unavailable: branch state could not be read after ${MAX_STARVATION_DROPS} sweeps for sub-repo(s) ${evidenceUnavailableRepos.join(", ")} — manual intervention required.`; + await this.store.updateTask(task.id, { status: "failed", error }); + await this.store.logEntry(task.id, error); + this.workspacePartialLandEvidenceDefers.delete(task.id); + await auditor.database({ + type: "task:reconcile-workspace-partial-land", + target: task.id, + metadata: { taskId: task.id, landedRepos, unlandedRepos, failedRepos: unrecoverableRepos, evidenceUnavailableRepos, action: "park-failed", reason: "evidence-unavailable-exhausted" }, + }).catch(() => undefined); + log.warn(`reconcileWorkspacePartialLands: parked ${task.id} failed after unavailable evidence (${evidenceUnavailableRepos.join(", ")})`); + recovered++; + } else { + await this.emitWorkspacePartialLandNoAction(task, "evidence-unavailable", []); + } + continue; + } + + this.workspacePartialLandEvidenceDefers.delete(task.id); + if (unrecoverableRepos.length > 0) { - // FORK-A: at least one repo can never land (branch gone, nothing landed) → park failed. + // FORK-A: at least one repo is proven branch-gone and not landed → park failed. const error = `Workspace partial-land unrecoverable: sub-repo(s) ${unrecoverableRepos.join(", ")} have no fusion/${task.id.toLowerCase()} branch and no landedSha — manual intervention required.`; await this.store.updateTask(task.id, { status: "failed", error }); await this.store.logEntry(task.id, error); @@ -10168,7 +10193,7 @@ const movedTask = await this.store.moveTask(task.id, completeLane); private async emitWorkspacePartialLandNoAction( task: Task, - reason: "auto-merge-off" | "user-paused" | "live-worktree" | "merge-pending", + reason: "auto-merge-off" | "user-paused" | "live-worktree" | "merge-pending" | "evidence-unavailable", livePaths: string[], ): Promise { try {