diff --git a/.changeset/fn-7802-phantom-worktree-merge-active-recovery.md b/.changeset/fn-7802-phantom-worktree-merge-active-recovery.md new file mode 100644 index 0000000000..b8e8560ed4 --- /dev/null +++ b/.changeset/fn-7802-phantom-worktree-merge-active-recovery.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Recover tasks stranded by missing worktrees during merge/review and allow retry. +category: fix +dev: Adds merge-active missing-worktree self-healing with no-action audits and signature-only retry resets across CLI, extension, and dashboard. diff --git a/AGENTS.md b/AGENTS.md index 147c00ee37..0726759a11 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,6 +224,7 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme - FN-7158: agent performance reflections emit `reflection:generated`, `reflection:skipped`, and `reflection:failed` with ids/counts/outcomes-only metadata; never persist reflection prose or prompt text in run-audit. - FN-7528: a deterministic, non-LLM post-task performance capture (`AgentReflectionService.captureTaskPerformance`) runs once per completed task and emits `reflection:captured` with ids/counts/outcomes-only metadata (`retryReworkCount?`, `filesTouchedCount?`, `packagesTouchedCount?`, `verificationFileScoped?`, `durationMs?`); never persists `verificationScopeReason` free-text or summary prose in run-audit. - FN-7787: `createResolvedAgentSession` enriches `session:runtime-resolved` with `noModelResolved: true` and `runtimeBuiltInFallbackModel` when a non-mock/non-test session reaches runtime creation without a complete provider/model pair; this is a visibility signal for runtime built-in fallback usage, not a fabricated model-resolution verdict. +- FN-7802: self-healing emits `task:reconcile-missing-worktree-merge-active` when it proves an `in-review` merge-active task (`merging`/`merging-pr`/`merging-fix`) is stranded by an unusable-worktree session-start failure, clears stale `worktree`/`branch`/`sessionFile`, resets the worktree-session retry budget, increments `recoveryRetryCount` as the bounded stale-metadata clear counter, and requeues to `todo`; it emits `task:reconcile-missing-worktree-merge-active-no-action` when `autoMerge:false`, workspace-task ownership, or triple-proof blocks the backward move. - FN-7011: self-healing emits `task:reconcile-engine-downtime-active-timing` when startup recovery shifts active task segment anchors to exclude proven engine-process downtime, and `task:reconcile-engine-downtime-active-timing-no-action` when no active task qualifies. - FN-5419: git run-audit now includes `pull:fast-forward` and `stash:pop-conflict`; dashboard git surfaces now include the extended `POST /api/git/pull` integration-worktree path plus companion `POST /api/git/stash-resolve`, `POST /api/git/stash-drop`, and `POST /api/git/stash-apply` routes. - FN-6292: self-healing emits `task:reconcile-dependency-blocking-lease` when it rebounds an in-progress holder whose stale file-scope lease blocks an unmet dependency, and `task:reconcile-dependency-blocking-lease-no-action` when triple-proof blocks that backward move. diff --git a/docs/architecture.md b/docs/architecture.md index 6b3ad56a37..ffb9f6d9cd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2157,7 +2157,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f - **Worktree metadata reconcile ordering (FN-4962)**: `reconcile-task-worktree-metadata` must run before `reclaim-stale-active-branches`; stale `task.worktree` metadata is rebound to live `fusion/` worktrees when present (`task:auto-recover-worktree-metadata-rebound`) or cleared (`task:auto-recover-worktree-metadata-cleared`) when absent. - **Completion fan-out is synchronous**: `SelfHealingManager.reconcileCompletedTask()` runs on `in-review → done`. Downstream stale `blockedBy` links and residual `fusion/` branch/worktree artifacts are reconciled immediately, not on a periodic sweep. - **In-review stall deadlock**: identical stalls (same code + reason) repeated past `inReviewStallDeadlockThreshold` (default 3) auto-pause with `pausedReason: "in-review-stall-deadlock"` and `status: "failed"`. User-initiated retry paths (dashboard retry, `fn_task_retry`, and CLI `task retry`) clear that automatic deadlock pause so the retry can execute, but they never override explicit/manual pauses or unrelated automatic pause reasons. -- **Restart recovery**: `RestartRecoveryCoordinator` classifies interrupted `in-progress` runs. Unusable-worktree session-start failures (`missing`, `incomplete`, `unregistered git worktree`) are recoverable; retries are capped at `MAX_WORKTREE_SESSION_RETRIES=3` before escalating. +- **Restart recovery**: `RestartRecoveryCoordinator` classifies interrupted `in-progress` runs. Unusable-worktree session-start failures (`missing`, `incomplete`, `unregistered git worktree`) are recoverable; retries are capped at `MAX_WORKTREE_SESSION_RETRIES=3` before escalating. `recoverMissingWorktreeReviewFailures` also owns durable unusable-worktree session-start failures that reached `in-review` with a merge-active status (`merging`, `merging-pr`, or `merging-fix`): before interrupted/deadlocked merge sweeps can re-drive the same phantom path, it applies auto-merge eligibility, workspace-task exclusion, and triple-proof, clears stale `worktree`/`branch`/`sessionFile`, resets the exhausted worktree-session retry budget, increments `recoveryRetryCount` as the bounded merge-active stale-metadata clear counter, and requeues to `todo` preserving progress. Operator retry surfaces (`fn_task_retry`, CLI `task retry`, dashboard retry) have the same signature-only reset primitive so a missing-worktree failure does not require a valid `merging` transition. - **Executor pre-session liveness gate (FN-4935/FN-6861)**: the gate now skips for fresh acquisitions (`acquisition.source === "fresh"`), emits structured `not_usable_task_worktree:` diagnostics (including canonicalized registered-path snapshots) and a `worktree:incomplete-detected` audit event with `source: "executor-liveness-gate"`, while preserving the existing `taskDoneRetryCount` / `MAX_TASK_DONE_REQUEUE_RETRIES` requeue contract. The project repo root is never a usable task worktree even though it is a legitimately registered Git worktree; `classifyTaskWorktree` returns `repo-root` for canonical root-equal paths, and resume acquisition treats that as self-healable stale metadata by clearing `task.worktree` and creating a fresh checkout under the configured worktrees directory. FN-5772 adds a bounded nested-root self-heal: when `task.worktree` points at a strict descendant of a registered worktree root inside the configured worktrees dir, executor re-anchors `task.worktree` to the git top-level, emits `worktree:reanchored` (`fromPath`, `toPath`, `source`), and proceeds; repo-root/outside-dir/unregistered top-level mismatches still fail. FN-4651 `worktreeSessionRetryCount` remains scoped to the in-review/session-start recovery path. - **Stale self-owned active-session reconcile on conflict cleanup (FN-4973)**: when executor worktree-conflict cleanup finds only a same-task stale `activeSessionRegistry` entry and no live in-memory `activeWorktrees` binding for that task/path, it must unregister the stale entry before `removeWorktree` (plus one-shot backstop reconcile on same-task `ActiveSessionWorktreeRemovalError` races). Foreign-task entries remain protected by FN-4811 and must never be reconciled by the requesting task. - **Same-task stale removal canonical helper (FN-5346)**: executor same-task cleanup paths now route pre-removal reconciliation through `reconcileSelfOwnedActiveSessionForRemoval` (via executor helper wiring), so stale self-owned `activeSessionRegistry` residues are cleared only when no live in-memory binding exists, while FN-4811 foreign-owner refusals and live-owner protections remain intact. @@ -2177,7 +2177,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f - **Empty-commit refusal + early empty-own-diff finalize (FN-5345/FN-5377)**: Fusion task worktrees install a `prepare-commit-msg` hook that refuses `git commit --allow-empty` and other zero-staged-diff commits, preventing verification-only tasks from manufacturing empty handoff commits that defeat the merger's no-op classifier. The hook allows legitimate empty-tree paths (amend, merge, squash, cherry-pick, revert, rebase). Amend detection tokenizes the parent process command line (`ps -o args=` with `/proc/$PPID/cmdline` fallback for Alpine/busybox) and stops at the first message-supplying flag (`-m`/`-F`/`--message`/`--file`) so a commit message containing the substring `--amend` cannot bypass the guard. In `aiMergeTask`, an early empty-own-diff fast-path runs BEFORE any reuse-handoff acquisition: when integration mode is `reuse-task-worktree`, the branch exists, `git rev-list --count ..` is > 0, and `git diff --quiet ..` exits 0, the task auto-finalizes as no-op with `mergeDetails.noOpMerge: true` and emits `task:auto-recover-finalize-already-on-main` with `reason: "empty-own-diff-early-fast-path"`. The fast-path best-effort removes the stranded worktree (FN-4811 same-task/foreign-owner guard) and deletes the `fusion/` branch so empty-own-diff residuals do not accumulate. This unsticks tasks where a stale empty handoff commit combined with drifted worktree↔branch mapping would otherwise wedge the handoff gate with `registered-branch-mismatch`. The explicit `cwd-integration-branch` mode is unchanged (`cwd-main` remains a deprecated alias normalized to it). `classifyOwnedLandedEvidence` also detects empty-own-diff (aheadCount > 0, zero net diff) and returns `proven-no-op` so downstream self-healing and post-handoff finalize paths benefit too. Additionally, merger's reuse-fallback path now consults `git worktree list --porcelain` before creating a new worktree: extant usable registrations of `fusion/` are reused directly (rather than blindly `git worktree add -f` producing a duplicate registration), and stale registrations are pruned first. The direct-reuse shortcut is guarded by FN-4811 (refuses paths owned by a different task in `activeSessionRegistry`) and FN-4954 (skipped when `recycleWorktrees=true` with a pool attached, so `WorktreePool.acquire` lease bookkeeping stays consistent). Two audit subtypes — `merge:reuse-fallback-pruned-stale-registration` and `merge:reuse-fallback-reused-existing-registration` — replace the prior overloading of `merge:reuse-fallback-new-worktree` for these cases. - **Verified no-op/duplicate executor completion (FN-6275/FN-7488)**: explicit `fn_task_done` may complete with zero branch commits only when the summary starts with a recognized sentinel (`PREMISE STALE:`, `NO-OP:`, `NOOP:`, `DUPLICATE: FN-NNNN ...`, or `REDUNDANT:`), the task already carries a no-commit contract, or the PROMPT declares a source-free gitignored task-artifact delivery. The source-free path is intentionally narrow: File Scope must be populated and limited to board/task artifacts such as `.fusion/tasks/...`, task documents/logs, or attachments; the prompt must forbid force-adding ignored `.fusion/` artifacts and fabricating empty commits or equivalently state that source-free/gitignored task artifacts are the only deliverables; and any tracked source/docs/config/test/changeset scope keeps the `no_commits` refusal active (even if `.fusion/` artifacts are also listed). These exemptions only relax the `no_commits` invariant; `wrong_toplevel`, `wrong_branch`, pending-step/review refusals, and scope-leak guards still run. Accepted sentinel completions persist `noCommitsExpected: true`, write task-log audit details with marker kind/reason/raw summary/run/agent IDs, and add a task timeline activity so the no-code terminal path remains explainable. Prompt-derived source-free completions log `prompt-derived source-free task-artifact contract` for operator audit. Ordinary zero-commit implementation completions without one of these contracts are still refused. - **In-review branch-binding self-heal (FN-5083/FN-6695)**: `reconcile-in-review-branch-rebind` runs after `reconcile-task-worktree-metadata` and before `reclaim-stale-active-branches`. It restores `task.branch` (and clears `task.worktree` for fresh acquisition) for `in-review` tasks when exactly one case-insensitive `fusion/` candidate branch has unique commits versus the integration base. Ambiguous candidates emit `task:auto-rebind-skipped` (`reason: "ambiguous-candidates"`) and are never auto-resolved. Unsafe metadata repair is also skipped with `task:auto-rebind-skipped`: `userPaused` preserves authoritative user intent, and `checkedOutBy` preserves live agent checkout ownership. Branch construction across executor/worktree-pool/worktree-acquisition/merger/self-healing canonicalizes to lowercase via `canonicalFusionBranchName`; `fn_task_done` wrong-branch checks now auto-canonicalize case-only mismatches and emit `branch:auto-canonicalize-case`. -- **In-review is terminal-until-merged under `autoMerge: false` (FN-5147)**: when a project sets `settings.autoMerge: false`, `in-review` is the intended resting state until a human merges the PR. No lifecycle-mutating self-healing sweep (`reclaimSelfOwnedBranchConflicts`, `recoverGhostReviewTasks`, `recoverStaleIncompleteReviewTasks`, `recoverInterruptedMergingTasks`, `recoverStuckMergeDeadlocks`, `recoverMissingWorktreeReviewFailures`, `recoverPartialProgressNoTaskDoneFailures`, `recoverCompletionHandoffLimbo`, `recoverPostDoneNonContinuableWedge`, `recoverMergeableReviewTasks`, `recoverMergedReviewTasks`, `recoverAlreadyMergedReviewTasks`, `recoverOrphanOnlyScopeViolations`, `recoverForeignOnlyContaminatedInReviewTasks`, `recoverReviewTasksWithFailedPreMergeSteps`, `finalizeNoOpReviewTasks`, `surfaceInReviewStalls`, `surfaceInReviewStalled`) may move the task out of `in-review`, mark it `paused`/`failed`, or re-enqueue it for execution. Explicit per-task overrides are distinguished by `task.autoMergeProvenance: "user"`; ambiguous legacy rows stamped `autoMerge: true` by the pre-FN-6245 review-entry path are marked `"legacy-stamp"` once and surfaced in run-audit/logs, but are only cleared by the operator-driven `reconcileLegacyAutoMergeStamps({ apply: true })` action. Scoped FN-5819 exception: shared-group members (`branchContext.assignmentMode === "shared"`) are still allowed through the member→`branch_groups.branchName` integration step while `autoMerge` is off; this is a soft pre-integration only and does not permit shared-branch → default-branch promotion. FN-7182 applies the same human-gated treatment to an open `PrInfo.manual` PR created or linked from the dashboard **Create PR** action: automatic merge queues and self-healing stand down until the PR is closed/merged or handled manually, while pipeline-created PRs without `manual` remain auto-merge eligible. RECONCILE-ONLY sweeps (branch rebind, blocker fan-out, stale-status clears, contamination metadata cleanup, attribution restore, PR refresh, misclassified-failure error clearing) continue to run. +- **In-review is terminal-until-merged under `autoMerge: false` (FN-5147)**: when a project sets `settings.autoMerge: false`, `in-review` is the intended resting state until a human merges the PR. No lifecycle-mutating self-healing sweep (`reclaimSelfOwnedBranchConflicts`, `recoverGhostReviewTasks`, `recoverStaleIncompleteReviewTasks`, `recoverInterruptedMergingTasks`, `recoverStuckMergeDeadlocks`, `recoverMissingWorktreeReviewFailures`, `recoverPartialProgressNoTaskDoneFailures`, `recoverCompletionHandoffLimbo`, `recoverPostDoneNonContinuableWedge`, `recoverMergeableReviewTasks`, `recoverMergedReviewTasks`, `recoverAlreadyMergedReviewTasks`, `recoverOrphanOnlyScopeViolations`, `recoverForeignOnlyContaminatedInReviewTasks`, `recoverReviewTasksWithFailedPreMergeSteps`, `finalizeNoOpReviewTasks`, `surfaceInReviewStalls`, `surfaceInReviewStalled`) may move the task out of `in-review`, mark it `paused`/`failed`, or re-enqueue it for execution. That includes merge-active unusable-worktree recovery: automation emits `task:reconcile-missing-worktree-merge-active-no-action` with `reason:"auto-merge-off"` instead of moving the row, while explicit operator retry remains supported because it is a manual reset request. Explicit per-task overrides are distinguished by `task.autoMergeProvenance: "user"`; ambiguous legacy rows stamped `autoMerge: true` by the pre-FN-6245 review-entry path are marked `"legacy-stamp"` once and surfaced in run-audit/logs, but are only cleared by the operator-driven `reconcileLegacyAutoMergeStamps({ apply: true })` action. Scoped FN-5819 exception: shared-group members (`branchContext.assignmentMode === "shared"`) are still allowed through the member→`branch_groups.branchName` integration step while `autoMerge` is off; this is a soft pre-integration only and does not permit shared-branch → default-branch promotion. FN-7182 applies the same human-gated treatment to an open `PrInfo.manual` PR created or linked from the dashboard **Create PR** action: automatic merge queues and self-healing stand down until the PR is closed/merged or handled manually, while pipeline-created PRs without `manual` remain auto-merge eligible. RECONCILE-ONLY sweeps (branch rebind, blocker fan-out, stale-status clears, contamination metadata cleanup, attribution restore, PR refresh, misclassified-failure error clearing) continue to run. - **Auto-merge integration-root default (FN-5279)**: direct auto-merge now defaults `mergeIntegrationWorktree` to `reuse-task-worktree`; merger must pass the reuse handoff gates or emit `merge:reuse-handoff-refused` and leave the task in `in-review` without silently falling back to `cwd-integration-branch` (`cwd-main` remains a deprecated alias normalized to that mode). - **Orphaned execution sweep is observation-only (FN-5337)**: `recoverOrphanedExecutions` only annotates stale in-progress candidates with `task:orphan-detected-no-action` and `[orphan-detected] ... no action (operator-decides)` logs. It must never move `in-progress`/`in-review` backward to `todo` or mutate lease/worktree metadata. Proof-based backward recovery remains exclusively in `recoverInProgressLimbo` (FN-5219), `RestartRecoveryCoordinator`, `recoverMissingWorktreeReviewFailures`, and explicit executor/merger failure paths. Reintroducing lifecycle mutation here requires hard git/session proof gating plus CEO+CTO+PM sign-off. - **Self-owned reclaim resume-limbo escalation (FN-5704)**: `reclaimSelfOwnedBranchConflicts` tracks `resumeLimboCount`, `resumeLimboTipSha`, and `resumeLimboStepSignature` for in-progress reclaim/unpause loops. If reclaim finds no progress (same tip, same step-status signature, and no active-session signal) for `MAX_NO_PROGRESS_RESUME_ATTEMPTS` consecutive sweeps, self-healing escalates by moving the task to `todo` with `preserveWorktree: true`, `preserveProgress: true`, and `preserveResumeState: true` instead of endlessly re-arming resume. Escalation emits `task:resume-limbo-escalated` run-audit metadata (`frozenTipSha`, `idleMs`, `resumeAttemptCount`, `currentStep`) and resets the limbo counter. diff --git a/docs/self-healing-backward-move-audit.md b/docs/self-healing-backward-move-audit.md index aef06faec6..67c8a13e9e 100644 --- a/docs/self-healing-backward-move-audit.md +++ b/docs/self-healing-backward-move-audit.md @@ -55,7 +55,7 @@ Stages that cannot satisfy all three must either (a) tighten predicate to requir | recoverOrphanedAgents | 6180 | dead parent/direct-report linkage | n/a | org topology checks | pause/delete/reparent decisions | RECONCILE-ONLY | keep | n/a | n/a | | recoverStaleHeartbeatRuns | 6372 | stale heartbeat run records | run age thresholds | pid/age mismatch | terminate run records | RECONCILE-ONLY | keep | n/a | n/a | | recoverNoProgressNoTaskDoneFailures | 6451 | in-progress failed no-task-done no progress | implicit (no explicit grace) | no-step-progress + no git work + not executing | clear metadata + move to todo | BACKWARD | tighten | triple proof + no-progress checks + recent liveness-audit absence | gate move; emit `task:no-progress-no-task-done-no-action` | -| recoverMissingWorktreeReviewFailures | 6516 | in-review failed session-start missing/unusable worktree | classifier-based | error classifier proof only | autoRecover requeue to todo | BACKWARD | tighten | triple proof + classifier proof | gate requeue; emit `task:missing-worktree-review-no-action` | +| recoverMissingWorktreeReviewFailures | 6516 | in-review failed OR merge-active (`merging`/`merging-pr`/`merging-fix`) session-start missing/unusable worktree | classifier-based | error classifier proof only | autoRecover requeue to todo | BACKWARD | tightened | triple proof + classifier proof + `allowsAutoMergeProcessing` + workspace-task exclusion | gate requeue; emit `task:missing-worktree-review-no-action` or `task:reconcile-missing-worktree-merge-active-no-action`; successful merge-active recovery clears `worktree`/`branch`/`sessionFile`, resets the worktree-session retry budget, increments `recoveryRetryCount` as the bounded stale-metadata clear counter, emits `task:reconcile-missing-worktree-merge-active`, and requeues to todo preserving progress | | recoverPartialProgressNoTaskDoneFailures | 6586 | in-review failed no-task-done with partial progress | bounded by `MAX_TASK_DONE_RETRIES` | no-task-done + partial progress + retry budget | clear error + move to todo preserveProgress | BACKWARD | tighten | triple proof + retry-budget predicates | gate move; emit `task:partial-progress-no-task-done-no-action` | | recoverApprovedTriageTasks | 6706 | triage planning specified stale | `APPROVED_TRIAGE_RECOVERY_GRACE_MS` | planning idle + valid PROMPT.md | recoverApprovedTriageTask callback | FORWARD | keep | n/a | n/a | | recoverStarvedRefinementTriageTasks | 6827 | refinement planning stale | `STARVED_REFINEMENT_RECOVERY_GRACE_MS` | no progress idle | requeue/annotation in triage | RECONCILE-ONLY | keep | n/a | n/a | @@ -89,7 +89,8 @@ Stages that cannot satisfy all three must either (a) tighten predicate to requir ### recoverMissingWorktreeReviewFailures - Existing classifier proof is necessary but not sufficient for backward movement. -- Tighten: require full triple-proof before auto requeue. +- Tightened: require full triple-proof before auto requeue, and keep `allowsAutoMergeProcessing` as the first lifecycle gate so `autoMerge:false` remains terminal-until-merged. +- Merge-active rows (`merging`, `merging-pr`, `merging-fix`) with the same unusable-worktree session-start signature are now owned here before interrupted/deadlocked merge sweeps can re-drive the phantom path. Workspace tasks are annotation-only in this path; per-repo land recovery remains workspace-owned. ### recoverPartialProgressNoTaskDoneFailures - Retry-budget alone can still reopen tasks under active/residual execution races. diff --git a/docs/task-management.md b/docs/task-management.md index 053e9352b7..a3853dfd46 100644 --- a/docs/task-management.md +++ b/docs/task-management.md @@ -263,7 +263,7 @@ Fusion task columns use persisted enum values as the API/filter contract. Caller - If merge/finalization hits a terminal error, tasks can remain in `in-review` with `status: "failed"` for explicit follow-up. This state is intentionally preserved by recovery (not auto-bounced to `todo`). - Retry behavior splits by execution-vs-merge signals: `in-review` tasks with incomplete steps (`pending`/`in-progress`) are treated as execution failures and retried back to `todo` with `preserveProgress: true`; zero-step `in-review` tasks use `mergeRetries` as the tie-breaker (`mergeRetries === 0` or undefined → execution failure path back to `todo`, `mergeRetries > 0` → merge/finalization retry in `in-review` with merge retry state reset); tasks whose steps are all terminal (`done`/`skipped`/`failed`) also stay on the merge/finalization retry path. - Manual retry now clears the full persisted retry-budget counter set (`stuckKillCount`, `recoveryRetryCount`, `taskDoneRetryCount`, `worktreeSessionRetryCount`, `workflowStepRetries`, `verificationFailureCount`, `postReviewFixCount`, `mergeConflictBounceCount`, `branchConflictRecoveryCount`, `reviewerContextRetryCount`, `reviewerFallbackRetryCount`, `completionHandoffLimboRecoveryCount`, `mergeAuditBounceCount`) plus `nextRecoveryAt`; merge retry counters clear only on merge-failure/generic retry paths. `retrySummary` is recomputed from persisted counters at read time, so manual retry resets the dashboard retry badge/details back to zero immediately. - - Persisted executor session state is resumed only when it still matches the task's current worktree context. If a retry fails with `Refusing to start coding agent in missing worktree: ...` and the persisted session points at stale worktree metadata, recovery clears stale session pointers and retries fresh so review retries do not reopen deleted worktree paths. + - Persisted executor session state is resumed only when it still matches the task's current worktree context. If a retry fails with `Refusing to start coding agent in missing worktree: ...` (or the incomplete/unregistered worktree variants) and the persisted session points at stale worktree metadata, recovery clears stale session pointers and retries fresh so review retries do not reopen deleted worktree paths. Merge-active self-healing resets the exhausted session-start counter only after a guarded stale-metadata clear and increments `recoveryRetryCount` so repeated recurrences still escalate for human inspection. The same supported reset is available from dashboard retry, CLI `task retry`, and `fn_task_retry` even when the row is stuck in a merge-active status (`merging`, `merging-pr`, `merging-fix`): only the unusable-worktree session-start signature bypasses the usual merge-active status gate, and the retry clears `worktree`/`branch`/`sessionFile` while preserving step progress. - Merge-confirmed tasks still respect `getTaskMergeBlocker()` before the final `in-review` → `done` move. If merge is confirmed but a blocker remains (for example, incomplete steps), Fusion parks the task in `in-review` with `status: "failed"` and an explicit blocker error instead of retry-looping auto-finalization. - Self-healing can still auto-finalize retry-exhausted failed review tasks when it can prove their branch content already landed on the merge target, so already-merged work does not deadlock in `in-review`. - Repeated engine merge-queue drops now escalate to an explicit recoverable review failure: if auto-recovery hits `Auto-merge starvation:` in the task `error`, Fusion has already seen three consecutive enqueue attempts rejected by the engine merge queue. Operators can recover by clearing the failed state from the dashboard, which lets the usual unpause/clear flow re-attempt merge once the underlying queue wedge is resolved. diff --git a/packages/cli/src/__tests__/extension.test.ts b/packages/cli/src/__tests__/extension.test.ts index 317997f301..47610fe473 100644 --- a/packages/cli/src/__tests__/extension.test.ts +++ b/packages/cli/src/__tests__/extension.test.ts @@ -3596,6 +3596,70 @@ describe("fn pi extension (runnable structured-output regression slice)", () => expect(task.retrySummary?.total ?? 0).toBe(0); }; + it("moves merge-active missing-worktree session failures to todo with phantom metadata cleared", async () => { + const store = createStore(); + await store.init(); + + const task = await store.createTask({ + title: "missing-worktree merge-active task", + description: "test", + column: "todo", + }); + await store.updateTask(task.id, { + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "pending" }, + ], + }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "merging", + error: "Refusing to start coding agent in missing worktree: /tmp/fusion-missing-worktree", + worktree: "/tmp/fusion-missing-worktree", + branch: `fusion/${task.id}`, + sessionFile: "/tmp/fusion-session.json", + mergeRetries: 3, + worktreeSessionRetryCount: 3, + nextRecoveryAt: new Date(Date.now() + 60_000).toISOString(), + ...nonZeroRetryCounters, + }); + + const retryTool = api.tools.get("fn_task_retry")!; + const result = await retryTool.execute("retry-missing-worktree-merge-active", { id: task.id }, undefined, undefined, makeCtx(tmpDir)); + + expect(result.isError).toBeFalsy(); + expect(result.details.newColumn).toBe("todo"); + + const updated = await store.getTask(task.id); + expect(updated?.column).toBe("todo"); + expect(updated?.status).toBeFalsy(); + expect(updated?.error).toBeFalsy(); + expect(updated?.worktree).toBeFalsy(); + expect(updated?.branch).toBeFalsy(); + expect(updated?.sessionFile).toBeFalsy(); + expect(updated?.steps[0].status).toBe("done"); + expectRetryCountersReset(updated); + expect(updated?.mergeRetries).toBe(0); + }); + + it("rejects unrelated merge-active tasks", async () => { + const store = createStore(); + await store.init(); + + const task = await store.createTask({ title: "ordinary merge-active task", description: "test", column: "todo" }); + await store.updateTask(task.id, { steps: [{ name: "Step 0", status: "done" }] }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { status: "merging", error: "ordinary merge still running", mergeRetries: 2 }); + + const retryTool = api.tools.get("fn_task_retry")!; + const result = await retryTool.execute("retry-unrelated-merge-active", { id: task.id }, undefined, undefined, makeCtx(tmpDir)); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("not in a retryable state"); + }); + it("clears the deadlock auto-pause for execution-failed in-review retries", async () => { const store = createStore(); await store.init(); diff --git a/packages/cli/src/__tests__/task-retry.test.ts b/packages/cli/src/__tests__/task-retry.test.ts index 2e4c794fac..aca71aa632 100644 --- a/packages/cli/src/__tests__/task-retry.test.ts +++ b/packages/cli/src/__tests__/task-retry.test.ts @@ -28,6 +28,55 @@ describe("runTaskRetry", () => { return store; } + it("retries merge-active missing-worktree session failures by clearing phantom metadata", async () => { + const store = await createStore(); + const task = await store.createTask({ + title: "missing worktree merge-active task", + description: "test", + column: "todo", + }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "merging", + error: "Refusing to start coding agent in missing worktree: /tmp/fusion-missing-worktree", + worktree: "/tmp/fusion-missing-worktree", + branch: `fusion/${task.id}`, + sessionFile: "/tmp/fusion-session.json", + steps: [{ name: "implemented", status: "done" }, { name: "fix", status: "pending" }], + worktreeSessionRetryCount: 3, + mergeRetries: 3, + }); + + await runTaskRetry(task.id); + + const verificationStore = await createStore(); + const updated = await verificationStore.getTask(task.id); + expect(updated.column).toBe("todo"); + expect(updated.status).toBeUndefined(); + expect(updated.error).toBeUndefined(); + expect(updated.worktree).toBeUndefined(); + expect(updated.branch).toBeUndefined(); + expect(updated.sessionFile).toBeUndefined(); + expect(updated.worktreeSessionRetryCount).toBe(0); + expect(updated.mergeRetries).toBe(0); + expect(updated.steps?.[0]?.status).toBe("done"); + }); + + it("rejects unrelated merge-active tasks without the missing-worktree signature", async () => { + const store = await createStore(); + const task = await store.createTask({ title: "ordinary merge", description: "test", column: "todo" }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "merging", + error: "ordinary merge still running", + steps: [{ name: "implemented", status: "done" }], + }); + + await expect(runTaskRetry(task.id)).rejects.toThrow(/not in a retryable state/); + }); + it("clears the deadlock auto-pause when retrying a failed task", async () => { const store = await createStore(); const task = await store.createTask({ diff --git a/packages/cli/src/commands/task.ts b/packages/cli/src/commands/task.ts index b818e19204..239888ede7 100644 --- a/packages/cli/src/commands/task.ts +++ b/packages/cli/src/commands/task.ts @@ -1,5 +1,5 @@ import { TaskStore, COLUMNS, COLUMN_LABELS, CentralCore, buildAutoPauseClearPatch, buildManualRetryResetPatch, extractIntentSignature, findNearDuplicates, getTaskDuplicateLineage, isWorkspaceTask, reconcileDeterministicDuplicate, resolveTaskGithubTracking, runDeterministicDuplicateGuard, type Settings, type Column, type ColumnId, type StepStatus, type AgentLogType, type AgentLogEntry, type IntentSignature, type NearDuplicateCandidate, type NearDuplicateMatch, type TaskDependencyMutation } from "@fusion/core"; -import { runAiMerge, landWorkspaceTask } from "@fusion/engine"; +import { isInReviewMissingWorktreeSessionStartFailure, runAiMerge, landWorkspaceTask } from "@fusion/engine"; import { createInterface } from "node:readline/promises"; import type { PlanningQuestion, PlanningSummary } from "@fusion/core"; import { createSession, submitResponse, RateLimitError, SessionNotFoundError, InvalidSessionStateError } from "@fusion/dashboard/planning"; @@ -1282,9 +1282,14 @@ export async function runTaskRetry(id: string, projectName?: string) { task.status === "stuck-killed" || isInReviewExecutionStall || isInReviewMergeRetryStall); + /* + FNXC:MissingWorktreeRetry 2026-07-10-18:28: + Upstream #1992 requires operator retry to recover an in-review task whose session start refused a missing/incomplete/unregistered worktree even when the row is stuck in an invalid merge-active status. This signature-only bypass clears stale session metadata instead of requiring a valid `merging` transition. + */ + const isMissingWorktreeSessionRetry = isInReviewMissingWorktreeSessionStartFailure(task); // Validate task is in a retryable state - if (task.status !== 'failed' && task.status !== 'stuck-killed' && !isInReviewRetry) { + if (task.status !== 'failed' && task.status !== 'stuck-killed' && !isInReviewRetry && !isMissingWorktreeSessionRetry) { throw new Error(`Task ${id} is not in a retryable state (status: ${task.status || 'none'})`); } @@ -1292,6 +1297,25 @@ export async function runTaskRetry(id: string, projectName?: string) { const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; const retryLogSuffix = clearedDeadlockAutoPause ? ", cleared deadlock auto-pause" : ""; + if (isMissingWorktreeSessionRetry) { + await retryBoardCall(context, id, "move task", () => context.store.moveTask(id, "todo", { preserveProgress: true })); + await retryBoardCall(context, id, "update task", () => context.store.updateTask(id, { + status: null, + error: null, + worktree: null, + branch: null, + sessionFile: null, + ...autoPauseClearPatch, + ...buildManualRetryResetPatch({ resetMergeRetries: true }), + })); + await retryBoardCall(context, id, "log entry", () => context.store.logEntry(id, `Retry requested from CLI (unusable worktree session-start recovery → todo, preserving progress${retryLogSuffix})`)); + + console.log(); + console.log(` ✓ Retried ${id} → todo (unusable worktree session metadata cleared)`); + console.log(); + return; + } + // In-review retry: distinguish between execution failures (incomplete steps) // and merge failures (all steps done). if (isInReviewRetry) { diff --git a/packages/cli/src/extension.ts b/packages/cli/src/extension.ts index 831d330d7c..733b4452e1 100644 --- a/packages/cli/src/extension.ts +++ b/packages/cli/src/extension.ts @@ -64,6 +64,7 @@ import { workflowDeleteParams, workflowSettingsParams, traitListParams, + isInReviewMissingWorktreeSessionStartFailure, } from "@fusion/engine"; import * as dashboard from "@fusion/dashboard"; import { resolve, relative, isAbsolute, sep, basename, extname, join } from "node:path"; @@ -1385,9 +1386,14 @@ export default function kbExtension(pi: ExtensionAPI) { task.status === "stuck-killed" || isInReviewExecutionStall || isInReviewMergeRetryStall); + /* + FNXC:MissingWorktreeRetry 2026-07-10-18:30: + Upstream #1992 requires fn_task_retry to recover an in-review unusable-worktree session-start failure even when status remains merge-active. Keep this status bypass constrained to the centrally classified missing/incomplete/unregistered worktree signature. + */ + const isMissingWorktreeSessionRetry = isInReviewMissingWorktreeSessionStartFailure(task); // Validate task is in a retryable state - if (task.status !== 'failed' && task.status !== 'stuck-killed' && !isInReviewRetry) { + if (task.status !== 'failed' && task.status !== 'stuck-killed' && !isInReviewRetry && !isMissingWorktreeSessionRetry) { return { content: [{ type: "text", text: `Task ${params.id} is not in a retryable state (status: ${task.status || 'none'})` }], isError: true, @@ -1399,6 +1405,24 @@ export default function kbExtension(pi: ExtensionAPI) { const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; const retryLogSuffix = clearedDeadlockAutoPause ? ", cleared deadlock auto-pause" : ""; + if (isMissingWorktreeSessionRetry) { + await store.updateTask(params.id, { + status: null, + error: null, + worktree: null, + branch: null, + sessionFile: null, + ...autoPauseClearPatch, + ...buildManualRetryResetPatch({ resetMergeRetries: true }), + }); + await store.logEntry(params.id, `Retry requested via Fusion extension (unusable worktree session-start recovery → todo, preserving progress${retryLogSuffix})`); + await store.moveTask(params.id, "todo", { preserveProgress: true }); + return { + content: [{ type: "text", text: `Retried ${params.id} → todo (unusable worktree session metadata cleared)` }], + details: { taskId: params.id, newColumn: 'todo' }, + }; + } + // In-review retry: distinguish between execution failures and merge failures. if (isInReviewRetry) { if (isExecutionFailureInReview) { diff --git a/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts b/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts index f978109ce4..6f96c79eda 100644 --- a/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts +++ b/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts @@ -813,6 +813,58 @@ describe("POST /tasks/:id/retry", () => { expect(store.moveTask).toHaveBeenCalledWith("KB-001", "todo"); }); + it("retries merge-active missing-worktree session failures by clearing phantom metadata", async () => { + const reviewTask = { + ...FAKE_TASK_DETAIL, + column: "in-review" as const, + status: "merging", + error: "Refusing to start coding agent in missing worktree: /tmp/fusion-missing-worktree", + worktree: "/tmp/fusion-missing-worktree", + branch: "fusion/KB-001", + sessionFile: "/tmp/fusion-session.json", + worktreeSessionRetryCount: 3, + mergeRetries: 3, + steps: [{ name: "Step 0", status: "done" }, { name: "Step 1", status: "pending" }], + }; + const movedTask = { ...reviewTask, column: "todo" as const, status: undefined, worktree: undefined, branch: undefined, sessionFile: undefined }; + (store.getTask as ReturnType).mockResolvedValueOnce(reviewTask); + (store.updateTask as ReturnType).mockResolvedValue(reviewTask); + (store.moveTask as ReturnType).mockResolvedValue(movedTask); + + const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/retry", JSON.stringify({}), { + "Content-Type": "application/json", + }); + + expect(res.status).toBe(200); + expect(store.clearWorkflowRunStepInstances).toHaveBeenCalledWith("KB-001"); + expect(store.updateTask).toHaveBeenCalledWith("KB-001", { + status: null, + error: null, + worktree: null, + branch: null, + sessionFile: null, + ...buildManualRetryResetPatch({ resetMergeRetries: true }), + }); + expect(store.moveTask).toHaveBeenCalledWith("KB-001", "todo", { preserveProgress: true }); + expect(store.logEntry).toHaveBeenCalledWith( + "KB-001", + "Retry requested from dashboard (unusable worktree session-start recovery → todo, preserving progress)", + ); + }); + + it("keeps unrelated merge-active tasks non-retryable", async () => { + const activeTask = { ...FAKE_TASK_DETAIL, column: "in-review" as const, status: "merging", error: "ordinary merge still running" }; + (store.getTask as ReturnType).mockResolvedValue(activeTask); + + const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/retry", JSON.stringify({}), { + "Content-Type": "application/json", + }); + + expect(res.status).toBe(400); + expect(res.body.error).toContain("not in a retryable state"); + expect(engine.clearTaskPauseAbortState).not.toHaveBeenCalled(); + }); + it("returns 400 when task is not in a retryable state", async () => { const activeTask = { ...FAKE_TASK_DETAIL, status: "executing" }; (store.getTask as ReturnType).mockResolvedValue(activeTask); diff --git a/packages/dashboard/src/routes/register-task-workflow-routes.ts b/packages/dashboard/src/routes/register-task-workflow-routes.ts index 731926e4df..b8d272ccfb 100644 --- a/packages/dashboard/src/routes/register-task-workflow-routes.ts +++ b/packages/dashboard/src/routes/register-task-workflow-routes.ts @@ -61,6 +61,7 @@ import { createAiUndoTask, prepareRevertPrBranch, prepareWorkspaceRevertPrBranches, + isInReviewMissingWorktreeSessionStartFailure, type AiUndoTaskResult, type PrepareRevertPrBranchResult, type PrepareWorkspaceRevertPrBranchesResult, @@ -2310,7 +2311,12 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork task.status === "stuck-killed" || isInReviewExecutionStall || isInReviewMergeRetryStall); - if (task.status !== "failed" && task.status !== "stuck-killed" && !retrySpecification && !isInReviewRetry) { + /* + FNXC:MissingWorktreeRetry 2026-07-10-18:32: + Dashboard retry must support the upstream #1992 signature where the task is stranded in a merge-active status but the durable failure is an unusable worktree session-start assertion. Only that classifier bypasses the merge-active status gate. + */ + const isMissingWorktreeSessionRetry = isInReviewMissingWorktreeSessionStartFailure(task); + if (task.status !== "failed" && task.status !== "stuck-killed" && !retrySpecification && !isInReviewRetry && !isMissingWorktreeSessionRetry) { throw badRequest(`Task is not in a retryable state (current status: ${task.status || 'none'})`); } @@ -2324,6 +2330,23 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; const retryLogSuffix = clearedDeadlockAutoPause ? ", cleared deadlock auto-pause" : ""; + if (isMissingWorktreeSessionRetry) { + clearRebuiltSpecWorkflowPins(scopedStore, req.params.id); + await scopedStore.updateTask(req.params.id, { + status: null, + error: null, + worktree: null, + branch: null, + sessionFile: null, + ...autoPauseClearPatch, + ...buildManualRetryResetPatch({ resetMergeRetries: true }), + }); + await scopedStore.logEntry(req.params.id, `Retry requested from dashboard (unusable worktree session-start recovery → todo, preserving progress${retryLogSuffix})`); + const updated = await scopedStore.moveTask(req.params.id, "todo", { preserveProgress: true }); + res.json(updated); + return; + } + // In-review retry: distinguish between execution failures (incomplete steps) // and merge failures (all steps done). if (isInReviewRetry) { diff --git a/packages/dashboard/src/test/mockCoreEngine.ts b/packages/dashboard/src/test/mockCoreEngine.ts index f45ecc5d4c..3729d6a6ae 100644 --- a/packages/dashboard/src/test/mockCoreEngine.ts +++ b/packages/dashboard/src/test/mockCoreEngine.ts @@ -66,6 +66,17 @@ export function createEngineMock(overrides: AnyModule = {}): AnyModule { */ createChatTaskDocumentTools: vi.fn(() => []), createChatArtifactTools: vi.fn(() => []), + /* + FNXC:MissingWorktreeRetry 2026-07-10-18:45: + Dashboard route tests mock @fusion/engine wholesale; the retry route must still exercise the upstream #1992 classifier so merge-active unusable-worktree failures are admitted while unrelated merging rows remain rejected. + */ + isInReviewMissingWorktreeSessionStartFailure: vi.fn((task: { column?: string; error?: unknown }) => ( + task.column === "in-review" + && typeof task.error === "string" + && (task.error.includes("Refusing to start coding agent in missing worktree:") + || task.error.includes("Refusing to start coding agent in incomplete worktree:") + || task.error.includes("Refusing to start coding agent in unregistered git worktree:")) + )), // FNXC:McpConfig 2026-07-02-13:45: Planning/mission route tests share this engine mock; MCP resolution must return the full shaped empty result so readonly session creation can proceed without importing real engine stores. resolveMcpServersForStore: vi.fn(async () => ({ servers: [], errors: [] })), ...overrides, diff --git a/packages/engine/src/__tests__/restart-recovery-coordinator.test.ts b/packages/engine/src/__tests__/restart-recovery-coordinator.test.ts index 0897f14ae3..d297dd7c71 100644 --- a/packages/engine/src/__tests__/restart-recovery-coordinator.test.ts +++ b/packages/engine/src/__tests__/restart-recovery-coordinator.test.ts @@ -4,6 +4,7 @@ import { RestartRecoveryCoordinator, extractMissingWorktreePathFromSessionStartFailure, isMissingWorktreeSessionStartFailure, + isMergeActiveMissingWorktreeSessionStartFailure, isRecoverableMissingWorktreeReviewFailure, isRecoverableMissingWorktreeReviewFailureNoProgress, isRecoverableMissingWorktreeReviewFailureWithProgress, @@ -76,6 +77,25 @@ describe("RestartRecoveryCoordinator", () => { } }); + it("recognizes missing-worktree failures in every merge-active review status", () => { + const baseTask = createTask({ + column: "in-review", + paused: false, + error: "Refusing to start coding agent in missing worktree: /tmp/wt", + steps: [{ id: "s1", title: "step", status: "done" }] as any, + }); + + for (const status of ["merging", "merging-pr", "merging-fix"] as const) { + const task = { ...baseTask, status }; + expect(isMergeActiveMissingWorktreeSessionStartFailure(task)).toBe(true); + expect(isRecoverableMissingWorktreeReviewFailure(task)).toBe(true); + } + + expect(isMergeActiveMissingWorktreeSessionStartFailure({ ...baseTask, status: "failed" })).toBe(false); + expect(isMergeActiveMissingWorktreeSessionStartFailure({ ...baseTask, status: null as any })).toBe(false); + expect(isMergeActiveMissingWorktreeSessionStartFailure({ ...baseTask, status: "merging", error: "ordinary merge failure" })).toBe(false); + }); + it("requeues interrupted failed tasks with no progress, then resumes remaining orphans", async () => { const store = { listTasks: vi.fn().mockResolvedValue([ diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 4fe89ed866..991c068c79 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -2757,6 +2757,216 @@ describe("SelfHealingManager", () => { managerWithRecovery.stop(); }); + it("recovers merge-active unusable-worktree failures before merge re-drive can reuse phantom metadata", async () => { + const enqueueMerge = vi.fn(); + const managerWithRecovery = new SelfHealingManager(store, { + rootDir: "/tmp/test-project", + enqueueMerge, + }); + const strandedTasks = (["merging", "merging-pr", "merging-fix"] as const).map((status) => ({ + id: `FN-7802-${status}`, + column: "in-review", + paused: false, + status, + scopeOverride: true, + scopeOverrideReason: "operator requested main-checkout retry", + worktree: `/tmp/project/.worktrees/${status}-phantom`, + branch: `fusion/fn-7802-${status}`, + sessionFile: `/tmp/project/.fusion/sessions/${status}.json`, + error: `Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/${status}-phantom`, + worktreeSessionRetryCount: 3, + steps: [{ status: "done" }, { status: "pending" }], + log: [], + })); + (store.listTasks as ReturnType).mockResolvedValue(strandedTasks); + + const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures(); + + expect(result).toBe(3); + for (const task of strandedTasks) { + expect(store.updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({ + status: null, + error: null, + worktree: null, + branch: null, + sessionFile: null, + worktreeSessionRetryCount: 0, + recoveryRetryCount: 1, + })); + expect(store.moveTask).toHaveBeenCalledWith(task.id, "todo", { preserveProgress: true, moveSource: "engine", recoveryRehome: true }); + } + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:reconcile-missing-worktree-merge-active" })); + expect(enqueueMerge).not.toHaveBeenCalled(); + + managerWithRecovery.stop(); + }); + + it("bounds repeated merge-active stale-metadata clears with recoveryRetryCount", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-MERGE-ACTIVE-CAP", + column: "in-review", + paused: false, + status: "merging", + scopeOverride: true, + worktree: "/tmp/project/.worktrees/fn-7802-cap", + branch: "fusion/FN-7802-MERGE-ACTIVE-CAP", + sessionFile: "/tmp/project/.fusion/sessions/fn-7802-cap.json", + error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-7802-cap", + worktreeSessionRetryCount: 3, + recoveryRetryCount: 3, + steps: [{ status: "done" }, { status: "pending" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures(); + + expect(result).toBe(0); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-7802-MERGE-ACTIVE-CAP", + "Auto-recovery exhausted (3/3) for merge-active unusable-worktree stale-metadata clears — leaving in-review for human inspection", + ); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "task:auto-recover-worktree-session-exhausted", + metadata: expect.objectContaining({ counter: "recoveryRetryCount", source: "merge-active-sweep" }), + })); + + managerWithRecovery.stop(); + }); + + it("recovers merge-active unusable-worktree failures even when task.worktree is already null", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-NULL-WORKTREE", + column: "in-review", + paused: false, + status: "merging-fix", + worktree: null, + branch: "fusion/FN-7802-NULL-WORKTREE", + sessionFile: "/tmp/project/.fusion/sessions/fn-7802-null.json", + error: "Refusing to start coding agent in unregistered git worktree: /tmp/project/.worktrees/fn-7802-null", + worktreeSessionRetryCount: 3, + steps: [{ status: "done" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures(); + + expect(result).toBe(1); + expect(store.updateTask).toHaveBeenCalledWith("FN-7802-NULL-WORKTREE", expect.objectContaining({ + worktree: null, + branch: null, + sessionFile: null, + worktreeSessionRetryCount: 0, + recoveryRetryCount: 1, + })); + expect(store.moveTask).toHaveBeenCalledWith("FN-7802-NULL-WORKTREE", "todo", { preserveProgress: true, moveSource: "engine", recoveryRehome: true }); + managerWithRecovery.stop(); + }); + + it("does not automate merge-active unusable-worktree recovery when auto-merge is off", async () => { + (store.getSettings as ReturnType).mockResolvedValue({ autoMerge: false, globalPause: false, enginePaused: false }); + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-AUTOMERGE-OFF", + column: "in-review", + paused: false, + status: "merging", + worktree: "/tmp/project/.worktrees/fn-7802-auto-off", + branch: "fusion/FN-7802-AUTOMERGE-OFF", + error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-7802-auto-off", + steps: [{ status: "done" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures(); + + expect(result).toBe(0); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "task:reconcile-missing-worktree-merge-active-no-action", + metadata: expect.objectContaining({ reason: "auto-merge-off" }), + })); + managerWithRecovery.stop(); + }); + + it("emits no-action for workspace tasks instead of single-repo missing-worktree recovery", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-WORKSPACE", + column: "in-review", + paused: false, + status: "merging", + worktree: null, + workspaceWorktrees: { app: { worktree: "/tmp/ws/app", branch: "fusion/FN-7802-WORKSPACE" } }, + error: "Refusing to start coding agent in missing worktree: /tmp/ws/app", + steps: [{ status: "done" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures(); + + expect(result).toBe(0); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "task:reconcile-missing-worktree-merge-active-no-action", + metadata: expect.objectContaining({ reason: "workspace-task" }), + })); + managerWithRecovery.stop(); + }); + + it("leaves live worktrees and status-none review rows out of merge-active recovery", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + mockedClassifyTaskWorktree.mockResolvedValueOnce({ ok: true }); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-LIVE", + column: "in-review", + paused: false, + status: "merging", + worktree: "/tmp/project/.worktrees/fn-7802-live", + branch: "fusion/FN-7802-LIVE", + error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-7802-live", + steps: [{ status: "done" }], + log: [], + }, + { + id: "FN-7802-NONE", + column: "in-review", + paused: false, + status: null, + worktree: "/tmp/project/.worktrees/fn-7802-none", + branch: "fusion/FN-7802-NONE", + error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-7802-none", + steps: [{ status: "done" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures(); + + expect(result).toBe(0); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "task:reconcile-missing-worktree-merge-active-no-action", + target: "FN-7802-LIVE", + })); + managerWithRecovery.stop(); + }); + it("does not requeue non-matching in-review failures", async () => { const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project", @@ -2802,6 +3012,93 @@ describe("SelfHealingManager", () => { }); }); + describe("reconcileTaskWorktreeMetadata", () => { + beforeEach(() => { + (store.getSettings as ReturnType).mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false }); + }); + + it("clears phantom active worktree metadata for scopeOverride main-checkout tasks", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + mockedExistsSync.mockReturnValue(false); + mockedGetRegisteredWorktreeBranchMap.mockResolvedValue(new Map()); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-SCOPE", + column: "in-review", + paused: false, + status: "merging-fix", + scopeOverride: true, + worktree: "/tmp/project/.worktrees/fn-7802-phantom", + branch: "fusion/FN-7802-SCOPE", + sessionFile: "/tmp/project/.fusion/sessions/fn-7802-scope.json", + steps: [{ status: "done" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.reconcileTaskWorktreeMetadata(); + + expect(result).toBe(1); + expect(store.updateTask).toHaveBeenCalledWith("FN-7802-SCOPE", { worktree: null, branch: null, sessionFile: null }); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:auto-recover-worktree-metadata-cleared" })); + managerWithRecovery.stop(); + }); + + it("does NOT clear worktree metadata for a scopeOverride task that is genuinely in-progress (FN-5256 guard)", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + mockedExistsSync.mockReturnValue(false); + mockedGetRegisteredWorktreeBranchMap.mockResolvedValue(new Map()); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-SCOPE-INPROGRESS", + column: "in-progress", + paused: false, + status: null, + scopeOverride: true, + worktree: "/tmp/project/.worktrees/fn-7802-live", + branch: "fusion/FN-7802-SCOPE-INPROGRESS", + sessionFile: "/tmp/project/.fusion/sessions/fn-7802-live.json", + steps: [{ status: "in-progress" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.reconcileTaskWorktreeMetadata(); + + expect(result).toBe(0); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:auto-recover-worktree-metadata-skipped-active" })); + managerWithRecovery.stop(); + }); + + it("does NOT clear worktree metadata for a scopeOverride in-review task mid-step (status: null)", async () => { + const managerWithRecovery = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + mockedExistsSync.mockReturnValue(false); + mockedGetRegisteredWorktreeBranchMap.mockResolvedValue(new Map()); + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-7802-SCOPE-REVIEW-STEP", + column: "in-review", + paused: false, + status: null, + scopeOverride: true, + worktree: "/tmp/project/.worktrees/fn-7802-review-live", + branch: "fusion/FN-7802-SCOPE-REVIEW-STEP", + sessionFile: "/tmp/project/.fusion/sessions/fn-7802-review-live.json", + steps: [{ status: "in-progress" }], + log: [], + }, + ]); + + const result = await managerWithRecovery.reconcileTaskWorktreeMetadata(); + + expect(result).toBe(0); + expect(store.updateTask).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ mutationType: "task:auto-recover-worktree-metadata-skipped-active" })); + managerWithRecovery.stop(); + }); + }); + describe("recoverMisclassifiedFailures", () => { it("clears failed status when all steps are done and error is no-task_done", async () => { const managerWithRecovery = new SelfHealingManager(store, { diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 6548cf1bf9..1b62544c42 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -115,6 +115,19 @@ export { type WorkflowRuntimePrimitiveProvider, type WorkflowRuntimePrimitiveFactory, } from "./workflow-runtime-primitive-provider.js"; +export { + MERGE_ACTIVE_MISSING_WORKTREE_STATUSES, + MISSING_WORKTREE_SESSION_PREFIXES, + classifyMissingWorktreeSessionStartFailure, + extractMissingWorktreePathFromSessionStartFailure, + hasStepProgress, + isInReviewMissingWorktreeSessionStartFailure, + isMergeActiveMissingWorktreeSessionStartFailure, + isMissingWorktreeSessionStartFailure, + isRecoverableMissingWorktreeReviewFailure, + isRecoverableMissingWorktreeReviewFailureNoProgress, + isRecoverableMissingWorktreeReviewFailureWithProgress, +} from "./restart-recovery-coordinator.js"; export { WorkflowCustomNodeExecutionService, type WorkflowCustomNodeExecutionServiceDeps, diff --git a/packages/engine/src/restart-recovery-coordinator.ts b/packages/engine/src/restart-recovery-coordinator.ts index de5418e1de..085f7be8f3 100644 --- a/packages/engine/src/restart-recovery-coordinator.ts +++ b/packages/engine/src/restart-recovery-coordinator.ts @@ -82,9 +82,26 @@ export function isRecoverableMissingWorktreeReviewFailureNoProgress(task: Task): && !hasStepProgress(task); } +export const MERGE_ACTIVE_MISSING_WORKTREE_STATUSES = ["merging", "merging-pr", "merging-fix"] as const; +const MERGE_ACTIVE_MISSING_WORKTREE_STATUS_SET = new Set(MERGE_ACTIVE_MISSING_WORKTREE_STATUSES); + +export function isMergeActiveMissingWorktreeSessionStartFailure(task: Task): boolean { + return task.column === "in-review" + && !task.paused + && typeof task.status === "string" + && MERGE_ACTIVE_MISSING_WORKTREE_STATUS_SET.has(task.status) + && isMissingWorktreeSessionStartFailure(task.error); +} + +export function isInReviewMissingWorktreeSessionStartFailure(task: Task): boolean { + return task.column === "in-review" + && isMissingWorktreeSessionStartFailure(task.error); +} + export function isRecoverableMissingWorktreeReviewFailure(task: Task): boolean { return isRecoverableMissingWorktreeReviewFailureWithProgress(task) - || isRecoverableMissingWorktreeReviewFailureNoProgress(task); + || isRecoverableMissingWorktreeReviewFailureNoProgress(task) + || isMergeActiveMissingWorktreeSessionStartFailure(task); } export class RestartRecoveryCoordinator { diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 1758f87050..32c00e1aa5 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -39,8 +39,10 @@ import { classifyMissingWorktreeSessionStartFailure, extractMissingWorktreePathFromSessionStartFailure, isMissingWorktreeSessionStartFailure, + isMergeActiveMissingWorktreeSessionStartFailure, isRecoverableMissingWorktreeReviewFailureNoProgress, isRecoverableMissingWorktreeReviewFailureWithProgress, + MERGE_ACTIVE_MISSING_WORKTREE_STATUSES, } from "./restart-recovery-coordinator.js"; import { classifyError, extractMissingModulePath, isNonContinuableSessionError, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js"; import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js"; @@ -470,6 +472,7 @@ const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000; */ const MAX_TASK_DONE_RETRIES = 3; export const MAX_WORKTREE_SESSION_RETRIES = 3; +const RECONCILE_SCOPE_OVERRIDE_MERGE_ACTIVE_STATUS_SET = new Set(MERGE_ACTIVE_MISSING_WORKTREE_STATUSES); /** * FNXC:WorkflowLifecycle 2026-06-20-00:00: single source of truth for the * pause-abort park error message markers. The executor's handleGraphFailure @@ -528,13 +531,46 @@ export async function autoRecoverWorktreeSessionStartFailure( task: Task, opts: { failure: unknown; - source: "executor-session-start" | "in-review-sweep" | "resume-guard"; + source: "executor-session-start" | "in-review-sweep" | "merge-active-sweep" | "resume-guard"; auditor: RunAuditor | null; + forceClearWorktreeMetadata?: boolean; + resetRetryBudgetOnStaleMetadataClear?: boolean; + staleMetadataClearRecoveryRetryCount?: number; }, ): Promise<{ outcome: "requeue-todo" | "escalate-exhausted"; retries: number; classification: "missing" | "incomplete" | "unregistered" | "unknown" }> { const classification = classifyMissingWorktreeSessionStartFailure(opts.failure); - const nextCount = (task.worktreeSessionRetryCount ?? 0) + 1; - if (nextCount > MAX_WORKTREE_SESSION_RETRIES) { + /* + FNXC:MissingWorktreeRecovery 2026-07-10-18:15: + Upstream #1992 showed merge-active review-fix sessions can exhaust the unusable-worktree retry budget while every retry reuses the same phantom worktree metadata. When a guarded recovery clears that stale worktree/branch/session reference, the next dispatch must get a fresh session-start retry budget instead of inheriting the exhausted context that caused the strand. + + FNXC:MissingWorktreeRecovery 2026-07-10-21:36: + The merge-active sweep still needs a bounded human-escalation circuit breaker after clearing stale metadata, so it tracks those guarded clears through recoveryRetryCount instead of repeatedly resetting worktreeSessionRetryCount to zero on every recurrence. + */ + const resetRetryBudget = opts.resetRetryBudgetOnStaleMetadataClear === true; + const staleMetadataClearRecoveryRetryCount = opts.staleMetadataClearRecoveryRetryCount; + const currentStaleMetadataClearRecoveryCount = staleMetadataClearRecoveryRetryCount ?? 0; + const nextStaleMetadataClearRecoveryCount = staleMetadataClearRecoveryRetryCount === undefined + ? undefined + : currentStaleMetadataClearRecoveryCount + 1; + if (nextStaleMetadataClearRecoveryCount !== undefined && nextStaleMetadataClearRecoveryCount > MAX_WORKTREE_SESSION_RETRIES) { + await store.logEntry( + task.id, + `Auto-recovery exhausted (${MAX_WORKTREE_SESSION_RETRIES}/${MAX_WORKTREE_SESSION_RETRIES}) for merge-active unusable-worktree stale-metadata clears — leaving in-review for human inspection`, + ); + await opts.auditor?.database({ + type: "task:auto-recover-worktree-session-exhausted", + target: task.id, + metadata: { + retries: currentStaleMetadataClearRecoveryCount, + maxRetries: MAX_WORKTREE_SESSION_RETRIES, + source: opts.source, + counter: "recoveryRetryCount", + }, + }); + return { outcome: "escalate-exhausted", retries: currentStaleMetadataClearRecoveryCount, classification }; + } + const nextCount = resetRetryBudget ? 0 : (task.worktreeSessionRetryCount ?? 0) + 1; + if (!resetRetryBudget && nextCount > MAX_WORKTREE_SESSION_RETRIES) { await store.logEntry( task.id, `Auto-recovery exhausted (${MAX_WORKTREE_SESSION_RETRIES}/${MAX_WORKTREE_SESSION_RETRIES}) for unusable-worktree session-start failure — leaving in-review for human inspection`, @@ -558,13 +594,15 @@ export async function autoRecoverWorktreeSessionStartFailure( && typeof missingWorktreePath === "string" && missingWorktreePath.length > 0 && resolve(staleWorktree) !== resolve(missingWorktreePath); const noProgress = !hasStepProgress(task); + const forceClearWorktreeMetadata = opts.forceClearWorktreeMetadata === true; await store.updateTask(task.id, { status: null, error: null, worktreeSessionRetryCount: nextCount, - worktree: noProgress ? null : (hasMismatchedLiveWorktree ? staleWorktree : null), - branch: noProgress ? null : (hasMismatchedLiveWorktree ? task.branch ?? null : null), + ...(nextStaleMetadataClearRecoveryCount === undefined ? {} : { recoveryRetryCount: nextStaleMetadataClearRecoveryCount }), + worktree: (noProgress || forceClearWorktreeMetadata) ? null : (hasMismatchedLiveWorktree ? staleWorktree : null), + branch: (noProgress || forceClearWorktreeMetadata) ? null : (hasMismatchedLiveWorktree ? task.branch ?? null : null), sessionFile: null, }); @@ -576,13 +614,16 @@ export async function autoRecoverWorktreeSessionStartFailure( const failureExcerpt = isMissingWorktreeSessionStartFailure(rawFailureExcerpt) ? "session-start unusable-worktree assertion" : rawFailureExcerpt; + const attemptLabel = resetRetryBudget + ? `retry budget reset from ${task.worktreeSessionRetryCount ?? 0}/${MAX_WORKTREE_SESSION_RETRIES}` + : `attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}`; await store.logEntry( task.id, noProgress - ? `Auto-recovered (no-progress): session-start refused unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})` - : hasMismatchedLiveWorktree - ? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})` - : `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`, + ? `Auto-recovered (no-progress): session-start refused unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (${attemptLabel}, failure: ${failureExcerpt})` + : hasMismatchedLiveWorktree && !forceClearWorktreeMetadata + ? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (${attemptLabel}, failure: ${failureExcerpt})` + : `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (${attemptLabel}, failure: ${failureExcerpt})`, ); if (noProgress) { // #1411: backward recovery move — recoveryRehome skips order-derived adjacency. @@ -1316,6 +1357,7 @@ export class SelfHealingManager { { name: "recover-stranded-completed-todo", fn: () => this.recoverStrandedCompletedTodoTasks().then(() => undefined) }, { name: "stale-incomplete-review", fn: () => this.recoverStaleIncompleteReviewTasks().then(() => undefined) }, { name: "failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps().then(() => undefined) }, + { name: "missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures().then(() => undefined) }, { name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) }, { name: "transient-merge-failures", fn: () => this.recoverTransientMergeFailures().then(() => undefined) }, { name: "done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata().then(() => undefined) }, @@ -1331,7 +1373,6 @@ export class SelfHealingManager { { name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations().then(() => undefined) }, { name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) }, { name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) }, - { name: "missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures().then(() => undefined) }, { name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) }, { name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) }, { name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) }, @@ -2521,6 +2562,7 @@ export class SelfHealingManager { { name: "recover-stranded-completed-todo", fn: () => this.recoverStrandedCompletedTodoTasks() }, { name: "recover-stale-incomplete-review", fn: () => this.recoverStaleIncompleteReviewTasks() }, { name: "recover-failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps() }, + { name: "recover-missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures() }, { name: "recover-interrupted-merging", fn: () => this.recoverInterruptedMergingTasks() }, { name: "recover-transient-merge-failures", fn: () => this.recoverTransientMergeFailures() }, { name: "recover-done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata() }, @@ -2542,7 +2584,6 @@ export class SelfHealingManager { { name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations() }, { name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() }, { name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() }, - { name: "recover-missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures() }, { name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() }, { name: "recover-paused-abort-failures", fn: () => this.recoverPausedAbortFailures() }, { name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() }, @@ -4516,6 +4557,33 @@ export class SelfHealingManager { continue; } + const scopeOverrideMergeActiveSafe = + task.scopeOverride === true + && task.column !== "in-progress" + && (task.column !== "in-review" || (typeof task.status === "string" && RECONCILE_SCOPE_OVERRIDE_MERGE_ACTIVE_STATUS_SET.has(task.status))); + if (scopeOverrideMergeActiveSafe) { + /* + FNXC:MissingWorktreeRecovery 2026-07-10-18:23: + Upstream #1992 reproduced with scopeOverride=1: a main-checkout-only task retained a stale sub-repo worktree pointer, so every session start refused the missing path before scope override could help. When no live fusion/ worktree exists, clear only the phantom metadata; scopeOverride remains a file-scope no-op. + + FNXC:MissingWorktreeRecovery 2026-07-10 (code review): the FN-5256 guard immediately below exists precisely because column==="in-progress"/"in-review" tasks can be live even when this heuristic's existsSync/registered-path check calls them stale (that's the guard's own stated rationale). Restrict this scopeOverride bypass to the narrow #1992 bug shape reproduced in the task report — in-review AND a merge-active sub-status (merging/merging-pr/merging-fix) — so a scopeOverride task that is genuinely in-progress, or in-review mid-step (status: null) with a live but momentarily undetected session, still falls through to the FN-5256 protection instead of having its worktree/branch/sessionFile yanked out from under it. + */ + await this.store.updateTask(task.id, { worktree: null, branch: null, sessionFile: null }); + await this.emitWorktreeMetadataAuditEvent({ + taskId: task.id, + mutationType: "task:auto-recover-worktree-metadata-cleared", + previousWorktree, + newWorktree: null, + previousBranch, + newBranch: null, + }); + worktreeMetadataReconcileLog.log( + `cleared scopeOverride ${task.id}: ${previousWorktree} (${previousBranch ?? ""})`, + ); + repaired++; + continue; + } + // FN-5256: never null out worktree/branch metadata for an active task. If a // live task's worktree looks stale here (and we couldn't rebind to a live // fusion/), the executor's own recovery paths will detect and recreate @@ -10528,9 +10596,9 @@ export class SelfHealingManager { if (settings.globalPause || settings.enginePaused) return 0; const tasks = await this.store.listTasks({ column: "in-review", slim: true }); const candidates = tasks.filter((task) => - allowsAutoMergeProcessing(task, settings) - && (isRecoverableMissingWorktreeReviewFailureWithProgress(task) - || isRecoverableMissingWorktreeReviewFailureNoProgress(task)), + isRecoverableMissingWorktreeReviewFailureWithProgress(task) + || isRecoverableMissingWorktreeReviewFailureNoProgress(task) + || isMergeActiveMissingWorktreeSessionStartFailure(task), ); if (candidates.length === 0) return 0; @@ -10540,17 +10608,10 @@ export class SelfHealingManager { let recovered = 0; for (const task of candidates) { try { - const proof = await this.evaluateBackwardMoveTripleProof(task, { - stage: "missing-worktree-review", - graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS, - stalenessAnchor: task.columnMovedAt ?? task.updatedAt, - reason: "missing-worktree-review-candidate", - }); - if (!proof.ok) { - await this.emitBackwardMoveNoAction(task, "missing-worktree-review", "task:missing-worktree-review-no-action", proof); - continue; - } - + const mergeActiveCandidate = isMergeActiveMissingWorktreeSessionStartFailure(task); + const stage = mergeActiveCandidate ? "missing-worktree-merge-active" : "missing-worktree-review"; + const noActionEvent = mergeActiveCandidate ? "task:reconcile-missing-worktree-merge-active-no-action" : "task:missing-worktree-review-no-action"; + const recoveryEvent = mergeActiveCandidate ? "task:reconcile-missing-worktree-merge-active" : "task:missing-worktree-review"; const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("self-heal", task.id), agentId: "self-healing", @@ -10558,12 +10619,56 @@ export class SelfHealingManager { taskLineageId: task.lineageId, phase: "maintenance", }); + if (!allowsAutoMergeProcessing(task, settings)) { + await auditor.database({ + type: noActionEvent as DatabaseMutationType, + target: task.id, + metadata: { reason: "auto-merge-off", priorStatus: task.status ?? null, priorWorktree: task.worktree ?? null }, + }); + continue; + } + if (isWorkspaceTask(task)) { + await auditor.database({ + type: noActionEvent as DatabaseMutationType, + target: task.id, + metadata: { reason: "workspace-task", priorStatus: task.status ?? null, priorWorktree: task.worktree ?? null }, + }); + continue; + } + const proof = await this.evaluateBackwardMoveTripleProof(task, { + stage, + graceMs: settings.taskStuckTimeoutMs ?? STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS, + stalenessAnchor: task.columnMovedAt ?? task.updatedAt, + reason: mergeActiveCandidate ? "missing-worktree-merge-active-candidate" : "missing-worktree-review-candidate", + extra: { priorStatus: task.status ?? null }, + }); + if (!proof.ok) { + await this.emitBackwardMoveNoAction(task, stage, noActionEvent, proof); + continue; + } + const result = await autoRecoverWorktreeSessionStartFailure(this.store, task, { failure: task.error, - source: "in-review-sweep", + source: mergeActiveCandidate ? "merge-active-sweep" : "in-review-sweep", auditor, + forceClearWorktreeMetadata: mergeActiveCandidate, + resetRetryBudgetOnStaleMetadataClear: mergeActiveCandidate, + staleMetadataClearRecoveryRetryCount: mergeActiveCandidate ? task.recoveryRetryCount ?? 0 : undefined, }); - if (result.outcome === "requeue-todo") recovered++; + if (result.outcome === "requeue-todo") { + await auditor.database({ + type: recoveryEvent as DatabaseMutationType, + target: task.id, + metadata: { + priorStatus: task.status ?? null, + priorWorktree: task.worktree ?? null, + priorBranch: task.branch ?? null, + classification: result.classification, + retries: result.retries, + }, + }); + recovered++; + } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); log.error(`Failed to recover unusable-worktree review failure ${task.id}: ${errorMessage}`);