From 0e69ed9a5bef28119f84f7c57d2fa19f37beeeb6 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 2 Aug 2026 19:05:59 -0700 Subject: [PATCH] fix: stop stale-active-branch rescue spam for done squash leftovers (#3311) ## Summary - Local `fusion/*` branches for **done** tasks kept unique tip SHAs after squash/AI merge, so self-healing logged `stale-active-branch-rescue-needed` on every maintenance sweep without ever deleting them. - **Completion fan-out** now force-deletes the task branch even when unique commits remain (squash-safe). - **`reclaim-stale-active-branches`** force-deletes complete-lane leftovers (`reason=complete-column-unique-commits-force`) and no longer emits rescue-needed for those lanes; non-complete columns still warn and preserve unmerged work. Archived lanes still skip reclaim entirely. Also cleaned up 116 leftover local fusion branches on this machine (done/orphan only; 9 active kept). ## Test plan - [x] `pnpm --filter @fusion/engine exec vitest run src/__tests__/self-healing-completion-fanout.test.ts src/__tests__/self-healing.test.ts -t "SelfHealingManager reclaimStaleActiveBranches|self-healing completion fan-out|force-deletes"` - [ ] After merge/restart engine: confirm logs no longer spam rescue-needed for done tasks - [ ] Confirm active todo/in-progress fusion branches still get rescue-needed when unique and no worktree ## Summary by CodeRabbit * **Bug Fixes** * Improved cleanup of stale task branches after completion, including branches with unique commits remaining after squash or AI-assisted merges. * Completed tasks now reliably remove residual branches and worktree metadata. * Non-completed tasks continue to preserve recoverable branches and display rescue warnings. * Added clearer recovery logging and audit records for forced branch cleanup. --- .changeset/stale-active-branch-done-spam.md | 7 ++ docs/architecture.md | 2 +- .../self-healing-completion-fanout.test.ts | 24 ++++++- .../engine/src/__tests__/self-healing.test.ts | 69 +++++++++++++++++++ packages/engine/src/self-healing.ts | 28 ++++++-- 5 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 .changeset/stale-active-branch-done-spam.md diff --git a/.changeset/stale-active-branch-done-spam.md b/.changeset/stale-active-branch-done-spam.md new file mode 100644 index 0000000000..c07b3416a1 --- /dev/null +++ b/.changeset/stale-active-branch-done-spam.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Stop reclaim log spam for done-task squash branches and delete them after completion. +category: fix +dev: reclaimStaleActiveBranches force-deletes complete-lane leftovers with unique commits; clearCompletionBranchIfSubsumed force-deletes post-done (squash-safe). diff --git a/docs/architecture.md b/docs/architecture.md index 172219c107..a089c36c58 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2246,7 +2246,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f - **Planning-recovery no-regression (FN-7977/FN-8361)**: a provider, model-selection, transport, or deterministic planning failure may only mutate a task after re-reading its live row and proving it remains in the planning stage. Execution/terminal columns, a worktree, or materialized steps prove advancement; stale triage recovery must leave that column, status, worktree, step progress, and `PROMPT.md` untouched. Patch writes use `updateTaskAtomic`; recovery releases use `moveTaskIf` and recovery duplicate deletion uses `deleteTaskIf`, each read-predicate-mutating under one task lock. A predicate skip is normal scheduler advancement and short-circuits the remaining recovery body. A genuine Plan Review `REVISE` remains a separate, explicit replan signal. - **Orphan `fusion/*` branches**: branches with zero unique commits vs `main` are pruned by `cleanupOrphanedBranches` (`branch:orphan-prune`). Branches with unique commits are not auto-rescued; operators inspect and clean them manually via standard git tooling (`git branch -D`, `git worktree remove`, etc.). -- **Stale active branches**: self-healing's `reclaim-stale-active-branches` stage prunes a `fusion/` branch with zero unique commits when no usable worktree mapping exists, then clears `task.branch`/`task.worktree`/`task.baseCommitSha`. It must defer reclaim (emit `branch:stale-active-reclaim-deferred`) when the task worktree is in `activeSessionRegistry`, when `executionStartedAt` is within `STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS` (10 minutes), or when the mapped worktree has uncommitted changes. +- **Stale active branches**: self-healing's `reclaim-stale-active-branches` stage prunes a `fusion/` branch with zero unique commits when no usable worktree mapping exists, then clears `task.branch`/`task.worktree`/`task.baseCommitSha`. For **complete**-role columns it also force-deletes branches that still have unique commits vs the integration base (squash/AI-merge tip SHAs are not ancestors of main) with reason `complete-column-unique-commits-force`, and does **not** emit the non-actionable `stale-active-branch-rescue-needed` warn for those lanes. Non-complete columns with unique commits still warn rescue-needed and leave the branch alone. Archived columns are skipped entirely. It must defer reclaim (emit `branch:stale-active-reclaim-deferred`) when the task worktree is in `activeSessionRegistry`, when `executionStartedAt` is within `STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS` (10 minutes), or when the mapped worktree has uncommitted changes. Completion fan-out (`clearCompletionBranchIfSubsumed`) force-deletes the task branch after done even when unique commits remain, so squash leftovers do not accumulate. - **Worktree metadata reconcile ordering (FN-4962)**: `reconcile-task-worktree-metadata` must run before `reclaim-stale-active-branches`; stale `task.worktree` metadata is rebound to live `fusion/` worktrees when present (`task:auto-recover-worktree-metadata-rebound`) or cleared (`task:auto-recover-worktree-metadata-cleared`) when absent. - **Completion fan-out is synchronous**: `SelfHealingManager.reconcileCompletedTask()` runs on `in-review → done`. Downstream stale `blockedBy` links and residual `fusion/` branch/worktree artifacts are reconciled immediately, not on a periodic sweep. - **In-review stall deadlock**: identical stalls (same code + reason) repeated past `inReviewStallDeadlockThreshold` (default 3) auto-pause with `pausedReason: "in-review-stall-deadlock"` and `status: "failed"`. User-initiated retry paths (dashboard retry, `fn_task_retry`, and CLI `task retry`) clear that automatic deadlock pause so the retry can execute, but they never override explicit/manual pauses or unrelated automatic pause reasons. diff --git a/packages/engine/src/__tests__/self-healing-completion-fanout.test.ts b/packages/engine/src/__tests__/self-healing-completion-fanout.test.ts index 342caf23b1..816e409e1b 100644 --- a/packages/engine/src/__tests__/self-healing-completion-fanout.test.ts +++ b/packages/engine/src/__tests__/self-healing-completion-fanout.test.ts @@ -145,7 +145,29 @@ describe("self-healing completion fan-out", () => { expect(execMock.mock.calls.some((c) => String(c[0]).includes("git worktree remove --force") && String(c[0]).includes("/wt/fn-c"))).toBe(true); expect((await store.getTask("FN-C"))?.worktree).toBeNull(); expect((await store.getTask("FN-C"))?.branch).toBeNull(); - expect(out.branchRemoved).toBe(false); + /* + FNXC:StaleActiveBranchDoneSpam 2026-08-03-01:47: + Post-completion branch cleanup force-deletes even when the tip still has unique commits vs main + (squash/AI-merge shape). Previously this case expected branchRemoved=false and left fusion/* forever. + */ + expect(out.branchRemoved).toBe(true); + expect(execMock.mock.calls.some((c) => String(c[0]).includes("git branch -D") && String(c[0]).includes("fusion/fn-c"))).toBe(true); + }); + + it("force-deletes completion branch when unique commits remain after squash", async () => { + uniqueCommitsMock.mockResolvedValue({ + commits: [{ sha: "deadbeef", subject: "feat: pre-squash tip" }] as any, + mainRef: "main", + degraded: false, + }); + const blocker = makeTask("FN-SQUASH", { column: "done", branch: "fusion/fn-squash" }); + const store = createStore([blocker]); + const mgr = new SelfHealingManager(store, { rootDir: "/repo" }); + + const out = await mgr.reconcileCompletedTask("FN-SQUASH"); + expect(out.branchRemoved).toBe(true); + expect(execMock.mock.calls.some((c) => String(c[0]).includes("git branch -D") && String(c[0]).includes("fusion/fn-squash"))).toBe(true); + expect(logger.log).toHaveBeenCalledWith(expect.stringContaining("force-deleting post-completion")); }); it("globalPause short-circuits", async () => { diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 3e0b7c3849..828fa1a829 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -11197,6 +11197,75 @@ describe("SelfHealingManager reclaimStaleActiveBranches (FN-4546)", () => { expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("stale-active-branch-rescue-needed FN-1001")); }); + /* + FNXC:StaleActiveBranchDoneSpam 2026-08-03-01:47: + Done/complete-lane squash leftovers used to emit rescue-needed forever (unique tip SHAs vs main). + Complete columns must force-delete after the no-worktree gates pass, and must NOT warn rescue-needed. + */ + it("force-deletes complete-lane branch with unique commits and does not warn rescue-needed", async () => { + getSelfHealingLogger().warn.mockClear(); + mockedIsUsableTaskWorktree.mockResolvedValue(false); + (store.listTasks as any).mockResolvedValueOnce([ + { id: "FN-1001", column: "done", checkedOutBy: null, userPaused: false, worktree: null, branch: "fusion/fn-1001", lineageId: "lin-done" }, + ]); + + mockedExecSync.mockImplementation((command: string) => { + if (command.includes("git branch --list 'fusion/*'")) return Buffer.from(" fusion/fn-1001\n"); + if (command.includes("git rev-parse --verify") && command.includes("fusion/fn-1001")) return Buffer.from("abc123def456\n"); + if (command.includes("git rev-list --count") && command.includes("fusion/fn-1001")) return Buffer.from("2\n"); + return Buffer.from(""); + }); + + const recovered = await manager.reclaimStaleActiveBranches(); + + expect(recovered).toBe(1); + expect(mockedExecSync).toHaveBeenCalledWith(expect.stringContaining("git branch -D \"fusion/fn-1001\""), expect.anything()); + expect(getSelfHealingLogger().warn).not.toHaveBeenCalledWith(expect.stringContaining("stale-active-branch-rescue-needed FN-1001")); + expect(store.updateTask).toHaveBeenCalledWith("FN-1001", { worktree: null, branch: null, baseCommitSha: null }); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-1001", + expect.stringContaining("reason=complete-column-unique-commits-force"), + ); + expect((store as any).recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + domain: "git", + mutationType: "branch:stale-active-reclaim", + target: "fusion/fn-1001", + })); + }); + + it("force-deletes unique-commit branches for RENAMED complete lanes (role-resolved)", async () => { + /* Role resolution, not the literal id "done": a custom complete column must take the force path. */ + getSelfHealingLogger().warn.mockClear(); + mockedIsUsableTaskWorktree.mockResolvedValue(false); + (store.listTasks as any).mockResolvedValueOnce([ + { id: "FN-1001", column: "shipped", checkedOutBy: null, userPaused: false, worktree: null, branch: null, lineageId: "lin-1" }, + ]); + (store as unknown as { listWorkflowDefinitions: unknown }).listWorkflowDefinitions = vi.fn(async () => [{ + ir: { + version: "v2", + id: "custom:renamed-complete", + nodes: [], + edges: [], + columns: [ + { id: "building", name: "building", traits: [{ trait: "wip", config: { limitSetting: "maxConcurrent" } }] }, + { id: "shipped", name: "shipped", traits: [{ trait: "complete" }] }, + ], + }, + }]); + mockedExecSync.mockImplementation((command: string) => { + if (command.includes("git branch --list 'fusion/*'")) return Buffer.from(" fusion/fn-1001\n"); + if (command.includes("git rev-parse --verify") && command.includes("fusion/fn-1001")) return Buffer.from("abc123def456\n"); + if (command.includes("git rev-list --count") && command.includes("fusion/fn-1001")) return Buffer.from("2\n"); + return Buffer.from(""); + }); + + const recovered = await manager.reclaimStaleActiveBranches(); + + expect(recovered).toBe(1); + expect(mockedExecSync).toHaveBeenCalledWith(expect.stringContaining("git branch -D \"fusion/fn-1001\""), expect.anything()); + expect(getSelfHealingLogger().warn).not.toHaveBeenCalledWith(expect.stringContaining("stale-active-branch-rescue-needed")); + }); + it("skips task with active heartbeat run", async () => { const agentStore = { listActiveHeartbeatRuns: vi.fn().mockResolvedValue([{ startedAt: new Date().toISOString(), contextSnapshot: { taskId: "FN-1001" } }]), diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 43a60dacd4..1ca5afc640 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -4684,8 +4684,14 @@ export class SelfHealingManager extends SelfHealingGitEvidence { } async reclaimStaleActiveBranches(): Promise { - /* FNXC:WorkflowLifecycleColumns 2026-07-31-22:30 (self-healing cluster): an archived card must not have its branch reclaimed, whatever that lane is named. Keyed on the literal this sweep answered "no" for every card on a renamed board. */ + /* + FNXC:WorkflowLifecycleColumns 2026-07-31-22:30 (self-healing cluster): an archived card must not have its branch reclaimed, whatever that lane is named. Keyed on the literal this sweep answered "no" for every card on a renamed board. + + FNXC:StaleActiveBranchDoneSpam 2026-08-03-01:47: + Complete-lane cards used to hit the unique-commit rescue-needed warn every maintenance sweep after squash/AI merge (feature tip SHAs are not ancestors of main). That spam was not actionable — the work already landed — and left local fusion/* refs forever. Resolve complete columns and force-delete their stale branches once the no-worktree / no-session gates pass; keep rescue-needed only for non-terminal columns where unique commits may still be real unmerged work. Archived still skips entirely (archive cleanup owns those refs). + */ const reclaimArchivedColumns = await resolveProjectColumnsForRoles(this.store, ["archived"]); + const reclaimCompleteColumns = await resolveProjectColumnsForRoles(this.store, ["complete"]); try { const settings = await this.store.getSettings(); if (settings.globalPause || settings.enginePaused) return 0; @@ -4800,11 +4806,16 @@ export class SelfHealingManager extends SelfHealingGitEvidence { const inspection = await this.inspectOrphanedBranch(branch); if (!inspection) continue; - if (inspection.uniqueCommitCount > 0) { + const isCompleteColumn = reclaimCompleteColumns.has(task.column); + if (inspection.uniqueCommitCount > 0 && !isCompleteColumn) { log.warn(`[recovery] stale-active-branch-rescue-needed ${task.id} branch=${branch} unique=${inspection.uniqueCommitCount} tip=${inspection.tipSha.slice(0, 12)}`); continue; } + const reclaimReason = inspection.uniqueCommitCount > 0 + ? "complete-column-unique-commits-force" + : "zero-unique-commits-no-worktree"; + await execAsync(`git branch -D ${JSON.stringify(branch)}`, { cwd: this.options.rootDir, timeout: 120_000, @@ -4826,7 +4837,7 @@ export class SelfHealingManager extends SelfHealingGitEvidence { }); await this.store.logEntry( task.id, - `[recovery] stale-active-branch-reclaim ${task.id} branch=${branch} reason=zero-unique-commits-no-worktree`, + `[recovery] stale-active-branch-reclaim ${task.id} branch=${branch} reason=${reclaimReason}`, ); try { @@ -4845,7 +4856,7 @@ export class SelfHealingManager extends SelfHealingGitEvidence { branch, tipSha: inspection.tipSha, uniqueCommitCount: inspection.uniqueCommitCount, - reason: "zero-unique-commits-no-worktree", + reason: reclaimReason, }, }); } catch (auditErr: unknown) { @@ -4885,6 +4896,10 @@ export class SelfHealingManager extends SelfHealingGitEvidence { */ private async clearCompletionBranchIfSubsumed(task: Task, branchName: string): Promise { + /* + FNXC:StaleActiveBranchDoneSpam 2026-08-03-01:47: + Completion fan-out used to skip deletion when the tip still had unique commits vs the integration base. Squash / AI-merge always leaves that shape (new main SHA, old fusion/* tip still "unique"), so done tasks kept local branches forever and reclaimStaleActiveBranches warned rescue-needed every sweep. After a successful complete, force-delete the task branch regardless of unique commit count; the landed content is already on the integration branch under a different SHA. Log unique-count force deletes at info, not as an open rescue. + */ try { await execAsync(`git rev-parse --verify ${shellQuote(branchName)}`, { cwd: this.options.rootDir, @@ -4897,10 +4912,9 @@ export class SelfHealingManager extends SelfHealingGitEvidence { const baseBranch = task.baseBranch || await resolveIntegrationBranch(this.options.rootDir, undefined); const comparison = await listUniqueBranchCommits(this.options.rootDir, baseBranch, branchName); if (comparison.commits.length > 0) { - log.warn( - `[self-healing] reconcileCompletedTask ${task.id}: branch ${branchName} has ${comparison.commits.length} unique commit(s) vs ${comparison.mainRef}; skip deletion`, + log.log( + `[self-healing] reconcileCompletedTask ${task.id}: branch ${branchName} has ${comparison.commits.length} unique commit(s) vs ${comparison.mainRef}; force-deleting post-completion (squash-safe)`, ); - return false; } try {