From de3273eba3041a41f4e870d75658414b22bcce29 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 3 Jun 2026 15:05:03 -0700 Subject: [PATCH] FN-5937: clear deadlock auto-pause on retry Clear deadlock auto-pauses when users retry failed in-review tasks. - add a shared core helper to clear only the in-review stall deadlock auto-pause without overriding manual or unrelated pauses - apply the auto-pause clear patch across dashboard, CLI, and extension retry flows while preserving existing merge retry reset behavior - add regression coverage for core, CLI, extension, and dashboard retry paths plus a patch changeset and architecture note Files changed: .changeset/fn-5937-clear-auto-pause-retry.md | 5 + docs/architecture.md | 2 +- packages/cli/src/__tests__/extension.test.ts | 126 ++++++++++++++++++ packages/cli/src/__tests__/task-retry.test.ts | 59 +++++++++ packages/cli/src/commands/task.ts | 12 +- packages/cli/src/extension.ts | 12 +- .../core/src/__tests__/manual-retry-reset.test.ts | 44 ++++++- packages/core/src/index.ts | 7 +- packages/core/src/manual-retry-reset.ts | 19 +++ packages/core/src/store.ts | 8 +- .../src/__tests__/routes-tasks-ops.test.ts | 141 +++++++++++++++++++++ .../src/routes/register-task-workflow-routes.ts | 12 +- 12 files changed, 436 insertions(+), 11 deletions(-) Fusion-Task-Id: FN-5937 Fusion-Task-Lineage: 8a15e01b-7b05-4126-ac6b-895467801f02 --- .changeset/fn-5937-clear-auto-pause-retry.md | 5 + docs/architecture.md | 2 +- packages/cli/src/__tests__/extension.test.ts | 126 ++++++++++++++++ packages/cli/src/__tests__/task-retry.test.ts | 59 ++++++++ packages/cli/src/commands/task.ts | 12 +- packages/cli/src/extension.ts | 12 +- .../src/__tests__/manual-retry-reset.test.ts | 44 +++++- packages/core/src/index.ts | 7 +- packages/core/src/manual-retry-reset.ts | 19 +++ packages/core/src/store.ts | 8 +- .../src/__tests__/routes-tasks-ops.test.ts | 141 ++++++++++++++++++ .../routes/register-task-workflow-routes.ts | 12 +- 12 files changed, 436 insertions(+), 11 deletions(-) create mode 100644 .changeset/fn-5937-clear-auto-pause-retry.md create mode 100644 packages/cli/src/__tests__/task-retry.test.ts diff --git a/.changeset/fn-5937-clear-auto-pause-retry.md b/.changeset/fn-5937-clear-auto-pause-retry.md new file mode 100644 index 0000000000..a45833d969 --- /dev/null +++ b/.changeset/fn-5937-clear-auto-pause-retry.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Clear the in-review stall deadlock auto-pause on user-initiated retry so dashboard, CLI, and extension retries can actually resume merge/execution work without overriding manual pauses. diff --git a/docs/architecture.md b/docs/architecture.md index 694e66504e..c768df610d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1720,7 +1720,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f - **Stale active branches**: self-healing's `reclaim-stale-active-branches` stage prunes a `fusion/` branch with zero unique commits when no usable worktree mapping exists, then clears `task.branch`/`task.worktree`/`task.baseCommitSha`. It must defer reclaim (emit `branch:stale-active-reclaim-deferred`) when the task worktree is in `activeSessionRegistry`, when `executionStartedAt` is within `STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS` (10 minutes), or when the mapped worktree has uncommitted changes. - **Worktree metadata reconcile ordering (FN-4962)**: `reconcile-task-worktree-metadata` must run before `reclaim-stale-active-branches`; stale `task.worktree` metadata is rebound to live `fusion/` worktrees when present (`task:auto-recover-worktree-metadata-rebound`) or cleared (`task:auto-recover-worktree-metadata-cleared`) when absent. - **Completion fan-out is synchronous**: `SelfHealingManager.reconcileCompletedTask()` runs on `in-review → done`. Downstream stale `blockedBy` links and residual `fusion/` branch/worktree artifacts are reconciled immediately, not on a periodic sweep. -- **In-review stall deadlock**: identical stalls (same code + reason) repeated past `inReviewStallDeadlockThreshold` (default 3) auto-pause with `pausedReason: "in-review-stall-deadlock"` and `status: "failed"`. +- **In-review stall deadlock**: identical stalls (same code + reason) repeated past `inReviewStallDeadlockThreshold` (default 3) auto-pause with `pausedReason: "in-review-stall-deadlock"` and `status: "failed"`. User-initiated retry paths (dashboard retry, `fn_task_retry`, and CLI `task retry`) clear that automatic deadlock pause so the retry can execute, but they never override explicit/manual pauses or unrelated automatic pause reasons. - **Restart recovery**: `RestartRecoveryCoordinator` classifies interrupted `in-progress` runs. Unusable-worktree session-start failures (`missing`, `incomplete`, `unregistered git worktree`) are recoverable; retries are capped at `MAX_WORKTREE_SESSION_RETRIES=3` before escalating. - **Executor pre-session liveness gate (FN-4935)**: the gate now skips for fresh acquisitions (`acquisition.source === "fresh"`), emits structured `not_usable_task_worktree:` diagnostics (including canonicalized registered-path snapshots) and a `worktree:incomplete-detected` audit event with `source: "executor-liveness-gate"`, while preserving the existing `taskDoneRetryCount` / `MAX_TASK_DONE_REQUEUE_RETRIES` requeue contract. FN-5772 adds a bounded nested-root self-heal: when `task.worktree` points at a strict descendant of a registered worktree root inside the configured worktrees dir, executor re-anchors `task.worktree` to the git top-level, emits `worktree:reanchored` (`fromPath`, `toPath`, `source`), and proceeds; repo-root/outside-dir/unregistered top-level mismatches still fail. FN-4651 `worktreeSessionRetryCount` remains scoped to the in-review/session-start recovery path. - **Stale self-owned active-session reconcile on conflict cleanup (FN-4973)**: when executor worktree-conflict cleanup finds only a same-task stale `activeSessionRegistry` entry and no live in-memory `activeWorktrees` binding for that task/path, it must unregister the stale entry before `removeWorktree` (plus one-shot backstop reconcile on same-task `ActiveSessionWorktreeRemovalError` races). Foreign-task entries remain protected by FN-4811 and must never be reconciled by the requesting task. diff --git a/packages/cli/src/__tests__/extension.test.ts b/packages/cli/src/__tests__/extension.test.ts index 2b4f7fd380..36261b1058 100644 --- a/packages/cli/src/__tests__/extension.test.ts +++ b/packages/cli/src/__tests__/extension.test.ts @@ -2601,6 +2601,51 @@ describe("fn pi extension (runnable structured-output regression slice)", () => expect(task.retrySummary?.total ?? 0).toBe(0); }; + it("clears the deadlock auto-pause for execution-failed in-review retries", async () => { + const store = new TaskStore(tmpDir); + await store.init(); + + const task = await store.createTask({ + title: "deadlock-paused execution-failed task", + description: "test", + column: "todo", + }); + await store.updateTask(task.id, { + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "in-progress" }, + { name: "Step 2", status: "pending" }, + ], + }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "failed", + error: "executor stalled after deadlock pause", + paused: true, + pausedReason: "in-review-stall-deadlock", + mergeRetries: 0, + nextRecoveryAt: new Date(Date.now() + 60_000).toISOString(), + ...nonZeroRetryCounters, + }); + + const retryTool = api.tools.get("fn_task_retry")!; + const result = await retryTool.execute("retry-deadlock-exec", { id: task.id }, undefined, undefined, makeCtx(tmpDir)); + + expect(result.isError).toBeFalsy(); + expect(result.details.newColumn).toBe("todo"); + + const updated = await store.getTask(task.id); + expect(updated?.column).toBe("todo"); + expect(updated?.status).toBeFalsy(); + expect(updated?.error).toBeFalsy(); + expect(updated?.paused).toBeUndefined(); + expect(updated?.pausedReason).toBeUndefined(); + expect(updated?.steps[1].status).toBe("in-progress"); + expectRetryCountersReset(updated); + expect(updated?.mergeRetries).toBe(0); + }); + it("moves execution-failed in-review task (incomplete steps) to todo preserving progress", async () => { const store = new TaskStore(tmpDir); await store.init(); @@ -2671,6 +2716,87 @@ describe("fn pi extension (runnable structured-output regression slice)", () => expect(updated?.mergeRetries).toBe(0); }); + it("clears the deadlock auto-pause for merge-failed in-review retries", async () => { + const store = new TaskStore(tmpDir); + await store.init(); + + const task = await store.createTask({ + title: "deadlock-paused merge-failed task", + description: "test", + column: "todo", + }); + await store.updateTask(task.id, { + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "done" }, + ], + }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "failed", + error: "merge deadlock", + paused: true, + pausedReason: "in-review-stall-deadlock", + mergeRetries: 3, + nextRecoveryAt: new Date(Date.now() + 60_000).toISOString(), + ...nonZeroRetryCounters, + }); + + const retryTool = api.tools.get("fn_task_retry")!; + const result = await retryTool.execute("retry-deadlock-merge", { id: task.id }, undefined, undefined, makeCtx(tmpDir)); + + expect(result.isError).toBeFalsy(); + expect(result.details.newColumn).toBe("in-review"); + + const updated = await store.getTask(task.id); + expect(updated?.column).toBe("in-review"); + expect(updated?.status).toBeFalsy(); + expect(updated?.error).toBeFalsy(); + expect(updated?.paused).toBeUndefined(); + expect(updated?.pausedReason).toBeUndefined(); + expectRetryCountersReset(updated); + expect(updated?.mergeRetries).toBe(0); + }); + + it("does not clear manual pauses for merge-failed in-review retries", async () => { + const store = new TaskStore(tmpDir); + await store.init(); + + const task = await store.createTask({ + title: "user-paused merge-failed task", + description: "test", + column: "todo", + }); + await store.updateTask(task.id, { + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "done" }, + ], + }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "failed", + error: "merge deadlock", + paused: true, + pausedReason: "manual", + mergeRetries: 3, + }); + + const retryTool = api.tools.get("fn_task_retry")!; + const result = await retryTool.execute("retry-user-paused-merge", { id: task.id }, undefined, undefined, makeCtx(tmpDir)); + + expect(result.isError).toBeFalsy(); + expect(result.details.newColumn).toBe("in-review"); + + const updated = await store.getTask(task.id); + expect(updated?.paused).toBe(true); + expect(updated?.pausedReason).toBe("manual"); + expect(updated?.status).toBeFalsy(); + expect(updated?.mergeRetries).toBe(0); + }); + it("keeps merge-failed in-review task (all steps done) in in-review and resets merge state", async () => { const store = new TaskStore(tmpDir); await store.init(); diff --git a/packages/cli/src/__tests__/task-retry.test.ts b/packages/cli/src/__tests__/task-retry.test.ts new file mode 100644 index 0000000000..8e18a28d65 --- /dev/null +++ b/packages/cli/src/__tests__/task-retry.test.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { TaskStore } from "@fusion/core"; +import { runTaskRetry } from "../commands/task.js"; + +describe("runTaskRetry", () => { + const originalCwd = process.cwd(); + let tmpDir: string; + let consoleLogSpy: ReturnType; + + beforeEach(async () => { + tmpDir = await mkdtemp(join(tmpdir(), "fusion-task-retry-")); + process.chdir(tmpDir); + consoleLogSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + }); + + afterEach(async () => { + consoleLogSpy.mockRestore(); + process.chdir(originalCwd); + await rm(tmpDir, { recursive: true, force: true }); + }); + + async function createStore() { + const store = new TaskStore(tmpDir); + await store.init(); + return store; + } + + it("clears the deadlock auto-pause when retrying a failed task", async () => { + const store = await createStore(); + const task = await store.createTask({ + title: "deadlock-paused task", + description: "test", + column: "todo", + }); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + await store.updateTask(task.id, { + status: "failed", + error: "merge deadlock", + paused: true, + pausedReason: "in-review-stall-deadlock", + mergeRetries: 4, + }); + + await runTaskRetry(task.id); + + const updated = await store.getTask(task.id); + expect(updated.column).toBe("todo"); + expect(updated.status).toBeUndefined(); + expect(updated.error).toBeUndefined(); + expect(updated.paused).toBeUndefined(); + expect(updated.pausedReason).toBeUndefined(); + expect(updated.mergeRetries).toBe(0); + }); + +}); diff --git a/packages/cli/src/commands/task.ts b/packages/cli/src/commands/task.ts index 248c1b347f..79db647e37 100644 --- a/packages/cli/src/commands/task.ts +++ b/packages/cli/src/commands/task.ts @@ -1,4 +1,4 @@ -import { TaskStore, COLUMNS, COLUMN_LABELS, CentralCore, buildManualRetryResetPatch, extractIntentSignature, findNearDuplicates, getTaskDuplicateLineage, reconcileDeterministicDuplicate, runDeterministicDuplicateGuard, type Settings, type Column, type StepStatus, type AgentLogType, type AgentLogEntry, type IntentSignature, type NearDuplicateCandidate, type NearDuplicateMatch, type TaskDependencyMutation } from "@fusion/core"; +import { TaskStore, COLUMNS, COLUMN_LABELS, CentralCore, buildAutoPauseClearPatch, buildManualRetryResetPatch, extractIntentSignature, findNearDuplicates, getTaskDuplicateLineage, reconcileDeterministicDuplicate, runDeterministicDuplicateGuard, type Settings, type Column, type StepStatus, type AgentLogType, type AgentLogEntry, type IntentSignature, type NearDuplicateCandidate, type NearDuplicateMatch, type TaskDependencyMutation } from "@fusion/core"; import { aiMergeTask } from "@fusion/engine"; import { createInterface } from "node:readline/promises"; import type { PlanningQuestion, PlanningSummary } from "@fusion/core"; @@ -1039,6 +1039,9 @@ export async function runTaskRetry(id: string, projectName?: string) { throw new Error(`Task ${id} is not in a retryable state (status: ${task.status || 'none'})`); } + const autoPauseClearPatch = buildAutoPauseClearPatch(task); + const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; + // Clear failure state and stale branch refs so retry can choose a fresh base. await store.updateTask(id, { status: null, @@ -1047,6 +1050,7 @@ export async function runTaskRetry(id: string, projectName?: string) { branch: null, baseBranch: null, baseCommitSha: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch({ resetMergeRetries: true }), }); @@ -1054,7 +1058,11 @@ export async function runTaskRetry(id: string, projectName?: string) { await store.moveTask(id, 'todo'); // Log the retry action - await store.logEntry(id, "Retry requested from CLI", "Task reset to todo for retry"); + await store.logEntry( + id, + clearedDeadlockAutoPause ? "Retry requested from CLI (cleared deadlock auto-pause)" : "Retry requested from CLI", + "Task reset to todo for retry", + ); console.log(); console.log(` ✓ Retried ${id} → todo (failure state cleared)`); diff --git a/packages/cli/src/extension.ts b/packages/cli/src/extension.ts index 7e5f5476b7..7c801bd5ac 100644 --- a/packages/cli/src/extension.ts +++ b/packages/cli/src/extension.ts @@ -5,6 +5,7 @@ import { TaskStore, COLUMNS, COLUMN_LABELS, + buildAutoPauseClearPatch, buildManualRetryResetPatch, validateNodeOverrideChange, type Task, @@ -990,6 +991,10 @@ export default function kbExtension(pi: ExtensionAPI) { }; } + const autoPauseClearPatch = buildAutoPauseClearPatch(task); + const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; + const retryLogSuffix = clearedDeadlockAutoPause ? ", cleared deadlock auto-pause" : ""; + // In-review retry: distinguish between execution failures and merge failures. if (task.column === 'in-review') { const hasIncompleteSteps = task.steps.some( @@ -1004,9 +1009,10 @@ export default function kbExtension(pi: ExtensionAPI) { await store.updateTask(params.id, { status: null, error: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch(), }); - await store.logEntry(params.id, "Retry requested via Fusion extension (execution failure in-review → todo, preserving progress)"); + await store.logEntry(params.id, `Retry requested via Fusion extension (execution failure in-review → todo, preserving progress${retryLogSuffix})`); await store.moveTask(params.id, "todo", { preserveProgress: true }); return { content: [{ type: "text", text: `Retried ${params.id} → todo (execution failure, preserving step progress)` }], @@ -1017,9 +1023,10 @@ export default function kbExtension(pi: ExtensionAPI) { await store.updateTask(params.id, { status: null, error: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch({ resetMergeRetries: true }), }); - await store.logEntry(params.id, "Retry requested via Fusion extension (in-review merge retry, mergeRetries reset)"); + await store.logEntry(params.id, `Retry requested via Fusion extension (in-review merge retry, mergeRetries reset${retryLogSuffix})`); return { content: [{ type: "text", text: `Retried ${params.id} → in-review (merge retry state cleared)` }], details: { taskId: params.id, newColumn: 'in-review' }, @@ -1030,6 +1037,7 @@ export default function kbExtension(pi: ExtensionAPI) { await store.updateTask(params.id, { status: null, error: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch({ resetMergeRetries: true }), }); diff --git a/packages/core/src/__tests__/manual-retry-reset.test.ts b/packages/core/src/__tests__/manual-retry-reset.test.ts index 46e455e78a..5b6a249fb1 100644 --- a/packages/core/src/__tests__/manual-retry-reset.test.ts +++ b/packages/core/src/__tests__/manual-retry-reset.test.ts @@ -1,9 +1,51 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { buildManualRetryResetPatch, MANUAL_RETRY_RESET_COUNTER_KEYS } from "../manual-retry-reset.js"; +import { + IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON, + MANUAL_RETRY_RESET_COUNTER_KEYS, + buildAutoPauseClearPatch, + buildManualRetryResetPatch, +} from "../manual-retry-reset.js"; const RETRY_SUMMARY_COUNTER_REGEX = /toCount\(task\.(\w+)\)/g; +describe("buildAutoPauseClearPatch", () => { + it("clears the deadlock auto-pause for auto-paused tasks", () => { + expect(buildAutoPauseClearPatch({ + paused: true, + userPaused: undefined, + pausedReason: IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON, + })).toEqual({ + paused: false, + pausedReason: null, + }); + }); + + it("does not clear an explicit user pause", () => { + expect(buildAutoPauseClearPatch({ + paused: true, + userPaused: true, + pausedReason: IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON, + })).toEqual({}); + }); + + it("does not clear unrelated automatic pause reasons", () => { + expect(buildAutoPauseClearPatch({ + paused: true, + userPaused: undefined, + pausedReason: "branch-conflict-unrecoverable", + })).toEqual({}); + }); + + it("is a no-op when the task is not paused", () => { + expect(buildAutoPauseClearPatch({ + paused: undefined, + userPaused: undefined, + pausedReason: IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON, + })).toEqual({}); + }); +}); + describe("buildManualRetryResetPatch", () => { it("resets all manual retry counters to zero", () => { const patch = buildManualRetryResetPatch(); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2b7ebeeb3e..d30b61be70 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -230,7 +230,12 @@ export { normalizeTitleForTaskId, } from "./task-title-id-drift.js"; export { getPrimaryPrInfo } from "./task-helpers.js"; -export { MANUAL_RETRY_RESET_COUNTER_KEYS, buildManualRetryResetPatch } from "./manual-retry-reset.js"; +export { + IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON, + MANUAL_RETRY_RESET_COUNTER_KEYS, + buildAutoPauseClearPatch, + buildManualRetryResetPatch, +} from "./manual-retry-reset.js"; export type { TaskIdIntegrityAnomaly, TaskIdIntegrityAnomalyKind, diff --git a/packages/core/src/manual-retry-reset.ts b/packages/core/src/manual-retry-reset.ts index 042f25afde..938177ca26 100644 --- a/packages/core/src/manual-retry-reset.ts +++ b/packages/core/src/manual-retry-reset.ts @@ -1,5 +1,7 @@ import type { Task } from "./types.js"; +export const IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON = "in-review-stall-deadlock"; + export const MANUAL_RETRY_RESET_COUNTER_KEYS = [ "stuckKillCount", "resumeLimboCount", @@ -17,6 +19,23 @@ export const MANUAL_RETRY_RESET_COUNTER_KEYS = [ "mergeAuditBounceCount", ] as const satisfies ReadonlyArray; +export function buildAutoPauseClearPatch( + task: Pick, +): Partial { + if ( + task.paused === true + && task.userPaused !== true + && task.pausedReason === IN_REVIEW_STALL_DEADLOCK_PAUSE_REASON + ) { + return { + paused: false, + pausedReason: null as unknown as Task["pausedReason"], + }; + } + + return {}; +} + export function buildManualRetryResetPatch(options?: { resetMergeRetries?: boolean }): Partial { const patch: Partial = { nextRecoveryAt: null as unknown as Task["nextRecoveryAt"], diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 463852e1d2..7334bbf3f7 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -88,6 +88,7 @@ interface TaskRow { blockedBy: string | null; overlapBlockedBy: string | null; paused: number | null; + pausedReason: string | null; userPaused: number | null; baseBranch: string | null; executionStartBranch: string | null; @@ -1476,6 +1477,7 @@ export class TaskStore extends EventEmitter { blockedBy: row.blockedBy || undefined, overlapBlockedBy: row.overlapBlockedBy || undefined, paused: row.paused ? true : undefined, + pausedReason: row.pausedReason || undefined, userPaused: row.userPaused ? true : undefined, baseBranch: row.baseBranch || undefined, executionStartBranch: row.executionStartBranch || undefined, @@ -2106,6 +2108,7 @@ export class TaskStore extends EventEmitter { task.blockedBy ?? null, task.overlapBlockedBy ?? null, task.paused ? 1 : 0, + task.pausedReason ?? null, task.userPaused ? 1 : 0, task.baseBranch ?? null, task.branch ?? null, @@ -2222,7 +2225,7 @@ export class TaskStore extends EventEmitter { this.db.prepare(` INSERT INTO tasks ( id, lineageId, title, description, priority, "column", status, size, reviewLevel, currentStep, - worktree, blockedBy, overlapBlockedBy, paused, userPaused, baseBranch, branch, autoMerge, executionStartBranch, baseCommitSha, modelPresetId, modelProvider, + worktree, blockedBy, overlapBlockedBy, paused, pausedReason, userPaused, baseBranch, branch, autoMerge, executionStartBranch, baseCommitSha, modelPresetId, modelProvider, modelId, validatorModelProvider, validatorModelId, planningModelProvider, planningModelId, mergeRetries, workflowStepRetries, stuckKillCount, resumeLimboCount, resumeLimboTipSha, resumeLimboStepSignature, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, worktreeSessionRetryCount, completionHandoffLimboRecoveryCount, verificationFailureCount, mergeConflictBounceCount, mergeAuditBounceCount, mergeTransientRetryCount, branchConflictRecoveryCount, reviewerContextRetryCount, reviewerFallbackRetryCount, nextRecoveryAt, error, summary, thinkingLevel, executionMode, tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens, @@ -2249,7 +2252,7 @@ export class TaskStore extends EventEmitter { this.db.prepare(` INSERT INTO tasks ( id, lineageId, title, description, priority, "column", status, size, reviewLevel, currentStep, - worktree, blockedBy, overlapBlockedBy, paused, userPaused, baseBranch, branch, autoMerge, executionStartBranch, baseCommitSha, modelPresetId, modelProvider, + worktree, blockedBy, overlapBlockedBy, paused, pausedReason, userPaused, baseBranch, branch, autoMerge, executionStartBranch, baseCommitSha, modelPresetId, modelProvider, modelId, validatorModelProvider, validatorModelId, planningModelProvider, planningModelId, mergeRetries, workflowStepRetries, stuckKillCount, resumeLimboCount, resumeLimboTipSha, resumeLimboStepSignature, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, worktreeSessionRetryCount, completionHandoffLimboRecoveryCount, verificationFailureCount, mergeConflictBounceCount, mergeAuditBounceCount, mergeTransientRetryCount, branchConflictRecoveryCount, reviewerContextRetryCount, reviewerFallbackRetryCount, nextRecoveryAt, error, summary, thinkingLevel, executionMode, tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens, @@ -2274,6 +2277,7 @@ export class TaskStore extends EventEmitter { blockedBy = excluded.blockedBy, overlapBlockedBy = excluded.overlapBlockedBy, paused = excluded.paused, + pausedReason = excluded.pausedReason, userPaused = excluded.userPaused, baseBranch = excluded.baseBranch, branch = excluded.branch, diff --git a/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts b/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts index f8a6ea8fc0..4cc88b5f46 100644 --- a/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts +++ b/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts @@ -498,6 +498,44 @@ describe("POST /tasks/:id/retry", () => { expect(updateCall.nextRecoveryAt).toBeNull(); }); + it("clears the deadlock auto-pause when retrying an execution-failed in-review task", async () => { + const executionFailedTask = { + ...FAKE_TASK_DETAIL, + column: "in-review" as const, + status: "failed", + paused: true, + pausedReason: "in-review-stall-deadlock", + mergeRetries: 0, + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "in-progress" }, + { name: "Step 2", status: "pending" }, + ], + }; + const movedTask = { ...executionFailedTask, column: "todo" as const, status: undefined, paused: undefined, pausedReason: undefined }; + (store.getTask as ReturnType).mockResolvedValueOnce(executionFailedTask); + (store.updateTask as ReturnType).mockResolvedValue(executionFailedTask); + (store.moveTask as ReturnType).mockResolvedValue(movedTask); + + const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/retry", JSON.stringify({}), { + "Content-Type": "application/json", + }); + + expect(res.status).toBe(200); + expect(store.updateTask).toHaveBeenCalledWith("KB-001", { + status: null, + error: null, + paused: false, + pausedReason: null, + ...buildManualRetryResetPatch(), + }); + expect(store.moveTask).toHaveBeenCalledWith("KB-001", "todo", { preserveProgress: true }); + expect(store.logEntry).toHaveBeenCalledWith( + "KB-001", + "Retry requested from dashboard (execution failure in-review → todo, preserving progress, cleared deadlock auto-pause)", + ); + }); + it("retries execution-failed in-review task by moving to todo with progress preserved", async () => { const executionFailedTask = { ...FAKE_TASK_DETAIL, @@ -531,6 +569,44 @@ describe("POST /tasks/:id/retry", () => { ); }); + it("clears the deadlock auto-pause when retrying a merge-failed in-review task", async () => { + const mergeFailedTask = { + ...FAKE_TASK_DETAIL, + column: "in-review" as const, + status: "failed", + paused: true, + pausedReason: "in-review-stall-deadlock", + mergeRetries: 3, + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "done" }, + { name: "Step 2", status: "done" }, + ], + }; + (store.getTask as ReturnType) + .mockResolvedValueOnce(mergeFailedTask) + .mockResolvedValueOnce({ ...mergeFailedTask, paused: undefined, pausedReason: undefined, status: undefined, mergeRetries: 0 }); + (store.updateTask as ReturnType).mockResolvedValue(mergeFailedTask); + + const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/retry", JSON.stringify({}), { + "Content-Type": "application/json", + }); + + expect(res.status).toBe(200); + expect(store.updateTask).toHaveBeenCalledWith("KB-001", { + status: null, + error: null, + paused: false, + pausedReason: null, + ...buildManualRetryResetPatch({ resetMergeRetries: true }), + }); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.logEntry).toHaveBeenCalledWith( + "KB-001", + "Retry requested from dashboard (in-review merge retry, mergeRetries reset, cleared deadlock auto-pause)", + ); + }); + it("retries merge-failed in-review task by staying in-review with mergeRetries reset", async () => { const mergeFailedTask = { ...FAKE_TASK_DETAIL, @@ -564,6 +640,71 @@ describe("POST /tasks/:id/retry", () => { ); }); + it("does not clear an explicit user pause when retrying in-review merge failure", async () => { + const mergeFailedTask = { + ...FAKE_TASK_DETAIL, + column: "in-review" as const, + status: "failed", + paused: true, + userPaused: true, + pausedReason: "manual", + mergeRetries: 3, + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "done" }, + ], + }; + (store.getTask as ReturnType) + .mockResolvedValueOnce(mergeFailedTask) + .mockResolvedValueOnce(mergeFailedTask); + (store.updateTask as ReturnType).mockResolvedValue(mergeFailedTask); + + const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/retry", JSON.stringify({}), { + "Content-Type": "application/json", + }); + + expect(res.status).toBe(200); + const updateCall = (store.updateTask as ReturnType).mock.calls[0][1]; + expect(updateCall).not.toHaveProperty("paused"); + expect(updateCall).not.toHaveProperty("pausedReason"); + expect(store.logEntry).toHaveBeenCalledWith( + "KB-001", + "Retry requested from dashboard (in-review merge retry, mergeRetries reset)", + ); + }); + + it("does not clear unrelated automatic pauses when retrying in-review merge failure", async () => { + const mergeFailedTask = { + ...FAKE_TASK_DETAIL, + column: "in-review" as const, + status: "failed", + paused: true, + pausedReason: "branch-conflict-unrecoverable", + mergeRetries: 3, + steps: [ + { name: "Step 0", status: "done" }, + { name: "Step 1", status: "done" }, + ], + }; + (store.getTask as ReturnType) + .mockResolvedValueOnce(mergeFailedTask) + .mockResolvedValueOnce(mergeFailedTask); + (store.updateTask as ReturnType).mockResolvedValue(mergeFailedTask); + + const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/retry", JSON.stringify({}), { + "Content-Type": "application/json", + }); + + expect(res.status).toBe(200); + const updateCall = (store.updateTask as ReturnType).mock.calls[0][1]; + expect(updateCall).not.toHaveProperty("paused"); + expect(updateCall).not.toHaveProperty("pausedReason"); + expect(store.logEntry).toHaveBeenCalledWith( + "KB-001", + "Retry requested from dashboard (in-review merge retry, mergeRetries reset)", + ); + }); + it("retries zero-step merge-failed in-review task with prior merge attempts by staying in-review", async () => { const mergeFailedTask = { ...FAKE_TASK_DETAIL, diff --git a/packages/dashboard/src/routes/register-task-workflow-routes.ts b/packages/dashboard/src/routes/register-task-workflow-routes.ts index b36255a4ed..147300d316 100644 --- a/packages/dashboard/src/routes/register-task-workflow-routes.ts +++ b/packages/dashboard/src/routes/register-task-workflow-routes.ts @@ -29,6 +29,7 @@ import { findDuplicateMatches, deterministicGuardLocks, runDeterministicDuplicateGuard, + buildAutoPauseClearPatch, buildManualRetryResetPatch, reconcileDeterministicDuplicate, extractIntentSignature, @@ -1390,6 +1391,10 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork throw badRequest(`Task is not in a retryable state (current status: ${task.status || 'none'})`); } + const autoPauseClearPatch = buildAutoPauseClearPatch(task); + const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; + const retryLogSuffix = clearedDeadlockAutoPause ? ", cleared deadlock auto-pause" : ""; + // In-review retry: distinguish between execution failures (incomplete steps) // and merge failures (all steps done). if (isInReviewRetry) { @@ -1405,11 +1410,12 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork await scopedStore.updateTask(req.params.id, { status: null, error: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch(), }); await scopedStore.logEntry( req.params.id, - "Retry requested from dashboard (execution failure in-review → todo, preserving progress)", + `Retry requested from dashboard (execution failure in-review → todo, preserving progress${retryLogSuffix})`, ); const updated = await scopedStore.moveTask(req.params.id, "todo", { preserveProgress: true }); res.json(updated); @@ -1419,9 +1425,10 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork await scopedStore.updateTask(req.params.id, { status: null, error: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch({ resetMergeRetries: true }), }); - await scopedStore.logEntry(req.params.id, "Retry requested from dashboard (in-review merge retry, mergeRetries reset)"); + await scopedStore.logEntry(req.params.id, `Retry requested from dashboard (in-review merge retry, mergeRetries reset${retryLogSuffix})`); const updated = await scopedStore.getTask(req.params.id); res.json(updated); return; @@ -1434,6 +1441,7 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork branch: null, baseBranch: null, baseCommitSha: null, + ...autoPauseClearPatch, ...buildManualRetryResetPatch({ resetMergeRetries: true }), });