diff --git a/.changeset/fn-7225-stale-pause-retry.md b/.changeset/fn-7225-stale-pause-retry.md new file mode 100644 index 0000000000..00e59c2cf5 --- /dev/null +++ b/.changeset/fn-7225-stale-pause-retry.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Prevent stale pause state from mislabeling workflow retries as engine pauses. +category: fix +dev: Clears executor pause-abort provenance on fresh dispatch, Plan Review replan, and manual retry. diff --git a/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts b/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts index 67d7991afe..1d14f739eb 100644 --- a/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts +++ b/packages/dashboard/src/__tests__/routes-tasks-ops.test.ts @@ -769,15 +769,20 @@ describe("POST /tasks/:id/steer", () => { describe("POST /tasks/:id/retry", () => { let store: TaskStore; + let engine: { getTaskStore: ReturnType; clearTaskPauseAbortState: ReturnType }; beforeEach(() => { store = createMockStore(); + engine = { + getTaskStore: vi.fn(() => store), + clearTaskPauseAbortState: vi.fn(), + }; }); function buildApp() { const app = express(); app.use(express.json()); - app.use("/api", createApiRoutes(store)); + app.use("/api", createApiRoutes(store, { engine: engine as any } as any)); return app; } @@ -793,6 +798,7 @@ describe("POST /tasks/:id/retry", () => { }); expect(res.status).toBe(200); + expect(engine.clearTaskPauseAbortState).toHaveBeenCalledWith("KB-001"); expect(store.updateTask).toHaveBeenCalledWith("KB-001", { status: null, error: null, @@ -815,6 +821,7 @@ describe("POST /tasks/:id/retry", () => { expect(res.status).toBe(400); expect(res.body.error).toContain("not in a retryable state"); + expect(engine.clearTaskPauseAbortState).not.toHaveBeenCalled(); }); it("retries a failed task in any column (not just in-progress)", async () => { @@ -4480,4 +4487,3 @@ describe("Attachment routes", () => { }); }); }); - diff --git a/packages/dashboard/src/routes/register-task-workflow-routes.ts b/packages/dashboard/src/routes/register-task-workflow-routes.ts index 4862320e20..bc59972961 100644 --- a/packages/dashboard/src/routes/register-task-workflow-routes.ts +++ b/packages/dashboard/src/routes/register-task-workflow-routes.ts @@ -1640,7 +1640,7 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork // Retry failed, stuck-killed, or stranded triage/planning task router.post("/tasks/:id/retry", async (req, res) => { try { - const { store: scopedStore } = await getProjectContext(req); + const { store: scopedStore, engine } = await getProjectContext(req); const task = await scopedStore.getTask(req.params.id); const retrySpecification = task.column === "triage" && @@ -1669,6 +1669,12 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork throw badRequest(`Task is not in a retryable state (current status: ${task.status || 'none'})`); } + /* + FNXC:ManualRetry 2026-06-29-00:57: + Dashboard retry is a fresh run boundary. Clear executor-only pause-abort provenance before mutating task state so stale pause/resume markers cannot relabel the next Plan Review or execution failure as an engine pause. + */ + engine?.clearTaskPauseAbortState?.(req.params.id); + const autoPauseClearPatch = buildAutoPauseClearPatch(task); const clearedDeadlockAutoPause = Object.keys(autoPauseClearPatch).length > 0; const retryLogSuffix = clearedDeadlockAutoPause ? ", cleared deadlock auto-pause" : ""; diff --git a/packages/engine/src/__tests__/workflow-graph-optional-step-fix.test.ts b/packages/engine/src/__tests__/workflow-graph-optional-step-fix.test.ts index 37b350a816..1140b35d7f 100644 --- a/packages/engine/src/__tests__/workflow-graph-optional-step-fix.test.ts +++ b/packages/engine/src/__tests__/workflow-graph-optional-step-fix.test.ts @@ -132,6 +132,7 @@ describe("TaskExecutor pre-merge optional-step fix seam", () => { store.getTask.mockResolvedValue(liveTask); store.getSettings.mockResolvedValue({ maxPostReviewFixes: 3 }); const executor = new TaskExecutor(store, "/tmp/test"); + (executor as any).markPausedAborted(liveTask.id); const sendBack = vi.spyOn(executor as any, "sendTaskBackForFix").mockResolvedValue(undefined); const scheduled = await (executor as any).requestPreMergeOptionalStepFix(liveTask.id, liveTask, { @@ -160,6 +161,59 @@ describe("TaskExecutor pre-merge optional-step fix seam", () => { graphResumeRetryCount: 0, }, undefined); expect(store.updateTask).not.toHaveBeenCalledWith("FN-7066", { postReviewFixCount: 1 }, undefined); + expect((executor as any).pausedAborted.has("FN-7066")).toBe(false); + }); + + it("clears stale pause-abort provenance before a fresh unpaused execution dispatch", async () => { + const store = createMockStore(); + const liveTask = task({ column: "todo", paused: false, userPaused: false }); + store.getSettings.mockResolvedValue({ globalPause: false }); + const executor = new TaskExecutor(store, "/tmp/test"); + (executor as any).markPausedAborted(liveTask.id); + + await (executor as any).clearStalePauseAbortBeforeDispatch(liveTask); + + expect((executor as any).pausedAborted.has("FN-7066")).toBe(false); + expect(store.logEntry).toHaveBeenCalledWith( + "FN-7066", + "Cleared stale pause-abort marker before unpaused execution dispatch", + undefined, + undefined, + ); + }); + + it("clears pause-abort provenance for manual retry", () => { + const store = createMockStore(); + const executor = new TaskExecutor(store, "/tmp/test"); + (executor as any).markPausedAborted("FN-7066"); + + executor.clearPauseAbortStateForManualRetry("FN-7066"); + + expect((executor as any).pausedAborted.has("FN-7066")).toBe(false); + }); + + it("preserves pause-abort provenance while the task or engine is actually paused", async () => { + for (const { taskPatch, settings } of [ + { taskPatch: { paused: true }, settings: { globalPause: false } }, + { taskPatch: { userPaused: true }, settings: { globalPause: false } }, + { taskPatch: { paused: false, userPaused: false }, settings: { globalPause: true } }, + ]) { + const store = createMockStore(); + const liveTask = task({ column: "todo", ...taskPatch }); + store.getSettings.mockResolvedValue(settings); + const executor = new TaskExecutor(store, "/tmp/test"); + (executor as any).markPausedAborted(liveTask.id); + + await (executor as any).clearStalePauseAbortBeforeDispatch(liveTask); + + expect((executor as any).pausedAborted.has("FN-7066")).toBe(true); + expect(store.logEntry).not.toHaveBeenCalledWith( + "FN-7066", + "Cleared stale pause-abort marker before unpaused execution dispatch", + undefined, + undefined, + ); + } }); it("uses the default budget of 3 for repeated fix passes and then declines when exhausted", async () => { diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index 67a34d2141..bb6b2c178c 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -1654,6 +1654,40 @@ export class TaskExecutor { this.completionFinalizedTaskIds.delete(taskId); } + private async clearStalePauseAbortBeforeDispatch(task: Task): Promise { + if (!this.pausedAborted.has(task.id)) return; + let globalPause = false; + try { + globalPause = (await this.store.getSettings()).globalPause === true; + } catch { + globalPause = false; + } + if (task.paused === true || task.userPaused === true || globalPause) return; + /* + * FNXC:WorkflowLifecycle 2026-06-29-00:57: + * A stale pause-abort marker must not survive into a fresh unpaused dispatch. + * FN-7225 showed graph-owned Plan Review and execution failures being logged + * as "engine pause/resume" even though the task row was not paused. Clear the + * volatile marker at dispatch entry so real workflow/execution failures keep + * their actual cause and do not loop through pause recovery. + */ + this.clearPausedAborted(task.id); + await this.store.logEntry( + task.id, + "Cleared stale pause-abort marker before unpaused execution dispatch", + undefined, + this.getRunContextFor(task.id), + ).catch(() => undefined); + } + + clearPauseAbortStateForManualRetry(taskId: string): void { + /* + FNXC:ManualRetry 2026-06-29-00:57: + User retry is a fresh execution boundary. Clear volatile pause-abort provenance so retries cannot inherit stale engine pause/resume classification from a prior run. + */ + this.clearPausedAborted(taskId); + } + /* FNXC:Workspace 2026-06-24-15:45 (concurrent workspace tasks — shared browse-root collision): In workspace mode `this.rootDir` is the SHARED browse-only (non-git) workspace root, and EVERY @@ -3890,6 +3924,7 @@ export class TaskExecutor { */ const feedback = info.feedback?.trim() || "Plan Review failed before execution. Revise the task plan, then continue execution."; + this.clearPausedAborted(taskId); await this.store.logEntry( taskId, "AI spec revision requested", @@ -7730,6 +7765,7 @@ export class TaskExecutor { async execute(task: Task): Promise { this.completionFinalizedTaskIds.delete(task.id); + await this.clearStalePauseAbortBeforeDispatch(task); // Workflow graph interpreter routing (cutover M-C): graph-selected tasks // are orchestrated by the interpreter. The execute seam re-enters this // method with a completion interceptor registered (which claims the task @@ -13071,6 +13107,7 @@ ${failureFeedback} // assumptions and proceed instead of parking on a question. Explicit opt-in // only (default false = board run); see runGraphCustomNode / KTD-3. const unattended = stepOptions?.unattended === true; + const isPlanReviewStep = workflowStep.id === "graph:plan-review-step" || workflowStep.name === "Plan Review"; // Compute the diff scope so the workflow step agent reviews only what THIS // task changed — not unrelated files it might wander into. Without this, @@ -13099,7 +13136,19 @@ ${failureFeedback} ? `${scopedFiles.slice(0, MAX_SCOPE_FILES).map((f) => `- ${f}`).join("\n")}\n- ... (${scopedFiles.length - MAX_SCOPE_FILES} more files truncated)` : scopedFiles.map((f) => `- ${f}`).join("\n"); - const scopeBlock = `Diff Scope (files changed by THIS task vs base): + /* + * FNXC:PlanReviewScope 2026-06-29-00:57: + * Plan Review validates the planned PROMPT.md before execution. It must not + * inherit the generic workflow-step diff scope, because dirty worktrees or + * unrelated local commits can make a plan-only gate reject implementation + * state and loop back to triage after the planner already approved the spec. + */ + const scopeBlock = isPlanReviewStep + ? `Plan Review Scope: +- Review the task plan artifact (PROMPT.md) and task metadata only. +- Do NOT judge current implementation diffs, uncommitted worktree changes, or unrelated repository changes. +- If PROMPT.md is internally consistent, complete, scoped, and verifiable, approve even when the worktree contains unrelated changes from another task.` + : `Diff Scope (files changed by THIS task vs base): ${scopeFileBlock}${diffShortstat ? `\nDiff stat: ${diffShortstat}` : ""} CRITICAL SCOPING RULES — read before doing anything else: diff --git a/packages/engine/src/project-engine.ts b/packages/engine/src/project-engine.ts index a3b5e183b6..9981dca08b 100644 --- a/packages/engine/src/project-engine.ts +++ b/packages/engine/src/project-engine.ts @@ -924,6 +924,10 @@ export class ProjectEngine { return this.runtime.getAgentStore(); } + clearTaskPauseAbortState(taskId: string): void { + this.runtime.clearTaskPauseAbortState?.(taskId); + } + /** Get the MessageStore (if initialized). Returns undefined before start(). */ getMessageStore(): import("@fusion/core").MessageStore | undefined { return this.runtime.getMessageStore(); diff --git a/packages/engine/src/project-runtime.ts b/packages/engine/src/project-runtime.ts index eb8f2077da..2b2b54b9cd 100644 --- a/packages/engine/src/project-runtime.ts +++ b/packages/engine/src/project-runtime.ts @@ -151,6 +151,12 @@ export interface ProjectRuntime extends EventEmitter { */ getScheduler(): Scheduler; + /** + * Clear volatile executor pause-abort provenance for a task before a manual retry. + * Optional because isolated runtimes do not expose in-memory executor state. + */ + clearTaskPauseAbortState?(taskId: string): void; + /** * Get current runtime metrics. * @returns Metrics including in-flight tasks, active agents, and memory usage diff --git a/packages/engine/src/runtimes/in-process-runtime.ts b/packages/engine/src/runtimes/in-process-runtime.ts index fbcc0558d2..2e4ca64d4b 100644 --- a/packages/engine/src/runtimes/in-process-runtime.ts +++ b/packages/engine/src/runtimes/in-process-runtime.ts @@ -1378,6 +1378,10 @@ export class InProcessRuntime return this.scheduler; } + clearTaskPauseAbortState(taskId: string): void { + this.executor?.clearPauseAbortStateForManualRetry(taskId); + } + configurePrMonitoring(options: { prMonitor: PrMonitor; onClosedPrFeedback?: (taskId: string, prInfo: PrInfo, comments: PrComment[]) => void | Promise;