From bc6dfd386e964f5c5fdc005e44b104bb62844754 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 15 Jun 2026 02:20:45 -0700 Subject: [PATCH] FN-6478: surface paused workflow graph failures Surface stranded paused workflow exits as actionable executor failures. - Treat paused or aborted graph exits as benign only while the live task remains in-progress. - Preserve terminal/review lifecycle state while recording operator-actionable failure evidence for advanced columns. - Cover user-paused, pause-aborted, existing-failure, in-progress, in-review, todo, and done column recovery paths. - Document the workflow lifecycle invariant and add a patch changeset. Files changed: .changeset/fn-6478-paused-workflow-executions.md | 5 + docs/architecture.md | 1 + .../engine/src/__tests__/executor-recovery.test.ts | 283 +++++++++++++++++++++ packages/engine/src/executor.ts | 30 ++- 4 files changed, 315 insertions(+), 4 deletions(-) Fusion-Task-Id: FN-6478 Fusion-Task-Lineage: 219d8612-6604-4dbc-9a3a-a1c7837419c1 --- .../fn-6478-paused-workflow-executions.md | 5 + docs/architecture.md | 1 + .../src/__tests__/executor-recovery.test.ts | 283 ++++++++++++++++++ packages/engine/src/executor.ts | 30 +- 4 files changed, 315 insertions(+), 4 deletions(-) create mode 100644 .changeset/fn-6478-paused-workflow-executions.md diff --git a/.changeset/fn-6478-paused-workflow-executions.md b/.changeset/fn-6478-paused-workflow-executions.md new file mode 100644 index 0000000000..6bfb1f0406 --- /dev/null +++ b/.changeset/fn-6478-paused-workflow-executions.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Surface paused workflow graph exits that occur outside `in-progress` as operator-actionable failures instead of leaving tasks stranded. diff --git a/docs/architecture.md b/docs/architecture.md index e8586cff29..2362c37aa5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1276,6 +1276,7 @@ The columns/traits track moved *board* policy (transitions, capacity, hold, merg - A `parse-steps` node reads a workflow-declared **artifact** (PROMPT.md is just the default workflow's declared `step-source` artifact) and runs a registry **parser** (`step-headings`, `json-steps`, or a plugin-contributed parser) to write `Task.steps[]`. It is the only graph-side step-list writer and must dominate any `foreach`. Parsers fail closed to a routable `outcome:parse-error`. - A `foreach(source:"task-steps")` node instantiates an inline template subgraph once per planned step, with `mode` (sequential/parallel) and `isolation` (shared/worktree) as explicit axes and per-instance run-state pinned + persisted for crash-safe resume. - Resume-limbo graph failures are retried only through a narrow persisted counter (`Task.graphResumeRetryCount`, max 2). The executor classifies a failure as transient only when it happens immediately after the engine restart/unpause resume log marker, reports no graph `reason`, has no completed step progress, and the task has no durable `lastError`/`failureReason`; it clears transient `status`/`error`, logs the auto-retry, and schedules one more graph execution. Any explicit graph reason, completed step progress, durable task error, missing resume marker, or exhausted counter remains a genuine `status:"failed"` disposition and goes to review handoff, preserving the FN-5704 anti-loop contract. +- Paused graph exits are benign only while the task is still in `in-progress`; that is the user-pause/engine-pause state where preserving the pause without requeueing is intentional. If the graph reports a pause/abort exit after the task has already advanced to another live column (for example `in-review` after an unpause/resume race), `TaskExecutor.handleGraphFailure()` surfaces the boundary as operator-actionable failure evidence (`status:"failed"`/`error` when no failure is already present, plus a task-log entry) and does **not** move, rewind, or auto-merge the task. `done` and `archived` remain terminal and keep their column/status, while existing failure details are preserved. - A `step-review` node surfaces reviewer verdicts (APPROVE/REVISE/RETHINK/UNAVAILABLE) as outcome edges; `rework` edges (the only legal graph cycles, bounded per instance) route REVISE/RETHINK back to `step-execute`, with RETHINK traversal triggering the reset seam. - A `code` node runs sandboxed TypeScript (esbuild + child process, clamped timeout, no store handle) for arbitrary computed routing/field logic — the same trust tier as project-local script steps. diff --git a/packages/engine/src/__tests__/executor-recovery.test.ts b/packages/engine/src/__tests__/executor-recovery.test.ts index 2067d04890..8d053f91df 100644 --- a/packages/engine/src/__tests__/executor-recovery.test.ts +++ b/packages/engine/src/__tests__/executor-recovery.test.ts @@ -944,6 +944,289 @@ describe("TaskExecutor bounded recovery retries", () => { expect(store.handoffToReview).not.toHaveBeenCalled(); }); + /* + FNXC:WorkflowLifecycle 2026-06-15-01:38: + FN-6478 established that a workflow graph exit while paused is benign only while the task remains in-progress. If the live row already advanced to in-review or another non-execution column, the executor must preserve explicit user pauses and autoMerge:false terminal review state while surfacing an operator-actionable workflow failure instead of the generic pause-preserved log. + */ + it("surfaces an operator-actionable failure for user-paused in-review graph exits", async () => { + const store = createMockStore(); + const steps = [ + { name: "Preflight", status: "pending" }, + { name: "Implement", status: "pending" }, + ]; + const task = { + id: "FN-001", + title: "Test", + description: "Test", + column: "in-progress", + status: undefined, + dependencies: [], + steps, + currentStep: 0, + log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as Task; + store.getTask.mockResolvedValue({ + ...task, + column: "in-review", + paused: true, + userPaused: true, + status: undefined, + error: null, + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + + await (executor as any).handleGraphFailure(task, { + visitedNodeIds: ["execute"], + }); + + const messages = store.logEntry.mock.calls.map((call) => call[1]).join("\n"); + expect(store.logEntry.mock.calls.map((call) => call[1])).toEqual([ + "Workflow graph failure surfaced after paused explicit user pause in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task", + ]); + expect(messages).toContain("Workflow graph failure surfaced"); + expect(messages).toContain("explicit user pause"); + expect(messages).toContain("operator action required"); + expect(messages).not.toContain("Workflow graph run ended while task is paused — pause state preserved"); + expect(store.updateTask).toHaveBeenCalledWith( + "FN-001", + { + error: "Workflow graph failure surfaced after paused explicit user pause in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task", + status: "failed", + }, + undefined, + ); + expect(store.handoffToReview).not.toHaveBeenCalled(); + }); + + it("surfaces pausedAborted in-review graph exits as workflow failures", async () => { + const store = createMockStore(); + const steps = [ + { name: "Preflight", status: "pending" }, + { name: "Implement", status: "pending" }, + ]; + const task = { + id: "FN-001", + title: "Test", + description: "Test", + column: "in-progress", + status: undefined, + dependencies: [], + steps, + currentStep: 0, + log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as Task; + store.getTask.mockResolvedValue({ + ...task, + column: "in-review", + paused: false, + status: undefined, + error: null, + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + (executor as any).pausedAborted.add("FN-001"); + + await (executor as any).handleGraphFailure(task, { + visitedNodeIds: ["execute"], + }); + + const messages = store.logEntry.mock.calls.map((call) => call[1]).join("\n"); + expect(store.logEntry.mock.calls.map((call) => call[1])).toEqual([ + "Workflow graph failure surfaced after paused engine abort during pause/resume in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task", + ]); + expect(messages).toContain("Workflow graph failure surfaced"); + expect(messages).toContain("engine abort during pause/resume"); + expect(messages).toContain("operator action required"); + expect(messages).not.toContain("Workflow graph run ended while task is paused — pause state preserved"); + expect(store.updateTask).toHaveBeenCalledWith( + "FN-001", + { + error: "Workflow graph failure surfaced after paused engine abort during pause/resume in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task", + status: "failed", + }, + undefined, + ); + expect(store.handoffToReview).not.toHaveBeenCalled(); + }); + + it("does not overwrite an already-surfaced in-review failure during paused abort cleanup", async () => { + const store = createMockStore(); + const task = { + id: "FN-001", + title: "Test", + description: "Test", + column: "in-progress", + status: undefined, + dependencies: [], + steps: [{ name: "Preflight", status: "pending" }], + currentStep: 0, + log: [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as Task; + store.getTask.mockResolvedValue({ + ...task, + column: "in-review", + paused: false, + status: "failed", + error: "Task reached in-review without calling fn_task_done", + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + (executor as any).pausedAborted.add("FN-001"); + + await (executor as any).handleGraphFailure(task, { + visitedNodeIds: ["execute"], + }); + + expect(store.logEntry).toHaveBeenCalledWith( + "FN-001", + "Workflow graph failure surfaced after paused engine abort during pause/resume in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task", + undefined, + undefined, + ); + expect(store.updateTask).not.toHaveBeenCalledWith( + "FN-001", + expect.objectContaining({ status: "failed" }), + expect.anything(), + ); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.handoffToReview).not.toHaveBeenCalled(); + }); + + it("keeps genuine in-progress user pauses benign even with partial step progress", async () => { + const store = createMockStore(); + const task = { + id: "FN-001", + title: "Test", + description: "Test", + column: "in-progress", + status: undefined, + dependencies: [], + steps: [ + { name: "Preflight", status: "done" }, + { name: "Implement", status: "pending" }, + ], + currentStep: 1, + log: [{ timestamp: new Date().toISOString(), action: "Started execution" }], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as Task; + store.getTask.mockResolvedValue({ + ...task, + paused: true, + userPaused: true, + status: undefined, + error: null, + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + + await (executor as any).handleGraphFailure(task, { + visitedNodeIds: ["execute"], + }); + + expect(store.logEntry).toHaveBeenCalledWith( + "FN-001", + "Workflow graph run ended while task is paused — pause state preserved", + undefined, + undefined, + ); + expect(store.updateTask).not.toHaveBeenCalledWith( + "FN-001", + expect.objectContaining({ status: "failed" }), + expect.anything(), + ); + expect(store.handoffToReview).not.toHaveBeenCalled(); + expect(store.moveTask).not.toHaveBeenCalled(); + }); + + it("surfaces non-in-progress paused graph exits even after partial progress without requeueing autoMerge-off review", async () => { + const store = createMockStore(); + const task = { + id: "FN-001", + title: "Test", + description: "Test", + column: "in-progress", + status: undefined, + dependencies: [], + steps: [ + { name: "Preflight", status: "done" }, + { name: "Implement", status: "pending" }, + ], + currentStep: 1, + log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as Task; + store.getTask.mockResolvedValue({ + ...task, + column: "in-review", + paused: true, + userPaused: true, + status: undefined, + error: null, + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + + await (executor as any).handleGraphFailure(task, { + visitedNodeIds: ["execute"], + }); + + const expectedMessage = "Workflow graph failure surfaced after paused explicit user pause in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task"; + expect(store.logEntry).toHaveBeenCalledWith("FN-001", expectedMessage, undefined, undefined); + expect(store.updateTask).toHaveBeenCalledWith("FN-001", { error: expectedMessage, status: "failed" }, undefined); + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.handoffToReview).not.toHaveBeenCalled(); + }); + + it.each(["todo", "done"] as const)( + "surfaces paused graph exits in already-advanced %s column without lifecycle movement", + async (column) => { + const store = createMockStore(); + const task = { + id: "FN-001", + title: "Test", + description: "Test", + column: "in-progress", + status: undefined, + dependencies: [], + steps: [{ name: "Preflight", status: "pending" }], + currentStep: 0, + log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as Task; + store.getTask.mockResolvedValue({ + ...task, + column, + paused: true, + status: undefined, + error: null, + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + + await (executor as any).handleGraphFailure(task, { + visitedNodeIds: ["execute"], + }); + + const expectedMessage = `Workflow graph failure surfaced after paused task pause in '${column}' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task`; + expect(store.logEntry).toHaveBeenCalledWith("FN-001", expectedMessage, undefined, undefined); + if (column === "done") { + expect(store.updateTask).not.toHaveBeenCalledWith( + "FN-001", + expect.objectContaining({ status: "failed" }), + expect.anything(), + ); + } else { + expect(store.updateTask).toHaveBeenCalledWith("FN-001", { error: expectedMessage, status: "failed" }, undefined); + } + expect(store.moveTask).not.toHaveBeenCalled(); + expect(store.handoffToReview).not.toHaveBeenCalled(); + }, + ); + it("auto-retries a bounded transient resume-after-restart graph failure instead of parking", async () => { const store = createMockStore(); const task = { diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index d183439ac0..6fae544a68 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -6317,11 +6317,33 @@ export class TaskExecutor { this.options.stuckTaskDetector?.untrackTask(task.id); try { const live = await this.store.getTask(task.id); - // A paused/aborted implementation is not a graph failure — leave the - // pause machinery in charge instead of parking the task in review. - if (live.paused || this.pausedAborted.has(task.id)) { + // A paused/aborted implementation is not a graph failure while the task + // is still in-progress — leave the pause machinery in charge instead of + // parking the task in review. + const pausedAborted = this.pausedAborted.has(task.id); + if (live.paused || pausedAborted) { + /* + FNXC:WorkflowLifecycle 2026-06-15-01:45: + FN-6478: a graph exit during an in-progress pause is recoverable by explicit unpause, but the same exit after the task has already left in-progress strands the workflow graph. Preserve userPaused and autoMerge:false review parking; surface non-in-progress paused exits as operator-actionable failures without moving the task backward or re-enqueueing execution. + */ + const pauseProvenance = live.userPaused + ? "explicit user pause" + : pausedAborted + ? "engine abort during pause/resume" + : "task pause"; + if (live.column !== "in-progress") { + const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown"; + const message = `Workflow graph failure surfaced after paused ${pauseProvenance} in '${live.column}' at node '${failedNode}' — operator action required; retry or explicitly unpause/resume after inspecting the task`; + executorLog.warn(`${task.id}: ${message}`); + await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); + if (live.column !== "done" && live.column !== "archived" && live.status == null && live.error == null) { + await this.store.updateTask(task.id, { error: message, status: "failed" }, this.getRunContextFor(task.id)); + } + await this.persistTokenUsage(task.id); + return; + } const benignMessage = "Workflow graph run ended while task is paused — pause state preserved"; - executorLog.log(`${task.id}: ${benignMessage}`); + executorLog.log(`${task.id}: ${benignMessage} (${pauseProvenance})`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); return; }