FN-6478: surface paused workflow graph failures
Surface stranded paused workflow exits as actionable executor failures. - Treat paused or aborted graph exits as benign only while the live task remains in-progress. - Preserve terminal/review lifecycle state while recording operator-actionable failure evidence for advanced columns. - Cover user-paused, pause-aborted, existing-failure, in-progress, in-review, todo, and done column recovery paths. - Document the workflow lifecycle invariant and add a patch changeset. Files changed: .changeset/fn-6478-paused-workflow-executions.md | 5 + docs/architecture.md | 1 + .../engine/src/__tests__/executor-recovery.test.ts | 283 +++++++++++++++++++++ packages/engine/src/executor.ts | 30 ++- 4 files changed, 315 insertions(+), 4 deletions(-) Fusion-Task-Id: FN-6478 Fusion-Task-Lineage: 219d8612-6604-4dbc-9a3a-a1c7837419c1
This commit is contained in:
gsxdsm
@@ -944,6 +944,289 @@ describe("TaskExecutor bounded recovery retries", () => {
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
/*
|
||||
FNXC:WorkflowLifecycle 2026-06-15-01:38:
|
||||
FN-6478 established that a workflow graph exit while paused is benign only while the task remains in-progress. If the live row already advanced to in-review or another non-execution column, the executor must preserve explicit user pauses and autoMerge:false terminal review state while surfacing an operator-actionable workflow failure instead of the generic pause-preserved log.
|
||||
*/
|
||||
it("surfaces an operator-actionable failure for user-paused in-review graph exits", async () => {
|
||||
const store = createMockStore();
|
||||
const steps = [
|
||||
{ name: "Preflight", status: "pending" },
|
||||
{ name: "Implement", status: "pending" },
|
||||
];
|
||||
const task = {
|
||||
id: "FN-001",
|
||||
title: "Test",
|
||||
description: "Test",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
dependencies: [],
|
||||
steps,
|
||||
currentStep: 0,
|
||||
log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} as Task;
|
||||
store.getTask.mockResolvedValue({
|
||||
...task,
|
||||
column: "in-review",
|
||||
paused: true,
|
||||
userPaused: true,
|
||||
status: undefined,
|
||||
error: null,
|
||||
});
|
||||
const executor = new TaskExecutor(store, "/tmp/test", {});
|
||||
|
||||
await (executor as any).handleGraphFailure(task, {
|
||||
visitedNodeIds: ["execute"],
|
||||
});
|
||||
|
||||
const messages = store.logEntry.mock.calls.map((call) => call[1]).join("\n");
|
||||
expect(store.logEntry.mock.calls.map((call) => call[1])).toEqual([
|
||||
"Workflow graph failure surfaced after paused explicit user pause in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task",
|
||||
]);
|
||||
expect(messages).toContain("Workflow graph failure surfaced");
|
||||
expect(messages).toContain("explicit user pause");
|
||||
expect(messages).toContain("operator action required");
|
||||
expect(messages).not.toContain("Workflow graph run ended while task is paused — pause state preserved");
|
||||
expect(store.updateTask).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
{
|
||||
error: "Workflow graph failure surfaced after paused explicit user pause in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task",
|
||||
status: "failed",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("surfaces pausedAborted in-review graph exits as workflow failures", async () => {
|
||||
const store = createMockStore();
|
||||
const steps = [
|
||||
{ name: "Preflight", status: "pending" },
|
||||
{ name: "Implement", status: "pending" },
|
||||
];
|
||||
const task = {
|
||||
id: "FN-001",
|
||||
title: "Test",
|
||||
description: "Test",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
dependencies: [],
|
||||
steps,
|
||||
currentStep: 0,
|
||||
log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} as Task;
|
||||
store.getTask.mockResolvedValue({
|
||||
...task,
|
||||
column: "in-review",
|
||||
paused: false,
|
||||
status: undefined,
|
||||
error: null,
|
||||
});
|
||||
const executor = new TaskExecutor(store, "/tmp/test", {});
|
||||
(executor as any).pausedAborted.add("FN-001");
|
||||
|
||||
await (executor as any).handleGraphFailure(task, {
|
||||
visitedNodeIds: ["execute"],
|
||||
});
|
||||
|
||||
const messages = store.logEntry.mock.calls.map((call) => call[1]).join("\n");
|
||||
expect(store.logEntry.mock.calls.map((call) => call[1])).toEqual([
|
||||
"Workflow graph failure surfaced after paused engine abort during pause/resume in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task",
|
||||
]);
|
||||
expect(messages).toContain("Workflow graph failure surfaced");
|
||||
expect(messages).toContain("engine abort during pause/resume");
|
||||
expect(messages).toContain("operator action required");
|
||||
expect(messages).not.toContain("Workflow graph run ended while task is paused — pause state preserved");
|
||||
expect(store.updateTask).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
{
|
||||
error: "Workflow graph failure surfaced after paused engine abort during pause/resume in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task",
|
||||
status: "failed",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not overwrite an already-surfaced in-review failure during paused abort cleanup", async () => {
|
||||
const store = createMockStore();
|
||||
const task = {
|
||||
id: "FN-001",
|
||||
title: "Test",
|
||||
description: "Test",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
dependencies: [],
|
||||
steps: [{ name: "Preflight", status: "pending" }],
|
||||
currentStep: 0,
|
||||
log: [],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} as Task;
|
||||
store.getTask.mockResolvedValue({
|
||||
...task,
|
||||
column: "in-review",
|
||||
paused: false,
|
||||
status: "failed",
|
||||
error: "Task reached in-review without calling fn_task_done",
|
||||
});
|
||||
const executor = new TaskExecutor(store, "/tmp/test", {});
|
||||
(executor as any).pausedAborted.add("FN-001");
|
||||
|
||||
await (executor as any).handleGraphFailure(task, {
|
||||
visitedNodeIds: ["execute"],
|
||||
});
|
||||
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"Workflow graph failure surfaced after paused engine abort during pause/resume in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task",
|
||||
undefined,
|
||||
undefined,
|
||||
);
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
expect.objectContaining({ status: "failed" }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps genuine in-progress user pauses benign even with partial step progress", async () => {
|
||||
const store = createMockStore();
|
||||
const task = {
|
||||
id: "FN-001",
|
||||
title: "Test",
|
||||
description: "Test",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
dependencies: [],
|
||||
steps: [
|
||||
{ name: "Preflight", status: "done" },
|
||||
{ name: "Implement", status: "pending" },
|
||||
],
|
||||
currentStep: 1,
|
||||
log: [{ timestamp: new Date().toISOString(), action: "Started execution" }],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} as Task;
|
||||
store.getTask.mockResolvedValue({
|
||||
...task,
|
||||
paused: true,
|
||||
userPaused: true,
|
||||
status: undefined,
|
||||
error: null,
|
||||
});
|
||||
const executor = new TaskExecutor(store, "/tmp/test", {});
|
||||
|
||||
await (executor as any).handleGraphFailure(task, {
|
||||
visitedNodeIds: ["execute"],
|
||||
});
|
||||
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
"Workflow graph run ended while task is paused — pause state preserved",
|
||||
undefined,
|
||||
undefined,
|
||||
);
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
expect.objectContaining({ status: "failed" }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("surfaces non-in-progress paused graph exits even after partial progress without requeueing autoMerge-off review", async () => {
|
||||
const store = createMockStore();
|
||||
const task = {
|
||||
id: "FN-001",
|
||||
title: "Test",
|
||||
description: "Test",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
dependencies: [],
|
||||
steps: [
|
||||
{ name: "Preflight", status: "done" },
|
||||
{ name: "Implement", status: "pending" },
|
||||
],
|
||||
currentStep: 1,
|
||||
log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} as Task;
|
||||
store.getTask.mockResolvedValue({
|
||||
...task,
|
||||
column: "in-review",
|
||||
paused: true,
|
||||
userPaused: true,
|
||||
status: undefined,
|
||||
error: null,
|
||||
});
|
||||
const executor = new TaskExecutor(store, "/tmp/test", {});
|
||||
|
||||
await (executor as any).handleGraphFailure(task, {
|
||||
visitedNodeIds: ["execute"],
|
||||
});
|
||||
|
||||
const expectedMessage = "Workflow graph failure surfaced after paused explicit user pause in 'in-review' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task";
|
||||
expect(store.logEntry).toHaveBeenCalledWith("FN-001", expectedMessage, undefined, undefined);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { error: expectedMessage, status: "failed" }, undefined);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["todo", "done"] as const)(
|
||||
"surfaces paused graph exits in already-advanced %s column without lifecycle movement",
|
||||
async (column) => {
|
||||
const store = createMockStore();
|
||||
const task = {
|
||||
id: "FN-001",
|
||||
title: "Test",
|
||||
description: "Test",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
dependencies: [],
|
||||
steps: [{ name: "Preflight", status: "pending" }],
|
||||
currentStep: 0,
|
||||
log: [{ timestamp: new Date().toISOString(), action: "Resuming execution after unpause" }],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} as Task;
|
||||
store.getTask.mockResolvedValue({
|
||||
...task,
|
||||
column,
|
||||
paused: true,
|
||||
status: undefined,
|
||||
error: null,
|
||||
});
|
||||
const executor = new TaskExecutor(store, "/tmp/test", {});
|
||||
|
||||
await (executor as any).handleGraphFailure(task, {
|
||||
visitedNodeIds: ["execute"],
|
||||
});
|
||||
|
||||
const expectedMessage = `Workflow graph failure surfaced after paused task pause in '${column}' at node 'execute' — operator action required; retry or explicitly unpause/resume after inspecting the task`;
|
||||
expect(store.logEntry).toHaveBeenCalledWith("FN-001", expectedMessage, undefined, undefined);
|
||||
if (column === "done") {
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||
"FN-001",
|
||||
expect.objectContaining({ status: "failed" }),
|
||||
expect.anything(),
|
||||
);
|
||||
} else {
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { error: expectedMessage, status: "failed" }, undefined);
|
||||
}
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("auto-retries a bounded transient resume-after-restart graph failure instead of parking", async () => {
|
||||
const store = createMockStore();
|
||||
const task = {
|
||||
|
||||
@@ -6317,11 +6317,33 @@ export class TaskExecutor {
|
||||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||||
try {
|
||||
const live = await this.store.getTask(task.id);
|
||||
// A paused/aborted implementation is not a graph failure — leave the
|
||||
// pause machinery in charge instead of parking the task in review.
|
||||
if (live.paused || this.pausedAborted.has(task.id)) {
|
||||
// A paused/aborted implementation is not a graph failure while the task
|
||||
// is still in-progress — leave the pause machinery in charge instead of
|
||||
// parking the task in review.
|
||||
const pausedAborted = this.pausedAborted.has(task.id);
|
||||
if (live.paused || pausedAborted) {
|
||||
/*
|
||||
FNXC:WorkflowLifecycle 2026-06-15-01:45:
|
||||
FN-6478: a graph exit during an in-progress pause is recoverable by explicit unpause, but the same exit after the task has already left in-progress strands the workflow graph. Preserve userPaused and autoMerge:false review parking; surface non-in-progress paused exits as operator-actionable failures without moving the task backward or re-enqueueing execution.
|
||||
*/
|
||||
const pauseProvenance = live.userPaused
|
||||
? "explicit user pause"
|
||||
: pausedAborted
|
||||
? "engine abort during pause/resume"
|
||||
: "task pause";
|
||||
if (live.column !== "in-progress") {
|
||||
const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown";
|
||||
const message = `Workflow graph failure surfaced after paused ${pauseProvenance} in '${live.column}' at node '${failedNode}' — operator action required; retry or explicitly unpause/resume after inspecting the task`;
|
||||
executorLog.warn(`${task.id}: ${message}`);
|
||||
await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id));
|
||||
if (live.column !== "done" && live.column !== "archived" && live.status == null && live.error == null) {
|
||||
await this.store.updateTask(task.id, { error: message, status: "failed" }, this.getRunContextFor(task.id));
|
||||
}
|
||||
await this.persistTokenUsage(task.id);
|
||||
return;
|
||||
}
|
||||
const benignMessage = "Workflow graph run ended while task is paused — pause state preserved";
|
||||
executorLog.log(`${task.id}: ${benignMessage}`);
|
||||
executorLog.log(`${task.id}: ${benignMessage} (${pauseProvenance})`);
|
||||
await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id));
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user