fix: preserve user pause across executor pause teardown (FN-7851 pause-bounce loop)
Pausing an in-progress task never stuck: the pause teardown re-queued the row to todo with a plain engine move, and the reopen block wiped paused/pausedByAgentId/pausedReason. The graph-failure classifier then saw an unpaused row, misread the hard-cancel as an engine-internal abort, and auto-continued the session (graphResumeRetryCount 1/2, 2/2); once the budget was exhausted the benign re-queue left the row dispatchable and the scheduler re-dispatched it seconds later — an indefinite pause/resume bounce, burning a fresh worktree + pnpm install per cycle. - store: new moveTask option `preservePause` keeps the pause park across a reopen-to-todo/triage move (flag-ON trait hook + flag-OFF legacy inline, kept in sync). It never SETS a pause, only prevents clearing one. - executor teardown: when the pause that caused the abort is still in force, move with preservePause so the row lands in todo still parked (scheduler skips paused/userPaused rows until explicit unpause). - classifier: a live task pause is labeled operator intent, never "engine abort during pause/resume"; the benign log now says "parked … awaiting explicit unpause" instead of the contradictory "cleared for normal scheduling" for parked rows. Surfaces covered by tests: flag-ON hook (preserve + never-set + default clear), classifier no-auto-continue for task-pause/user-pause/global-pause rows in todo, provenance labels. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
7
.changeset/pause-survives-teardown.md
Normal file
7
.changeset/pause-survives-teardown.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Pausing an in-progress task now sticks — the pause survives session teardown instead of auto-resuming.
|
||||||
|
category: fix
|
||||||
|
dev: New `preservePause` moveTask option; the executor pause teardown passes it so the todo re-queue keeps `paused`/`pausedByAgentId`/`pausedReason`. The graph-failure classifier now labels a preserved task pause as operator intent (never "engine abort during pause/resume" auto-continue), and the benign re-queue log says "parked … awaiting explicit unpause" for paused rows.
|
||||||
@@ -84,4 +84,41 @@ describe("default-workflow-hooks registry wiring", () => {
|
|||||||
applyDefaultWorkflowMoveEffects(engineCtx);
|
applyDefaultWorkflowMoveEffects(engineCtx);
|
||||||
expect(engineCtx.task.userPaused).toBeUndefined();
|
expect(engineCtx.task.userPaused).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// FN-7851 pause-bounce regression: the executor's pause teardown re-queues a
|
||||||
|
// user-paused in-progress task to todo. Without preservePause the reopen
|
||||||
|
// block wiped the pause flags, leaving the row dispatchable — the scheduler
|
||||||
|
// re-dispatched it seconds after the user paused it.
|
||||||
|
it("preservePause keeps the pause park across an engine reopen to todo", () => {
|
||||||
|
registerDefaultWorkflowHooks();
|
||||||
|
const ctx = makeCtx({ fromColumn: "in-progress", toColumn: "todo", moveSource: "engine", options: { preservePause: true } });
|
||||||
|
ctx.task.paused = true;
|
||||||
|
ctx.task.pausedByAgentId = "agent-1";
|
||||||
|
ctx.task.pausedReason = "operator pause";
|
||||||
|
ctx.task.userPaused = true;
|
||||||
|
applyDefaultWorkflowMoveEffects(ctx);
|
||||||
|
expect(ctx.task.paused).toBe(true);
|
||||||
|
expect(ctx.task.pausedByAgentId).toBe("agent-1");
|
||||||
|
expect(ctx.task.pausedReason).toBe("operator pause");
|
||||||
|
expect(ctx.task.userPaused).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preservePause never SETS a pause on an unpaused reopen, and default reopen still clears one", () => {
|
||||||
|
registerDefaultWorkflowHooks();
|
||||||
|
// preservePause on an unpaused task: nothing appears.
|
||||||
|
const unpausedCtx = makeCtx({ fromColumn: "in-progress", toColumn: "todo", moveSource: "engine", options: { preservePause: true } });
|
||||||
|
applyDefaultWorkflowMoveEffects(unpausedCtx);
|
||||||
|
expect(unpausedCtx.task.paused).toBeUndefined();
|
||||||
|
expect(unpausedCtx.task.userPaused).toBeUndefined();
|
||||||
|
|
||||||
|
// Default (no preservePause) engine reopen still clears an existing pause.
|
||||||
|
const defaultCtx = makeCtx({ fromColumn: "in-progress", toColumn: "todo", moveSource: "engine" });
|
||||||
|
defaultCtx.task.paused = true;
|
||||||
|
defaultCtx.task.pausedByAgentId = "agent-1";
|
||||||
|
defaultCtx.task.pausedReason = "operator pause";
|
||||||
|
applyDefaultWorkflowMoveEffects(defaultCtx);
|
||||||
|
expect(defaultCtx.task.paused).toBeUndefined();
|
||||||
|
expect(defaultCtx.task.pausedByAgentId).toBeUndefined();
|
||||||
|
expect(defaultCtx.task.pausedReason).toBeUndefined();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ export interface DefaultWorkflowMoveContext {
|
|||||||
preserveResumeState?: boolean;
|
preserveResumeState?: boolean;
|
||||||
preserveProgress?: boolean;
|
preserveProgress?: boolean;
|
||||||
preserveWorktree?: boolean;
|
preserveWorktree?: boolean;
|
||||||
|
preservePause?: boolean;
|
||||||
};
|
};
|
||||||
/** Reset all steps to pending + currentStep 0 (store owns the impl). */
|
/** Reset all steps to pending + currentStep 0 (store owns the impl). */
|
||||||
resetSteps: () => void;
|
resetSteps: () => void;
|
||||||
@@ -134,19 +135,28 @@ export function applyResetOnEntryEffects(ctx: DefaultWorkflowMoveContext): void
|
|||||||
(toColumn === "todo" || toColumn === "triage");
|
(toColumn === "todo" || toColumn === "triage");
|
||||||
if (!isReopenToTodoOrTriage) return;
|
if (!isReopenToTodoOrTriage) return;
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:WorkflowLifecycle 2026-07-12-09:05:
|
||||||
|
Pause-bounce loop (observed on FN-7851, 2026-07-12): a user pause of an in-progress task hard-cancels the session and the executor teardown re-queues the row to todo. This reopen block unconditionally wiped `paused`/`pausedByAgentId`/`pausedReason`, so the pause NEVER survived its own teardown — the graph-failure classifier then saw an unpaused row, misread the abort as engine-internal, and auto-continued the session (and after the retry budget, the scheduler re-dispatched the unpaused todo row). `preservePause` lets the pause-caused teardown move keep the park; the scheduler skips paused/userPaused todo rows until an explicit unpause.
|
||||||
|
`userPaused` promotion for user-source moves is unchanged; preservePause only prevents CLEARING an existing park, never sets one.
|
||||||
|
*/
|
||||||
if (!options.preserveStatus) {
|
if (!options.preserveStatus) {
|
||||||
task.status = undefined;
|
task.status = undefined;
|
||||||
task.error = undefined;
|
task.error = undefined;
|
||||||
task.pausedReason = undefined;
|
if (!options.preservePause) {
|
||||||
|
task.pausedReason = undefined;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
task.blockedBy = undefined;
|
task.blockedBy = undefined;
|
||||||
task.overlapBlockedBy = undefined;
|
task.overlapBlockedBy = undefined;
|
||||||
task.paused = undefined;
|
if (!options.preservePause) {
|
||||||
task.pausedByAgentId = undefined;
|
task.paused = undefined;
|
||||||
|
task.pausedByAgentId = undefined;
|
||||||
|
}
|
||||||
// abort-on-exit userPaused: only for user-source moves to todo (KTD-9).
|
// abort-on-exit userPaused: only for user-source moves to todo (KTD-9).
|
||||||
if (moveSource === "user" && toColumn === "todo") {
|
if (moveSource === "user" && toColumn === "todo") {
|
||||||
task.userPaused = true;
|
task.userPaused = true;
|
||||||
} else {
|
} else if (!options.preservePause) {
|
||||||
task.userPaused = undefined;
|
task.userPaused = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1436,6 +1436,15 @@ interface MoveTaskOptions {
|
|||||||
preserveProgress?: boolean;
|
preserveProgress?: boolean;
|
||||||
preserveWorktree?: boolean;
|
preserveWorktree?: boolean;
|
||||||
preserveStatus?: boolean;
|
preserveStatus?: boolean;
|
||||||
|
/**
|
||||||
|
* FNXC:WorkflowLifecycle 2026-07-12-09:05:
|
||||||
|
* Keep `paused`/`pausedByAgentId`/`pausedReason` (and any existing
|
||||||
|
* `userPaused`) across a reopen-to-todo/triage move. Used by the executor's
|
||||||
|
* pause teardown so a user pause survives its own hard-cancel re-queue and
|
||||||
|
* the row stays parked until an explicit unpause (FN-7851 pause-bounce loop).
|
||||||
|
* Never SETS a pause — only prevents the reopen block from clearing one.
|
||||||
|
*/
|
||||||
|
preservePause?: boolean;
|
||||||
allocateWorktree?: (reservedNames: Set<string>) => string | null;
|
allocateWorktree?: (reservedNames: Set<string>) => string | null;
|
||||||
moveSource?: "user" | "engine" | "scheduler";
|
moveSource?: "user" | "engine" | "scheduler";
|
||||||
workflowMoveActor?: WorkflowMovePolicyInput["actor"];
|
workflowMoveActor?: WorkflowMovePolicyInput["actor"];
|
||||||
@@ -7885,6 +7894,7 @@ ${TASK_UPSERT_SQL_ASSIGNMENTS}
|
|||||||
preserveResumeState: options?.preserveResumeState,
|
preserveResumeState: options?.preserveResumeState,
|
||||||
preserveProgress: options?.preserveProgress,
|
preserveProgress: options?.preserveProgress,
|
||||||
preserveWorktree: options?.preserveWorktree,
|
preserveWorktree: options?.preserveWorktree,
|
||||||
|
preservePause: options?.preservePause,
|
||||||
},
|
},
|
||||||
resetSteps: () => this.resetAllStepsToPending(task),
|
resetSteps: () => this.resetAllStepsToPending(task),
|
||||||
};
|
};
|
||||||
@@ -7946,18 +7956,26 @@ ${TASK_UPSERT_SQL_ASSIGNMENTS}
|
|||||||
&& (toColumn === "todo" || toColumn === "triage");
|
&& (toColumn === "todo" || toColumn === "triage");
|
||||||
|
|
||||||
if (isReopenToTodoOrTriage) {
|
if (isReopenToTodoOrTriage) {
|
||||||
|
// FNXC:WorkflowLifecycle 2026-07-12-09:05: keep this flag-OFF inline
|
||||||
|
// block in sync with applyResetOnEntryEffects (default-workflow-hooks.ts)
|
||||||
|
// — `preservePause` keeps a pause-caused teardown move from clearing the
|
||||||
|
// user's park (FN-7851 pause-bounce loop).
|
||||||
if (!options?.preserveStatus) {
|
if (!options?.preserveStatus) {
|
||||||
task.status = undefined;
|
task.status = undefined;
|
||||||
task.error = undefined;
|
task.error = undefined;
|
||||||
task.pausedReason = undefined;
|
if (!options?.preservePause) {
|
||||||
|
task.pausedReason = undefined;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
task.blockedBy = undefined;
|
task.blockedBy = undefined;
|
||||||
task.overlapBlockedBy = undefined;
|
task.overlapBlockedBy = undefined;
|
||||||
task.paused = undefined;
|
if (!options?.preservePause) {
|
||||||
task.pausedByAgentId = undefined;
|
task.paused = undefined;
|
||||||
|
task.pausedByAgentId = undefined;
|
||||||
|
}
|
||||||
if (moveSource === "user" && toColumn === "todo") {
|
if (moveSource === "user" && toColumn === "todo") {
|
||||||
task.userPaused = true;
|
task.userPaused = true;
|
||||||
} else {
|
} else if (!options?.preservePause) {
|
||||||
task.userPaused = undefined;
|
task.userPaused = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -193,22 +193,47 @@ describe("pause-abort benign requeue-to-todo (FN-6782)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it.each([
|
it.each([
|
||||||
{ label: "explicit user pause", overrides: { column: "todo", userPaused: true }, provenance: "hard-cancel" as const },
|
{
|
||||||
{ label: "global pause", overrides: { column: "todo" }, provenance: "global-pause" as const },
|
label: "explicit user pause",
|
||||||
|
overrides: { column: "todo", userPaused: true },
|
||||||
|
provenance: "hard-cancel" as const,
|
||||||
|
expectedBenign: "benign, paused awaiting explicit unpause",
|
||||||
|
expectedProvenance: "explicit user pause",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "global pause",
|
||||||
|
overrides: { column: "todo" },
|
||||||
|
provenance: "global-pause" as const,
|
||||||
|
expectedBenign: "benign, cleared for normal scheduling",
|
||||||
|
expectedProvenance: "global pause",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// FN-7851 pause-bounce regression: a pause-button pause that survived the
|
||||||
|
// executor teardown via preservePause lands in todo with `paused: true`.
|
||||||
|
// It must be classified as a task pause (NOT an engine-internal abort),
|
||||||
|
// stay parked, and never auto-continue the session.
|
||||||
|
label: "task pause (pause button, preserved across teardown)",
|
||||||
|
overrides: { column: "todo", paused: true },
|
||||||
|
provenance: "hard-cancel" as const,
|
||||||
|
expectedBenign: "benign, paused awaiting explicit unpause",
|
||||||
|
expectedProvenance: "task pause",
|
||||||
|
},
|
||||||
])(
|
])(
|
||||||
"does NOT auto-resume a $label that landed in todo",
|
"does NOT auto-resume a $label that landed in todo",
|
||||||
async ({ overrides, provenance }) => {
|
async ({ overrides, provenance, expectedBenign, expectedProvenance }) => {
|
||||||
// The auto-continue is scoped strictly to the engine-internal abort
|
// The auto-continue is scoped strictly to the engine-internal abort
|
||||||
// provenance. A genuine operator pause (userPaused) or a global engine
|
// provenance. A genuine operator pause (userPaused OR a preserved
|
||||||
// pause that ended up in todo must stay parked-benign and wait for
|
// pause-button `paused`) or a global engine pause that ended up in todo
|
||||||
// explicit resume — auto-resuming it would override the operator's intent.
|
// must stay parked-benign and wait for explicit resume — auto-resuming it
|
||||||
|
// would override the operator's intent.
|
||||||
const { store, task, executor } = makeHarness(overrides, provenance);
|
const { store, task, executor } = makeHarness(overrides, provenance);
|
||||||
(executor as any).addActiveWorktree(task.id, task.worktree);
|
(executor as any).addActiveWorktree(task.id, task.worktree);
|
||||||
const executeSpy = vi.spyOn(executor as any, "execute").mockResolvedValue(undefined);
|
const executeSpy = vi.spyOn(executor as any, "execute").mockResolvedValue(undefined);
|
||||||
|
|
||||||
await invokeGraphFailure(executor, task);
|
await invokeGraphFailure(executor, task);
|
||||||
|
|
||||||
expect(logText(store)).toContain("benign, cleared for normal scheduling");
|
expect(logText(store)).toContain(expectedBenign);
|
||||||
|
expect(logText(store)).toContain(`during ${expectedProvenance} with task`);
|
||||||
expect(logText(store)).not.toContain("auto-continuing the agent session");
|
expect(logText(store)).not.toContain("auto-continuing the agent session");
|
||||||
await flushScheduledRetry();
|
await flushScheduledRetry();
|
||||||
expect(executeSpy).not.toHaveBeenCalled();
|
expect(executeSpy).not.toHaveBeenCalled();
|
||||||
|
|||||||
@@ -8739,13 +8739,20 @@ export class TaskExecutor {
|
|||||||
FNXC:WorkflowLifecycle 2026-06-17-23:32:
|
FNXC:WorkflowLifecycle 2026-06-17-23:32:
|
||||||
FN-6625: completion-finalize aborts are teardown artifacts after a completed/no-commit execution has already advanced to in-review. Without excluding that provenance, the FN-6614 execute-node tail failure was mislabeled as an operator-action pause abort and re-parked failed.
|
FN-6625: completion-finalize aborts are teardown artifacts after a completed/no-commit execution has already advanced to in-review. Without excluding that provenance, the FN-6614 execute-node tail failure was mislabeled as an operator-action pause abort and re-parked failed.
|
||||||
*/
|
*/
|
||||||
|
// FNXC:WorkflowLifecycle 2026-07-12-09:05: check `live.paused` BEFORE the
|
||||||
|
// bare pausedAborted marker — a task-pause park that survived teardown
|
||||||
|
// (preservePause, FN-7851) is operator intent, not an engine-internal
|
||||||
|
// abort, and must be labeled as such so the benign re-queue log below
|
||||||
|
// does not misreport it as engine churn.
|
||||||
const pauseProvenance = live.userPaused
|
const pauseProvenance = live.userPaused
|
||||||
? "explicit user pause"
|
? "explicit user pause"
|
||||||
: abortProvenance === "global-pause"
|
: abortProvenance === "global-pause"
|
||||||
? "global pause"
|
? "global pause"
|
||||||
: pausedAborted
|
: live.paused
|
||||||
? "engine abort during pause/resume"
|
? "task pause"
|
||||||
: "task pause";
|
: pausedAborted
|
||||||
|
? "engine abort during pause/resume"
|
||||||
|
: "task pause";
|
||||||
// Typed discriminant for the engine-internal abort case (mirrors the
|
// Typed discriminant for the engine-internal abort case (mirrors the
|
||||||
// `pauseProvenance === "engine abort during pause/resume"` arm above):
|
// `pauseProvenance === "engine abort during pause/resume"` arm above):
|
||||||
// a hard-cancel teardown that is NOT a user pause or global pause. Used
|
// a hard-cancel teardown that is NOT a user pause or global pause. Used
|
||||||
@@ -8864,7 +8871,13 @@ export class TaskExecutor {
|
|||||||
// shared budget and falls back to plain todo re-queueing.
|
// shared budget and falls back to plain todo re-queueing.
|
||||||
executorLog.warn(`${task.id}: engine abort during pause/resume exhausted ${MAX_TRANSIENT_GRAPH_RESUME_RETRIES} internal retries — falling back to benign todo re-queue`);
|
executorLog.warn(`${task.id}: engine abort during pause/resume exhausted ${MAX_TRANSIENT_GRAPH_RESUME_RETRIES} internal retries — falling back to benign todo re-queue`);
|
||||||
}
|
}
|
||||||
const todoBenign = `Workflow graph run ended during ${pauseProvenance} with task re-queued to todo — benign, cleared for normal scheduling`;
|
// FNXC:WorkflowLifecycle 2026-07-12-09:05: a row still carrying a
|
||||||
|
// pause park (paused/userPaused) is NOT "cleared for normal
|
||||||
|
// scheduling" — the scheduler skips it until an explicit unpause.
|
||||||
|
// Say so, or the log contradicts the board (FN-7851 misdiagnosis).
|
||||||
|
const todoBenign = live.paused || live.userPaused
|
||||||
|
? `Workflow graph run ended during ${pauseProvenance} with task parked in todo — benign, paused awaiting explicit unpause`
|
||||||
|
: `Workflow graph run ended during ${pauseProvenance} with task re-queued to todo — benign, cleared for normal scheduling`;
|
||||||
executorLog.log(`${task.id}: ${todoBenign}`);
|
executorLog.log(`${task.id}: ${todoBenign}`);
|
||||||
await this.store.logEntry(task.id, todoBenign, undefined, this.getRunContextFor(task.id));
|
await this.store.logEntry(task.id, todoBenign, undefined, this.getRunContextFor(task.id));
|
||||||
// FNXC:WorkflowLifecycle 2026-06-20-19:58: reconcile a stale
|
// FNXC:WorkflowLifecycle 2026-06-20-19:58: reconcile a stale
|
||||||
@@ -11588,13 +11601,28 @@ export class TaskExecutor {
|
|||||||
const hasResumableProgress =
|
const hasResumableProgress =
|
||||||
(progressSource.currentStep ?? 0) > 0
|
(progressSource.currentStep ?? 0) > 0
|
||||||
|| (progressSource.steps?.some((step) => step.status === "done" || step.status === "in-progress") ?? false);
|
|| (progressSource.steps?.some((step) => step.status === "done" || step.status === "in-progress") ?? false);
|
||||||
|
/*
|
||||||
|
FNXC:WorkflowLifecycle 2026-07-12-09:05:
|
||||||
|
Pause-bounce loop (observed on FN-7851): this teardown runs BECAUSE the user paused the task, but the plain move-to-todo below wiped the pause flags (store reopen block), leaving an unpaused dispatchable todo row. The graph-failure classifier then read `paused=false, userPaused=false`, misclassified the abort as engine-internal, and auto-continued the session; once the shared graphResumeRetryCount budget was exhausted the scheduler simply re-dispatched the row seconds later — so pausing an in-progress task could never stick. When the pause that caused this abort is still in force at teardown time, move with `preservePause` so the row lands in todo still parked (`paused` kept; scheduler skips paused/userPaused todo rows) and the classifier sees the pause and routes benignly. An unpause during the teardown window leaves `paused` unset and restores the old requeue-for-normal-scheduling behavior.
|
||||||
|
*/
|
||||||
|
const pauseStillInForce = latestTask?.paused === true;
|
||||||
await this.store.updateTask(
|
await this.store.updateTask(
|
||||||
task.id,
|
task.id,
|
||||||
hasResumableProgress ? { worktree: undefined } : { worktree: undefined, branch: undefined },
|
hasResumableProgress ? { worktree: undefined } : { worktree: undefined, branch: undefined },
|
||||||
);
|
);
|
||||||
await this.store.logEntry(task.id, "Execution paused — agent terminated, moved to todo", undefined, this.getRunContextFor(task.id));
|
await this.store.logEntry(
|
||||||
|
task.id,
|
||||||
|
pauseStillInForce
|
||||||
|
? "Execution paused — agent terminated, parked in todo (pause preserved, awaiting explicit unpause)"
|
||||||
|
: "Execution paused — agent terminated, moved to todo",
|
||||||
|
undefined,
|
||||||
|
this.getRunContextFor(task.id),
|
||||||
|
);
|
||||||
this.markGraphExecuteSelfRequeued(task.id);
|
this.markGraphExecuteSelfRequeued(task.id);
|
||||||
await this.store.moveTask(task.id, "todo", hasResumableProgress ? { preserveResumeState: true } : undefined);
|
await this.store.moveTask(task.id, "todo", {
|
||||||
|
...(hasResumableProgress ? { preserveResumeState: true } : {}),
|
||||||
|
...(pauseStillInForce ? { preservePause: true } : {}),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
} else if (this.stuckAborted.has(task.id)) {
|
} else if (this.stuckAborted.has(task.id)) {
|
||||||
// Task was killed by stuck task detector — defer requeue to finally block
|
// Task was killed by stuck task detector — defer requeue to finally block
|
||||||
|
|||||||
Reference in New Issue
Block a user