From 9a8fc409ff6c4fbd4e2b8269ac20d349f3f9c3d1 Mon Sep 17 00:00:00 2001 From: Phil Larson Date: Wed, 29 Jul 2026 00:04:28 -0700 Subject: [PATCH] fix: persist manual task pauses (#2536) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - persist an explicit `userPaused` latch when operators pause tasks through CLI, MCP, dashboard task routes, or mission stop - keep automatic/internal pauses distinct (`userPaused` remains false unless explicitly requested) - clear the latch on unpause - route the flag through in-memory and PostgreSQL task stores - add contract coverage across core, CLI, MCP, dashboard task routes, and mission stop ## Why A manually paused task could lose the reason for its pause across dashboard/runtime restart. Startup recovery then treated it like an internally interrupted task and reclaimed it, restarting automation against the operator’s intent. Manual pauses must survive restart and remain non-runnable until explicitly unpaused. ## Verification - core pause durability tests: 2 passed - CLI task/extension tests: 150 passed; PostgreSQL integration lane remains active in CI - dashboard route tests: 261 passed - `@fusion/core`, `@runfusion/fusion`, and `@fusion/dashboard` typechecks passed - full workspace build passed with pnpm 10.33.0 - changeset validation and `git diff --check` passed - live aggregate runtime verification also confirmed `paused=true,userPaused=true` survived a normal dashboard restart with zero active tasks ## Summary by CodeRabbit - **Bug Fixes** - Manual task pauses now persist across application restarts and recovery. - Pauses initiated via the CLI, dashboard, MCP tools, and mission stop controls are recorded as explicit user actions. - Automatically paused tasks remain eligible for recovery. - Unpausing clears the durable manual-pause state. --- .changeset/persist-manual-task-pauses.md | 7 ++ .../__tests__/extension-task-tools.test.ts | 15 ++++ packages/cli/src/__tests__/extension.test.ts | 4 + .../cli/src/commands/__tests__/task.test.ts | 2 +- packages/cli/src/commands/task.ts | 2 +- packages/cli/src/extension.ts | 2 +- .../store-manual-pause-durability.test.ts | 89 +++++++++++++++++++ packages/core/src/store.ts | 2 +- .../core/src/task-store/branch-group-ops.ts | 5 +- .../src/__tests__/mission-stop-pause.test.ts | 46 ++++++++++ .../src/__tests__/routes-auth.test.ts | 2 +- packages/dashboard/src/mission-routes.ts | 54 +++++++---- ...ister-task-workflow-routes.unpause.test.ts | 7 +- .../routes/register-task-workflow-routes.ts | 2 +- 14 files changed, 213 insertions(+), 26 deletions(-) create mode 100644 .changeset/persist-manual-task-pauses.md create mode 100644 packages/core/src/__tests__/store-manual-pause-durability.test.ts create mode 100644 packages/dashboard/src/__tests__/mission-stop-pause.test.ts diff --git a/.changeset/persist-manual-task-pauses.md b/.changeset/persist-manual-task-pauses.md new file mode 100644 index 0000000000..2dcf2b1a8f --- /dev/null +++ b/.changeset/persist-manual-task-pauses.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Persist explicit user intent across manual task pauses so startup recovery cannot reclaim paused work. +category: fix +dev: CLI, dashboard, MCP tool, and mission pause controls now set the durable userPaused latch while automatic holds remain recoverable. diff --git a/packages/cli/src/__tests__/extension-task-tools.test.ts b/packages/cli/src/__tests__/extension-task-tools.test.ts index 431d71ef79..5929ec1966 100644 --- a/packages/cli/src/__tests__/extension-task-tools.test.ts +++ b/packages/cli/src/__tests__/extension-task-tools.test.ts @@ -163,4 +163,19 @@ pgTest("extension task tools resolve repo root from worktrees", () => { expect(Array.isArray(show.content)).toBe(true); expect(show.content[0]?.text).toContain(created.id); }); + + it("persists manual pause intent through fn_task_pause", async () => { + const store = h.store(); + const created = await store.createTask({ description: "Operator-paused task" }); + const api = createMockApi(); + registerExtension(api); + + const pauseTool = requireTool(api, "fn_task_pause"); + await pauseTool.execute("pause", { id: created.id }, undefined, undefined, { cwd: h.rootDir() }); + + await expect(store.getTask(created.id)).resolves.toMatchObject({ + paused: true, + userPaused: true, + }); + }); }); diff --git a/packages/cli/src/__tests__/extension.test.ts b/packages/cli/src/__tests__/extension.test.ts index 6dbea90dbe..e130293896 100644 --- a/packages/cli/src/__tests__/extension.test.ts +++ b/packages/cli/src/__tests__/extension.test.ts @@ -1360,6 +1360,10 @@ legacyDescribe("fn pi extension (legacy exhaustive suite)", () => { makeCtx(tmpDir), ); expect(pauseResult.content[0].text).toContain("Paused FN-001"); + await expect(h.store().getTask("FN-001")).resolves.toMatchObject({ + paused: true, + userPaused: true, + }); // Verify it's paused const showTool = api.tools.get("fn_task_show")!; diff --git a/packages/cli/src/commands/__tests__/task.test.ts b/packages/cli/src/commands/__tests__/task.test.ts index 248c30a678..55ca36356a 100644 --- a/packages/cli/src/commands/__tests__/task.test.ts +++ b/packages/cli/src/commands/__tests__/task.test.ts @@ -1217,7 +1217,7 @@ describe("project-aware task command behavior", () => { await runTaskPause("FN-123", "demo-project"); await runTaskUnpause("FN-123", "demo-project"); - expect(pauseTask).toHaveBeenNthCalledWith(1, "FN-123", true); + expect(pauseTask).toHaveBeenNthCalledWith(1, "FN-123", true, undefined, { userPaused: true }); expect(pauseTask).toHaveBeenNthCalledWith(2, "FN-123", false); }); diff --git a/packages/cli/src/commands/task.ts b/packages/cli/src/commands/task.ts index 0f34874137..6c3c16693f 100644 --- a/packages/cli/src/commands/task.ts +++ b/packages/cli/src/commands/task.ts @@ -1157,7 +1157,7 @@ export async function runTaskAttach(id: string, filePath: string, projectName?: export async function runTaskPause(id: string, projectName?: string) { // FNXC:CliBoardMutation 2026-07-09-00:00 (FN-7734): single board write. await withBoardWrite(projectName, { id, action: "pause task" }, async (context) => { - const task = await context.store.pauseTask(id, true); + const task = await context.store.pauseTask(id, true, undefined, { userPaused: true }); console.log(); console.log(` ✓ Paused ${task.id}`); diff --git a/packages/cli/src/extension.ts b/packages/cli/src/extension.ts index b571a90218..4515447f80 100644 --- a/packages/cli/src/extension.ts +++ b/packages/cli/src/extension.ts @@ -1799,7 +1799,7 @@ export default function kbExtension(pi: ExtensionAPI) { async execute(_toolCallId, params, _signal, _onUpdate, ctx) { const store = await getStore(ctx.cwd); - const task = await store.pauseTask(params.id, true); + const task = await store.pauseTask(params.id, true, undefined, { userPaused: true }); return { content: [{ type: "text", text: `Paused ${task.id}` }], diff --git a/packages/core/src/__tests__/store-manual-pause-durability.test.ts b/packages/core/src/__tests__/store-manual-pause-durability.test.ts new file mode 100644 index 0000000000..0acf06f73b --- /dev/null +++ b/packages/core/src/__tests__/store-manual-pause-durability.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { pauseTaskImpl } from "../task-store/branch-group-ops.js"; +import type { TaskStore } from "../store.js"; +import type { Task } from "../types.js"; + +describe("TaskStore manual pause durability", () => { + it("marks a manual pause as user-paused", async () => { + let persisted = { + id: "FN-001", + column: "in-progress", + status: "executing", + log: [], + } as unknown as Task; + + const store = { + withTaskLock: async (_id: string, operation: () => Promise) => operation(), + taskDir: () => "/tmp/FN-001", + readTaskJson: async () => ({ ...persisted, log: [...(persisted.log ?? [])] }), + atomicWriteTaskJson: async (_dir: string, task: Task) => { + persisted = task; + }, + isWatching: false, + emit: () => undefined, + } as unknown as TaskStore; + + const paused = await pauseTaskImpl(store, "FN-001", true, undefined, { userPaused: true }); + + expect(paused).toMatchObject({ paused: true, userPaused: true, status: "paused" }); + expect(persisted).toMatchObject({ paused: true, userPaused: true, status: "paused" }); + }); + + it("does not mark an automatic hold as user-paused", async () => { + let persisted = { + id: "FN-002", + column: "in-progress", + status: "executing", + log: [], + } as unknown as Task; + + const store = { + withTaskLock: async (_id: string, operation: () => Promise) => operation(), + taskDir: () => "/tmp/FN-002", + readTaskJson: async () => ({ ...persisted, log: [...(persisted.log ?? [])] }), + atomicWriteTaskJson: async (_dir: string, task: Task) => { + persisted = task; + }, + isWatching: false, + emit: () => undefined, + } as unknown as TaskStore; + + const paused = await pauseTaskImpl(store, "FN-002", true, undefined, { + pausedReason: "token_budget_exceeded", + }); + + expect(paused).toMatchObject({ paused: true, status: "paused" }); + expect(paused.userPaused).toBeUndefined(); + expect(persisted.userPaused).toBeUndefined(); + }); + + it("clears the durable user-pause latch when unpaused", async () => { + let persisted = { + id: "FN-003", + column: "in-progress", + status: "executing", + log: [], + } as unknown as Task; + + const store = { + withTaskLock: async (_id: string, operation: () => Promise) => operation(), + taskDir: () => "/tmp/FN-003", + readTaskJson: async () => ({ ...persisted, log: [...(persisted.log ?? [])] }), + atomicWriteTaskJson: async (_dir: string, task: Task) => { + persisted = task; + }, + isWatching: false, + emit: () => undefined, + } as unknown as TaskStore; + + await pauseTaskImpl(store, "FN-003", true, undefined, { userPaused: true }); + const unpaused = await pauseTaskImpl(store, "FN-003", false); + + expect(unpaused.paused).toBeUndefined(); + expect(unpaused.status).toBeUndefined(); + expect(unpaused.userPaused).toBeUndefined(); + expect(persisted.paused).toBeUndefined(); + expect(persisted.status).toBeUndefined(); + expect(persisted.userPaused).toBeUndefined(); + }); +}); diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index cd176ea636..337788dd76 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -1353,7 +1353,7 @@ export class TaskStore extends EventEmitter { public async updateTaskUnlocked( id: string, updates: Parameters[1], runContext?: RunMutationContext, ): Promise { return updateTaskUnlockedImpl(this, id, updates, runContext); } - async pauseTask( id: string, paused: boolean, runContext?: RunMutationContext, agentOptions?: { pausedByAgentId?: string; pausedReason?: string }, ): Promise { + async pauseTask( id: string, paused: boolean, runContext?: RunMutationContext, agentOptions?: { pausedByAgentId?: string; pausedReason?: string; userPaused?: boolean }, ): Promise { return pauseTaskImpl(this, id, paused, runContext, agentOptions); } diff --git a/packages/core/src/task-store/branch-group-ops.ts b/packages/core/src/task-store/branch-group-ops.ts index 0f043e837f..175601a33b 100644 --- a/packages/core/src/task-store/branch-group-ops.ts +++ b/packages/core/src/task-store/branch-group-ops.ts @@ -172,7 +172,7 @@ export async function selectNextTaskForAgentImpl(store: TaskStore, agentId: stri return null; } -export async function pauseTaskImpl(store: TaskStore, id: string, paused: boolean, runContext?: RunMutationContext, agentOptions?: { pausedByAgentId?: string; pausedReason?: string },): Promise { +export async function pauseTaskImpl(store: TaskStore, id: string, paused: boolean, runContext?: RunMutationContext, agentOptions?: { pausedByAgentId?: string; pausedReason?: string; userPaused?: boolean },): Promise { return store.withTaskLock(id, async () => { const dir = store.taskDir(id); const task = await store.readTaskJson(dir); @@ -184,6 +184,9 @@ export async function pauseTaskImpl(store: TaskStore, id: string, paused: boolea const previousPausedByAgentId = task.pausedByAgentId; task.paused = paused || undefined; + if (paused && agentOptions?.userPaused) { + task.userPaused = true; + } if (paused && agentOptions?.pausedByAgentId) { task.pausedByAgentId = agentOptions.pausedByAgentId; } diff --git a/packages/dashboard/src/__tests__/mission-stop-pause.test.ts b/packages/dashboard/src/__tests__/mission-stop-pause.test.ts new file mode 100644 index 0000000000..72a26e5797 --- /dev/null +++ b/packages/dashboard/src/__tests__/mission-stop-pause.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TaskStore } from "@fusion/core"; +import { pauseMissionTasksForOperatorStop } from "../mission-routes.js"; + +describe("pauseMissionTasksForOperatorStop", () => { + it("durably marks every linked mission task as user-paused", async () => { + const pauseTask = vi.fn().mockResolvedValue(undefined); + const store = { pauseTask } as unknown as TaskStore; + const hierarchy = { + milestones: [ + { + slices: [ + { + features: [ + { taskId: "FN-001" }, + {}, + { taskId: "FN-002" }, + ], + }, + ], + }, + ], + }; + + await expect(pauseMissionTasksForOperatorStop(store, hierarchy)).resolves.toEqual([ + "FN-001", + "FN-002", + ]); + expect(pauseTask).toHaveBeenNthCalledWith(1, "FN-001", true, undefined, { userPaused: true }); + expect(pauseTask).toHaveBeenNthCalledWith(2, "FN-002", true, undefined, { userPaused: true }); + }); + + it("continues after one linked task can no longer be paused", async () => { + const pauseTask = vi.fn() + .mockRejectedValueOnce(new Error("task not found")) + .mockResolvedValueOnce(undefined); + const store = { pauseTask } as unknown as TaskStore; + const hierarchy = { + milestones: [{ slices: [{ features: [{ taskId: "FN-gone" }, { taskId: "FN-live" }] }] }], + }; + + await expect(pauseMissionTasksForOperatorStop(store, hierarchy)).resolves.toEqual(["FN-live"]); + expect(pauseTask).toHaveBeenNthCalledWith(1, "FN-gone", true, undefined, { userPaused: true }); + expect(pauseTask).toHaveBeenNthCalledWith(2, "FN-live", true, undefined, { userPaused: true }); + }); +}); diff --git a/packages/dashboard/src/__tests__/routes-auth.test.ts b/packages/dashboard/src/__tests__/routes-auth.test.ts index 02e699ab5d..853359340b 100644 --- a/packages/dashboard/src/__tests__/routes-auth.test.ts +++ b/packages/dashboard/src/__tests__/routes-auth.test.ts @@ -3704,7 +3704,7 @@ describe("Pause/Unpause endpoints", () => { const res = await REQUEST(buildApp(), "POST", "/api/tasks/KB-001/pause"); expect(res.status).toBe(200); expect(res.body).toEqual({ id: "FN-001", paused: true }); - expect(store.pauseTask).toHaveBeenCalledWith("KB-001", true); + expect(store.pauseTask).toHaveBeenCalledWith("KB-001", true, undefined, { userPaused: true }); }); it("POST /tasks/:id/unpause — unpauses a task", async () => { diff --git a/packages/dashboard/src/mission-routes.ts b/packages/dashboard/src/mission-routes.ts index 788d41624d..ff2646e344 100644 --- a/packages/dashboard/src/mission-routes.ts +++ b/packages/dashboard/src/mission-routes.ts @@ -16,6 +16,7 @@ import { Router, type Request, type Response, type NextFunction } from "express" import { AsyncLocalStorage } from "node:async_hooks"; import { TaskStore, + createLogger, resolvePlanningSettingsModel, AgentStore, THINKING_LEVELS, @@ -71,6 +72,8 @@ import { import type { AiSessionStore } from "./ai-session-store.js"; import { resolveBranchAssignmentContext, resolveBranchSelection } from "./routes/branch-selection.js"; +const missionRoutesLog = createLogger("dashboard-mission-routes"); + /** Resolve the mission-start override through the planning settings hierarchy. */ export function resolveMissionInterviewThinkingLevel( settings: Partial | undefined, @@ -79,6 +82,39 @@ export function resolveMissionInterviewThinkingLevel( return resolvePlanningThinkingLevel(settings, thinkingLevel) as ThinkingLevel | undefined; } +type MissionTaskHierarchy = { + milestones: Array<{ + slices: Array<{ + features: Array<{ taskId?: string }>; + }>; + }>; +}; + +export async function pauseMissionTasksForOperatorStop( + store: Pick, + hierarchy: MissionTaskHierarchy, +): Promise { + const pausedTaskIds: string[] = []; + for (const milestone of hierarchy.milestones) { + for (const slice of milestone.slices) { + for (const feature of slice.features) { + if (!feature.taskId) continue; + try { + await store.pauseTask(feature.taskId, true, undefined, { userPaused: true }); + pausedTaskIds.push(feature.taskId); + } catch (error) { + // Continue stopping the mission if a linked task is already gone, but + // keep unexpected pause failures visible to operators. + missionRoutesLog.warn( + `Failed to pause mission-linked task ${feature.taskId}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + } + } + return pausedTaskIds; +} + // ── Validation Utilities ──────────────────────────────────────────────────── /* @@ -3012,22 +3048,8 @@ export function createMissionRouter( // Set mission status to blocked const updated = await missionStore.updateMission(missionId, { status: "blocked" }, { actor: DASHBOARD_MISSION_ACTOR }); - // Pause all tasks linked to features in this mission - const pausedTaskIds: string[] = []; - for (const milestone of hierarchy.milestones) { - for (const slice of milestone.slices) { - for (const feature of slice.features) { - if (feature.taskId) { - try { - await store.pauseTask(feature.taskId, true); - pausedTaskIds.push(feature.taskId); - } catch (_err) { - // Log but don't fail — task may already be paused or not found - } - } - } - } - } + // Pause all tasks linked to features in this mission. + const pausedTaskIds = await pauseMissionTasksForOperatorStop(getScopedStore(), hierarchy); res.json({ ...updated, pausedTaskIds }); }) diff --git a/packages/dashboard/src/routes/__tests__/register-task-workflow-routes.unpause.test.ts b/packages/dashboard/src/routes/__tests__/register-task-workflow-routes.unpause.test.ts index 7c9defa09e..c113ed33e0 100644 --- a/packages/dashboard/src/routes/__tests__/register-task-workflow-routes.unpause.test.ts +++ b/packages/dashboard/src/routes/__tests__/register-task-workflow-routes.unpause.test.ts @@ -33,11 +33,11 @@ const createPauseRouteHarness = (initialTaskState: any) => { */ getProjectScopedPluginMcpServers: vi.fn(async () => []), getTask: vi.fn(async () => taskState), - pauseTask: vi.fn(async (_id: string, paused: boolean) => { + pauseTask: vi.fn(async (_id: string, paused: boolean, _runContext, options) => { taskState = { ...taskState, paused: paused ? true : undefined, - userPaused: paused ? taskState.userPaused : undefined, + userPaused: paused ? (options?.userPaused ? true : taskState.userPaused) : undefined, pausedByAgentId: paused ? taskState.pausedByAgentId : undefined, }; return taskState; @@ -90,6 +90,7 @@ describe("task workflow pause routes", () => { expect(res.status).toBe(200); expect(getTaskState().paused).toBe(true); - expect(store.pauseTask).toHaveBeenCalledWith("FN-001", true); + expect(getTaskState().userPaused).toBe(true); + expect(store.pauseTask).toHaveBeenCalledWith("FN-001", true, undefined, { userPaused: true }); }); }); diff --git a/packages/dashboard/src/routes/register-task-workflow-routes.ts b/packages/dashboard/src/routes/register-task-workflow-routes.ts index 6bcdb14702..0eb629e0c2 100644 --- a/packages/dashboard/src/routes/register-task-workflow-routes.ts +++ b/packages/dashboard/src/routes/register-task-workflow-routes.ts @@ -3549,7 +3549,7 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork try { const { store: scopedStore } = await getProjectContext(req); await scopedStore.getTask(req.params.id); - const updated = await scopedStore.pauseTask(req.params.id, true); + const updated = await scopedStore.pauseTask(req.params.id, true, undefined, { userPaused: true }); res.json(updated); } catch (err: unknown) { if (err instanceof ApiError) {