FN-087: resume failed workspace merges

Recover retryable workspace merge failures immediately while preserving ownership and safety guards.

- Dispatch operator retries only after a fail-closed pending-owner check.
- Reconcile partial workspace lands during startup using existing guarded recovery logic.
- Add dashboard and self-healing regression coverage.

Files changed:
 .../routes-task-retry-stale-merge-status.test.ts   | 110 ++++++++++++++++++++-
 .../src/routes/register-task-workflow-routes.ts    |  24 ++++-
 .../src/__tests__/self-healing-workspace.test.ts   |  29 ++++++
 packages/engine/src/self-healing.ts                |   7 ++
 4 files changed, 164 insertions(+), 6 deletions(-)

Fusion-Task-Id: FN-087

Fusion-Task-Lineage: fbfc4db5-e154-4e41-90b4-4b4514627f47

Co-authored-by: Fusion <noreply@runfusion.ai>
This commit is contained in:
Fusion Agent
2026-08-20 21:03:43 +00:00
parent aa9fb0553c
commit ab812880d0
4 changed files with 164 additions and 6 deletions

View File

@@ -61,8 +61,14 @@ function mkMergeTask(overrides: Partial<Task> = {}): Task {
} as unknown as Task;
}
function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleMergingStatusMinAgeMs?: number }) {
const updateTask = vi.fn(async () => input.task);
function buildApp(input: {
task: Task;
activeMergeTaskId?: string | null;
staleMergingStatusMinAgeMs?: number;
settings?: { autoMerge?: boolean };
engine?: { isMergePending: ReturnType<typeof vi.fn>; enqueueMerge: ReturnType<typeof vi.fn> };
}) {
const updateTask = vi.fn(async (_id: string, patch: Partial<Task>) => Object.assign(input.task, patch));
const moveTask = vi.fn(async () => input.task);
const logEntry = vi.fn(async () => {});
const store = {
@@ -71,8 +77,8 @@ function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleM
updateTask,
moveTask,
logEntry,
getSettings: async () => ({}),
getSettingsFast: async () => ({}),
getSettings: async () => ({ autoMerge: true, ...input.settings }),
getSettingsFast: async () => ({ autoMerge: true, ...input.settings }),
getRootDir: () => "/tmp/does-not-exist",
listTasks: async () => [input.task],
// FNXC:TaskWedgeNotifications 2026-08-15-05:10: dashboard Retry now clears the spent generic-terminal auto-recovery budget before mutating task state; the fixture must expose the seam or every retry 500s.
@@ -90,7 +96,7 @@ function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleM
chatLogger: runtimeLogger as never,
getProjectIdFromRequest: () => undefined,
getScopedStore: async () => store,
getProjectContext: async () => ({ store, engine: undefined as never, projectId: "p-1" }),
getProjectContext: async () => ({ store, engine: input.engine as never, projectId: "p-1" }),
prioritizeProjectsForCurrentDirectory: (projects: unknown) => projects,
emitRemoteRouteDiagnostic: () => {},
emitAuthSyncAuditLog: () => {},
@@ -135,6 +141,34 @@ function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleM
return { app, updateTask, moveTask, logEntry };
}
/** The reported zero-land workspace lease-loss state, with completed execution progress. */
function mkFailedWorkspaceTask(overrides: Partial<Task> = {}): Task {
return mkMergeTask({
id: "MRG-040",
status: "failed",
error: "Workspace partial land for MRG-040: 0 repo(s) landed, 1 failed — Merge: Workspace lease is no longer valid",
workspaceWorktrees: {
Merge: { worktreePath: "/workspace/Merge/.worktrees/fast-olive", branch: "fusion/mrg-040" },
"Merge-Auth": { worktreePath: "/workspace/Merge-Auth/.worktrees/swift-eagle", branch: "fusion/mrg-040" },
},
...overrides,
} as Partial<Task>);
}
type WorkspaceRetryGateInput = {
pending?: boolean;
settings?: { autoMerge?: boolean };
task?: Partial<Task>;
probeError?: Error;
};
const workspaceRetrySafetyCases: Array<[string, WorkspaceRetryGateInput]> = [
["a pending local or remote merge owner", { pending: true }],
["an effective auto-merge hold", { settings: { autoMerge: false } }],
["a user-controlled pause", { task: { userPaused: true } }],
["an unreadable pending-owner probe", { probeError: new Error("remote lease unavailable") }],
];
describe("POST /api/tasks/:id/retry — orphaned merge-active status (FN-8004)", () => {
it("retries a task stranded in 'landing' by a killed merger", async () => {
const { app, updateTask, moveTask } = buildApp({ task: mkMergeTask() });
@@ -206,4 +240,70 @@ describe("POST /api/tasks/:id/retry — orphaned merge-active status (FN-8004)",
);
expect(moveTask).not.toHaveBeenCalled();
});
it("promptly queues a failed zero-land workspace retry after a clear pending-owner probe", async () => {
const task = mkFailedWorkspaceTask();
const workspaceWorktrees = task.workspaceWorktrees;
const engine = { isMergePending: vi.fn().mockResolvedValue(false), enqueueMerge: vi.fn().mockReturnValue(true) };
const { app, updateTask, moveTask } = buildApp({ task, engine });
const res = await performRequest(app, "POST", "/api/tasks/MRG-040/retry", "{}", { "content-type": "application/json" });
expect(res.status).toBe(200);
expect(updateTask).toHaveBeenCalledWith("MRG-040", expect.objectContaining({ status: null, error: null, mergeRetries: 0 }));
expect(task.column).toBe("in-review");
expect(task.steps.every((step) => step.status === "done")).toBe(true);
expect(task.workspaceWorktrees).toBe(workspaceWorktrees);
expect(moveTask).not.toHaveBeenCalled();
expect(engine.isMergePending).toHaveBeenCalledOnce();
expect(engine.isMergePending).toHaveBeenCalledWith("MRG-040");
expect(engine.enqueueMerge).toHaveBeenCalledOnce();
expect(engine.enqueueMerge).toHaveBeenCalledWith("MRG-040");
});
it("keeps a workspace retry successful when no engine is available", async () => {
const task = mkFailedWorkspaceTask();
const workspaceWorktrees = task.workspaceWorktrees;
const { app, moveTask } = buildApp({ task });
const res = await performRequest(app, "POST", "/api/tasks/MRG-040/retry", "{}", { "content-type": "application/json" });
expect(res.status).toBe(200);
expect(task.status).toBeNull();
expect(task.workspaceWorktrees).toBe(workspaceWorktrees);
expect(moveTask).not.toHaveBeenCalled();
});
it("does not send a non-workspace merge retry through the workspace queue", async () => {
const engine = { isMergePending: vi.fn().mockResolvedValue(false), enqueueMerge: vi.fn().mockReturnValue(true) };
const { app } = buildApp({ task: mkMergeTask({ status: "failed" }), engine });
const res = await performRequest(app, "POST", "/api/tasks/FN-8004/retry", "{}", { "content-type": "application/json" });
expect(res.status).toBe(200);
expect(engine.isMergePending).not.toHaveBeenCalled();
expect(engine.enqueueMerge).not.toHaveBeenCalled();
});
it.each(workspaceRetrySafetyCases)("does not double-dispatch a workspace retry with %s", async (_label, input) => {
const engine = {
isMergePending: input.probeError ? vi.fn().mockRejectedValue(input.probeError) : vi.fn().mockResolvedValue(input.pending === true),
enqueueMerge: vi.fn().mockReturnValue(true),
};
const task = mkFailedWorkspaceTask(input.task);
const workspaceWorktrees = task.workspaceWorktrees;
const { app, moveTask } = buildApp({ task, engine, settings: input.settings });
const res = await performRequest(app, "POST", "/api/tasks/MRG-040/retry", "{}", { "content-type": "application/json" });
expect(res.status).toBe(200);
expect(task.status).toBeNull();
expect(task.error).toBeNull();
expect(task.column).toBe("in-review");
expect(task.workspaceWorktrees).toBe(workspaceWorktrees);
expect(moveTask).not.toHaveBeenCalled();
if (input.settings || input.task) expect(engine.isMergePending).not.toHaveBeenCalled();
else expect(engine.isMergePending).toHaveBeenCalledOnce();
expect(engine.enqueueMerge).not.toHaveBeenCalled();
});
});

View File

@@ -130,7 +130,7 @@ import {
import { buildBoardWorkflowsPayload } from "./board-workflows.js";
import { resolveNativeStructurePreview } from "../native-structure-preview.js";
import { isBackwardMoveBlockedByOpenPr, PR_OPEN_BLOCKS_MOVE_BACK_MESSAGE } from "./register-pull-requests-routes.js";
import { computePlanApprovalFingerprint, isTaskAwaitingPlanning, isWorkspaceTask, type RunAuditEventInput } from "@fusion/core";
import { allowsAutoMergeProcessing, computePlanApprovalFingerprint, isTaskAwaitingPlanning, isWorkspaceTask, type RunAuditEventInput } from "@fusion/core";
import { FUSION_CLIENT_HEADER, resolveHttpDeleteCallerKind, isValidTaskBranchName } from "@fusion/core";
import { ApiError, badRequest, conflict, notFound } from "../api-error.js";
// FNXC:TaskLookup404 2026-07-26-11:40: shared task-miss -> 404 mapping seam.
@@ -3514,6 +3514,28 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
...buildManualRetryResetPatch({ resetMergeRetries: true }),
});
await scopedStore.logEntry(req.params.id, `Retry requested from dashboard (in-review merge retry, mergeRetries reset${retryLogSuffix})`);
/*
FNXC:WorkspaceRetry 2026-08-20-20:46:
A lease-loss workspace merge must resume promptly when an operator selects Retry, without
waiting for periodic recovery. Delegate only to ProjectEngine's fenced queue after its
authoritative pending-owner probe says no local or remote owner exists; probe failures stay
fail-closed so this route never duplicates an active land attempt or handles leases itself.
*/
const isCompletedWorkspaceMerge = isWorkspaceTask(task)
&& task.steps.every((step) => step.status === "done" || step.status === "skipped");
const isUserControlledPause = task.userPaused === true || (task.paused === true && !task.pausedReason);
if (engine && isCompletedWorkspaceMerge && !isUserControlledPause) {
const settings = await scopedStore.getSettings();
if (allowsAutoMergeProcessing(task, settings)) {
try {
if (!(await engine.isMergePending(task.id))) engine.enqueueMerge(task.id);
} catch {
// An unreadable pending-owner probe must preserve the retry reset but not dispatch.
}
}
}
const updated = await scopedStore.getTask(req.params.id);
res.json(updated);
return;

View File

@@ -402,6 +402,35 @@ describeIfGit("workspace-aware self-healing (Phase D U1)", () => {
expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed");
});
it("partial-land reconciler re-enqueues a failed zero-land lease-loss workspace task", async () => {
fx = await createWorkspaceFixture(["repo-a", "repo-b"]);
addRepoBranch(fx, "repo-a", "a\n");
addRepoBranch(fx, "repo-b", "b\n");
const task = workspaceTask(
{
"repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH },
"repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH },
},
{
status: "failed",
error: "Workspace partial land: 0 repo(s) landed — Workspace lease is no longer valid",
steps: [{ status: "done" }, { status: "done" }],
},
);
const workspaceWorktrees = task.workspaceWorktrees;
const store = createStore([task]);
const manager = makeManager(store, fx.rootDir);
const recovered = await manager.reconcileWorkspacePartialLands();
expect(recovered).toBe(1);
expect(store.enqueued).toEqual([TASK_ID]);
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.emitted.some((event) => event.event === "task:merged")).toBe(false);
expect(store.tasks.get(TASK_ID)?.workspaceWorktrees).toBe(workspaceWorktrees);
expect(store.tasks.get(TASK_ID)?.column).toBe("in-review");
});
// ── KTD1 P1: zero-landed mergeable workspace task admitted ─────────────────
it("recoverMergeableReviewTasks re-enqueues a zero-landed mergeable workspace task (P1)", async () => {
fx = await createWorkspaceFixture(["repo-a", "repo-b"]);

View File

@@ -1794,6 +1794,13 @@ export class SelfHealingManager extends SelfHealingGitEvidence {
}
} },
{ name: "reconcile-pending-workspace-land-intents", fn: () => this.reconcilePendingWorkspaceLandIntents().then(() => undefined) },
/*
FNXC:WorkspaceRetry 2026-08-20-20:46:
A startup after a lease-loss must not leave an otherwise retryable workspace partial land
waiting for periodic maintenance. Reuse the guarded reconciler so its human-control,
liveness, pending-owner, branch-evidence, and bounded-queue rules remain authoritative.
*/
{ name: "reconcile-workspace-partial-lands", fn: () => this.reconcileWorkspacePartialLands().then(() => undefined) },
{ name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) },
{ name: "wedged-active-merge", fn: () => this.recoverWedgedActiveMerge().then(() => undefined) },
{ name: "transient-merge-failures", fn: () => this.recoverTransientMergeFailures().then(() => undefined) },