FN-087: resume failed workspace merges
Recover retryable workspace merge failures immediately while preserving ownership and safety guards. - Dispatch operator retries only after a fail-closed pending-owner check. - Reconcile partial workspace lands during startup using existing guarded recovery logic. - Add dashboard and self-healing regression coverage. Files changed: .../routes-task-retry-stale-merge-status.test.ts | 110 ++++++++++++++++++++- .../src/routes/register-task-workflow-routes.ts | 24 ++++- .../src/__tests__/self-healing-workspace.test.ts | 29 ++++++ packages/engine/src/self-healing.ts | 7 ++ 4 files changed, 164 insertions(+), 6 deletions(-) Fusion-Task-Id: FN-087 Fusion-Task-Lineage: fbfc4db5-e154-4e41-90b4-4b4514627f47 Co-authored-by: Fusion <noreply@runfusion.ai>
This commit is contained in:
@@ -61,8 +61,14 @@ function mkMergeTask(overrides: Partial<Task> = {}): Task {
|
||||
} as unknown as Task;
|
||||
}
|
||||
|
||||
function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleMergingStatusMinAgeMs?: number }) {
|
||||
const updateTask = vi.fn(async () => input.task);
|
||||
function buildApp(input: {
|
||||
task: Task;
|
||||
activeMergeTaskId?: string | null;
|
||||
staleMergingStatusMinAgeMs?: number;
|
||||
settings?: { autoMerge?: boolean };
|
||||
engine?: { isMergePending: ReturnType<typeof vi.fn>; enqueueMerge: ReturnType<typeof vi.fn> };
|
||||
}) {
|
||||
const updateTask = vi.fn(async (_id: string, patch: Partial<Task>) => Object.assign(input.task, patch));
|
||||
const moveTask = vi.fn(async () => input.task);
|
||||
const logEntry = vi.fn(async () => {});
|
||||
const store = {
|
||||
@@ -71,8 +77,8 @@ function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleM
|
||||
updateTask,
|
||||
moveTask,
|
||||
logEntry,
|
||||
getSettings: async () => ({}),
|
||||
getSettingsFast: async () => ({}),
|
||||
getSettings: async () => ({ autoMerge: true, ...input.settings }),
|
||||
getSettingsFast: async () => ({ autoMerge: true, ...input.settings }),
|
||||
getRootDir: () => "/tmp/does-not-exist",
|
||||
listTasks: async () => [input.task],
|
||||
// FNXC:TaskWedgeNotifications 2026-08-15-05:10: dashboard Retry now clears the spent generic-terminal auto-recovery budget before mutating task state; the fixture must expose the seam or every retry 500s.
|
||||
@@ -90,7 +96,7 @@ function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleM
|
||||
chatLogger: runtimeLogger as never,
|
||||
getProjectIdFromRequest: () => undefined,
|
||||
getScopedStore: async () => store,
|
||||
getProjectContext: async () => ({ store, engine: undefined as never, projectId: "p-1" }),
|
||||
getProjectContext: async () => ({ store, engine: input.engine as never, projectId: "p-1" }),
|
||||
prioritizeProjectsForCurrentDirectory: (projects: unknown) => projects,
|
||||
emitRemoteRouteDiagnostic: () => {},
|
||||
emitAuthSyncAuditLog: () => {},
|
||||
@@ -135,6 +141,34 @@ function buildApp(input: { task: Task; activeMergeTaskId?: string | null; staleM
|
||||
return { app, updateTask, moveTask, logEntry };
|
||||
}
|
||||
|
||||
/** The reported zero-land workspace lease-loss state, with completed execution progress. */
|
||||
function mkFailedWorkspaceTask(overrides: Partial<Task> = {}): Task {
|
||||
return mkMergeTask({
|
||||
id: "MRG-040",
|
||||
status: "failed",
|
||||
error: "Workspace partial land for MRG-040: 0 repo(s) landed, 1 failed — Merge: Workspace lease is no longer valid",
|
||||
workspaceWorktrees: {
|
||||
Merge: { worktreePath: "/workspace/Merge/.worktrees/fast-olive", branch: "fusion/mrg-040" },
|
||||
"Merge-Auth": { worktreePath: "/workspace/Merge-Auth/.worktrees/swift-eagle", branch: "fusion/mrg-040" },
|
||||
},
|
||||
...overrides,
|
||||
} as Partial<Task>);
|
||||
}
|
||||
|
||||
type WorkspaceRetryGateInput = {
|
||||
pending?: boolean;
|
||||
settings?: { autoMerge?: boolean };
|
||||
task?: Partial<Task>;
|
||||
probeError?: Error;
|
||||
};
|
||||
|
||||
const workspaceRetrySafetyCases: Array<[string, WorkspaceRetryGateInput]> = [
|
||||
["a pending local or remote merge owner", { pending: true }],
|
||||
["an effective auto-merge hold", { settings: { autoMerge: false } }],
|
||||
["a user-controlled pause", { task: { userPaused: true } }],
|
||||
["an unreadable pending-owner probe", { probeError: new Error("remote lease unavailable") }],
|
||||
];
|
||||
|
||||
describe("POST /api/tasks/:id/retry — orphaned merge-active status (FN-8004)", () => {
|
||||
it("retries a task stranded in 'landing' by a killed merger", async () => {
|
||||
const { app, updateTask, moveTask } = buildApp({ task: mkMergeTask() });
|
||||
@@ -206,4 +240,70 @@ describe("POST /api/tasks/:id/retry — orphaned merge-active status (FN-8004)",
|
||||
);
|
||||
expect(moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("promptly queues a failed zero-land workspace retry after a clear pending-owner probe", async () => {
|
||||
const task = mkFailedWorkspaceTask();
|
||||
const workspaceWorktrees = task.workspaceWorktrees;
|
||||
const engine = { isMergePending: vi.fn().mockResolvedValue(false), enqueueMerge: vi.fn().mockReturnValue(true) };
|
||||
const { app, updateTask, moveTask } = buildApp({ task, engine });
|
||||
|
||||
const res = await performRequest(app, "POST", "/api/tasks/MRG-040/retry", "{}", { "content-type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(updateTask).toHaveBeenCalledWith("MRG-040", expect.objectContaining({ status: null, error: null, mergeRetries: 0 }));
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(task.steps.every((step) => step.status === "done")).toBe(true);
|
||||
expect(task.workspaceWorktrees).toBe(workspaceWorktrees);
|
||||
expect(moveTask).not.toHaveBeenCalled();
|
||||
expect(engine.isMergePending).toHaveBeenCalledOnce();
|
||||
expect(engine.isMergePending).toHaveBeenCalledWith("MRG-040");
|
||||
expect(engine.enqueueMerge).toHaveBeenCalledOnce();
|
||||
expect(engine.enqueueMerge).toHaveBeenCalledWith("MRG-040");
|
||||
});
|
||||
|
||||
it("keeps a workspace retry successful when no engine is available", async () => {
|
||||
const task = mkFailedWorkspaceTask();
|
||||
const workspaceWorktrees = task.workspaceWorktrees;
|
||||
const { app, moveTask } = buildApp({ task });
|
||||
|
||||
const res = await performRequest(app, "POST", "/api/tasks/MRG-040/retry", "{}", { "content-type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(task.status).toBeNull();
|
||||
expect(task.workspaceWorktrees).toBe(workspaceWorktrees);
|
||||
expect(moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not send a non-workspace merge retry through the workspace queue", async () => {
|
||||
const engine = { isMergePending: vi.fn().mockResolvedValue(false), enqueueMerge: vi.fn().mockReturnValue(true) };
|
||||
const { app } = buildApp({ task: mkMergeTask({ status: "failed" }), engine });
|
||||
|
||||
const res = await performRequest(app, "POST", "/api/tasks/FN-8004/retry", "{}", { "content-type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(engine.isMergePending).not.toHaveBeenCalled();
|
||||
expect(engine.enqueueMerge).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(workspaceRetrySafetyCases)("does not double-dispatch a workspace retry with %s", async (_label, input) => {
|
||||
const engine = {
|
||||
isMergePending: input.probeError ? vi.fn().mockRejectedValue(input.probeError) : vi.fn().mockResolvedValue(input.pending === true),
|
||||
enqueueMerge: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
const task = mkFailedWorkspaceTask(input.task);
|
||||
const workspaceWorktrees = task.workspaceWorktrees;
|
||||
const { app, moveTask } = buildApp({ task, engine, settings: input.settings });
|
||||
|
||||
const res = await performRequest(app, "POST", "/api/tasks/MRG-040/retry", "{}", { "content-type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(task.status).toBeNull();
|
||||
expect(task.error).toBeNull();
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(task.workspaceWorktrees).toBe(workspaceWorktrees);
|
||||
expect(moveTask).not.toHaveBeenCalled();
|
||||
if (input.settings || input.task) expect(engine.isMergePending).not.toHaveBeenCalled();
|
||||
else expect(engine.isMergePending).toHaveBeenCalledOnce();
|
||||
expect(engine.enqueueMerge).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -130,7 +130,7 @@ import {
|
||||
import { buildBoardWorkflowsPayload } from "./board-workflows.js";
|
||||
import { resolveNativeStructurePreview } from "../native-structure-preview.js";
|
||||
import { isBackwardMoveBlockedByOpenPr, PR_OPEN_BLOCKS_MOVE_BACK_MESSAGE } from "./register-pull-requests-routes.js";
|
||||
import { computePlanApprovalFingerprint, isTaskAwaitingPlanning, isWorkspaceTask, type RunAuditEventInput } from "@fusion/core";
|
||||
import { allowsAutoMergeProcessing, computePlanApprovalFingerprint, isTaskAwaitingPlanning, isWorkspaceTask, type RunAuditEventInput } from "@fusion/core";
|
||||
import { FUSION_CLIENT_HEADER, resolveHttpDeleteCallerKind, isValidTaskBranchName } from "@fusion/core";
|
||||
import { ApiError, badRequest, conflict, notFound } from "../api-error.js";
|
||||
// FNXC:TaskLookup404 2026-07-26-11:40: shared task-miss -> 404 mapping seam.
|
||||
@@ -3514,6 +3514,28 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
|
||||
...buildManualRetryResetPatch({ resetMergeRetries: true }),
|
||||
});
|
||||
await scopedStore.logEntry(req.params.id, `Retry requested from dashboard (in-review merge retry, mergeRetries reset${retryLogSuffix})`);
|
||||
|
||||
/*
|
||||
FNXC:WorkspaceRetry 2026-08-20-20:46:
|
||||
A lease-loss workspace merge must resume promptly when an operator selects Retry, without
|
||||
waiting for periodic recovery. Delegate only to ProjectEngine's fenced queue after its
|
||||
authoritative pending-owner probe says no local or remote owner exists; probe failures stay
|
||||
fail-closed so this route never duplicates an active land attempt or handles leases itself.
|
||||
*/
|
||||
const isCompletedWorkspaceMerge = isWorkspaceTask(task)
|
||||
&& task.steps.every((step) => step.status === "done" || step.status === "skipped");
|
||||
const isUserControlledPause = task.userPaused === true || (task.paused === true && !task.pausedReason);
|
||||
if (engine && isCompletedWorkspaceMerge && !isUserControlledPause) {
|
||||
const settings = await scopedStore.getSettings();
|
||||
if (allowsAutoMergeProcessing(task, settings)) {
|
||||
try {
|
||||
if (!(await engine.isMergePending(task.id))) engine.enqueueMerge(task.id);
|
||||
} catch {
|
||||
// An unreadable pending-owner probe must preserve the retry reset but not dispatch.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const updated = await scopedStore.getTask(req.params.id);
|
||||
res.json(updated);
|
||||
return;
|
||||
|
||||
@@ -402,6 +402,35 @@ describeIfGit("workspace-aware self-healing (Phase D U1)", () => {
|
||||
expect(store.tasks.get(TASK_ID)?.status).not.toBe("failed");
|
||||
});
|
||||
|
||||
it("partial-land reconciler re-enqueues a failed zero-land lease-loss workspace task", async () => {
|
||||
fx = await createWorkspaceFixture(["repo-a", "repo-b"]);
|
||||
addRepoBranch(fx, "repo-a", "a\n");
|
||||
addRepoBranch(fx, "repo-b", "b\n");
|
||||
const task = workspaceTask(
|
||||
{
|
||||
"repo-a": { worktreePath: fx.repoPath("repo-a"), branch: BRANCH },
|
||||
"repo-b": { worktreePath: fx.repoPath("repo-b"), branch: BRANCH },
|
||||
},
|
||||
{
|
||||
status: "failed",
|
||||
error: "Workspace partial land: 0 repo(s) landed — Workspace lease is no longer valid",
|
||||
steps: [{ status: "done" }, { status: "done" }],
|
||||
},
|
||||
);
|
||||
const workspaceWorktrees = task.workspaceWorktrees;
|
||||
const store = createStore([task]);
|
||||
const manager = makeManager(store, fx.rootDir);
|
||||
|
||||
const recovered = await manager.reconcileWorkspacePartialLands();
|
||||
|
||||
expect(recovered).toBe(1);
|
||||
expect(store.enqueued).toEqual([TASK_ID]);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.emitted.some((event) => event.event === "task:merged")).toBe(false);
|
||||
expect(store.tasks.get(TASK_ID)?.workspaceWorktrees).toBe(workspaceWorktrees);
|
||||
expect(store.tasks.get(TASK_ID)?.column).toBe("in-review");
|
||||
});
|
||||
|
||||
// ── KTD1 P1: zero-landed mergeable workspace task admitted ─────────────────
|
||||
it("recoverMergeableReviewTasks re-enqueues a zero-landed mergeable workspace task (P1)", async () => {
|
||||
fx = await createWorkspaceFixture(["repo-a", "repo-b"]);
|
||||
|
||||
@@ -1794,6 +1794,13 @@ export class SelfHealingManager extends SelfHealingGitEvidence {
|
||||
}
|
||||
} },
|
||||
{ name: "reconcile-pending-workspace-land-intents", fn: () => this.reconcilePendingWorkspaceLandIntents().then(() => undefined) },
|
||||
/*
|
||||
FNXC:WorkspaceRetry 2026-08-20-20:46:
|
||||
A startup after a lease-loss must not leave an otherwise retryable workspace partial land
|
||||
waiting for periodic maintenance. Reuse the guarded reconciler so its human-control,
|
||||
liveness, pending-owner, branch-evidence, and bounded-queue rules remain authoritative.
|
||||
*/
|
||||
{ name: "reconcile-workspace-partial-lands", fn: () => this.reconcileWorkspacePartialLands().then(() => undefined) },
|
||||
{ name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) },
|
||||
{ name: "wedged-active-merge", fn: () => this.recoverWedgedActiveMerge().then(() => undefined) },
|
||||
{ name: "transient-merge-failures", fn: () => this.recoverTransientMergeFailures().then(() => undefined) },
|
||||
|
||||
Reference in New Issue
Block a user