root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(engine): prevent auto-merge cooldown loop on unresolvable conflicts

Browse Source 
Tasks were getting stuck in `in-review` forever when auto-merge could not
resolve conflicts within MAX_AUTO_MERGE_RETRIES. The conflict-exhaustion
branch silently cleared `status` (no error, no log entry, no comment),
and the 30-min cooldown sweep would reset retries and re-attempt the
same impossible merge — looping silently with no user-facing surface.

Why:
- FN-2918 and FN-2903 both spent hours in this loop with no error/comment
  visible on the task. The only log evidence was repeated
  "Auto-merge retry cooldown elapsed (30m idle)" entries with no
  follow-up outcome.

How to apply:
- Every merge failure now writes a `<Manual|Auto>-merge failed: <msg>`
  entry to the task log so the dashboard surfaces the reason.
- Conflict-retry exhaustion now bounces the task back to `in-progress`
  with a comment + log entry so the executor re-rebases against main
  and retries — mirroring the verification-failure-bounce pattern.
- New `mergeConflictBounceCount` task field caps outer bounces
  (`MAX_MERGE_CONFLICT_BOUNCES = 2`); past the cap, the task is parked
  in `in-review` with `status="failed"` and a follow-up triage task is
  created so a human can resolve the conflict manually.
- Non-conflict and non-direct-strategy errors now also set
  `status="failed"` so the cooldown sweep can't re-pick them up.
- `canMergeTask` skips tasks with `status="failed"` so terminal
  failures (verification cap, bounce cap, non-conflict error) are no
  longer eligible for cooldown re-attempts.

Schema migration v52 adds the `mergeConflictBounceCount` column.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-28 23:17:10 -07:00
parent d155b8964a
commit 15e4cba5e9
11 changed files with 242 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
packages/engine/src/__tests__/merge-error-recovery.test.ts
View File

@@ -36,6 +36,9 @@ type MockTask = {
status: string | null;
error: string | null;
verificationFailureCount?: number;
mergeConflictBounceCount?: number;
branch?: string;
worktree?: string;
updatedAt: string;
log: Array<{ action?: string }>;
};
@@ -158,20 +161,36 @@ describe("ProjectEngine merge error recovery", () => {
logSpy = vi.spyOn(runtimeLog, "log").mockImplementation(() => undefined);
});
it("clears status when conflict retries are exhausted and recovery update succeeds", async () => {
it("bounces task to in-progress when conflict retries are exhausted (under bounce cap)", async () => {
const store = makeStore({
tasks: [makeTask({ mergeRetries: 2 }), makeTask({ mergeRetries: 3 })],
tasks: [makeTask({ mergeRetries: 2 }), makeTask({ mergeRetries: 3, branch: "fusion/fn-2084" })],
});
vi.mocked(aiMergeTask).mockRejectedValueOnce(new Error("merge conflict detected"));
const engine = createEngine(store);
await runMergeCycle(engine);
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, { status: null });
expect(hasErrorLog(errorSpy, "failed to clear status on")).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
status: null,
mergeRetries: 0,
error: null,
mergeConflictBounceCount: 1,
});
expect(store.moveTask).toHaveBeenCalledWith(TASK_ID, "in-progress");
expect(store.addTaskComment).toHaveBeenCalledWith(
TASK_ID,
expect.stringContaining("Bouncing back to in-progress"),
"agent",
);
expect(store.logEntry).toHaveBeenCalledWith(
TASK_ID,
expect.stringContaining("bounced to in-progress"),
"MergeConflictBounce",
);
expect(hasErrorLog(errorSpy, "failed to bounce")).toBe(false);
});
it("logs when clearing status fails after conflict retries are exhausted", async () => {
it("logs when bouncing fails after conflict retries are exhausted", async () => {
const store = makeStore({
tasks: [makeTask({ mergeRetries: 2 }), makeTask({ mergeRetries: 3 })],
updateTask: vi.fn(async () => {
@@ -183,10 +202,36 @@ describe("ProjectEngine merge error recovery", () => {
const engine = createEngine(store);
await expect(runMergeCycle(engine)).resolves.toBeUndefined();
expect(hasErrorLog(errorSpy, `failed to clear status on ${TASK_ID}`)).toBe(true);
expect(hasErrorLog(errorSpy, `failed to bounce ${TASK_ID}`)).toBe(true);
expect(hasErrorLog(errorSpy, "db write failed")).toBe(true);
});
it("parks task and creates follow-up when conflict bounce cap is exceeded", async () => {
// Already bounced twice (cap is 2) — next bounce would be 3, exceeding cap
const store = makeStore({
tasks: [
makeTask({ mergeRetries: 2, mergeConflictBounceCount: 2, branch: "fusion/fn-2084" }),
makeTask({ mergeRetries: 3, mergeConflictBounceCount: 2, branch: "fusion/fn-2084" }),
],
});
vi.mocked(aiMergeTask).mockRejectedValueOnce(new Error("merge conflict detected"));
const engine = createEngine(store);
await runMergeCycle(engine);
expect(store.moveTask).not.toHaveBeenCalledWith(TASK_ID, "in-progress");
expect(store.updateTask).toHaveBeenCalledWith(
TASK_ID,
expect.objectContaining({
status: "failed",
mergeRetries: 3,
}),
);
expect(store.createTask).toHaveBeenCalledWith(
expect.objectContaining({ column: "triage", priority: "high" }),
);
});
it("stores terminal merge metadata for non-conflict direct merge errors", async () => {
const store = makeStore();
vi.mocked(aiMergeTask).mockRejectedValueOnce(new Error("remote branch missing"));
@@ -195,7 +240,7 @@ describe("ProjectEngine merge error recovery", () => {
await runMergeCycle(engine);
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
status: null,
status: "failed",
mergeRetries: 3,
error: "remote branch missing",
});
@@ -238,7 +283,7 @@ describe("ProjectEngine merge error recovery", () => {
expect(processPullRequestMerge).toHaveBeenCalledTimes(1);
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
status: null,
status: "failed",
mergeRetries: 3,
error: "PR API timeout",
});

145
packages/engine/src/project-engine.ts
View File

@@ -174,6 +174,12 @@ export class ProjectEngine {
* a follow-up triage task so a fresh agent (or human) can investigate
* the underlying flake/regression instead of looping forever. */
private static readonly MAX_VERIFICATION_FAILURE_BOUNCES = 3;
/** Cap on outer in-review→in-progress bounces caused by auto-merge conflict
* retries being exhausted. After this many bounces the task is parked in
* in-review with status=failed and a follow-up task is created, so the
* 30-minute cooldown sweep cannot loop forever on a merge that requires
* human intervention. */
private static readonly MAX_MERGE_CONFLICT_BOUNCES = 2;
/** 30-minute cooldown before a retry-exhausted task gets another sweep attempt */
private static readonly AUTO_MERGE_COOLDOWN_MS = 30 * 60 * 1000;
@@ -938,6 +944,10 @@ export class ProjectEngine {
// Already-confirmed merges always eligible — just need to move to done
if (task.mergeDetails?.mergeConfirmed) return true;
if (this.options.getTaskMergeBlocker?.(task as Task)) return false;
// Terminal failure: don't let the cooldown sweep re-attempt a merge that
// already gave up (verification cap, conflict-bounce cap, or non-conflict
// error). The task is parked for human/follow-up intervention.
if (task.status === "failed") return false;
return (
(task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES ||
this.hasAutoHealableVerificationBufferFailure(task) ||
@@ -1153,6 +1163,20 @@ export class ProjectEngine {
runtimeLog.error(`${manualResolver ? "Manual" : "Auto"}-merge failed for ${taskId}: ${errorMsg}`);
// Surface every merge failure on the task log so the dashboard shows
// *why* a merge didn't complete instead of silently looping.
await store
.logEntry(
taskId,
`${manualResolver ? "Manual" : "Auto"}-merge failed: ${errorMsg}`,
err instanceof Error ? err.name : undefined,
)
.catch((logErr: unknown) => {
runtimeLog.warn(
`Auto-merge: failed to log merge-failure entry on ${taskId}: ${logErr instanceof Error ? logErr.message : String(logErr)}`,
);
});
// If this was a manual merge, reject the promise and skip auto-retry logic
if (manualResolver) {
this.manualMergeResolvers.delete(taskId);
@@ -1276,23 +1300,122 @@ export class ProjectEngine {
if (!this.shuttingDown) this.internalEnqueueMerge(taskId);
}, delayMs);
} else {
// Max retries exceeded or auto-resolve disabled
try {
await store.updateTask(taskId, { status: null });
} catch (recoveryErr) {
runtimeLog.error(
`Auto-merge: failed to clear status on ${taskId} after max retries exceeded: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
);
// Conflict retries exhausted (or auto-resolve disabled).
// Previous behavior: silently clear status, leaving the task in
// in-review with mergeRetries=MAX. The 30-min cooldown sweep
// would then reset retries and re-attempt the same impossible
// merge forever, with no error surface for the user.
//
// New behavior: bounce the task back to in-progress so the
// executor can rebase against the latest main and retry. Cap
// bounces at MAX_MERGE_CONFLICT_BOUNCES — past that, park in
// in-review with status=failed and create a follow-up task so
// a human can resolve the conflict manually.
const previousBounces = taskOnErr.mergeConflictBounceCount ?? 0;
const nextBounces = previousBounces + 1;
const bounceCap = ProjectEngine.MAX_MERGE_CONFLICT_BOUNCES;
const autoResolveDisabled =
(settingsOnErr as Settings).autoResolveConflicts === false;
if (autoResolveDisabled || nextBounces > bounceCap) {
// Park for human intervention.
const reason = autoResolveDisabled
? "autoResolveConflicts is disabled"
: `merge-conflict bounce cap reached (${nextBounces - 1}/${bounceCap})`;
try {
await store.updateTask(taskId, {
status: "failed",
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
error: `Auto-merge gave up: ${reason}. ${errorMsg}`,
});
await store.addTaskComment(
taskId,
`Auto-merge gave up after ${ProjectEngine.MAX_AUTO_MERGE_RETRIES} conflict-resolution retries (${reason}). ` +
`Resolve the conflict on branch \`${taskOnErr.branch ?? "?"}\` manually, then unpause/retry.`,
"agent",
);
await store.logEntry(
taskId,
`Auto-merge gave up after conflict retries exhausted (${reason}); task parked for human intervention`,
"MergeConflictGiveUp",
);
if (!autoResolveDisabled) {
// Create a follow-up only when we capped on bounces; if
// auto-resolve is just disabled, the user is presumed to
// be handling merges manually and a follow-up is noise.
try {
const followUp = await store.createTask({
description:
`Resolve auto-merge conflict on ${taskId} (${taskOnErr.title || "untitled"}). ` +
`Auto-merge attempted to rebase + resolve ${nextBounces - 1} times against main and exhausted retries each pass. ` +
`Branch: \`${taskOnErr.branch ?? "?"}\`. Worktree: \`${taskOnErr.worktree ?? "?"}\`. ` +
`Last merge error: ${errorMsg}`,
column: "triage",
priority: "high",
});
await store.addTaskComment(
taskId,
`Created follow-up ${followUp.id} to track manual conflict resolution.`,
"agent",
);
} catch (followUpErr) {
runtimeLog.warn(
`Auto-merge: failed to create follow-up for ${taskId}: ${followUpErr instanceof Error ? followUpErr.message : String(followUpErr)}`,
);
}
}
} catch (recoveryErr) {
runtimeLog.error(
`Auto-merge: failed to park ${taskId} after conflict-bounce cap: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
);
}
} else {
// Bounce to in-progress for a fresh rebase + retry pass.
try {
await store.addTaskComment(
taskId,
`Auto-merge could not resolve conflicts within ${ProjectEngine.MAX_AUTO_MERGE_RETRIES} retries (bounce ${nextBounces}/${bounceCap}). ` +
`Bouncing back to in-progress for a fresh rebase against main; the executor will re-run quality gates and re-attempt the merge.`,
"agent",
);
await store.updateTask(taskId, {
status: null,
mergeRetries: 0,
error: null,
mergeConflictBounceCount: nextBounces,
});
await store.moveTask(taskId, "in-progress");
await store.logEntry(
taskId,
`Auto-merge conflicts unresolved (${ProjectEngine.MAX_AUTO_MERGE_RETRIES}/${ProjectEngine.MAX_AUTO_MERGE_RETRIES}) — bounced to in-progress for re-rebase (bounce ${nextBounces}/${bounceCap})`,
"MergeConflictBounce",
);
runtimeLog.log(
`Auto-merge: ${taskId} conflict retries exhausted — bounced to in-progress (${nextBounces}/${bounceCap})`,
);
} catch (recoveryErr) {
runtimeLog.error(
`Auto-merge: failed to bounce ${taskId} after conflict exhaustion: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
);
}
}
}
} else {
// Non-conflict error — stop retrying until user intervenes
// Non-conflict error — stop retrying until user intervenes.
// Mark status=failed so the cooldown sweep won't silently
// re-attempt; the catch-block-top logEntry already recorded the
// failure on the task log.
try {
await store.updateTask(taskId, {
status: null,
status: "failed",
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
error: errorMsg,
});
await store.addTaskComment(
taskId,
`Auto-merge failed with a non-conflict error and stopped retrying: ${errorMsg}`,
"agent",
);
} catch (recoveryErr) {
runtimeLog.error(
`Auto-merge: failed to update ${taskId} after non-conflict error: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
@@ -1300,9 +1423,11 @@ export class ProjectEngine {
}
}
} else {
// Non-direct merge strategy (e.g. pull-request) errored — park as
// failed so the cooldown sweep stops re-attempting silently.
try {
await store.updateTask(taskId, {
status: null,
status: "failed",
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
error: errorMsg,
});