FN-5888: retry non-continuable sessions with a fresh session
Retry incomplete executor work with a fresh session after non-continuable session errors. - add executor recovery handling that clears session state and requeues incomplete tasks to todo while retry budget remains - preserve terminal failure behavior once the fresh-session retry budget is exhausted - extend reliability coverage and AGENTS.md backstop notes for the new retry path Files changed: AGENTS.md | 1 + .../post-done-continuation-no-wedge.test.ts | 48 +++++++++++++++++++++- packages/engine/src/executor.ts | 40 ++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) Fusion-Task-Id: FN-5888 Fusion-Task-Lineage: 4ebce260-8923-4599-aced-541533b94543
This commit is contained in:
@@ -173,6 +173,7 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme
|
||||
- FN-5830 backstop: `packages/engine/src/__tests__/reliability-interactions/branch-group-promotion.test.ts` guards branch-group completion-gate + promotion lifecycle so completion detection drives exactly one shared→default promotion, re-calls stay idempotent, and gated paths emit promotion-gated telemetry without promoting.
|
||||
- FN-5820 backstop: `packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts` guards the full shared-branch-group lifecycle—concurrent distinct-worktree execution, member→shared-branch accumulation, single shared→main completion-gate promotion with idempotent re-evaluation, gate-disabled integration-without-promotion, and per-task-derived/ungrouped no-regression.
|
||||
- FN-5866 backstop: `packages/engine/src/__tests__/reliability-interactions/post-done-continuation-no-wedge.test.ts` guards the post-done non-continuable-session seam so completed executor work stays cleanly in `in-review` while incomplete tasks still fail normally.
|
||||
- FN-5888 backstop: `packages/engine/src/__tests__/reliability-interactions/post-done-continuation-no-wedge.test.ts` also covers the incomplete-task non-continuable-session fresh-session retry path, ensuring within-budget failures clear `sessionFile` and requeue to `todo` with preserved resume state while exhausted budgets still fall through to terminal failure.
|
||||
- FN-5874 backstop: `packages/engine/src/__tests__/reliability-interactions/ai-merge-ff-landed-files.test.ts` guards AI-merge fast-forward finalizer persistence of `mergeDetails.commitSha`, `landedFiles`, and `modifiedFiles`, verifies no-op landings do not fabricate metadata, and confirms normal squash landings do not set FN-5103 attribution-restriction flags; companion coverage in `packages/engine/src/__tests__/self-healing.test.ts` extends `recoverDoneTaskMergeMetadata` so done tasks with empty `mergeDetails` but a recorded `baseCommitSha` are backfilled via owned-commit discovery while FN-5103 skip guards still prevent overwrite.
|
||||
|
||||
---
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { Task, TaskStore } from "@fusion/core";
|
||||
import "../executor-test-helpers.js";
|
||||
import { TaskExecutor } from "../../executor.js";
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
import { MAX_RECOVERY_RETRIES } from "../../recovery-policy.js";
|
||||
import { mockedCreateFnAgent, resetExecutorMocks } from "../executor-test-helpers.js";
|
||||
|
||||
function makeTask(overrides: Partial<Task> = {}): Task {
|
||||
@@ -142,8 +143,46 @@ describe("FN-5866 reliability interactions: post-done continuation no wedge", ()
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("still marks incomplete work failed when the same session error happens before completion", async () => {
|
||||
const task = makeTask({ id: "FN-5866-INCOMPLETE" });
|
||||
it("requeues incomplete work with a fresh session when the session is not continuable", async () => {
|
||||
const task = makeTask({
|
||||
id: "FN-5866-INCOMPLETE",
|
||||
sessionFile: "/tmp/test/.fusion/sessions/FN-5866-INCOMPLETE.json",
|
||||
});
|
||||
const store = createStore(task);
|
||||
const onError = vi.fn();
|
||||
|
||||
mockedCreateFnAgent.mockResolvedValue({
|
||||
session: {
|
||||
prompt: vi.fn().mockRejectedValue(new Error("Cannot continue from message role: assistant")),
|
||||
dispose: vi.fn(),
|
||||
getSessionStats: vi.fn().mockResolvedValue({
|
||||
tokens: { input: 5, output: 0, cacheRead: 0, cacheWrite: 0, total: 5 },
|
||||
}),
|
||||
},
|
||||
} as any);
|
||||
|
||||
const executor = new TaskExecutor(store, "/tmp/test", { onError });
|
||||
await executor.execute(task);
|
||||
|
||||
expect(task.column).toBe("todo");
|
||||
expect(task.status).toBeUndefined();
|
||||
expect(task.error).toBeUndefined();
|
||||
expect(task.sessionFile).toBeNull();
|
||||
expect(task.recoveryRetryCount).toBe(1);
|
||||
expect(task.nextRecoveryAt).toEqual(expect.any(String));
|
||||
expect(store.moveTask).toHaveBeenCalledWith(task.id, "todo", { preserveResumeState: true });
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect((task.log ?? []).some((entry: any) => entry.action.includes("Non-continuable session — fresh-session retry"))).toBe(true);
|
||||
});
|
||||
|
||||
it("falls through to terminal failure after the non-continuable fresh-session retry budget is exhausted", async () => {
|
||||
const task = makeTask({
|
||||
id: "FN-5866-INCOMPLETE-EXHAUSTED",
|
||||
recoveryRetryCount: MAX_RECOVERY_RETRIES,
|
||||
nextRecoveryAt: "2026-06-02T00:05:00.000Z",
|
||||
sessionFile: "/tmp/test/.fusion/sessions/FN-5866-INCOMPLETE-EXHAUSTED.json",
|
||||
});
|
||||
const store = createStore(task);
|
||||
const onError = vi.fn();
|
||||
|
||||
@@ -163,7 +202,12 @@ describe("FN-5866 reliability interactions: post-done continuation no wedge", ()
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(task.status).toBe("failed");
|
||||
expect(task.error).toContain("Cannot continue from message role: assistant");
|
||||
expect(task.recoveryRetryCount).toBeNull();
|
||||
expect(task.nextRecoveryAt).toBeNull();
|
||||
expect(task.sessionFile).toBeNull();
|
||||
expect(store.moveTask).not.toHaveBeenCalledWith(task.id, "todo", { preserveResumeState: true });
|
||||
expect(store.handoffToReview).toHaveBeenCalledTimes(1);
|
||||
expect(onError).toHaveBeenCalledTimes(1);
|
||||
expect((task.log ?? []).some((entry: any) => entry.action.includes("fresh-session retries exhausted"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2598,6 +2598,44 @@ export class TaskExecutor {
|
||||
return true;
|
||||
}
|
||||
|
||||
private async handleNonContinuableSessionRetry(task: Task, errorMessage: string): Promise<boolean> {
|
||||
if (!isNonContinuableSessionError(errorMessage)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const liveTask = await this.store.getTask(task.id);
|
||||
if (!liveTask) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const decision = computeRecoveryDecision({
|
||||
recoveryRetryCount: liveTask.recoveryRetryCount,
|
||||
nextRecoveryAt: liveTask.nextRecoveryAt,
|
||||
});
|
||||
|
||||
if (decision.shouldRetry) {
|
||||
const attempt = decision.nextState.recoveryRetryCount;
|
||||
const delay = formatDelay(decision.delayMs);
|
||||
executorLog.warn(`⚡ ${task.id} non-continuable session — fresh-session retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`);
|
||||
await this.store.logEntry(task.id, `Non-continuable session — fresh-session retry (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id));
|
||||
await this.store.updateTask(task.id, {
|
||||
recoveryRetryCount: decision.nextState.recoveryRetryCount,
|
||||
nextRecoveryAt: decision.nextState.nextRecoveryAt,
|
||||
sessionFile: null,
|
||||
});
|
||||
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
executorLog.error(`✗ ${task.id} non-continuable session fresh-session retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
|
||||
await this.store.logEntry(task.id, `Non-continuable session fresh-session retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.getRunContextFor(task.id));
|
||||
await this.store.updateTask(task.id, {
|
||||
recoveryRetryCount: null,
|
||||
nextRecoveryAt: null,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
private async getTaskCompletionBlocker(task: Task): Promise<string | undefined> {
|
||||
return getTaskCompletionBlockerForStore(this.store, task);
|
||||
}
|
||||
@@ -5446,6 +5484,8 @@ export class TaskExecutor {
|
||||
return;
|
||||
} else if (await this.handleNonContinuableSessionError(task, taskDone, errorMessage)) {
|
||||
return;
|
||||
} else if (await this.handleNonContinuableSessionRetry(task, errorMessage)) {
|
||||
return;
|
||||
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
|
||||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
|
||||
} else if (isTransientError(errorMessage)) {
|
||||
|
||||
Reference in New Issue
Block a user