FN-5888: retry non-continuable sessions with a fresh session

Retry incomplete executor work with a fresh session after non-continuable session errors.

- add executor recovery handling that clears session state and requeues incomplete tasks to todo while retry budget remains
- preserve terminal failure behavior once the fresh-session retry budget is exhausted
- extend reliability coverage and AGENTS.md backstop notes for the new retry path

Files changed:
 AGENTS.md                                          |  1 +
 .../post-done-continuation-no-wedge.test.ts        | 48 +++++++++++++++++++++-
 packages/engine/src/executor.ts                    | 40 ++++++++++++++++++
 3 files changed, 87 insertions(+), 2 deletions(-)

Fusion-Task-Id: FN-5888

Fusion-Task-Lineage: 4ebce260-8923-4599-aced-541533b94543
This commit is contained in:
gsxdsm
2026-06-02 10:14:20 -07:00
parent a66b128089
commit 742a564491
3 changed files with 87 additions and 2 deletions

View File

@@ -4,6 +4,7 @@ import type { Task, TaskStore } from "@fusion/core";
import "../executor-test-helpers.js";
import { TaskExecutor } from "../../executor.js";
import { SelfHealingManager } from "../../self-healing.js";
import { MAX_RECOVERY_RETRIES } from "../../recovery-policy.js";
import { mockedCreateFnAgent, resetExecutorMocks } from "../executor-test-helpers.js";
function makeTask(overrides: Partial<Task> = {}): Task {
@@ -142,8 +143,46 @@ describe("FN-5866 reliability interactions: post-done continuation no wedge", ()
manager.stop();
});
it("still marks incomplete work failed when the same session error happens before completion", async () => {
const task = makeTask({ id: "FN-5866-INCOMPLETE" });
it("requeues incomplete work with a fresh session when the session is not continuable", async () => {
const task = makeTask({
id: "FN-5866-INCOMPLETE",
sessionFile: "/tmp/test/.fusion/sessions/FN-5866-INCOMPLETE.json",
});
const store = createStore(task);
const onError = vi.fn();
mockedCreateFnAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockRejectedValue(new Error("Cannot continue from message role: assistant")),
dispose: vi.fn(),
getSessionStats: vi.fn().mockResolvedValue({
tokens: { input: 5, output: 0, cacheRead: 0, cacheWrite: 0, total: 5 },
}),
},
} as any);
const executor = new TaskExecutor(store, "/tmp/test", { onError });
await executor.execute(task);
expect(task.column).toBe("todo");
expect(task.status).toBeUndefined();
expect(task.error).toBeUndefined();
expect(task.sessionFile).toBeNull();
expect(task.recoveryRetryCount).toBe(1);
expect(task.nextRecoveryAt).toEqual(expect.any(String));
expect(store.moveTask).toHaveBeenCalledWith(task.id, "todo", { preserveResumeState: true });
expect(store.handoffToReview).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
expect((task.log ?? []).some((entry: any) => entry.action.includes("Non-continuable session — fresh-session retry"))).toBe(true);
});
it("falls through to terminal failure after the non-continuable fresh-session retry budget is exhausted", async () => {
const task = makeTask({
id: "FN-5866-INCOMPLETE-EXHAUSTED",
recoveryRetryCount: MAX_RECOVERY_RETRIES,
nextRecoveryAt: "2026-06-02T00:05:00.000Z",
sessionFile: "/tmp/test/.fusion/sessions/FN-5866-INCOMPLETE-EXHAUSTED.json",
});
const store = createStore(task);
const onError = vi.fn();
@@ -163,7 +202,12 @@ describe("FN-5866 reliability interactions: post-done continuation no wedge", ()
expect(task.column).toBe("in-review");
expect(task.status).toBe("failed");
expect(task.error).toContain("Cannot continue from message role: assistant");
expect(task.recoveryRetryCount).toBeNull();
expect(task.nextRecoveryAt).toBeNull();
expect(task.sessionFile).toBeNull();
expect(store.moveTask).not.toHaveBeenCalledWith(task.id, "todo", { preserveResumeState: true });
expect(store.handoffToReview).toHaveBeenCalledTimes(1);
expect(onError).toHaveBeenCalledTimes(1);
expect((task.log ?? []).some((entry: any) => entry.action.includes("fresh-session retries exhausted"))).toBe(true);
});
});

View File

@@ -2598,6 +2598,44 @@ export class TaskExecutor {
return true;
}
private async handleNonContinuableSessionRetry(task: Task, errorMessage: string): Promise<boolean> {
if (!isNonContinuableSessionError(errorMessage)) {
return false;
}
const liveTask = await this.store.getTask(task.id);
if (!liveTask) {
return false;
}
const decision = computeRecoveryDecision({
recoveryRetryCount: liveTask.recoveryRetryCount,
nextRecoveryAt: liveTask.nextRecoveryAt,
});
if (decision.shouldRetry) {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
executorLog.warn(`${task.id} non-continuable session — fresh-session retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`);
await this.store.logEntry(task.id, `Non-continuable session — fresh-session retry (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id));
await this.store.updateTask(task.id, {
recoveryRetryCount: decision.nextState.recoveryRetryCount,
nextRecoveryAt: decision.nextState.nextRecoveryAt,
sessionFile: null,
});
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
return true;
}
executorLog.error(`${task.id} non-continuable session fresh-session retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
await this.store.logEntry(task.id, `Non-continuable session fresh-session retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.getRunContextFor(task.id));
await this.store.updateTask(task.id, {
recoveryRetryCount: null,
nextRecoveryAt: null,
});
return false;
}
private async getTaskCompletionBlocker(task: Task): Promise<string | undefined> {
return getTaskCompletionBlockerForStore(this.store, task);
}
@@ -5446,6 +5484,8 @@ export class TaskExecutor {
return;
} else if (await this.handleNonContinuableSessionError(task, taskDone, errorMessage)) {
return;
} else if (await this.handleNonContinuableSessionRetry(task, errorMessage)) {
return;
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
} else if (isTransientError(errorMessage)) {