FN-5888: retry non-continuable sessions with a fresh session
Retry incomplete executor work with a fresh session after non-continuable session errors. - add executor recovery handling that clears session state and requeues incomplete tasks to todo while retry budget remains - preserve terminal failure behavior once the fresh-session retry budget is exhausted - extend reliability coverage and AGENTS.md backstop notes for the new retry path Files changed: AGENTS.md | 1 + .../post-done-continuation-no-wedge.test.ts | 48 +++++++++++++++++++++- packages/engine/src/executor.ts | 40 ++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) Fusion-Task-Id: FN-5888 Fusion-Task-Lineage: 4ebce260-8923-4599-aced-541533b94543
This commit is contained in:
@@ -4,6 +4,7 @@ import type { Task, TaskStore } from "@fusion/core";
|
||||
import "../executor-test-helpers.js";
|
||||
import { TaskExecutor } from "../../executor.js";
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
import { MAX_RECOVERY_RETRIES } from "../../recovery-policy.js";
|
||||
import { mockedCreateFnAgent, resetExecutorMocks } from "../executor-test-helpers.js";
|
||||
|
||||
function makeTask(overrides: Partial<Task> = {}): Task {
|
||||
@@ -142,8 +143,46 @@ describe("FN-5866 reliability interactions: post-done continuation no wedge", ()
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("still marks incomplete work failed when the same session error happens before completion", async () => {
|
||||
const task = makeTask({ id: "FN-5866-INCOMPLETE" });
|
||||
it("requeues incomplete work with a fresh session when the session is not continuable", async () => {
|
||||
const task = makeTask({
|
||||
id: "FN-5866-INCOMPLETE",
|
||||
sessionFile: "/tmp/test/.fusion/sessions/FN-5866-INCOMPLETE.json",
|
||||
});
|
||||
const store = createStore(task);
|
||||
const onError = vi.fn();
|
||||
|
||||
mockedCreateFnAgent.mockResolvedValue({
|
||||
session: {
|
||||
prompt: vi.fn().mockRejectedValue(new Error("Cannot continue from message role: assistant")),
|
||||
dispose: vi.fn(),
|
||||
getSessionStats: vi.fn().mockResolvedValue({
|
||||
tokens: { input: 5, output: 0, cacheRead: 0, cacheWrite: 0, total: 5 },
|
||||
}),
|
||||
},
|
||||
} as any);
|
||||
|
||||
const executor = new TaskExecutor(store, "/tmp/test", { onError });
|
||||
await executor.execute(task);
|
||||
|
||||
expect(task.column).toBe("todo");
|
||||
expect(task.status).toBeUndefined();
|
||||
expect(task.error).toBeUndefined();
|
||||
expect(task.sessionFile).toBeNull();
|
||||
expect(task.recoveryRetryCount).toBe(1);
|
||||
expect(task.nextRecoveryAt).toEqual(expect.any(String));
|
||||
expect(store.moveTask).toHaveBeenCalledWith(task.id, "todo", { preserveResumeState: true });
|
||||
expect(store.handoffToReview).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect((task.log ?? []).some((entry: any) => entry.action.includes("Non-continuable session — fresh-session retry"))).toBe(true);
|
||||
});
|
||||
|
||||
it("falls through to terminal failure after the non-continuable fresh-session retry budget is exhausted", async () => {
|
||||
const task = makeTask({
|
||||
id: "FN-5866-INCOMPLETE-EXHAUSTED",
|
||||
recoveryRetryCount: MAX_RECOVERY_RETRIES,
|
||||
nextRecoveryAt: "2026-06-02T00:05:00.000Z",
|
||||
sessionFile: "/tmp/test/.fusion/sessions/FN-5866-INCOMPLETE-EXHAUSTED.json",
|
||||
});
|
||||
const store = createStore(task);
|
||||
const onError = vi.fn();
|
||||
|
||||
@@ -163,7 +202,12 @@ describe("FN-5866 reliability interactions: post-done continuation no wedge", ()
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(task.status).toBe("failed");
|
||||
expect(task.error).toContain("Cannot continue from message role: assistant");
|
||||
expect(task.recoveryRetryCount).toBeNull();
|
||||
expect(task.nextRecoveryAt).toBeNull();
|
||||
expect(task.sessionFile).toBeNull();
|
||||
expect(store.moveTask).not.toHaveBeenCalledWith(task.id, "todo", { preserveResumeState: true });
|
||||
expect(store.handoffToReview).toHaveBeenCalledTimes(1);
|
||||
expect(onError).toHaveBeenCalledTimes(1);
|
||||
expect((task.log ?? []).some((entry: any) => entry.action.includes("fresh-session retries exhausted"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2598,6 +2598,44 @@ export class TaskExecutor {
|
||||
return true;
|
||||
}
|
||||
|
||||
private async handleNonContinuableSessionRetry(task: Task, errorMessage: string): Promise<boolean> {
|
||||
if (!isNonContinuableSessionError(errorMessage)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const liveTask = await this.store.getTask(task.id);
|
||||
if (!liveTask) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const decision = computeRecoveryDecision({
|
||||
recoveryRetryCount: liveTask.recoveryRetryCount,
|
||||
nextRecoveryAt: liveTask.nextRecoveryAt,
|
||||
});
|
||||
|
||||
if (decision.shouldRetry) {
|
||||
const attempt = decision.nextState.recoveryRetryCount;
|
||||
const delay = formatDelay(decision.delayMs);
|
||||
executorLog.warn(`⚡ ${task.id} non-continuable session — fresh-session retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`);
|
||||
await this.store.logEntry(task.id, `Non-continuable session — fresh-session retry (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id));
|
||||
await this.store.updateTask(task.id, {
|
||||
recoveryRetryCount: decision.nextState.recoveryRetryCount,
|
||||
nextRecoveryAt: decision.nextState.nextRecoveryAt,
|
||||
sessionFile: null,
|
||||
});
|
||||
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
executorLog.error(`✗ ${task.id} non-continuable session fresh-session retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
|
||||
await this.store.logEntry(task.id, `Non-continuable session fresh-session retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.getRunContextFor(task.id));
|
||||
await this.store.updateTask(task.id, {
|
||||
recoveryRetryCount: null,
|
||||
nextRecoveryAt: null,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
private async getTaskCompletionBlocker(task: Task): Promise<string | undefined> {
|
||||
return getTaskCompletionBlockerForStore(this.store, task);
|
||||
}
|
||||
@@ -5446,6 +5484,8 @@ export class TaskExecutor {
|
||||
return;
|
||||
} else if (await this.handleNonContinuableSessionError(task, taskDone, errorMessage)) {
|
||||
return;
|
||||
} else if (await this.handleNonContinuableSessionRetry(task, errorMessage)) {
|
||||
return;
|
||||
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
|
||||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
|
||||
} else if (isTransientError(errorMessage)) {
|
||||
|
||||
Reference in New Issue
Block a user