FN-5866: suppress post-done non-continuable session errors
Keep completed executor work in review when session continuation is no longer possible. - detect non-continuable session errors separately from unsupported message role failures - suppress post-done continuation errors once task work is already complete and clear any failed state before review handoff - add regression coverage for completed vs incomplete continuation failures and document the FN-5866 backstop Files changed: AGENTS.md | 1 + .../post-done-continuation-no-wedge.test.ts | 169 +++++++++++++++++++++ .../src/__tests__/transient-error-detector.test.ts | 27 ++++ packages/engine/src/executor.ts | 41 ++++- packages/engine/src/transient-error-detector.ts | 8 + 5 files changed, 245 insertions(+), 1 deletion(-) Fusion-Task-Id: FN-5866 Fusion-Task-Lineage: b5cffce4-7c14-40f6-b12e-84b9acdf7f50
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { Task, TaskStore } from "@fusion/core";
|
||||
import "../executor-test-helpers.js";
|
||||
import { TaskExecutor } from "../../executor.js";
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
import { mockedCreateFnAgent, resetExecutorMocks } from "../executor-test-helpers.js";
|
||||
|
||||
function makeTask(overrides: Partial<Task> = {}): Task {
|
||||
return {
|
||||
id: "FN-5866",
|
||||
title: "Prevent executor from continuing post-done sessions",
|
||||
description: "regression fixture",
|
||||
column: "in-progress",
|
||||
status: undefined,
|
||||
error: undefined,
|
||||
paused: false,
|
||||
userPaused: false,
|
||||
dependencies: [],
|
||||
steps: [{ name: "Implement", status: "pending" as const }],
|
||||
currentStep: 0,
|
||||
workflowStepResults: [],
|
||||
log: [],
|
||||
prompt: "# Task\n\n## Steps\n\n### Step 0: Implement\n- [ ] do the work\n",
|
||||
branch: "fusion/fn-5866",
|
||||
worktree: "/tmp/test/.worktrees/swift-falcon",
|
||||
createdAt: "2026-06-02T00:00:00.000Z",
|
||||
updatedAt: "2026-06-02T00:00:00.000Z",
|
||||
...overrides,
|
||||
} as Task;
|
||||
}
|
||||
|
||||
function createStore(task: Task, settingsOverrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
|
||||
const emitter = new EventEmitter() as TaskStore & EventEmitter;
|
||||
const audits: any[] = [];
|
||||
|
||||
(emitter as any).__audits = audits;
|
||||
(emitter as any).getTask = vi.fn().mockImplementation(async () => task);
|
||||
(emitter as any).listTasks = vi.fn().mockImplementation(async ({ column }: { column?: string } = {}) => {
|
||||
if (!column) return [task];
|
||||
return task.column === column ? [task] : [];
|
||||
});
|
||||
(emitter as any).getSettings = vi.fn().mockResolvedValue({
|
||||
autoMerge: true,
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
maxConcurrent: 2,
|
||||
maxWorktrees: 4,
|
||||
pollIntervalMs: 15_000,
|
||||
groupOverlappingFiles: false,
|
||||
inReviewStallDeadlockThreshold: 3,
|
||||
taskStuckTimeoutMs: 60_000,
|
||||
...settingsOverrides,
|
||||
});
|
||||
(emitter as any).updateTask = vi.fn().mockImplementation(async (_taskId: string, updates: Partial<Task>) => {
|
||||
Object.assign(task, updates, { updatedAt: new Date(Date.now()).toISOString() });
|
||||
return task;
|
||||
});
|
||||
(emitter as any).moveTask = vi.fn().mockImplementation(async (_taskId: string, column: Task["column"]) => {
|
||||
task.column = column;
|
||||
task.updatedAt = new Date(Date.now()).toISOString();
|
||||
return task;
|
||||
});
|
||||
(emitter as any).handoffToReview = vi.fn().mockImplementation(async () => {
|
||||
task.column = "in-review";
|
||||
task.updatedAt = new Date(Date.now()).toISOString();
|
||||
return { ...task, autoMerge: task.autoMerge ?? true };
|
||||
});
|
||||
(emitter as any).mergeTask = vi.fn().mockResolvedValue(task);
|
||||
(emitter as any).logEntry = vi.fn().mockImplementation(async (_taskId: string, action: string, detail?: string) => {
|
||||
task.log = task.log ?? [];
|
||||
task.log.push({ timestamp: new Date(Date.now()).toISOString(), action, detail } as any);
|
||||
});
|
||||
(emitter as any).recordRunAuditEvent = vi.fn().mockImplementation(async (event: any) => {
|
||||
audits.push(event);
|
||||
});
|
||||
(emitter as any).appendAgentLog = vi.fn().mockResolvedValue(undefined);
|
||||
(emitter as any).getGoalStore = vi.fn().mockReturnValue({ listGoals: vi.fn().mockReturnValue([]) });
|
||||
(emitter as any).getFusionDir = vi.fn().mockReturnValue("/tmp/test/.fusion");
|
||||
(emitter as any).clearStaleExecutionStartBranchReferences = vi.fn().mockReturnValue([]);
|
||||
(emitter as any).listWorkflowSteps = vi.fn().mockResolvedValue([]);
|
||||
(emitter as any).getWorkflowStep = vi.fn().mockResolvedValue(undefined);
|
||||
(emitter as any).setPluginWorkflowStepTemplates = vi.fn().mockResolvedValue(undefined);
|
||||
(emitter as any).updateStep = vi.fn().mockResolvedValue(undefined);
|
||||
(emitter as any).parseStepsFromPrompt = vi.fn().mockResolvedValue([]);
|
||||
(emitter as any).parseFileScopeFromPrompt = vi.fn().mockResolvedValue([]);
|
||||
(emitter as any).getAgentLogs = vi.fn().mockResolvedValue([]);
|
||||
(emitter as any).updateSettings = vi.fn().mockResolvedValue(undefined);
|
||||
(emitter as any).emit = emitter.emit.bind(emitter);
|
||||
|
||||
return emitter;
|
||||
}
|
||||
|
||||
describe("FN-5866 reliability interactions: post-done continuation no wedge", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
resetExecutorMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("keeps completed work cleanly in-review and avoids stall deadlock after a post-done continuation error", async () => {
|
||||
const task = makeTask();
|
||||
const store = createStore(task);
|
||||
const onComplete = vi.fn();
|
||||
const onError = vi.fn();
|
||||
|
||||
mockedCreateFnAgent.mockResolvedValue({
|
||||
session: {
|
||||
prompt: vi.fn().mockImplementation(async () => {
|
||||
task.steps = [{ name: "Implement", status: "done" as const }];
|
||||
task.currentStep = 1;
|
||||
task.column = "in-review";
|
||||
task.status = undefined;
|
||||
task.error = undefined;
|
||||
throw new Error("Cannot continue from message role: assistant");
|
||||
}),
|
||||
dispose: vi.fn(),
|
||||
getSessionStats: vi.fn().mockResolvedValue({
|
||||
tokens: { input: 11, output: 7, cacheRead: 0, cacheWrite: 0, total: 18 },
|
||||
}),
|
||||
},
|
||||
} as any);
|
||||
|
||||
const executor = new TaskExecutor(store, "/tmp/test", { onComplete, onError });
|
||||
await executor.execute(task);
|
||||
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(task.status).toBeUndefined();
|
||||
expect(task.error).toBeUndefined();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect(onComplete).toHaveBeenCalled();
|
||||
expect((store.handoffToReview as any).mock.calls.length).toBe(0);
|
||||
expect((task.log ?? []).some((entry: any) => entry.action.includes("Post-done session continuation suppressed"))).toBe(true);
|
||||
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||
expect(await manager.surfaceInReviewStalls()).toBe(0);
|
||||
expect(task.paused).toBe(false);
|
||||
expect(((store as any).__audits as any[]).some((event) => event.mutationType === "task:in-review-stall-deadlock-disposed")).toBe(false);
|
||||
manager.stop();
|
||||
});
|
||||
|
||||
it("still marks incomplete work failed when the same session error happens before completion", async () => {
|
||||
const task = makeTask({ id: "FN-5866-INCOMPLETE" });
|
||||
const store = createStore(task);
|
||||
const onError = vi.fn();
|
||||
|
||||
mockedCreateFnAgent.mockResolvedValue({
|
||||
session: {
|
||||
prompt: vi.fn().mockRejectedValue(new Error("Cannot continue from message role: assistant")),
|
||||
dispose: vi.fn(),
|
||||
getSessionStats: vi.fn().mockResolvedValue({
|
||||
tokens: { input: 5, output: 0, cacheRead: 0, cacheWrite: 0, total: 5 },
|
||||
}),
|
||||
},
|
||||
} as any);
|
||||
|
||||
const executor = new TaskExecutor(store, "/tmp/test", { onError });
|
||||
await executor.execute(task);
|
||||
|
||||
expect(task.column).toBe("in-review");
|
||||
expect(task.status).toBe("failed");
|
||||
expect(task.error).toContain("Cannot continue from message role: assistant");
|
||||
expect(store.handoffToReview).toHaveBeenCalledTimes(1);
|
||||
expect(onError).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
isOperatorActionableAgentError,
|
||||
isStaleWorktreeModuleResolutionError,
|
||||
isUnsupportedMessageRoleError,
|
||||
isNonContinuableSessionError,
|
||||
TRANSIENT_ERROR_PATTERNS,
|
||||
} from "../transient-error-detector.js";
|
||||
import { isUsageLimitError } from "../usage-limit-detector.js";
|
||||
@@ -298,6 +299,32 @@ describe("Transient Error Detector", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isNonContinuableSessionError", () => {
|
||||
it("returns true for the reported assistant-role continuation error", () => {
|
||||
expect(isNonContinuableSessionError("Cannot continue from message role: assistant")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for quoted and role-variant forms", () => {
|
||||
expect(isNonContinuableSessionError("Cannot continue from message role 'assistant'"))
|
||||
.toBe(true);
|
||||
expect(isNonContinuableSessionError('Cannot continue from message role "assistant"'))
|
||||
.toBe(true);
|
||||
expect(isNonContinuableSessionError("cannot continue from message role=`tool`"))
|
||||
.toBe(true);
|
||||
expect(isNonContinuableSessionError("Cannot continue from message role user."))
|
||||
.toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for unrelated and provider role-validation errors", () => {
|
||||
expect(isNonContinuableSessionError("socket hang up")).toBe(false);
|
||||
expect(
|
||||
isNonContinuableSessionError(
|
||||
"developer is not one of ['system', 'assistant', 'user', 'tool', 'function'] - 'messages.[0].role'",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isOperatorActionableAgentError", () => {
|
||||
it("returns true for credential/model/billing errors", () => {
|
||||
expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
|
||||
|
||||
@@ -85,7 +85,7 @@ import { AgentLogger } from "./agent-logger.js";
|
||||
import { createLogger, executorLog, reviewerLog, formatError } from "./logger.js";
|
||||
import { TokenCapDetector } from "./token-cap-detector.js";
|
||||
import { isUsageLimitError, checkSessionError, type UsageLimitPauser } from "./usage-limit-detector.js";
|
||||
import { isTransientError, isSilentTransientError } from "./transient-error-detector.js";
|
||||
import { isNonContinuableSessionError, isTransientError, isSilentTransientError } from "./transient-error-detector.js";
|
||||
import { withRateLimitRetry } from "./rate-limit-retry.js";
|
||||
import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js";
|
||||
import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js";
|
||||
@@ -2561,6 +2561,43 @@ export class TaskExecutor {
|
||||
return this.isTaskWorkComplete(task);
|
||||
}
|
||||
|
||||
private isTaskAlreadyCompleteForNonContinuableSession(task: Task, taskDone: boolean): boolean {
|
||||
return taskDone || task.column === "in-review" || this.isTaskWorkComplete(task);
|
||||
}
|
||||
|
||||
private async handleNonContinuableSessionError(task: Task, taskDone: boolean, errorMessage: string): Promise<boolean> {
|
||||
if (!isNonContinuableSessionError(errorMessage)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const liveTask = await this.store.getTask(task.id);
|
||||
if (!liveTask || !this.isTaskAlreadyCompleteForNonContinuableSession(liveTask, taskDone)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const diagnosticMessage = "Post-done session continuation suppressed — session not continuable (last role assistant); task work already complete, leaving clean in-review";
|
||||
executorLog.warn(`${task.id} ${diagnosticMessage}`);
|
||||
await this.store.logEntry(task.id, diagnosticMessage, errorMessage, this.getRunContextFor(task.id));
|
||||
|
||||
if (liveTask.status === "failed" || liveTask.error) {
|
||||
await this.store.updateTask(task.id, { status: undefined, error: undefined });
|
||||
}
|
||||
|
||||
await this.persistTokenUsage(task.id);
|
||||
|
||||
if (liveTask.column === "in-review") {
|
||||
this.clearCompletedTaskWatchdog(task.id);
|
||||
this.options.onComplete?.(liveTask);
|
||||
return true;
|
||||
}
|
||||
|
||||
const refreshedTask = await this.store.getTask(task.id);
|
||||
await this.handoffTaskToReview(refreshedTask ?? liveTask, "post-done-noncontinuable");
|
||||
this.clearCompletedTaskWatchdog(task.id);
|
||||
this.options.onComplete?.(refreshedTask ?? liveTask);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async getTaskCompletionBlocker(task: Task): Promise<string | undefined> {
|
||||
return getTaskCompletionBlockerForStore(this.store, task);
|
||||
}
|
||||
@@ -5407,6 +5444,8 @@ export class TaskExecutor {
|
||||
return;
|
||||
}
|
||||
return;
|
||||
} else if (await this.handleNonContinuableSessionError(task, taskDone, errorMessage)) {
|
||||
return;
|
||||
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
|
||||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
|
||||
} else if (isTransientError(errorMessage)) {
|
||||
|
||||
@@ -192,6 +192,7 @@ export function extractMissingModulePath(errorMessage: string): string | null {
|
||||
}
|
||||
|
||||
const UNSUPPORTED_MESSAGE_ROLE_PATTERN = /\bmessages\.\[\d+\]\.role\b[\s\S]*\bis not one of\b|\bis not one of\b[\s\S]*\bmessages\.\[\d+\]\.role\b/i;
|
||||
const NON_CONTINUABLE_SESSION_PATTERN = /cannot continue from message role\s*[:=-]?\s*(?:['"`]?)(assistant|tool|function|system|user)(?:['"`]?)\b/i;
|
||||
|
||||
export function isUnsupportedMessageRoleError(errorMessage: string): boolean {
|
||||
if (!errorMessage || typeof errorMessage !== "string") {
|
||||
@@ -200,6 +201,13 @@ export function isUnsupportedMessageRoleError(errorMessage: string): boolean {
|
||||
return UNSUPPORTED_MESSAGE_ROLE_PATTERN.test(errorMessage);
|
||||
}
|
||||
|
||||
export function isNonContinuableSessionError(errorMessage: string): boolean {
|
||||
if (!errorMessage || typeof errorMessage !== "string") {
|
||||
return false;
|
||||
}
|
||||
return NON_CONTINUABLE_SESSION_PATTERN.test(errorMessage);
|
||||
}
|
||||
|
||||
const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [
|
||||
/invalid api key/i,
|
||||
/authentication failed/i,
|
||||
|
||||
Reference in New Issue
Block a user