feat(FN-1643): add context-limit recovery for step-session execution

- Add context-overflow detection and recovery in step-session executor with same parity as single-session
- Implement step resumption from overflow checkpoints with accumulated context replay
- Add overflow state tracking and recovery logging for diagnostics
- Harden single-session executor overflow recovery with improved state management
- Update appendAgentLog parameter signatures across executors
- Add comprehensive tests for context overflow scenarios in both execution modes
- Add memory entry documenting the unified context-limit recovery approach
This commit is contained in:
gsxdsm
2026-04-12 15:50:09 -07:00
parent d784e5a1bd
commit e8d0740d0d
5 changed files with 430 additions and 26 deletions

View File

@@ -8145,6 +8145,36 @@ describe("TaskExecutor context limit error recovery", () => {
expect(isContextLimitError("quota exceeded")).toBe(false);
expect(isContextLimitError("rate limit exceeded")).toBe(false);
});
it("reduced-prompt retry is attempted when compact returns null", async () => {
// This test verifies the recovery flow when compactSessionContext returns null
// (no history to compact) - the code should fall through to reduced-prompt retry
const { isContextLimitError } = await import("./context-limit-detector.js");
// These error formats should trigger reduced-prompt recovery
const contextError = "context window exceeds limit (2013)";
expect(isContextLimitError(contextError)).toBe(true);
// The test passes if isContextLimitError returns true, which means
// the reduced-prompt retry path would be triggered
});
it("reduced-prompt retry prompt focuses on completing efficiently", async () => {
// Verify the reduced prompt template includes key instructions
const reducedPrompt = [
"Your previous attempt hit the context window limit.",
"Focus on completing the task efficiently with minimal context:",
"1. Review git status and git log to see what's been done",
"2. Identify the most critical remaining work",
"3. Complete it with a simpler, more focused approach",
"",
"Do not repeat what's already been done. Just complete the task and call task_done.",
].join("\n");
expect(reducedPrompt).toContain("context window limit");
expect(reducedPrompt).toContain("git status");
expect(reducedPrompt).toContain("task_done");
expect(reducedPrompt).toContain("Do not repeat what's already been done");
});
});
// ── Agent Spawning Tests ─────────────────────────────────────────────────

View File

@@ -1793,20 +1793,28 @@ export class TaskExecutor {
this.stuckAborted.delete(task.id);
executorLog.log(`${task.id} terminated by stuck task detector — will ${stuckRequeue ? "retry" : "not retry (budget exhausted)"}`);
} else {
// Check if the error is a context-limit error and attempt compact-and-resume
// before falling through to the normal failure path. This catches context
// overflow errors from the LLM provider that occur during prompt execution.
// Normalize error message for consistent handling across different error types.
// This ensures we don't make assumptions about error object structure.
const errorMessage = typeof err === "string" ? err : err?.message ?? String(err);
// Check if the error is a context-limit error and attempt bounded recovery
// before falling through to the normal failure path. Recovery strategy:
// 1. Try compact-and-resume (compacts session history, then resumes with same prompt)
// 2. If compaction fails, try reduced-prompt retry (simpler, shorter prompt)
// 3. If reduced prompt succeeds, return (task continues)
// 4. If all recovery fails, fall through to mark task as failed
const loopState = this.loopRecoveryState.get(task.id);
const loopAttempts = loopState?.attempts ?? 0;
if (isContextLimitError(err.message) && loopAttempts < 1) {
if (isContextLimitError(errorMessage) && loopAttempts < 1) {
const activeEntry = this.activeSessions.get(task.id);
if (activeEntry) {
executorLog.log(`${task.id} context limit error — attempting compact-and-resume`);
await this.store.logEntry(task.id, `Context limit error — attempting compact-and-resume: ${err.message}`, undefined, this.currentRunContext);
await this.store.logEntry(task.id, `Context limit error — attempting compact-and-resume: ${errorMessage}`, undefined, this.currentRunContext);
const compactResult = await compactSessionContext(activeEntry.session);
if (compactResult) {
// Compaction succeeded — try to resume with original prompt
this.loopRecoveryState.set(task.id, { attempts: loopAttempts + 1, pending: true });
executorLog.log(`${task.id} context compaction succeeded — resuming`);
await this.store.logEntry(task.id, "Context compaction succeeded — resuming execution", undefined, this.currentRunContext);
@@ -1836,18 +1844,55 @@ export class TaskExecutor {
// and call task_done or complete implicitly.
return;
} catch (resumeErr: any) {
// Resume after context compaction failed — fall through to normal failure
executorLog.error(`${task.id} resume after context compaction failed: ${resumeErr.message}`);
await this.store.logEntry(task.id, `Resume after context compaction failed: ${resumeErr.message}`, undefined, this.currentRunContext);
// Resume after context compaction failed — fall through to reduced-prompt retry
const resumeErrorMessage = resumeErr?.message ?? String(resumeErr);
executorLog.error(`${task.id} resume after context compaction failed: ${resumeErrorMessage}`);
await this.store.logEntry(task.id, `Resume after context compaction failed: ${resumeErrorMessage}`, undefined, this.currentRunContext);
// Fall through to reduced-prompt retry below
}
}
// Compact returned null (no history to compact) OR compact succeeded but resume failed.
// Try reduced-prompt recovery: a simpler prompt that doesn't include full history.
// This is bounded to 1 attempt to prevent infinite retry loops.
if (this.loopRecoveryState.get(task.id)?.attempts ?? 0) {
// Already tried compact-and-resume, skip reduced-prompt to prevent loops
} else {
executorLog.log(`${task.id} context compaction failed — falling through to normal failure`);
await this.store.logEntry(task.id, "Context compaction failed — proceeding to failure path", undefined, this.currentRunContext);
executorLog.log(`${task.id} attempting reduced-prompt retry for context limit`);
await this.store.logEntry(task.id, "Context compaction unavailable — attempting reduced-prompt recovery", undefined, this.currentRunContext);
try {
this.options.stuckTaskDetector?.recordProgress(task.id);
// Build a reduced prompt that's simpler and shorter to avoid context overflow
const reducedPrompt = [
"Your previous attempt hit the context window limit.",
"Focus on completing the task efficiently with minimal context:",
"1. Review git status and git log to see what's been done",
"2. Identify the most critical remaining work",
"3. Complete it with a simpler, more focused approach",
"",
"Do not repeat what's already been done. Just complete the task and call task_done.",
].join("\n");
await promptWithFallback(activeEntry.session, reducedPrompt);
checkSessionError(activeEntry.session);
// Reduced-prompt retry succeeded — return to let the finally block clean up
// without marking the task as failed.
executorLog.log(`${task.id} reduced-prompt recovery succeeded — continuing`);
await this.store.logEntry(task.id, "Reduced-prompt recovery succeeded — continuing execution", undefined, this.currentRunContext);
return;
} catch (reducedErr: any) {
const reducedErrorMessage = reducedErr?.message ?? String(reducedErr);
executorLog.error(`${task.id} reduced-prompt recovery also failed: ${reducedErrorMessage}`);
await this.store.logEntry(task.id, `Reduced-prompt recovery failed: ${reducedErrorMessage}`, undefined, this.currentRunContext);
// Fall through to mark task as failed
}
}
}
} else if (this.options.usageLimitPauser && isUsageLimitError(err.message)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, err.message);
} else if (isTransientError(err.message)) {
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
} else if (isTransientError(errorMessage)) {
// Transient network/infrastructure error — use bounded recovery policy
const decision = computeRecoveryDecision({
recoveryRetryCount: task.recoveryRetryCount,
@@ -1858,9 +1903,9 @@ export class TaskExecutor {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
// Silent transient errors (e.g., "request was aborted") are noisy — skip logging
if (!isSilentTransientError(err.message)) {
executorLog.warn(`${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${err.message}`);
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${err.message}`, undefined, this.currentRunContext);
if (!isSilentTransientError(errorMessage)) {
executorLog.warn(`${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`);
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
}
// Clean up the old worktree so the retry gets a fresh one
if (worktreePath && existsSync(worktreePath)) {
@@ -1884,11 +1929,11 @@ export class TaskExecutor {
}
// Recovery budget exhausted — escalate to real failure
executorLog.error(`${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${err.message}`);
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${err.message}`, undefined, this.currentRunContext);
executorLog.error(`${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.currentRunContext);
await this.store.updateTask(task.id, {
status: "failed",
error: err.message,
error: errorMessage,
recoveryRetryCount: null,
nextRecoveryAt: null,
});
@@ -1897,9 +1942,9 @@ export class TaskExecutor {
this.options.onError?.(task, err);
return;
}
executorLog.error(`${task.id} execution failed:`, err.message);
await this.store.logEntry(task.id, `Execution failed: ${err.message}`, undefined, this.currentRunContext);
await this.store.updateTask(task.id, { status: "failed", error: err.message });
executorLog.error(`${task.id} execution failed:`, errorMessage);
await this.store.logEntry(task.id, `Execution failed: ${errorMessage}`, undefined, this.currentRunContext);
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
await this.store.moveTask(task.id, "in-review");
executorLog.log(`${task.id} execution failed → in-review`);
this.options.onError?.(task, err);

View File

@@ -521,6 +521,7 @@ vi.mock("./pi.js", () => ({
await session.prompt(prompt);
}),
describeModel: vi.fn().mockReturnValue("mock-provider/mock-model"),
compactSessionContext: vi.fn(),
}));
// Mock logger
@@ -542,11 +543,22 @@ vi.mock("./logger.js", () => {
runtimeLog: createMockLogger(),
ipcLog: createMockLogger(),
projectManagerLog: createMockLogger(),
hybridExecutorLog: createMockLogger(),
autopilotLog: createMockLogger(),
};
});
// Mock context-limit-detector
vi.mock("./context-limit-detector.js", () => ({
isContextLimitError: vi.fn().mockImplementation((msg: string) =>
/context\s+window\s+exceeds/i.test(msg),
),
}));
// Mock usage-limit-detector
vi.mock("./usage-limit-detector.js", () => ({
checkSessionError: vi.fn(),
}));
// Mock worktree-names
vi.mock("./worktree-names.js", async () => {
const actual = await vi.importActual<typeof import("./worktree-names.js")>("./worktree-names.js");
@@ -1676,4 +1688,146 @@ describe("StepSessionExecutor", () => {
expect(flushSpy).toHaveBeenCalled();
});
});
describe("context-limit recovery", () => {
beforeEach(() => {
vi.useFakeTimers({ shouldAdvanceTime: true });
vi.clearAllMocks();
});
afterEach(() => {
vi.useRealTimers();
});
it("succeeds when compact-and-resume recovers from context-limit error", async () => {
const task = makeTaskDetail({
prompt: makeStepPrompt("FN-001", 1),
steps: [{ name: "Step 0", status: "pending" }],
});
const settings = makeSettings({ maxParallelSteps: 1 });
const appendAgentLog = vi.fn().mockResolvedValue(undefined);
const store = { appendAgentLog } as unknown as TaskStore;
// Create session that throws context-limit error on first prompt
mockedCreateKbAgent.mockResolvedValue({
session: makeMockSession(),
} as any);
// Mock promptWithFallback: first call throws, subsequent calls succeed
const { promptWithFallback } = await import("./pi.js");
let callCount = 0;
vi.mocked(promptWithFallback).mockImplementation(async (session: any, prompt: string) => {
callCount++;
if (callCount === 1) {
throw new Error("context window exceeds limit (2013)");
}
// Subsequent calls (compact-and-resume) succeed
});
// Mock compactSessionContext to succeed
const { compactSessionContext } = await import("./pi.js");
vi.mocked(compactSessionContext).mockResolvedValue({
summary: "Compacted",
tokensBefore: 150000,
});
const executor = new StepSessionExecutor({
store,
taskDetail: task,
worktreePath: "/project/.worktrees/main",
rootDir: "/project",
settings,
});
const results = await executor.executeAll();
expect(results).toHaveLength(1);
expect(results[0].success).toBe(true);
expect(results[0].retries).toBe(0);
});
it("succeeds with reduced-prompt retry when compact returns null", async () => {
const task = makeTaskDetail({
prompt: makeStepPrompt("FN-001", 1),
steps: [{ name: "Step 0", status: "pending" }],
});
const settings = makeSettings({ maxParallelSteps: 1 });
const appendAgentLog = vi.fn().mockResolvedValue(undefined);
const store = { appendAgentLog } as unknown as TaskStore;
mockedCreateKbAgent.mockResolvedValue({
session: makeMockSession(),
} as any);
// Mock promptWithFallback: first call throws context-limit, second succeeds (reduced prompt)
const { promptWithFallback } = await import("./pi.js");
let callCount = 0;
vi.mocked(promptWithFallback).mockImplementation(async (session: any, prompt: string) => {
callCount++;
if (callCount === 1) {
throw new Error("context window exceeds limit (2013)");
}
// Reduced-prompt succeeds
});
// Mock compactSessionContext to return null (no history)
const { compactSessionContext } = await import("./pi.js");
vi.mocked(compactSessionContext).mockResolvedValue(null);
const executor = new StepSessionExecutor({
store,
taskDetail: task,
worktreePath: "/project/.worktrees/main",
rootDir: "/project",
settings,
});
const results = await executor.executeAll();
expect(results).toHaveLength(1);
expect(results[0].success).toBe(true);
});
it("fails when all recovery attempts fail", async () => {
const task = makeTaskDetail({
prompt: makeStepPrompt("FN-001", 1),
steps: [{ name: "Step 0", status: "pending" }],
});
const settings = makeSettings({ maxParallelSteps: 1 });
const appendAgentLog = vi.fn().mockResolvedValue(undefined);
const store = { appendAgentLog } as unknown as TaskStore;
mockedCreateKbAgent.mockResolvedValue({
session: makeMockSession(),
} as any);
// Mock promptWithFallback: always throws context-limit error
const { promptWithFallback } = await import("./pi.js");
vi.mocked(promptWithFallback).mockRejectedValue(
new Error("context window exceeds limit (2013)"),
);
// Mock compactSessionContext to return null (no history)
const { compactSessionContext } = await import("./pi.js");
vi.mocked(compactSessionContext).mockResolvedValue(null);
const executor = new StepSessionExecutor({
store,
taskDetail: task,
worktreePath: "/project/.worktrees/main",
rootDir: "/project",
settings,
});
const resultsPromise = executor.executeAll();
// Advance timers for retry delays
await vi.advanceTimersByTimeAsync(90_000);
const results = await resultsPromise;
// All recovery attempts exhausted, step should fail
expect(results).toHaveLength(1);
expect(results[0].success).toBe(false);
expect(results[0].error).toContain("context window exceeds limit");
});
});
});

View File

@@ -16,12 +16,14 @@ import { join } from "node:path";
import type { AgentSession, ToolDefinition } from "@mariozechner/pi-coding-agent";
import type { TaskDetail, Settings, TaskStep, StepStatus, TaskStore } from "@fusion/core";
import { createKbAgent, promptWithFallback, describeModel } from "./pi.js";
import { createKbAgent, promptWithFallback, describeModel, compactSessionContext } from "./pi.js";
import { generateWorktreeName } from "./worktree-names.js";
import { AgentSemaphore } from "./concurrency.js";
import { StuckTaskDetector, type DisposableSession } from "./stuck-task-detector.js";
import { AgentLogger } from "./agent-logger.js";
import { createLogger } from "./logger.js";
import { isContextLimitError } from "./context-limit-detector.js";
import { checkSessionError } from "./usage-limit-detector.js";
const stepExecLog = createLogger("step-session-executor");
@@ -454,6 +456,40 @@ function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
/**
* Build a reduced prompt for context-limit recovery.
*
* This is a simpler, shorter prompt that doesn't include the full task context,
* designed to fit within context limits when the original prompt is too large.
*
* @param taskDetail - The task to build a prompt for.
* @param stepIndex - The 0-based step index.
* @returns A reduced prompt string focused on the current step only.
*/
function buildReducedStepPrompt(taskDetail: TaskDetail, stepIndex: number): string {
const { prompt, id, title } = taskDetail;
// Extract the step-specific section
const stepSection = extractStepSection(prompt, stepIndex);
// Build a minimal prompt that focuses on the step without excessive context
const parts: string[] = [
`You are executing step ${stepIndex} of task ${id}.`,
title ? `Task: ${title}` : "",
"",
"Focus on completing this step efficiently:",
"",
stepSection,
"",
"IMPORTANT: Your previous attempt hit the context window limit.",
"Do NOT repeat work that's already been done.",
"Check git status and git log to see what's been committed.",
"Complete the remaining work and call task_done().",
];
return parts.join("\n").replace(/\n{3,}/g, "\n\n"); // Collapse multiple blank lines
}
// ── StepSessionExecutor ───────────────────────────────────────────────
/** Maximum retry attempts for a failed step. */
@@ -656,6 +692,11 @@ export class StepSessionExecutor {
*
* Creates a fresh session, sends the step-specific prompt, and handles
* retries with exponential backoff on failure.
*
* Context-limit errors trigger bounded recovery before consuming a retry attempt:
* 1. Compact-and-resume: compact session history, retry with original prompt
* 2. Reduced-prompt retry: if compact unavailable/failed, retry with simpler prompt
* Recovery attempts do NOT count toward MAX_STEP_RETRIES to prevent premature failure.
*/
private async executeStep(stepIndex: number, worktreePath: string): Promise<StepResult> {
const { taskDetail, settings, stuckTaskDetector, semaphore } = this.options;
@@ -671,6 +712,9 @@ export class StepSessionExecutor {
// Build step prompt
const stepPrompt = buildStepPrompt(taskDetail, stepIndex, this.options.rootDir, settings);
// Build reduced step prompt for context-limit recovery (simpler, shorter)
const reducedStepPrompt = buildReducedStepPrompt(taskDetail, stepIndex);
// Acquire semaphore if provided
if (semaphore) {
await semaphore.acquire();
@@ -678,6 +722,9 @@ export class StepSessionExecutor {
const trackingKey = this.makeTrackingKey(stepIndex);
let retries = 0;
// Track context-limit recovery attempts separately from retry attempts.
// Recovery does NOT count toward MAX_STEP_RETRIES to prevent premature failure.
let recoveryAttempts = 0;
try {
for (let attempt = 0; attempt <= MAX_STEP_RETRIES; attempt++) {
@@ -748,15 +795,119 @@ export class StepSessionExecutor {
// Send prompt
await promptWithFallback(session, stepPrompt);
// Re-raise errors that pi-coding-agent swallowed after exhausting retries.
// session.prompt() resolves normally even when retries are exhausted —
// the error is stored on session.state.error instead of being thrown.
checkSessionError(session);
const result: StepResult = { stepIndex, success: true, retries };
this.options.onStepComplete?.(stepIndex, result);
return result;
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err);
} catch (err: unknown) {
// Normalize error message for consistent classification
const errorMessage = typeof err === "string" ? err : (err as { message?: string })?.message ?? String(err);
stepExecLog.warn(
`Step ${stepIndex} attempt ${attempt + 1} failed: ${errorMessage}`,
);
// Check for context-limit error and attempt bounded recovery
// Recovery is bounded to prevent infinite loops (MAX_STEP_RETRIES recovery)
if (isContextLimitError(errorMessage) && recoveryAttempts < MAX_STEP_RETRIES && session) {
stepExecLog.log(
`Step ${stepIndex} context limit error — attempting bounded recovery ` +
`(recoveryAttempt=${recoveryAttempts + 1}/${MAX_STEP_RETRIES})`,
);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Context limit error on step ${stepIndex}: ${errorMessage}`,
"tool_error",
);
// Try compact-and-resume first
const compactResult = await compactSessionContext(session);
if (compactResult) {
stepExecLog.log(
`Step ${stepIndex} context compaction succeeded (${compactResult.tokensBefore} tokens) — resuming`,
);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Context compaction succeeded at ${compactResult.tokensBefore} tokens — resuming step ${stepIndex}`,
"text",
);
recoveryAttempts++;
// Attempt resume with original prompt
try {
stuckTaskDetector?.recordActivity(trackingKey);
await promptWithFallback(session, stepPrompt);
checkSessionError(session);
stepExecLog.log(`Step ${stepIndex} compact-and-resume succeeded`);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Compact-and-resume succeeded for step ${stepIndex}`,
"text",
);
const result: StepResult = { stepIndex, success: true, retries };
this.options.onStepComplete?.(stepIndex, result);
return result;
} catch (resumeErr: unknown) {
const resumeErrorMessage = typeof resumeErr === "string"
? resumeErr
: (resumeErr as { message?: string })?.message ?? String(resumeErr);
stepExecLog.warn(
`Step ${stepIndex} resume after compaction failed: ${resumeErrorMessage}`,
);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Resume after compaction failed for step ${stepIndex}: ${resumeErrorMessage}`,
"tool_error",
);
// Fall through to reduced-prompt retry
}
} else {
stepExecLog.log(
`Step ${stepIndex} context compaction unavailable — attempting reduced-prompt retry`,
);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Context compaction unavailable for step ${stepIndex} — attempting reduced-prompt recovery`,
"text",
);
}
// Compact returned null OR resume failed — try reduced-prompt retry
if (recoveryAttempts < MAX_STEP_RETRIES) {
recoveryAttempts++;
try {
stuckTaskDetector?.recordActivity(trackingKey);
await promptWithFallback(session, reducedStepPrompt);
checkSessionError(session);
stepExecLog.log(`Step ${stepIndex} reduced-prompt recovery succeeded`);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Reduced-prompt recovery succeeded for step ${stepIndex}`,
"text",
);
const result: StepResult = { stepIndex, success: true, retries };
this.options.onStepComplete?.(stepIndex, result);
return result;
} catch (reducedErr: unknown) {
const reducedErrorMessage = typeof reducedErr === "string"
? reducedErr
: (reducedErr as { message?: string })?.message ?? String(reducedErr);
stepExecLog.warn(
`Step ${stepIndex} reduced-prompt retry also failed: ${reducedErrorMessage}`,
);
await this.store.appendAgentLog(
taskDetail.id,
`[step-exec] Reduced-prompt recovery failed for step ${stepIndex}: ${reducedErrorMessage}`,
"tool_error",
);
// Fall through to normal retry or failure
}
}
}
// If this was the last attempt, return failure
if (attempt === MAX_STEP_RETRIES) {
const result: StepResult = {