feat(FN-1643): add context-limit recovery for step-session execution
- Add context-overflow detection and recovery in step-session executor with same parity as single-session - Implement step resumption from overflow checkpoints with accumulated context replay - Add overflow state tracking and recovery logging for diagnostics - Harden single-session executor overflow recovery with improved state management - Update appendAgentLog parameter signatures across executors - Add comprehensive tests for context overflow scenarios in both execution modes - Add memory entry documenting the unified context-limit recovery approach
This commit is contained in:
@@ -8145,6 +8145,36 @@ describe("TaskExecutor context limit error recovery", () => {
|
||||
expect(isContextLimitError("quota exceeded")).toBe(false);
|
||||
expect(isContextLimitError("rate limit exceeded")).toBe(false);
|
||||
});
|
||||
|
||||
it("reduced-prompt retry is attempted when compact returns null", async () => {
|
||||
// This test verifies the recovery flow when compactSessionContext returns null
|
||||
// (no history to compact) - the code should fall through to reduced-prompt retry
|
||||
const { isContextLimitError } = await import("./context-limit-detector.js");
|
||||
|
||||
// These error formats should trigger reduced-prompt recovery
|
||||
const contextError = "context window exceeds limit (2013)";
|
||||
expect(isContextLimitError(contextError)).toBe(true);
|
||||
// The test passes if isContextLimitError returns true, which means
|
||||
// the reduced-prompt retry path would be triggered
|
||||
});
|
||||
|
||||
it("reduced-prompt retry prompt focuses on completing efficiently", async () => {
|
||||
// Verify the reduced prompt template includes key instructions
|
||||
const reducedPrompt = [
|
||||
"Your previous attempt hit the context window limit.",
|
||||
"Focus on completing the task efficiently with minimal context:",
|
||||
"1. Review git status and git log to see what's been done",
|
||||
"2. Identify the most critical remaining work",
|
||||
"3. Complete it with a simpler, more focused approach",
|
||||
"",
|
||||
"Do not repeat what's already been done. Just complete the task and call task_done.",
|
||||
].join("\n");
|
||||
|
||||
expect(reducedPrompt).toContain("context window limit");
|
||||
expect(reducedPrompt).toContain("git status");
|
||||
expect(reducedPrompt).toContain("task_done");
|
||||
expect(reducedPrompt).toContain("Do not repeat what's already been done");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Agent Spawning Tests ─────────────────────────────────────────────────
|
||||
|
||||
@@ -1793,20 +1793,28 @@ export class TaskExecutor {
|
||||
this.stuckAborted.delete(task.id);
|
||||
executorLog.log(`${task.id} terminated by stuck task detector — will ${stuckRequeue ? "retry" : "not retry (budget exhausted)"}`);
|
||||
} else {
|
||||
// Check if the error is a context-limit error and attempt compact-and-resume
|
||||
// before falling through to the normal failure path. This catches context
|
||||
// overflow errors from the LLM provider that occur during prompt execution.
|
||||
// Normalize error message for consistent handling across different error types.
|
||||
// This ensures we don't make assumptions about error object structure.
|
||||
const errorMessage = typeof err === "string" ? err : err?.message ?? String(err);
|
||||
|
||||
// Check if the error is a context-limit error and attempt bounded recovery
|
||||
// before falling through to the normal failure path. Recovery strategy:
|
||||
// 1. Try compact-and-resume (compacts session history, then resumes with same prompt)
|
||||
// 2. If compaction fails, try reduced-prompt retry (simpler, shorter prompt)
|
||||
// 3. If reduced prompt succeeds, return (task continues)
|
||||
// 4. If all recovery fails, fall through to mark task as failed
|
||||
const loopState = this.loopRecoveryState.get(task.id);
|
||||
const loopAttempts = loopState?.attempts ?? 0;
|
||||
|
||||
if (isContextLimitError(err.message) && loopAttempts < 1) {
|
||||
if (isContextLimitError(errorMessage) && loopAttempts < 1) {
|
||||
const activeEntry = this.activeSessions.get(task.id);
|
||||
if (activeEntry) {
|
||||
executorLog.log(`${task.id} context limit error — attempting compact-and-resume`);
|
||||
await this.store.logEntry(task.id, `Context limit error — attempting compact-and-resume: ${err.message}`, undefined, this.currentRunContext);
|
||||
await this.store.logEntry(task.id, `Context limit error — attempting compact-and-resume: ${errorMessage}`, undefined, this.currentRunContext);
|
||||
|
||||
const compactResult = await compactSessionContext(activeEntry.session);
|
||||
if (compactResult) {
|
||||
// Compaction succeeded — try to resume with original prompt
|
||||
this.loopRecoveryState.set(task.id, { attempts: loopAttempts + 1, pending: true });
|
||||
executorLog.log(`${task.id} context compaction succeeded — resuming`);
|
||||
await this.store.logEntry(task.id, "Context compaction succeeded — resuming execution", undefined, this.currentRunContext);
|
||||
@@ -1836,18 +1844,55 @@ export class TaskExecutor {
|
||||
// and call task_done or complete implicitly.
|
||||
return;
|
||||
} catch (resumeErr: any) {
|
||||
// Resume after context compaction failed — fall through to normal failure
|
||||
executorLog.error(`${task.id} resume after context compaction failed: ${resumeErr.message}`);
|
||||
await this.store.logEntry(task.id, `Resume after context compaction failed: ${resumeErr.message}`, undefined, this.currentRunContext);
|
||||
// Resume after context compaction failed — fall through to reduced-prompt retry
|
||||
const resumeErrorMessage = resumeErr?.message ?? String(resumeErr);
|
||||
executorLog.error(`${task.id} resume after context compaction failed: ${resumeErrorMessage}`);
|
||||
await this.store.logEntry(task.id, `Resume after context compaction failed: ${resumeErrorMessage}`, undefined, this.currentRunContext);
|
||||
// Fall through to reduced-prompt retry below
|
||||
}
|
||||
}
|
||||
|
||||
// Compact returned null (no history to compact) OR compact succeeded but resume failed.
|
||||
// Try reduced-prompt recovery: a simpler prompt that doesn't include full history.
|
||||
// This is bounded to 1 attempt to prevent infinite retry loops.
|
||||
if (this.loopRecoveryState.get(task.id)?.attempts ?? 0) {
|
||||
// Already tried compact-and-resume, skip reduced-prompt to prevent loops
|
||||
} else {
|
||||
executorLog.log(`${task.id} context compaction failed — falling through to normal failure`);
|
||||
await this.store.logEntry(task.id, "Context compaction failed — proceeding to failure path", undefined, this.currentRunContext);
|
||||
executorLog.log(`${task.id} attempting reduced-prompt retry for context limit`);
|
||||
await this.store.logEntry(task.id, "Context compaction unavailable — attempting reduced-prompt recovery", undefined, this.currentRunContext);
|
||||
|
||||
try {
|
||||
this.options.stuckTaskDetector?.recordProgress(task.id);
|
||||
// Build a reduced prompt that's simpler and shorter to avoid context overflow
|
||||
const reducedPrompt = [
|
||||
"Your previous attempt hit the context window limit.",
|
||||
"Focus on completing the task efficiently with minimal context:",
|
||||
"1. Review git status and git log to see what's been done",
|
||||
"2. Identify the most critical remaining work",
|
||||
"3. Complete it with a simpler, more focused approach",
|
||||
"",
|
||||
"Do not repeat what's already been done. Just complete the task and call task_done.",
|
||||
].join("\n");
|
||||
|
||||
await promptWithFallback(activeEntry.session, reducedPrompt);
|
||||
checkSessionError(activeEntry.session);
|
||||
|
||||
// Reduced-prompt retry succeeded — return to let the finally block clean up
|
||||
// without marking the task as failed.
|
||||
executorLog.log(`${task.id} reduced-prompt recovery succeeded — continuing`);
|
||||
await this.store.logEntry(task.id, "Reduced-prompt recovery succeeded — continuing execution", undefined, this.currentRunContext);
|
||||
return;
|
||||
} catch (reducedErr: any) {
|
||||
const reducedErrorMessage = reducedErr?.message ?? String(reducedErr);
|
||||
executorLog.error(`${task.id} reduced-prompt recovery also failed: ${reducedErrorMessage}`);
|
||||
await this.store.logEntry(task.id, `Reduced-prompt recovery failed: ${reducedErrorMessage}`, undefined, this.currentRunContext);
|
||||
// Fall through to mark task as failed
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (this.options.usageLimitPauser && isUsageLimitError(err.message)) {
|
||||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, err.message);
|
||||
} else if (isTransientError(err.message)) {
|
||||
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
|
||||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
|
||||
} else if (isTransientError(errorMessage)) {
|
||||
// Transient network/infrastructure error — use bounded recovery policy
|
||||
const decision = computeRecoveryDecision({
|
||||
recoveryRetryCount: task.recoveryRetryCount,
|
||||
@@ -1858,9 +1903,9 @@ export class TaskExecutor {
|
||||
const attempt = decision.nextState.recoveryRetryCount;
|
||||
const delay = formatDelay(decision.delayMs);
|
||||
// Silent transient errors (e.g., "request was aborted") are noisy — skip logging
|
||||
if (!isSilentTransientError(err.message)) {
|
||||
executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${err.message}`);
|
||||
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${err.message}`, undefined, this.currentRunContext);
|
||||
if (!isSilentTransientError(errorMessage)) {
|
||||
executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`);
|
||||
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
|
||||
}
|
||||
// Clean up the old worktree so the retry gets a fresh one
|
||||
if (worktreePath && existsSync(worktreePath)) {
|
||||
@@ -1884,11 +1929,11 @@ export class TaskExecutor {
|
||||
}
|
||||
|
||||
// Recovery budget exhausted — escalate to real failure
|
||||
executorLog.error(`✗ ${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${err.message}`);
|
||||
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${err.message}`, undefined, this.currentRunContext);
|
||||
executorLog.error(`✗ ${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
|
||||
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.currentRunContext);
|
||||
await this.store.updateTask(task.id, {
|
||||
status: "failed",
|
||||
error: err.message,
|
||||
error: errorMessage,
|
||||
recoveryRetryCount: null,
|
||||
nextRecoveryAt: null,
|
||||
});
|
||||
@@ -1897,9 +1942,9 @@ export class TaskExecutor {
|
||||
this.options.onError?.(task, err);
|
||||
return;
|
||||
}
|
||||
executorLog.error(`✗ ${task.id} execution failed:`, err.message);
|
||||
await this.store.logEntry(task.id, `Execution failed: ${err.message}`, undefined, this.currentRunContext);
|
||||
await this.store.updateTask(task.id, { status: "failed", error: err.message });
|
||||
executorLog.error(`✗ ${task.id} execution failed:`, errorMessage);
|
||||
await this.store.logEntry(task.id, `Execution failed: ${errorMessage}`, undefined, this.currentRunContext);
|
||||
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
|
||||
await this.store.moveTask(task.id, "in-review");
|
||||
executorLog.log(`✗ ${task.id} execution failed → in-review`);
|
||||
this.options.onError?.(task, err);
|
||||
|
||||
@@ -521,6 +521,7 @@ vi.mock("./pi.js", () => ({
|
||||
await session.prompt(prompt);
|
||||
}),
|
||||
describeModel: vi.fn().mockReturnValue("mock-provider/mock-model"),
|
||||
compactSessionContext: vi.fn(),
|
||||
}));
|
||||
|
||||
// Mock logger
|
||||
@@ -542,11 +543,22 @@ vi.mock("./logger.js", () => {
|
||||
runtimeLog: createMockLogger(),
|
||||
ipcLog: createMockLogger(),
|
||||
projectManagerLog: createMockLogger(),
|
||||
hybridExecutorLog: createMockLogger(),
|
||||
autopilotLog: createMockLogger(),
|
||||
};
|
||||
});
|
||||
|
||||
// Mock context-limit-detector
|
||||
vi.mock("./context-limit-detector.js", () => ({
|
||||
isContextLimitError: vi.fn().mockImplementation((msg: string) =>
|
||||
/context\s+window\s+exceeds/i.test(msg),
|
||||
),
|
||||
}));
|
||||
|
||||
// Mock usage-limit-detector
|
||||
vi.mock("./usage-limit-detector.js", () => ({
|
||||
checkSessionError: vi.fn(),
|
||||
}));
|
||||
|
||||
// Mock worktree-names
|
||||
vi.mock("./worktree-names.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("./worktree-names.js")>("./worktree-names.js");
|
||||
@@ -1676,4 +1688,146 @@ describe("StepSessionExecutor", () => {
|
||||
expect(flushSpy).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("context-limit recovery", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("succeeds when compact-and-resume recovers from context-limit error", async () => {
|
||||
const task = makeTaskDetail({
|
||||
prompt: makeStepPrompt("FN-001", 1),
|
||||
steps: [{ name: "Step 0", status: "pending" }],
|
||||
});
|
||||
const settings = makeSettings({ maxParallelSteps: 1 });
|
||||
const appendAgentLog = vi.fn().mockResolvedValue(undefined);
|
||||
const store = { appendAgentLog } as unknown as TaskStore;
|
||||
|
||||
// Create session that throws context-limit error on first prompt
|
||||
mockedCreateKbAgent.mockResolvedValue({
|
||||
session: makeMockSession(),
|
||||
} as any);
|
||||
|
||||
// Mock promptWithFallback: first call throws, subsequent calls succeed
|
||||
const { promptWithFallback } = await import("./pi.js");
|
||||
let callCount = 0;
|
||||
vi.mocked(promptWithFallback).mockImplementation(async (session: any, prompt: string) => {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw new Error("context window exceeds limit (2013)");
|
||||
}
|
||||
// Subsequent calls (compact-and-resume) succeed
|
||||
});
|
||||
|
||||
// Mock compactSessionContext to succeed
|
||||
const { compactSessionContext } = await import("./pi.js");
|
||||
vi.mocked(compactSessionContext).mockResolvedValue({
|
||||
summary: "Compacted",
|
||||
tokensBefore: 150000,
|
||||
});
|
||||
|
||||
const executor = new StepSessionExecutor({
|
||||
store,
|
||||
taskDetail: task,
|
||||
worktreePath: "/project/.worktrees/main",
|
||||
rootDir: "/project",
|
||||
settings,
|
||||
});
|
||||
|
||||
const results = await executor.executeAll();
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].success).toBe(true);
|
||||
expect(results[0].retries).toBe(0);
|
||||
});
|
||||
|
||||
it("succeeds with reduced-prompt retry when compact returns null", async () => {
|
||||
const task = makeTaskDetail({
|
||||
prompt: makeStepPrompt("FN-001", 1),
|
||||
steps: [{ name: "Step 0", status: "pending" }],
|
||||
});
|
||||
const settings = makeSettings({ maxParallelSteps: 1 });
|
||||
const appendAgentLog = vi.fn().mockResolvedValue(undefined);
|
||||
const store = { appendAgentLog } as unknown as TaskStore;
|
||||
|
||||
mockedCreateKbAgent.mockResolvedValue({
|
||||
session: makeMockSession(),
|
||||
} as any);
|
||||
|
||||
// Mock promptWithFallback: first call throws context-limit, second succeeds (reduced prompt)
|
||||
const { promptWithFallback } = await import("./pi.js");
|
||||
let callCount = 0;
|
||||
vi.mocked(promptWithFallback).mockImplementation(async (session: any, prompt: string) => {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw new Error("context window exceeds limit (2013)");
|
||||
}
|
||||
// Reduced-prompt succeeds
|
||||
});
|
||||
|
||||
// Mock compactSessionContext to return null (no history)
|
||||
const { compactSessionContext } = await import("./pi.js");
|
||||
vi.mocked(compactSessionContext).mockResolvedValue(null);
|
||||
|
||||
const executor = new StepSessionExecutor({
|
||||
store,
|
||||
taskDetail: task,
|
||||
worktreePath: "/project/.worktrees/main",
|
||||
rootDir: "/project",
|
||||
settings,
|
||||
});
|
||||
|
||||
const results = await executor.executeAll();
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].success).toBe(true);
|
||||
});
|
||||
|
||||
it("fails when all recovery attempts fail", async () => {
|
||||
const task = makeTaskDetail({
|
||||
prompt: makeStepPrompt("FN-001", 1),
|
||||
steps: [{ name: "Step 0", status: "pending" }],
|
||||
});
|
||||
const settings = makeSettings({ maxParallelSteps: 1 });
|
||||
const appendAgentLog = vi.fn().mockResolvedValue(undefined);
|
||||
const store = { appendAgentLog } as unknown as TaskStore;
|
||||
|
||||
mockedCreateKbAgent.mockResolvedValue({
|
||||
session: makeMockSession(),
|
||||
} as any);
|
||||
|
||||
// Mock promptWithFallback: always throws context-limit error
|
||||
const { promptWithFallback } = await import("./pi.js");
|
||||
vi.mocked(promptWithFallback).mockRejectedValue(
|
||||
new Error("context window exceeds limit (2013)"),
|
||||
);
|
||||
|
||||
// Mock compactSessionContext to return null (no history)
|
||||
const { compactSessionContext } = await import("./pi.js");
|
||||
vi.mocked(compactSessionContext).mockResolvedValue(null);
|
||||
|
||||
const executor = new StepSessionExecutor({
|
||||
store,
|
||||
taskDetail: task,
|
||||
worktreePath: "/project/.worktrees/main",
|
||||
rootDir: "/project",
|
||||
settings,
|
||||
});
|
||||
|
||||
const resultsPromise = executor.executeAll();
|
||||
// Advance timers for retry delays
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
const results = await resultsPromise;
|
||||
|
||||
// All recovery attempts exhausted, step should fail
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].success).toBe(false);
|
||||
expect(results[0].error).toContain("context window exceeds limit");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,12 +16,14 @@ import { join } from "node:path";
|
||||
import type { AgentSession, ToolDefinition } from "@mariozechner/pi-coding-agent";
|
||||
import type { TaskDetail, Settings, TaskStep, StepStatus, TaskStore } from "@fusion/core";
|
||||
|
||||
import { createKbAgent, promptWithFallback, describeModel } from "./pi.js";
|
||||
import { createKbAgent, promptWithFallback, describeModel, compactSessionContext } from "./pi.js";
|
||||
import { generateWorktreeName } from "./worktree-names.js";
|
||||
import { AgentSemaphore } from "./concurrency.js";
|
||||
import { StuckTaskDetector, type DisposableSession } from "./stuck-task-detector.js";
|
||||
import { AgentLogger } from "./agent-logger.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { isContextLimitError } from "./context-limit-detector.js";
|
||||
import { checkSessionError } from "./usage-limit-detector.js";
|
||||
|
||||
const stepExecLog = createLogger("step-session-executor");
|
||||
|
||||
@@ -454,6 +456,40 @@ function escapeRegex(str: string): string {
|
||||
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a reduced prompt for context-limit recovery.
|
||||
*
|
||||
* This is a simpler, shorter prompt that doesn't include the full task context,
|
||||
* designed to fit within context limits when the original prompt is too large.
|
||||
*
|
||||
* @param taskDetail - The task to build a prompt for.
|
||||
* @param stepIndex - The 0-based step index.
|
||||
* @returns A reduced prompt string focused on the current step only.
|
||||
*/
|
||||
function buildReducedStepPrompt(taskDetail: TaskDetail, stepIndex: number): string {
|
||||
const { prompt, id, title } = taskDetail;
|
||||
|
||||
// Extract the step-specific section
|
||||
const stepSection = extractStepSection(prompt, stepIndex);
|
||||
|
||||
// Build a minimal prompt that focuses on the step without excessive context
|
||||
const parts: string[] = [
|
||||
`You are executing step ${stepIndex} of task ${id}.`,
|
||||
title ? `Task: ${title}` : "",
|
||||
"",
|
||||
"Focus on completing this step efficiently:",
|
||||
"",
|
||||
stepSection,
|
||||
"",
|
||||
"IMPORTANT: Your previous attempt hit the context window limit.",
|
||||
"Do NOT repeat work that's already been done.",
|
||||
"Check git status and git log to see what's been committed.",
|
||||
"Complete the remaining work and call task_done().",
|
||||
];
|
||||
|
||||
return parts.join("\n").replace(/\n{3,}/g, "\n\n"); // Collapse multiple blank lines
|
||||
}
|
||||
|
||||
// ── StepSessionExecutor ───────────────────────────────────────────────
|
||||
|
||||
/** Maximum retry attempts for a failed step. */
|
||||
@@ -656,6 +692,11 @@ export class StepSessionExecutor {
|
||||
*
|
||||
* Creates a fresh session, sends the step-specific prompt, and handles
|
||||
* retries with exponential backoff on failure.
|
||||
*
|
||||
* Context-limit errors trigger bounded recovery before consuming a retry attempt:
|
||||
* 1. Compact-and-resume: compact session history, retry with original prompt
|
||||
* 2. Reduced-prompt retry: if compact unavailable/failed, retry with simpler prompt
|
||||
* Recovery attempts do NOT count toward MAX_STEP_RETRIES to prevent premature failure.
|
||||
*/
|
||||
private async executeStep(stepIndex: number, worktreePath: string): Promise<StepResult> {
|
||||
const { taskDetail, settings, stuckTaskDetector, semaphore } = this.options;
|
||||
@@ -671,6 +712,9 @@ export class StepSessionExecutor {
|
||||
// Build step prompt
|
||||
const stepPrompt = buildStepPrompt(taskDetail, stepIndex, this.options.rootDir, settings);
|
||||
|
||||
// Build reduced step prompt for context-limit recovery (simpler, shorter)
|
||||
const reducedStepPrompt = buildReducedStepPrompt(taskDetail, stepIndex);
|
||||
|
||||
// Acquire semaphore if provided
|
||||
if (semaphore) {
|
||||
await semaphore.acquire();
|
||||
@@ -678,6 +722,9 @@ export class StepSessionExecutor {
|
||||
|
||||
const trackingKey = this.makeTrackingKey(stepIndex);
|
||||
let retries = 0;
|
||||
// Track context-limit recovery attempts separately from retry attempts.
|
||||
// Recovery does NOT count toward MAX_STEP_RETRIES to prevent premature failure.
|
||||
let recoveryAttempts = 0;
|
||||
|
||||
try {
|
||||
for (let attempt = 0; attempt <= MAX_STEP_RETRIES; attempt++) {
|
||||
@@ -748,15 +795,119 @@ export class StepSessionExecutor {
|
||||
// Send prompt
|
||||
await promptWithFallback(session, stepPrompt);
|
||||
|
||||
// Re-raise errors that pi-coding-agent swallowed after exhausting retries.
|
||||
// session.prompt() resolves normally even when retries are exhausted —
|
||||
// the error is stored on session.state.error instead of being thrown.
|
||||
checkSessionError(session);
|
||||
|
||||
const result: StepResult = { stepIndex, success: true, retries };
|
||||
this.options.onStepComplete?.(stepIndex, result);
|
||||
return result;
|
||||
} catch (err) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
} catch (err: unknown) {
|
||||
// Normalize error message for consistent classification
|
||||
const errorMessage = typeof err === "string" ? err : (err as { message?: string })?.message ?? String(err);
|
||||
stepExecLog.warn(
|
||||
`Step ${stepIndex} attempt ${attempt + 1} failed: ${errorMessage}`,
|
||||
);
|
||||
|
||||
// Check for context-limit error and attempt bounded recovery
|
||||
// Recovery is bounded to prevent infinite loops (MAX_STEP_RETRIES recovery)
|
||||
if (isContextLimitError(errorMessage) && recoveryAttempts < MAX_STEP_RETRIES && session) {
|
||||
stepExecLog.log(
|
||||
`Step ${stepIndex} context limit error — attempting bounded recovery ` +
|
||||
`(recoveryAttempt=${recoveryAttempts + 1}/${MAX_STEP_RETRIES})`,
|
||||
);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Context limit error on step ${stepIndex}: ${errorMessage}`,
|
||||
"tool_error",
|
||||
);
|
||||
|
||||
// Try compact-and-resume first
|
||||
const compactResult = await compactSessionContext(session);
|
||||
if (compactResult) {
|
||||
stepExecLog.log(
|
||||
`Step ${stepIndex} context compaction succeeded (${compactResult.tokensBefore} tokens) — resuming`,
|
||||
);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Context compaction succeeded at ${compactResult.tokensBefore} tokens — resuming step ${stepIndex}`,
|
||||
"text",
|
||||
);
|
||||
recoveryAttempts++;
|
||||
|
||||
// Attempt resume with original prompt
|
||||
try {
|
||||
stuckTaskDetector?.recordActivity(trackingKey);
|
||||
await promptWithFallback(session, stepPrompt);
|
||||
checkSessionError(session);
|
||||
stepExecLog.log(`Step ${stepIndex} compact-and-resume succeeded`);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Compact-and-resume succeeded for step ${stepIndex}`,
|
||||
"text",
|
||||
);
|
||||
const result: StepResult = { stepIndex, success: true, retries };
|
||||
this.options.onStepComplete?.(stepIndex, result);
|
||||
return result;
|
||||
} catch (resumeErr: unknown) {
|
||||
const resumeErrorMessage = typeof resumeErr === "string"
|
||||
? resumeErr
|
||||
: (resumeErr as { message?: string })?.message ?? String(resumeErr);
|
||||
stepExecLog.warn(
|
||||
`Step ${stepIndex} resume after compaction failed: ${resumeErrorMessage}`,
|
||||
);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Resume after compaction failed for step ${stepIndex}: ${resumeErrorMessage}`,
|
||||
"tool_error",
|
||||
);
|
||||
// Fall through to reduced-prompt retry
|
||||
}
|
||||
} else {
|
||||
stepExecLog.log(
|
||||
`Step ${stepIndex} context compaction unavailable — attempting reduced-prompt retry`,
|
||||
);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Context compaction unavailable for step ${stepIndex} — attempting reduced-prompt recovery`,
|
||||
"text",
|
||||
);
|
||||
}
|
||||
|
||||
// Compact returned null OR resume failed — try reduced-prompt retry
|
||||
if (recoveryAttempts < MAX_STEP_RETRIES) {
|
||||
recoveryAttempts++;
|
||||
try {
|
||||
stuckTaskDetector?.recordActivity(trackingKey);
|
||||
await promptWithFallback(session, reducedStepPrompt);
|
||||
checkSessionError(session);
|
||||
stepExecLog.log(`Step ${stepIndex} reduced-prompt recovery succeeded`);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Reduced-prompt recovery succeeded for step ${stepIndex}`,
|
||||
"text",
|
||||
);
|
||||
const result: StepResult = { stepIndex, success: true, retries };
|
||||
this.options.onStepComplete?.(stepIndex, result);
|
||||
return result;
|
||||
} catch (reducedErr: unknown) {
|
||||
const reducedErrorMessage = typeof reducedErr === "string"
|
||||
? reducedErr
|
||||
: (reducedErr as { message?: string })?.message ?? String(reducedErr);
|
||||
stepExecLog.warn(
|
||||
`Step ${stepIndex} reduced-prompt retry also failed: ${reducedErrorMessage}`,
|
||||
);
|
||||
await this.store.appendAgentLog(
|
||||
taskDetail.id,
|
||||
`[step-exec] Reduced-prompt recovery failed for step ${stepIndex}: ${reducedErrorMessage}`,
|
||||
"tool_error",
|
||||
);
|
||||
// Fall through to normal retry or failure
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If this was the last attempt, return failure
|
||||
if (attempt === MAX_STEP_RETRIES) {
|
||||
const result: StepResult = {
|
||||
|
||||
Reference in New Issue
Block a user