feat(FN-4601): complete Step 2 — recover missing-worktree retry failures
Fusion-Task-Id: FN-4601 Fusion-Task-Lineage: 136888a7-aee7-4eda-a165-16448a98e626
This commit is contained in:
@@ -72,6 +72,10 @@ import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
|||||||
import type { AgentReflectionService } from "./agent-reflection.js";
|
import type { AgentReflectionService } from "./agent-reflection.js";
|
||||||
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
|
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
|
||||||
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
|
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
|
||||||
|
import {
|
||||||
|
extractMissingWorktreePathFromSessionStartFailure,
|
||||||
|
isMissingWorktreeSessionStartFailure,
|
||||||
|
} from "./restart-recovery-coordinator.js";
|
||||||
import { BranchWorktreeAutoRecoveryHandler } from "./auto-recovery-handlers/branch-worktree.js";
|
import { BranchWorktreeAutoRecoveryHandler } from "./auto-recovery-handlers/branch-worktree.js";
|
||||||
import { ContaminationAutoRecoveryHandler } from "./auto-recovery-handlers/contamination.js";
|
import { ContaminationAutoRecoveryHandler } from "./auto-recovery-handlers/contamination.js";
|
||||||
import { createFileScopeAutoRecoveryHandler } from "./auto-recovery-handlers/file-scope.js";
|
import { createFileScopeAutoRecoveryHandler } from "./auto-recovery-handlers/file-scope.js";
|
||||||
@@ -3835,92 +3839,116 @@ export class TaskExecutor {
|
|||||||
this.tokenUsageBaselines.delete(task.id);
|
this.tokenUsageBaselines.delete(task.id);
|
||||||
session.dispose();
|
session.dispose();
|
||||||
|
|
||||||
const { session: retrySession, sessionFile: retrySessionFile } = await createResolvedAgentSession({
|
let retrySession: AgentSession | null = null;
|
||||||
sessionPurpose: "executor",
|
try {
|
||||||
runtimeHint: executorRuntimeHint,
|
const createdRetrySession = await createResolvedAgentSession({
|
||||||
pluginRunner: this.options.pluginRunner,
|
sessionPurpose: "executor",
|
||||||
cwd: worktreePath,
|
runtimeHint: executorRuntimeHint,
|
||||||
systemPrompt: executorSystemPromptFinal,
|
pluginRunner: this.options.pluginRunner,
|
||||||
systemPromptLayers: executorLayers,
|
cwd: worktreePath,
|
||||||
tools: "coding",
|
systemPrompt: executorSystemPromptFinal,
|
||||||
customTools,
|
systemPromptLayers: executorLayers,
|
||||||
onText: agentLogger.onText,
|
tools: "coding",
|
||||||
onThinking: agentLogger.onThinking,
|
customTools,
|
||||||
onToolStart: agentLogger.onToolStart,
|
onText: agentLogger.onText,
|
||||||
onToolEnd: agentLogger.onToolEnd,
|
onThinking: agentLogger.onThinking,
|
||||||
defaultProvider: executorProvider,
|
onToolStart: agentLogger.onToolStart,
|
||||||
defaultModelId: executorModelId,
|
onToolEnd: agentLogger.onToolEnd,
|
||||||
fallbackProvider: executorFallbackProvider,
|
defaultProvider: executorProvider,
|
||||||
fallbackModelId: executorFallbackModelId,
|
defaultModelId: executorModelId,
|
||||||
defaultThinkingLevel: executorThinkingLevel,
|
fallbackProvider: executorFallbackProvider,
|
||||||
sessionManager: SessionManager.create(worktreePath),
|
fallbackModelId: executorFallbackModelId,
|
||||||
taskEnv,
|
defaultThinkingLevel: executorThinkingLevel,
|
||||||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
sessionManager: SessionManager.create(worktreePath),
|
||||||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
taskEnv,
|
||||||
actionGateContext: this.buildActionGateContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
|
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||||||
permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
|
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||||||
});
|
actionGateContext: this.buildActionGateContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
|
||||||
if (retrySessionFile) {
|
permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
|
||||||
this.store.updateTask(task.id, { sessionFile: retrySessionFile }).catch((err: unknown) => {
|
|
||||||
const msg = err instanceof Error ? err.message : String(err);
|
|
||||||
executorLog.warn(`${task.id} failed to persist retry sessionFile: ${msg}`);
|
|
||||||
});
|
});
|
||||||
|
retrySession = createdRetrySession.session;
|
||||||
|
if (createdRetrySession.sessionFile) {
|
||||||
|
this.store.updateTask(task.id, { sessionFile: createdRetrySession.sessionFile }).catch((err: unknown) => {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
executorLog.warn(`${task.id} failed to persist retry sessionFile: ${msg}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
session = retrySession;
|
||||||
|
sessionRef.current = retrySession;
|
||||||
|
this.activeSessions.set(task.id, {
|
||||||
|
session: retrySession,
|
||||||
|
seenSteeringIds,
|
||||||
|
lastResolvedModelProvider: executorProvider,
|
||||||
|
lastResolvedModelId: executorModelId,
|
||||||
|
lastTaskModelProvider: detail.modelProvider,
|
||||||
|
lastTaskModelId: detail.modelId,
|
||||||
|
lastAssignedAgentId: detail.assignedAgentId ?? null,
|
||||||
|
});
|
||||||
|
stuckDetector?.trackTask(task.id, retrySession);
|
||||||
|
|
||||||
|
let retryPrompt: string;
|
||||||
|
if (pseudoPause.kind !== "none") {
|
||||||
|
const shortMatch = (pseudoPause.matched ?? "").slice(0, 120);
|
||||||
|
retryPrompt = [
|
||||||
|
`Your previous turn ended with a pseudo-pause: "${shortMatch}". This is forbidden.`,
|
||||||
|
"",
|
||||||
|
"Turn-ending rules you violated:",
|
||||||
|
"- You MUST NOT end a turn by asking the user a question, summarizing progress, or requesting permission to continue.",
|
||||||
|
"- Phrases like 'If you want, I can continue', 'Should I proceed?', 'Let me know if...' are FORBIDDEN turn-endings.",
|
||||||
|
"- The user is not watching this conversation. Questions written as prose are ignored.",
|
||||||
|
"- If you genuinely cannot proceed, call fn_task_done with a clear explanation — never write the blocker as plain prose.",
|
||||||
|
"",
|
||||||
|
"What you must do now:",
|
||||||
|
"1. Review the PROMPT.md steps and identify the next pending step.",
|
||||||
|
"2. Do the work for that step immediately — call fn_task_update, write code, run tests.",
|
||||||
|
"3. Continue until all steps are done, then call fn_task_done.",
|
||||||
|
"Do NOT ask for permission. Do NOT write a summary. Just call a tool and keep working.",
|
||||||
|
"",
|
||||||
|
"Original task:",
|
||||||
|
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath, this.options.pluginRunner),
|
||||||
|
].join("\n");
|
||||||
|
} else {
|
||||||
|
retryPrompt = [
|
||||||
|
"Your previous session ended without calling the fn_task_done tool.",
|
||||||
|
"The task may already be complete — review the current state of the worktree and either:",
|
||||||
|
"1. If the work is done, call fn_task_done with a summary of what was accomplished.",
|
||||||
|
"2. If there is remaining work, finish it and then call fn_task_done.",
|
||||||
|
"",
|
||||||
|
"Original task:",
|
||||||
|
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath, this.options.pluginRunner),
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
stuckDetector?.recordActivity(task.id);
|
||||||
|
await promptWithFallback(retrySession, retryPrompt);
|
||||||
|
checkSessionError(retrySession);
|
||||||
|
await accumulateSessionTokenUsage(this.store, task.id, retrySession, {
|
||||||
|
agentId: task.assignedAgentId ?? undefined,
|
||||||
|
role: "executor",
|
||||||
|
});
|
||||||
|
} catch (retryError) {
|
||||||
|
const retryErrorText = retryError instanceof Error ? retryError.message : String(retryError);
|
||||||
|
if (!isMissingWorktreeSessionStartFailure(retryErrorText)) {
|
||||||
|
throw retryError;
|
||||||
|
}
|
||||||
|
const recoveredPath = extractMissingWorktreePathFromSessionStartFailure(retryErrorText) ?? worktreePath;
|
||||||
|
const reclaimMessage = `${task.id}: no-fn_task_done retry hit missing worktree session-start failure (${recoveredPath}) — clearing stale metadata and requeueing`;
|
||||||
|
executorLog.log(reclaimMessage);
|
||||||
|
await this.store.logEntry(task.id, reclaimMessage, undefined, this.currentRunContext);
|
||||||
|
await this.store.updateTask(task.id, {
|
||||||
|
sessionFile: null,
|
||||||
|
worktree: null,
|
||||||
|
branch: null,
|
||||||
|
baseCommitSha: null,
|
||||||
|
});
|
||||||
|
this.activeSessions.delete(task.id);
|
||||||
|
this.tokenUsageBaselines.delete(task.id);
|
||||||
|
retrySession?.dispose();
|
||||||
|
retryAbortedDueToReclaim = true;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
session = retrySession;
|
|
||||||
sessionRef.current = retrySession;
|
|
||||||
this.activeSessions.set(task.id, {
|
|
||||||
session: retrySession,
|
|
||||||
seenSteeringIds,
|
|
||||||
lastResolvedModelProvider: executorProvider,
|
|
||||||
lastResolvedModelId: executorModelId,
|
|
||||||
lastTaskModelProvider: detail.modelProvider,
|
|
||||||
lastTaskModelId: detail.modelId,
|
|
||||||
lastAssignedAgentId: detail.assignedAgentId ?? null,
|
|
||||||
});
|
|
||||||
stuckDetector?.trackTask(task.id, retrySession);
|
|
||||||
|
|
||||||
let retryPrompt: string;
|
|
||||||
if (pseudoPause.kind !== "none") {
|
|
||||||
const shortMatch = (pseudoPause.matched ?? "").slice(0, 120);
|
|
||||||
retryPrompt = [
|
|
||||||
`Your previous turn ended with a pseudo-pause: "${shortMatch}". This is forbidden.`,
|
|
||||||
"",
|
|
||||||
"Turn-ending rules you violated:",
|
|
||||||
"- You MUST NOT end a turn by asking the user a question, summarizing progress, or requesting permission to continue.",
|
|
||||||
"- Phrases like 'If you want, I can continue', 'Should I proceed?', 'Let me know if...' are FORBIDDEN turn-endings.",
|
|
||||||
"- The user is not watching this conversation. Questions written as prose are ignored.",
|
|
||||||
"- If you genuinely cannot proceed, call fn_task_done with a clear explanation — never write the blocker as plain prose.",
|
|
||||||
"",
|
|
||||||
"What you must do now:",
|
|
||||||
"1. Review the PROMPT.md steps and identify the next pending step.",
|
|
||||||
"2. Do the work for that step immediately — call fn_task_update, write code, run tests.",
|
|
||||||
"3. Continue until all steps are done, then call fn_task_done.",
|
|
||||||
"Do NOT ask for permission. Do NOT write a summary. Just call a tool and keep working.",
|
|
||||||
"",
|
|
||||||
"Original task:",
|
|
||||||
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath, this.options.pluginRunner),
|
|
||||||
].join("\n");
|
|
||||||
} else {
|
|
||||||
retryPrompt = [
|
|
||||||
"Your previous session ended without calling the fn_task_done tool.",
|
|
||||||
"The task may already be complete — review the current state of the worktree and either:",
|
|
||||||
"1. If the work is done, call fn_task_done with a summary of what was accomplished.",
|
|
||||||
"2. If there is remaining work, finish it and then call fn_task_done.",
|
|
||||||
"",
|
|
||||||
"Original task:",
|
|
||||||
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath, this.options.pluginRunner),
|
|
||||||
].join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
stuckDetector?.recordActivity(task.id);
|
|
||||||
await promptWithFallback(retrySession, retryPrompt);
|
|
||||||
checkSessionError(retrySession);
|
|
||||||
await accumulateSessionTokenUsage(this.store, task.id, retrySession, {
|
|
||||||
agentId: task.assignedAgentId ?? undefined,
|
|
||||||
role: "executor",
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!taskDone) {
|
if (!taskDone) {
|
||||||
const implicitCheck = await this.store.getTask(task.id);
|
const implicitCheck = await this.store.getTask(task.id);
|
||||||
if (implicitCheck.steps.length > 0 &&
|
if (implicitCheck.steps.length > 0 &&
|
||||||
|
|||||||
Reference in New Issue
Block a user