fix(engine): honor project execution model overrides in spawned children + workflow-step timeout fallback

Two related executor fixes:

1. Spawned child agents previously bypassed the executor model lane hierarchy
   and used settings.defaultProvider/defaultModelId directly, ignoring
   project-level executionProvider/executionModelId from .fusion/config.json.
   Resolve via resolveExecutorModelPair() so children honor the same
   precedence as the parent executor.

2. Pre-merge workflow step AI calls now have a wall-clock timeout
   (settings.workflowStepTimeoutMs, default 6 min) and fall back to the
   configured validatorFallback / fallback model on timeout. The 20-min
   stuck-detector kill loop was the only escape hatch when a provider's
   streaming API hung mid-response, and the kill triggered a same-provider
   retry — guaranteeing repeat hangs. The runner now races the prompt against
   a timeout; on timeout it disposes the session, logs a clear entry, and
   re-runs the step once with a distinct fallback provider/model. If neither
   completes (or no fallback is configured), the step returns a normal
   failure that flows into the existing handleWorkflowStepFailure retry path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-28 14:20:23 -07:00
parent 58510e179f
commit 621bee1cd6
3 changed files with 115 additions and 37 deletions

View File

@@ -114,6 +114,7 @@ export const DEFAULT_PROJECT_SETTINGS = {
smartConflictResolution: true, smartConflictResolution: true,
worktreeRebaseBeforeMerge: true, worktreeRebaseBeforeMerge: true,
worktreeRebaseRemote: "", worktreeRebaseRemote: "",
workflowStepTimeoutMs: 360_000,
strictScopeEnforcement: false, strictScopeEnforcement: false,
buildRetryCount: 0, buildRetryCount: 0,
verificationFixRetries: 3, verificationFixRetries: 3,

View File

@@ -1500,6 +1500,11 @@ export interface ProjectSettings {
* (typically `origin`). Exposed as a dropdown in the dashboard's * (typically `origin`). Exposed as a dropdown in the dashboard's
* Worktrees settings. */ * Worktrees settings. */
worktreeRebaseRemote?: string; worktreeRebaseRemote?: string;
/** Wall-clock timeout (ms) for a single pre-merge workflow step's AI call.
* When a step exceeds this, the session is aborted and the executor is
* given one shot to retry with the configured fallback model before the
* step is reported as failed. Default: 360_000 (6 minutes). */
workflowStepTimeoutMs?: number;
/** When true, out-of-scope file changes block merge instead of just logging warnings. /** When true, out-of-scope file changes block merge instead of just logging warnings.
* Useful for teams that want strict enforcement of declared File Scope. * Useful for teams that want strict enforcement of declared File Scope.
* Default: false (soft guardrail — warnings only). */ * Default: false (soft guardrail — warnings only). */

View File

@@ -216,6 +216,10 @@ export interface WorkflowStepOutcome {
revisionRequested?: boolean; revisionRequested?: boolean;
output?: string; output?: string;
error?: string; error?: string;
/** Set when the call exceeded `settings.workflowStepTimeoutMs`. Signals the
* caller to escalate to the fallback model rather than treat the failure
* as a generic revision request. */
timedOut?: boolean;
} }
/** /**
@@ -3962,12 +3966,30 @@ and show an appropriate message to the user.\`
}, },
}); });
try { // Determine primary model and an explicit fallback. The workflow step's
// Determine model: prefer workflow step override, fall back to global settings // own override takes precedence; otherwise we use the global default. The
const stepProvider = workflowStep.modelProvider || settings.defaultProvider; // fallback is the per-step override's missing-counterpart settings, then
const stepModelId = workflowStep.modelId || settings.defaultModelId; // the global validator/fallback pair, then the executor's `fallbackProvider`.
const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId); const primaryProvider = workflowStep.modelProvider || settings.defaultProvider;
const primaryModelId = workflowStep.modelId || settings.defaultModelId;
const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId);
type ModelTuple = { provider?: string; modelId?: string };
const fallbackCandidates: Array<ModelTuple & { label: string }> = [
{ provider: settings.validatorFallbackProvider, modelId: settings.validatorFallbackModelId, label: "validatorFallback" },
{ provider: settings.fallbackProvider, modelId: settings.fallbackModelId, label: "globalFallback" },
];
const fallback = fallbackCandidates.find(
(c) => c.provider && c.modelId && (c.provider !== primaryProvider || c.modelId !== primaryModelId),
);
const timeoutMs = Math.max(60_000, settings.workflowStepTimeoutMs ?? 360_000);
const runOnce = async (
provider: string | undefined,
modelId: string | undefined,
attemptLabel: string,
): Promise<WorkflowStepOutcome> => {
// Workflow step agents inherit executor instructions // Workflow step agents inherit executor instructions
const stepInstructions = await this.resolveInstructionsForRole("executor"); const stepInstructions = await this.resolveInstructionsForRole("executor");
const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions); const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions);
@@ -3992,8 +4014,8 @@ and show an appropriate message to the user.\`
cwd: worktreePath, cwd: worktreePath,
systemPrompt: stepSystemPrompt, systemPrompt: stepSystemPrompt,
tools: toolMode, tools: toolMode,
defaultProvider: stepProvider, defaultProvider: provider,
defaultModelId: stepModelId, defaultModelId: modelId,
fallbackProvider: settings.fallbackProvider, fallbackProvider: settings.fallbackProvider,
fallbackModelId: settings.fallbackModelId, fallbackModelId: settings.fallbackModelId,
defaultThinkingLevel: settings.defaultThinkingLevel, defaultThinkingLevel: settings.defaultThinkingLevel,
@@ -4001,8 +4023,11 @@ and show an appropriate message to the user.\`
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}), ...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
}); });
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' using model ${describeModel(session)}${useOverride ? " (workflow step override)" : ""}`); executorLog.log(`${task.id}: workflow step '${workflowStep.name}' using model ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`);
await this.store.logEntry(task.id, `Workflow step '${workflowStep.name}' using model: ${describeModel(session)}${useOverride ? " (workflow step override)" : ""}`); await this.store.logEntry(
task.id,
`Workflow step '${workflowStep.name}' using model: ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`,
);
let output = ""; let output = "";
session.subscribe((event) => { session.subscribe((event) => {
@@ -4023,37 +4048,78 @@ and show an appropriate message to the user.\`
} }
}); });
await promptWithFallback( let timedOut = false;
session, let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
`Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` + const timeoutPromise = new Promise<"timeout">((resolveTimeout) => {
`Review the work done in this worktree and evaluate it against the criteria in your instructions.`, timeoutHandle = setTimeout(() => {
); timedOut = true;
resolveTimeout("timeout");
}, timeoutMs);
});
checkSessionError(session); try {
await accumulateSessionTokenUsage(this.store, task.id, session); const promptPromise = promptWithFallback(
session.dispose(); session,
await agentLogger.flush(); `Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` +
`Review the work done in this worktree and evaluate it against the criteria in your instructions.`,
);
// Check if the output contains a revision request const outcome = await Promise.race([
const trimmedOutput = output.trim(); promptPromise.then(() => "completed" as const),
const revisionMatch = trimmedOutput.match(/^REQUEST REVISION\s*\n*/i); timeoutPromise,
if (revisionMatch) { ]);
// Extract the feedback after "REQUEST REVISION"
const feedbackStart = revisionMatch[0].length; if (outcome === "timeout") {
const feedback = trimmedOutput.slice(feedbackStart).trim(); executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' (${attemptLabel}) timed out after ${timeoutMs}ms — disposing session`);
return { await this.store.logEntry(
success: false, task.id,
revisionRequested: true, `Workflow step '${workflowStep.name}' ${attemptLabel === "primary" ? "primary" : "fallback"} model timed out after ${Math.round(timeoutMs / 1000)}s — aborting session`,
output: feedback, );
}; try { session.dispose(); } catch { /* best-effort */ }
await agentLogger.flush();
return { success: false, error: `workflow step timed out after ${timeoutMs}ms`, timedOut: true };
}
// Completed within the timeout — let any post-completion errors surface.
checkSessionError(session);
await accumulateSessionTokenUsage(this.store, task.id, session);
session.dispose();
await agentLogger.flush();
const trimmedOutput = output.trim();
const revisionMatch = trimmedOutput.match(/^REQUEST REVISION\s*\n*/i);
if (revisionMatch) {
const feedbackStart = revisionMatch[0].length;
const feedback = trimmedOutput.slice(feedbackStart).trim();
return { success: false, revisionRequested: true, output: feedback };
}
return { success: true, output };
} catch (err: unknown) {
await agentLogger.flush();
try { session.dispose(); } catch { /* best-effort */ }
const errorMessage = err instanceof Error ? err.message : String(err);
return { success: false, error: errorMessage };
} finally {
if (timeoutHandle) clearTimeout(timeoutHandle);
// Suppress unused-variable warning; `timedOut` documents intent.
void timedOut;
} }
};
return { success: true, output }; const primaryOutcome = await runOnce(primaryProvider, primaryModelId, "primary");
} catch (err: unknown) { if (!primaryOutcome.timedOut) return primaryOutcome;
const errorMessage = err instanceof Error ? err.message : String(err);
await agentLogger.flush(); if (!fallback) {
return { success: false, error: errorMessage }; executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' timed out and no fallback model is configured`);
await this.store.logEntry(
task.id,
`Workflow step '${workflowStep.name}' timed out — no fallback model configured (set settings.validatorFallbackProvider/Id or fallbackProvider/Id)`,
);
return primaryOutcome;
} }
executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with fallback ${fallback.provider}/${fallback.modelId} (label=${fallback.label})`);
return runOnce(fallback.provider, fallback.modelId, "fallback");
} }
private MAX_WORKTREE_RETRIES = 3; private MAX_WORKTREE_RETRIES = 3;
@@ -5160,6 +5226,12 @@ and show an appropriate message to the user.\`
const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig) const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig)
?? extractRuntimeHint(parentAgent?.runtimeConfig); ?? extractRuntimeHint(parentAgent?.runtimeConfig);
// Resolve executor model via canonical lane hierarchy so child agents
// honor project executionProvider/executionModelId overrides (parity
// with main executor at the top of agentWork()).
const { provider: childExecutorProvider, modelId: childExecutorModelId } =
resolveExecutorModelPair(undefined, undefined, settings);
// Create child agent session // Create child agent session
const { session: childSession } = await createResolvedAgentSession({ const { session: childSession } = await createResolvedAgentSession({
sessionPurpose: "executor", sessionPurpose: "executor",
@@ -5168,8 +5240,8 @@ and show an appropriate message to the user.\`
cwd: childWorktreePath, cwd: childWorktreePath,
systemPrompt: childSystemPrompt, systemPrompt: childSystemPrompt,
tools: "coding", tools: "coding",
defaultProvider: settings.defaultProvider, defaultProvider: childExecutorProvider,
defaultModelId: settings.defaultModelId, defaultModelId: childExecutorModelId,
fallbackProvider: settings.fallbackProvider, fallbackProvider: settings.fallbackProvider,
fallbackModelId: settings.fallbackModelId, fallbackModelId: settings.fallbackModelId,
// Skill selection: use assigned agent skills if available, otherwise role fallback // Skill selection: use assigned agent skills if available, otherwise role fallback