fix(engine): honor project execution model overrides in spawned children + workflow-step timeout fallback
Two related executor fixes: 1. Spawned child agents previously bypassed the executor model lane hierarchy and used settings.defaultProvider/defaultModelId directly, ignoring project-level executionProvider/executionModelId from .fusion/config.json. Resolve via resolveExecutorModelPair() so children honor the same precedence as the parent executor. 2. Pre-merge workflow step AI calls now have a wall-clock timeout (settings.workflowStepTimeoutMs, default 6 min) and fall back to the configured validatorFallback / fallback model on timeout. The 20-min stuck-detector kill loop was the only escape hatch when a provider's streaming API hung mid-response, and the kill triggered a same-provider retry — guaranteeing repeat hangs. The runner now races the prompt against a timeout; on timeout it disposes the session, logs a clear entry, and re-runs the step once with a distinct fallback provider/model. If neither completes (or no fallback is configured), the step returns a normal failure that flows into the existing handleWorkflowStepFailure retry path. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -114,6 +114,7 @@ export const DEFAULT_PROJECT_SETTINGS = {
|
|||||||
smartConflictResolution: true,
|
smartConflictResolution: true,
|
||||||
worktreeRebaseBeforeMerge: true,
|
worktreeRebaseBeforeMerge: true,
|
||||||
worktreeRebaseRemote: "",
|
worktreeRebaseRemote: "",
|
||||||
|
workflowStepTimeoutMs: 360_000,
|
||||||
strictScopeEnforcement: false,
|
strictScopeEnforcement: false,
|
||||||
buildRetryCount: 0,
|
buildRetryCount: 0,
|
||||||
verificationFixRetries: 3,
|
verificationFixRetries: 3,
|
||||||
|
|||||||
@@ -1500,6 +1500,11 @@ export interface ProjectSettings {
|
|||||||
* (typically `origin`). Exposed as a dropdown in the dashboard's
|
* (typically `origin`). Exposed as a dropdown in the dashboard's
|
||||||
* Worktrees settings. */
|
* Worktrees settings. */
|
||||||
worktreeRebaseRemote?: string;
|
worktreeRebaseRemote?: string;
|
||||||
|
/** Wall-clock timeout (ms) for a single pre-merge workflow step's AI call.
|
||||||
|
* When a step exceeds this, the session is aborted and the executor is
|
||||||
|
* given one shot to retry with the configured fallback model before the
|
||||||
|
* step is reported as failed. Default: 360_000 (6 minutes). */
|
||||||
|
workflowStepTimeoutMs?: number;
|
||||||
/** When true, out-of-scope file changes block merge instead of just logging warnings.
|
/** When true, out-of-scope file changes block merge instead of just logging warnings.
|
||||||
* Useful for teams that want strict enforcement of declared File Scope.
|
* Useful for teams that want strict enforcement of declared File Scope.
|
||||||
* Default: false (soft guardrail — warnings only). */
|
* Default: false (soft guardrail — warnings only). */
|
||||||
|
|||||||
@@ -216,6 +216,10 @@ export interface WorkflowStepOutcome {
|
|||||||
revisionRequested?: boolean;
|
revisionRequested?: boolean;
|
||||||
output?: string;
|
output?: string;
|
||||||
error?: string;
|
error?: string;
|
||||||
|
/** Set when the call exceeded `settings.workflowStepTimeoutMs`. Signals the
|
||||||
|
* caller to escalate to the fallback model rather than treat the failure
|
||||||
|
* as a generic revision request. */
|
||||||
|
timedOut?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -3962,12 +3966,30 @@ and show an appropriate message to the user.\`
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
// Determine primary model and an explicit fallback. The workflow step's
|
||||||
// Determine model: prefer workflow step override, fall back to global settings
|
// own override takes precedence; otherwise we use the global default. The
|
||||||
const stepProvider = workflowStep.modelProvider || settings.defaultProvider;
|
// fallback is the per-step override's missing-counterpart settings, then
|
||||||
const stepModelId = workflowStep.modelId || settings.defaultModelId;
|
// the global validator/fallback pair, then the executor's `fallbackProvider`.
|
||||||
const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId);
|
const primaryProvider = workflowStep.modelProvider || settings.defaultProvider;
|
||||||
|
const primaryModelId = workflowStep.modelId || settings.defaultModelId;
|
||||||
|
const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId);
|
||||||
|
|
||||||
|
type ModelTuple = { provider?: string; modelId?: string };
|
||||||
|
const fallbackCandidates: Array<ModelTuple & { label: string }> = [
|
||||||
|
{ provider: settings.validatorFallbackProvider, modelId: settings.validatorFallbackModelId, label: "validatorFallback" },
|
||||||
|
{ provider: settings.fallbackProvider, modelId: settings.fallbackModelId, label: "globalFallback" },
|
||||||
|
];
|
||||||
|
const fallback = fallbackCandidates.find(
|
||||||
|
(c) => c.provider && c.modelId && (c.provider !== primaryProvider || c.modelId !== primaryModelId),
|
||||||
|
);
|
||||||
|
|
||||||
|
const timeoutMs = Math.max(60_000, settings.workflowStepTimeoutMs ?? 360_000);
|
||||||
|
|
||||||
|
const runOnce = async (
|
||||||
|
provider: string | undefined,
|
||||||
|
modelId: string | undefined,
|
||||||
|
attemptLabel: string,
|
||||||
|
): Promise<WorkflowStepOutcome> => {
|
||||||
// Workflow step agents inherit executor instructions
|
// Workflow step agents inherit executor instructions
|
||||||
const stepInstructions = await this.resolveInstructionsForRole("executor");
|
const stepInstructions = await this.resolveInstructionsForRole("executor");
|
||||||
const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions);
|
const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions);
|
||||||
@@ -3992,8 +4014,8 @@ and show an appropriate message to the user.\`
|
|||||||
cwd: worktreePath,
|
cwd: worktreePath,
|
||||||
systemPrompt: stepSystemPrompt,
|
systemPrompt: stepSystemPrompt,
|
||||||
tools: toolMode,
|
tools: toolMode,
|
||||||
defaultProvider: stepProvider,
|
defaultProvider: provider,
|
||||||
defaultModelId: stepModelId,
|
defaultModelId: modelId,
|
||||||
fallbackProvider: settings.fallbackProvider,
|
fallbackProvider: settings.fallbackProvider,
|
||||||
fallbackModelId: settings.fallbackModelId,
|
fallbackModelId: settings.fallbackModelId,
|
||||||
defaultThinkingLevel: settings.defaultThinkingLevel,
|
defaultThinkingLevel: settings.defaultThinkingLevel,
|
||||||
@@ -4001,8 +4023,11 @@ and show an appropriate message to the user.\`
|
|||||||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' using model ${describeModel(session)}${useOverride ? " (workflow step override)" : ""}`);
|
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' using model ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`);
|
||||||
await this.store.logEntry(task.id, `Workflow step '${workflowStep.name}' using model: ${describeModel(session)}${useOverride ? " (workflow step override)" : ""}`);
|
await this.store.logEntry(
|
||||||
|
task.id,
|
||||||
|
`Workflow step '${workflowStep.name}' using model: ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
let output = "";
|
let output = "";
|
||||||
session.subscribe((event) => {
|
session.subscribe((event) => {
|
||||||
@@ -4023,37 +4048,78 @@ and show an appropriate message to the user.\`
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
await promptWithFallback(
|
let timedOut = false;
|
||||||
session,
|
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
|
||||||
`Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` +
|
const timeoutPromise = new Promise<"timeout">((resolveTimeout) => {
|
||||||
`Review the work done in this worktree and evaluate it against the criteria in your instructions.`,
|
timeoutHandle = setTimeout(() => {
|
||||||
);
|
timedOut = true;
|
||||||
|
resolveTimeout("timeout");
|
||||||
|
}, timeoutMs);
|
||||||
|
});
|
||||||
|
|
||||||
checkSessionError(session);
|
try {
|
||||||
await accumulateSessionTokenUsage(this.store, task.id, session);
|
const promptPromise = promptWithFallback(
|
||||||
session.dispose();
|
session,
|
||||||
await agentLogger.flush();
|
`Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` +
|
||||||
|
`Review the work done in this worktree and evaluate it against the criteria in your instructions.`,
|
||||||
|
);
|
||||||
|
|
||||||
// Check if the output contains a revision request
|
const outcome = await Promise.race([
|
||||||
const trimmedOutput = output.trim();
|
promptPromise.then(() => "completed" as const),
|
||||||
const revisionMatch = trimmedOutput.match(/^REQUEST REVISION\s*\n*/i);
|
timeoutPromise,
|
||||||
if (revisionMatch) {
|
]);
|
||||||
// Extract the feedback after "REQUEST REVISION"
|
|
||||||
const feedbackStart = revisionMatch[0].length;
|
if (outcome === "timeout") {
|
||||||
const feedback = trimmedOutput.slice(feedbackStart).trim();
|
executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' (${attemptLabel}) timed out after ${timeoutMs}ms — disposing session`);
|
||||||
return {
|
await this.store.logEntry(
|
||||||
success: false,
|
task.id,
|
||||||
revisionRequested: true,
|
`Workflow step '${workflowStep.name}' ${attemptLabel === "primary" ? "primary" : "fallback"} model timed out after ${Math.round(timeoutMs / 1000)}s — aborting session`,
|
||||||
output: feedback,
|
);
|
||||||
};
|
try { session.dispose(); } catch { /* best-effort */ }
|
||||||
|
await agentLogger.flush();
|
||||||
|
return { success: false, error: `workflow step timed out after ${timeoutMs}ms`, timedOut: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Completed within the timeout — let any post-completion errors surface.
|
||||||
|
checkSessionError(session);
|
||||||
|
await accumulateSessionTokenUsage(this.store, task.id, session);
|
||||||
|
session.dispose();
|
||||||
|
await agentLogger.flush();
|
||||||
|
|
||||||
|
const trimmedOutput = output.trim();
|
||||||
|
const revisionMatch = trimmedOutput.match(/^REQUEST REVISION\s*\n*/i);
|
||||||
|
if (revisionMatch) {
|
||||||
|
const feedbackStart = revisionMatch[0].length;
|
||||||
|
const feedback = trimmedOutput.slice(feedbackStart).trim();
|
||||||
|
return { success: false, revisionRequested: true, output: feedback };
|
||||||
|
}
|
||||||
|
return { success: true, output };
|
||||||
|
} catch (err: unknown) {
|
||||||
|
await agentLogger.flush();
|
||||||
|
try { session.dispose(); } catch { /* best-effort */ }
|
||||||
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||||
|
return { success: false, error: errorMessage };
|
||||||
|
} finally {
|
||||||
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||||
|
// Suppress unused-variable warning; `timedOut` documents intent.
|
||||||
|
void timedOut;
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
return { success: true, output };
|
const primaryOutcome = await runOnce(primaryProvider, primaryModelId, "primary");
|
||||||
} catch (err: unknown) {
|
if (!primaryOutcome.timedOut) return primaryOutcome;
|
||||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
||||||
await agentLogger.flush();
|
if (!fallback) {
|
||||||
return { success: false, error: errorMessage };
|
executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' timed out and no fallback model is configured`);
|
||||||
|
await this.store.logEntry(
|
||||||
|
task.id,
|
||||||
|
`Workflow step '${workflowStep.name}' timed out — no fallback model configured (set settings.validatorFallbackProvider/Id or fallbackProvider/Id)`,
|
||||||
|
);
|
||||||
|
return primaryOutcome;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with fallback ${fallback.provider}/${fallback.modelId} (label=${fallback.label})`);
|
||||||
|
return runOnce(fallback.provider, fallback.modelId, "fallback");
|
||||||
}
|
}
|
||||||
|
|
||||||
private MAX_WORKTREE_RETRIES = 3;
|
private MAX_WORKTREE_RETRIES = 3;
|
||||||
@@ -5160,6 +5226,12 @@ and show an appropriate message to the user.\`
|
|||||||
const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig)
|
const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig)
|
||||||
?? extractRuntimeHint(parentAgent?.runtimeConfig);
|
?? extractRuntimeHint(parentAgent?.runtimeConfig);
|
||||||
|
|
||||||
|
// Resolve executor model via canonical lane hierarchy so child agents
|
||||||
|
// honor project executionProvider/executionModelId overrides (parity
|
||||||
|
// with main executor at the top of agentWork()).
|
||||||
|
const { provider: childExecutorProvider, modelId: childExecutorModelId } =
|
||||||
|
resolveExecutorModelPair(undefined, undefined, settings);
|
||||||
|
|
||||||
// Create child agent session
|
// Create child agent session
|
||||||
const { session: childSession } = await createResolvedAgentSession({
|
const { session: childSession } = await createResolvedAgentSession({
|
||||||
sessionPurpose: "executor",
|
sessionPurpose: "executor",
|
||||||
@@ -5168,8 +5240,8 @@ and show an appropriate message to the user.\`
|
|||||||
cwd: childWorktreePath,
|
cwd: childWorktreePath,
|
||||||
systemPrompt: childSystemPrompt,
|
systemPrompt: childSystemPrompt,
|
||||||
tools: "coding",
|
tools: "coding",
|
||||||
defaultProvider: settings.defaultProvider,
|
defaultProvider: childExecutorProvider,
|
||||||
defaultModelId: settings.defaultModelId,
|
defaultModelId: childExecutorModelId,
|
||||||
fallbackProvider: settings.fallbackProvider,
|
fallbackProvider: settings.fallbackProvider,
|
||||||
fallbackModelId: settings.fallbackModelId,
|
fallbackModelId: settings.fallbackModelId,
|
||||||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||||||
|
|||||||
Reference in New Issue
Block a user