Files
fusion/packages/engine/src/executor.ts
Fusion cced0cec9a feat(FN-3352): improve in-process runtime cleanup reliability
Adds comprehensive tests for cleanup reliability in the in-process runtime (131-line test suite) plus related coverage in the executor tests, with a small fix to the runtime implementation itself to address the reliability issue.

Fusion-Task-Id: FN-3352
2026-05-04 18:08:31 -07:00

7142 lines
321 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { exec } from "node:child_process";
import { promisify } from "node:util";
const execAsync = promisify(exec);
import { isAbsolute, join, relative, resolve as resolvePath } from "node:path";
import { existsSync } from "node:fs";
import { readFile, writeFile } from "node:fs/promises";
import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext } from "@fusion/core";
import {
buildExecutionMemoryInstructions,
getTaskMergeBlocker,
resolveAgentPrompt,
resolveProjectDefaultModel,
type RunCommandResult,
} from "@fusion/core";
import { findWorktreeUser } from "./merger.js";
import {
runVerificationCommand,
summarizeVerificationOutput,
VERIFICATION_LOG_MAX_CHARS,
type VerificationResult,
} from "./verification-utils.js";
import { generateWorktreeName, slugify } from "./worktree-names.js";
import { Type, type Static } from "@mariozechner/pi-ai";
import { describeModel, promptWithFallback, compactSessionContext } from "./pi.js";
import { accumulateSessionTokenUsage } from "./session-token-usage.js";
import { createResolvedAgentSession, extractRuntimeHint } from "./agent-session-helpers.js";
import { buildSessionSkillContext } from "./session-skill-context.js";
import { reviewStep, type ReviewVerdict } from "./reviewer.js";
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@mariozechner/pi-coding-agent";
import { PRIORITY_EXECUTE, type AgentSemaphore } from "./concurrency.js";
import { getRegisteredWorktreePaths, isGitRepository, isRegisteredGitWorktree, isUsableTaskWorktree, type WorktreePool } from "./worktree-pool.js";
import { AgentLogger } from "./agent-logger.js";
import { executorLog, reviewerLog, formatError } from "./logger.js";
import { TokenCapDetector } from "./token-cap-detector.js";
import { isUsageLimitError, checkSessionError, type UsageLimitPauser } from "./usage-limit-detector.js";
import { isTransientError, isSilentTransientError } from "./transient-error-detector.js";
import { withRateLimitRetry } from "./rate-limit-retry.js";
import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js";
import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js";
import type { PluginRunner } from "./plugin-runner.js";
import { isContextLimitError } from "./context-limit-detector.js";
import { StepSessionExecutor } from "./step-session-executor.js";
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
import type { AgentReflectionService } from "./agent-reflection.js";
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext } from "./run-audit.js";
import { evaluateSpecStaleness, getPromptPath } from "./spec-staleness.js";
import {
createDelegateTaskTool,
createGetAgentConfigTool,
createListAgentsTool,
createMemoryTools,
createReadMessagesTool,
createReflectOnPerformanceTool,
createUpdateAgentConfigTool,
createResearchTools,
createSendMessageTool,
createTaskCreateTool as sharedCreateTaskCreateTool,
createTaskDocumentReadTool as sharedCreateTaskDocumentReadTool,
createTaskDocumentWriteTool as sharedCreateTaskDocumentWriteTool,
createTaskLogTool as sharedCreateTaskLogTool,
} from "./agent-tools.js";
import { getTaskCompletionBlockerForStore } from "./task-completion.js";
import { createFusionAuthStorage, getModelRegistryModelsPath } from "./auth-storage.js";
import { createRunVerificationTool } from "./run-verification-tool.js";
import { createFallbackModelObserver } from "./fallback-model-observer.js";
// Re-export for backward compatibility (tests import from executor.ts)
export { summarizeToolArgs } from "./agent-logger.js";
export {
createDelegateTaskTool,
createGetAgentConfigTool,
createListAgentsTool,
createReadMessagesTool,
createUpdateAgentConfigTool,
createSendMessageTool,
createTaskCreateTool,
createTaskDocumentReadTool,
createTaskDocumentWriteTool,
createTaskLogTool,
delegateTaskParams,
listAgentsParams,
memoryAppendParams,
memoryGetParams,
memorySearchParams,
readMessagesParams,
sendMessageParams,
taskCreateParams,
taskLogParams,
} from "./agent-tools.js";
const STEP_STATUSES: StepStatus[] = ["pending", "in-progress", "done", "skipped"];
/** Maximum retry attempts for workflow step hard failures before giving up */
const MAX_WORKFLOW_STEP_RETRIES = 3;
/** Maximum in-session retries when an agent exits without calling fn_task_done(). */
const MAX_TASK_DONE_SESSION_RETRIES = 3;
/** Maximum todo requeues after exhausting in-session fn_task_done retries. */
const MAX_TASK_DONE_REQUEUE_RETRIES = 3;
/** How long to wait before recovering a completed task still stuck in in-progress. */
const COMPLETED_TASK_WATCHDOG_MS = 60_000;
/** How long to wait before retrying a workflow rerun handoff that never reached in-progress. */
const WORKFLOW_RERUN_WATCHDOG_MS = 15_000;
/**
* @deprecated Kept exported so existing unit tests in executor.test.ts still
* link, but no longer called from the executor. Revision feedback is applied
* as an in-place fix via `reopenLastStepForRevision` — earlier completed
* steps stay done instead of being replayed.
*/
export function determineRevisionResetStart(
steps: ReadonlyArray<{ name: string }>,
feedback: string,
): number {
const total = steps.length;
if (total === 0) return 0;
const skipPreflight = /preflight/i.test(steps[0].name);
const firstCandidate = skipPreflight ? 1 : 0;
if (firstCandidate >= total) return total;
const fb = feedback.toLowerCase();
for (let i = firstCandidate; i < total; i++) {
const tokens = steps[i].name.toLowerCase().match(/[a-z][a-z]{4,}/g) ?? [];
if (tokens.some((t) => fb.includes(t))) return i;
}
return firstCandidate;
}
const WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS = 4_000;
class NonRetryableWorktreeError extends Error {}
function truncateWorkflowScriptOutput(output: string): string {
if (output.length <= WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS) return output;
return `... output truncated to last ${WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS} characters ...\n${output.slice(-WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS)}`;
}
function configuredCommandErrorMessage(result: RunCommandResult): string {
if (result.spawnError) return result.spawnError.message;
const parts: string[] = [];
if (result.timedOut) parts.push("Timed out");
if (result.exitCode !== null) parts.push(`Exit code: ${result.exitCode}`);
if (result.signal) parts.push(`Signal: ${result.signal}`);
const stdout = result.stdout.trim();
const stderr = result.stderr.trim();
if (stdout) parts.push(`stdout: ${truncateWorkflowScriptOutput(stdout)}`);
if (stderr) parts.push(`stderr: ${truncateWorkflowScriptOutput(stderr)}`);
return parts.length ? parts.join("\n") : "Command failed";
}
async function runConfiguredCommand(
command: string,
cwd: string,
timeoutMs: number,
): Promise<RunCommandResult> {
try {
const { stdout, stderr } = await execAsync(command, {
cwd,
timeout: timeoutMs,
maxBuffer: 10 * 1024 * 1024,
encoding: "utf-8",
});
return {
stdout: stdout?.toString?.() ?? "",
stderr: stderr?.toString?.() ?? "",
exitCode: 0,
signal: null,
bufferExceeded: false,
timedOut: false,
};
} catch (error) {
const errObj = error as Record<string, unknown>;
const code = errObj?.code;
const status = typeof errObj?.status === "number" ? errObj.status : null;
const exitCode = typeof code === "number" ? code : status;
const message = String(errObj?.message ?? "");
return {
stdout: typeof (errObj?.stdout as { toString?: unknown })?.toString === "function" ? String(errObj.stdout) : "",
stderr: typeof (errObj?.stderr as { toString?: unknown })?.toString === "function" ? String(errObj.stderr) : "",
exitCode,
signal: (errObj?.signal as NodeJS.Signals | null | undefined) ?? null,
bufferExceeded:
code === "ENOBUFS"
|| code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"
|| message.includes("maxBuffer"),
timedOut:
code === "ETIMEDOUT"
|| (errObj?.killed === true && (errObj?.signal === "SIGTERM" || message.includes("timed out"))),
spawnError: code === "ENOENT" || code === "EACCES" ? (error as Error) : undefined,
};
}
}
// ── Tool parameter schemas (module-level for reuse in ToolDefinition generics) ──
const taskUpdateParams = Type.Object({
step: Type.Number({ description: "Step number (0-indexed)" }),
status: Type.Union(
STEP_STATUSES.map((s) => Type.Literal(s)),
{ description: "New status: pending, in-progress, done, or skipped" },
),
});
// taskLogParams and taskCreateParams are imported from agent-tools.ts
const taskAddDepParams = Type.Object({
task_id: Type.String({ description: "The ID of the task to depend on (e.g. \"KB-001\")" }),
confirm: Type.Optional(Type.Boolean({ description: "Set to true to confirm adding the dependency. Required because adding a dep to an in-progress task will stop execution and discard current work." })),
});
const spawnAgentParams = Type.Object({
name: Type.String({ description: "Name for the child agent" }),
role: Type.Union([
Type.Literal("triage"),
Type.Literal("executor"),
Type.Literal("reviewer"),
Type.Literal("merger"),
Type.Literal("engineer"),
Type.Literal("custom"),
], { description: "Role for the child agent" }),
task: Type.String({ description: "Task description for the child agent to execute" }),
});
/** Result returned from fn_spawn_agent tool */
interface SpawnAgentResult {
agentId: string;
name: string;
state: AgentState;
role: AgentCapability;
message: string;
}
/**
* Outcome of a single workflow step execution.
* Supports three states: pass, hard failure, or revision requested with feedback.
*/
export interface WorkflowStepOutcome {
success: boolean;
revisionRequested?: boolean;
output?: string;
error?: string;
/** Set when the call exceeded `settings.workflowStepTimeoutMs`. Signals the
* caller to escalate to the fallback model rather than treat the failure
* as a generic revision request. */
timedOut?: boolean;
}
/**
* Result of running all pre-merge workflow steps.
* Returns true if all passed, false if any hard failure, or a structured
* revision result if a revision was requested.
*/
export type WorkflowStepResult =
| { allPassed: true }
| { allPassed: false; revisionRequested: false; feedback: string; stepName: string }
| { allPassed: false; revisionRequested: true; feedback: string; stepName: string };
const reviewStepParams = Type.Object({
step: Type.Number({ description: "Step number to review" }),
type: Type.Union(
[Type.Literal("plan"), Type.Literal("code")],
{ description: 'Review type: "plan" or "code"' },
),
step_name: Type.String({ description: "Name of the step being reviewed" }),
baseline: Type.Optional(
Type.String({
description:
"Git commit SHA for code review diff baseline. " +
"Capture HEAD before starting a step and pass it here.",
}),
),
});
const EXECUTOR_SYSTEM_PROMPT = `You are a task execution agent for "fn", an AI-orchestrated task board.
You are working in a git worktree isolated from the main branch. Your job is to implement the task described in the PROMPT.md specification you're given.
## Your Role in the System
You are the primary implementation agent in Fusion.
You execute task specs in isolated worktrees, produce production-quality changes, and hand off work that can pass independent review and merge.
## Turn-ending rules — read carefully
You MUST end every turn by either:
- (a) calling another tool to make progress, OR
- (b) calling \`fn_task_done\` if the entire task is complete, OR
- (c) calling \`fn_task_done\` with a summary explaining what is blocked, if you cannot make progress for any reason
You MUST NOT end a turn by writing prose that asks the user a question, summarizes progress, or requests permission to continue. The following are FORBIDDEN turn-endings:
- "If you want, I can continue with..."
- "Should I proceed with...?"
- "Let me know if you'd like me to..."
- "Ready to move on to step N. Want me to continue?"
- Any markdown progress summary at the end of a turn instead of a tool call
If you have just finished a step's work, immediately call \`fn_task_update\` to mark the step done and continue with the next pending step in the SAME turn. Do not pause to summarize.
The user is not watching this conversation in real-time. They will read the final result. Asking permission wastes a full retry cycle and may orphan committed work.
If you genuinely cannot proceed (blocked on a dependency, missing information, or an unresolvable error), call \`fn_task_done\` with a clear explanation of what is blocked and what is needed to unblock it. Never write the question as plain prose.
## How to work
1. Read the PROMPT.md carefully — it contains your mission, steps, file scope, acceptance criteria, and Do NOT constraints
2. Before touching code, read all files listed in "Context to Read First" and understand the full step outcome
3. Check existing patterns in the codebase before introducing new structure, naming, or APIs
4. Work through each step in order
5. Write clean, production-quality code
6. Test your changes continuously
7. Commit at meaningful boundaries (step completion)
## Reporting progress via tools
You have tools to report progress. The board updates in real-time.
**Step lifecycle:**
- Before starting a step: \`fn_task_update(step=N, status="in-progress")\`
- After completing a step: \`fn_task_update(step=N, status="done")\`
- If skipping a step: \`fn_task_update(step=N, status="skipped")\`
**Logging important actions:** \`fn_task_log(message="what happened")\`
**Out-of-scope work found during execution:** \`fn_task_create(description="what needs doing")\`
When creating multiple related tasks, declare dependencies between them:
\`fn_task_create(description="load door sounds", dependencies=[])\` → returns KB-050
\`fn_task_create(description="play sound on door open/close", dependencies=["KB-050"])\`
**Discovered a dependency:** \`fn_task_add_dep(task_id="KB-XXX")\` — use when you discover mid-execution that another task must be completed first. This will return a warning first — you must call again with \`confirm=true\` to proceed. Adding a dependency stops execution, discards current work, and moves the task to triage for re-planning.
## Cross-model review via fn_review_step tool
You have a \`fn_review_step\` tool. It spawns a SEPARATE reviewer agent (different
model, read-only access) to independently assess your work.
**When to call it** — based on the Review Level in the PROMPT.md:
| Review Level | Before implementing | After implementing + committing |
|-------------|--------------------|---------------------------------|
| 0 (None) | — | — |
| 1 (Plan) | \`fn_review_step(step, "plan", step_name)\` | — |
| 2 (Plan+Code) | \`fn_review_step(step, "plan", step_name)\` | \`fn_review_step(step, "code", step_name, baseline)\` |
| 3 (Full) | plan review | code review + test review |
**Skip reviews for** Step 0 (Preflight) and the final documentation/delivery step.
**Code review flow:**
1. Before starting a step, capture baseline: \`git rev-parse HEAD\`
2. Implement the step
3. Commit
4. Call \`fn_review_step\` with the baseline SHA so the reviewer sees only your changes
**Handling verdicts:**
- **APPROVE** → proceed to next step
- **REVISE (code review)** → **enforced**. You MUST fix the issues, commit again,
and re-run \`fn_review_step(type="code")\` before the step can be marked done.
\`fn_task_update(status="done")\` will be rejected until the code review passes.
- **REVISE (plan review)** → advisory. Incorporate the feedback at your discretion
and proceed with implementation. No re-review is required.
- **RETHINK (code review)** → your code changes have been reverted and conversation rewound. Read the feedback carefully and take a fundamentally different approach. Do NOT repeat the rejected strategy.
- **RETHINK (plan review)** → conversation rewound to before the step (no git reset since no code was written). Read the feedback and take a fundamentally different approach to planning this step.
## Task Documents
You can save and retrieve named documents for this task. Use these to store planning notes, research findings, or any persistent data that should survive across sessions.
- **Save a document:** \`fn_task_document_write(key="plan", content="...")\`
- **Read a document:** \`fn_task_document_read(key="plan")\`
- **List all documents:** \`fn_task_document_read()\` (no key)
Documents are versioned — each write creates a new revision. Use meaningful keys like "plan", "notes", "research", "architecture".
## Research tools
When implementation needs external context, you may use research tools (
\`fn_research_run\`, \`fn_research_list\`, \`fn_research_get\`, \`fn_research_cancel\`) to run bounded research.
Keep runs focused and short, and persist durable conclusions into task documents (for example key="research").
If research is disabled or providers are not configured, use the actionable tool response and continue with available local context.
**IMPORTANT — Save your deliverables as documents:** When your task produces written output (documentation, specifications, reports, API references, README updates, guides, or any other content), you MUST save that content as a task document using \`fn_task_document_write\`. Use a key that describes the deliverable (e.g., key="readme", key="api-docs", key="changelog"). Do this in addition to writing the file to disk — the document persists in the task for review even after the worktree is cleaned up.
If the task's PROMPT.md includes a "Documentation Requirements" section listing files to update, save each updated file's final content as a task document with a matching key.
## Git discipline
- Commit after completing each step (not after every file change)
- Use conventional commit messages prefixed with the task ID
- When the task has a GitHub issue reference, include \`Ref: owner/repo#N\` in the commit body
- Do NOT commit broken or half-implemented code
Good commit message examples:
- \`feat(FN-1234): complete Step 2 — add retry guard for workflow step timeouts\`
- \`test(FN-1234): add regression tests for paused-session cleanup\`
Bad commit message examples:
- \`misc updates\`
- \`fix stuff\`
- \`wip\`
## Worktree Boundaries
You are running in an **isolated git worktree**. This means:
- **All code changes must be made inside the current worktree directory.** Do not modify files outside the worktree — the worktree is your isolated execution environment.
- **Exception — Project memory:** You MAY read and write to files under .fusion/memory/ at the project root to save durable project learnings (architecture patterns, conventions, pitfalls).
- **Exception — Task attachments:** You MAY read files under .fusion/tasks/{taskId}/attachments/ at the project root for context screenshots and documents attached to this task.
- **Exception — Sibling task specs:** You MAY read .fusion/tasks/{taskId}/PROMPT.md and .fusion/tasks/{taskId}/task.json at the project root (read-only) to consult dependency tasks' specifications.
- **Shell commands** run inside the worktree by default. Avoid using cd to navigate outside the worktree.
If you attempt to write to a path outside the worktree, the file tools will reject the operation with an error explaining the boundary.
## Guardrails
- **NEVER kill processes on port 4040.** Port 4040 is the production dashboard. Do not run \`kill\`, \`pkill\`, \`killall\`, or \`lsof -ti:4040 | xargs kill\` against it. If you need to start a test server, use \`--port 0\` for a random free port. If port 4040 is occupied, pick a different port — do NOT kill the occupant.
- Treat the File Scope in PROMPT.md as the expected starting scope, not a hard boundary when quality gates fail
- Read "Context to Read First" files before starting
- Follow the "Do NOT" section strictly — these are hard constraints, not suggestions
- If tests, lint, build, or typecheck fail and the fix requires touching code outside the declared File Scope, fix those failures directly and keep the repo green
- Use \`fn_task_create\` for genuinely separate follow-up work, not for mandatory fixes required to make this task land cleanly
- Update documentation listed in "Must Update" and check "Check If Affected"
- NEVER delete, remove, or gut modules, interfaces, settings, exports, or test files outside your File Scope
- NEVER remove features as "cleanup" — if something seems unused, create a task for investigation instead
- Removing code is acceptable ONLY when it is explicitly part of your task's mission
- If you remove existing functionality, you MUST create a changeset in \`.changeset/\` explaining the removal and rationale
## Spawning Child Agents
You can spawn child agents to handle parallel work or specialized sub-tasks:
**When to use \`fn_spawn_agent\`:**
- Parallel work that can be divided into independent chunks with minimal overlap
- Specialized tasks requiring different expertise or tools
- Delegation of sub-tasks whose outputs can be validated independently
**When NOT to spawn:**
- The work is small enough to finish directly in your current step
- Subtasks are tightly coupled and would create merge/cherry-pick overhead
- You have not yet clarified expected outputs and acceptance criteria for the child
**How to spawn:**
\`\`\`javascript
fn_spawn_agent({
name: "researcher",
role: "engineer",
task: "Research best practices for authentication in React applications"
})
\`\`\`
**Child agent behavior:**
- Each child runs in its own git worktree (branched from your worktree)
- Children execute autonomously and report completion
- When you end (fn_task_done), all spawned children are terminated
- Check AgentStore for spawned agent status
**Limits:**
- Max 5 spawned agents per parent by default (configurable via settings)
- Max 20 total spawned agents system-wide (configurable via settings)
## Completion
After all steps are done, lint passes, tests pass, typecheck passes, and docs are updated:
\`\`\`bash
Call \`fn_task_done()\` to signal completion.
\`\`\`
If a project build command is listed in the prompt, it is a hard completion gate:
- Run the exact build command in the current worktree before \`fn_task_done()\`
- Do not claim the build passes unless you actually ran it and got exit code 0
- If the build fails, do NOT call \`fn_task_done()\`; keep working until it passes
Lint, tests, and typecheck are also hard quality gates:
- Keep fixing failures until lint, the configured/full test suite, and typecheck all pass
- If the repository exposes a typecheck command, run it and keep fixing failures until it passes
- Do not stop at "out of scope" if additional fixes are required to restore green lint, tests, build, or typecheck
- When tests fail, first identify whether the failure is caused by your change, a pre-existing defect, or an outdated test expectation; then fix code or tests accordingly so behavior and assertions match
- Update tests when intended behavior changed; fix implementation when behavior regressed unintentionally
- **CRITICAL: Resolve ALL lint failures and test failures before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.
## Verification commands — use fn_run_verification
For ALL test/lint/build/typecheck verification, use the \`fn_run_verification\` tool, NOT raw bash.
The tool prevents your session from being killed by the inactivity watchdog during long compiles.
- Prefer **package-scoped** verification first: e.g. \`pnpm --filter @fusion/<pkg> test\` with \`scope: "package"\`. This is faster and isolated.
- Only run **workspace-scoped** verification (\`pnpm test\`, \`pnpm lint\`, \`pnpm build\` from root) at the FINAL integration step, when you are about to call \`fn_task_done\`.
- If you need to run \`pnpm install\` (e.g. you added a new package), use \`fn_run_verification\` with \`scope: "workspace"\` and \`timeoutSec: 600\`.
- If a verification command times out, do NOT blindly retry — investigate. Check for hung subprocesses, infinite test loops, or tests waiting on missing dependencies. Use \`node_modules/.modules.yaml\` presence to confirm bootstrap.
## Common Pitfalls
- Editing files outside the assigned worktree (except allowed memory/attachment paths)
- Skipping or partially running required quality gates
- Leaving TODO/FIXME placeholders instead of completing required implementation
- Introducing new patterns when existing local patterns should be reused
- Marking a step done before required review/tooling gates are satisfied`;
/** Resolve the executor system prompt from settings, falling back to the hardcoded constant. */
function getExecutorSystemPrompt(settings: Settings): string {
const customPrompt = resolveAgentPrompt("executor", settings.agentPrompts);
return customPrompt || EXECUTOR_SYSTEM_PROMPT;
}
function resolveExecutorModelPair(
taskModelProvider: string | undefined,
taskModelId: string | undefined,
settings: Partial<Settings> | undefined,
): { provider: string | undefined; modelId: string | undefined } {
if (taskModelProvider && taskModelId) {
return { provider: taskModelProvider, modelId: taskModelId };
}
if (settings?.executionProvider && settings?.executionModelId) {
return {
provider: settings.executionProvider,
modelId: settings.executionModelId,
};
}
if (settings?.executionGlobalProvider && settings?.executionGlobalModelId) {
return {
provider: settings.executionGlobalProvider,
modelId: settings.executionGlobalModelId,
};
}
if (settings?.defaultProviderOverride && settings?.defaultModelIdOverride) {
return {
provider: settings.defaultProviderOverride,
modelId: settings.defaultModelIdOverride,
};
}
if (settings?.defaultProvider && settings?.defaultModelId) {
return {
provider: settings.defaultProvider,
modelId: settings.defaultModelId,
};
}
return { provider: undefined, modelId: undefined };
}
export interface TaskExecutorOptions {
semaphore?: AgentSemaphore;
/** Worktree pool for recycling idle worktrees across tasks. */
pool?: WorktreePool;
/** Usage limit pauser — triggers global pause when API limits are detected. */
usageLimitPauser?: UsageLimitPauser;
/** Stuck task detector — monitors agent sessions for stagnation and triggers recovery. */
stuckTaskDetector?: StuckTaskDetector;
/** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */
agentStore?: import("@fusion/core").AgentStore;
/** Reflection service used to generate self-reflection insights for agents. */
reflectionService?: AgentReflectionService;
/** Plugin runner for invoking plugin hooks and providing plugin tools. */
pluginRunner?: PluginRunner;
/** MessageStore for sending messages to other agents. When provided, executor agents gain fn_send_message capability. */
messageStore?: import("@fusion/core").MessageStore;
missionStore?: MissionStore;
onSliceComplete?: (slice: Slice) => void;
onStart?: (task: Task, worktreePath: string) => void;
onComplete?: (task: Task) => void;
onError?: (task: Task, error: Error) => void;
onAgentText?: (taskId: string, delta: string) => void;
onAgentTool?: (taskId: string, toolName: string) => void;
}
export class TaskExecutor {
private activeWorktrees = new Map<string, string>();
private executing = new Set<string>();
/** Tasks currently being prepared for unpause resume, before execute() has registered them. */
private resumingUnpaused = new Set<string>();
/** Completed orphan recovery tasks currently running during startup. */
private recoveringCompleted = new Set<string>();
/** Tracks tasks whose workflow-rerun bounce is in flight (todo→in-progress).
* Prevents the task:moved handler from dispatching execute() before the
* bounce finishes its own dispatch. */
private workflowRerunPending = new Set<string>();
/** Active agent sessions per task, used to terminate on pause and inject steering. */
private activeSessions = new Map<string, {
session: AgentSession;
seenSteeringIds: Set<string>;
lastModelProvider?: string | null;
lastModelId?: string | null;
}>();
/** Active step-session executors per task (mutually exclusive with activeSessions). */
private activeStepExecutors = new Map<string, StepSessionExecutor>();
/** Active pre-merge workflow step sessions per task. */
private activeWorkflowStepSessions = new Map<string, AgentSession>();
/**
* Reviewer subagent sessions per task. Reviewers (`reviewer.ts`) create their
* own AgentSessions that aren't part of `activeSessions`/`activeStepExecutors`,
* so without this map they survive when the parent task is stopped — they
* keep producing log entries and step transitions after the user thinks they
* killed the task. Disposed alongside the main session in the move-out,
* pause, and global-pause handlers below.
*/
private activeSubagentSessions = new Map<string, Set<AgentSession>>();
/** Tasks that were paused mid-execution (to avoid marking them as "failed"). */
private pausedAborted = new Set<string>();
/** Tasks that had a dependency added mid-execution (abort + discard worktree). */
private depAborted = new Set<string>();
/** Tasks killed by stuck task detector. Value = shouldRequeue (budget not exhausted). */
private stuckAborted = new Map<string, boolean>();
/** In-memory loop recovery state per task. Keyed by taskId, not persisted.
* Tracks compact-and-resume attempt count per execute() lifecycle.
* Reset at execute() lifecycle end (finally block). */
private loopRecoveryState = new Map<string, { attempts: number; pending: boolean }>();
/** Spawned child agent IDs per parent task ID. Used for lifecycle tracking. */
private spawnedAgents = new Map<string, Set<string>>();
/** Per-task baseline of session stats used for delta persistence across repeated updates. */
private tokenUsageBaselines = new Map<string, { inputTokens: number; outputTokens: number; cachedTokens: number; totalTokens: number }>();
/** One-shot watchdogs for completed tasks that should have transitioned to in-review. */
private completedTaskWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
/** One-shot watchdogs for workflow reruns that should have bounced back to in-progress. */
private workflowRerunWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
/** Set of ephemeral spawned agent IDs with scheduled cleanup (prevents duplicate deletion attempts). */
private pendingEphemeralDeletions = new Set<string>();
/** Map of spawned agent IDs to scheduled cleanup timer handles for shutdown disposal. */
private ephemeralCleanupTimers = new Map<string, ReturnType<typeof setTimeout>>();
private async finalizeAlreadyReviewedTask(taskId: string): Promise<"merged" | "blocked" | "missing"> {
const latestTask = await this.store.getTask(taskId);
if (!latestTask || latestTask.column !== "in-review") {
return "missing";
}
const blocker = getTaskMergeBlocker(latestTask);
if (blocker) {
await this.store.logEntry(taskId, "Task already in-review; merge deferred", blocker, this.currentRunContext);
return "blocked";
}
await this.store.logEntry(
taskId,
"Task already in-review after completion — finalizing merge",
undefined,
this.currentRunContext,
);
await this.store.mergeTask(taskId);
return "merged";
}
private async getExecutionPauseLabel(): Promise<"global pause" | "engine pause" | null> {
const settings = await this.store.getSettings();
if (settings.globalPause) return "global pause";
if (settings.enginePaused) return "engine pause";
return null;
}
private async shouldDeferCompletionForGlobalPause(
taskId: string,
context: string,
): Promise<boolean> {
const settings = await this.store.getSettings();
if (!settings.globalPause) {
return false;
}
this.clearCompletedTaskWatchdog(taskId);
executorLog.log(`${taskId}: completion handoff deferred — global pause active (${context})`);
await this.store.logEntry(
taskId,
`Completion handoff deferred — global pause active (${context})`,
undefined,
this.currentRunContext,
).catch(() => undefined);
return true;
}
private async shouldDeferWorkflowStepCompletion(
taskId: string,
context: string,
): Promise<boolean> {
let latestTask: Task | null = null;
try {
latestTask = await this.store.getTask(taskId);
} catch {
latestTask = null;
}
if (latestTask?.paused || this.pausedAborted.has(taskId)) {
this.clearCompletedTaskWatchdog(taskId);
executorLog.log(`${taskId}: completion handoff deferred — task paused (${context})`);
await this.store.logEntry(
taskId,
`Completion handoff deferred — task paused (${context})`,
undefined,
this.currentRunContext,
).catch(() => undefined);
return true;
}
return this.shouldDeferCompletionForGlobalPause(taskId, context);
}
private async parkTaskAfterWorkflowStepPause(taskId: string): Promise<boolean> {
let latestTask: Task | null = null;
try {
latestTask = await this.store.getTask(taskId);
} catch {
latestTask = null;
}
if (!latestTask?.paused) {
return false;
}
executorLog.log(`${taskId}: workflow step interrupted by task pause — moving to todo`);
await this.store.logEntry(
taskId,
"Execution paused during pre-merge workflow step — moved to todo",
undefined,
this.currentRunContext,
).catch(() => undefined);
if (latestTask.column === "in-progress") {
await this.store.moveTask(taskId, "todo", { preserveResumeState: true });
}
return true;
}
/** Child agent sessions keyed by agent ID. Used for termination. */
private childSessions = new Map<string, AgentSession>();
/** Total count of currently spawned agents (across all parents). */
private totalSpawnedCount = 0;
/** Token cap detector for proactive context compaction. */
private tokenCapDetector = new TokenCapDetector();
private _modelRegistry?: ModelRegistry;
/** Current run context for mutation correlation. Set at execute() start, cleared in finally. */
private currentRunContext: RunMutationContext | undefined;
private get modelRegistry(): ModelRegistry {
if (!this._modelRegistry) {
const authStorage = createFusionAuthStorage();
this._modelRegistry = ModelRegistry.create(authStorage, getModelRegistryModelsPath());
this._modelRegistry.refresh();
}
return this._modelRegistry;
}
/** Returns the set of task IDs currently being executed. */
getExecutingTaskIds(): Set<string> {
return new Set([...this.executing, ...this.recoveringCompleted, ...this.resumingUnpaused]);
}
isEphemeralDeletionPending(agentId: string): boolean {
return this.pendingEphemeralDeletions.has(agentId);
}
disposeEphemeralTimers(): void {
const timerCount = this.ephemeralCleanupTimers.size;
for (const timerId of this.ephemeralCleanupTimers.values()) {
clearTimeout(timerId);
}
this.ephemeralCleanupTimers.clear();
this.pendingEphemeralDeletions.clear();
if (timerCount > 0) {
executorLog.log(`Cleared ${timerCount} pending spawned-agent cleanup timer(s)`);
}
}
private isBenignEphemeralDeleteRaceError(agentId: string, err: unknown): boolean {
const msg = err instanceof Error ? err.message : String(err);
const lower = msg.toLowerCase();
if (lower.includes("not found") || lower.includes("already deleted") || lower.includes("does not exist")) {
executorLog.log(`Skip spawned-agent cleanup for ${agentId}: already deleted by another pathway`);
return true;
}
return false;
}
/**
* Abort the in-flight bash subprocess (if any) on every active agent session.
*
* Invoked at runtime shutdown so detached subprocess trees spawned by agent
* bash tools — including grandchildren like vitest workers — are killed via
* pi-coding-agent's killProcessTree. Without this, when the worker is killed
* those process groups are orphaned because they're detached.
*
* Sessions are not disposed here so any near-complete agent loop still has a
* chance to wrap up during the runtime's graceful drain window.
*/
/**
* Register a subagent session (e.g. reviewer) under its parent task ID so it
* can be disposed when the parent stops. Used as the `onSessionCreated`
* callback passed to `reviewStep`.
*/
private registerSubagentSession(taskId: string, session: AgentSession): void {
let set = this.activeSubagentSessions.get(taskId);
if (!set) {
set = new Set();
this.activeSubagentSessions.set(taskId, set);
}
set.add(session);
}
/**
* Deregister a subagent session that has finished naturally. The reviewer's
* own `finally` block disposes the session — this just removes it from the
* map.
*/
private unregisterSubagentSession(taskId: string, session: AgentSession): void {
const set = this.activeSubagentSessions.get(taskId);
if (!set) return;
set.delete(session);
if (set.size === 0) this.activeSubagentSessions.delete(taskId);
}
/**
* Dispose all subagent sessions for a task and remove them from the map.
* Called by the kill paths (move-out-of-in-progress, pause, global pause)
* so subagents stop alongside the main session.
*/
private disposeSubagentsForTask(taskId: string, reason: string): void {
const set = this.activeSubagentSessions.get(taskId);
if (!set || set.size === 0) return;
executorLog.log(`${taskId}: disposing ${set.size} subagent session(s) — ${reason}`);
for (const session of set) {
try {
session.dispose();
} catch (err) {
executorLog.warn(`${taskId}: failed to dispose subagent session: ${err}`);
}
}
this.activeSubagentSessions.delete(taskId);
}
abortAllSessionBash(): void {
for (const [taskId, { session }] of this.activeSessions) {
try {
session.abortBash();
} catch (err) {
executorLog.warn(`abortAllSessionBash: failed for task ${taskId}: ${err}`);
}
}
for (const [agentId, session] of this.childSessions) {
try {
session.abortBash();
} catch (err) {
executorLog.warn(`abortAllSessionBash: failed for child agent ${agentId}: ${err}`);
}
}
for (const [taskId, stepExecutor] of this.activeStepExecutors) {
try {
stepExecutor.abortAllSessionBash();
} catch (err) {
executorLog.warn(`abortAllSessionBash: failed for step executor ${taskId}: ${err}`);
}
}
}
/**
* @param store — Task store instance (also used to listen for events)
* @param rootDir — Project root directory
* @param options — Executor configuration
*
* Listens for `task:moved` to auto-execute tasks moved to `in-progress`,
* `task:updated` to terminate agent sessions when individual tasks are paused,
* and `settings:updated` to terminate **all** active agent sessions when
* `globalPause` transitions from `false` to `true`. `enginePaused` only
* prevents new work dispatch — running sessions continue to completion.
* Paused tasks are moved back to `todo` rather than marked as `failed`.
*/
constructor(
private store: TaskStore,
private rootDir: string,
private options: TaskExecutorOptions = {},
) {
executorLog.log(`TaskExecutor constructed (rootDir=${rootDir}, hasSemaphore=${!!options.semaphore}, hasStuckDetector=${!!options.stuckTaskDetector})`);
store.on("task:moved", ({ task, from, to }) => {
executorLog.log(`[event:task:moved] ${task.id}: ${from} → ${to}`);
if (to === "in-progress") {
this.clearWorkflowRerunWatchdog(task.id);
executorLog.log(`[event:task:moved] Initiating execute() for ${task.id}`);
void (async () => {
const taskForExecution = await this.resetMergeStateIfNeeded(task, from);
await this.execute(taskForExecution);
})().catch((err) =>
executorLog.error(`Failed to start ${task.id}:`, err),
);
} else if (from === "in-progress") {
this.clearCompletedTaskWatchdog(task.id);
// Task moved away from in-progress — terminate any active sessions
if (this.activeSessions.has(task.id)) {
executorLog.log(`${task.id} moved from in-progress to ${to} — terminating agent session`);
this.pausedAborted.add(task.id);
this.options.stuckTaskDetector?.untrackTask(task.id);
const { session } = this.activeSessions.get(task.id)!;
session.dispose();
this.activeSessions.delete(task.id);
}
if (this.activeStepExecutors.has(task.id)) {
executorLog.log(`${task.id} moved from in-progress to ${to} — terminating step sessions`);
this.pausedAborted.add(task.id);
this.options.stuckTaskDetector?.untrackTask(task.id);
const stepExecutor = this.activeStepExecutors.get(task.id)!;
stepExecutor.terminateAllSessions().catch((err) =>
executorLog.error(`Failed to terminate step sessions for ${task.id}:`, err),
);
this.activeStepExecutors.delete(task.id);
}
if (this.activeWorkflowStepSessions.has(task.id)) {
executorLog.log(`${task.id} moved from in-progress to ${to} — terminating workflow step session`);
this.pausedAborted.add(task.id);
this.options.stuckTaskDetector?.untrackTask(task.id);
const workflowSession = this.activeWorkflowStepSessions.get(task.id)!;
const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise<void> };
if (typeof sessionWithAbort.abort === "function") {
void sessionWithAbort.abort().catch((err) => {
executorLog.warn(`Failed to abort workflow step session for ${task.id}: ${err}`);
});
}
workflowSession.dispose();
this.activeWorkflowStepSessions.delete(task.id);
}
// Reviewer subagents run in their own sessions outside `activeSessions`
// and `activeStepExecutors`, so the loops above don't reach them.
// Without this, a reviewer keeps running (and emitting verdicts that
// trigger step transitions) even after the parent task was moved out
// of in-progress.
this.disposeSubagentsForTask(task.id, `parent moved from in-progress to ${to}`);
// Clean up all in-memory state for this task so nothing leaks across runs.
// This prevents zombie state from persisting when a task moves away from
// in-progress while execute() is still unwinding, or when the scheduler
// moves a task out before the executor's task:moved fires.
this.loopRecoveryState.delete(task.id);
this.spawnedAgents.delete(task.id);
this.stuckAborted.delete(task.id);
}
});
// When a task is paused while executing, terminate the agent session.
// When steering comments are added during execution, inject them into the running session.
//
// Real-time steering comment injection mechanism:
// 1. When execution starts, we initialize seenSteeringIds with all existing comment IDs
// 2. On each task:updated event, we check if there are new comments not in seenSteeringIds
// 3. New comments are injected via session.steer() which queues them for delivery
// after the current assistant turn completes (before the next LLM call)
// 4. Comments are marked as seen BEFORE injection to prevent retry loops on failure
// 5. Each injection is logged to the task for user visibility
store.on("task:updated", async (task) => {
try {
// Handle pause - terminate the agent session or step sessions
if (task.paused && this.activeSessions.has(task.id)) {
executorLog.log(`Pausing ${task.id} — terminating agent session`);
this.pausedAborted.add(task.id);
this.options.stuckTaskDetector?.untrackTask(task.id);
const { session } = this.activeSessions.get(task.id)!;
session.dispose();
// Also clean up in-memory state to prevent leaks if the task is later unpaused
this.loopRecoveryState.delete(task.id);
this.spawnedAgents.delete(task.id);
this.stuckAborted.delete(task.id);
this.disposeSubagentsForTask(task.id, "task paused");
return;
}
if (task.paused && this.activeStepExecutors.has(task.id)) {
executorLog.log(`Pausing ${task.id} — terminating step sessions`);
this.pausedAborted.add(task.id);
this.options.stuckTaskDetector?.untrackTask(task.id);
const stepExecutor = this.activeStepExecutors.get(task.id)!;
await stepExecutor.terminateAllSessions();
// Also clean up in-memory state to prevent leaks if the task is later unpaused
this.loopRecoveryState.delete(task.id);
this.spawnedAgents.delete(task.id);
this.stuckAborted.delete(task.id);
this.disposeSubagentsForTask(task.id, "task paused");
return;
}
if (task.paused && this.activeWorkflowStepSessions.has(task.id)) {
executorLog.log(`Pausing ${task.id} — terminating workflow step session`);
this.pausedAborted.add(task.id);
this.options.stuckTaskDetector?.untrackTask(task.id);
const workflowSession = this.activeWorkflowStepSessions.get(task.id)!;
const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise<void> };
if (typeof sessionWithAbort.abort === "function") {
await sessionWithAbort.abort().catch((err) =>
executorLog.warn(`Failed to abort workflow step session for pause ${task.id}: ${err}`),
);
}
workflowSession.dispose();
this.activeWorkflowStepSessions.delete(task.id);
this.loopRecoveryState.delete(task.id);
this.spawnedAgents.delete(task.id);
this.stuckAborted.delete(task.id);
this.disposeSubagentsForTask(task.id, "task paused");
return;
}
// Handle unpause of an in-progress task with no active session.
// This covers orphaned states (e.g., engine restarted while task was
// paused in-progress) where the task needs to resume execution.
// The executing/resuming guards prevent duplicate runs.
if (
!task.paused
&& task.column === "in-progress"
&& !this.activeSessions.has(task.id)
&& !this.activeStepExecutors.has(task.id)
&& !this.activeWorkflowStepSessions.has(task.id)
) {
if (
!this.executing.has(task.id)
&& !this.resumingUnpaused.has(task.id)
&& !this.recoveringCompleted.has(task.id)
) {
const pauseLabel = await this.getExecutionPauseLabel();
if (pauseLabel) {
executorLog.log(`Skipping unpause resume for ${task.id} — ${pauseLabel} active`);
return;
}
if (this.isTaskWorkComplete(task) && !task.mergeDetails) {
this.recoveringCompleted.add(task.id);
executorLog.log(`${task.id} unpaused with completed work and no session — recovering directly to in-review`);
void this.recoverCompletedTask(task)
.catch((err) =>
executorLog.error(`Failed to recover completed unpaused task ${task.id}:`, err),
)
.finally(() => {
this.recoveringCompleted.delete(task.id);
});
return;
}
this.resumingUnpaused.add(task.id);
executorLog.log(`Unpaused ${task.id} in-progress with no session — resuming execution`);
try {
await this.clearResumeFailureState(task);
await this.store.logEntry(task.id, "Resuming execution after unpause", undefined, this.currentRunContext);
await this.recoverApprovedStepsOnResume(task.id);
} catch (clearErr) {
executorLog.warn(`${task.id} clearResumeFailureState failed during unpause: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`);
}
this.execute(task)
.catch((err) =>
executorLog.error(`Failed to resume unpaused ${task.id}:`, err),
)
.finally(() => {
this.resumingUnpaused.delete(task.id);
});
}
return;
}
// Handle executor model hot-swap on active single-session executions
if (this.activeSessions.has(task.id) && !task.paused) {
const activeEntry = this.activeSessions.get(task.id)!;
const providerChanged = task.modelProvider !== activeEntry.lastModelProvider;
const modelIdChanged = task.modelId !== activeEntry.lastModelId;
if (providerChanged || modelIdChanged) {
activeEntry.lastModelProvider = task.modelProvider;
activeEntry.lastModelId = task.modelId;
const settings = await this.store.getSettings();
// Resolve model using canonical lane hierarchy for hot-swap
const { provider: newProvider, modelId: newModelId } = resolveExecutorModelPair(
task.modelProvider,
task.modelId,
settings,
);
if (newProvider && newModelId) {
try {
const model = this.modelRegistry.find(newProvider, newModelId);
if (model) {
await activeEntry.session.setModel(model);
executorLog.log(`${task.id}: executor model hot-swapped to ${newProvider}/${newModelId}`);
await this.store.logEntry(task.id, `Model changed to ${newProvider}/${newModelId}`, undefined, this.currentRunContext);
} else {
executorLog.log(`${task.id}: model ${newProvider}/${newModelId} not found in registry for hot-swap`);
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`${task.id}: failed to hot-swap model: ${errorMessage}`);
await this.store.logEntry(task.id, `Model change failed: ${errorMessage}`, undefined, this.currentRunContext);
}
}
}
}
// Handle steering comments - inject new ones into the running session
// Only process if session is active (activeSessions check is sufficient
// since entries are only added when a task is in-progress)
if (this.activeSessions.has(task.id) && task.steeringComments) {
const activeSession = this.activeSessions.get(task.id)!;
const { session, seenSteeringIds } = activeSession;
// Find new steering comments that haven't been seen yet
const newComments = task.steeringComments.filter(c => !seenSteeringIds.has(c.id));
if (newComments.length > 0) {
for (const comment of newComments) {
const summary = comment.text.length > 80
? comment.text.slice(0, 80) + "..."
: comment.text;
// Mark as seen BEFORE attempting injection to prevent retry loops on failure
seenSteeringIds.add(comment.id);
// Format and inject the comment
const commentMessage = formatCommentForInjection(comment);
try {
executorLog.log(`Injecting comment into ${task.id}: ${summary}`);
await session.steer(commentMessage);
executorLog.log(`Successfully injected comment into ${task.id}`);
// Log to the task that comment was received
await this.store.logEntry(
task.id,
`Comment received mid-execution: ${summary}`,
`by ${comment.author}`
);
} catch (err) {
executorLog.error(`Failed to inject comment for ${task.id}:`, err);
// Comment is already marked as seen - we won't retry to avoid spamming
// the agent with failed injections. The error is logged for debugging.
}
}
// After injecting comments, check for review handoff intent
// Only detect handoff in agent-authored comments when policy is enabled
const settings = await this.store.getSettings();
if (settings.reviewHandoffPolicy === "comment-triggered") {
const agentComments = newComments.filter(c => c.author !== "user");
for (const comment of agentComments) {
if (detectReviewHandoffIntent(comment.text)) {
executorLog.log(`Review handoff detected in ${task.id}: ${comment.text.slice(0, 50)}...`);
await this.executeReviewHandoff(task, session, activeSession);
return; // Exit early - handoff handles session disposal
}
}
}
}
}
} catch (err) {
executorLog.error("Uncaught error in task:updated listener:", err);
}
});
// When globalPause transitions from false → true, terminate all active agent sessions.
store.on("settings:updated", ({ settings, previous }) => {
if (settings.globalPause && !previous.globalPause) {
// Dispose every reviewer subagent across every task. The per-task loops
// below handle main + step sessions; reviewers live in their own map
// and would otherwise outlive the global pause.
for (const taskId of [...this.activeSubagentSessions.keys()]) {
this.disposeSubagentsForTask(taskId, "global pause");
}
for (const [taskId, { session }] of this.activeSessions) {
executorLog.log(`Global pause — terminating agent session for ${taskId}`);
this.pausedAborted.add(taskId);
this.options.stuckTaskDetector?.untrackTask(taskId);
// abort() interrupts any in-flight LLM stream / tool call;
// dispose() then releases session resources.
const sessionWithAbort = session as unknown as { abort?: () => Promise<void> };
if (typeof sessionWithAbort.abort === "function") {
void sessionWithAbort.abort().catch((err) => {
executorLog.warn(`Failed to abort agent session for ${taskId}: ${err}`);
});
}
session.dispose();
// Clean up all in-memory state so nothing leaks when tasks are later unpaused
this.loopRecoveryState.delete(taskId);
this.spawnedAgents.delete(taskId);
this.stuckAborted.delete(taskId);
}
for (const [taskId, stepExecutor] of this.activeStepExecutors) {
executorLog.log(`Global pause — terminating step sessions for ${taskId}`);
this.pausedAborted.add(taskId);
this.options.stuckTaskDetector?.untrackTask(taskId);
stepExecutor.terminateAllSessions().catch(err =>
executorLog.warn(`Failed to terminate step sessions for global pause ${taskId}: ${err}`)
);
// Clean up all in-memory state so nothing leaks when tasks are later unpaused
this.loopRecoveryState.delete(taskId);
this.spawnedAgents.delete(taskId);
this.stuckAborted.delete(taskId);
}
for (const [taskId, workflowSession] of this.activeWorkflowStepSessions) {
executorLog.log(`Global pause — terminating workflow step session for ${taskId}`);
this.pausedAborted.add(taskId);
this.options.stuckTaskDetector?.untrackTask(taskId);
const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise<void> };
if (typeof sessionWithAbort.abort === "function") {
void sessionWithAbort.abort().catch((err) => {
executorLog.warn(`Failed to abort workflow step session for ${taskId}: ${err}`);
});
}
workflowSession.dispose();
this.activeWorkflowStepSessions.delete(taskId);
this.loopRecoveryState.delete(taskId);
this.spawnedAgents.delete(taskId);
this.stuckAborted.delete(taskId);
}
}
});
}
/**
* Check whether a task's work is complete — all steps are done or skipped.
* Used to detect tasks that called fn_task_done() but never transitioned to in-review
* (e.g., killed by stuck detector after fn_task_done but before moveTask).
*/
private isTaskWorkComplete(task: Task): boolean {
if (task.steps.length === 0) return false;
return task.steps.every((s) => s.status === "done" || s.status === "skipped");
}
private async resetMergeStateIfNeeded(task: Task, from: Task["column"]): Promise<Task> {
if (from !== "in-review" && from !== "done") {
return task;
}
const hasMergeEvidence = Boolean(task.mergeDetails)
|| (task.mergeRetries ?? 0) > 0
|| (task.verificationFailureCount ?? 0) > 0
|| task.status === "merging"
|| task.status === "merging-pr"
|| task.status === "merging-fix";
if (!hasMergeEvidence) {
return task;
}
return this.cleanupMergeStateForReverification(
task,
`Task returned to in-progress from ${from} column — resetting verification steps and merge state for re-verification`,
{
// Keep deterministic merge-verification bounce budget across remediation
// cycles. Status may be cleared by intermediate paths, so the counter is
// the canonical signal once a bounce has started.
preserveVerificationFailureCount: (task.verificationFailureCount ?? 0) > 0,
},
);
}
private async cleanupMergeStateForReverification(
task: Task,
logMessage: string,
options?: { preserveVerificationFailureCount?: boolean },
): Promise<Task> {
await this.store.updateTask(task.id, {
mergeDetails: null,
mergeRetries: 0,
verificationFailureCount: options?.preserveVerificationFailureCount ? task.verificationFailureCount ?? 0 : 0,
workflowStepResults: [],
});
const refreshedTask = await this.store.getTask(task.id);
const steps = refreshedTask.steps ?? [];
if (steps.length > 0) {
const allStepsComplete = this.isTaskWorkComplete(refreshedTask);
if (allStepsComplete) {
await this.reopenLastStepForRevision(task.id, refreshedTask);
} else {
const resetIndexes = new Set<number>();
for (let i = 0; i < steps.length; i++) {
const name = steps[i].name.toLowerCase();
if (/testing|verification/.test(name) || /documentation|delivery/.test(name)) {
resetIndexes.add(i);
}
}
if (resetIndexes.size === 0) {
const reopened = await this.reopenLastStepForRevision(task.id, refreshedTask);
if (reopened) {
resetIndexes.add(reopened.index);
}
} else {
for (const index of resetIndexes) {
if (steps[index].status !== "pending") {
await this.store.updateStep(task.id, index, "pending");
}
}
const earliestIndex = Math.min(...Array.from(resetIndexes));
await this.store.updateTask(task.id, { currentStep: earliestIndex });
}
}
}
await this.store.logEntry(task.id, logMessage, undefined, this.currentRunContext);
return this.store.getTask(task.id);
}
private isNoProgressNoTaskDoneFailure(task: Task): boolean {
return task.status === "failed" &&
task.error?.includes("without calling fn_task_done") === true &&
task.steps.every((step) => step.status === "pending");
}
private async clearResumeFailureState(task: Task): Promise<void> {
const updates: { status?: null; error?: null; blockedBy?: null } = {};
if (task.status === "failed" || task.error) {
updates.status = null;
updates.error = null;
}
// Pre-dispatch gating state must not survive into a resumed in-progress run.
// The scheduler sets status="queued" + blockedBy on dep/file-scope conflicts
// (scheduler.ts:618, 660) and clears them on the todo→in-progress transition
// (scheduler.ts:696). Resume paths (unpause, drift recovery, engine restart)
// bypass that clear, so a task can end up actively executing while still
// labeled "queued" in the UI.
if (task.status === "queued") {
updates.status = null;
}
if (task.blockedBy) {
updates.blockedBy = null;
}
if (Object.keys(updates).length > 0) {
await this.store.updateTask(task.id, updates);
}
}
private clearCompletedTaskWatchdog(taskId: string): void {
const handle = this.completedTaskWatchdogs.get(taskId);
if (!handle) return;
clearTimeout(handle);
this.completedTaskWatchdogs.delete(taskId);
}
private clearWorkflowRerunWatchdog(taskId: string): void {
const handle = this.workflowRerunWatchdogs.get(taskId);
if (!handle) return;
clearTimeout(handle);
this.workflowRerunWatchdogs.delete(taskId);
}
private scheduleCompletedTaskWatchdog(taskId: string, trigger: string): void {
this.clearCompletedTaskWatchdog(taskId);
const handle = setTimeout(async () => {
this.completedTaskWatchdogs.delete(taskId);
// Claim recovery slot atomically (synchronously) before any async work.
// Without this, two paths can pass the in-flight guards on the same
// event-loop turn and both call recoverCompletedTask() concurrently.
if (
this.recoveringCompleted.has(taskId)
|| this.executing.has(taskId)
|| this.activeSessions.has(taskId)
|| this.activeStepExecutors.has(taskId)
|| this.activeWorkflowStepSessions.has(taskId)
|| this.resumingUnpaused.has(taskId)
) {
return;
}
this.recoveringCompleted.add(taskId);
try {
const pauseLabel = await this.getExecutionPauseLabel();
if (pauseLabel) {
return;
}
let currentTask: Task | null = null;
try {
currentTask = await this.store.getTask(taskId);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: completed-task watchdog could not read latest task state: ${errorMessage}`);
return;
}
if (!currentTask || currentTask.column !== "in-progress" || currentTask.paused) {
return;
}
if (!this.isTaskWorkComplete(currentTask)) {
return;
}
executorLog.warn(
`${taskId}: completed-task watchdog fired after ${COMPLETED_TASK_WATCHDOG_MS / 1000}s ` +
`(${trigger}) — attempting direct recovery to in-review`,
);
await this.store.logEntry(
taskId,
`Watchdog: task remained in-progress ${COMPLETED_TASK_WATCHDOG_MS / 1000}s after ${trigger} — attempting direct recovery to in-review`,
).catch(() => undefined);
const recovered = await this.recoverCompletedTask(currentTask);
if (!recovered) {
await this.store.logEntry(
taskId,
"Watchdog recovery attempt could not finalize completed task — leaving for follow-up recovery",
).catch(() => undefined);
}
} finally {
this.recoveringCompleted.delete(taskId);
}
}, COMPLETED_TASK_WATCHDOG_MS);
this.completedTaskWatchdogs.set(taskId, handle);
}
/**
* Result of a workflow-rerun bounce attempt.
*
* - `bounced` — the move sequence completed successfully and the task is
* back in `in-progress` ready for re-execution.
* - `skipped-pending` — another bounce for the same task is mid-flight;
* this attempt is a no-op. Callers (notably the watchdog) must NOT log
* this as a successful retry, since the original bounce may itself be
* stuck.
*/
private async performWorkflowRerunBounce(
taskId: string,
worktreePath: string,
preserveResumeState: boolean = true,
): Promise<"bounced" | "skipped-pending" | "deferred-paused"> {
const pauseLabel = await this.getExecutionPauseLabel();
if (pauseLabel) {
executorLog.log(`${taskId}: workflow rerun deferred — ${pauseLabel} active`);
return "deferred-paused";
}
// Re-entry guard: if a previous bounce for the same task is still
// mid-flight (e.g., the watchdog fired before the original sequence
// completed), skip rather than racing two concurrent moveTask sequences.
if (this.workflowRerunPending.has(taskId)) {
executorLog.warn(`${taskId}: workflow rerun bounce already in flight — skipping re-entry`);
return "skipped-pending";
}
this.workflowRerunPending.add(taskId);
try {
// moveTask(in-progress → todo) clears `task.worktree`; restore it before
// the return trip so the dashboard never renders the task under
// "Unassigned" and self-healing can't reclaim the worktree as idle.
const latestTask = await this.store.getTask(taskId);
if (!latestTask) {
throw new Error("task missing during workflow rerun bounce");
}
if (latestTask.paused) {
executorLog.log(`${taskId}: workflow rerun deferred — task is paused`);
return "deferred-paused";
}
if (latestTask.column === "in-progress") {
const originalExecutionStartedAt = latestTask.executionStartedAt;
// Preserve step progress across the in-progress → todo hop:
// moveTask's default reopen-to-todo path resets every step to
// pending and rewrites PROMPT.md checkboxes, which would discard
// the partial progress this bounce is supposed to retry on top of.
if (preserveResumeState) {
await this.store.moveTask(taskId, "todo", { preserveResumeState: true });
} else {
await this.store.moveTask(taskId, "todo");
}
await this.store.updateTask(taskId, {
worktree: worktreePath,
executionStartedAt: originalExecutionStartedAt ?? null,
});
const pauseLabelAfterTodo = await this.getExecutionPauseLabel();
if (pauseLabelAfterTodo) {
executorLog.log(`${taskId}: workflow rerun parked in todo — ${pauseLabelAfterTodo} became active during bounce`);
return "deferred-paused";
}
await this.store.moveTask(taskId, "in-progress");
return "bounced";
}
if (latestTask.column === "todo") {
await this.store.updateTask(taskId, { worktree: worktreePath });
const pauseLabelBeforeResume = await this.getExecutionPauseLabel();
if (pauseLabelBeforeResume) {
executorLog.log(`${taskId}: workflow rerun parked in todo — ${pauseLabelBeforeResume} became active before resume`);
return "deferred-paused";
}
await this.store.moveTask(taskId, "in-progress");
return "bounced";
}
throw new Error(`task is in '${latestTask.column}', cannot bounce to in-progress`);
} finally {
this.workflowRerunPending.delete(taskId);
}
}
private scheduleWorkflowRerun(
taskId: string,
worktreePath: string,
successMessage: string,
preserveResumeState: boolean = true,
): void {
this.clearWorkflowRerunWatchdog(taskId);
setTimeout(async () => {
try {
const outcome = await this.performWorkflowRerunBounce(taskId, worktreePath, preserveResumeState);
if (outcome === "bounced") {
executorLog.log(successMessage);
} else if (outcome === "skipped-pending") {
executorLog.warn(`${taskId}: rerun bounce skipped — another bounce already in flight`);
} else {
executorLog.log(`${taskId}: rerun bounce deferred while pause is active`);
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`${taskId}: failed to schedule rerun bounce: ${errorMessage}`);
}
}, 0);
const watchdog = setTimeout(async () => {
this.workflowRerunWatchdogs.delete(taskId);
const pauseLabel = await this.getExecutionPauseLabel();
if (pauseLabel) {
executorLog.log(`${taskId}: workflow rerun watchdog skipped — ${pauseLabel} active`);
return;
}
let currentTask: Task | null = null;
try {
currentTask = await this.store.getTask(taskId);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: workflow rerun watchdog could not read latest task state: ${errorMessage}`);
return;
}
if (!currentTask || currentTask.paused || currentTask.column === "in-progress") {
return;
}
executorLog.warn(
`${taskId}: workflow rerun watchdog fired after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s ` +
`— task is still ${currentTask.column}; retrying handoff once`,
);
await this.store.logEntry(
taskId,
`Watchdog: workflow rerun handoff stalled for ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s ` +
`(still ${currentTask.column}) — retrying once`,
).catch(() => undefined);
try {
const outcome = await this.performWorkflowRerunBounce(taskId, worktreePath, preserveResumeState);
if (outcome === "bounced") {
executorLog.warn(`${taskId}: workflow rerun watchdog retry succeeded`);
} else if (outcome === "skipped-pending") {
// The original bounce is still mid-flight, which means *it* is the
// one that's hung — not us. Log honestly so operators don't see a
// false "succeeded" message while the task is actually stranded.
executorLog.error(
`${taskId}: workflow rerun watchdog retry skipped — original bounce still in flight after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s; task may be stuck`,
);
await this.store.logEntry(
taskId,
`Workflow rerun watchdog retry skipped — original bounce still in flight after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s; task may be stuck`,
).catch(() => undefined);
} else {
executorLog.log(`${taskId}: workflow rerun watchdog retry deferred while pause is active`);
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`${taskId}: workflow rerun watchdog retry failed: ${errorMessage}`);
}
}, WORKFLOW_RERUN_WATCHDOG_MS);
this.workflowRerunWatchdogs.set(taskId, watchdog);
}
private async shouldFinalizeCompletedTask(taskId: string, taskDone: boolean): Promise<boolean> {
const task = await this.store.getTask(taskId);
const completionBlocker = await this.getTaskCompletionBlocker(task);
if (completionBlocker) {
executorLog.log(`${taskId} completion blocked — ${completionBlocker}`);
return false;
}
if (taskDone) return true;
return this.isTaskWorkComplete(task);
}
private async getTaskCompletionBlocker(task: Task): Promise<string | undefined> {
return getTaskCompletionBlockerForStore(this.store, task);
}
private accumulateTokenUsage(
existing: TaskTokenUsage | undefined,
delta: Pick<TaskTokenUsage, "inputTokens" | "outputTokens" | "cachedTokens" | "totalTokens"> | undefined,
timestamp = new Date().toISOString(),
): TaskTokenUsage | undefined {
if (!delta) return existing;
const merged: TaskTokenUsage = {
inputTokens: (existing?.inputTokens ?? 0) + delta.inputTokens,
outputTokens: (existing?.outputTokens ?? 0) + delta.outputTokens,
cachedTokens: (existing?.cachedTokens ?? 0) + delta.cachedTokens,
totalTokens: (existing?.totalTokens ?? 0) + delta.totalTokens,
firstUsedAt: existing?.firstUsedAt ?? timestamp,
lastUsedAt: timestamp,
};
return merged;
}
private async extractSessionTokenUsage(
session: AgentSession | undefined,
): Promise<Pick<TaskTokenUsage, "inputTokens" | "outputTokens" | "cachedTokens" | "totalTokens"> | undefined> {
if (!session) return undefined;
try {
const statsResult = (session as AgentSession & {
getSessionStats?: () =>
| {
tokens?: {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
};
}
| Promise<{
tokens?: {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
};
}>;
}).getSessionStats?.();
const stats = await Promise.resolve(statsResult);
const tokens = stats?.tokens;
if (!tokens) return undefined;
const inputTokens = tokens.input ?? 0;
const outputTokens = tokens.output ?? 0;
const cacheReadTokens = tokens.cacheRead ?? 0;
const cacheWriteTokens = tokens.cacheWrite ?? 0;
const cachedTokens = cacheReadTokens + cacheWriteTokens;
const totalTokens = tokens.total ?? (inputTokens + outputTokens + cachedTokens);
return {
inputTokens,
outputTokens,
cachedTokens,
totalTokens,
};
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
executorLog.warn(`Failed to read session stats for token usage: ${message}`);
return undefined;
}
}
private async persistTokenUsage(taskId: string, session?: AgentSession): Promise<void> {
const activeSession = session ?? this.activeSessions.get(taskId)?.session;
const currentUsage = await this.extractSessionTokenUsage(activeSession);
if (!currentUsage) return;
const baseline = this.tokenUsageBaselines.get(taskId);
this.tokenUsageBaselines.set(taskId, currentUsage);
const delta = baseline
? {
inputTokens: Math.max(0, currentUsage.inputTokens - baseline.inputTokens),
outputTokens: Math.max(0, currentUsage.outputTokens - baseline.outputTokens),
cachedTokens: Math.max(0, currentUsage.cachedTokens - baseline.cachedTokens),
totalTokens: Math.max(0, currentUsage.totalTokens - baseline.totalTokens),
}
: currentUsage;
if (
delta.inputTokens === 0
&& delta.outputTokens === 0
&& delta.cachedTokens === 0
&& delta.totalTokens === 0
) {
return;
}
const task = await this.store.getTask(taskId);
const merged = this.accumulateTokenUsage(task.tokenUsage, delta);
if (!merged) return;
await this.store.updateTask(taskId, { tokenUsage: merged });
}
/**
* Execute a review handoff: move the task to in-review column with
* awaiting-user-review status, assign the requesting user, and dispose
* the agent session.
*/
private async executeReviewHandoff(
task: Task,
_session: AgentSession,
_sessionEntry: { session: AgentSession; seenSteeringIds: Set<string>; lastModelProvider?: string | null; lastModelId?: string | null },
): Promise<void> {
try {
executorLog.log(`Executing review handoff for ${task.id}`);
// Log the handoff event
await this.store.logEntry(
task.id,
"Review handoff requested by agent — moving to in-review for user review",
undefined,
this.currentRunContext
);
// Update task with awaiting-user-review status and assignee
// Use a single updateTask call for atomicity
await this.store.updateTask(
task.id,
{
status: "awaiting-user-review",
assigneeUserId: "requesting-user",
},
this.currentRunContext
);
// Move the task to in-review column (this will also emit task:moved event)
// The task:moved handler will clean up activeSessions
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
// Dispose the agent session (this may already be done by task:moved handler)
// but we do it here to be explicit
if (this.activeSessions.has(task.id)) {
const { session: activeSession } = this.activeSessions.get(task.id)!;
activeSession.dispose();
this.activeSessions.delete(task.id);
}
// Untrack from stuck detector
this.options.stuckTaskDetector?.untrackTask(task.id);
executorLog.log(`Review handoff complete for ${task.id} — task moved to in-review`);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to execute review handoff for ${task.id}: ${errorMessage}`);
}
}
/**
* Fast-path a completed task directly to in-review without spawning a new agent.
* Captures modified files, runs workflow steps, and transitions the task.
*
* @returns true if the task was successfully transitioned, false otherwise.
*/
async recoverCompletedTask(task: Task): Promise<boolean> {
try {
if (
this.executing.has(task.id)
|| this.activeSessions.has(task.id)
|| this.activeStepExecutors.has(task.id)
|| this.activeWorkflowStepSessions.has(task.id)
|| this.resumingUnpaused.has(task.id)
) {
executorLog.log(`${task.id}: skipping recoverCompletedTask — task has active execution in flight`);
return false;
}
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) {
executorLog.log(
`${task.id}: skipping recoverCompletedTask — ${
settings.globalPause ? "global pause" : "engine pause"
} active`,
);
return false;
}
// Capture modified files if the worktree still exists
if (task.worktree && existsSync(task.worktree)) {
const modifiedFiles = await this.captureModifiedFiles(task.worktree, task.baseCommitSha);
if (modifiedFiles.length > 0) {
await this.store.updateTask(task.id, { modifiedFiles });
executorLog.log(`${task.id}: recovered ${modifiedFiles.length} modified files`);
}
// Run workflow steps before transitioning — skip in fast mode
if (task.executionMode !== "fast") {
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps during completed-task recovery")) {
return false;
}
const workflowResult = await this.runWorkflowSteps(task, task.worktree, settings);
if (workflowResult === "deferred-paused") {
if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
}
return false;
}
if (!workflowResult.allPassed) {
// For recovery path, treat any failure (including revision) as hard failure
// Send back to in-progress so executor can attempt to fix the issues
await this.sendTaskBackForFix(task, task.worktree!, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed during recovery", false);
return true; // Still transitioned out of in-progress
}
} else {
executorLog.log(`${task.id}: fast mode — skipping workflow steps on auto-recovery`);
}
}
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition during completed-task recovery")) {
return false;
}
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
this.clearCompletedTaskWatchdog(task.id);
await this.store.logEntry(task.id, "Auto-recovered: task work was complete but stuck in in-progress — moved to in-review");
executorLog.log(`✓ ${task.id} auto-recovered completed task → in-review`);
this.options.onComplete?.(task);
return true;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to recover completed task ${task.id}: ${errorMessage}`);
return false;
}
}
/**
* Auto-revive an `in-review` task whose pre-merge workflow step(s) failed, by
* replaying the same send-back-for-fix flow the executor uses during a live
* run. Invoked by SelfHealingManager's `recoverReviewTasksWithFailedPreMergeSteps`
* scan when a task is parked in review with a failed pre-merge step and no
* active session.
*
* Picks the latest failed pre-merge workflow step result (there is usually only
* one, but if several ran we want the most recent), injects its feedback into
* `PROMPT.md`, resets steps, and schedules todo → in-progress. The call site
* is responsible for enforcing the `maxPostReviewFixes` budget before invoking
* this method — this method itself does no accounting.
*
* @returns true when the task was sent back, false when no eligible failed
* step exists (caller should skip).
*/
async recoverFailedPreMergeWorkflowStep(task: Task): Promise<boolean> {
try {
const failed = (task.workflowStepResults ?? [])
.filter((r) => (r.phase || "pre-merge") === "pre-merge" && r.status === "failed")
.sort((a, b) => {
const aTs = Date.parse(a.completedAt || a.startedAt || "");
const bTs = Date.parse(b.completedAt || b.startedAt || "");
return (Number.isFinite(bTs) ? bTs : 0) - (Number.isFinite(aTs) ? aTs : 0);
});
const target = failed[0];
if (!target) {
executorLog.warn(`${task.id}: no failed pre-merge workflow step to recover from`);
return false;
}
const feedback = target.output?.trim() || "(no feedback captured)";
const stepName = target.workflowStepName || target.workflowStepId || "Unknown";
await this.sendTaskBackForFix(
task,
task.worktree ?? "",
feedback,
stepName,
`Auto-revived from in-review: pre-merge workflow step "${stepName}" had failed`,
);
return true;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to recover failed pre-merge workflow step for ${task.id}: ${errorMessage}`);
return false;
}
}
/**
* Resume orphaned in-progress tasks (e.g., after crash/restart).
* Call once after engine startup.
*
* Tasks that are already complete (all steps done/skipped) are fast-pathed
* directly to in-review without spawning a new agent session.
*/
async resumeOrphaned(): Promise<void> {
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) {
executorLog.log(
`resumeOrphaned skipped — ${
settings.globalPause ? "global pause" : "engine pause"
} is active`,
);
return;
}
const tasks = await this.store.listTasks({ slim: true, column: "in-progress" });
const inProgress = tasks.filter(
(t) => t.column === "in-progress" && !this.executing.has(t.id) && !t.paused,
);
if (inProgress.length === 0) return;
executorLog.log(`Found ${inProgress.length} orphaned in-progress task(s)`);
for (const task of inProgress) {
// Fast-path: if the task already completed its work (all steps done),
// move it directly to in-review instead of re-executing from scratch.
if (this.isTaskWorkComplete(task) && !task.mergeDetails) {
if (this.recoveringCompleted.has(task.id)) {
executorLog.log(`${task.id} completed-task recovery already running - skipping duplicate startup recovery`);
continue;
}
executorLog.log(`${task.id} is already complete — fast-pathing to in-review`);
this.recoveringCompleted.add(task.id);
void this.recoverCompletedTask(task)
.catch((err) =>
executorLog.error(`Failed to recover completed orphan ${task.id}:`, err),
)
.finally(() => {
this.recoveringCompleted.delete(task.id);
});
continue;
}
if (this.isNoProgressNoTaskDoneFailure(task)) {
executorLog.log(`${task.id} failed without fn_task_done and has no step progress — leaving for self-healing requeue`);
continue;
}
executorLog.log(`Resuming ${task.id}: ${task.title || task.description.slice(0, 60)}`);
try {
await this.clearResumeFailureState(task);
await this.store.logEntry(task.id, "Resumed after engine restart");
await this.recoverApprovedStepsOnResume(task.id);
} catch (err) {
executorLog.error(`Failed to write resume log for ${task.id}:`, err);
}
this.execute(task).catch((err) =>
executorLog.error(`Failed to resume ${task.id}:`, err),
);
}
}
/**
* Execute a task in an isolated git worktree.
*
* Worktree acquisition flow:
* 1. If the worktree already exists on disk (resume after crash), reuse it.
* 2. If a {@link WorktreePool} is provided and `recycleWorktrees` is enabled,
* attempt to acquire a warm worktree from the pool. Pooled worktrees skip
* the `worktreeInitCommand` since their build caches are already warm.
* 3. Otherwise, create a fresh worktree via `git worktree add` and run the
* `worktreeInitCommand` if configured.
*/
/**
* Resolve custom instructions for a given agent role by looking up agents
* in the AgentStore that have instructions configured.
* Returns an empty string if no instructions are found.
*/
private async resolveInstructionsForRole(role: string): Promise<string> {
if (!this.options.agentStore) return "";
try {
const agents = await this.options.agentStore.listAgents({ role: role as AgentCapability });
for (const agent of agents) {
if (agent.instructionsText || agent.instructionsPath) {
try {
const ratingSummary = await this.options.agentStore.getRatingSummary(agent.id);
return await resolveAgentInstructions(agent, this.rootDir, ratingSummary);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${agent.id}: failed to load rating summary for instruction resolution, falling back to default instructions: ${msg}`);
return await resolveAgentInstructions(agent, this.rootDir);
}
}
}
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`Failed to resolve instructions for role '${role}', continuing without custom instructions: ${msg}`);
}
return "";
}
private resolveDependencyWorktree(task: Task, allTasks: Task[]): string | null {
if (task.dependencies.length === 0) return null;
for (const depId of task.dependencies) {
const dep = allTasks.find((t) => t.id === depId);
if (
dep &&
dep.worktree &&
(dep.column === "done" || dep.column === "in-review") &&
existsSync(dep.worktree)
) {
return dep.worktree;
}
}
return null;
}
/**
* Reuse an existing worktree directory from a dependency task.
* Instead of creating a new worktree with `git worktree add`, this creates
* a new branch in the existing worktree via `git checkout -b`. The worktree
* directory (and its build caches) are preserved.
*/
private async reuseWorktree(branch: string, worktreePath: string): Promise<void> {
await execAsync(`git checkout -b "${branch}"`, {
cwd: worktreePath,
});
executorLog.log(`Reused worktree at ${worktreePath}, created branch ${branch}`);
}
/**
* Execute a task in an isolated git worktree.
*
* **Worktree assignment:** New worktrees get humanized random names
* (e.g., `.worktrees/swift-falcon/`) via `generateWorktreeName()` rather
* than being named after the task ID. This decouples directory names from
* tasks, enabling worktree reuse across dependency chains. When resuming
* a task that already has `task.worktree` set, the existing path is used
* as-is. Branches remain task-scoped (`fusion/{task-id}`).
*/
async execute(task: Task): Promise<void> {
executorLog.log(`execute() called for ${task.id} (already executing=${this.executing.has(task.id)})`);
if (this.executing.has(task.id)) return;
this.executing.add(task.id);
executorLog.log(`Starting ${task.id}: ${task.title || task.description.slice(0, 60)}`);
// Fetch settings early — needed for worktree naming and later configuration
const settings = await this.store.getSettings();
// Read execution mode to determine whether to skip review and workflow steps
const executionMode = task.executionMode ?? "standard";
// Construct run context for mutation correlation
// Use a synthetic correlation ID: task ID + timestamp + random suffix
const syntheticRunId = generateSyntheticRunId("exec", task.id);
this.currentRunContext = {
runId: syntheticRunId,
agentId: task.assignedAgentId ?? "executor",
};
// Build engine run context for audit instrumentation (FN-1404)
const engineRunContext: EngineRunContext = {
runId: syntheticRunId,
agentId: task.assignedAgentId ?? "executor",
taskId: task.id,
phase: "execute",
};
// Create run auditor for TaskStore-backed audit emission (no-ops if store doesn't support it)
const audit = createRunAuditor(this.store, engineRunContext);
// Stale spec enforcement: check if PROMPT.md has aged beyond the configured threshold.
// When enabled, stale tasks are moved back to triage with status "needs-replan"
// so they receive fresh specification before execution. This guard runs early in
// execute() to prevent stale tasks from entering worktree creation or agent sessions.
// If timestamp evaluation is skipped (missing/unreadable file), continue with execution
// so existing filesystem validation paths remain authoritative.
// Skip for tasks that are already in-progress, in-review, merging, or done —
// these should not be interrupted and sent back to triage for re-planning.
const activeColumns = new Set(["in-progress", "in-review", "done"]);
const activeMergeStatuses = new Set(["merging", "merging-pr", "merging-fix"]);
const isActiveTask = activeColumns.has(task.column) || activeMergeStatuses.has(task.status ?? "");
if (!isActiveTask) {
const tasksDir = join(this.store.getFusionDir(), "tasks");
const promptPath = getPromptPath(tasksDir, task.id);
const staleness = await evaluateSpecStaleness({ settings, promptPath });
if (staleness.isStale) {
executorLog.warn(`Task ${task.id} specification is stale — ${staleness.reason}`);
// Move to triage first, then set status so the task enters triage with needs-replan
await this.store.moveTask(task.id, "triage");
await this.store.updateTask(task.id, { status: "needs-replan" });
await this.store.logEntry(task.id, staleness.reason, undefined, this.currentRunContext);
return;
}
}
// Drift detection: a task that is already in-progress (i.e. we're not
// dispatching it fresh from todo) should always carry a `worktree`. If it
// doesn't, some prior update — most likely a partial pause/abort sequence
// where updateTask({ worktree: null }) succeeded but the subsequent
// moveTask()/status write failed — left the row in a half-state. The
// executor can still recover by falling through to the fresh-worktree
// path below, but we emit a loud audit record so these states stop being
// silent.
if (task.column === "in-progress" && task.mergeDetails) {
executorLog.warn(`${task.id}: stale mergeDetails found while executing in-progress task — resetting merge state before continuing`);
task = await this.cleanupMergeStateForReverification(
task,
"Executor detected stale merge state while task was in-progress — reset verification steps and merge metadata before resuming",
);
}
if (task.column === "in-progress" && !task.worktree) {
executorLog.error(
`${task.id}: drift detected — task is in-progress with no worktree. ` +
`Recovering by creating a fresh worktree. This usually indicates a partial ` +
`updateTask/moveTask sequence failed somewhere upstream.`,
);
await this.store.logEntry(
task.id,
"Drift detected: in-progress with no worktree — creating fresh worktree to recover",
undefined,
this.currentRunContext,
);
}
// Hoist worktreePath so it's accessible in the catch block for dep-abort cleanup
// Determine worktree name based on settings
let worktreePath: string;
if (task.worktree) {
worktreePath = task.worktree;
} else {
const naming = settings.worktreeNaming || "random";
let worktreeName: string;
switch (naming) {
case "task-id":
worktreeName = task.id.toLowerCase();
break;
case "task-title":
worktreeName = slugify(task.title || task.description.slice(0, 60));
break;
case "random":
default:
worktreeName = generateWorktreeName(this.rootDir);
break;
}
worktreePath = join(this.rootDir, ".worktrees", worktreeName);
}
// Set by stuck-abort handlers; the actual moveTask("todo") is deferred to
// the finally block so this.executing is cleared first (prevents re-dispatch race).
// true = requeue to todo, false = budget exhausted (already marked failed).
let stuckRequeue: boolean | null = null;
let taskDone = false;
try {
// Check dependencies
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
const unmetDeps = task.dependencies.filter((depId) => {
const dep = allTasks.find((t) => t.id === depId);
return dep && dep.column !== "done" && dep.column !== "in-review" && dep.column !== "archived";
});
if (unmetDeps.length > 0) {
executorLog.log(`${task.id} blocked by: ${unmetDeps.join(", ")} — deferring`);
return;
}
if (!await isGitRepository(this.rootDir)) {
await this.store.logEntry(
task.id,
"Cannot execute task: project directory is not a Git repository. Fusion requires a Git repository for worktree-based task execution.",
);
throw new Error(
"Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.",
);
}
// Create or reuse worktree — try pool first when recycling is enabled
// Prefer the persisted branch from a prior run so the agent resumes on the
// same branch instead of creating a fresh fusion/fn-XXXX-2, -3, etc.
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
// Use generateWorktreeName for human-friendly directory names (adjective-noun pattern)
// instead of task.id, so worktrees are named like ".worktrees/swift-falcon"
let isResume = existsSync(worktreePath);
let acquiredFromPool = false;
// Resolve the base branch — set by the scheduler when a dep is in-review
const baseBranch = task.baseBranch || null;
if (task.worktree && isResume && !await isUsableTaskWorktree(this.rootDir, worktreePath)) {
const invalidWorktreePath = worktreePath;
executorLog.log(`${task.id}: assigned worktree is not usable; creating a fresh worktree instead: ${invalidWorktreePath}`);
await this.store.logEntry(
task.id,
`Assigned worktree is not a registered, usable git worktree; creating a fresh worktree instead`,
invalidWorktreePath,
this.currentRunContext,
);
await this.store.updateTask(task.id, { worktree: null, branch: null });
worktreePath = join(this.rootDir, ".worktrees", generateWorktreeName(this.rootDir));
isResume = existsSync(worktreePath);
}
if (!isResume) {
// Try acquiring a warm worktree from the pool
if (this.options.pool && settings.recycleWorktrees) {
const pooled = this.options.pool.acquire();
if (pooled) {
try {
const actualBranch = await this.options.pool.prepareForTask(pooled, branchName, baseBranch ?? undefined);
worktreePath = pooled;
acquiredFromPool = true;
executorLog.log(`Acquired worktree from pool: ${pooled}`);
await this.store.updateTask(task.id, { worktree: worktreePath, branch: actualBranch });
// Audit trail: record worktree reuse (FN-1404)
await audit.git({ type: "worktree:reuse", target: worktreePath, metadata: { branch: actualBranch } });
if (actualBranch !== branchName) {
executorLog.log(`Branch conflict resolved: using ${actualBranch} instead of ${branchName}`);
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath} (branch conflict: using ${actualBranch})`, undefined, this.currentRunContext);
} else {
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath}`, undefined, this.currentRunContext);
}
} catch (poolErr: unknown) {
// Pool preparation failed — release the worktree back and fall through
// to fresh worktree creation
const poolErrMessage = poolErr instanceof Error ? poolErr.message : String(poolErr);
this.options.pool.release(pooled);
executorLog.log(`Pool prepareForTask failed, falling through to fresh worktree: ${poolErrMessage}`);
await this.store.logEntry(
task.id,
`Pool worktree preparation failed (${poolErrMessage}), creating fresh worktree`,
undefined,
this.currentRunContext,
);
}
}
}
// Fall through to fresh worktree creation if pool had nothing
if (!acquiredFromPool) {
const created = await this.createWorktree(branchName, worktreePath, task.id, baseBranch ?? undefined);
worktreePath = created.path;
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
// Audit trail: record worktree creation and branch creation (FN-1404)
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
await audit.git({ type: "branch:create", target: created.branch });
if (created.branch !== branchName) {
executorLog.log(`Branch conflict resolved: using ${created.branch} instead of ${branchName}`);
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (branch conflict: using ${created.branch})`, undefined, this.currentRunContext);
} else if (baseBranch) {
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (based on ${baseBranch})`, undefined, this.currentRunContext);
} else {
await this.store.logEntry(task.id, `Worktree created at ${worktreePath}`, undefined, this.currentRunContext);
}
// Run worktree init command for fresh worktrees (skip for pooled — caches are warm).
// The init command should deterministically install the full dependency
// graph required by test/typecheck/build commands (for pnpm workspaces,
// prefer `pnpm install --frozen-lockfile`) so transitive modules and
// declarations like @vitest/runner, loupe, debug, @types/express, and
// node-pty are present after bootstrap.
//
// NOTE: This is distinct from the separate workspace-export failure
// class where internal packages fail to resolve because exports point
// to missing dist/* outputs (e.g. "@fusion/core entry not found").
// 5-minute timeout accommodates larger dependency installs.
if (settings.worktreeInitCommand) {
const initStartedAt = Date.now();
try {
const initResult = await runConfiguredCommand(settings.worktreeInitCommand, worktreePath, 300_000);
if (initResult.spawnError || initResult.timedOut || initResult.exitCode !== 0) {
throw new Error(configuredCommandErrorMessage(initResult));
}
await this.store.logEntry(task.id, `[timing] Worktree init command completed in ${Date.now() - initStartedAt}ms`, settings.worktreeInitCommand, this.currentRunContext);
} catch (err: unknown) {
await this.store.logEntry(task.id, `[timing] Worktree init command failed after ${Date.now() - initStartedAt}ms`, undefined, this.currentRunContext);
const execError = err instanceof Error ? err : new Error(String(err));
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
? String((execError as Record<string, unknown>).stderr)
: execError.message;
executorLog.error(`${task.id}: worktree init command failed — first test run will likely fail: ${message}`);
await this.store.logEntry(
task.id,
`Worktree init command failed (first test run will likely fail): ${message}`,
undefined,
this.currentRunContext,
);
}
}
// Run setup script for fresh worktrees (after worktreeInitCommand)
if (settings.setupScript) {
const scriptCommand = settings.scripts?.[settings.setupScript];
if (scriptCommand) {
const setupStartedAt = Date.now();
try {
const setupResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000);
if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) {
throw new Error(configuredCommandErrorMessage(setupResult));
}
await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.currentRunContext);
} catch (err: unknown) {
const execError = err instanceof Error ? err : new Error(String(err));
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
? String((execError as Record<string, unknown>).stderr)
: execError.message;
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.currentRunContext);
}
} else {
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.currentRunContext);
}
}
}
} else if (task.worktree) {
// Task already had a worktree assigned and it exists on disk — reuse it
executorLog.log(`Reusing existing worktree: ${worktreePath}`);
} else {
// Directory exists at generated path but task has no worktree — create via normal flow
const created = await this.createWorktree(branchName, worktreePath, task.id);
worktreePath = created.path;
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
// Audit trail: record worktree creation and branch creation (FN-1404)
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
await audit.git({ type: "branch:create", target: created.branch });
}
// Capture the base commit SHA for diff computation whenever a task
// starts with a newly assigned worktree. Recycled worktrees must
// overwrite any prior task baseline instead of inheriting it.
if (!isResume) {
try {
const { stdout } = await execAsync("git rev-parse HEAD", {
cwd: worktreePath,
encoding: "utf-8",
});
const baseCommitSha = stdout.trim();
await this.store.updateTask(task.id, { baseCommitSha });
executorLog.log(`${task.id}: captured baseCommitSha ${baseCommitSha.slice(0, 7)}`);
// Audit trail: record base commit capture for later diff computation (FN-1404)
await audit.git({ type: "commit:create", target: baseCommitSha, metadata: { purpose: "base" } });
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.log(`Failed to capture baseCommitSha for ${task.id}: ${errorMessage}`);
// Non-fatal: task can continue without baseCommitSha
}
}
this.activeWorktrees.set(task.id, worktreePath);
executorLog.log(`${task.id}: worktree ready at ${worktreePath}`);
this.options.onStart?.(task, worktreePath);
const detail = await this.store.getTask(task.id);
executorLog.log(`${task.id}: fetched task detail (${detail.steps.length} steps, prompt length=${detail.prompt?.length ?? 0})`);
// Initialize steps from PROMPT.md if empty
if (detail.steps.length === 0) {
const steps = await this.store.parseStepsFromPrompt(task.id);
if (steps.length > 0) {
await this.store.updateStep(task.id, 0, "pending");
}
}
// On resume (task.branch already set from a prior run), reconcile step
// statuses from git history so the agent doesn't redo already-committed work.
if (isResume && task.branch && detail.steps.length > 0) {
await this.reconcileStepsFromGitHistory(task.id, detail, worktreePath);
}
// ── Step-Session vs Single-Session execution path ──
// When runStepsInNewSessions is enabled, each step runs in its own
// fresh agent session via StepSessionExecutor. Otherwise, the existing
// single-session flow runs all steps in one monolithic session.
// Build skill selection context early so it's available in both paths
const skillContext = await buildSessionSkillContext({
agentStore: this.options.agentStore!,
task: detail,
sessionPurpose: "executor",
projectRootDir: this.rootDir,
pluginRunner: this.options.pluginRunner,
});
if (settings.runStepsInNewSessions) {
// ── Step-Session Path ──────────────────────────────────────────
executorLog.log(`${task.id}: using step-session mode (maxParallel=${settings.maxParallelSteps ?? 2})`);
const stepSessionAgent = detail.assignedAgentId && this.options.agentStore
? await this.options.agentStore.getAgent(detail.assignedAgentId).catch(() => null)
: null;
const stepSessionRuntimeHint = extractRuntimeHint(stepSessionAgent?.runtimeConfig);
let accumulatedStepTokenUsage = detail.tokenUsage;
const tokenUsageRecordedSteps = new Set<number>();
const stepExecutor = new StepSessionExecutor({
store: this.store,
taskDetail: detail,
worktreePath,
rootDir: this.rootDir,
settings,
semaphore: this.options.semaphore,
stuckTaskDetector: this.options.stuckTaskDetector,
pluginRunner: this.options.pluginRunner,
runtimeHint: stepSessionRuntimeHint,
// Pass skill selection context from the main executor session
skillSelection: skillContext.skillSelectionContext,
// Pass agentStore and messageStore for delegation and messaging tools
agentStore: this.options.agentStore,
messageStore: this.options.messageStore,
onStepStart: (stepIndex) => {
this.options.stuckTaskDetector?.recordProgress(task.id);
try {
this.store.updateStep(task.id, stepIndex, "in-progress").catch((err) => {
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status to in-progress: ${err}`);
});
} catch (err) {
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status to in-progress: ${err}`);
}
},
onStepComplete: (stepIndex, result) => {
executorLog.log(`${task.id}: step ${stepIndex} ${result.success ? "succeeded" : "failed"} (${result.retries} retries)`);
try {
this.store.updateStep(task.id, stepIndex, result.success ? "done" : "skipped").catch((err) => {
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status: ${err}`);
});
} catch (err) {
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status: ${err}`);
}
if (!result.tokenUsage) {
return;
}
accumulatedStepTokenUsage = this.accumulateTokenUsage(accumulatedStepTokenUsage, result.tokenUsage);
tokenUsageRecordedSteps.add(stepIndex);
if (!accumulatedStepTokenUsage) {
return;
}
this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage }).catch((err) => {
executorLog.warn(`${task.id}: failed to persist token usage on step ${stepIndex} complete: ${err}`);
});
},
});
this.activeStepExecutors.set(task.id, stepExecutor);
const stepWork = async () => {
const results = await stepExecutor.executeAll();
// Check abort conditions after execution completes
if (this.depAborted.has(task.id)) {
this.depAborted.delete(task.id);
await this.handleDepAbortCleanup(task.id, worktreePath);
return;
}
if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
await this.store.logEntry(task.id, "Execution paused — step sessions terminated, moved to todo", undefined, this.currentRunContext);
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
return;
}
if (this.stuckAborted.has(task.id)) {
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
this.stuckAborted.delete(task.id);
return;
}
for (const result of results) {
if (!result.tokenUsage || tokenUsageRecordedSteps.has(result.stepIndex)) {
continue;
}
accumulatedStepTokenUsage = this.accumulateTokenUsage(accumulatedStepTokenUsage, result.tokenUsage);
}
if (accumulatedStepTokenUsage) {
await this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage });
}
const allSuccess = results.every(r => r.success);
if (allSuccess) {
const updatedTask = await this.store.getTask(task.id);
const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha);
if (modifiedFiles.length > 0) {
await this.store.updateTask(task.id, { modifiedFiles });
executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`);
// Audit trail: record filesystem mutation (FN-1404)
await audit.filesystem({ type: "file:capture-modified", target: task.id, metadata: { files: modifiedFiles } });
}
this.scheduleCompletedTaskWatchdog(task.id, "step-session completion");
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after step-session completion")) {
return;
}
// ── Deterministic verification gate (FN-3345) ──────────
// Run testCommand/buildCommand after all steps succeed but BEFORE
// workflow steps and the in-review transition. Skipped in fast mode
// and when no verification commands are configured.
if (executionMode !== "fast") {
if (settings.testCommand?.trim() || settings.buildCommand?.trim()) {
const verificationResult = await this.runExecutorDeterministicVerification(task, worktreePath, settings);
if (!verificationResult.allPassed) {
const failedType = verificationResult.failedCommand === "testCommand" ? "test" : "build";
const failedResult = failedType === "test" ? verificationResult.testResult! : verificationResult.buildResult!;
const failedCommand = failedResult.command;
const failureOutput = failedResult.stderr || failedResult.stdout || "Unknown error";
const summary = summarizeVerificationOutput(failureOutput, failedType);
executorLog.log(`${task.id}: [verification] ${failedType} failed — attempting fix agent`);
await this.store.logEntry(
task.id,
`[verification] ${failedType} command failed (exit ${failedResult.exitCode}). Attempting fix agent...`,
summary,
this.currentRunContext,
);
const maxFixRetries = Math.min(settings.verificationFixRetries ?? 3, 3);
if (maxFixRetries === 0) {
executorLog.log(`${task.id}: [verification] fix retries set to 0 — sending task back immediately`);
await this.sendTaskBackForFix(
task, worktreePath,
`${failedType} command \`${failedCommand}\` failed (exit ${failedResult.exitCode}):\n${summary}`,
`Verification (${failedType})`,
`Deterministic verification failed (${failedType})`,
true,
true,
);
return;
}
let fixSucceeded = false;
for (let attempt = 1; attempt <= maxFixRetries; attempt++) {
const fixed = await this.attemptExecutorVerificationFix(
task, worktreePath,
{
command: failedCommand,
exitCode: failedResult.exitCode,
output: failureOutput,
type: failedType,
},
settings,
attempt,
maxFixRetries,
);
if (fixed) {
fixSucceeded = true;
executorLog.log(`${task.id}: [verification] fix agent succeeded on attempt ${attempt}/${maxFixRetries}`);
await this.store.logEntry(
task.id,
`[verification] Fix agent succeeded on attempt ${attempt}/${maxFixRetries}. Verification now passing.`,
undefined,
this.currentRunContext,
);
break;
}
executorLog.log(`${task.id}: [verification] fix agent attempt ${attempt}/${maxFixRetries} failed`);
await this.store.logEntry(
task.id,
`[verification] Fix agent attempt ${attempt}/${maxFixRetries} failed`,
undefined,
this.currentRunContext,
);
}
if (!fixSucceeded) {
executorLog.log(`${task.id}: [verification] all fix attempts exhausted (${maxFixRetries}/${maxFixRetries}) — sending task back`);
await this.sendTaskBackForFix(
task, worktreePath,
`${failedType} command \`${failedCommand}\` failed (exit ${failedResult.exitCode}) after ${maxFixRetries} fix attempts:\n${summary}`,
`Verification (${failedType})`,
`Deterministic verification failed after ${maxFixRetries} fix attempts`,
true,
true,
);
return;
}
}
}
}
// Run workflow steps before moving to in-review — skip in fast mode
if (executionMode !== "fast") {
const workflowResult = await this.runWorkflowSteps(task, worktreePath, settings);
if (workflowResult === "deferred-paused") {
if (await this.parkTaskAfterWorkflowStepPause(task.id)) {
this.pausedAborted.delete(task.id);
return;
}
if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
}
return;
}
if (!workflowResult.allPassed) {
// Check if revision was requested
if (workflowResult.revisionRequested) {
await this.handleWorkflowRevisionRequest(task, worktreePath, workflowResult.feedback, workflowResult.stepName);
return;
}
// Try to fix workflow step failures with retries
const retried = await this.handleWorkflowStepFailure(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown");
if (retried) {
return; // Retry scheduled
}
// Retries exhausted - send back to in-progress for remediation
await this.sendTaskBackForFix(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed");
return;
}
} else {
executorLog.log(`${task.id}: fast mode — skipping pre-merge workflow steps`);
await this.store.logEntry(task.id, "Fast mode — pre-merge workflow steps skipped", undefined, this.currentRunContext);
}
// Reset retry counters on success
await this.store.updateTask(task.id, { workflowStepRetries: undefined, taskDoneRetryCount: null });
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after step-session completion")) {
return;
}
await this.store.moveTask(task.id, "in-review");
this.clearCompletedTaskWatchdog(task.id);
// Audit trail: record task move (FN-1404)
await audit.database({ type: "task:move", target: task.id, metadata: { to: "in-review" } });
executorLog.log(`✓ ${task.id} completed (step-session) → in-review`);
this.options.onComplete?.(task);
} else {
const failedSteps = results.filter(r => !r.success);
const errorSummary = failedSteps.map(r => `Step ${r.stepIndex}: ${r.error || "unknown error"}`).join("; ");
await this.store.updateTask(task.id, { status: "failed", error: errorSummary });
await this.store.moveTask(task.id, "in-review");
executorLog.log(`✗ ${task.id} step-session failed → in-review: ${errorSummary}`);
this.options.onError?.(task, new Error(errorSummary));
}
};
const retryableStepWork = () => withRateLimitRetry(stepWork, {
onRetry: (attempt, delayMs, error) => {
const delaySec = Math.round(delayMs / 1000);
executorLog.warn(`⏳ ${task.id} rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`);
this.store.logEntry(task.id, `Rate limited — retry ${attempt} in ${delaySec}s`, undefined, this.currentRunContext).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id} failed to log rate-limit retry: ${msg}`);
});
},
});
try {
if (this.options.semaphore) {
await this.options.semaphore.run(retryableStepWork, PRIORITY_EXECUTE);
} else {
await retryableStepWork();
}
} catch (err: unknown) {
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
if (this.depAborted.has(task.id)) {
this.depAborted.delete(task.id);
await this.handleDepAbortCleanup(task.id, worktreePath);
} else if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
await this.store.logEntry(task.id, "Execution paused during step-session", undefined, this.currentRunContext);
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
} else if (this.stuckAborted.has(task.id)) {
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
this.stuckAborted.delete(task.id);
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
} else if (isTransientError(errorMessage)) {
const decision = computeRecoveryDecision({
recoveryRetryCount: task.recoveryRetryCount,
nextRecoveryAt: task.nextRecoveryAt,
});
if (decision.shouldRetry) {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
if (!isSilentTransientError(errorMessage)) {
executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`);
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
}
if (worktreePath && existsSync(worktreePath)) {
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
// Audit trail: record worktree removal (FN-1404)
await audit.git({ type: "worktree:remove", target: worktreePath });
} catch (wtErr: unknown) {
const msg = wtErr instanceof Error ? wtErr.message : String(wtErr);
executorLog.warn(`${task.id}: worktree removal failed during transient-error retry cleanup (${worktreePath}): ${msg}`);
}
}
await this.store.updateTask(task.id, {
recoveryRetryCount: decision.nextState.recoveryRetryCount,
nextRecoveryAt: decision.nextState.nextRecoveryAt,
worktree: null,
branch: null,
});
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
stuckRequeue = null; // Prevent outer finally from re-processing
return;
}
executorLog.error(`✗ ${task.id} transient error retries exhausted: ${errorDetail}`);
if (errorStack) {
await this.store.logEntry(task.id, `Transient error retries exhausted: ${errorMessage}`, errorStack, this.currentRunContext);
}
await this.store.updateTask(task.id, {
status: "failed",
error: errorMessage,
recoveryRetryCount: null,
nextRecoveryAt: null,
});
if (accumulatedStepTokenUsage) {
await this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage });
}
await this.store.moveTask(task.id, "in-review");
executorLog.log(`✗ ${task.id} transient retries exhausted → in-review`);
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
} else {
executorLog.error(`✗ ${task.id} step-session execution failed:`, errorDetail);
await this.store.logEntry(task.id, `Step-session execution failed: ${errorMessage}`, errorStack ?? errorDetail, this.currentRunContext);
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
if (accumulatedStepTokenUsage) {
await this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage });
}
await this.store.moveTask(task.id, "in-review");
executorLog.log(`✗ ${task.id} step-session execution failed → in-review`);
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
}
} finally {
this.executing.delete(task.id);
this.loopRecoveryState.delete(task.id);
// Wrap cleanup in try/catch so activeStepExecutors.delete() always runs.
// If cleanup() throws, the executor continues to clean up the in-memory map
// and requeue logic without leaking the reference.
try {
await stepExecutor.cleanup();
} catch (cleanupErr) {
executorLog.warn(`StepSessionExecutor cleanup failed for ${task.id}: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
}
this.activeStepExecutors.delete(task.id);
// Stuck-requeue: clean up worktree and move to todo
if (stuckRequeue === true) {
try {
// Reset steps whose work was never committed before destroying the worktree
const latestTask = await this.store.getTask(task.id);
await this.resetStepsIfWorkLost(latestTask);
if (worktreePath && existsSync(worktreePath)) {
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
} catch (wtErr: unknown) {
const msg = wtErr instanceof Error ? wtErr.message : String(wtErr);
executorLog.warn(`${task.id}: worktree removal failed during stuck-requeue cleanup (${worktreePath}): ${msg}`);
}
}
await this.store.updateTask(task.id, { status: "stuck-killed", worktree: null, branch: null });
if (task.column !== "todo") {
await this.store.moveTask(task.id, "todo");
executorLog.log(`${task.id} moved to todo for retry after stuck kill`);
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to requeue stuck task ${task.id}: ${errorMessage}`);
}
stuckRequeue = null; // Prevent outer finally from re-processing
}
}
// Step-session path handled completely — return before outer catch/finally
return;
}
// ── Single-Session Path (default) ────────────────────────────────
// Build custom tools for the worker
// Track the last code review verdict per step so we can enforce REVISE
// (block fn_task_update status="done" until the agent re-reviews and gets APPROVE).
const codeReviewVerdicts = new Map<number, ReviewVerdict>();
let wasPaused = false;
// Mutable ref — populated after createFnAgent, tools access lazily via closure
const sessionRef: { current: AgentSession | null } = { current: null };
const stepCheckpoints = new Map<number, string>();
const stuckDetector = this.options.stuckTaskDetector;
const assignedAgentId = detail.assignedAgentId?.trim();
const reflectionTools = this.options.reflectionService && settings.reflectionEnabled && assignedAgentId
? [createReflectOnPerformanceTool(this.options.reflectionService, assignedAgentId)]
: [];
const assignedAgent = assignedAgentId && this.options.agentStore
? await this.options.agentStore.getAgent(assignedAgentId).catch(() => null)
: null;
const executorRuntimeHint = extractRuntimeHint(assignedAgent?.runtimeConfig);
// Log fast mode status
if (executionMode === "fast") {
executorLog.log(`${task.id}: fast mode — fn_review_step tool not injected`);
}
const customTools = [
this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stepCheckpoints, stuckDetector),
this.createTaskLogTool(task.id),
this.createTaskCreateTool(),
this.createTaskAddDepTool(task.id),
this.createTaskDoneTool(task.id, () => { taskDone = true; }),
createRunVerificationTool({
worktreePath,
rootDir: this.rootDir,
taskId: task.id,
recordActivity: () => stuckDetector?.recordActivity(task.id),
log: {
info: (s) => executorLog.log(s),
warn: (s) => executorLog.warn(s),
error: (s) => executorLog.warn(s),
},
}),
// Skip fn_review_step tool in fast mode — fast mode bypasses automated review gates
...(executionMode !== "fast" ? [
this.createReviewStepTool(task.id, worktreePath, detail.prompt, codeReviewVerdicts, sessionRef, stepCheckpoints, detail, stuckDetector),
] : []),
this.createSpawnAgentTool(task.id, worktreePath, settings),
this.createTaskDocumentWriteTool(task.id),
this.createTaskDocumentReadTool(task.id),
...createResearchTools({
store: this.store,
rootDir: this.rootDir,
getSettings: async () => this.store.getSettings(),
}),
...createMemoryTools(this.rootDir, settings, assignedAgent ? {
agentMemory: {
agentId: assignedAgent.id,
agentName: assignedAgent.name,
memory: assignedAgent.memory,
},
} : undefined),
// Conditionally add agent self-reflection when enabled and task has an assigned agent.
...reflectionTools,
// Agent delegation tools — discover and delegate work to other agents.
...(this.options.agentStore ? [
createListAgentsTool(this.options.agentStore),
createDelegateTaskTool(this.options.agentStore, this.store, { rootDir: this.rootDir }),
...(assignedAgentId ? [
createGetAgentConfigTool(this.options.agentStore, assignedAgentId),
createUpdateAgentConfigTool(this.options.agentStore, assignedAgentId),
] : []),
] : []),
// Messaging tools — allows executor agents to send and receive messages.
...(this.options.messageStore && assignedAgentId ? [
createSendMessageTool(this.options.messageStore, assignedAgentId),
createReadMessagesTool(this.options.messageStore, assignedAgentId),
] : []),
// Add plugin tools from PluginRunner
...(this.options.pluginRunner?.getPluginTools() ?? []),
];
// Accumulates the full assistant text output for the most recent session.
// Reset to "" each time a new session begins so detectPseudoPause only
// sees the last session's output, not the entire conversation history.
let lastAssistantText = "";
const agentLogger = new AgentLogger({
store: this.store,
taskId: task.id,
agent: "executor",
persistAgentToolOutput: settings.persistAgentToolOutput,
onAgentText: (taskId, delta) => {
lastAssistantText += delta;
stuckDetector?.recordActivity(taskId);
this.options.onAgentText?.(taskId, delta);
},
onAgentTool: (taskId, toolName) => {
stuckDetector?.recordActivity(taskId);
this.options.onAgentTool?.(taskId, toolName);
},
});
const agentWork = async () => {
// Resolve model settings using canonical lane hierarchy:
// 1. Task override pair (modelProvider + modelId)
// 2. Project execution lane pair (executionProvider + executionModelId)
// 3. Global execution lane pair (executionGlobalProvider + executionGlobalModelId)
// 4. Project default override pair (defaultProviderOverride + defaultModelIdOverride)
// 5. Global default pair (defaultProvider + defaultModelId)
const { provider: executorProvider, modelId: executorModelId } = resolveExecutorModelPair(
detail.modelProvider,
detail.modelId,
settings,
);
const executorFallbackProvider = settings.fallbackProvider;
const executorFallbackModelId = settings.fallbackModelId;
const executorThinkingLevel = detail.thinkingLevel ?? settings.defaultThinkingLevel;
// Determine whether we're resuming a previous session (pause/resume)
// or starting fresh. Use file-based sessions so conversation state
// persists across pause/unpause cycles.
const isResuming = !!task.sessionFile && existsSync(task.sessionFile);
const sessionManager = isResuming
? SessionManager.open(task.sessionFile!)
: SessionManager.create(worktreePath);
executorLog.log(`${task.id}: creating agent session (provider=${executorProvider ?? "default"}, model=${executorModelId ?? "default"}, resuming=${isResuming})`);
// Resolve per-agent custom instructions for the executor role
const executorInstructions = await this.resolveInstructionsForRole("executor");
const executorSystemPrompt = buildSystemPromptWithInstructions(
getExecutorSystemPrompt(settings),
executorInstructions,
);
// sessionFile must be let because it's destructured alongside session which is reassigned
// eslint-disable-next-line prefer-const
let { session, sessionFile } = await createResolvedAgentSession({
sessionPurpose: "executor",
runtimeHint: executorRuntimeHint,
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: executorSystemPrompt,
tools: "coding",
customTools,
onText: agentLogger.onText,
onThinking: agentLogger.onThinking,
onToolStart: agentLogger.onToolStart,
onToolEnd: agentLogger.onToolEnd,
defaultProvider: executorProvider,
defaultModelId: executorModelId,
fallbackProvider: executorFallbackProvider,
fallbackModelId: executorFallbackModelId,
defaultThinkingLevel: executorThinkingLevel,
sessionManager,
// Skill selection: use assigned agent skills if available, otherwise role fallback
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
taskId: task.id,
taskTitle: detail.title,
onFallbackModelUsed: createFallbackModelObserver({
agent: "executor",
label: "executor",
store: this.store,
taskId: task.id,
taskTitle: detail.title,
}),
});
if (isResuming) {
executorLog.log(`${task.id}: resumed session from ${task.sessionFile}`);
await this.store.logEntry(task.id, `Resumed agent session after unpause (model: ${describeModel(session)})`, undefined, this.currentRunContext);
} else {
executorLog.log(`${task.id}: using model ${describeModel(session)}`);
await this.store.logEntry(task.id, `Executor using model: ${describeModel(session)}`, undefined, this.currentRunContext);
// Persist session file path so pause/resume can reopen it
if (sessionFile) {
await this.store.updateTask(task.id, { sessionFile });
}
}
// Make session available to custom tools (fn_task_update checkpoint capture, fn_review_step rewind)
sessionRef.current = session;
// Register session so the pause listener can terminate it
// Initialize with empty set of seen comments
const seenSteeringIds = new Set<string>();
if (detail.comments) {
for (const comment of detail.comments) {
seenSteeringIds.add(comment.id);
}
}
this.activeSessions.set(task.id, {
session,
seenSteeringIds,
lastModelProvider: detail.modelProvider,
lastModelId: detail.modelId,
});
// Register with stuck task detector for heartbeat monitoring
stuckDetector?.trackTask(task.id, session);
executorLog.log(`${task.id}: session registered (model=${describeModel(session)}, stuckDetector=${!!stuckDetector})`);
// Invoke plugin onAgentRunStart hook (fire-and-forget)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
void (this.options.pluginRunner as any)?.invokeHook("onAgentRunStart", task.id);
try {
// Record activity on prompt start (heartbeat for stuck detection)
stuckDetector?.recordActivity(task.id);
executorLog.log(`${task.id}: calling promptWithFallback()...`);
if (isResuming) {
// Session already has full conversation history — just tell the
// agent it was paused and should pick up where it left off.
await promptWithFallback(session, [
"Your session was paused and has now been resumed.",
"Continue working on the task from where you left off.",
"Review the current state of your worktree and proceed with the next pending step.",
].join("\n"));
} else {
const agentPrompt = buildExecutionPrompt(detail, this.rootDir, settings, worktreePath);
await promptWithFallback(session, agentPrompt);
}
// Re-raise errors that pi-coding-agent swallowed after exhausting retries.
// session.prompt() resolves normally even when retries are exhausted —
// the error is stored on session.state.error instead of being thrown.
checkSessionError(session);
await accumulateSessionTokenUsage(this.store, task.id, session);
// Check if proactive context compaction is needed based on token cap setting.
// This runs after the main prompt completes to avoid interrupting active work.
try {
const capResult = await this.tokenCapDetector.checkAndCompact(
session,
task.id,
settings.tokenCap,
async (s) => {
const compactResult = await compactSessionContext(s);
if (compactResult) {
await this.store.logEntry(
task.id,
`Context compacted at ${compactResult.tokensBefore} tokens (token cap: ${settings.tokenCap})`,
undefined,
this.currentRunContext,
);
}
return compactResult;
},
);
if (capResult.triggered) {
executorLog.log(`${task.id} token cap check: ${capResult.message}`);
}
} catch (err) {
executorLog.log(`${task.id} token cap check failed (non-fatal): ${err}`);
}
// If loop recovery is pending (compact-and-resume was triggered by
// handleLoopDetected), consume the pending state and resume with a
// deterministic prompt. The session has already been compacted, so
// we just need to send a fresh prompt to continue execution.
const loopState = this.loopRecoveryState.get(task.id);
if (loopState?.pending) {
loopState.pending = false;
executorLog.log(`${task.id} consuming loop recovery — resuming with fresh context`);
await this.store.logEntry(task.id, "Resuming execution after context compaction — taking a different approach", undefined, this.currentRunContext);
// Reset activity tracking so the detector doesn't immediately re-trigger
stuckDetector?.recordProgress(task.id);
const resumePrompt = [
"Your conversation was compacted because you were looping without making progress.",
"Review the current state of the worktree carefully:",
"1. Check `git log --oneline` to see what's already been committed",
"2. Read the files you were working on to understand current state",
"3. Review the PROMPT.md steps to see which are still pending",
"",
"Take a DIFFERENT approach from what you were doing before.",
"If the current step is complete, call fn_task_update to mark it done and move to the next step.",
"If you're stuck on a problem, try a simpler or alternative solution.",
"",
"Continue the task from where you left off.",
].join("\n");
await promptWithFallback(session, resumePrompt);
checkSessionError(session);
await accumulateSessionTokenUsage(this.store, task.id, session);
}
// If dependency was added during execution, discard worktree and move to triage
if (this.depAborted.has(task.id)) {
this.depAborted.delete(task.id);
await this.handleDepAbortCleanup(task.id, worktreePath);
return;
}
// If paused during execution, move to todo so the scheduler can resume
// after unpause. This path fires when session.dispose() causes the
// prompt to resolve gracefully instead of throwing.
if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
wasPaused = true;
if (await this.shouldFinalizeCompletedTask(task.id, taskDone)) {
if (await this.shouldDeferCompletionForGlobalPause(task.id, "paused after completion")) {
return;
}
executorLog.log(`${task.id} paused after completion (graceful session exit) — finalizing to in-review`);
await this.store.logEntry(task.id, "Execution paused after completion — finalizing to in-review");
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
this.clearCompletedTaskWatchdog(task.id);
this.options.onComplete?.(task);
} else {
executorLog.log(`${task.id} paused (graceful session exit) — moving to todo`);
await this.store.logEntry(task.id, "Execution paused — session preserved for resume, moved to todo");
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
}
return;
}
// If the stuck task detector disposed the session and the agent exited
// cleanly, stop here. The requeue is deferred to the finally block
// (after this.executing is cleared) to prevent a race where the
// scheduler re-dispatches while the old execution guard is still set.
if (this.stuckAborted.has(task.id)) {
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
this.stuckAborted.delete(task.id);
executorLog.log(`${task.id} terminated by stuck task detector (graceful session exit)`);
return;
}
// If the agent didn't explicitly call fn_task_done, check whether
// all steps are already complete — treat as implicit done to avoid
// unnecessary retry sessions for context-overflow / compaction cases.
if (!taskDone) {
const implicitCheck = await this.store.getTask(task.id);
if (implicitCheck.steps.length > 0 &&
implicitCheck.steps.every((s) => s.status === "done" || s.status === "skipped")) {
taskDone = true;
executorLog.log(`${task.id} all steps done — treating as implicit fn_task_done`);
await this.store.logEntry(task.id, "All steps complete — implicit fn_task_done (agent did not call tool explicitly)", undefined, this.currentRunContext);
this.scheduleCompletedTaskWatchdog(task.id, "implicit fn_task_done");
}
}
if (taskDone) {
// Capture modified files before running workflow steps
const updatedTask = await this.store.getTask(task.id);
const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha);
if (modifiedFiles.length > 0) {
await this.store.updateTask(task.id, { modifiedFiles });
executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`);
}
this.scheduleCompletedTaskWatchdog(task.id, "task completion");
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after task completion")) {
return;
}
// Run workflow steps before moving to in-review — skip in fast mode
if (executionMode !== "fast") {
const workflowResult = await this.runWorkflowSteps(task, worktreePath, settings);
if (workflowResult === "deferred-paused") {
if (await this.parkTaskAfterWorkflowStepPause(task.id)) {
this.pausedAborted.delete(task.id);
wasPaused = true;
return;
}
if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
wasPaused = true;
}
return;
}
if (!workflowResult.allPassed) {
// Check if revision was requested
if (workflowResult.revisionRequested) {
await this.handleWorkflowRevisionRequest(task, worktreePath, workflowResult.feedback, workflowResult.stepName);
return;
}
// Try to fix workflow step failures with retries
const retried = await this.handleWorkflowStepFailure(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown");
if (retried) {
return; // Retry scheduled
}
// Retries exhausted - send back to in-progress for remediation
await this.sendTaskBackForFix(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed");
return;
}
} else {
executorLog.log(`${task.id}: fast mode — skipping pre-merge workflow steps`);
await this.store.logEntry(task.id, "Fast mode — pre-merge workflow steps skipped", undefined, this.currentRunContext);
}
// Reset retry counters on success
await this.store.updateTask(task.id, { workflowStepRetries: undefined, taskDoneRetryCount: null });
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after task completion")) {
return;
}
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
this.clearCompletedTaskWatchdog(task.id);
executorLog.log(`✓ ${task.id} completed → in-review`);
this.options.onComplete?.(task);
} else {
let taskDoneSessionRetries = 0;
while (!taskDone && taskDoneSessionRetries < MAX_TASK_DONE_SESSION_RETRIES) {
taskDoneSessionRetries++;
executorLog.log(
`⚠ ${task.id} finished without fn_task_done — retrying with new session (${taskDoneSessionRetries}/${MAX_TASK_DONE_SESSION_RETRIES})`,
);
await this.store.logEntry(
task.id,
`Agent finished without calling fn_task_done — retrying with new session (${taskDoneSessionRetries}/${MAX_TASK_DONE_SESSION_RETRIES})`,
undefined,
this.currentRunContext,
);
// Capture and analyse the previous session's text before resetting.
const previousSessionText = lastAssistantText;
const pseudoPause = detectPseudoPause(previousSessionText);
if (pseudoPause.kind !== "none") {
const shortMatch = (pseudoPause.matched ?? "").slice(0, 120);
await this.store.logEntry(
task.id,
`Pseudo-pause detected (kind=${pseudoPause.kind}, matched='${shortMatch}')`,
undefined,
this.currentRunContext,
);
executorLog.log(`${task.id} pseudo-pause detected (kind=${pseudoPause.kind}): ${shortMatch}`);
}
// Dispose old session and create a fresh one.
// Reset lastAssistantText so the new session's text is tracked cleanly.
lastAssistantText = "";
this.activeSessions.delete(task.id);
this.tokenUsageBaselines.delete(task.id);
session.dispose();
const { session: retrySession, sessionFile: retrySessionFile } = await createResolvedAgentSession({
sessionPurpose: "executor",
runtimeHint: executorRuntimeHint,
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: executorSystemPrompt,
tools: "coding",
customTools,
onText: agentLogger.onText,
onThinking: agentLogger.onThinking,
onToolStart: agentLogger.onToolStart,
onToolEnd: agentLogger.onToolEnd,
defaultProvider: executorProvider,
defaultModelId: executorModelId,
fallbackProvider: executorFallbackProvider,
fallbackModelId: executorFallbackModelId,
defaultThinkingLevel: executorThinkingLevel,
sessionManager: SessionManager.create(worktreePath),
// Skill selection: use assigned agent skills if available, otherwise role fallback
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
});
if (retrySessionFile) {
this.store.updateTask(task.id, { sessionFile: retrySessionFile }).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id} failed to persist retry sessionFile: ${msg}`);
});
}
session = retrySession;
sessionRef.current = retrySession;
this.activeSessions.set(task.id, {
session: retrySession,
seenSteeringIds,
lastModelProvider: detail.modelProvider,
lastModelId: detail.modelId,
});
stuckDetector?.trackTask(task.id, retrySession);
let retryPrompt: string;
if (pseudoPause.kind !== "none") {
const shortMatch = (pseudoPause.matched ?? "").slice(0, 120);
retryPrompt = [
`Your previous turn ended with a pseudo-pause: "${shortMatch}". This is forbidden.`,
"",
"Turn-ending rules you violated:",
"- You MUST NOT end a turn by asking the user a question, summarizing progress, or requesting permission to continue.",
"- Phrases like 'If you want, I can continue', 'Should I proceed?', 'Let me know if...' are FORBIDDEN turn-endings.",
"- The user is not watching this conversation. Questions written as prose are ignored.",
"- If you genuinely cannot proceed, call fn_task_done with a clear explanation — never write the blocker as plain prose.",
"",
"What you must do now:",
"1. Review the PROMPT.md steps and identify the next pending step.",
"2. Do the work for that step immediately — call fn_task_update, write code, run tests.",
"3. Continue until all steps are done, then call fn_task_done.",
"Do NOT ask for permission. Do NOT write a summary. Just call a tool and keep working.",
"",
"Original task:",
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath),
].join("\n");
} else {
retryPrompt = [
"Your previous session ended without calling the fn_task_done tool.",
"The task may already be complete — review the current state of the worktree and either:",
"1. If the work is done, call fn_task_done with a summary of what was accomplished.",
"2. If there is remaining work, finish it and then call fn_task_done.",
"",
"Original task:",
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath),
].join("\n");
}
stuckDetector?.recordActivity(task.id);
await promptWithFallback(retrySession, retryPrompt);
checkSessionError(retrySession);
await accumulateSessionTokenUsage(this.store, task.id, retrySession);
if (!taskDone) {
const implicitCheck = await this.store.getTask(task.id);
if (implicitCheck.steps.length > 0 &&
implicitCheck.steps.every((s) => s.status === "done" || s.status === "skipped")) {
taskDone = true;
executorLog.log(`${task.id} all steps done — treating as implicit fn_task_done`);
await this.store.logEntry(task.id, "All steps complete — implicit fn_task_done (agent did not call tool explicitly)", undefined, this.currentRunContext);
this.scheduleCompletedTaskWatchdog(task.id, "implicit fn_task_done");
}
}
}
if (taskDone) {
const updatedTask = await this.store.getTask(task.id);
const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha);
if (modifiedFiles.length > 0) {
await this.store.updateTask(task.id, { modifiedFiles });
executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`);
}
this.scheduleCompletedTaskWatchdog(task.id, "task completion retry");
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after task completion retry")) {
return;
}
// Run workflow steps before moving to in-review — skip in fast mode
if (executionMode !== "fast") {
const workflowResult = await this.runWorkflowSteps(task, worktreePath, settings);
if (workflowResult === "deferred-paused") {
if (await this.parkTaskAfterWorkflowStepPause(task.id)) {
this.pausedAborted.delete(task.id);
wasPaused = true;
return;
}
if (this.pausedAborted.has(task.id)) {
this.pausedAborted.delete(task.id);
wasPaused = true;
}
return;
}
if (!workflowResult.allPassed) {
if (workflowResult.revisionRequested) {
await this.handleWorkflowRevisionRequest(task, worktreePath, workflowResult.feedback, workflowResult.stepName);
return;
}
await this.sendTaskBackForFix(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed on retry");
return;
}
} else {
executorLog.log(`${task.id}: fast mode — skipping pre-merge workflow steps`);
await this.store.logEntry(task.id, "Fast mode — pre-merge workflow steps skipped", undefined, this.currentRunContext);
}
await this.store.updateTask(task.id, { workflowStepRetries: undefined, taskDoneRetryCount: null });
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after task completion retry")) {
return;
}
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
this.clearCompletedTaskWatchdog(task.id);
executorLog.log(`✓ ${task.id} completed on retry → in-review`);
this.options.onComplete?.(task);
} else {
const priorRequeues = task.taskDoneRetryCount ?? 0;
const nextRequeueCount = priorRequeues + 1;
const errorMessage = `Agent finished without calling fn_task_done (after ${MAX_TASK_DONE_SESSION_RETRIES} retries)`;
if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) {
await this.store.updateTask(task.id, {
status: "failed",
error: errorMessage,
taskDoneRetryCount: nextRequeueCount,
});
await this.store.logEntry(
task.id,
`${errorMessage} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`,
undefined,
this.currentRunContext,
);
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
executorLog.log(`✗ ${task.id} failed after ${MAX_TASK_DONE_SESSION_RETRIES} retries — requeued to todo (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`);
} else {
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
await this.store.logEntry(task.id, `${errorMessage} — moved to in-review for inspection`, undefined, this.currentRunContext);
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
executorLog.log(`✗ ${task.id} failed after ${MAX_TASK_DONE_SESSION_RETRIES} retries — no fn_task_done → in-review`);
}
this.options.onError?.(task, new Error(errorMessage));
}
}
} finally {
this.activeSessions.delete(task.id);
stuckDetector?.untrackTask(task.id);
await agentLogger.flush();
await this.persistTokenUsage(task.id, session).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id}: failed to persist final single-session token usage before dispose: ${msg}`);
});
session.dispose();
// Terminate all spawned child agents when parent session ends
await this.terminateAllChildren(task.id);
// Clear session file when task completes or fails (not when paused —
// the file is preserved so unpause can resume the conversation).
// Check both the local flag (graceful exit) and the instance set
// (error path where dispose caused prompt to throw).
if (!wasPaused && !this.pausedAborted.has(task.id)) {
this.store.updateTask(task.id, { sessionFile: null }).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id} failed to clear sessionFile: ${msg}`);
});
}
// Invoke plugin onAgentRunEnd hook (fire-and-forget)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
void (this.options.pluginRunner as any)?.invokeHook("onAgentRunEnd", task.id);
}
};
const retryableWork = () => withRateLimitRetry(agentWork, {
onRetry: (attempt, delayMs, error) => {
const delaySec = Math.round(delayMs / 1000);
executorLog.warn(`⏳ ${task.id} rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`);
this.store.logEntry(task.id, `Rate limited — retry ${attempt} in ${delaySec}s`, undefined, this.currentRunContext).catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id} failed to log rate-limit retry: ${msg}`);
});
},
});
if (this.options.semaphore) {
await this.options.semaphore.run(retryableWork, PRIORITY_EXECUTE);
} else {
await retryableWork();
}
} catch (err: unknown) {
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
if (this.depAborted.has(task.id)) {
// Dependency added mid-execution — discard worktree and move to triage
this.depAborted.delete(task.id);
await this.handleDepAbortCleanup(task.id, worktreePath);
} else if (errorMessage.includes("Invalid transition")) {
// Task was moved by user/process while executor was running — already in desired state
// This check must come before pausedAborted since it's more specific
const transitionMatch = errorMessage.match(/Invalid transition: '([^']+)' → '([^']+)'/);
const fromColumn = transitionMatch?.[1] ?? "unknown";
const toColumn = transitionMatch?.[2] ?? "unknown";
const logMessage = `Task already moved from '${fromColumn}' — skipping transition to '${toColumn}'`;
executorLog.log(`${task.id} ${logMessage}`);
await this.store.logEntry(task.id, logMessage, errorMessage, this.currentRunContext);
if (fromColumn === "in-review" && toColumn === "in-review") {
try {
const finalizeResult = await this.finalizeAlreadyReviewedTask(task.id);
executorLog.log(`${task.id} duplicate in-review finalization result: ${finalizeResult}`);
} catch (finalizeErr: unknown) {
const finalizeErrMessage = finalizeErr instanceof Error ? finalizeErr.message : String(finalizeErr);
executorLog.warn(`${task.id} failed to finalize duplicate in-review transition: ${finalizeErrMessage}`);
}
}
// Task finished successfully (just already moved), so call onComplete
this.options.onComplete?.(task);
} else if (this.pausedAborted.has(task.id)) {
// Task was paused mid-execution — clean up worktree and move to todo
this.pausedAborted.delete(task.id);
if (await this.shouldFinalizeCompletedTask(task.id, taskDone)) {
if (await this.shouldDeferCompletionForGlobalPause(task.id, "paused after completion")) {
return;
}
executorLog.log(`${task.id} paused after completion — finalizing to in-review`);
await this.store.logEntry(task.id, "Execution paused after completion — finalizing to in-review", undefined, this.currentRunContext);
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
this.options.onComplete?.(task);
} else {
executorLog.log(`${task.id} paused — moving to todo`);
if (worktreePath && existsSync(worktreePath)) {
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
executorLog.log(`Removed old worktree for paused task: ${worktreePath}`);
// Audit trail: record worktree removal (FN-1404)
await audit.git({ type: "worktree:remove", target: worktreePath });
} catch (cleanupErr: unknown) {
const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr);
executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`);
}
}
await this.store.updateTask(task.id, { worktree: undefined, branch: undefined });
await this.store.logEntry(task.id, "Execution paused — agent terminated, moved to todo", undefined, this.currentRunContext);
await this.store.moveTask(task.id, "todo");
}
} else if (this.stuckAborted.has(task.id)) {
// Task was killed by stuck task detector — defer requeue to finally block
// (after this.executing is cleared) to prevent re-dispatch race.
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
this.stuckAborted.delete(task.id);
executorLog.log(`${task.id} terminated by stuck task detector — will ${stuckRequeue ? "retry" : "not retry (budget exhausted)"}`);
} else {
// Context-limit error reached the executor after promptWithFallback's auto-compaction
// already attempted to recover. Recovery strategy (in order):
// 1. Reduced-prompt retry in the same session (up to MAX_REDUCED_PROMPT_ATTEMPTS)
// 2. Fresh-session requeue — terminate the saturated session and move the task
// back to "todo" so the next dispatch gets a clean session (bounded by
// recoveryRetryCount / MAX_RECOVERY_RETRIES).
// FN-2182 class: Step 7 overflow after earlier compaction used to hit the
// loopAttempts<1 guard and fail permanently; the requeue path below recovers
// by restarting with a fresh session against the already-written step output.
const MAX_REDUCED_PROMPT_ATTEMPTS = 3;
const loopState = this.loopRecoveryState.get(task.id);
const loopAttempts = loopState?.attempts ?? 0;
const isContextError = isContextLimitError(errorMessage);
if (isContextError && loopAttempts < MAX_REDUCED_PROMPT_ATTEMPTS) {
const activeEntry = this.activeSessions.get(task.id);
if (activeEntry) {
executorLog.log(`${task.id} context limit error after auto-compaction — attempting reduced-prompt retry (${loopAttempts + 1}/${MAX_REDUCED_PROMPT_ATTEMPTS})`);
await this.store.logEntry(task.id, `Context limit error after auto-compaction — attempting reduced-prompt retry (${loopAttempts + 1}/${MAX_REDUCED_PROMPT_ATTEMPTS}): ${errorMessage}`, undefined, this.currentRunContext);
this.loopRecoveryState.set(task.id, { attempts: loopAttempts + 1, pending: false });
try {
this.options.stuckTaskDetector?.recordProgress(task.id);
// Build a reduced prompt that's simpler and shorter to avoid context overflow
const reducedPrompt = [
"Your previous attempt hit the context window limit.",
"Focus on completing the task efficiently with minimal context:",
"1. Review git status and git log to see what's been done",
"2. Identify the most critical remaining work",
"3. Complete it with a simpler, more focused approach",
"",
"Do not repeat what's already been done. Just complete the task and call fn_task_done.",
].join("\n");
await promptWithFallback(activeEntry.session, reducedPrompt);
checkSessionError(activeEntry.session);
await accumulateSessionTokenUsage(this.store, task.id, activeEntry.session);
// Reduced-prompt retry succeeded — return to let the finally block clean up
// without marking the task as failed.
executorLog.log(`${task.id} reduced-prompt recovery succeeded — continuing`);
await this.store.logEntry(task.id, "Reduced-prompt recovery succeeded — continuing execution", undefined, this.currentRunContext);
return;
} catch (reducedErr: unknown) {
const reducedErrorMessage = reducedErr instanceof Error ? reducedErr.message : String(reducedErr);
if (!isContextLimitError(reducedErrorMessage)) {
executorLog.error(`${task.id} reduced-prompt recovery also failed: ${reducedErrorMessage}`);
await this.store.logEntry(task.id, `Reduced-prompt recovery failed: ${reducedErrorMessage}`, undefined, this.currentRunContext);
// Non-context failure — fall through to mark task as failed
} else {
// Still a context error — the session is saturated beyond recovery.
// Fall through to the fresh-session requeue path below.
executorLog.warn(`${task.id} session still saturated after reduced-prompt retry — will attempt fresh-session requeue`);
await this.store.logEntry(task.id, `Reduced-prompt retry still over context — will attempt fresh-session requeue`, undefined, this.currentRunContext);
}
}
}
}
// Fresh-session requeue for context-limit errors: the saturated session
// cannot be salvaged, but the task's git state is intact. Move the task
// back to todo so the next scheduling pass creates a new session.
if (isContextError) {
const decision = computeRecoveryDecision({
recoveryRetryCount: task.recoveryRetryCount,
nextRecoveryAt: task.nextRecoveryAt,
});
if (decision.shouldRetry) {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
executorLog.warn(`⚡ ${task.id} context-overflow fresh-session requeue ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`);
await this.store.logEntry(task.id, `Context-overflow fresh-session requeue (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
// Retain the worktree and accumulated step progress so the fresh
// session resumes where the saturated one left off, but clear
// sessionFile synchronously here so the next dispatch is forced
// to spawn a brand-new session instead of reopening the
// over-context one. The session-end finally block also clears
// sessionFile, but it runs as fire-and-forget — if moveTask
// wins the task lock first, the next executor pass would
// observe a stale sessionFile and resume into the saturated
// session, looping on the same context-limit failure.
await this.store.updateTask(task.id, {
recoveryRetryCount: decision.nextState.recoveryRetryCount,
nextRecoveryAt: decision.nextState.nextRecoveryAt,
sessionFile: null,
});
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
return;
}
executorLog.error(`✗ ${task.id} context-overflow requeue budget exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
await this.store.logEntry(task.id, `Context-overflow requeues exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.currentRunContext);
// Reset so downstream failure path can persist cleanly
await this.store.updateTask(task.id, {
recoveryRetryCount: null,
nextRecoveryAt: null,
});
// Fall through to terminal failure marking
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
} else if (isTransientError(errorMessage)) {
// Transient network/infrastructure error — use bounded recovery policy
const decision = computeRecoveryDecision({
recoveryRetryCount: task.recoveryRetryCount,
nextRecoveryAt: task.nextRecoveryAt,
});
if (decision.shouldRetry) {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
// Silent transient errors (e.g., "request was aborted") are noisy — skip logging
if (!isSilentTransientError(errorMessage)) {
executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`);
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
}
// Clean up the old worktree so the retry gets a fresh one
if (worktreePath && existsSync(worktreePath)) {
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
executorLog.log(`Removed old worktree for transient retry: ${worktreePath}`);
// Audit trail: record worktree removal (FN-1404)
await audit.git({ type: "worktree:remove", target: worktreePath });
} catch (cleanupErr: unknown) {
const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr);
executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`);
}
}
await this.store.updateTask(task.id, {
recoveryRetryCount: decision.nextState.recoveryRetryCount,
nextRecoveryAt: decision.nextState.nextRecoveryAt,
worktree: null,
branch: null,
});
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
return;
}
// Recovery budget exhausted — escalate to real failure
executorLog.error(`✗ ${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorDetail}`);
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, errorStack ?? errorDetail, this.currentRunContext);
await this.store.updateTask(task.id, {
status: "failed",
error: errorMessage,
recoveryRetryCount: null,
nextRecoveryAt: null,
});
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
executorLog.log(`✗ ${task.id} transient retries exhausted → in-review`);
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
return;
}
executorLog.error(`✗ ${task.id} execution failed:`, errorDetail);
await this.store.logEntry(task.id, `Execution failed: ${errorMessage}`, errorStack ?? errorDetail, this.currentRunContext);
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
await this.persistTokenUsage(task.id);
await this.store.moveTask(task.id, "in-review");
executorLog.log(`✗ ${task.id} execution failed → in-review`);
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
}
} finally {
this.executing.delete(task.id);
// Clear run context at end of execute() lifecycle
this.currentRunContext = undefined;
// Terminate all spawned child agents on ALL exit paths.
// This must run here (in the outer finally) rather than only in agentWork's
// finally block, because failures during worktree creation or before
// agentWork is entered leave children orphaned with no other cleanup path.
try {
await this.terminateAllChildren(task.id);
} catch (err) {
executorLog.warn(`terminateAllChildren failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
}
// Reset loop recovery state at end of execute() lifecycle.
// State is in-memory and per-run — should not persist across attempts.
this.loopRecoveryState.delete(task.id);
this.tokenUsageBaselines.delete(task.id);
// Requeue stuck-killed task AFTER this.executing is cleared.
// This prevents the race where the scheduler re-dispatches the task
// (via task:moved → execute()) while the old execution guard is still set,
// which caused the new execute() call to silently no-op, stranding the
// task in "in-progress" with no active session or worktree.
if (stuckRequeue === true) {
try {
// Reset steps whose work was never committed before destroying the worktree
const latestTask = await this.store.getTask(task.id);
await this.resetStepsIfWorkLost(latestTask);
// Clean up the old worktree so the retry gets a fresh one
if (worktreePath && existsSync(worktreePath)) {
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
executorLog.log(`Removed old worktree for stuck-killed retry: ${worktreePath}`);
// Audit trail: record worktree removal (FN-1404)
await audit.git({ type: "worktree:remove", target: worktreePath });
} catch (cleanupErr: unknown) {
const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr);
executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`);
}
}
await this.store.updateTask(task.id, { status: "stuck-killed", worktree: null, branch: null });
// Only move to todo if not already there. The task.column check uses the
// captured task object from execute() start — if the task was already in "todo"
// when execute() started (e.g., resumed orphan), we skip the redundant move.
if (task.column !== "todo") {
await this.store.moveTask(task.id, "todo");
// Audit trail: record task move (FN-1404)
await audit.database({ type: "task:move", target: task.id, metadata: { to: "todo" } });
executorLog.log(`${task.id} moved to todo for retry after stuck kill`);
} else {
executorLog.log(`${task.id} already in todo — skipping redundant move`);
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to requeue stuck task ${task.id}: ${errorMessage}`);
}
}
}
}
// ── Custom tools for the worker agent ──────────────────────────────
private createTaskUpdateTool(
taskId: string,
codeReviewVerdicts: Map<number, ReviewVerdict>,
sessionRef: { current: AgentSession | null },
stepCheckpoints: Map<number, string>,
stuckDetector?: StuckTaskDetector,
): ToolDefinition {
const store = this.store;
return {
name: "fn_task_update",
label: "Update Step",
description:
"Update a step's status. Call before starting a step (in-progress), " +
"after completing it (done), or to skip it (skipped). " +
"The board updates in real-time.",
parameters: taskUpdateParams,
execute: async (_id: string, params: Static<typeof taskUpdateParams>) => {
const { step, status } = params;
// Record step progress for stuck task detection.
// Step transitions (in-progress, done, skipped) indicate real progress
// and reset the loop detection counter. Generic activity (text deltas,
// tool calls) is tracked separately via recordActivity in AgentLogger.
if (status === "in-progress" || status === "done" || status === "skipped") {
stuckDetector?.recordProgress(taskId);
}
// Enforce code review REVISE: block advancing to "done" when the last
// code review for this step returned REVISE. The agent must fix the
// issues and call fn_review_step(type="code") again before proceeding.
if (status === "done" && codeReviewVerdicts.get(step) === "REVISE") {
return {
content: [{
type: "text" as const,
text: `Cannot mark Step ${step} as done — the last code review returned REVISE. ` +
`Fix the issues from the code review, commit your changes, and call ` +
`fn_review_step(step=${step}, type="code") again. The step can only advance ` +
`after the code review passes.`,
}],
details: {},
};
}
const stepIndex = step - 1;
if (!Number.isInteger(stepIndex) || stepIndex < 0) {
return {
content: [{
type: "text" as const,
text: `Invalid step number: ${step}. Steps are 1-indexed.`,
}],
details: {},
};
}
const task = await store.updateStep(taskId, stepIndex, status as StepStatus);
const stepInfo = task.steps[stepIndex];
const persistedStatus = stepInfo.status;
const progress = task.steps.filter((s) => s.status === "done").length;
// Capture session checkpoint only when the store actually moved the
// step to in-progress, so RETHINK can rewind to it. Doing this AFTER
// updateStep means a regression that updateStep ignores (e.g. the
// agent re-marking an already-done step) cannot replace the
// pre-step leaf with a later one.
if (
status === "in-progress" &&
persistedStatus === "in-progress" &&
sessionRef.current
) {
const leafId = sessionRef.current.sessionManager.getLeafId();
if (leafId) {
stepCheckpoints.set(step, leafId);
}
}
// If the persisted status doesn't match the requested status, the
// store rejected the transition (currently: in-progress regression
// on a done/skipped step). Tell the agent honestly so it doesn't
// assume the step reopened.
if (persistedStatus !== status) {
return {
content: [{
type: "text" as const,
text: `Step ${step} (${stepInfo.name}) is already ${persistedStatus} — ${status} request ignored to preserve completed work. Progress: ${progress}/${task.steps.length} done.`,
}],
details: {},
};
}
return {
content: [{
type: "text" as const,
text: `Step ${step} (${stepInfo.name}) → ${persistedStatus}. Progress: ${progress}/${task.steps.length} done.`,
}],
details: {},
};
},
};
}
private createTaskLogTool(taskId: string): ToolDefinition {
return sharedCreateTaskLogTool(this.store, taskId);
}
private createTaskCreateTool(): ToolDefinition {
return sharedCreateTaskCreateTool(this.store, { sourceType: "api" }, { rootDir: this.rootDir });
}
private createTaskDocumentWriteTool(taskId: string): ToolDefinition {
return sharedCreateTaskDocumentWriteTool(this.store, taskId);
}
private createTaskDocumentReadTool(taskId: string): ToolDefinition {
return sharedCreateTaskDocumentReadTool(this.store, taskId);
}
private createTaskAddDepTool(taskId: string): ToolDefinition {
const store = this.store;
return {
name: "fn_task_add_dep",
label: "Add Dependency",
description:
"Declare a dependency on an existing task. Use when you discover " +
"mid-execution that another task must be completed first. " +
"Adding a dependency to an in-progress task will stop execution " +
"and discard current work, so confirm=true is required. " +
"Without confirm=true, a warning is returned first.",
parameters: taskAddDepParams,
execute: async (_id: string, params: Static<typeof taskAddDepParams>) => {
const targetId = params.task_id;
// Prevent self-dependency
if (targetId === taskId) {
return {
content: [{
type: "text" as const,
text: `Cannot add self-dependency: ${taskId} cannot depend on itself.`,
}],
details: {},
};
}
// Validate target task exists
try {
await store.getTask(targetId);
} catch {
return {
content: [{
type: "text" as const,
text: `Task ${targetId} not found. Cannot add dependency on a non-existent task.`,
}],
details: {},
};
}
// Read current task to get existing dependencies
const currentTask = await store.getTask(taskId);
const existing = currentTask.dependencies;
// Dedup check
if (existing.includes(targetId)) {
return {
content: [{
type: "text" as const,
text: `${targetId} is already a dependency of ${taskId}. No changes made.`,
}],
details: {},
};
}
// Confirmation gate — destructive action for in-progress tasks
if (!params.confirm) {
return {
content: [{
type: "text" as const,
text: `Warning: adding a dependency to an in-progress task will stop execution and discard current work. Call with confirm=true to proceed.`,
}],
details: {},
};
}
// Add the dependency
await store.updateTask(taskId, { dependencies: [...existing, targetId] });
await store.logEntry(taskId, `Added dependency on ${targetId} — stopping execution for re-planning`);
// Trigger abort flow (same pattern as pausedAborted)
this.depAborted.add(taskId);
const activeSession = this.activeSessions.get(taskId);
activeSession?.session.dispose();
// Also terminate step sessions if active
const stepExecutor = this.activeStepExecutors.get(taskId);
if (stepExecutor) {
stepExecutor.terminateAllSessions().catch(err =>
executorLog.warn(`Failed to terminate step sessions for dep-abort ${taskId}: ${err}`)
);
}
return {
content: [{
type: "text" as const,
text: `Added dependency on ${targetId}. Stopping execution — task will move to triage for re-planning.`,
}],
details: {},
};
},
};
}
private createTaskDoneTool(taskId: string, onDone: () => void): ToolDefinition {
const store = this.store;
return {
name: "fn_task_done",
label: "Mark Task Done",
description:
"Signal that all steps are complete, tests pass, and documentation is updated. " +
"Call this as the final action after finishing all work. " +
"Automatically marks all remaining steps as done. " +
"Optionally provide a summary of what was changed/fixed.",
parameters: Type.Object({
summary: Type.Optional(Type.String({
description: "Optional summary of what was changed/fixed and what was verified (2-4 sentences)",
})),
}),
execute: async (_id: string, params: { summary?: string }) => {
const task = await store.getTask(taskId);
const completionBlocker = await this.getTaskCompletionBlocker(task);
if (completionBlocker) {
return {
content: [{
type: "text" as const,
text: `Cannot mark task done yet — ${completionBlocker}. Resolve the blocker before calling fn_task_done().`,
}],
details: {},
};
}
onDone();
// Mark all pending/in-progress steps as done
for (let i = 0; i < task.steps.length; i++) {
if (task.steps[i].status !== "done" && task.steps[i].status !== "skipped") {
await store.updateStep(taskId, i, "done");
}
}
// Save summary if provided
if (params.summary) {
await store.updateTask(taskId, { summary: params.summary });
}
const settings = await store.getSettings();
const hardPauseActive = Boolean(task.paused || settings.globalPause);
if (hardPauseActive) {
await store.updateTask(taskId, { status: null });
} else {
await store.updateTask(taskId, { paused: false, status: null });
}
await store.logEntry(taskId, "Task marked done by agent");
const latestTask = await store.getTask(taskId);
let latestColumn = latestTask.column;
if (latestColumn === "todo") {
await store.logEntry(
taskId,
hardPauseActive
? "fn_task_done called while task was in todo during pause — promoting to in-progress for deferred completion handoff"
: "fn_task_done called while task was in todo — promoting to in-progress before completion handoff",
);
await store.moveTask(taskId, "in-progress");
latestColumn = "in-progress";
}
if (latestColumn === "in-progress" && !hardPauseActive) {
this.scheduleCompletedTaskWatchdog(taskId, "fn_task_done");
}
const successMessage = hardPauseActive
? "Task marked complete. Completion handoff deferred until pause is cleared."
: params.summary
? "Task marked complete with summary. All steps done. Moving to in-review."
: "Task marked complete. All steps done. Moving to in-review.";
return {
content: [{ type: "text" as const, text: successMessage }],
details: {},
};
},
};
}
/**
* Create the fn_review_step tool for the executor agent.
*
* When the reviewer returns a RETHINK verdict, this tool:
* 1. Runs `git reset --hard <baseline>` to revert file changes
* 2. Rewinds the conversation to the pre-step checkpoint via `session.navigateTree()`
* 3. Resets the step status to "pending"
* 4. Returns a re-prompt instructing the agent to take a different approach
*/
private createReviewStepTool(
taskId: string,
worktreePath: string,
promptContent: string,
codeReviewVerdicts: Map<number, ReviewVerdict>,
sessionRef: { current: AgentSession | null },
stepCheckpoints: Map<number, string>,
detail: TaskDetail,
stuckDetector?: StuckTaskDetector,
): ToolDefinition {
const store = this.store;
const options = this.options;
return {
name: "fn_review_step",
label: "Review Step",
description:
"Spawn a reviewer agent to evaluate your plan or code for a step. " +
"Returns APPROVE, REVISE, RETHINK, or UNAVAILABLE. " +
"Call at step boundaries based on the task's review level. " +
"Skip reviews for Step 0 (Preflight) and the final documentation step.",
parameters: reviewStepParams,
execute: async (_toolCallId: string, params: Static<typeof reviewStepParams>) => {
const { step, type: reviewType, step_name, baseline } = params;
reviewerLog.log(`${taskId}: ${reviewType} review for Step ${step} (${step_name})`);
await store.logEntry(taskId, `${reviewType} review requested for Step ${step} (${step_name})`);
try {
const settings = await store.getSettings();
// Run the reviewer via semaphore.runNested so its slot accounting
// is honest: activeCount transiently bumps to reflect the second
// agent session, but the reviewer doesn't enter the wait queue
// (avoiding a fairness regression where unrelated work could
// overtake this task at low maxConcurrent). The parent (this
// executor) makes no LLM calls while suspended awaiting the tool
// result, so the soft breach of `limit` does not push real
// LLM-active concurrency above the configured cap.
const sem = options.semaphore;
const invokeReviewer = () => reviewStep(
worktreePath, taskId, step, step_name,
reviewType, promptContent, baseline,
{
onText: (delta) => options.onAgentText?.(taskId, delta),
// Execution defaults as final fallback
defaultProvider: settings.defaultProvider,
defaultModelId: settings.defaultModelId,
fallbackProvider: settings.fallbackProvider,
fallbackModelId: settings.fallbackModelId,
defaultThinkingLevel: detail.thinkingLevel ?? settings.defaultThinkingLevel,
// Task-level validator override (from task)
taskValidatorProvider: detail.validatorModelProvider,
taskValidatorModelId: detail.validatorModelId,
// Project-level validator override
projectValidatorProvider: settings.validatorProvider,
projectValidatorModelId: settings.validatorModelId,
// Project-level validator fallback
projectValidatorFallbackProvider: settings.validatorFallbackProvider,
projectValidatorFallbackModelId: settings.validatorFallbackModelId,
// Global validator lane
globalValidatorProvider: settings.validatorGlobalProvider,
globalValidatorModelId: settings.validatorGlobalModelId,
// Project-level default override (fallback before execution defaults)
projectDefaultOverrideProvider: settings.defaultProviderOverride,
projectDefaultOverrideModelId: settings.defaultModelIdOverride,
store,
taskId,
task: detail,
agentPrompts: settings.agentPrompts,
agentStore: this.options.agentStore,
rootDir: this.rootDir,
settings,
// Track the reviewer's session under this task so it's disposed
// alongside the main session when the task moves out of
// in-progress, is paused, or the engine globally pauses.
onSessionCreated: (s) => this.registerSubagentSession(taskId, s),
onSessionEnded: (s) => this.unregisterSubagentSession(taskId, s),
},
);
const result = sem
? await sem.runNested(invokeReviewer)
: await invokeReviewer();
await store.logEntry(
taskId,
`${reviewType} review Step ${step}: ${result.verdict}`,
result.summary,
);
reviewerLog.log(`${taskId}: Step ${step} ${reviewType} → ${result.verdict}`);
stuckDetector?.recordProgress(taskId);
// Track code review verdicts for enforcement. Plan reviews remain
// advisory — only code reviews write to the verdict map.
if (reviewType === "code") {
if (result.verdict === "REVISE") {
codeReviewVerdicts.set(step, "REVISE");
} else if (result.verdict === "APPROVE") {
codeReviewVerdicts.delete(step);
}
}
let text: string;
switch (result.verdict) {
case "APPROVE": text = "APPROVE"; break;
case "REVISE":
if (reviewType === "code") {
text = `REVISE — this step cannot be marked done until the code review passes.\n\n` +
`Fix the issues below, commit your changes, and call fn_review_step(step=${step}, ` +
`type="code", step_name="${step_name}", baseline="<new SHA>") again.\n\n${result.review}`;
} else {
text = `REVISE\n\n${result.review}`;
}
break;
case "RETHINK": {
// For code reviews: git reset to baseline to revert file changes
// For plan reviews: skip git reset (no code has been written yet)
if (reviewType === "code" && baseline) {
try {
await execAsync(`git reset --hard ${baseline}`, { cwd: worktreePath });
executorLog.log(`${taskId}: RETHINK — git reset --hard ${baseline}`);
} catch (gitErr: unknown) {
const gitErrMessage = gitErr instanceof Error ? gitErr.message : String(gitErr);
executorLog.error(`${taskId}: RETHINK git reset failed: ${gitErrMessage}`);
}
} else if (reviewType === "code") {
executorLog.log(`${taskId}: RETHINK — no baseline SHA, skipping git reset`);
}
// Rewind conversation to pre-step checkpoint
const checkpointId = stepCheckpoints.get(step);
if (checkpointId && sessionRef.current) {
try {
await sessionRef.current.navigateTree(checkpointId, { summarize: false });
executorLog.log(`${taskId}: RETHINK — session rewound to checkpoint ${checkpointId}`);
} catch (rewindErr: unknown) {
const msg = rewindErr instanceof Error ? rewindErr.message : String(rewindErr);
executorLog.warn(`${taskId}: RETHINK navigateTree rewind failed, falling back to branchWithSummary: ${msg}`);
// Fallback to branchWithSummary
try {
sessionRef.current.sessionManager.branchWithSummary(
checkpointId,
`RETHINK: ${result.summary || "Approach rejected by reviewer"}`,
);
executorLog.log(`${taskId}: RETHINK — branched from checkpoint ${checkpointId}`);
} catch (branchErr: unknown) {
const branchErrMessage = branchErr instanceof Error ? branchErr.message : String(branchErr);
executorLog.error(`${taskId}: RETHINK session rewind failed: ${branchErrMessage}`);
}
}
} else {
executorLog.log(`${taskId}: RETHINK — no session checkpoint for step ${step}, skipping rewind`);
}
// Reset step status to pending
await store.updateStep(taskId, step, "pending");
if (reviewType === "plan") {
await store.logEntry(
taskId,
`RETHINK: Step ${step} plan rewound — session checkpoint ${checkpointId || "N/A"}`,
result.summary,
);
text = `RETHINK\n\nYour plan was rejected. Here is why:\n\n${result.review}\n\nTake a different approach to planning this step. Do NOT repeat the rejected strategy.`;
} else {
await store.logEntry(
taskId,
`RETHINK: Step ${step} rewound — git reset to ${baseline || "N/A"}, session checkpoint ${checkpointId || "N/A"}`,
result.summary,
);
text = `RETHINK\n\nYour previous approach was rejected. Here is why:\n\n${result.review}\n\nTake a different approach. Do NOT repeat the rejected strategy. Re-read the step requirements and find an alternative solution.`;
}
break;
}
default: text = "UNAVAILABLE — reviewer did not produce a usable verdict.";
}
return { content: [{ type: "text" as const, text }], details: {} };
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
reviewerLog.error(`${taskId}: review failed: ${errorMessage}`);
await store.logEntry(taskId, `${reviewType} review failed: ${errorMessage}`);
return {
content: [{ type: "text" as const, text: `UNAVAILABLE — reviewer error: ${errorMessage}` }],
details: {},
};
}
},
};
}
/**
* Clean up after a dep-abort: remove worktree, delete branch, move task to triage.
* Shared between the try-block (graceful return) and catch-block (error) paths.
*/
private async handleDepAbortCleanup(taskId: string, worktreePath: string): Promise<void> {
executorLog.log(`${taskId} dependency added — work discarded, moved to triage for re-planning`);
// Remove worktree
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: failed to remove worktree during dep-abort cleanup (${worktreePath}): ${msg}`);
}
// Delete the branch — use stored branch name if available, fall back to convention
const task = await this.store.getTask(taskId);
const branch = task.branch || `fusion/${taskId.toLowerCase()}`;
let branchDeleted = false;
try {
await execAsync(`git branch -D "${branch}"`, { cwd: this.rootDir });
branchDeleted = true;
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: failed to delete branch during dep-abort cleanup (${branch}): ${msg}`);
}
if (branchDeleted) {
// FN-2165 regression guard: null baseBranch on any task that stored this branch
try { this.store.clearStaleBaseBranchReferences([branch], taskId); } catch { /* best-effort */ }
}
// Clear worktree tracking
this.activeWorktrees.delete(taskId);
// Update task: clear worktree and status, move to triage
await this.store.updateTask(taskId, { worktree: null, status: null });
await this.store.moveTask(taskId, "triage");
await this.store.logEntry(taskId, "Execution stopped — work discarded, moved to triage for re-planning");
}
/**
* Handle a workflow step revision request.
*
* Re-opens ONLY the last step so the executor has exactly one pending slot
* to re-enter through. All earlier done steps stay done — the agent reads
* the injected feedback from PROMPT.md and applies an in-place fix rather
* than redoing any completed step.
*/
private async handleWorkflowRevisionRequest(
task: Task,
worktreePath: string,
feedback: string,
stepName: string,
): Promise<void> {
executorLog.log(`${task.id}: workflow revision requested by step "${stepName}"`);
this.clearCompletedTaskWatchdog(task.id);
const updatedTask = await this.store.getTask(task.id);
const reopen = await this.reopenLastStepForRevision(task.id, updatedTask);
const reopenSummary = reopen
? `re-opening Step ${reopen.index + 1} ("${reopen.name}") for in-place fix`
: "no step to re-open (none were completed)";
await this.store.logEntry(
task.id,
`Workflow step "${stepName}" requested revision — ${reopenSummary}`,
feedback,
);
await this.injectWorkflowRevisionInstructions(task, feedback);
await this.store.updateTask(task.id, {
status: null,
sessionFile: null,
});
// 4. Schedule fresh execution after guard unwinds
// This prevents the race condition where the scheduler re-dispatches
// while the old execution guard is still set.
executorLog.log(`${task.id}: scheduling fresh execution after revision request`);
this.scheduleWorkflowRerun(
task.id,
worktreePath,
`${task.id}: revision rerun scheduled — moved to todo then in-progress`,
);
}
/**
* Re-open the last non-pending step so a revision/failure handler gives the
* executor exactly one pending slot to re-enter through. Returns the index
* and name of the step that was flipped to `pending`, or null when there
* was nothing to re-open.
*/
private async reopenLastStepForRevision(
taskId: string,
task: Task,
): Promise<{ index: number; name: string } | null> {
const steps = task.steps;
if (steps.length === 0) return null;
let targetIndex = -1;
for (let i = steps.length - 1; i >= 0; i--) {
if (steps[i].status !== "pending") {
targetIndex = i;
break;
}
}
if (targetIndex === -1) {
await this.store.updateTask(taskId, { currentStep: 0 });
return null;
}
await this.store.updateStep(taskId, targetIndex, "pending");
await this.store.updateTask(taskId, { currentStep: targetIndex });
return { index: targetIndex, name: steps[targetIndex].name };
}
/**
* Inject or update the "Workflow Revision Instructions" section in PROMPT.md.
* This section contains feedback from workflow steps that requested revisions.
* The section is replaced entirely to avoid accumulation of old feedback.
*/
private async injectWorkflowRevisionInstructions(
task: Task,
feedback: string,
): Promise<void> {
const promptPath = join(this.store.getFusionDir(), "tasks", task.id, "PROMPT.md");
// Read existing PROMPT.md
let content: string;
try {
content = await readFile(promptPath, "utf-8");
} catch {
executorLog.warn(`${task.id}: PROMPT.md not found at ${promptPath}, skipping revision injection`);
return;
}
// All prior steps stay done — agent applies the feedback as an in-place
// patch rather than re-planning or re-executing earlier steps.
const scopeLine = "All prior steps remain **done**. Apply the feedback above as an in-place fix (make the necessary code changes, commit, and call `fn_task_done()` when complete). Do **not** re-run or re-plan any earlier step unless the feedback explicitly calls it out.";
// Check for existing Workflow Revision Instructions section
const revisionSectionHeader = "## Workflow Revision Instructions";
const revisionSectionContent = `${revisionSectionHeader}
The following feedback was received from quality gates and requires implementation changes:
${feedback}
**Important:** ${scopeLine}
`;
let newContent: string;
if (content.includes(revisionSectionHeader)) {
// Replace existing section
const sectionRegex = new RegExp(
`${revisionSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`,
"i"
);
if (sectionRegex.test(content)) {
newContent = content.replace(sectionRegex, revisionSectionContent);
} else {
// Fallback: append at end
newContent = content + "\n" + revisionSectionContent;
}
} else {
// Append new section before any closing markers or at end
// Look for common markers like "## Acceptance Criteria" or just append
const acceptanceCriteriaMatch = content.match(/\n##\s+Acceptance Criteria\n/);
if (acceptanceCriteriaMatch) {
const insertIdx = acceptanceCriteriaMatch.index!;
newContent = content.slice(0, insertIdx) + "\n" + revisionSectionContent + content.slice(insertIdx);
} else {
newContent = content + "\n" + revisionSectionContent;
}
}
// Write updated content
try {
await writeFile(promptPath, newContent);
executorLog.log(`${task.id}: injected workflow revision instructions into PROMPT.md`);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`${task.id}: failed to inject revision instructions: ${errorMessage}`);
}
}
/**
* Handle workflow step hard failures by retrying execution up to MAX_WORKFLOW_STEP_RETRIES times.
* This gives the executor a chance to fix workflow step failures automatically before
* moving the task to in-review with failed status.
*
* @returns true if a retry was scheduled, false if retries are exhausted
*/
/**
* Run deterministic verification (test + build commands) in the task's worktree.
* Returns a structured result indicating whether all commands passed.
*/
private async runExecutorDeterministicVerification(
task: Task,
worktreePath: string,
settings: Settings,
): Promise<VerificationResult> {
const testCommand = settings.testCommand?.trim();
const buildCommand = settings.buildCommand?.trim();
if (!testCommand && !buildCommand) {
executorLog.log(`${task.id}: no test/build commands configured — skipping verification`);
return { allPassed: true };
}
const parts: string[] = [];
if (testCommand) parts.push(`test: ${testCommand}`);
if (buildCommand) parts.push(`build: ${buildCommand}`);
executorLog.log(`${task.id}: [verification] running deterministic verification (${parts.join(", ")})`);
await this.store.logEntry(
task.id,
`[verification] Running deterministic verification (${parts.join(", ")})`,
undefined,
this.currentRunContext,
);
const result: VerificationResult = { allPassed: true };
// Run test command first if configured
if (testCommand) {
const testResult = await runVerificationCommand(
this.store, worktreePath, task.id, testCommand, "test", undefined, executorLog, "executor",
);
result.testResult = testResult;
if (!testResult.success) {
result.allPassed = false;
result.failedCommand = "testCommand";
executorLog.log(`${task.id}: [verification] test failed (exit ${testResult.exitCode})`);
return result;
}
}
// Run build command second if configured
if (buildCommand) {
const buildResult = await runVerificationCommand(
this.store, worktreePath, task.id, buildCommand, "build", undefined, executorLog, "executor",
);
result.buildResult = buildResult;
if (!buildResult.success) {
result.allPassed = false;
result.failedCommand = "buildCommand";
executorLog.log(`${task.id}: [verification] build failed (exit ${buildResult.exitCode})`);
return result;
}
}
executorLog.log(`${task.id}: [verification] passed`);
await this.store.logEntry(
task.id,
`[verification] Deterministic verification passed`,
undefined,
this.currentRunContext,
);
return result;
}
/**
* Attempt to fix verification failures by spawning a dedicated AI fix agent.
* Follows the pattern established by the merger's attemptInMergeVerificationFix.
* Returns true if verification passes after the fix attempt, false otherwise.
*/
private async attemptExecutorVerificationFix(
task: Task,
worktreePath: string,
failureContext: {
command: string;
exitCode: number | null;
output: string;
type: "test" | "build";
},
settings: Settings,
retryNumber: number,
maxRetries: number,
): Promise<boolean> {
try {
executorLog.log(`${task.id}: spawning executor verification fix agent (attempt ${retryNumber}/${maxRetries})`);
const logger = new AgentLogger({
store: this.store,
taskId: task.id,
agent: "executor",
persistAgentToolOutput: settings.persistAgentToolOutput,
onAgentText: this.options.onAgentText,
onAgentTool: this.options.onAgentTool,
});
// Build skill selection context
let skillContext: Awaited<ReturnType<typeof buildSessionSkillContext>> | undefined;
if (this.options.agentStore) {
try {
skillContext = await buildSessionSkillContext({
agentStore: this.options.agentStore,
task,
sessionPurpose: "executor",
projectRootDir: worktreePath,
pluginRunner: this.options.pluginRunner,
});
} catch {
// Graceful fallback - no skill selection
}
}
// Resolve model using the executor's model hierarchy
const { provider: executorProvider, modelId: executorModelId } = resolveExecutorModelPair(
task.modelProvider,
task.modelId,
settings,
);
// Create the fix agent session
const { session } = await createResolvedAgentSession({
sessionPurpose: "executor",
pluginRunner: this.options.pluginRunner,
cwd: worktreePath, // Run in the task's worktree
systemPrompt: `You are a verification fix agent running during task execution in a worktree.
All step-session steps completed successfully but the deterministic verification command failed. Your job is to fix the failing code directly in the working directory.
## Scope
Only fix what is required to make the failing verification pass.
Do not refactor, rename broadly, or make opportunistic improvements.
## Rules
1. Read the error output carefully to understand what is failing before editing anything
2. Before assuming a code fix is needed, check whether the failure is caused by stale/missing build artifacts in a sibling workspace package — typical signatures: \`Failed to resolve import "./X.js"\` pointing into another package's \`dist/\`, \`Cannot find module\`, or \`ERR_MODULE_NOT_FOUND\` referencing a workspace-internal path. In that case, rebuild the affected package(s) (e.g. \`pnpm --filter <pkg> build\`, or \`pnpm --filter "<scope>/*" build\` for a group) and re-run verification before editing source files.
3. Make targeted fixes to the failing code path
4. After fixing, run the verification command to confirm the fix works
5. Do NOT make any git commits — just fix the code
6. You MAY modify any files needed to make the verification pass, including files unrelated to this task's original change. Pre-existing build/test breakage is in scope: fix it. Prefer the smallest change that makes verification green.
7. If you cannot fix the issue within scope, explain why and what evidence indicates a deeper/root problem`,
tools: "coding",
onText: logger.onText,
onThinking: logger.onThinking,
onToolStart: logger.onToolStart,
onToolEnd: logger.onToolEnd,
defaultProvider: executorProvider,
defaultModelId: executorModelId,
defaultThinkingLevel: settings.defaultThinkingLevel,
...(skillContext?.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
});
await this.store.logEntry(
task.id,
`Executor verification fix agent started (model: ${describeModel(session)}, attempt ${retryNumber}/${maxRetries})`,
undefined,
this.currentRunContext,
);
await this.store.appendAgentLog(
task.id,
`Fix agent started (model: ${describeModel(session)}, attempt ${retryNumber}/${maxRetries})`,
"text",
undefined,
"executor",
);
try {
// Build the fix prompt
const fixPrompt = `Fix the failing ${failureContext.type} verification for task ${task.id}.
## Failed command
Command: \`${failureContext.command}\`
Exit code: ${failureContext.exitCode}
## Error output
${failureContext.output.slice(0, VERIFICATION_LOG_MAX_CHARS)}
## Instructions
1. Read the error output and identify the root cause
2. Make targeted fixes to resolve the failure
3. Run the verification command \`${failureContext.command}\` to confirm your fix works
4. If the fix doesn't work, try a different approach
5. Do NOT make any git commits`;
// Run the agent with rate limit retry
await withRateLimitRetry(async () => {
await promptWithFallback(session, fixPrompt);
}, {
onRetry: (attempt, delayMs, error) => {
const delaySec = Math.round(delayMs / 1000);
executorLog.warn(`⏳ ${task.id} executor fix agent rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`);
},
});
await accumulateSessionTokenUsage(this.store, task.id, session);
// Re-run full deterministic verification (test AND build) after the fix attempt
executorLog.log(`${task.id}: re-running deterministic verification after fix attempt ${retryNumber}/${maxRetries}`);
await this.store.logEntry(
task.id,
`Re-running deterministic verification (attempt ${retryNumber}/${maxRetries})`,
undefined,
this.currentRunContext,
);
await this.store.appendAgentLog(
task.id,
`Re-running verification (attempt ${retryNumber}/${maxRetries})`,
"text",
undefined,
"executor",
);
const reRunResult = await this.runExecutorDeterministicVerification(task, worktreePath, settings);
return reRunResult.allPassed;
} finally {
await logger.flush();
await session.dispose();
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id}: executor verification fix agent error: ${errorMessage}`);
await this.store.logEntry(
task.id,
`Executor verification fix agent encountered an error`,
errorMessage,
this.currentRunContext,
);
await this.store.appendAgentLog(
task.id,
"Fix agent encountered an error",
"tool_error",
errorMessage,
"executor",
);
return false;
}
}
private async handleWorkflowStepFailure(
task: Task,
worktreePath: string,
failureFeedback: string,
stepName: string,
): Promise<boolean> {
this.clearCompletedTaskWatchdog(task.id);
const currentRetries = task.workflowStepRetries ?? 0;
if (currentRetries >= MAX_WORKFLOW_STEP_RETRIES) {
// Retries exhausted — caller should fall through to hard failure
executorLog.warn(`${task.id}: workflow step "${stepName}" failed — retries exhausted (${MAX_WORKFLOW_STEP_RETRIES}/${MAX_WORKFLOW_STEP_RETRIES})`);
return false;
}
const retryCount = currentRetries + 1;
executorLog.log(`${task.id}: workflow step "${stepName}" failed — retry ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES} (executor will attempt to fix)`);
// 1. Update the workflowStepRetries counter on the task
await this.store.updateTask(task.id, {
workflowStepRetries: retryCount,
});
// 2. Inject failure feedback into PROMPT.md
await this.injectWorkflowStepFailureInstructions(task, failureFeedback, stepName, retryCount);
// 3. Re-open only the last step so the executor has a single pending
// slot to re-enter. Earlier done steps stay done.
const updatedTask = await this.store.getTask(task.id);
await this.reopenLastStepForRevision(task.id, updatedTask);
// 4. Clear any session file so we get a fresh session
await this.store.updateTask(task.id, {
status: null,
sessionFile: null,
});
// 5. Schedule fresh execution after guard unwinds
executorLog.log(`${task.id}: scheduling fresh execution after workflow step failure (retry ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES})`);
this.scheduleWorkflowRerun(
task.id,
worktreePath,
`${task.id}: workflow step retry scheduled — moved to todo then in-progress`,
);
return true;
}
/**
* Send a task back to in-progress after verification failure.
* Injects failure feedback into PROMPT.md, resets steps, clears session,
* and schedules a move to todo → in-progress after the executing guard clears.
*/
private async sendTaskBackForFix(
task: Task,
worktreePath: string,
failureFeedback: string,
stepName: string,
reason: string,
preserveResumeState: boolean = true,
mergeVerificationFailure: boolean = false,
): Promise<void> {
const taskId = task.id;
this.clearCompletedTaskWatchdog(taskId);
// 1. Add a task comment explaining the failure
await this.store.addTaskComment(
taskId,
`${reason}. The failing workflow step was "${stepName}". ` +
`Feedback:\n${failureFeedback}\n\n` +
`Please fix the issues so the verification can pass on the next attempt.`,
"agent",
);
// 2. Log an entry explaining the task was sent back
await this.store.logEntry(
taskId,
`${reason} — moved back to in-progress for remediation`,
);
// 3. Inject failure feedback into PROMPT.md using the existing method
// Pass MAX_WORKFLOW_STEP_RETRIES to indicate retries are exhausted (shows "3/3 (0 remaining)")
await this.injectWorkflowStepFailureInstructions(task, failureFeedback, stepName, MAX_WORKFLOW_STEP_RETRIES);
// 4. Re-open only the last step for a single in-place fix pass. Earlier
// done steps stay done so the executor doesn't redo finished work.
const updatedTask = await this.store.getTask(taskId);
await this.reopenLastStepForRevision(taskId, updatedTask);
// 5. Clear error/status/session fields and reset workflow step retries
await this.store.updateTask(taskId, {
status: mergeVerificationFailure ? "merging-fix" : null,
error: null,
sessionFile: null,
workflowStepRetries: 0,
});
// 6. Schedule the move after the guard unwinds (per guard-unwind requirement)
this.scheduleWorkflowRerun(
taskId,
worktreePath,
`${taskId}: sent back to in-progress for remediation`,
preserveResumeState,
);
}
/**
* Inject or update the "Workflow Step Failure" section in PROMPT.md.
* This section contains failure feedback from workflow steps that hard-failed.
* The section is replaced entirely to avoid accumulation of old feedback.
*/
private async injectWorkflowStepFailureInstructions(
task: Task,
failureFeedback: string,
stepName: string,
retryCount: number,
): Promise<void> {
const promptPath = join(this.store.getFusionDir(), "tasks", task.id, "PROMPT.md");
// Read existing PROMPT.md
let content: string;
try {
content = await readFile(promptPath, "utf-8");
} catch {
executorLog.warn(`${task.id}: PROMPT.md not found at ${promptPath}, skipping workflow failure injection`);
return;
}
const remainingRetries = MAX_WORKFLOW_STEP_RETRIES - retryCount;
const failureSectionHeader = "## Workflow Step Failure";
const failureSectionContent = `${failureSectionHeader}
The following workflow step failed and requires implementation fixes:
**Step:** ${stepName}
**Failure Feedback:**
${failureFeedback}
**Retry:** ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES} (${remainingRetries} remaining)
**Important:** This is a workflow step failure — fix the issues above by making the necessary code changes. The task has been sent back to in-progress for remediation. The executor will attempt to fix the issues on the next pass.
`;
let newContent: string;
if (content.includes(failureSectionHeader)) {
// Replace existing section
const sectionRegex = new RegExp(
`${failureSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`,
"i"
);
if (sectionRegex.test(content)) {
newContent = content.replace(sectionRegex, failureSectionContent);
} else {
// Fallback: append at end
newContent = content + "\n" + failureSectionContent;
}
} else {
// Remove any existing Workflow Revision Instructions section first (conflicting state)
const revisionSectionHeader = "## Workflow Revision Instructions";
if (content.includes(revisionSectionHeader)) {
const revisionRegex = new RegExp(
`${revisionSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`,
"i"
);
content = content.replace(revisionRegex, "");
}
// Append new section before any closing markers or at end
const acceptanceCriteriaMatch = content.match(/\n##\s+Acceptance Criteria\n/);
if (acceptanceCriteriaMatch) {
const insertIdx = acceptanceCriteriaMatch.index!;
newContent = content.slice(0, insertIdx) + "\n" + failureSectionContent + content.slice(insertIdx);
} else {
newContent = content + "\n" + failureSectionContent;
}
}
// Write updated content
try {
await writeFile(promptPath, newContent);
executorLog.log(`${task.id}: injected workflow step failure instructions into PROMPT.md (retry ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES})`);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`${task.id}: failed to inject workflow step failure instructions: ${errorMessage}`);
}
}
/**
* Capture the list of files modified during agent execution.
* Uses git diff against the stored baseCommitSha to determine what changed.
* Returns an empty array if no changes or if git commands fail.
*/
private async resolveDiffBaseRef(worktreePath: string, baseCommitSha?: string): Promise<string | undefined> {
if (baseCommitSha) return baseCommitSha;
try {
const { stdout } = await execAsync(
"git merge-base HEAD origin/main 2>/dev/null || git merge-base HEAD main",
{ cwd: worktreePath, encoding: "utf-8" },
);
const ref = stdout.trim();
if (ref) return ref;
} catch (mergeBaseErr: unknown) {
const mergeBaseMsg = mergeBaseErr instanceof Error ? mergeBaseErr.message : String(mergeBaseErr);
executorLog.warn(`Failed merge-base lookup for diff base in ${worktreePath}, trying HEAD~1 fallback: ${mergeBaseMsg}`);
}
try {
const { stdout } = await execAsync("git rev-parse HEAD~1", {
cwd: worktreePath,
encoding: "utf-8",
});
return stdout.trim() || undefined;
} catch {
executorLog.log(`Could not determine base commit for diff in ${worktreePath}`);
return undefined;
}
}
private async captureModifiedFiles(worktreePath: string, baseCommitSha?: string): Promise<string[]> {
try {
const baseRef = await this.resolveDiffBaseRef(worktreePath, baseCommitSha);
if (!baseRef) {
return [];
}
// Get list of modified files using git diff --name-only
const { stdout } = await execAsync(`git diff --name-only ${baseRef}..HEAD`, {
cwd: worktreePath,
encoding: "utf-8",
});
const output = stdout.trim();
if (!output) {
return [];
}
return output.split("\n").filter(Boolean);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.log(`Failed to capture modified files: ${errorMessage}`);
return [];
}
}
// ── Worktree management ────────────────────────────────────────────
/**
* Create a git worktree at `path` on a new branch.
*
* @param branch — Branch name (e.g., `fusion/fn-042`)
* @param path — Absolute worktree directory path
* @param startPoint — Optional git ref to branch from (e.g., `fusion/fn-041`).
* When provided, the worktree starts from that ref instead of HEAD.
*/
/**
* Run workflow step agents sequentially after main task execution completes.
* Each workflow step spawns a separate agent with the step's prompt.
* Returns structured result: all passed, all passed (true), failed (false), or revision requested.
*/
private async runWorkflowSteps(
task: Task,
worktreePath: string,
settings: Settings,
): Promise<WorkflowStepResult | "deferred-paused"> {
// Check if task has enabled workflow steps
const currentTask = await this.store.getTask(task.id);
if (!currentTask.enabledWorkflowSteps?.length) return { allPassed: true };
const workflowStepIds = currentTask.enabledWorkflowSteps;
const results: import("@fusion/core").WorkflowStepResult[] = [];
for (const wsId of workflowStepIds) {
const ws = await this.store.getWorkflowStep(wsId);
if (!ws) {
await this.store.logEntry(task.id, `[pre-merge] Workflow step ${wsId} not found — skipping`);
results.push({
workflowStepId: wsId,
workflowStepName: "Unknown",
phase: "pre-merge",
status: "skipped",
output: "Workflow step definition not found",
});
await this.store.updateTask(task.id, { workflowStepResults: results });
continue;
}
// Normalize legacy steps: undefined phase → "pre-merge"
const stepPhase = ws.phase || "pre-merge";
// readonly review steps always run pre-merge to reuse the coding worktree — see FN-2185 post-mortem.
// Skip non-readonly post-merge steps — those run in the merger after merge.
if (stepPhase === "post-merge" && ws.toolMode !== "readonly") continue;
// Normalize legacy steps without mode to prompt-mode
const stepMode: "prompt" | "script" = ws.mode || "prompt";
// Skip validation per mode
if (stepMode === "prompt" && !ws.prompt?.trim()) {
await this.store.logEntry(task.id, `[pre-merge] Workflow step '${ws.name}' has no prompt — skipping`);
results.push({
workflowStepId: ws.id,
workflowStepName: ws.name,
phase: stepPhase,
status: "skipped",
output: "No prompt configured for this workflow step",
});
await this.store.updateTask(task.id, { workflowStepResults: results });
continue;
}
if (stepMode === "script" && !ws.scriptName?.trim()) {
await this.store.logEntry(task.id, `[pre-merge] Workflow step '${ws.name}' has no scriptName — skipping`);
results.push({
workflowStepId: ws.id,
workflowStepName: ws.name,
phase: stepPhase,
status: "skipped",
output: "No scriptName configured for this workflow step",
});
await this.store.updateTask(task.id, { workflowStepResults: results });
continue;
}
if (await this.shouldDeferWorkflowStepCompletion(task.id, `before workflow step '${ws.name}'`)) {
return "deferred-paused";
}
await this.store.logEntry(task.id, `[pre-merge] Starting workflow step: ${ws.name} (${stepMode} mode)`);
executorLog.log(`${task.id} — [pre-merge] running workflow step: ${ws.name} (${stepMode} mode)`);
const startedAt = new Date().toISOString();
const stepStartedAtMs = Date.now();
// Push pending entry BEFORE execution so dashboard can show live status
results.push({
workflowStepId: ws.id,
workflowStepName: ws.name,
phase: stepPhase,
status: "pending",
startedAt,
});
await this.store.updateTask(task.id, { workflowStepResults: results });
try {
const result: WorkflowStepOutcome = stepMode === "script"
? await this.executeScriptWorkflowStep(task, ws, worktreePath, settings)
: await this.executeWorkflowStep(task, ws, worktreePath, settings);
if (await this.shouldDeferWorkflowStepCompletion(task.id, `workflow step '${ws.name}'`)) {
return "deferred-paused";
}
const completedAt = new Date().toISOString();
if (result.success) {
await this.store.logEntry(task.id, `[timing] Workflow step '${ws.name}' completed in ${Date.now() - stepStartedAtMs}ms`);
await this.store.logEntry(task.id, `[pre-merge] Workflow step completed: ${ws.name}`);
executorLog.log(`${task.id} — [pre-merge] workflow step passed: ${ws.name}`);
// Update existing pending entry in place
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
if (existingIdx >= 0) {
results[existingIdx] = {
...results[existingIdx],
status: "passed",
output: result.output,
completedAt,
};
}
await this.store.updateTask(task.id, { workflowStepResults: results });
} else if (result.revisionRequested) {
// Revision requested — this is a structured outcome that routes back to executor
await this.store.logEntry(task.id, `[timing] Workflow step '${ws.name}' requested revision after ${Date.now() - stepStartedAtMs}ms`);
await this.store.logEntry(
task.id,
`[pre-merge] Workflow step requested revision: ${ws.name}`,
result.output,
);
executorLog.log(`${task.id} — [pre-merge] workflow step requested revision: ${ws.name}`);
// Update existing pending entry in place
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
if (existingIdx >= 0) {
results[existingIdx] = {
...results[existingIdx],
status: "failed",
output: result.output || "Revision requested",
completedAt,
};
}
await this.store.updateTask(task.id, { workflowStepResults: results });
return {
allPassed: false,
revisionRequested: true,
feedback: result.output || "Workflow step requested revision",
stepName: ws.name,
};
} else {
// Hard failure
await this.store.logEntry(task.id, `[timing] Workflow step '${ws.name}' failed after ${Date.now() - stepStartedAtMs}ms`);
await this.store.logEntry(
task.id,
`[pre-merge] Workflow step failed: ${ws.name}`,
result.error || "Unknown error",
);
executorLog.error(`${task.id} — [pre-merge] workflow step failed: ${ws.name}; output captured in task log`);
// Update existing pending entry in place
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
if (existingIdx >= 0) {
results[existingIdx] = {
...results[existingIdx],
status: "failed",
output: result.error || "Workflow step failed",
completedAt,
};
}
await this.store.updateTask(task.id, { workflowStepResults: results });
return {
allPassed: false,
revisionRequested: false,
feedback: result.error || "Workflow step failed",
stepName: ws.name,
};
}
} catch (err: unknown) {
if (await this.shouldDeferWorkflowStepCompletion(task.id, `workflow step '${ws.name}'`)) {
return "deferred-paused";
}
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
const completedAt = new Date().toISOString();
await this.store.logEntry(
task.id,
`[pre-merge] Workflow step failed: ${ws.name}`,
errorStack ?? errorDetail,
);
executorLog.error(`${task.id} — [pre-merge] workflow step error: ${ws.name} — ${errorDetail}`);
// Update existing pending entry in place
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
if (existingIdx >= 0) {
results[existingIdx] = {
...results[existingIdx],
status: "failed",
output: errorMessage || "Workflow step error",
completedAt,
};
}
await this.store.updateTask(task.id, { workflowStepResults: results });
return {
allPassed: false,
revisionRequested: false,
feedback: errorMessage || "Workflow step error",
stepName: ws.name,
};
}
}
return { allPassed: true };
}
/**
* Execute a script-mode workflow step by resolving the scriptName to a command
* from project settings and running it in the task worktree.
*/
private async executeScriptWorkflowStep(
task: Task,
workflowStep: WorkflowStep,
worktreePath: string,
settings: Settings,
): Promise<{ success: boolean; output?: string; error?: string }> {
const scriptName = workflowStep.scriptName!.trim();
const scriptCommand = settings.scripts?.[scriptName];
if (!scriptCommand) {
const available = settings.scripts ? Object.keys(settings.scripts).join(", ") : "none";
const msg = `Script '${scriptName}' not found in project settings. Available scripts: ${available}`;
await this.store.logEntry(task.id, msg);
return { success: false, error: msg };
}
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' executing script '${scriptName}': ${scriptCommand}`);
await this.store.logEntry(task.id, `Workflow step '${workflowStep.name}' executing script '${scriptName}': ${scriptCommand}`);
try {
const scriptResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000);
if (scriptResult.spawnError || scriptResult.timedOut || scriptResult.exitCode !== 0) {
return { success: false, error: configuredCommandErrorMessage(scriptResult) };
}
return { success: true, output: `Script '${scriptName}' completed successfully` };
} catch (err: unknown) {
const execError = err instanceof Error ? err : new Error(String(err));
const stderr = "stderr" in execError && typeof execError.stderr === "string" ? execError.stderr.trim() : "";
const stdout = "stdout" in execError && typeof execError.stdout === "string" ? execError.stdout.trim() : "";
const exitCode = "code" in execError ? execError.code : ("status" in execError ? execError.status : undefined);
const parts: string[] = [];
if (exitCode !== undefined) parts.push(`Exit code: ${exitCode}`);
if (stdout) parts.push(`stdout: ${truncateWorkflowScriptOutput(stdout)}`);
if (stderr) parts.push(`stderr: ${truncateWorkflowScriptOutput(stderr)}`);
if (!parts.length) parts.push(execError.message || "Unknown error");
const errorOutput = parts.join("\n");
return { success: false, error: errorOutput };
}
}
/**
* Execute a single workflow step by spawning an agent with the step's prompt.
* Returns structured outcome with support for revision requests.
*/
private async executeWorkflowStep(
task: Task,
workflowStep: WorkflowStep,
worktreePath: string,
settings: Settings,
): Promise<WorkflowStepOutcome> {
const toolMode: "coding" | "readonly" = workflowStep.toolMode || "readonly";
// Compute the diff scope so the workflow step agent reviews only what THIS
// task changed — not unrelated files it might wander into. Without this,
// open-ended review prompts (e.g. "verify visual polish") have been
// observed to spend the entire timeout budget reading pre-existing files
// that match the task description's keywords. See FN-3327 post-mortem.
const scopedFiles = await this.captureModifiedFiles(worktreePath, task.baseCommitSha);
let diffShortstat: string | undefined;
try {
const baseRef = await this.resolveDiffBaseRef(worktreePath, task.baseCommitSha);
if (baseRef) {
const { stdout } = await execAsync(`git diff --shortstat ${baseRef}..HEAD`, {
cwd: worktreePath,
encoding: "utf-8",
});
diffShortstat = stdout.trim() || undefined;
}
} catch {
// best-effort — fall through with no shortstat
}
const MAX_SCOPE_FILES = 100;
const scopeFileBlock = scopedFiles.length === 0
? "(no modified files detected for this task — review the worktree directly, but do NOT browse unrelated files)"
: scopedFiles.length > MAX_SCOPE_FILES
? `${scopedFiles.slice(0, MAX_SCOPE_FILES).map((f) => `- ${f}`).join("\n")}\n- ... (${scopedFiles.length - MAX_SCOPE_FILES} more files truncated)`
: scopedFiles.map((f) => `- ${f}`).join("\n");
const scopeBlock = `Diff Scope (files changed by THIS task vs base):
${scopeFileBlock}${diffShortstat ? `\nDiff stat: ${diffShortstat}` : ""}
CRITICAL SCOPING RULES — read before doing anything else:
- Review ONLY the files listed above. Do NOT analyze unmodified files or unrelated parts of the codebase.
- If NONE of the files in the diff scope are relevant to your review category (e.g. a UX/design reviewer with no UI/CSS/component files in scope, a security reviewer with no auth/network code in scope, an a11y reviewer with no markup changes), respond IMMEDIATELY with a single short approval line such as "No relevant changes in scope — approved." and STOP. Do not start exploring the codebase.
- Your wall-clock budget is short. Spending it browsing unmodified files will cause this step to time out and block merge.`;
const systemPrompt = `You are a workflow step agent executing: ${workflowStep.name}
Task Context:
- Task ID: ${task.id}
- Task Description: ${task.description}
- Worktree: ${worktreePath}
${scopeBlock}
Your role:
- Execute this workflow step exactly as scoped.
- Prioritize high-impact correctness/risk findings over stylistic nits.
- Keep feedback actionable and directly tied to evidence in files/outputs.
Your Instructions:
${workflowStep.prompt}
You have access to the file system to review changes.
## Feedback Format
When your review is complete, you MUST use one of these exact formats:
**For PASS (no issues found):**
Simply state your findings and approval. No special formatting required.
**For REVISION REQUESTED (issues found that require code changes):**
Your response MUST start with the exact phrase:
\`REQUEST REVISION\`
Followed by a clear, actionable description of what needs to be fixed.
Be specific: reference exact files, line numbers, or functions that need changes.
Example:
\`REQUEST REVISION
The login function in src/auth.ts does not handle the case where the user
account is locked. Add proper error handling for the LOCKED_ACCOUNT error code
and show an appropriate message to the user.\`
**Important:**
- Only use "REQUEST REVISION" when the implementation needs code changes.
- If the code is correct and no changes are needed, just state your findings.
- Be constructive and actionable — vague feedback wastes the executor's time.`;
const agentLogger = new AgentLogger({
store: this.store,
taskId: task.id,
agent: "reviewer",
persistAgentToolOutput: settings.persistAgentToolOutput,
onAgentText: (taskId, delta) => {
this.options.onAgentText?.(taskId, delta);
},
onAgentTool: (taskId, toolName) => {
this.options.onAgentTool?.(taskId, toolName);
},
});
// Determine primary model and an explicit fallback. The workflow step's
// own override takes precedence; otherwise we use the project default
// override before falling through to the global default. The
// fallback is the per-step override's missing-counterpart settings, then
// the global validator/fallback pair, then the executor's `fallbackProvider`.
const defaultModel = resolveProjectDefaultModel(settings);
const primaryProvider = workflowStep.modelProvider || defaultModel.provider;
const primaryModelId = workflowStep.modelId || defaultModel.modelId;
const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId);
type ModelTuple = { provider?: string; modelId?: string };
const fallbackCandidates: Array<ModelTuple & { label: string }> = [
{ provider: settings.validatorFallbackProvider, modelId: settings.validatorFallbackModelId, label: "validatorFallback" },
{ provider: settings.fallbackProvider, modelId: settings.fallbackModelId, label: "globalFallback" },
];
const fallback = fallbackCandidates.find(
(c) => c.provider && c.modelId && (c.provider !== primaryProvider || c.modelId !== primaryModelId),
);
const timeoutMs = Math.max(60_000, settings.workflowStepTimeoutMs ?? 360_000);
const runOnce = async (
provider: string | undefined,
modelId: string | undefined,
attemptLabel: string,
): Promise<WorkflowStepOutcome> => {
// Workflow step agents inherit executor instructions
const stepInstructions = await this.resolveInstructionsForRole("executor");
const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions);
// Build skill selection context for workflow step session
const skillContext = await buildSessionSkillContext({
agentStore: this.options.agentStore!,
task,
sessionPurpose: "executor",
projectRootDir: this.rootDir,
pluginRunner: this.options.pluginRunner,
});
const workflowAgent = task.assignedAgentId && this.options.agentStore
? await this.options.agentStore.getAgent(task.assignedAgentId).catch(() => null)
: null;
const workflowRuntimeHint = extractRuntimeHint(workflowAgent?.runtimeConfig);
const { session } = await createResolvedAgentSession({
sessionPurpose: "executor",
runtimeHint: workflowRuntimeHint,
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: stepSystemPrompt,
tools: toolMode,
defaultProvider: provider,
defaultModelId: modelId,
fallbackProvider: settings.fallbackProvider,
fallbackModelId: settings.fallbackModelId,
defaultThinkingLevel: settings.defaultThinkingLevel,
// Skill selection: use assigned agent skills if available, otherwise role fallback
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
});
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' using model ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`);
await this.store.logEntry(
task.id,
`Workflow step '${workflowStep.name}' using model: ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`,
);
this.activeWorkflowStepSessions.set(task.id, session);
let output = "";
session.subscribe((event) => {
if (event.type === "message_update") {
const msgEvent = event.assistantMessageEvent;
if (msgEvent.type === "text_delta") {
output += msgEvent.delta;
agentLogger.onText(msgEvent.delta);
} else if (msgEvent.type === "thinking_delta") {
agentLogger.onThinking(msgEvent.delta);
}
}
if (event.type === "tool_execution_start") {
agentLogger.onToolStart(event.toolName, event.args as Record<string, unknown> | undefined);
}
if (event.type === "tool_execution_end") {
agentLogger.onToolEnd(event.toolName, event.isError, event.result);
}
});
let timedOut = false;
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<"timeout">((resolveTimeout) => {
timeoutHandle = setTimeout(() => {
timedOut = true;
resolveTimeout("timeout");
}, timeoutMs);
});
try {
const promptPromise = promptWithFallback(
session,
`Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` +
`Review the work done in this worktree and evaluate it against the criteria in your instructions.`,
);
const outcome = await Promise.race([
promptPromise.then(() => "completed" as const),
timeoutPromise,
]);
if (outcome === "timeout") {
executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' (${attemptLabel}) timed out after ${timeoutMs}ms — disposing session`);
await this.store.logEntry(
task.id,
`Workflow step '${workflowStep.name}' ${attemptLabel === "primary" ? "primary" : "fallback"} model timed out after ${Math.round(timeoutMs / 1000)}s — aborting session`,
);
try { session.dispose(); } catch { /* best-effort */ }
await agentLogger.flush();
return { success: false, error: `workflow step timed out after ${timeoutMs}ms`, timedOut: true };
}
// Completed within the timeout — let any post-completion errors surface.
checkSessionError(session);
await accumulateSessionTokenUsage(this.store, task.id, session);
session.dispose();
await agentLogger.flush();
const trimmedOutput = output.trim();
const revisionMatch = trimmedOutput.match(/^REQUEST REVISION\s*\n*/i);
if (revisionMatch) {
const feedbackStart = revisionMatch[0].length;
const feedback = trimmedOutput.slice(feedbackStart).trim();
return { success: false, revisionRequested: true, output: feedback };
}
return { success: true, output };
} catch (err: unknown) {
await agentLogger.flush();
try { session.dispose(); } catch { /* best-effort */ }
const errorMessage = err instanceof Error ? err.message : String(err);
return { success: false, error: errorMessage };
} finally {
if (timeoutHandle) clearTimeout(timeoutHandle);
const activeWorkflowStepSession = this.activeWorkflowStepSessions.get(task.id);
if (activeWorkflowStepSession === session) {
this.activeWorkflowStepSessions.delete(task.id);
}
// Suppress unused-variable warning; `timedOut` documents intent.
void timedOut;
}
};
const primaryOutcome = await runOnce(primaryProvider, primaryModelId, "primary");
if (!primaryOutcome.timedOut) return primaryOutcome;
if (!fallback) {
executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' timed out and no fallback model is configured`);
await this.store.logEntry(
task.id,
`Workflow step '${workflowStep.name}' timed out — no fallback model configured (set settings.validatorFallbackProvider/Id or fallbackProvider/Id)`,
);
return primaryOutcome;
}
executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with fallback ${fallback.provider}/${fallback.modelId} (label=${fallback.label})`);
return runOnce(fallback.provider, fallback.modelId, "fallback");
}
private MAX_WORKTREE_RETRIES = 3;
private WORKTREE_RETRY_DELAYS = [100, 500, 1000]; // ms
/**
* Create a git worktree with automatic recovery from conflicts.
* Implements retry logic with exponential backoff for transient failures.
*
* @param branch - The branch name to create (e.g., "fusion/fn-123")
* @param path - The desired worktree path
* @param taskId - The task ID for logging
* @param startPoint - Optional base branch/commit for new branch
* @returns The actual worktree path (may differ if recovery generated new name)
*/
private async createWorktree(
branch: string,
path: string,
taskId: string,
startPoint?: string,
): Promise<{ path: string; branch: string }> {
// Track the worktree path we're attempting to use (may change during recovery)
const currentPath = path;
let resolvedStartPoint: string | undefined;
if (startPoint) {
const resolved = await this.resolveWorktreeStartPoint(startPoint, taskId);
if (resolved === null) {
// Stored baseBranch no longer exists (e.g., upstream dep merged and branch
// deleted while this task sat queued/stuck). Clear it on the task so any
// subsequent retry branches from the default base, and proceed from HEAD.
await this.store.updateTask(taskId, { baseBranch: null });
} else {
resolvedStartPoint = resolved;
}
}
// When the task declares a non-main base (a sibling task's branch), the
// legacy behavior was to fork the worktree from that branch's tip,
// inheriting all of its commits. That caused content leakage when the
// dep was later squash-merged to main: the dep's raw commits became
// orphans whose content already existed in main, blocking the
// dependent's own merge with phantom conflicts.
//
// Prevention: instead of forking from the dep's tip, fork from `main`
// (or the configured remote/main if rebase-from-remote is enabled) and
// then `git merge --squash` the dep's content into a single import
// commit. The dependent branch then carries main's history + 1 commit
// for the dep's content; if the dep is later squash-merged to main, the
// patch-id on that import commit will match main's squash and Layer 2
// recovery (or a clean rebase) handles it.
//
// Fall-soft: any failure in this path falls back to the legacy behavior
// so we don't break worktree creation for setups where the squash flow
// can't run (no main branch resolvable, network down, etc.).
const squashImport = resolvedStartPoint
? await this.planSquashImportFromDep(taskId, resolvedStartPoint, startPoint)
: null;
const initialStartPoint = squashImport ? squashImport.mainBase : resolvedStartPoint;
for (let attempt = 0; attempt < this.MAX_WORKTREE_RETRIES; attempt++) {
try {
const result = await this.tryCreateWorktree(branch, currentPath, taskId, initialStartPoint, attempt);
// Squash-import dep content into the freshly created worktree so the
// branch contains main's history + 1 import commit instead of the
// dep's raw commits.
if (squashImport) {
await this.squashImportDepIntoWorktree(
result.path,
taskId,
squashImport.depTip,
squashImport.label,
).catch((importErr: unknown) => {
executorLog.warn(
`Squash-import of ${squashImport.label} into ${result.branch} failed for ${taskId} (continuing without): ${importErr instanceof Error ? importErr.message : String(importErr)}`,
);
});
}
// Mirror the merge-time rebase behavior: when worktreeRebaseBeforeMerge
// is enabled, fetch the remote and rebase the just-created task branch
// onto the latest <remote>/<defaultBranch>. This makes the worktree
// start from origin/main + local main both, so divergence only matters
// if the user actively skips this setting. Best-effort: failures here
// don't abort task setup.
await this.rebaseNewWorktreeOntoRemote(result.path, result.branch, taskId).catch((err: unknown) => {
executorLog.warn(
`Post-create worktree rebase failed for ${taskId} (continuing): ${err instanceof Error ? err.message : String(err)}`,
);
});
return result;
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
const isLastAttempt = attempt === this.MAX_WORKTREE_RETRIES - 1;
const isTerminalWorktreeError = error instanceof NonRetryableWorktreeError;
if (isLastAttempt || isTerminalWorktreeError) {
await this.store.logEntry(
taskId,
`Worktree creation failed after ${this.MAX_WORKTREE_RETRIES} attempts`,
errorMessage,
);
throw new Error(
`Failed to create worktree after ${this.MAX_WORKTREE_RETRIES} attempts: ${errorMessage}`,
);
}
// Wait before retry (exponential backoff)
const delay = this.WORKTREE_RETRY_DELAYS[attempt] || 1000;
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
// Should never reach here, but TypeScript needs a return
throw new Error("Unexpected exit from worktree creation retry loop");
}
private quoteShellArg(value: string): string {
return `'${value.replace(/'/g, "'\\''")}'`;
}
/**
* Decide whether a task's declared dep base should be squash-imported
* (instead of forked from). Returns the planned operation's data when the
* dep tip differs from the resolvable main base; returns null when no
* import is needed (dep is already at main) or when no main base is
* resolvable (caller falls back to legacy fork-from-dep).
*
* `originalStartPoint` is the user-facing label (typically the branch name
* like `fusion/fn-2729`) used purely for log messages. `depTip` is the
* resolved SHA of the dep's tip — that's what gets squash-merged.
*/
private async planSquashImportFromDep(
taskId: string,
depTip: string,
originalStartPoint: string | undefined,
): Promise<{ depTip: string; mainBase: string; label: string } | null> {
let settings;
try {
settings = await this.store.getSettings();
} catch {
return null;
}
// Resolve the main base. Preference order:
// 1. <remote>/<defaultBranch> when worktreeRebaseBeforeMerge is enabled
// and a remote is resolvable (settings.worktreeRebaseRemote wins;
// otherwise fall back to "origin" or the lone remote).
// 2. rootDir's HEAD (i.e., whatever local main is currently checked out
// to). Used when remote rebase is disabled or no remote exists.
let mainBase = "";
if (settings.worktreeRebaseBeforeMerge !== false) {
let remote = settings.worktreeRebaseRemote?.trim() || "";
if (!remote) {
try {
const { stdout } = await execAsync("git remote", { cwd: this.rootDir });
const remotes = stdout.split("\n").map((s) => s.trim()).filter(Boolean);
if (remotes.includes("origin")) remote = "origin";
else if (remotes.length === 1) remote = remotes[0];
} catch {
// No remote resolvable.
}
}
if (remote) {
let defaultBranch = "";
try {
const { stdout } = await execAsync(
`git rev-parse --abbrev-ref ${this.quoteShellArg(remote)}/HEAD`,
{ cwd: this.rootDir },
);
defaultBranch = stdout.trim().replace(new RegExp(`^${remote}/`), "");
} catch {
// origin/HEAD not set; will fall through to local HEAD below.
}
if (defaultBranch && defaultBranch !== "HEAD") {
// Fetch best-effort so the remote ref reflects upstream tip.
await execAsync(
`git fetch ${this.quoteShellArg(remote)} ${this.quoteShellArg(defaultBranch)}`,
{ cwd: this.rootDir },
).catch(() => undefined);
try {
const { stdout } = await execAsync(
`git rev-parse --verify "${remote}/${defaultBranch}^{commit}"`,
{ cwd: this.rootDir, encoding: "utf-8" },
);
mainBase = stdout.trim();
} catch {
// Couldn't resolve remote ref — fall through.
}
}
}
}
if (!mainBase) {
try {
const { stdout } = await execAsync("git rev-parse HEAD", {
cwd: this.rootDir,
encoding: "utf-8",
});
mainBase = stdout.trim();
} catch {
return null;
}
}
if (!mainBase) return null;
// If the dep tip is already an ancestor of main, no squash import is
// needed — the dep's content is already represented in main.
try {
await execAsync(
`git merge-base --is-ancestor ${this.quoteShellArg(depTip)} ${this.quoteShellArg(mainBase)}`,
{ cwd: this.rootDir },
);
// Exit code 0 → ancestor → no import needed; legacy fork-from-main is fine.
// Returning the plan with mainBase but signalling "no work" via dep===main.
if (depTip === mainBase) return null;
// Dep is ancestor of main but its tip SHA differs from main's tip; the
// worktree should still branch off main, no squash needed.
return { depTip: mainBase, mainBase, label: originalStartPoint || depTip.slice(0, 8) };
} catch {
// Not an ancestor — squash-import is the safer path.
}
return { depTip, mainBase, label: originalStartPoint || depTip.slice(0, 8) };
}
/**
* Squash-merge the dep's content into a worktree that's already branched
* off main. Produces one commit on the worktree branch carrying the dep's
* content, instead of inheriting the dep's individual commits. Best-effort:
* any failure (conflict, hooks, IO) leaves the worktree at main and the
* caller proceeds — the dependent task will then need to import the dep's
* content itself, but the worktree itself is still usable.
*/
private async squashImportDepIntoWorktree(
worktreePath: string,
taskId: string,
depTip: string,
label: string,
): Promise<void> {
// No-op when dep is already represented in the worktree's history.
try {
await execAsync(
`git merge-base --is-ancestor ${this.quoteShellArg(depTip)} HEAD`,
{ cwd: worktreePath },
);
return;
} catch {
// Not an ancestor — proceed.
}
// Try a squash-merge. `--no-commit` is implied by `--squash`; the merge
// either stages the dep's diff or fails (conflicts / unrelated histories).
try {
await execAsync(
`git merge --squash --allow-unrelated-histories ${this.quoteShellArg(depTip)}`,
{ cwd: worktreePath },
);
} catch (err) {
// Reset any partial state so the worktree stays usable, then rethrow
// so the caller can decide whether to log/fall-through.
await execAsync("git reset --hard HEAD", { cwd: worktreePath }).catch(
() => undefined,
);
throw err;
}
// If no diff was staged the dep is content-equivalent to main; nothing
// to commit.
try {
await execAsync("git diff --cached --quiet", { cwd: worktreePath });
return; // exit 0 → no staged changes, nothing to commit
} catch {
// exit non-zero → staged changes exist, proceed to commit.
}
// Always non-empty (subject + body via two -m args). Drop
// --allow-empty-message: we never want git to silently accept an empty
// message — a missing message here would make the commit hard to
// attribute / explain in `git log` and break downstream consumers that
// parse merge metadata from commit messages.
const subject = `chore(${taskId}): import dependency content from ${label}`;
const body =
`Squash-imported the working tree of ${label} as a single commit so this ` +
`branch carries the dep's content without inheriting its individual commits. ` +
`If the dep is later squash-merged to main, this commit's patch-id should ` +
`match the merge and rebase cleanly.`;
try {
await execAsync(
`git commit -m ${this.quoteShellArg(subject)} -m ${this.quoteShellArg(body)}`,
{ cwd: worktreePath },
);
} catch (commitErr) {
await execAsync("git reset --hard HEAD", { cwd: worktreePath }).catch(
() => undefined,
);
throw commitErr;
}
await this.store.logEntry(
taskId,
`Squash-imported dependency content from ${label} into worktree (single import commit instead of inheriting raw commits)`,
);
}
/**
* After creating a fresh task worktree, fetch the configured remote and
* rebase the task branch onto `<remote>/<defaultBranch>`. The result is a
* branch that contains origin's tip plus any local main commits, so the
* eventual merge has fewer surprises and the executor sees the freshest
* code its peers/CI may have published.
*
* No-op when `worktreeRebaseBeforeMerge` is disabled, no remote is
* configured/resolvable, or the rebase produces conflicts (we abort and
* leave the worktree as-is so the executor can still run).
*/
private async rebaseNewWorktreeOntoRemote(
worktreePath: string,
branch: string,
taskId: string,
): Promise<void> {
let settings;
try {
settings = await this.store.getSettings();
} catch {
return;
}
if (settings.worktreeRebaseBeforeMerge === false) return;
let remote = settings.worktreeRebaseRemote?.trim() || "";
if (!remote) {
try {
const { stdout } = await execAsync("git remote", { cwd: this.rootDir });
const remotes = stdout.split("\n").map((s) => s.trim()).filter(Boolean);
if (remotes.includes("origin")) remote = "origin";
else if (remotes.length === 1) remote = remotes[0];
} catch {
// No remote resolvable — nothing to rebase against.
}
}
if (!remote) return;
let defaultBranch = "";
try {
const { stdout } = await execAsync(`git rev-parse --abbrev-ref ${remote}/HEAD`, { cwd: this.rootDir });
defaultBranch = stdout.trim().replace(new RegExp(`^${remote}/`), "");
} catch {
// origin/HEAD not set — fall back to current branch in rootDir.
}
if (!defaultBranch) {
try {
const { stdout } = await execAsync("git rev-parse --abbrev-ref HEAD", { cwd: this.rootDir });
defaultBranch = stdout.trim();
} catch {
return;
}
}
if (!defaultBranch || defaultBranch === "HEAD") return;
const remoteRef = `${remote}/${defaultBranch}`;
try {
await execAsync(`git fetch ${this.quoteShellArg(remote)} ${this.quoteShellArg(defaultBranch)}`, { cwd: this.rootDir });
} catch (err) {
executorLog.warn(
`Worktree rebase: fetch ${remote} ${defaultBranch} failed for ${taskId}: ${err instanceof Error ? err.message : String(err)}`,
);
return;
}
try {
await execAsync(`git rebase ${this.quoteShellArg(remoteRef)}`, { cwd: worktreePath });
await this.store.logEntry(
taskId,
`Rebased new worktree branch ${branch} onto ${remoteRef}`,
);
} catch (rebaseErr) {
const msg = rebaseErr instanceof Error ? rebaseErr.message : String(rebaseErr);
executorLog.warn(
`Worktree rebase: rebase onto ${remoteRef} failed for ${taskId} — aborting and leaving local base intact: ${msg}`,
);
try {
await execAsync("git rebase --abort", { cwd: worktreePath });
} catch {
// best-effort
}
await this.store.logEntry(
taskId,
`Could not rebase new worktree onto ${remoteRef} — kept local base. The merge-time rebase will retry with conflict resolution.`,
);
}
}
/**
* Resolve a stored baseBranch to a concrete commit SHA.
*
* Returns `null` (not throw) when the ref cannot be resolved — typically
* because the upstream dep's branch was merged and deleted while this task
* sat queued/stuck. Callers should treat null as "fall back to default base"
* rather than fail the task permanently.
*/
private async resolveWorktreeStartPoint(startPoint: string, taskId: string): Promise<string | null> {
const command = isAbsolute(startPoint) && existsSync(startPoint)
? `git -C "${startPoint}" rev-parse --verify HEAD^{commit}`
: `git rev-parse --verify "${startPoint}^{commit}"`;
try {
const { stdout } = await execAsync(command, { cwd: this.rootDir });
return stdout.trim() || startPoint;
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
await this.store.logEntry(
taskId,
`Worktree base ref "${startPoint}" is missing — falling back to default base`,
errorMessage,
);
return null;
}
}
/**
* Single attempt to create a worktree with conflict detection and recovery.
* Returns the actual worktree path used (may differ from input if recovery generated new name).
*/
private async tryCreateWorktree(
branch: string,
path: string,
taskId: string,
startPoint?: string,
attemptNumber = 0,
recoveryDepth = 0,
): Promise<{ path: string; branch: string }> {
// Guard: refuse to create a worktree nested inside another worktree.
// Nested worktrees happen when the executor is launched with rootDir pointed
// at a worktree directory instead of the main repo — produces paths like
// `.worktrees/green-finch/.worktrees/amber-panda` that bloat the filesystem
// and confuse every tool that walks git state.
await this.assertWorktreePathNotNested(path, taskId);
// If directory exists but is not a registered worktree, remove it first
if (existsSync(path)) {
const isRegistered = await this.isRegisteredWorktree(path);
if (!isRegistered) {
await this.store.logEntry(
taskId,
`Removing existing directory (not a registered worktree): ${path}`,
);
try {
await execAsync(`rm -rf "${path}"`, { cwd: this.rootDir });
} catch (e: unknown) {
const eMessage = e instanceof Error ? e.message : String(e);
throw new Error(`Failed to remove existing directory ${path}: ${eMessage}`);
}
} else {
executorLog.log(`Worktree already exists: ${path}`);
return { path, branch };
}
}
const createWithBranch = async (branchToCreate: string) => {
const cmd = startPoint
? `git worktree add -b "${branchToCreate}" "${path}" "${startPoint}"`
: `git worktree add -b "${branchToCreate}" "${path}"`;
try {
await execAsync(cmd, { cwd: this.rootDir });
} catch (err) {
// Remove any partial directory left behind so the invariant holds:
// "if .worktrees/<slug> exists on disk, it is a fully registered git worktree."
try {
await execAsync(`rm -rf "${path}"`, { cwd: this.rootDir });
} catch {
// best-effort cleanup; log but don't mask the original error
executorLog.log(`Warning: failed to remove partial worktree directory after creation failure: ${path}`);
}
throw err;
}
};
const createFromExistingBranch = async () => {
try {
await execAsync(`git worktree add "${path}" "${branch}"`, { cwd: this.rootDir });
} catch (err) {
// Remove any partial directory left behind so the invariant holds:
// "if .worktrees/<slug> exists on disk, it is a fully registered git worktree."
try {
await execAsync(`rm -rf "${path}"`, { cwd: this.rootDir });
} catch {
// best-effort cleanup; log but don't mask the original error
executorLog.log(`Warning: failed to remove partial worktree directory after creation failure: ${path}`);
}
throw err;
}
};
try {
await createWithBranch(branch);
executorLog.log(`Worktree created: ${path}${startPoint ? ` (from ${startPoint})` : ""}`);
if (attemptNumber > 0) {
await this.store.logEntry(taskId, `Worktree created on attempt ${attemptNumber + 1}`, path);
}
return { path, branch };
} catch (initialError: unknown) {
const conflictInfo = this.extractWorktreeConflictInfo(initialError);
if (conflictInfo.type === "not-git-repo") {
throw new NonRetryableWorktreeError(
"Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.",
);
}
// Handle "already used by worktree" conflict
if (conflictInfo.type === "already-used" && conflictInfo.path) {
const result = await this.handleWorktreeConflict(
conflictInfo.path,
branch,
path,
taskId,
startPoint,
attemptNumber,
);
if (result) {
return result;
}
throw new Error(
`Worktree conflict at ${conflictInfo.path}: automatic cleanup failed`,
);
}
// Handle "invalid reference" - stale branch that doesn't exist
if (conflictInfo.type === "invalid-reference") {
if (recoveryDepth >= this.MAX_WORKTREE_RETRIES - 1) {
throw new NonRetryableWorktreeError(
`Stale branch reference for ${branch} remained invalid after ${this.MAX_WORKTREE_RETRIES} cleanup attempts`,
);
}
const branchCleaned = await this.cleanupStaleBranch(branch, taskId);
if (branchCleaned) {
await this.store.logEntry(taskId, `Removed stale branch reference, retrying`);
return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, recoveryDepth + 1);
}
throw new Error(
`Invalid reference for branch ${branch}: unable to clean up stale reference`,
);
}
// Handle "could not create leading directories" - permission/path issues
if (conflictInfo.type === "leading-directories") {
throw new Error(
`Cannot create worktree at ${path}: permission or path issue. ` +
`Check that parent directories are writable.`,
);
}
// Try creating from existing branch (branch might already exist)
try {
await createFromExistingBranch();
executorLog.log(`Worktree created from existing branch: ${path}`);
return { path, branch };
} catch (fallbackError: unknown) {
const fallbackErrorMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
// Check if the fallback also hit an "already used" conflict
const fallbackConflictInfo = this.extractWorktreeConflictInfo(fallbackError);
if (fallbackConflictInfo.type === "not-git-repo") {
throw new NonRetryableWorktreeError(
"Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.",
);
}
if (fallbackConflictInfo.type === "already-used" && fallbackConflictInfo.path) {
const result = await this.handleWorktreeConflict(
fallbackConflictInfo.path,
branch,
path,
taskId,
startPoint,
attemptNumber,
);
if (result) {
return result;
}
throw new Error(
`Worktree conflict at ${fallbackConflictInfo.path}: automatic cleanup failed`,
);
}
// Handle stale reference in fallback path too
if (fallbackConflictInfo.type === "invalid-reference") {
if (recoveryDepth >= this.MAX_WORKTREE_RETRIES - 1) {
throw new NonRetryableWorktreeError(
`Stale branch reference for ${branch} remained invalid after ${this.MAX_WORKTREE_RETRIES} cleanup attempts`,
);
}
const branchCleaned = await this.cleanupStaleBranch(branch, taskId);
if (branchCleaned) {
await this.store.logEntry(taskId, `Cleaned up stale reference in fallback, retrying`);
return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, recoveryDepth + 1);
}
}
throw new Error(`Failed to create worktree: ${fallbackErrorMessage}`);
}
}
}
/**
* Handle "already used by worktree" conflict.
* Either generates a new worktree name (if conflicting worktree is in use by active task)
* or cleans up the conflicting worktree and retries.
*
* @returns The worktree path if recovery succeeded, null if recovery failed
*/
private async handleWorktreeConflict(
conflictPath: string,
branch: string,
path: string,
taskId: string,
startPoint?: string,
attemptNumber?: number,
): Promise<{ path: string; branch: string } | null> {
const shouldGenerateNewName = await this.shouldGenerateNewWorktreeName(
conflictPath,
taskId,
);
if (shouldGenerateNewName) {
// Conflicting worktree belongs to an active task — generate new path AND
// use a suffixed branch name so git doesn't conflict with the branch
// already checked out in the existing worktree. Branch conflicts here
// mean the original task branch already exists and is checked out
// elsewhere, so suffix retries must branch from that task branch tip
// rather than the stale base ref to preserve the task's commits.
const conflictStartPoint = branch;
const newPath = join(this.rootDir, ".worktrees", generateWorktreeName(this.rootDir));
for (let suffix = 2; suffix <= 6; suffix++) {
const suffixedBranch = `${branch}-${suffix}`;
try {
await this.store.logEntry(
taskId,
`Conflicting worktree in use by active task, trying new path with branch ${suffixedBranch}`,
newPath,
);
return await this.tryCreateWorktree(suffixedBranch, newPath, taskId, conflictStartPoint, attemptNumber);
} catch (suffixErr: unknown) {
const info = this.extractWorktreeConflictInfo(suffixErr);
if (info.type === "already-used") {
// This suffixed branch is also in use — try next suffix
continue;
}
throw suffixErr;
}
}
throw new Error(
`Cannot create branch for task: "${branch}" and suffixes -2 through -6 are all in use by other worktrees`,
);
}
// Safe to clean up - conflicting worktree is not in use
const cleanupSuccess = await this.cleanupConflictingWorktree(conflictPath, branch, taskId);
if (cleanupSuccess) {
await this.store.logEntry(taskId, `Cleaned up conflicting worktree, retrying`, path);
return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber);
}
return null;
}
/**
* Check if a path is registered as a git worktree.
*/
private async isRegisteredWorktree(path: string): Promise<boolean> {
return isRegisteredGitWorktree(this.rootDir, path);
}
/**
* Throw if `path` lies inside an existing registered worktree other than the
* repo root. The repo root itself is a worktree (main branch) and must be
* allowed — we only reject paths strictly *inside* a non-root worktree.
*/
private async assertWorktreePathNotNested(path: string, taskId: string): Promise<void> {
const target = resolvePath(path);
const rootResolved = resolvePath(this.rootDir);
const registered = await getRegisteredWorktreePaths(this.rootDir);
for (const wt of registered) {
if (wt === rootResolved) continue; // root is allowed as ancestor
if (wt === target) continue; // exact match handled later as "already registered"
const rel = relative(wt, target);
if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
await this.store.logEntry(
taskId,
`Refusing to create nested worktree`,
`target ${target} is inside registered worktree ${wt}`,
);
throw new NonRetryableWorktreeError(
`Refusing to create worktree at ${target}: path is nested inside existing worktree ${wt}. ` +
`This usually means the executor was launched with rootDir pointing at a worktree instead of the main repo.`,
);
}
}
}
/**
* Determine if we should generate a new worktree name instead of cleaning up.
* Returns true if the conflicting worktree is used by an active task.
*/
private async shouldGenerateNewWorktreeName(
conflictPath: string,
currentTaskId: string,
): Promise<boolean> {
// Check if conflicting worktree is in our active set
for (const [taskId, worktreePath] of this.activeWorktrees) {
if (taskId !== currentTaskId && worktreePath === conflictPath) {
return true;
}
}
// Check if another non-done task uses this worktree
const otherUser = await findWorktreeUser(this.store, conflictPath, currentTaskId);
return otherUser !== null;
}
/**
* Clean up a conflicting worktree and its branch.
* Handles locked worktrees by unlocking first.
* Returns true if cleanup succeeded.
*/
private async cleanupConflictingWorktree(
worktreePath: string,
branch: string,
taskId: string,
): Promise<boolean> {
try {
// Check if worktree is locked and unlock if needed
try {
await execAsync(`git worktree unlock "${worktreePath}"`, {
cwd: this.rootDir,
});
await this.store.logEntry(taskId, `Unlocked worktree`, worktreePath);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: failed to unlock conflicting worktree ${worktreePath} before cleanup: ${msg}`);
}
// Remove the worktree
await execAsync(`git worktree remove "${worktreePath}" --force`, {
cwd: this.rootDir,
});
await this.store.logEntry(taskId, `Removed conflicting worktree`, worktreePath);
// Delete the branch if it exists
try {
await execAsync(`git branch -D "${branch}"`, {
cwd: this.rootDir,
});
await this.store.logEntry(taskId, `Deleted branch`, branch);
// FN-2165 regression guard: null baseBranch on any task that stored this branch
this.store.clearStaleBaseBranchReferences([branch], taskId);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: failed to delete conflicting branch ${branch}: ${msg}`);
}
return true;
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
await this.store.logEntry(
taskId,
`Failed to clean up conflicting worktree`,
`${worktreePath}: ${errorMessage}`,
);
return false;
}
}
/**
* Clean up a stale branch that no longer has a valid reference.
*
* Recovery strategy (in order):
* 1. `git worktree prune` — remove stale worktree metadata that may
* hold a lock on the branch reference
* 2. `git branch -D` — delete the branch normally
* 3. `git update-ref -d refs/heads/<branch>` — force-remove a corrupted
* or dangling reference when `git branch -D` fails
*
* Each step is logged so operators can trace the recovery path.
* Returns true if the branch reference was successfully removed.
*/
private async cleanupStaleBranch(branch: string, taskId: string): Promise<boolean> {
// Step 1: Prune stale worktree metadata that may hold a lock on the branch
try {
await execAsync("git worktree prune", { cwd: this.rootDir });
await this.store.logEntry(taskId, `Pruned stale worktree metadata`, branch);
} catch {
// Prune is best-effort — continue even if it fails
}
// Step 2: Try normal branch deletion
try {
await execAsync(`git branch -D "${branch}"`, {
cwd: this.rootDir,
});
await this.store.logEntry(taskId, `Removed stale branch`, branch);
// FN-2165 regression guard: null baseBranch on any task that stored this branch
try { this.store.clearStaleBaseBranchReferences([branch], taskId); } catch { /* best-effort */ }
return true;
} catch (branchDeleteError: unknown) {
const branchDeleteErrorMessage = branchDeleteError instanceof Error ? branchDeleteError.message : String(branchDeleteError);
await this.store.logEntry(
taskId,
`git branch -D failed for stale branch, trying update-ref`,
`${branch}: ${branchDeleteErrorMessage}`,
);
}
// Step 3: Force-remove the reference directly
try {
const refPath = `refs/heads/${branch}`;
await execAsync(`git update-ref -d "${refPath}"`, {
cwd: this.rootDir,
});
await this.store.logEntry(taskId, `Force-removed stale branch reference via update-ref`, refPath);
// FN-2165 regression guard: null baseBranch on any task that stored this branch
try { this.store.clearStaleBaseBranchReferences([branch], taskId); } catch { /* best-effort */ }
return true;
} catch (updateRefError: unknown) {
const updateRefErrorMessage = updateRefError instanceof Error ? updateRefError.message : String(updateRefError);
await this.store.logEntry(
taskId,
`Failed to remove stale branch reference`,
`${branch}: ${updateRefErrorMessage}`,
);
return false;
}
}
/**
* Extract conflict information from git worktree error output.
* Handles multiple error patterns:
* - "already used by worktree at '...'"
* - "invalid reference" / "unable to resolve reference" / "stale file handle"
* - "could not create leading directories"
* - "working tree already exists"
*/
private extractWorktreeConflictInfo(error: unknown): {
type: "already-used" | "invalid-reference" | "leading-directories" | "already-exists" | "not-git-repo" | "unknown";
path?: string;
message?: string;
} {
const execError = error instanceof Error ? error : new Error(String(error));
const output = [
execError.message,
"stderr" in execError && typeof execError.stderr === "string" ? execError.stderr.toString() : undefined,
"stdout" in execError && typeof execError.stdout === "string" ? execError.stdout.toString() : undefined,
]
.filter(Boolean)
.join("\n");
// Pattern: already used by worktree at '/path/to/worktree'
const alreadyUsedMatch = output.match(/already used by worktree at '([^']+)'/);
if (alreadyUsedMatch) {
return { type: "already-used", path: alreadyUsedMatch[1], message: output };
}
// Pattern: already checked out at '/path/to/worktree'
const alreadyCheckedOutMatch = output.match(/is already checked out at '([^']+)'/);
if (alreadyCheckedOutMatch) {
return { type: "already-used", path: alreadyCheckedOutMatch[1], message: output };
}
// Pattern: invalid reference: 'branch-name'
// Also covers: unable to resolve reference, stale file handle, not a valid ref
if (
output.match(/invalid reference/i) ||
output.match(/unable to resolve reference/i) ||
output.match(/stale file handle/i) ||
output.match(/not a valid ref/i) ||
output.match(/unable to delete.*ref/i)
) {
return { type: "invalid-reference", message: output };
}
// Pattern: could not create leading directories
if (output.match(/could not create leading directories/i)) {
return { type: "leading-directories", message: output };
}
// Pattern: working tree already exists
if (output.match(/working tree already exists/i)) {
return { type: "already-exists", message: output };
}
// Pattern: not a git repository / not a git repo
if (output.match(/not a git repo(sitory)?/i)) {
return { type: "not-git-repo", message: output };
}
return { type: "unknown", message: output };
}
/**
* Remove a task's worktree, but only if no other in-progress or todo task
* shares the same worktree path (dependency-chain reuse). The branch is
* always cleaned up by the merger on a per-task basis.
*/
async cleanup(taskId: string): Promise<void> {
const worktreePath = this.activeWorktrees.get(taskId);
if (!worktreePath) return;
this.activeWorktrees.delete(taskId);
// Check if another task still needs this worktree
const otherUser = await findWorktreeUser(this.store, worktreePath, taskId);
if (otherUser) {
executorLog.log(`Worktree retained for ${taskId} — still needed by ${otherUser}`);
return;
}
try {
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
executorLog.log(`Cleaned up worktree for ${taskId}`);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to clean up worktree for ${taskId}:`, errorMessage);
}
}
/**
* When the engine restarts mid-step, an `in-progress` step may have already
* passed its code review (log: `code review Step N: APPROVE`) but not yet
* been flipped to `done` by the agent's next `fn_task_update` call. Without
* intervention, the next executor pass re-enters the step and replays plan
* + code review, which we've measured at 5–20 min of pure waste per restart.
*
* This reconciler scans the task log for any in-progress step whose most
* recent approved code review is newer than its most recent `→ pending`
* transition, and marks those steps `done`. Subsequent resume logic then
* advances to the next actually-pending step.
*/
private async recoverApprovedStepsOnResume(taskId: string): Promise<void> {
let detail: TaskDetail;
try {
detail = await this.store.getTask(taskId);
} catch (err) {
executorLog.warn(`${taskId}: recoverApprovedStepsOnResume getTask failed: ${err instanceof Error ? err.message : String(err)}`);
return;
}
const log = detail.log ?? [];
if (log.length === 0) return;
let recovered = 0;
for (let i = 0; i < detail.steps.length; i++) {
if (detail.steps[i].status !== "in-progress") continue;
let lastPendingAt = -1;
let lastApproveAt = -1;
const stepName = detail.steps[i].name;
// Matches "Step 3 (My Step) → pending"; name is user-controlled, so match
// on prefix rather than a regex built from the name.
const transitionPrefix = `Step ${i} (${stepName}) → `;
const approvePrefix = `code review Step ${i}:`;
for (let j = 0; j < log.length; j++) {
const action = log[j].action || "";
if (action.startsWith(transitionPrefix)) {
const status = action.slice(transitionPrefix.length).trim();
if (status === "pending") lastPendingAt = j;
} else if (action.startsWith(approvePrefix) && action.includes("APPROVE")) {
lastApproveAt = j;
}
}
if (lastApproveAt > lastPendingAt) {
executorLog.log(
`${taskId}: step ${i} ("${stepName}") already has an approved code review — marking done on resume (skipping review replay)`,
);
try {
await this.store.logEntry(
taskId,
`Step ${i} (${stepName}) recovered as done on resume — code review had already approved before the engine stopped`,
);
await this.store.updateStep(taskId, i, "done");
recovered++;
} catch (err) {
executorLog.warn(
`${taskId}: failed to recover step ${i} on resume: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
}
if (recovered > 0) {
executorLog.log(`${taskId}: recovered ${recovered} approved step(s) on resume`);
}
}
/**
* On resume (task already has a branch from a prior run), walk git history
* and mark steps as done when a commit matching the step-completion convention
* is found. This prevents the agent from redoing already-committed work after
* an auto-requeue.
*
* Commit message convention (case-insensitive):
* feat|chore|fix(FN-XXXX): complete Step N
*
* Called after the worktree is acquired and before the agent session starts.
*/
private async reconcileStepsFromGitHistory(taskId: string, detail: TaskDetail, worktreePath: string): Promise<void> {
const baseCommitSha = detail.baseCommitSha;
if (!baseCommitSha) return;
const pendingOrInProgressSteps = detail.steps.filter(
(s, i) => (s.status === "pending" || s.status === "in-progress") && i > 0,
);
if (pendingOrInProgressSteps.length === 0) return;
let logOutput: string;
try {
const { stdout } = await execAsync(
`git log "${baseCommitSha}..HEAD" --oneline`,
{ cwd: worktreePath },
);
logOutput = stdout;
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${taskId}: reconcileStepsFromGitHistory — git log failed: ${msg}`);
return;
}
if (!logOutput.trim()) return;
// Match: feat(FN-2978): complete Step 3 / chore(fn-2978)!: Complete step 3
const stepCommitRegex = /^(?:feat|chore|fix)\([Ff][Nn]-\d+\)(?:!)?:\s*complete\s+step\s+(\d+)/i;
const reconciledStepIndices = new Set<number>();
for (const line of logOutput.split("\n")) {
// git log --oneline format: "<sha> <message>"
const message = line.replace(/^[0-9a-f]+ /, "").trim();
const match = message.match(stepCommitRegex);
if (!match) continue;
const stepIndex = parseInt(match[1], 10);
if (Number.isNaN(stepIndex) || stepIndex < 0 || stepIndex >= detail.steps.length) continue;
const step = detail.steps[stepIndex];
if (step.status === "pending" || step.status === "in-progress") {
reconciledStepIndices.add(stepIndex);
}
}
for (const stepIndex of reconciledStepIndices) {
await this.store.updateStep(taskId, stepIndex, "done");
await this.store.logEntry(
taskId,
`Reconciled Step ${stepIndex} as done from git history (resume)`,
undefined,
this.currentRunContext,
);
executorLog.log(`${taskId}: reconciled Step ${stepIndex} as done from git history`);
}
if (reconciledStepIndices.size > 0) {
// Refresh task and update currentStep to the lowest pending index
const updated = await this.store.getTask(taskId);
const lowestPending = updated.steps.findIndex((s) => s.status === "pending" || s.status === "in-progress");
if (lowestPending >= 0 && lowestPending !== updated.currentStep) {
await this.store.updateTask(taskId, { currentStep: lowestPending });
executorLog.log(`${taskId}: set currentStep to ${lowestPending} after step reconciliation`);
}
}
}
/**
* Check whether the task's branch has any unique commits compared to main.
* If the branch has no unique commits and the task has steps marked done,
* those steps represent lost uncommitted work — reset them to "pending"
* so the next execution doesn't skip them.
*
* Called during stuck-kill cleanup when the worktree is about to be destroyed.
*/
private async resetStepsIfWorkLost(task: Task): Promise<void> {
const completedSteps = task.steps.filter(
(s) => s.status === "done" || s.status === "in-progress",
);
if (completedSteps.length === 0) return;
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
try {
// Check if the branch has any unique commits vs main
const { stdout: mergeBaseStdout } = await execAsync(
`git merge-base "${branchName}" HEAD 2>/dev/null`,
{ cwd: this.rootDir, encoding: "utf-8" },
);
const { stdout: branchHeadStdout } = await execAsync(
`git rev-parse "${branchName}" 2>/dev/null`,
{ cwd: this.rootDir, encoding: "utf-8" },
);
const mergeBase = mergeBaseStdout.trim();
const branchHead = branchHeadStdout.trim();
if (mergeBase === branchHead) {
// Branch has no unique commits — all step work was lost
executorLog.warn(
`${task.id} branch has no unique commits — resetting ${completedSteps.length} step(s) to pending`,
);
for (let i = 0; i < task.steps.length; i++) {
if (task.steps[i].status === "done" || task.steps[i].status === "in-progress") {
await this.store.updateStep(task.id, i, "pending");
}
}
await this.store.logEntry(
task.id,
`Reset ${completedSteps.length} step(s) to pending — branch had no commits (uncommitted work lost with worktree)`,
);
}
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`${task.id}: step-reset-on-work-lost failed (non-fatal, steps keep current status): ${msg}`);
// Branch may not exist or git commands may fail — non-fatal.
// Steps keep their current status (safe default: agent can
// inspect the worktree and decide).
}
}
/**
* Mark a task as stuck-aborted so the executor's error handling
* knows not to treat the disposed session as a genuine failure.
* Called by the stuck task detector's onStuck callback.
*
* @param shouldRequeue — true to move the task back to "todo" for retry,
* false if the stuck kill budget is exhausted (task already marked failed).
*/
markStuckAborted(taskId: string, shouldRequeue: boolean = true): void {
// Terminate step-session executor if active
const stepExecutor = this.activeStepExecutors.get(taskId);
if (stepExecutor) {
stepExecutor.terminateAllSessions().catch(err =>
executorLog.warn(`Failed to terminate step sessions for stuck task ${taskId}: ${err}`)
);
}
this.stuckAborted.set(taskId, shouldRequeue);
// Safety net: if the executor's Promise never resolves (e.g. a bash subprocess
// is blocking the agent session even after dispose()), force-requeue the task
// directly after a short grace period. Without this, a task with a hung tool
// call stays stranded in "in-progress" until the engine restarts.
if (shouldRequeue && this.executing.has(taskId)) {
const FORCE_REQUEUE_GRACE_MS = 60_000; // 60 s — generous, but bounded
setTimeout(async () => {
if (!this.executing.has(taskId)) return; // executor unwound normally — nothing to do
executorLog.warn(
`${taskId} still executing ${FORCE_REQUEUE_GRACE_MS / 1000}s after stuck-kill signal ` +
`(likely a hung subprocess) — force-requeueing`,
);
try {
await this.store.logEntry(
taskId,
`Force-requeued after stuck-kill: executor did not unwind within ${FORCE_REQUEUE_GRACE_MS / 1000}s (hung subprocess)`,
);
await this.store.updateTask(taskId, { status: "stuck-killed", worktree: null, branch: null });
await this.store.moveTask(taskId, "todo");
// Remove from executing so the scheduler can re-dispatch normally.
// The old Promise is still running but the executing guard is cleared so
// a fresh execute() call won't be blocked.
this.executing.delete(taskId);
this.stuckAborted.delete(taskId);
executorLog.log(`${taskId} force-requeued to todo`);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`Failed to force-requeue stuck task ${taskId}: ${errorMessage}`);
}
}, FORCE_REQUEUE_GRACE_MS);
}
}
/**
* Handle a loop-detected event from the stuck task detector.
* Attempts an in-process compact-and-resume before falling back to kill/requeue.
*
* This method is the `onLoopDetected` callback wired through the dashboard.
* It:
* 1. Checks if the task has an active session
* 2. Rejects if the one-attempt ceiling has been reached
* 3. Calls `compactSessionContext()` to compact the conversation
* 4. Sets recovery-pending state so the execution flow can resume
*
* @returns true if the executor accepted recovery ownership (detector skips kill),
* false if recovery should not be attempted (detector proceeds with kill/requeue)
*/
async handleLoopDetected(event: StuckTaskEvent): Promise<boolean> {
const { taskId } = event;
const activeEntry = this.activeSessions.get(taskId);
// No active session — can't compact, let detector kill/requeue
if (!activeEntry) {
executorLog.log(`${taskId} loop detected but no active session — falling back to kill/requeue`);
return false;
}
// Check attempt ceiling (max 1 compact-and-resume per execute() lifecycle)
const state = this.loopRecoveryState.get(taskId);
if (state && state.attempts >= 1) {
executorLog.log(`${taskId} loop detected but compact ceiling reached — falling back to kill/requeue`);
return false;
}
// Attempt compaction
const attempt = (state?.attempts ?? 0) + 1;
executorLog.log(`${taskId} loop detected (attempt ${attempt}) — attempting compact-and-resume`);
await this.store.logEntry(taskId, `Loop detected (${event.activitySinceProgress} events since last progress) — attempting compact-and-resume (attempt ${attempt})`);
const compactResult = await compactSessionContext(activeEntry.session);
if (!compactResult) {
executorLog.log(`${taskId} compaction failed or unavailable — falling back to kill/requeue`);
await this.store.logEntry(taskId, "Context compaction failed or unavailable — falling back to kill/requeue");
return false;
}
executorLog.log(`${taskId} compaction succeeded (freed ${compactResult.tokensBefore} tokens) — setting recovery-pending`);
await this.store.logEntry(taskId, `Context compacted successfully — will resume with fresh context`);
// Mark recovery-pending so the execution flow can consume it
this.loopRecoveryState.set(taskId, { attempts: attempt, pending: true });
// Steer the session with a resume prompt to break the loop
try {
await activeEntry.session.steer(
"⚠️ Loop detected: you were repeating actions without making progress. " +
"The conversation has been compacted. Review the current state carefully, " +
"check what's already been done (git log, file contents), and take a different " +
"approach. Do NOT repeat the same actions. Advance to the next step if the " +
"current work is complete.",
);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.error(`${taskId} failed to steer after compaction: ${errorMessage}`);
// Recovery-pending is still set — the execution flow will handle it
}
return true;
}
getWorktreePath(taskId: string): string | undefined {
return this.activeWorktrees.get(taskId);
}
// ── Agent Spawning ─────────────────────────────────────────────────────
/**
* Terminate all child agents spawned by a parent task.
* Called from the finally block of agentWork when the parent session ends.
*/
private async terminateAllChildren(parentTaskId: string): Promise<void> {
const childIds = this.spawnedAgents.get(parentTaskId);
if (!childIds || childIds.size === 0) return;
executorLog.log(`Terminating ${childIds.size} child agents for parent ${parentTaskId}`);
for (const childId of childIds) {
await this.terminateChildAgent(childId);
}
this.spawnedAgents.delete(parentTaskId);
}
/**
* Terminate a single child agent by ID.
* Disposes the session, updates AgentStore state, and cleans up tracking Maps.
*/
private async terminateChildAgent(childId: string): Promise<void> {
const childSession = this.childSessions.get(childId);
if (childSession) {
childSession.dispose();
this.childSessions.delete(childId);
}
try {
await this.options.agentStore?.updateAgentState(childId, "terminated");
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`Failed to update spawned child ${childId} state to 'terminated' during cleanup: ${msg}`);
}
// Auto-delete the child agent after a short delay so the UI can observe
// the terminal state before the agent is removed.
this.pendingEphemeralDeletions.add(childId);
const timerId = setTimeout(async () => {
this.ephemeralCleanupTimers.delete(childId);
this.pendingEphemeralDeletions.delete(childId);
try {
await this.options.agentStore?.deleteAgent(childId);
} catch (err: unknown) {
if (this.isBenignEphemeralDeleteRaceError(childId, err)) {
return;
}
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`Failed to delete spawned agent ${childId}: ${msg}`);
}
}, 5000);
this.ephemeralCleanupTimers.set(childId, timerId);
this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1);
}
/**
* Run a spawned child agent's task to completion.
* Handles state transitions and cleanup.
*/
private async runSpawnedChild(
agentId: string,
childSession: AgentSession,
taskPrompt: string,
): Promise<void> {
try {
await this.options.agentStore?.updateAgentState(agentId, "running");
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
executorLog.warn(`Failed to update spawned child ${agentId} state to 'running': ${msg}`);
}
try {
await promptWithFallback(childSession, taskPrompt);
// Normal completion — mark as active (available)
try {
await this.options.agentStore?.updateAgentState(agentId, "active");
} catch (markActiveErr) {
executorLog.warn(`Child agent ${agentId} updateAgentState(active) failed: ${markActiveErr instanceof Error ? markActiveErr.message : String(markActiveErr)}`);
}
} catch (err: unknown) {
// Error during execution — mark as error
try {
await this.options.agentStore?.updateAgentState(agentId, "error");
} catch (markErrorErr) {
executorLog.warn(`Child agent ${agentId} updateAgentState(error) failed: ${markErrorErr instanceof Error ? markErrorErr.message : String(markErrorErr)}`);
}
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.warn(`Child agent ${agentId} failed: ${errorMessage}`);
} finally {
this.childSessions.delete(agentId);
this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1);
}
}
/**
* Create the fn_spawn_agent tool definition.
* Allows the parent agent to spawn child agents with delegated tasks.
*/
private createSpawnAgentTool(taskId: string, worktreePath: string, settings: Settings): ToolDefinition {
return {
name: "fn_spawn_agent",
label: "Spawn Agent",
description:
"Spawn a child agent to handle parallel work or specialized sub-tasks. " +
"Each child runs in its own git worktree (branched from your worktree) and executes autonomously. " +
"When you end (fn_task_done), all spawned children are terminated.",
parameters: spawnAgentParams,
execute: async (_id: string, params: Static<typeof spawnAgentParams>) => {
const { name, role, task: taskPrompt } = params;
// Check if AgentStore is available
if (!this.options.agentStore) {
return {
content: [{ type: "text" as const, text: "Agent spawning is not available (no AgentStore configured)" }],
details: { agentId: "", state: "error" },
};
}
// Read spawn limits from settings
const maxPerParent = settings.maxSpawnedAgentsPerParent ?? 5;
const maxGlobal = settings.maxSpawnedAgentsGlobal ?? 20;
// Check per-parent limit
const currentPerParent = this.spawnedAgents.get(taskId)?.size ?? 0;
if (currentPerParent >= maxPerParent) {
return {
content: [{ type: "text" as const, text: `Per-parent spawn limit reached (${currentPerParent}/${maxPerParent}). Wait for children to finish or reduce parallelism.` }],
details: { agentId: "", state: "error" },
};
}
// Check global limit
if (this.totalSpawnedCount >= maxGlobal) {
return {
content: [{ type: "text" as const, text: `Global spawn limit reached (${this.totalSpawnedCount}/${maxGlobal}). Cannot spawn more agents.` }],
details: { agentId: "", state: "error" },
};
}
try {
// Create agent in AgentStore with reportsTo = parent task ID
const agent = await this.options.agentStore.createAgent({
name: name.trim(),
role: role as AgentCapability,
reportsTo: taskId,
metadata: { type: "spawned", parentTaskId: taskId },
});
// Create git worktree for child (branched from parent's worktree)
const childWorktreeName = generateWorktreeName(this.rootDir);
const childWorktreePath = join(this.rootDir, ".worktrees", childWorktreeName);
const childBranch = `fusion/spawn-${agent.id}`;
await this.createWorktree(childBranch, childWorktreePath, taskId, worktreePath);
// Transition agent to active state
await this.options.agentStore.updateAgentState(agent.id, "active");
// Child agents inherit executor instructions
const childInstructions = await this.resolveInstructionsForRole("executor");
const childBasePrompt = `You are a child agent spawned by a parent task executor.
Your role:
- Complete the delegated task in your own worktree.
- Work autonomously, but stay tightly scoped to the delegated request.
- Prefer existing project patterns over inventing new ones.
- Run relevant tests and report what you verified.
- Do not widen scope or refactor unrelated areas.
Output expectations:
- Provide a concise summary of what you changed.
- Call out files touched and validations run.
- Explicitly mention unresolved blockers if you could not finish.
Parent task: ${taskId}
Child agent: ${agent.id} (${name})`;
const childSystemPrompt = buildSystemPromptWithInstructions(childBasePrompt, childInstructions);
// Build skill selection context for child agent session
const childTask = await this.store.getTask(taskId);
const skillContext = await buildSessionSkillContext({
agentStore: this.options.agentStore!,
task: childTask,
sessionPurpose: "executor",
projectRootDir: this.rootDir,
pluginRunner: this.options.pluginRunner,
});
const parentAgent = childTask.assignedAgentId
? await this.options.agentStore.getAgent(childTask.assignedAgentId).catch(() => null)
: null;
const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig)
?? extractRuntimeHint(parentAgent?.runtimeConfig);
// Resolve executor model via canonical lane hierarchy so child agents
// honor project executionProvider/executionModelId overrides (parity
// with main executor at the top of agentWork()).
const { provider: childExecutorProvider, modelId: childExecutorModelId } =
resolveExecutorModelPair(undefined, undefined, settings);
// Create child agent session
const { session: childSession } = await createResolvedAgentSession({
sessionPurpose: "executor",
runtimeHint: childRuntimeHint,
pluginRunner: this.options.pluginRunner,
cwd: childWorktreePath,
systemPrompt: childSystemPrompt,
tools: "coding",
defaultProvider: childExecutorProvider,
defaultModelId: childExecutorModelId,
fallbackProvider: settings.fallbackProvider,
fallbackModelId: settings.fallbackModelId,
// Skill selection: use assigned agent skills if available, otherwise role fallback
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
});
// Store tracking state
this.childSessions.set(agent.id, childSession);
if (!this.spawnedAgents.has(taskId)) {
this.spawnedAgents.set(taskId, new Set());
}
this.spawnedAgents.get(taskId)!.add(agent.id);
this.totalSpawnedCount++;
// Run child asynchronously (don't await — parent continues working)
this.runSpawnedChild(agent.id, childSession, taskPrompt).catch((err: unknown) => {
const errorMessage = err instanceof Error ? err.message : String(err);
executorLog.warn(`Child agent ${agent.id} async error: ${errorMessage}`);
});
const result: SpawnAgentResult = {
agentId: agent.id,
name: agent.name,
state: "running",
role: agent.role,
message: `Agent "${name}" spawned and executing task: ${taskPrompt.slice(0, 100)}${taskPrompt.length > 100 ? "..." : ""}`,
};
return {
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
details: result,
};
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
return {
content: [{ type: "text" as const, text: `Failed to spawn agent: ${errorMessage}` }],
details: { agentId: "", state: "error", message: errorMessage },
};
}
},
};
}
}
/**
* Format a timestamp for display in steering comments.
* Returns relative time for recent comments, absolute date for older ones.
*/
function formatTimestamp(iso: string): string {
const date = new Date(iso);
const now = new Date();
const diffMs = now.getTime() - date.getTime();
const diffMin = Math.floor(diffMs / 60000);
const diffHr = Math.floor(diffMin / 60);
const diffDay = Math.floor(diffHr / 24);
if (diffMin < 1) return "just now";
if (diffMin < 60) return `${diffMin}m ago`;
if (diffHr < 24) return `${diffHr}h ago`;
if (diffDay < 7) return `${diffDay}d ago`;
return date.toLocaleDateString();
}
// Project commands are injected here (for reliability) and also in the PROMPT.md (by triage).
// This ensures the executor agent always sees the authoritative commands from settings,
// even if the PROMPT.md was written manually or before commands were configured.
function scopePromptToWorktree(prompt: string, rootDir?: string, worktreePath?: string): string {
if (!rootDir || !worktreePath || rootDir === worktreePath || !prompt.includes(rootDir)) {
return prompt;
}
return prompt
.replaceAll(`${rootDir}/`, `${worktreePath}/`)
.replaceAll(`${worktreePath}/.fusion/`, `${rootDir}/.fusion/`);
}
export function buildExecutionPrompt(task: TaskDetail, rootDir?: string, settings?: Settings, worktreePath?: string): string {
const prompt = scopePromptToWorktree(task.prompt, rootDir, worktreePath);
const reviewMatch = prompt.match(/##\s*Review Level[:\s]*(\d)/);
const reviewLevel = reviewMatch ? parseInt(reviewMatch[1], 10) : 0;
// Build author arg for git commits based on settings
const authorArg = settings?.commitAuthorEnabled !== false
? ` --author="${settings?.commitAuthorName || "Fusion"} <${settings?.commitAuthorEmail || "noreply@runfusion.ai"}>"`
: "";
const sourceIssueRef = task.sourceIssue?.provider === "github" && task.sourceIssue.repository && task.sourceIssue.issueNumber
? `${task.sourceIssue.repository}#${task.sourceIssue.issueNumber}`
: "";
// Build step progress for resume
const hasProgress = task.steps.length > 0 && task.steps.some((s) => s.status !== "pending");
let progressSection = "";
if (hasProgress) {
const doneSteps = task.steps
.map((s, i) => ({ ...s, index: i }))
.filter((s) => s.status === "done");
const currentStep = task.currentStep;
const currentStepInfo = task.steps[currentStep];
progressSection = `
## ⚠️ RESUMING — Previous progress exists
This task was already partially executed. DO NOT redo completed steps.
### Step status:
${task.steps.map((s, i) => `- Step ${i} (${s.name}): **${s.status}**`).join("\n")}
### Resume from: Step ${currentStep}${currentStepInfo ? ` (${currentStepInfo.name})` : ""}
${doneSteps.length > 0 ? `Steps ${doneSteps.map((s) => s.index).join(", ")} are already complete — skip them entirely.` : ""}
Check the git log to understand what was already implemented:
\`\`\`bash
git log --oneline
\`\`\`
`;
}
// Build attachments section
let attachmentsSection = "";
if (task.attachments && task.attachments.length > 0 && rootDir) {
const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
const lines = ["## Attachments", ""];
for (const att of task.attachments) {
const absPath = `${rootDir}/.fusion/tasks/${task.id}/attachments/${att.filename}`;
if (IMAGE_MIMES.has(att.mimeType)) {
lines.push(`- **${att.originalName}** (screenshot): \`${absPath}\``);
} else {
lines.push(`- **${att.originalName}** (${att.mimeType}): \`${absPath}\` — read for context`);
}
}
attachmentsSection = "\n" + lines.join("\n") + "\n";
}
// Build project commands section from settings
let commandsSection = "";
if (settings?.testCommand || settings?.buildCommand) {
const lines = ["## Project Commands"];
if (settings.testCommand) lines.push(`- **Test:** \`${settings.testCommand}\``);
if (settings.buildCommand) lines.push(`- **Build:** \`${settings.buildCommand}\``);
commandsSection = "\n" + lines.join("\n") + "\n";
}
// Build project memory section from settings
// When enabled, agents consult and update project memory for durable project learnings.
// Backend-aware: instructions branch based on memoryBackendType (file, readonly, qmd)
const memoryEnabled = settings?.memoryEnabled !== false;
let memorySection = "";
if (memoryEnabled && rootDir) {
memorySection = "\n" + buildExecutionMemoryInstructions(rootDir, settings);
}
// Build steering comments section (last 10 comments only to avoid context bloat)
let steeringSection = "";
if (task.steeringComments && task.steeringComments.length > 0) {
const recentComments = [...task.steeringComments].slice(-10);
const lines = [
"",
"## Steering Comments",
"",
"The following comments were added by the user during execution. Consider adjusting your approach or replanning remaining steps based on this feedback.",
"",
];
for (const comment of recentComments) {
const timestamp = formatTimestamp(comment.createdAt);
lines.push(`**${comment.author}** — ${timestamp}`);
lines.push(`> ${comment.text}`);
lines.push("");
}
steeringSection = lines.join("\n");
}
return `Execute this task.
## Task: ${task.id}
${task.title ? `**${task.title}**` : ""}
${task.dependencies.length > 0 ? `Dependencies: ${task.dependencies.join(", ")}` : ""}
## PROMPT.md
${prompt}
${attachmentsSection}${commandsSection}${memorySection}${progressSection}${steeringSection}
## Review level: ${reviewLevel}
${reviewLevel === 0 ? "No reviews required. Implement directly." : ""}
${reviewLevel >= 1 ? `Before implementing each step (except Step 0 and the final step), call:
\`fn_review_step(step=N, type="plan", step_name="...")\`` : ""}
${reviewLevel >= 2 ? `After implementing + committing each step, call:
\`fn_review_step(step=N, type="code", step_name="...", baseline="<SHA from before step>")\`` : ""}
${reviewLevel >= 3 ? `After tests, also call fn_review_step with type="code" for test review.` : ""}
## Worktree Boundaries
You are running in an **isolated git worktree**. This means:
- **All code changes must be made inside the current worktree directory.** Do not modify files outside the worktree.
- **Exception — Project memory:** You MAY read and write to files under \`.fusion/memory/\` at the project root to save durable project learnings.
- **Exception — Task attachments:** You MAY read files under \`.fusion/tasks/{taskId}/attachments/\` at the project root for context.
- **Exception — Sibling task specs:** You MAY read \`.fusion/tasks/{taskId}/PROMPT.md\` and \`.fusion/tasks/{taskId}/task.json\` at the project root (read-only) to consult dependency tasks' specifications.
- **Shell commands** run inside the worktree by default. Avoid using \`cd\` to navigate outside the worktree.
## Begin
${hasProgress
? `Resume from Step ${task.currentStep}. Do NOT redo completed steps.`
: "Start with Step 0 (Preflight). Work through each step in order."}
Use \`fn_task_update\` to report progress on every step transition.
Use \`fn_task_log\` for important actions and decisions.
Use \`fn_task_create\` for truly separate follow-up work, not for fixes required to get tests, build, or typecheck back to green.
Commit at step boundaries: \`git commit -m "feat(${task.id}): complete Step N — description"${sourceIssueRef ? ` -m "Ref: ${sourceIssueRef}"` : ""}${authorArg}\`
When all steps are complete: call \`fn_task_done()\`
If a build command is configured, run that exact command in this worktree before calling \`fn_task_done()\`.
Treat a non-zero exit code as a blocking failure. Do not claim success without a real passing run.
Run the configured/full test suite and fix failures even when that requires edits outside the original File Scope.
If the repo has a lint command (e.g. \`pnpm lint\`, \`npm run lint\`), run it before \`fn_task_done()\` and fix any failures it reports.
If the repo has a typecheck command, run it before \`fn_task_done()\` and fix any failures it reports.
Use \`fn_task_create\` for truly separate follow-up work, not for fixes required to get tests, build, or typecheck back to green.
If lint is configured and failing, fix that too before completion.
**CRITICAL: Resolve ALL test failures (and any lint/typecheck failures) before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.`;
}
/**
* Format a comment for injection into a running agent session.
* Used for real-time steering during task execution.
*/
function formatCommentForInjection(comment: import("@fusion/core").SteeringComment): string {
const timestamp = formatTimestamp(comment.createdAt);
return `📣 **New feedback** — ${timestamp} (${comment.author}):\n\n${comment.text}\n\nPlease adjust your approach based on this feedback.`;
}
/**
* Result of a pseudo-pause detection check.
*/
export interface PseudoPauseResult {
/** Detection method: "regex" if a regex pattern matched, "structural" for structural
* heuristics, or "none" if no pseudo-pause was detected. */
kind: "regex" | "structural" | "none";
/** The matched text or pattern description when kind is not "none". */
matched?: string;
}
/**
* Detect whether the last assistant text output looks like a "pseudo-pause" —
* where the agent ended a turn by asking for permission or summarizing progress
* instead of calling a tool.
*
* Returns a {@link PseudoPauseResult} describing the detection kind and the
* matched text/pattern. Returns `{ kind: "none" }` when no pseudo-pause is found.
*
* @param lastText - The last assistant text output from the session.
*/
export function detectPseudoPause(lastText: string): PseudoPauseResult {
if (!lastText || lastText.trim().length === 0) {
return { kind: "none" };
}
const regexPatterns: RegExp[] = [
/\bif you (?:want|wish|need|like|prefer|'?d like)\b/i,
/\bshould I (?:continue|proceed|go ahead|move on|start|begin)\b/i,
/\blet me know\b/i,
/\b(?:want|would you like) me to (?:continue|proceed|finish|complete|do)\b/i,
/\bready to (?:proceed|continue|move on|begin)\b/i,
/\bshall I\b/i,
/\b(?:awaiting|waiting for) (?:your )?(?:approval|confirmation|go-ahead|response)\b/i,
];
for (const pattern of regexPatterns) {
const match = pattern.exec(lastText);
if (match) {
// Capture surrounding context (up to 120 chars around the match)
const start = Math.max(0, match.index - 40);
const end = Math.min(lastText.length, match.index + match[0].length + 80);
const snippet = lastText.slice(start, end).replace(/\n+/g, " ").trim();
return { kind: "regex", matched: snippet };
}
}
// Structural fallback: long output that ends with a question or a markdown "next steps" heading
const trimmed = lastText.trimEnd();
if (trimmed.length > 200) {
if (trimmed.endsWith("?")) {
const lastLine = trimmed.split("\n").at(-1) ?? trimmed;
return { kind: "structural", matched: lastLine.trim() };
}
const nextStepsPattern = /(?:^|\n)#+\s*(?:notes?|next steps?|summary|what'?s? next)\s*:?\s*$/i;
if (nextStepsPattern.test(trimmed)) {
const lastLine = trimmed.split("\n").at(-1) ?? trimmed;
return { kind: "structural", matched: lastLine.trim() };
}
// Also catch plain "Next steps:" or "### Next steps" at the very end
if (/next steps?\s*:?\s*$/i.test(trimmed)) {
const lastLine = trimmed.split("\n").at(-1) ?? trimmed;
return { kind: "structural", matched: lastLine.trim() };
}
}
return { kind: "none" };
}
/**
* Detect if a steering comment contains a review handoff request.
* Matches common handoff phrases that agents can use to request
* human review of their work.
*/
export function detectReviewHandoffIntent(commentText: string): boolean {
const text = commentText.toLowerCase();
const handoffPhrases = [
"send it back to me",
"hand off to user",
"needs human review",
"assign to user",
"return to user",
"user review needed",
"requesting user review",
];
return handoffPhrases.some((phrase) => text.includes(phrase));
}