Adds comprehensive tests for cleanup reliability in the in-process runtime (131-line test suite) plus related coverage in the executor tests, with a small fix to the runtime implementation itself to address the reliability issue. Fusion-Task-Id: FN-3352
7142 lines
321 KiB
TypeScript
7142 lines
321 KiB
TypeScript
import { exec } from "node:child_process";
|
||
import { promisify } from "node:util";
|
||
|
||
const execAsync = promisify(exec);
|
||
import { isAbsolute, join, relative, resolve as resolvePath } from "node:path";
|
||
import { existsSync } from "node:fs";
|
||
import { readFile, writeFile } from "node:fs/promises";
|
||
import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext } from "@fusion/core";
|
||
import {
|
||
buildExecutionMemoryInstructions,
|
||
getTaskMergeBlocker,
|
||
resolveAgentPrompt,
|
||
resolveProjectDefaultModel,
|
||
type RunCommandResult,
|
||
} from "@fusion/core";
|
||
import { findWorktreeUser } from "./merger.js";
|
||
import {
|
||
runVerificationCommand,
|
||
summarizeVerificationOutput,
|
||
VERIFICATION_LOG_MAX_CHARS,
|
||
type VerificationResult,
|
||
} from "./verification-utils.js";
|
||
import { generateWorktreeName, slugify } from "./worktree-names.js";
|
||
import { Type, type Static } from "@mariozechner/pi-ai";
|
||
import { describeModel, promptWithFallback, compactSessionContext } from "./pi.js";
|
||
import { accumulateSessionTokenUsage } from "./session-token-usage.js";
|
||
import { createResolvedAgentSession, extractRuntimeHint } from "./agent-session-helpers.js";
|
||
import { buildSessionSkillContext } from "./session-skill-context.js";
|
||
import { reviewStep, type ReviewVerdict } from "./reviewer.js";
|
||
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@mariozechner/pi-coding-agent";
|
||
import { PRIORITY_EXECUTE, type AgentSemaphore } from "./concurrency.js";
|
||
import { getRegisteredWorktreePaths, isGitRepository, isRegisteredGitWorktree, isUsableTaskWorktree, type WorktreePool } from "./worktree-pool.js";
|
||
import { AgentLogger } from "./agent-logger.js";
|
||
import { executorLog, reviewerLog, formatError } from "./logger.js";
|
||
import { TokenCapDetector } from "./token-cap-detector.js";
|
||
import { isUsageLimitError, checkSessionError, type UsageLimitPauser } from "./usage-limit-detector.js";
|
||
import { isTransientError, isSilentTransientError } from "./transient-error-detector.js";
|
||
import { withRateLimitRetry } from "./rate-limit-retry.js";
|
||
import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js";
|
||
import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js";
|
||
import type { PluginRunner } from "./plugin-runner.js";
|
||
import { isContextLimitError } from "./context-limit-detector.js";
|
||
import { StepSessionExecutor } from "./step-session-executor.js";
|
||
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
|
||
import type { AgentReflectionService } from "./agent-reflection.js";
|
||
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext } from "./run-audit.js";
|
||
import { evaluateSpecStaleness, getPromptPath } from "./spec-staleness.js";
|
||
import {
|
||
createDelegateTaskTool,
|
||
createGetAgentConfigTool,
|
||
createListAgentsTool,
|
||
createMemoryTools,
|
||
createReadMessagesTool,
|
||
createReflectOnPerformanceTool,
|
||
createUpdateAgentConfigTool,
|
||
createResearchTools,
|
||
createSendMessageTool,
|
||
createTaskCreateTool as sharedCreateTaskCreateTool,
|
||
createTaskDocumentReadTool as sharedCreateTaskDocumentReadTool,
|
||
createTaskDocumentWriteTool as sharedCreateTaskDocumentWriteTool,
|
||
createTaskLogTool as sharedCreateTaskLogTool,
|
||
} from "./agent-tools.js";
|
||
import { getTaskCompletionBlockerForStore } from "./task-completion.js";
|
||
import { createFusionAuthStorage, getModelRegistryModelsPath } from "./auth-storage.js";
|
||
import { createRunVerificationTool } from "./run-verification-tool.js";
|
||
import { createFallbackModelObserver } from "./fallback-model-observer.js";
|
||
|
||
// Re-export for backward compatibility (tests import from executor.ts)
|
||
export { summarizeToolArgs } from "./agent-logger.js";
|
||
export {
|
||
createDelegateTaskTool,
|
||
createGetAgentConfigTool,
|
||
createListAgentsTool,
|
||
createReadMessagesTool,
|
||
createUpdateAgentConfigTool,
|
||
createSendMessageTool,
|
||
createTaskCreateTool,
|
||
createTaskDocumentReadTool,
|
||
createTaskDocumentWriteTool,
|
||
createTaskLogTool,
|
||
delegateTaskParams,
|
||
listAgentsParams,
|
||
memoryAppendParams,
|
||
memoryGetParams,
|
||
memorySearchParams,
|
||
readMessagesParams,
|
||
sendMessageParams,
|
||
taskCreateParams,
|
||
taskLogParams,
|
||
} from "./agent-tools.js";
|
||
|
||
const STEP_STATUSES: StepStatus[] = ["pending", "in-progress", "done", "skipped"];
|
||
|
||
/** Maximum retry attempts for workflow step hard failures before giving up */
|
||
const MAX_WORKFLOW_STEP_RETRIES = 3;
|
||
/** Maximum in-session retries when an agent exits without calling fn_task_done(). */
|
||
const MAX_TASK_DONE_SESSION_RETRIES = 3;
|
||
/** Maximum todo requeues after exhausting in-session fn_task_done retries. */
|
||
const MAX_TASK_DONE_REQUEUE_RETRIES = 3;
|
||
/** How long to wait before recovering a completed task still stuck in in-progress. */
|
||
const COMPLETED_TASK_WATCHDOG_MS = 60_000;
|
||
/** How long to wait before retrying a workflow rerun handoff that never reached in-progress. */
|
||
const WORKFLOW_RERUN_WATCHDOG_MS = 15_000;
|
||
|
||
/**
|
||
* @deprecated Kept exported so existing unit tests in executor.test.ts still
|
||
* link, but no longer called from the executor. Revision feedback is applied
|
||
* as an in-place fix via `reopenLastStepForRevision` — earlier completed
|
||
* steps stay done instead of being replayed.
|
||
*/
|
||
export function determineRevisionResetStart(
|
||
steps: ReadonlyArray<{ name: string }>,
|
||
feedback: string,
|
||
): number {
|
||
const total = steps.length;
|
||
if (total === 0) return 0;
|
||
const skipPreflight = /preflight/i.test(steps[0].name);
|
||
const firstCandidate = skipPreflight ? 1 : 0;
|
||
if (firstCandidate >= total) return total;
|
||
const fb = feedback.toLowerCase();
|
||
for (let i = firstCandidate; i < total; i++) {
|
||
const tokens = steps[i].name.toLowerCase().match(/[a-z][a-z]{4,}/g) ?? [];
|
||
if (tokens.some((t) => fb.includes(t))) return i;
|
||
}
|
||
return firstCandidate;
|
||
}
|
||
const WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS = 4_000;
|
||
|
||
class NonRetryableWorktreeError extends Error {}
|
||
|
||
function truncateWorkflowScriptOutput(output: string): string {
|
||
if (output.length <= WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS) return output;
|
||
return `... output truncated to last ${WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS} characters ...\n${output.slice(-WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS)}`;
|
||
}
|
||
|
||
function configuredCommandErrorMessage(result: RunCommandResult): string {
|
||
if (result.spawnError) return result.spawnError.message;
|
||
const parts: string[] = [];
|
||
if (result.timedOut) parts.push("Timed out");
|
||
if (result.exitCode !== null) parts.push(`Exit code: ${result.exitCode}`);
|
||
if (result.signal) parts.push(`Signal: ${result.signal}`);
|
||
const stdout = result.stdout.trim();
|
||
const stderr = result.stderr.trim();
|
||
if (stdout) parts.push(`stdout: ${truncateWorkflowScriptOutput(stdout)}`);
|
||
if (stderr) parts.push(`stderr: ${truncateWorkflowScriptOutput(stderr)}`);
|
||
return parts.length ? parts.join("\n") : "Command failed";
|
||
}
|
||
|
||
async function runConfiguredCommand(
|
||
command: string,
|
||
cwd: string,
|
||
timeoutMs: number,
|
||
): Promise<RunCommandResult> {
|
||
try {
|
||
const { stdout, stderr } = await execAsync(command, {
|
||
cwd,
|
||
timeout: timeoutMs,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
encoding: "utf-8",
|
||
});
|
||
|
||
return {
|
||
stdout: stdout?.toString?.() ?? "",
|
||
stderr: stderr?.toString?.() ?? "",
|
||
exitCode: 0,
|
||
signal: null,
|
||
bufferExceeded: false,
|
||
timedOut: false,
|
||
};
|
||
} catch (error) {
|
||
const errObj = error as Record<string, unknown>;
|
||
const code = errObj?.code;
|
||
const status = typeof errObj?.status === "number" ? errObj.status : null;
|
||
const exitCode = typeof code === "number" ? code : status;
|
||
const message = String(errObj?.message ?? "");
|
||
|
||
return {
|
||
stdout: typeof (errObj?.stdout as { toString?: unknown })?.toString === "function" ? String(errObj.stdout) : "",
|
||
stderr: typeof (errObj?.stderr as { toString?: unknown })?.toString === "function" ? String(errObj.stderr) : "",
|
||
exitCode,
|
||
signal: (errObj?.signal as NodeJS.Signals | null | undefined) ?? null,
|
||
bufferExceeded:
|
||
code === "ENOBUFS"
|
||
|| code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"
|
||
|| message.includes("maxBuffer"),
|
||
timedOut:
|
||
code === "ETIMEDOUT"
|
||
|| (errObj?.killed === true && (errObj?.signal === "SIGTERM" || message.includes("timed out"))),
|
||
spawnError: code === "ENOENT" || code === "EACCES" ? (error as Error) : undefined,
|
||
};
|
||
}
|
||
}
|
||
|
||
// ── Tool parameter schemas (module-level for reuse in ToolDefinition generics) ──
|
||
|
||
const taskUpdateParams = Type.Object({
|
||
step: Type.Number({ description: "Step number (0-indexed)" }),
|
||
status: Type.Union(
|
||
STEP_STATUSES.map((s) => Type.Literal(s)),
|
||
{ description: "New status: pending, in-progress, done, or skipped" },
|
||
),
|
||
});
|
||
|
||
// taskLogParams and taskCreateParams are imported from agent-tools.ts
|
||
|
||
const taskAddDepParams = Type.Object({
|
||
task_id: Type.String({ description: "The ID of the task to depend on (e.g. \"KB-001\")" }),
|
||
confirm: Type.Optional(Type.Boolean({ description: "Set to true to confirm adding the dependency. Required because adding a dep to an in-progress task will stop execution and discard current work." })),
|
||
});
|
||
|
||
const spawnAgentParams = Type.Object({
|
||
name: Type.String({ description: "Name for the child agent" }),
|
||
role: Type.Union([
|
||
Type.Literal("triage"),
|
||
Type.Literal("executor"),
|
||
Type.Literal("reviewer"),
|
||
Type.Literal("merger"),
|
||
Type.Literal("engineer"),
|
||
Type.Literal("custom"),
|
||
], { description: "Role for the child agent" }),
|
||
task: Type.String({ description: "Task description for the child agent to execute" }),
|
||
});
|
||
|
||
/** Result returned from fn_spawn_agent tool */
|
||
interface SpawnAgentResult {
|
||
agentId: string;
|
||
name: string;
|
||
state: AgentState;
|
||
role: AgentCapability;
|
||
message: string;
|
||
}
|
||
|
||
/**
|
||
* Outcome of a single workflow step execution.
|
||
* Supports three states: pass, hard failure, or revision requested with feedback.
|
||
*/
|
||
export interface WorkflowStepOutcome {
|
||
success: boolean;
|
||
revisionRequested?: boolean;
|
||
output?: string;
|
||
error?: string;
|
||
/** Set when the call exceeded `settings.workflowStepTimeoutMs`. Signals the
|
||
* caller to escalate to the fallback model rather than treat the failure
|
||
* as a generic revision request. */
|
||
timedOut?: boolean;
|
||
}
|
||
|
||
/**
|
||
* Result of running all pre-merge workflow steps.
|
||
* Returns true if all passed, false if any hard failure, or a structured
|
||
* revision result if a revision was requested.
|
||
*/
|
||
export type WorkflowStepResult =
|
||
| { allPassed: true }
|
||
| { allPassed: false; revisionRequested: false; feedback: string; stepName: string }
|
||
| { allPassed: false; revisionRequested: true; feedback: string; stepName: string };
|
||
|
||
|
||
const reviewStepParams = Type.Object({
|
||
step: Type.Number({ description: "Step number to review" }),
|
||
type: Type.Union(
|
||
[Type.Literal("plan"), Type.Literal("code")],
|
||
{ description: 'Review type: "plan" or "code"' },
|
||
),
|
||
step_name: Type.String({ description: "Name of the step being reviewed" }),
|
||
baseline: Type.Optional(
|
||
Type.String({
|
||
description:
|
||
"Git commit SHA for code review diff baseline. " +
|
||
"Capture HEAD before starting a step and pass it here.",
|
||
}),
|
||
),
|
||
});
|
||
|
||
const EXECUTOR_SYSTEM_PROMPT = `You are a task execution agent for "fn", an AI-orchestrated task board.
|
||
|
||
You are working in a git worktree isolated from the main branch. Your job is to implement the task described in the PROMPT.md specification you're given.
|
||
|
||
## Your Role in the System
|
||
You are the primary implementation agent in Fusion.
|
||
You execute task specs in isolated worktrees, produce production-quality changes, and hand off work that can pass independent review and merge.
|
||
|
||
## Turn-ending rules — read carefully
|
||
|
||
You MUST end every turn by either:
|
||
- (a) calling another tool to make progress, OR
|
||
- (b) calling \`fn_task_done\` if the entire task is complete, OR
|
||
- (c) calling \`fn_task_done\` with a summary explaining what is blocked, if you cannot make progress for any reason
|
||
|
||
You MUST NOT end a turn by writing prose that asks the user a question, summarizes progress, or requests permission to continue. The following are FORBIDDEN turn-endings:
|
||
- "If you want, I can continue with..."
|
||
- "Should I proceed with...?"
|
||
- "Let me know if you'd like me to..."
|
||
- "Ready to move on to step N. Want me to continue?"
|
||
- Any markdown progress summary at the end of a turn instead of a tool call
|
||
|
||
If you have just finished a step's work, immediately call \`fn_task_update\` to mark the step done and continue with the next pending step in the SAME turn. Do not pause to summarize.
|
||
|
||
The user is not watching this conversation in real-time. They will read the final result. Asking permission wastes a full retry cycle and may orphan committed work.
|
||
|
||
If you genuinely cannot proceed (blocked on a dependency, missing information, or an unresolvable error), call \`fn_task_done\` with a clear explanation of what is blocked and what is needed to unblock it. Never write the question as plain prose.
|
||
|
||
## How to work
|
||
1. Read the PROMPT.md carefully — it contains your mission, steps, file scope, acceptance criteria, and Do NOT constraints
|
||
2. Before touching code, read all files listed in "Context to Read First" and understand the full step outcome
|
||
3. Check existing patterns in the codebase before introducing new structure, naming, or APIs
|
||
4. Work through each step in order
|
||
5. Write clean, production-quality code
|
||
6. Test your changes continuously
|
||
7. Commit at meaningful boundaries (step completion)
|
||
|
||
## Reporting progress via tools
|
||
|
||
You have tools to report progress. The board updates in real-time.
|
||
|
||
**Step lifecycle:**
|
||
- Before starting a step: \`fn_task_update(step=N, status="in-progress")\`
|
||
- After completing a step: \`fn_task_update(step=N, status="done")\`
|
||
- If skipping a step: \`fn_task_update(step=N, status="skipped")\`
|
||
|
||
**Logging important actions:** \`fn_task_log(message="what happened")\`
|
||
|
||
**Out-of-scope work found during execution:** \`fn_task_create(description="what needs doing")\`
|
||
When creating multiple related tasks, declare dependencies between them:
|
||
\`fn_task_create(description="load door sounds", dependencies=[])\` → returns KB-050
|
||
\`fn_task_create(description="play sound on door open/close", dependencies=["KB-050"])\`
|
||
|
||
**Discovered a dependency:** \`fn_task_add_dep(task_id="KB-XXX")\` — use when you discover mid-execution that another task must be completed first. This will return a warning first — you must call again with \`confirm=true\` to proceed. Adding a dependency stops execution, discards current work, and moves the task to triage for re-planning.
|
||
|
||
## Cross-model review via fn_review_step tool
|
||
|
||
You have a \`fn_review_step\` tool. It spawns a SEPARATE reviewer agent (different
|
||
model, read-only access) to independently assess your work.
|
||
|
||
**When to call it** — based on the Review Level in the PROMPT.md:
|
||
|
||
| Review Level | Before implementing | After implementing + committing |
|
||
|-------------|--------------------|---------------------------------|
|
||
| 0 (None) | — | — |
|
||
| 1 (Plan) | \`fn_review_step(step, "plan", step_name)\` | — |
|
||
| 2 (Plan+Code) | \`fn_review_step(step, "plan", step_name)\` | \`fn_review_step(step, "code", step_name, baseline)\` |
|
||
| 3 (Full) | plan review | code review + test review |
|
||
|
||
**Skip reviews for** Step 0 (Preflight) and the final documentation/delivery step.
|
||
|
||
**Code review flow:**
|
||
1. Before starting a step, capture baseline: \`git rev-parse HEAD\`
|
||
2. Implement the step
|
||
3. Commit
|
||
4. Call \`fn_review_step\` with the baseline SHA so the reviewer sees only your changes
|
||
|
||
**Handling verdicts:**
|
||
- **APPROVE** → proceed to next step
|
||
- **REVISE (code review)** → **enforced**. You MUST fix the issues, commit again,
|
||
and re-run \`fn_review_step(type="code")\` before the step can be marked done.
|
||
\`fn_task_update(status="done")\` will be rejected until the code review passes.
|
||
- **REVISE (plan review)** → advisory. Incorporate the feedback at your discretion
|
||
and proceed with implementation. No re-review is required.
|
||
- **RETHINK (code review)** → your code changes have been reverted and conversation rewound. Read the feedback carefully and take a fundamentally different approach. Do NOT repeat the rejected strategy.
|
||
- **RETHINK (plan review)** → conversation rewound to before the step (no git reset since no code was written). Read the feedback and take a fundamentally different approach to planning this step.
|
||
|
||
## Task Documents
|
||
|
||
You can save and retrieve named documents for this task. Use these to store planning notes, research findings, or any persistent data that should survive across sessions.
|
||
|
||
- **Save a document:** \`fn_task_document_write(key="plan", content="...")\`
|
||
- **Read a document:** \`fn_task_document_read(key="plan")\`
|
||
- **List all documents:** \`fn_task_document_read()\` (no key)
|
||
|
||
Documents are versioned — each write creates a new revision. Use meaningful keys like "plan", "notes", "research", "architecture".
|
||
|
||
## Research tools
|
||
When implementation needs external context, you may use research tools (
|
||
\`fn_research_run\`, \`fn_research_list\`, \`fn_research_get\`, \`fn_research_cancel\`) to run bounded research.
|
||
Keep runs focused and short, and persist durable conclusions into task documents (for example key="research").
|
||
If research is disabled or providers are not configured, use the actionable tool response and continue with available local context.
|
||
|
||
**IMPORTANT — Save your deliverables as documents:** When your task produces written output (documentation, specifications, reports, API references, README updates, guides, or any other content), you MUST save that content as a task document using \`fn_task_document_write\`. Use a key that describes the deliverable (e.g., key="readme", key="api-docs", key="changelog"). Do this in addition to writing the file to disk — the document persists in the task for review even after the worktree is cleaned up.
|
||
|
||
If the task's PROMPT.md includes a "Documentation Requirements" section listing files to update, save each updated file's final content as a task document with a matching key.
|
||
|
||
## Git discipline
|
||
- Commit after completing each step (not after every file change)
|
||
- Use conventional commit messages prefixed with the task ID
|
||
- When the task has a GitHub issue reference, include \`Ref: owner/repo#N\` in the commit body
|
||
- Do NOT commit broken or half-implemented code
|
||
|
||
Good commit message examples:
|
||
- \`feat(FN-1234): complete Step 2 — add retry guard for workflow step timeouts\`
|
||
- \`test(FN-1234): add regression tests for paused-session cleanup\`
|
||
|
||
Bad commit message examples:
|
||
- \`misc updates\`
|
||
- \`fix stuff\`
|
||
- \`wip\`
|
||
|
||
## Worktree Boundaries
|
||
|
||
You are running in an **isolated git worktree**. This means:
|
||
|
||
- **All code changes must be made inside the current worktree directory.** Do not modify files outside the worktree — the worktree is your isolated execution environment.
|
||
- **Exception — Project memory:** You MAY read and write to files under .fusion/memory/ at the project root to save durable project learnings (architecture patterns, conventions, pitfalls).
|
||
- **Exception — Task attachments:** You MAY read files under .fusion/tasks/{taskId}/attachments/ at the project root for context screenshots and documents attached to this task.
|
||
- **Exception — Sibling task specs:** You MAY read .fusion/tasks/{taskId}/PROMPT.md and .fusion/tasks/{taskId}/task.json at the project root (read-only) to consult dependency tasks' specifications.
|
||
- **Shell commands** run inside the worktree by default. Avoid using cd to navigate outside the worktree.
|
||
|
||
If you attempt to write to a path outside the worktree, the file tools will reject the operation with an error explaining the boundary.
|
||
|
||
## Guardrails
|
||
- **NEVER kill processes on port 4040.** Port 4040 is the production dashboard. Do not run \`kill\`, \`pkill\`, \`killall\`, or \`lsof -ti:4040 | xargs kill\` against it. If you need to start a test server, use \`--port 0\` for a random free port. If port 4040 is occupied, pick a different port — do NOT kill the occupant.
|
||
- Treat the File Scope in PROMPT.md as the expected starting scope, not a hard boundary when quality gates fail
|
||
- Read "Context to Read First" files before starting
|
||
- Follow the "Do NOT" section strictly — these are hard constraints, not suggestions
|
||
- If tests, lint, build, or typecheck fail and the fix requires touching code outside the declared File Scope, fix those failures directly and keep the repo green
|
||
- Use \`fn_task_create\` for genuinely separate follow-up work, not for mandatory fixes required to make this task land cleanly
|
||
- Update documentation listed in "Must Update" and check "Check If Affected"
|
||
- NEVER delete, remove, or gut modules, interfaces, settings, exports, or test files outside your File Scope
|
||
- NEVER remove features as "cleanup" — if something seems unused, create a task for investigation instead
|
||
- Removing code is acceptable ONLY when it is explicitly part of your task's mission
|
||
- If you remove existing functionality, you MUST create a changeset in \`.changeset/\` explaining the removal and rationale
|
||
|
||
## Spawning Child Agents
|
||
|
||
You can spawn child agents to handle parallel work or specialized sub-tasks:
|
||
|
||
**When to use \`fn_spawn_agent\`:**
|
||
- Parallel work that can be divided into independent chunks with minimal overlap
|
||
- Specialized tasks requiring different expertise or tools
|
||
- Delegation of sub-tasks whose outputs can be validated independently
|
||
|
||
**When NOT to spawn:**
|
||
- The work is small enough to finish directly in your current step
|
||
- Subtasks are tightly coupled and would create merge/cherry-pick overhead
|
||
- You have not yet clarified expected outputs and acceptance criteria for the child
|
||
|
||
**How to spawn:**
|
||
\`\`\`javascript
|
||
fn_spawn_agent({
|
||
name: "researcher",
|
||
role: "engineer",
|
||
task: "Research best practices for authentication in React applications"
|
||
})
|
||
\`\`\`
|
||
|
||
**Child agent behavior:**
|
||
- Each child runs in its own git worktree (branched from your worktree)
|
||
- Children execute autonomously and report completion
|
||
- When you end (fn_task_done), all spawned children are terminated
|
||
- Check AgentStore for spawned agent status
|
||
|
||
**Limits:**
|
||
- Max 5 spawned agents per parent by default (configurable via settings)
|
||
- Max 20 total spawned agents system-wide (configurable via settings)
|
||
|
||
## Completion
|
||
After all steps are done, lint passes, tests pass, typecheck passes, and docs are updated:
|
||
\`\`\`bash
|
||
Call \`fn_task_done()\` to signal completion.
|
||
\`\`\`
|
||
|
||
If a project build command is listed in the prompt, it is a hard completion gate:
|
||
- Run the exact build command in the current worktree before \`fn_task_done()\`
|
||
- Do not claim the build passes unless you actually ran it and got exit code 0
|
||
- If the build fails, do NOT call \`fn_task_done()\`; keep working until it passes
|
||
|
||
Lint, tests, and typecheck are also hard quality gates:
|
||
- Keep fixing failures until lint, the configured/full test suite, and typecheck all pass
|
||
- If the repository exposes a typecheck command, run it and keep fixing failures until it passes
|
||
- Do not stop at "out of scope" if additional fixes are required to restore green lint, tests, build, or typecheck
|
||
- When tests fail, first identify whether the failure is caused by your change, a pre-existing defect, or an outdated test expectation; then fix code or tests accordingly so behavior and assertions match
|
||
- Update tests when intended behavior changed; fix implementation when behavior regressed unintentionally
|
||
- **CRITICAL: Resolve ALL lint failures and test failures before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.
|
||
|
||
## Verification commands — use fn_run_verification
|
||
|
||
For ALL test/lint/build/typecheck verification, use the \`fn_run_verification\` tool, NOT raw bash.
|
||
The tool prevents your session from being killed by the inactivity watchdog during long compiles.
|
||
|
||
- Prefer **package-scoped** verification first: e.g. \`pnpm --filter @fusion/<pkg> test\` with \`scope: "package"\`. This is faster and isolated.
|
||
- Only run **workspace-scoped** verification (\`pnpm test\`, \`pnpm lint\`, \`pnpm build\` from root) at the FINAL integration step, when you are about to call \`fn_task_done\`.
|
||
- If you need to run \`pnpm install\` (e.g. you added a new package), use \`fn_run_verification\` with \`scope: "workspace"\` and \`timeoutSec: 600\`.
|
||
- If a verification command times out, do NOT blindly retry — investigate. Check for hung subprocesses, infinite test loops, or tests waiting on missing dependencies. Use \`node_modules/.modules.yaml\` presence to confirm bootstrap.
|
||
|
||
## Common Pitfalls
|
||
- Editing files outside the assigned worktree (except allowed memory/attachment paths)
|
||
- Skipping or partially running required quality gates
|
||
- Leaving TODO/FIXME placeholders instead of completing required implementation
|
||
- Introducing new patterns when existing local patterns should be reused
|
||
- Marking a step done before required review/tooling gates are satisfied`;
|
||
|
||
/** Resolve the executor system prompt from settings, falling back to the hardcoded constant. */
|
||
function getExecutorSystemPrompt(settings: Settings): string {
|
||
const customPrompt = resolveAgentPrompt("executor", settings.agentPrompts);
|
||
return customPrompt || EXECUTOR_SYSTEM_PROMPT;
|
||
}
|
||
|
||
function resolveExecutorModelPair(
|
||
taskModelProvider: string | undefined,
|
||
taskModelId: string | undefined,
|
||
settings: Partial<Settings> | undefined,
|
||
): { provider: string | undefined; modelId: string | undefined } {
|
||
if (taskModelProvider && taskModelId) {
|
||
return { provider: taskModelProvider, modelId: taskModelId };
|
||
}
|
||
if (settings?.executionProvider && settings?.executionModelId) {
|
||
return {
|
||
provider: settings.executionProvider,
|
||
modelId: settings.executionModelId,
|
||
};
|
||
}
|
||
if (settings?.executionGlobalProvider && settings?.executionGlobalModelId) {
|
||
return {
|
||
provider: settings.executionGlobalProvider,
|
||
modelId: settings.executionGlobalModelId,
|
||
};
|
||
}
|
||
if (settings?.defaultProviderOverride && settings?.defaultModelIdOverride) {
|
||
return {
|
||
provider: settings.defaultProviderOverride,
|
||
modelId: settings.defaultModelIdOverride,
|
||
};
|
||
}
|
||
if (settings?.defaultProvider && settings?.defaultModelId) {
|
||
return {
|
||
provider: settings.defaultProvider,
|
||
modelId: settings.defaultModelId,
|
||
};
|
||
}
|
||
return { provider: undefined, modelId: undefined };
|
||
}
|
||
|
||
export interface TaskExecutorOptions {
|
||
semaphore?: AgentSemaphore;
|
||
/** Worktree pool for recycling idle worktrees across tasks. */
|
||
pool?: WorktreePool;
|
||
/** Usage limit pauser — triggers global pause when API limits are detected. */
|
||
usageLimitPauser?: UsageLimitPauser;
|
||
/** Stuck task detector — monitors agent sessions for stagnation and triggers recovery. */
|
||
stuckTaskDetector?: StuckTaskDetector;
|
||
/** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */
|
||
agentStore?: import("@fusion/core").AgentStore;
|
||
/** Reflection service used to generate self-reflection insights for agents. */
|
||
reflectionService?: AgentReflectionService;
|
||
/** Plugin runner for invoking plugin hooks and providing plugin tools. */
|
||
pluginRunner?: PluginRunner;
|
||
/** MessageStore for sending messages to other agents. When provided, executor agents gain fn_send_message capability. */
|
||
messageStore?: import("@fusion/core").MessageStore;
|
||
missionStore?: MissionStore;
|
||
onSliceComplete?: (slice: Slice) => void;
|
||
onStart?: (task: Task, worktreePath: string) => void;
|
||
onComplete?: (task: Task) => void;
|
||
onError?: (task: Task, error: Error) => void;
|
||
onAgentText?: (taskId: string, delta: string) => void;
|
||
onAgentTool?: (taskId: string, toolName: string) => void;
|
||
}
|
||
|
||
export class TaskExecutor {
|
||
private activeWorktrees = new Map<string, string>();
|
||
private executing = new Set<string>();
|
||
/** Tasks currently being prepared for unpause resume, before execute() has registered them. */
|
||
private resumingUnpaused = new Set<string>();
|
||
/** Completed orphan recovery tasks currently running during startup. */
|
||
private recoveringCompleted = new Set<string>();
|
||
/** Tracks tasks whose workflow-rerun bounce is in flight (todo→in-progress).
|
||
* Prevents the task:moved handler from dispatching execute() before the
|
||
* bounce finishes its own dispatch. */
|
||
private workflowRerunPending = new Set<string>();
|
||
/** Active agent sessions per task, used to terminate on pause and inject steering. */
|
||
private activeSessions = new Map<string, {
|
||
session: AgentSession;
|
||
seenSteeringIds: Set<string>;
|
||
lastModelProvider?: string | null;
|
||
lastModelId?: string | null;
|
||
}>();
|
||
/** Active step-session executors per task (mutually exclusive with activeSessions). */
|
||
private activeStepExecutors = new Map<string, StepSessionExecutor>();
|
||
/** Active pre-merge workflow step sessions per task. */
|
||
private activeWorkflowStepSessions = new Map<string, AgentSession>();
|
||
/**
|
||
* Reviewer subagent sessions per task. Reviewers (`reviewer.ts`) create their
|
||
* own AgentSessions that aren't part of `activeSessions`/`activeStepExecutors`,
|
||
* so without this map they survive when the parent task is stopped — they
|
||
* keep producing log entries and step transitions after the user thinks they
|
||
* killed the task. Disposed alongside the main session in the move-out,
|
||
* pause, and global-pause handlers below.
|
||
*/
|
||
private activeSubagentSessions = new Map<string, Set<AgentSession>>();
|
||
/** Tasks that were paused mid-execution (to avoid marking them as "failed"). */
|
||
private pausedAborted = new Set<string>();
|
||
/** Tasks that had a dependency added mid-execution (abort + discard worktree). */
|
||
private depAborted = new Set<string>();
|
||
/** Tasks killed by stuck task detector. Value = shouldRequeue (budget not exhausted). */
|
||
private stuckAborted = new Map<string, boolean>();
|
||
/** In-memory loop recovery state per task. Keyed by taskId, not persisted.
|
||
* Tracks compact-and-resume attempt count per execute() lifecycle.
|
||
* Reset at execute() lifecycle end (finally block). */
|
||
private loopRecoveryState = new Map<string, { attempts: number; pending: boolean }>();
|
||
/** Spawned child agent IDs per parent task ID. Used for lifecycle tracking. */
|
||
private spawnedAgents = new Map<string, Set<string>>();
|
||
/** Per-task baseline of session stats used for delta persistence across repeated updates. */
|
||
private tokenUsageBaselines = new Map<string, { inputTokens: number; outputTokens: number; cachedTokens: number; totalTokens: number }>();
|
||
/** One-shot watchdogs for completed tasks that should have transitioned to in-review. */
|
||
private completedTaskWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
|
||
/** One-shot watchdogs for workflow reruns that should have bounced back to in-progress. */
|
||
private workflowRerunWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
|
||
/** Set of ephemeral spawned agent IDs with scheduled cleanup (prevents duplicate deletion attempts). */
|
||
private pendingEphemeralDeletions = new Set<string>();
|
||
/** Map of spawned agent IDs to scheduled cleanup timer handles for shutdown disposal. */
|
||
private ephemeralCleanupTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||
|
||
private async finalizeAlreadyReviewedTask(taskId: string): Promise<"merged" | "blocked" | "missing"> {
|
||
const latestTask = await this.store.getTask(taskId);
|
||
if (!latestTask || latestTask.column !== "in-review") {
|
||
return "missing";
|
||
}
|
||
|
||
const blocker = getTaskMergeBlocker(latestTask);
|
||
if (blocker) {
|
||
await this.store.logEntry(taskId, "Task already in-review; merge deferred", blocker, this.currentRunContext);
|
||
return "blocked";
|
||
}
|
||
|
||
await this.store.logEntry(
|
||
taskId,
|
||
"Task already in-review after completion — finalizing merge",
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
await this.store.mergeTask(taskId);
|
||
return "merged";
|
||
}
|
||
|
||
private async getExecutionPauseLabel(): Promise<"global pause" | "engine pause" | null> {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause) return "global pause";
|
||
if (settings.enginePaused) return "engine pause";
|
||
return null;
|
||
}
|
||
|
||
private async shouldDeferCompletionForGlobalPause(
|
||
taskId: string,
|
||
context: string,
|
||
): Promise<boolean> {
|
||
const settings = await this.store.getSettings();
|
||
if (!settings.globalPause) {
|
||
return false;
|
||
}
|
||
|
||
this.clearCompletedTaskWatchdog(taskId);
|
||
executorLog.log(`${taskId}: completion handoff deferred — global pause active (${context})`);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Completion handoff deferred — global pause active (${context})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
).catch(() => undefined);
|
||
return true;
|
||
}
|
||
|
||
private async shouldDeferWorkflowStepCompletion(
|
||
taskId: string,
|
||
context: string,
|
||
): Promise<boolean> {
|
||
let latestTask: Task | null = null;
|
||
try {
|
||
latestTask = await this.store.getTask(taskId);
|
||
} catch {
|
||
latestTask = null;
|
||
}
|
||
|
||
if (latestTask?.paused || this.pausedAborted.has(taskId)) {
|
||
this.clearCompletedTaskWatchdog(taskId);
|
||
executorLog.log(`${taskId}: completion handoff deferred — task paused (${context})`);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Completion handoff deferred — task paused (${context})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
).catch(() => undefined);
|
||
return true;
|
||
}
|
||
|
||
return this.shouldDeferCompletionForGlobalPause(taskId, context);
|
||
}
|
||
|
||
private async parkTaskAfterWorkflowStepPause(taskId: string): Promise<boolean> {
|
||
let latestTask: Task | null = null;
|
||
try {
|
||
latestTask = await this.store.getTask(taskId);
|
||
} catch {
|
||
latestTask = null;
|
||
}
|
||
|
||
if (!latestTask?.paused) {
|
||
return false;
|
||
}
|
||
|
||
executorLog.log(`${taskId}: workflow step interrupted by task pause — moving to todo`);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
"Execution paused during pre-merge workflow step — moved to todo",
|
||
undefined,
|
||
this.currentRunContext,
|
||
).catch(() => undefined);
|
||
if (latestTask.column === "in-progress") {
|
||
await this.store.moveTask(taskId, "todo", { preserveResumeState: true });
|
||
}
|
||
return true;
|
||
}
|
||
/** Child agent sessions keyed by agent ID. Used for termination. */
|
||
private childSessions = new Map<string, AgentSession>();
|
||
/** Total count of currently spawned agents (across all parents). */
|
||
private totalSpawnedCount = 0;
|
||
/** Token cap detector for proactive context compaction. */
|
||
private tokenCapDetector = new TokenCapDetector();
|
||
private _modelRegistry?: ModelRegistry;
|
||
/** Current run context for mutation correlation. Set at execute() start, cleared in finally. */
|
||
private currentRunContext: RunMutationContext | undefined;
|
||
|
||
private get modelRegistry(): ModelRegistry {
|
||
if (!this._modelRegistry) {
|
||
const authStorage = createFusionAuthStorage();
|
||
this._modelRegistry = ModelRegistry.create(authStorage, getModelRegistryModelsPath());
|
||
this._modelRegistry.refresh();
|
||
}
|
||
return this._modelRegistry;
|
||
}
|
||
|
||
/** Returns the set of task IDs currently being executed. */
|
||
getExecutingTaskIds(): Set<string> {
|
||
return new Set([...this.executing, ...this.recoveringCompleted, ...this.resumingUnpaused]);
|
||
}
|
||
|
||
isEphemeralDeletionPending(agentId: string): boolean {
|
||
return this.pendingEphemeralDeletions.has(agentId);
|
||
}
|
||
|
||
disposeEphemeralTimers(): void {
|
||
const timerCount = this.ephemeralCleanupTimers.size;
|
||
for (const timerId of this.ephemeralCleanupTimers.values()) {
|
||
clearTimeout(timerId);
|
||
}
|
||
this.ephemeralCleanupTimers.clear();
|
||
this.pendingEphemeralDeletions.clear();
|
||
if (timerCount > 0) {
|
||
executorLog.log(`Cleared ${timerCount} pending spawned-agent cleanup timer(s)`);
|
||
}
|
||
}
|
||
|
||
private isBenignEphemeralDeleteRaceError(agentId: string, err: unknown): boolean {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
const lower = msg.toLowerCase();
|
||
if (lower.includes("not found") || lower.includes("already deleted") || lower.includes("does not exist")) {
|
||
executorLog.log(`Skip spawned-agent cleanup for ${agentId}: already deleted by another pathway`);
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Abort the in-flight bash subprocess (if any) on every active agent session.
|
||
*
|
||
* Invoked at runtime shutdown so detached subprocess trees spawned by agent
|
||
* bash tools — including grandchildren like vitest workers — are killed via
|
||
* pi-coding-agent's killProcessTree. Without this, when the worker is killed
|
||
* those process groups are orphaned because they're detached.
|
||
*
|
||
* Sessions are not disposed here so any near-complete agent loop still has a
|
||
* chance to wrap up during the runtime's graceful drain window.
|
||
*/
|
||
|
||
/**
|
||
* Register a subagent session (e.g. reviewer) under its parent task ID so it
|
||
* can be disposed when the parent stops. Used as the `onSessionCreated`
|
||
* callback passed to `reviewStep`.
|
||
*/
|
||
private registerSubagentSession(taskId: string, session: AgentSession): void {
|
||
let set = this.activeSubagentSessions.get(taskId);
|
||
if (!set) {
|
||
set = new Set();
|
||
this.activeSubagentSessions.set(taskId, set);
|
||
}
|
||
set.add(session);
|
||
}
|
||
|
||
/**
|
||
* Deregister a subagent session that has finished naturally. The reviewer's
|
||
* own `finally` block disposes the session — this just removes it from the
|
||
* map.
|
||
*/
|
||
private unregisterSubagentSession(taskId: string, session: AgentSession): void {
|
||
const set = this.activeSubagentSessions.get(taskId);
|
||
if (!set) return;
|
||
set.delete(session);
|
||
if (set.size === 0) this.activeSubagentSessions.delete(taskId);
|
||
}
|
||
|
||
/**
|
||
* Dispose all subagent sessions for a task and remove them from the map.
|
||
* Called by the kill paths (move-out-of-in-progress, pause, global pause)
|
||
* so subagents stop alongside the main session.
|
||
*/
|
||
private disposeSubagentsForTask(taskId: string, reason: string): void {
|
||
const set = this.activeSubagentSessions.get(taskId);
|
||
if (!set || set.size === 0) return;
|
||
executorLog.log(`${taskId}: disposing ${set.size} subagent session(s) — ${reason}`);
|
||
for (const session of set) {
|
||
try {
|
||
session.dispose();
|
||
} catch (err) {
|
||
executorLog.warn(`${taskId}: failed to dispose subagent session: ${err}`);
|
||
}
|
||
}
|
||
this.activeSubagentSessions.delete(taskId);
|
||
}
|
||
|
||
abortAllSessionBash(): void {
|
||
for (const [taskId, { session }] of this.activeSessions) {
|
||
try {
|
||
session.abortBash();
|
||
} catch (err) {
|
||
executorLog.warn(`abortAllSessionBash: failed for task ${taskId}: ${err}`);
|
||
}
|
||
}
|
||
for (const [agentId, session] of this.childSessions) {
|
||
try {
|
||
session.abortBash();
|
||
} catch (err) {
|
||
executorLog.warn(`abortAllSessionBash: failed for child agent ${agentId}: ${err}`);
|
||
}
|
||
}
|
||
for (const [taskId, stepExecutor] of this.activeStepExecutors) {
|
||
try {
|
||
stepExecutor.abortAllSessionBash();
|
||
} catch (err) {
|
||
executorLog.warn(`abortAllSessionBash: failed for step executor ${taskId}: ${err}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @param store — Task store instance (also used to listen for events)
|
||
* @param rootDir — Project root directory
|
||
* @param options — Executor configuration
|
||
*
|
||
* Listens for `task:moved` to auto-execute tasks moved to `in-progress`,
|
||
* `task:updated` to terminate agent sessions when individual tasks are paused,
|
||
* and `settings:updated` to terminate **all** active agent sessions when
|
||
* `globalPause` transitions from `false` to `true`. `enginePaused` only
|
||
* prevents new work dispatch — running sessions continue to completion.
|
||
* Paused tasks are moved back to `todo` rather than marked as `failed`.
|
||
*/
|
||
constructor(
|
||
private store: TaskStore,
|
||
private rootDir: string,
|
||
private options: TaskExecutorOptions = {},
|
||
) {
|
||
executorLog.log(`TaskExecutor constructed (rootDir=${rootDir}, hasSemaphore=${!!options.semaphore}, hasStuckDetector=${!!options.stuckTaskDetector})`);
|
||
|
||
store.on("task:moved", ({ task, from, to }) => {
|
||
executorLog.log(`[event:task:moved] ${task.id}: ${from} → ${to}`);
|
||
if (to === "in-progress") {
|
||
this.clearWorkflowRerunWatchdog(task.id);
|
||
executorLog.log(`[event:task:moved] Initiating execute() for ${task.id}`);
|
||
void (async () => {
|
||
const taskForExecution = await this.resetMergeStateIfNeeded(task, from);
|
||
await this.execute(taskForExecution);
|
||
})().catch((err) =>
|
||
executorLog.error(`Failed to start ${task.id}:`, err),
|
||
);
|
||
} else if (from === "in-progress") {
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
// Task moved away from in-progress — terminate any active sessions
|
||
if (this.activeSessions.has(task.id)) {
|
||
executorLog.log(`${task.id} moved from in-progress to ${to} — terminating agent session`);
|
||
this.pausedAborted.add(task.id);
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
const { session } = this.activeSessions.get(task.id)!;
|
||
session.dispose();
|
||
this.activeSessions.delete(task.id);
|
||
}
|
||
if (this.activeStepExecutors.has(task.id)) {
|
||
executorLog.log(`${task.id} moved from in-progress to ${to} — terminating step sessions`);
|
||
this.pausedAborted.add(task.id);
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
const stepExecutor = this.activeStepExecutors.get(task.id)!;
|
||
stepExecutor.terminateAllSessions().catch((err) =>
|
||
executorLog.error(`Failed to terminate step sessions for ${task.id}:`, err),
|
||
);
|
||
this.activeStepExecutors.delete(task.id);
|
||
}
|
||
if (this.activeWorkflowStepSessions.has(task.id)) {
|
||
executorLog.log(`${task.id} moved from in-progress to ${to} — terminating workflow step session`);
|
||
this.pausedAborted.add(task.id);
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
const workflowSession = this.activeWorkflowStepSessions.get(task.id)!;
|
||
const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise<void> };
|
||
if (typeof sessionWithAbort.abort === "function") {
|
||
void sessionWithAbort.abort().catch((err) => {
|
||
executorLog.warn(`Failed to abort workflow step session for ${task.id}: ${err}`);
|
||
});
|
||
}
|
||
workflowSession.dispose();
|
||
this.activeWorkflowStepSessions.delete(task.id);
|
||
}
|
||
// Reviewer subagents run in their own sessions outside `activeSessions`
|
||
// and `activeStepExecutors`, so the loops above don't reach them.
|
||
// Without this, a reviewer keeps running (and emitting verdicts that
|
||
// trigger step transitions) even after the parent task was moved out
|
||
// of in-progress.
|
||
this.disposeSubagentsForTask(task.id, `parent moved from in-progress to ${to}`);
|
||
// Clean up all in-memory state for this task so nothing leaks across runs.
|
||
// This prevents zombie state from persisting when a task moves away from
|
||
// in-progress while execute() is still unwinding, or when the scheduler
|
||
// moves a task out before the executor's task:moved fires.
|
||
this.loopRecoveryState.delete(task.id);
|
||
this.spawnedAgents.delete(task.id);
|
||
this.stuckAborted.delete(task.id);
|
||
}
|
||
});
|
||
|
||
// When a task is paused while executing, terminate the agent session.
|
||
// When steering comments are added during execution, inject them into the running session.
|
||
//
|
||
// Real-time steering comment injection mechanism:
|
||
// 1. When execution starts, we initialize seenSteeringIds with all existing comment IDs
|
||
// 2. On each task:updated event, we check if there are new comments not in seenSteeringIds
|
||
// 3. New comments are injected via session.steer() which queues them for delivery
|
||
// after the current assistant turn completes (before the next LLM call)
|
||
// 4. Comments are marked as seen BEFORE injection to prevent retry loops on failure
|
||
// 5. Each injection is logged to the task for user visibility
|
||
store.on("task:updated", async (task) => {
|
||
try {
|
||
// Handle pause - terminate the agent session or step sessions
|
||
if (task.paused && this.activeSessions.has(task.id)) {
|
||
executorLog.log(`Pausing ${task.id} — terminating agent session`);
|
||
this.pausedAborted.add(task.id);
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
const { session } = this.activeSessions.get(task.id)!;
|
||
session.dispose();
|
||
// Also clean up in-memory state to prevent leaks if the task is later unpaused
|
||
this.loopRecoveryState.delete(task.id);
|
||
this.spawnedAgents.delete(task.id);
|
||
this.stuckAborted.delete(task.id);
|
||
this.disposeSubagentsForTask(task.id, "task paused");
|
||
return;
|
||
}
|
||
if (task.paused && this.activeStepExecutors.has(task.id)) {
|
||
executorLog.log(`Pausing ${task.id} — terminating step sessions`);
|
||
this.pausedAborted.add(task.id);
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
const stepExecutor = this.activeStepExecutors.get(task.id)!;
|
||
await stepExecutor.terminateAllSessions();
|
||
// Also clean up in-memory state to prevent leaks if the task is later unpaused
|
||
this.loopRecoveryState.delete(task.id);
|
||
this.spawnedAgents.delete(task.id);
|
||
this.stuckAborted.delete(task.id);
|
||
this.disposeSubagentsForTask(task.id, "task paused");
|
||
return;
|
||
}
|
||
if (task.paused && this.activeWorkflowStepSessions.has(task.id)) {
|
||
executorLog.log(`Pausing ${task.id} — terminating workflow step session`);
|
||
this.pausedAborted.add(task.id);
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
const workflowSession = this.activeWorkflowStepSessions.get(task.id)!;
|
||
const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise<void> };
|
||
if (typeof sessionWithAbort.abort === "function") {
|
||
await sessionWithAbort.abort().catch((err) =>
|
||
executorLog.warn(`Failed to abort workflow step session for pause ${task.id}: ${err}`),
|
||
);
|
||
}
|
||
workflowSession.dispose();
|
||
this.activeWorkflowStepSessions.delete(task.id);
|
||
this.loopRecoveryState.delete(task.id);
|
||
this.spawnedAgents.delete(task.id);
|
||
this.stuckAborted.delete(task.id);
|
||
this.disposeSubagentsForTask(task.id, "task paused");
|
||
return;
|
||
}
|
||
|
||
// Handle unpause of an in-progress task with no active session.
|
||
// This covers orphaned states (e.g., engine restarted while task was
|
||
// paused in-progress) where the task needs to resume execution.
|
||
// The executing/resuming guards prevent duplicate runs.
|
||
if (
|
||
!task.paused
|
||
&& task.column === "in-progress"
|
||
&& !this.activeSessions.has(task.id)
|
||
&& !this.activeStepExecutors.has(task.id)
|
||
&& !this.activeWorkflowStepSessions.has(task.id)
|
||
) {
|
||
if (
|
||
!this.executing.has(task.id)
|
||
&& !this.resumingUnpaused.has(task.id)
|
||
&& !this.recoveringCompleted.has(task.id)
|
||
) {
|
||
const pauseLabel = await this.getExecutionPauseLabel();
|
||
if (pauseLabel) {
|
||
executorLog.log(`Skipping unpause resume for ${task.id} — ${pauseLabel} active`);
|
||
return;
|
||
}
|
||
|
||
if (this.isTaskWorkComplete(task) && !task.mergeDetails) {
|
||
this.recoveringCompleted.add(task.id);
|
||
executorLog.log(`${task.id} unpaused with completed work and no session — recovering directly to in-review`);
|
||
void this.recoverCompletedTask(task)
|
||
.catch((err) =>
|
||
executorLog.error(`Failed to recover completed unpaused task ${task.id}:`, err),
|
||
)
|
||
.finally(() => {
|
||
this.recoveringCompleted.delete(task.id);
|
||
});
|
||
return;
|
||
}
|
||
|
||
this.resumingUnpaused.add(task.id);
|
||
executorLog.log(`Unpaused ${task.id} in-progress with no session — resuming execution`);
|
||
try {
|
||
await this.clearResumeFailureState(task);
|
||
await this.store.logEntry(task.id, "Resuming execution after unpause", undefined, this.currentRunContext);
|
||
await this.recoverApprovedStepsOnResume(task.id);
|
||
} catch (clearErr) {
|
||
executorLog.warn(`${task.id} clearResumeFailureState failed during unpause: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`);
|
||
}
|
||
this.execute(task)
|
||
.catch((err) =>
|
||
executorLog.error(`Failed to resume unpaused ${task.id}:`, err),
|
||
)
|
||
.finally(() => {
|
||
this.resumingUnpaused.delete(task.id);
|
||
});
|
||
}
|
||
return;
|
||
}
|
||
|
||
// Handle executor model hot-swap on active single-session executions
|
||
if (this.activeSessions.has(task.id) && !task.paused) {
|
||
const activeEntry = this.activeSessions.get(task.id)!;
|
||
const providerChanged = task.modelProvider !== activeEntry.lastModelProvider;
|
||
const modelIdChanged = task.modelId !== activeEntry.lastModelId;
|
||
|
||
if (providerChanged || modelIdChanged) {
|
||
activeEntry.lastModelProvider = task.modelProvider;
|
||
activeEntry.lastModelId = task.modelId;
|
||
|
||
const settings = await this.store.getSettings();
|
||
// Resolve model using canonical lane hierarchy for hot-swap
|
||
const { provider: newProvider, modelId: newModelId } = resolveExecutorModelPair(
|
||
task.modelProvider,
|
||
task.modelId,
|
||
settings,
|
||
);
|
||
|
||
if (newProvider && newModelId) {
|
||
try {
|
||
const model = this.modelRegistry.find(newProvider, newModelId);
|
||
if (model) {
|
||
await activeEntry.session.setModel(model);
|
||
executorLog.log(`${task.id}: executor model hot-swapped to ${newProvider}/${newModelId}`);
|
||
await this.store.logEntry(task.id, `Model changed to ${newProvider}/${newModelId}`, undefined, this.currentRunContext);
|
||
} else {
|
||
executorLog.log(`${task.id}: model ${newProvider}/${newModelId} not found in registry for hot-swap`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`${task.id}: failed to hot-swap model: ${errorMessage}`);
|
||
await this.store.logEntry(task.id, `Model change failed: ${errorMessage}`, undefined, this.currentRunContext);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Handle steering comments - inject new ones into the running session
|
||
// Only process if session is active (activeSessions check is sufficient
|
||
// since entries are only added when a task is in-progress)
|
||
if (this.activeSessions.has(task.id) && task.steeringComments) {
|
||
const activeSession = this.activeSessions.get(task.id)!;
|
||
const { session, seenSteeringIds } = activeSession;
|
||
|
||
// Find new steering comments that haven't been seen yet
|
||
const newComments = task.steeringComments.filter(c => !seenSteeringIds.has(c.id));
|
||
|
||
if (newComments.length > 0) {
|
||
for (const comment of newComments) {
|
||
const summary = comment.text.length > 80
|
||
? comment.text.slice(0, 80) + "..."
|
||
: comment.text;
|
||
|
||
// Mark as seen BEFORE attempting injection to prevent retry loops on failure
|
||
seenSteeringIds.add(comment.id);
|
||
|
||
// Format and inject the comment
|
||
const commentMessage = formatCommentForInjection(comment);
|
||
try {
|
||
executorLog.log(`Injecting comment into ${task.id}: ${summary}`);
|
||
await session.steer(commentMessage);
|
||
executorLog.log(`Successfully injected comment into ${task.id}`);
|
||
|
||
// Log to the task that comment was received
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Comment received mid-execution: ${summary}`,
|
||
`by ${comment.author}`
|
||
);
|
||
} catch (err) {
|
||
executorLog.error(`Failed to inject comment for ${task.id}:`, err);
|
||
// Comment is already marked as seen - we won't retry to avoid spamming
|
||
// the agent with failed injections. The error is logged for debugging.
|
||
}
|
||
}
|
||
|
||
// After injecting comments, check for review handoff intent
|
||
// Only detect handoff in agent-authored comments when policy is enabled
|
||
const settings = await this.store.getSettings();
|
||
if (settings.reviewHandoffPolicy === "comment-triggered") {
|
||
const agentComments = newComments.filter(c => c.author !== "user");
|
||
for (const comment of agentComments) {
|
||
if (detectReviewHandoffIntent(comment.text)) {
|
||
executorLog.log(`Review handoff detected in ${task.id}: ${comment.text.slice(0, 50)}...`);
|
||
await this.executeReviewHandoff(task, session, activeSession);
|
||
return; // Exit early - handoff handles session disposal
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (err) {
|
||
executorLog.error("Uncaught error in task:updated listener:", err);
|
||
}
|
||
});
|
||
|
||
// When globalPause transitions from false → true, terminate all active agent sessions.
|
||
store.on("settings:updated", ({ settings, previous }) => {
|
||
if (settings.globalPause && !previous.globalPause) {
|
||
// Dispose every reviewer subagent across every task. The per-task loops
|
||
// below handle main + step sessions; reviewers live in their own map
|
||
// and would otherwise outlive the global pause.
|
||
for (const taskId of [...this.activeSubagentSessions.keys()]) {
|
||
this.disposeSubagentsForTask(taskId, "global pause");
|
||
}
|
||
for (const [taskId, { session }] of this.activeSessions) {
|
||
executorLog.log(`Global pause — terminating agent session for ${taskId}`);
|
||
this.pausedAborted.add(taskId);
|
||
this.options.stuckTaskDetector?.untrackTask(taskId);
|
||
// abort() interrupts any in-flight LLM stream / tool call;
|
||
// dispose() then releases session resources.
|
||
const sessionWithAbort = session as unknown as { abort?: () => Promise<void> };
|
||
if (typeof sessionWithAbort.abort === "function") {
|
||
void sessionWithAbort.abort().catch((err) => {
|
||
executorLog.warn(`Failed to abort agent session for ${taskId}: ${err}`);
|
||
});
|
||
}
|
||
session.dispose();
|
||
// Clean up all in-memory state so nothing leaks when tasks are later unpaused
|
||
this.loopRecoveryState.delete(taskId);
|
||
this.spawnedAgents.delete(taskId);
|
||
this.stuckAborted.delete(taskId);
|
||
}
|
||
for (const [taskId, stepExecutor] of this.activeStepExecutors) {
|
||
executorLog.log(`Global pause — terminating step sessions for ${taskId}`);
|
||
this.pausedAborted.add(taskId);
|
||
this.options.stuckTaskDetector?.untrackTask(taskId);
|
||
stepExecutor.terminateAllSessions().catch(err =>
|
||
executorLog.warn(`Failed to terminate step sessions for global pause ${taskId}: ${err}`)
|
||
);
|
||
// Clean up all in-memory state so nothing leaks when tasks are later unpaused
|
||
this.loopRecoveryState.delete(taskId);
|
||
this.spawnedAgents.delete(taskId);
|
||
this.stuckAborted.delete(taskId);
|
||
}
|
||
for (const [taskId, workflowSession] of this.activeWorkflowStepSessions) {
|
||
executorLog.log(`Global pause — terminating workflow step session for ${taskId}`);
|
||
this.pausedAborted.add(taskId);
|
||
this.options.stuckTaskDetector?.untrackTask(taskId);
|
||
const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise<void> };
|
||
if (typeof sessionWithAbort.abort === "function") {
|
||
void sessionWithAbort.abort().catch((err) => {
|
||
executorLog.warn(`Failed to abort workflow step session for ${taskId}: ${err}`);
|
||
});
|
||
}
|
||
workflowSession.dispose();
|
||
this.activeWorkflowStepSessions.delete(taskId);
|
||
this.loopRecoveryState.delete(taskId);
|
||
this.spawnedAgents.delete(taskId);
|
||
this.stuckAborted.delete(taskId);
|
||
}
|
||
}
|
||
});
|
||
|
||
}
|
||
|
||
/**
|
||
* Check whether a task's work is complete — all steps are done or skipped.
|
||
* Used to detect tasks that called fn_task_done() but never transitioned to in-review
|
||
* (e.g., killed by stuck detector after fn_task_done but before moveTask).
|
||
*/
|
||
private isTaskWorkComplete(task: Task): boolean {
|
||
if (task.steps.length === 0) return false;
|
||
return task.steps.every((s) => s.status === "done" || s.status === "skipped");
|
||
}
|
||
|
||
private async resetMergeStateIfNeeded(task: Task, from: Task["column"]): Promise<Task> {
|
||
if (from !== "in-review" && from !== "done") {
|
||
return task;
|
||
}
|
||
|
||
const hasMergeEvidence = Boolean(task.mergeDetails)
|
||
|| (task.mergeRetries ?? 0) > 0
|
||
|| (task.verificationFailureCount ?? 0) > 0
|
||
|| task.status === "merging"
|
||
|| task.status === "merging-pr"
|
||
|| task.status === "merging-fix";
|
||
|
||
if (!hasMergeEvidence) {
|
||
return task;
|
||
}
|
||
|
||
return this.cleanupMergeStateForReverification(
|
||
task,
|
||
`Task returned to in-progress from ${from} column — resetting verification steps and merge state for re-verification`,
|
||
{
|
||
// Keep deterministic merge-verification bounce budget across remediation
|
||
// cycles. Status may be cleared by intermediate paths, so the counter is
|
||
// the canonical signal once a bounce has started.
|
||
preserveVerificationFailureCount: (task.verificationFailureCount ?? 0) > 0,
|
||
},
|
||
);
|
||
}
|
||
|
||
private async cleanupMergeStateForReverification(
|
||
task: Task,
|
||
logMessage: string,
|
||
options?: { preserveVerificationFailureCount?: boolean },
|
||
): Promise<Task> {
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails: null,
|
||
mergeRetries: 0,
|
||
verificationFailureCount: options?.preserveVerificationFailureCount ? task.verificationFailureCount ?? 0 : 0,
|
||
workflowStepResults: [],
|
||
});
|
||
|
||
const refreshedTask = await this.store.getTask(task.id);
|
||
const steps = refreshedTask.steps ?? [];
|
||
if (steps.length > 0) {
|
||
const allStepsComplete = this.isTaskWorkComplete(refreshedTask);
|
||
if (allStepsComplete) {
|
||
await this.reopenLastStepForRevision(task.id, refreshedTask);
|
||
} else {
|
||
const resetIndexes = new Set<number>();
|
||
for (let i = 0; i < steps.length; i++) {
|
||
const name = steps[i].name.toLowerCase();
|
||
if (/testing|verification/.test(name) || /documentation|delivery/.test(name)) {
|
||
resetIndexes.add(i);
|
||
}
|
||
}
|
||
|
||
if (resetIndexes.size === 0) {
|
||
const reopened = await this.reopenLastStepForRevision(task.id, refreshedTask);
|
||
if (reopened) {
|
||
resetIndexes.add(reopened.index);
|
||
}
|
||
} else {
|
||
for (const index of resetIndexes) {
|
||
if (steps[index].status !== "pending") {
|
||
await this.store.updateStep(task.id, index, "pending");
|
||
}
|
||
}
|
||
const earliestIndex = Math.min(...Array.from(resetIndexes));
|
||
await this.store.updateTask(task.id, { currentStep: earliestIndex });
|
||
}
|
||
}
|
||
}
|
||
|
||
await this.store.logEntry(task.id, logMessage, undefined, this.currentRunContext);
|
||
return this.store.getTask(task.id);
|
||
}
|
||
|
||
private isNoProgressNoTaskDoneFailure(task: Task): boolean {
|
||
return task.status === "failed" &&
|
||
task.error?.includes("without calling fn_task_done") === true &&
|
||
task.steps.every((step) => step.status === "pending");
|
||
}
|
||
|
||
private async clearResumeFailureState(task: Task): Promise<void> {
|
||
const updates: { status?: null; error?: null; blockedBy?: null } = {};
|
||
if (task.status === "failed" || task.error) {
|
||
updates.status = null;
|
||
updates.error = null;
|
||
}
|
||
// Pre-dispatch gating state must not survive into a resumed in-progress run.
|
||
// The scheduler sets status="queued" + blockedBy on dep/file-scope conflicts
|
||
// (scheduler.ts:618, 660) and clears them on the todo→in-progress transition
|
||
// (scheduler.ts:696). Resume paths (unpause, drift recovery, engine restart)
|
||
// bypass that clear, so a task can end up actively executing while still
|
||
// labeled "queued" in the UI.
|
||
if (task.status === "queued") {
|
||
updates.status = null;
|
||
}
|
||
if (task.blockedBy) {
|
||
updates.blockedBy = null;
|
||
}
|
||
if (Object.keys(updates).length > 0) {
|
||
await this.store.updateTask(task.id, updates);
|
||
}
|
||
}
|
||
|
||
private clearCompletedTaskWatchdog(taskId: string): void {
|
||
const handle = this.completedTaskWatchdogs.get(taskId);
|
||
if (!handle) return;
|
||
clearTimeout(handle);
|
||
this.completedTaskWatchdogs.delete(taskId);
|
||
}
|
||
|
||
private clearWorkflowRerunWatchdog(taskId: string): void {
|
||
const handle = this.workflowRerunWatchdogs.get(taskId);
|
||
if (!handle) return;
|
||
clearTimeout(handle);
|
||
this.workflowRerunWatchdogs.delete(taskId);
|
||
}
|
||
|
||
private scheduleCompletedTaskWatchdog(taskId: string, trigger: string): void {
|
||
this.clearCompletedTaskWatchdog(taskId);
|
||
|
||
const handle = setTimeout(async () => {
|
||
this.completedTaskWatchdogs.delete(taskId);
|
||
|
||
// Claim recovery slot atomically (synchronously) before any async work.
|
||
// Without this, two paths can pass the in-flight guards on the same
|
||
// event-loop turn and both call recoverCompletedTask() concurrently.
|
||
if (
|
||
this.recoveringCompleted.has(taskId)
|
||
|| this.executing.has(taskId)
|
||
|| this.activeSessions.has(taskId)
|
||
|| this.activeStepExecutors.has(taskId)
|
||
|| this.activeWorkflowStepSessions.has(taskId)
|
||
|| this.resumingUnpaused.has(taskId)
|
||
) {
|
||
return;
|
||
}
|
||
this.recoveringCompleted.add(taskId);
|
||
|
||
try {
|
||
const pauseLabel = await this.getExecutionPauseLabel();
|
||
if (pauseLabel) {
|
||
return;
|
||
}
|
||
|
||
let currentTask: Task | null = null;
|
||
try {
|
||
currentTask = await this.store.getTask(taskId);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: completed-task watchdog could not read latest task state: ${errorMessage}`);
|
||
return;
|
||
}
|
||
|
||
if (!currentTask || currentTask.column !== "in-progress" || currentTask.paused) {
|
||
return;
|
||
}
|
||
if (!this.isTaskWorkComplete(currentTask)) {
|
||
return;
|
||
}
|
||
|
||
executorLog.warn(
|
||
`${taskId}: completed-task watchdog fired after ${COMPLETED_TASK_WATCHDOG_MS / 1000}s ` +
|
||
`(${trigger}) — attempting direct recovery to in-review`,
|
||
);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Watchdog: task remained in-progress ${COMPLETED_TASK_WATCHDOG_MS / 1000}s after ${trigger} — attempting direct recovery to in-review`,
|
||
).catch(() => undefined);
|
||
|
||
const recovered = await this.recoverCompletedTask(currentTask);
|
||
if (!recovered) {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
"Watchdog recovery attempt could not finalize completed task — leaving for follow-up recovery",
|
||
).catch(() => undefined);
|
||
}
|
||
} finally {
|
||
this.recoveringCompleted.delete(taskId);
|
||
}
|
||
}, COMPLETED_TASK_WATCHDOG_MS);
|
||
|
||
this.completedTaskWatchdogs.set(taskId, handle);
|
||
}
|
||
|
||
/**
|
||
* Result of a workflow-rerun bounce attempt.
|
||
*
|
||
* - `bounced` — the move sequence completed successfully and the task is
|
||
* back in `in-progress` ready for re-execution.
|
||
* - `skipped-pending` — another bounce for the same task is mid-flight;
|
||
* this attempt is a no-op. Callers (notably the watchdog) must NOT log
|
||
* this as a successful retry, since the original bounce may itself be
|
||
* stuck.
|
||
*/
|
||
private async performWorkflowRerunBounce(
|
||
taskId: string,
|
||
worktreePath: string,
|
||
preserveResumeState: boolean = true,
|
||
): Promise<"bounced" | "skipped-pending" | "deferred-paused"> {
|
||
const pauseLabel = await this.getExecutionPauseLabel();
|
||
if (pauseLabel) {
|
||
executorLog.log(`${taskId}: workflow rerun deferred — ${pauseLabel} active`);
|
||
return "deferred-paused";
|
||
}
|
||
|
||
// Re-entry guard: if a previous bounce for the same task is still
|
||
// mid-flight (e.g., the watchdog fired before the original sequence
|
||
// completed), skip rather than racing two concurrent moveTask sequences.
|
||
if (this.workflowRerunPending.has(taskId)) {
|
||
executorLog.warn(`${taskId}: workflow rerun bounce already in flight — skipping re-entry`);
|
||
return "skipped-pending";
|
||
}
|
||
this.workflowRerunPending.add(taskId);
|
||
try {
|
||
// moveTask(in-progress → todo) clears `task.worktree`; restore it before
|
||
// the return trip so the dashboard never renders the task under
|
||
// "Unassigned" and self-healing can't reclaim the worktree as idle.
|
||
const latestTask = await this.store.getTask(taskId);
|
||
if (!latestTask) {
|
||
throw new Error("task missing during workflow rerun bounce");
|
||
}
|
||
if (latestTask.paused) {
|
||
executorLog.log(`${taskId}: workflow rerun deferred — task is paused`);
|
||
return "deferred-paused";
|
||
}
|
||
|
||
if (latestTask.column === "in-progress") {
|
||
const originalExecutionStartedAt = latestTask.executionStartedAt;
|
||
// Preserve step progress across the in-progress → todo hop:
|
||
// moveTask's default reopen-to-todo path resets every step to
|
||
// pending and rewrites PROMPT.md checkboxes, which would discard
|
||
// the partial progress this bounce is supposed to retry on top of.
|
||
if (preserveResumeState) {
|
||
await this.store.moveTask(taskId, "todo", { preserveResumeState: true });
|
||
} else {
|
||
await this.store.moveTask(taskId, "todo");
|
||
}
|
||
await this.store.updateTask(taskId, {
|
||
worktree: worktreePath,
|
||
executionStartedAt: originalExecutionStartedAt ?? null,
|
||
});
|
||
const pauseLabelAfterTodo = await this.getExecutionPauseLabel();
|
||
if (pauseLabelAfterTodo) {
|
||
executorLog.log(`${taskId}: workflow rerun parked in todo — ${pauseLabelAfterTodo} became active during bounce`);
|
||
return "deferred-paused";
|
||
}
|
||
await this.store.moveTask(taskId, "in-progress");
|
||
return "bounced";
|
||
}
|
||
|
||
if (latestTask.column === "todo") {
|
||
await this.store.updateTask(taskId, { worktree: worktreePath });
|
||
const pauseLabelBeforeResume = await this.getExecutionPauseLabel();
|
||
if (pauseLabelBeforeResume) {
|
||
executorLog.log(`${taskId}: workflow rerun parked in todo — ${pauseLabelBeforeResume} became active before resume`);
|
||
return "deferred-paused";
|
||
}
|
||
await this.store.moveTask(taskId, "in-progress");
|
||
return "bounced";
|
||
}
|
||
|
||
throw new Error(`task is in '${latestTask.column}', cannot bounce to in-progress`);
|
||
} finally {
|
||
this.workflowRerunPending.delete(taskId);
|
||
}
|
||
}
|
||
|
||
private scheduleWorkflowRerun(
|
||
taskId: string,
|
||
worktreePath: string,
|
||
successMessage: string,
|
||
preserveResumeState: boolean = true,
|
||
): void {
|
||
this.clearWorkflowRerunWatchdog(taskId);
|
||
|
||
setTimeout(async () => {
|
||
try {
|
||
const outcome = await this.performWorkflowRerunBounce(taskId, worktreePath, preserveResumeState);
|
||
if (outcome === "bounced") {
|
||
executorLog.log(successMessage);
|
||
} else if (outcome === "skipped-pending") {
|
||
executorLog.warn(`${taskId}: rerun bounce skipped — another bounce already in flight`);
|
||
} else {
|
||
executorLog.log(`${taskId}: rerun bounce deferred while pause is active`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`${taskId}: failed to schedule rerun bounce: ${errorMessage}`);
|
||
}
|
||
}, 0);
|
||
|
||
const watchdog = setTimeout(async () => {
|
||
this.workflowRerunWatchdogs.delete(taskId);
|
||
|
||
const pauseLabel = await this.getExecutionPauseLabel();
|
||
if (pauseLabel) {
|
||
executorLog.log(`${taskId}: workflow rerun watchdog skipped — ${pauseLabel} active`);
|
||
return;
|
||
}
|
||
|
||
let currentTask: Task | null = null;
|
||
try {
|
||
currentTask = await this.store.getTask(taskId);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: workflow rerun watchdog could not read latest task state: ${errorMessage}`);
|
||
return;
|
||
}
|
||
|
||
if (!currentTask || currentTask.paused || currentTask.column === "in-progress") {
|
||
return;
|
||
}
|
||
|
||
executorLog.warn(
|
||
`${taskId}: workflow rerun watchdog fired after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s ` +
|
||
`— task is still ${currentTask.column}; retrying handoff once`,
|
||
);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Watchdog: workflow rerun handoff stalled for ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s ` +
|
||
`(still ${currentTask.column}) — retrying once`,
|
||
).catch(() => undefined);
|
||
|
||
try {
|
||
const outcome = await this.performWorkflowRerunBounce(taskId, worktreePath, preserveResumeState);
|
||
if (outcome === "bounced") {
|
||
executorLog.warn(`${taskId}: workflow rerun watchdog retry succeeded`);
|
||
} else if (outcome === "skipped-pending") {
|
||
// The original bounce is still mid-flight, which means *it* is the
|
||
// one that's hung — not us. Log honestly so operators don't see a
|
||
// false "succeeded" message while the task is actually stranded.
|
||
executorLog.error(
|
||
`${taskId}: workflow rerun watchdog retry skipped — original bounce still in flight after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s; task may be stuck`,
|
||
);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Workflow rerun watchdog retry skipped — original bounce still in flight after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s; task may be stuck`,
|
||
).catch(() => undefined);
|
||
} else {
|
||
executorLog.log(`${taskId}: workflow rerun watchdog retry deferred while pause is active`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`${taskId}: workflow rerun watchdog retry failed: ${errorMessage}`);
|
||
}
|
||
}, WORKFLOW_RERUN_WATCHDOG_MS);
|
||
|
||
this.workflowRerunWatchdogs.set(taskId, watchdog);
|
||
}
|
||
|
||
private async shouldFinalizeCompletedTask(taskId: string, taskDone: boolean): Promise<boolean> {
|
||
const task = await this.store.getTask(taskId);
|
||
const completionBlocker = await this.getTaskCompletionBlocker(task);
|
||
if (completionBlocker) {
|
||
executorLog.log(`${taskId} completion blocked — ${completionBlocker}`);
|
||
return false;
|
||
}
|
||
if (taskDone) return true;
|
||
return this.isTaskWorkComplete(task);
|
||
}
|
||
|
||
private async getTaskCompletionBlocker(task: Task): Promise<string | undefined> {
|
||
return getTaskCompletionBlockerForStore(this.store, task);
|
||
}
|
||
|
||
private accumulateTokenUsage(
|
||
existing: TaskTokenUsage | undefined,
|
||
delta: Pick<TaskTokenUsage, "inputTokens" | "outputTokens" | "cachedTokens" | "totalTokens"> | undefined,
|
||
timestamp = new Date().toISOString(),
|
||
): TaskTokenUsage | undefined {
|
||
if (!delta) return existing;
|
||
|
||
const merged: TaskTokenUsage = {
|
||
inputTokens: (existing?.inputTokens ?? 0) + delta.inputTokens,
|
||
outputTokens: (existing?.outputTokens ?? 0) + delta.outputTokens,
|
||
cachedTokens: (existing?.cachedTokens ?? 0) + delta.cachedTokens,
|
||
totalTokens: (existing?.totalTokens ?? 0) + delta.totalTokens,
|
||
firstUsedAt: existing?.firstUsedAt ?? timestamp,
|
||
lastUsedAt: timestamp,
|
||
};
|
||
|
||
return merged;
|
||
}
|
||
|
||
private async extractSessionTokenUsage(
|
||
session: AgentSession | undefined,
|
||
): Promise<Pick<TaskTokenUsage, "inputTokens" | "outputTokens" | "cachedTokens" | "totalTokens"> | undefined> {
|
||
if (!session) return undefined;
|
||
|
||
try {
|
||
const statsResult = (session as AgentSession & {
|
||
getSessionStats?: () =>
|
||
| {
|
||
tokens?: {
|
||
input?: number;
|
||
output?: number;
|
||
cacheRead?: number;
|
||
cacheWrite?: number;
|
||
total?: number;
|
||
};
|
||
}
|
||
| Promise<{
|
||
tokens?: {
|
||
input?: number;
|
||
output?: number;
|
||
cacheRead?: number;
|
||
cacheWrite?: number;
|
||
total?: number;
|
||
};
|
||
}>;
|
||
}).getSessionStats?.();
|
||
const stats = await Promise.resolve(statsResult);
|
||
const tokens = stats?.tokens;
|
||
if (!tokens) return undefined;
|
||
|
||
const inputTokens = tokens.input ?? 0;
|
||
const outputTokens = tokens.output ?? 0;
|
||
const cacheReadTokens = tokens.cacheRead ?? 0;
|
||
const cacheWriteTokens = tokens.cacheWrite ?? 0;
|
||
const cachedTokens = cacheReadTokens + cacheWriteTokens;
|
||
const totalTokens = tokens.total ?? (inputTokens + outputTokens + cachedTokens);
|
||
|
||
return {
|
||
inputTokens,
|
||
outputTokens,
|
||
cachedTokens,
|
||
totalTokens,
|
||
};
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Failed to read session stats for token usage: ${message}`);
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
private async persistTokenUsage(taskId: string, session?: AgentSession): Promise<void> {
|
||
const activeSession = session ?? this.activeSessions.get(taskId)?.session;
|
||
const currentUsage = await this.extractSessionTokenUsage(activeSession);
|
||
if (!currentUsage) return;
|
||
|
||
const baseline = this.tokenUsageBaselines.get(taskId);
|
||
this.tokenUsageBaselines.set(taskId, currentUsage);
|
||
|
||
const delta = baseline
|
||
? {
|
||
inputTokens: Math.max(0, currentUsage.inputTokens - baseline.inputTokens),
|
||
outputTokens: Math.max(0, currentUsage.outputTokens - baseline.outputTokens),
|
||
cachedTokens: Math.max(0, currentUsage.cachedTokens - baseline.cachedTokens),
|
||
totalTokens: Math.max(0, currentUsage.totalTokens - baseline.totalTokens),
|
||
}
|
||
: currentUsage;
|
||
|
||
if (
|
||
delta.inputTokens === 0
|
||
&& delta.outputTokens === 0
|
||
&& delta.cachedTokens === 0
|
||
&& delta.totalTokens === 0
|
||
) {
|
||
return;
|
||
}
|
||
|
||
const task = await this.store.getTask(taskId);
|
||
const merged = this.accumulateTokenUsage(task.tokenUsage, delta);
|
||
if (!merged) return;
|
||
|
||
await this.store.updateTask(taskId, { tokenUsage: merged });
|
||
}
|
||
|
||
/**
|
||
* Execute a review handoff: move the task to in-review column with
|
||
* awaiting-user-review status, assign the requesting user, and dispose
|
||
* the agent session.
|
||
*/
|
||
private async executeReviewHandoff(
|
||
task: Task,
|
||
_session: AgentSession,
|
||
_sessionEntry: { session: AgentSession; seenSteeringIds: Set<string>; lastModelProvider?: string | null; lastModelId?: string | null },
|
||
): Promise<void> {
|
||
try {
|
||
executorLog.log(`Executing review handoff for ${task.id}`);
|
||
|
||
// Log the handoff event
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Review handoff requested by agent — moving to in-review for user review",
|
||
undefined,
|
||
this.currentRunContext
|
||
);
|
||
|
||
// Update task with awaiting-user-review status and assignee
|
||
// Use a single updateTask call for atomicity
|
||
await this.store.updateTask(
|
||
task.id,
|
||
{
|
||
status: "awaiting-user-review",
|
||
assigneeUserId: "requesting-user",
|
||
},
|
||
this.currentRunContext
|
||
);
|
||
|
||
// Move the task to in-review column (this will also emit task:moved event)
|
||
// The task:moved handler will clean up activeSessions
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
|
||
// Dispose the agent session (this may already be done by task:moved handler)
|
||
// but we do it here to be explicit
|
||
if (this.activeSessions.has(task.id)) {
|
||
const { session: activeSession } = this.activeSessions.get(task.id)!;
|
||
activeSession.dispose();
|
||
this.activeSessions.delete(task.id);
|
||
}
|
||
|
||
// Untrack from stuck detector
|
||
this.options.stuckTaskDetector?.untrackTask(task.id);
|
||
|
||
executorLog.log(`Review handoff complete for ${task.id} — task moved to in-review`);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to execute review handoff for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Fast-path a completed task directly to in-review without spawning a new agent.
|
||
* Captures modified files, runs workflow steps, and transitions the task.
|
||
*
|
||
* @returns true if the task was successfully transitioned, false otherwise.
|
||
*/
|
||
async recoverCompletedTask(task: Task): Promise<boolean> {
|
||
try {
|
||
if (
|
||
this.executing.has(task.id)
|
||
|| this.activeSessions.has(task.id)
|
||
|| this.activeStepExecutors.has(task.id)
|
||
|| this.activeWorkflowStepSessions.has(task.id)
|
||
|| this.resumingUnpaused.has(task.id)
|
||
) {
|
||
executorLog.log(`${task.id}: skipping recoverCompletedTask — task has active execution in flight`);
|
||
return false;
|
||
}
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) {
|
||
executorLog.log(
|
||
`${task.id}: skipping recoverCompletedTask — ${
|
||
settings.globalPause ? "global pause" : "engine pause"
|
||
} active`,
|
||
);
|
||
return false;
|
||
}
|
||
|
||
// Capture modified files if the worktree still exists
|
||
if (task.worktree && existsSync(task.worktree)) {
|
||
const modifiedFiles = await this.captureModifiedFiles(task.worktree, task.baseCommitSha);
|
||
if (modifiedFiles.length > 0) {
|
||
await this.store.updateTask(task.id, { modifiedFiles });
|
||
executorLog.log(`${task.id}: recovered ${modifiedFiles.length} modified files`);
|
||
}
|
||
|
||
// Run workflow steps before transitioning — skip in fast mode
|
||
if (task.executionMode !== "fast") {
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps during completed-task recovery")) {
|
||
return false;
|
||
}
|
||
const workflowResult = await this.runWorkflowSteps(task, task.worktree, settings);
|
||
if (workflowResult === "deferred-paused") {
|
||
if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
}
|
||
return false;
|
||
}
|
||
if (!workflowResult.allPassed) {
|
||
// For recovery path, treat any failure (including revision) as hard failure
|
||
// Send back to in-progress so executor can attempt to fix the issues
|
||
await this.sendTaskBackForFix(task, task.worktree!, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed during recovery", false);
|
||
return true; // Still transitioned out of in-progress
|
||
}
|
||
} else {
|
||
executorLog.log(`${task.id}: fast mode — skipping workflow steps on auto-recovery`);
|
||
}
|
||
}
|
||
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition during completed-task recovery")) {
|
||
return false;
|
||
}
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
await this.store.logEntry(task.id, "Auto-recovered: task work was complete but stuck in in-progress — moved to in-review");
|
||
executorLog.log(`✓ ${task.id} auto-recovered completed task → in-review`);
|
||
this.options.onComplete?.(task);
|
||
return true;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to recover completed task ${task.id}: ${errorMessage}`);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Auto-revive an `in-review` task whose pre-merge workflow step(s) failed, by
|
||
* replaying the same send-back-for-fix flow the executor uses during a live
|
||
* run. Invoked by SelfHealingManager's `recoverReviewTasksWithFailedPreMergeSteps`
|
||
* scan when a task is parked in review with a failed pre-merge step and no
|
||
* active session.
|
||
*
|
||
* Picks the latest failed pre-merge workflow step result (there is usually only
|
||
* one, but if several ran we want the most recent), injects its feedback into
|
||
* `PROMPT.md`, resets steps, and schedules todo → in-progress. The call site
|
||
* is responsible for enforcing the `maxPostReviewFixes` budget before invoking
|
||
* this method — this method itself does no accounting.
|
||
*
|
||
* @returns true when the task was sent back, false when no eligible failed
|
||
* step exists (caller should skip).
|
||
*/
|
||
async recoverFailedPreMergeWorkflowStep(task: Task): Promise<boolean> {
|
||
try {
|
||
const failed = (task.workflowStepResults ?? [])
|
||
.filter((r) => (r.phase || "pre-merge") === "pre-merge" && r.status === "failed")
|
||
.sort((a, b) => {
|
||
const aTs = Date.parse(a.completedAt || a.startedAt || "");
|
||
const bTs = Date.parse(b.completedAt || b.startedAt || "");
|
||
return (Number.isFinite(bTs) ? bTs : 0) - (Number.isFinite(aTs) ? aTs : 0);
|
||
});
|
||
const target = failed[0];
|
||
if (!target) {
|
||
executorLog.warn(`${task.id}: no failed pre-merge workflow step to recover from`);
|
||
return false;
|
||
}
|
||
|
||
const feedback = target.output?.trim() || "(no feedback captured)";
|
||
const stepName = target.workflowStepName || target.workflowStepId || "Unknown";
|
||
|
||
await this.sendTaskBackForFix(
|
||
task,
|
||
task.worktree ?? "",
|
||
feedback,
|
||
stepName,
|
||
`Auto-revived from in-review: pre-merge workflow step "${stepName}" had failed`,
|
||
);
|
||
return true;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to recover failed pre-merge workflow step for ${task.id}: ${errorMessage}`);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Resume orphaned in-progress tasks (e.g., after crash/restart).
|
||
* Call once after engine startup.
|
||
*
|
||
* Tasks that are already complete (all steps done/skipped) are fast-pathed
|
||
* directly to in-review without spawning a new agent session.
|
||
*/
|
||
async resumeOrphaned(): Promise<void> {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) {
|
||
executorLog.log(
|
||
`resumeOrphaned skipped — ${
|
||
settings.globalPause ? "global pause" : "engine pause"
|
||
} is active`,
|
||
);
|
||
return;
|
||
}
|
||
|
||
const tasks = await this.store.listTasks({ slim: true, column: "in-progress" });
|
||
const inProgress = tasks.filter(
|
||
(t) => t.column === "in-progress" && !this.executing.has(t.id) && !t.paused,
|
||
);
|
||
|
||
if (inProgress.length === 0) return;
|
||
|
||
executorLog.log(`Found ${inProgress.length} orphaned in-progress task(s)`);
|
||
for (const task of inProgress) {
|
||
// Fast-path: if the task already completed its work (all steps done),
|
||
// move it directly to in-review instead of re-executing from scratch.
|
||
if (this.isTaskWorkComplete(task) && !task.mergeDetails) {
|
||
if (this.recoveringCompleted.has(task.id)) {
|
||
executorLog.log(`${task.id} completed-task recovery already running - skipping duplicate startup recovery`);
|
||
continue;
|
||
}
|
||
executorLog.log(`${task.id} is already complete — fast-pathing to in-review`);
|
||
this.recoveringCompleted.add(task.id);
|
||
void this.recoverCompletedTask(task)
|
||
.catch((err) =>
|
||
executorLog.error(`Failed to recover completed orphan ${task.id}:`, err),
|
||
)
|
||
.finally(() => {
|
||
this.recoveringCompleted.delete(task.id);
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (this.isNoProgressNoTaskDoneFailure(task)) {
|
||
executorLog.log(`${task.id} failed without fn_task_done and has no step progress — leaving for self-healing requeue`);
|
||
continue;
|
||
}
|
||
|
||
executorLog.log(`Resuming ${task.id}: ${task.title || task.description.slice(0, 60)}`);
|
||
try {
|
||
await this.clearResumeFailureState(task);
|
||
await this.store.logEntry(task.id, "Resumed after engine restart");
|
||
await this.recoverApprovedStepsOnResume(task.id);
|
||
} catch (err) {
|
||
executorLog.error(`Failed to write resume log for ${task.id}:`, err);
|
||
}
|
||
this.execute(task).catch((err) =>
|
||
executorLog.error(`Failed to resume ${task.id}:`, err),
|
||
);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Execute a task in an isolated git worktree.
|
||
*
|
||
* Worktree acquisition flow:
|
||
* 1. If the worktree already exists on disk (resume after crash), reuse it.
|
||
* 2. If a {@link WorktreePool} is provided and `recycleWorktrees` is enabled,
|
||
* attempt to acquire a warm worktree from the pool. Pooled worktrees skip
|
||
* the `worktreeInitCommand` since their build caches are already warm.
|
||
* 3. Otherwise, create a fresh worktree via `git worktree add` and run the
|
||
* `worktreeInitCommand` if configured.
|
||
*/
|
||
|
||
/**
|
||
* Resolve custom instructions for a given agent role by looking up agents
|
||
* in the AgentStore that have instructions configured.
|
||
* Returns an empty string if no instructions are found.
|
||
*/
|
||
private async resolveInstructionsForRole(role: string): Promise<string> {
|
||
if (!this.options.agentStore) return "";
|
||
try {
|
||
const agents = await this.options.agentStore.listAgents({ role: role as AgentCapability });
|
||
for (const agent of agents) {
|
||
if (agent.instructionsText || agent.instructionsPath) {
|
||
try {
|
||
const ratingSummary = await this.options.agentStore.getRatingSummary(agent.id);
|
||
return await resolveAgentInstructions(agent, this.rootDir, ratingSummary);
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${agent.id}: failed to load rating summary for instruction resolution, falling back to default instructions: ${msg}`);
|
||
return await resolveAgentInstructions(agent, this.rootDir);
|
||
}
|
||
}
|
||
}
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Failed to resolve instructions for role '${role}', continuing without custom instructions: ${msg}`);
|
||
}
|
||
return "";
|
||
}
|
||
|
||
private resolveDependencyWorktree(task: Task, allTasks: Task[]): string | null {
|
||
if (task.dependencies.length === 0) return null;
|
||
|
||
for (const depId of task.dependencies) {
|
||
const dep = allTasks.find((t) => t.id === depId);
|
||
if (
|
||
dep &&
|
||
dep.worktree &&
|
||
(dep.column === "done" || dep.column === "in-review") &&
|
||
existsSync(dep.worktree)
|
||
) {
|
||
return dep.worktree;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* Reuse an existing worktree directory from a dependency task.
|
||
* Instead of creating a new worktree with `git worktree add`, this creates
|
||
* a new branch in the existing worktree via `git checkout -b`. The worktree
|
||
* directory (and its build caches) are preserved.
|
||
*/
|
||
private async reuseWorktree(branch: string, worktreePath: string): Promise<void> {
|
||
await execAsync(`git checkout -b "${branch}"`, {
|
||
cwd: worktreePath,
|
||
});
|
||
executorLog.log(`Reused worktree at ${worktreePath}, created branch ${branch}`);
|
||
}
|
||
|
||
/**
|
||
* Execute a task in an isolated git worktree.
|
||
*
|
||
* **Worktree assignment:** New worktrees get humanized random names
|
||
* (e.g., `.worktrees/swift-falcon/`) via `generateWorktreeName()` rather
|
||
* than being named after the task ID. This decouples directory names from
|
||
* tasks, enabling worktree reuse across dependency chains. When resuming
|
||
* a task that already has `task.worktree` set, the existing path is used
|
||
* as-is. Branches remain task-scoped (`fusion/{task-id}`).
|
||
*/
|
||
async execute(task: Task): Promise<void> {
|
||
executorLog.log(`execute() called for ${task.id} (already executing=${this.executing.has(task.id)})`);
|
||
if (this.executing.has(task.id)) return;
|
||
this.executing.add(task.id);
|
||
|
||
executorLog.log(`Starting ${task.id}: ${task.title || task.description.slice(0, 60)}`);
|
||
|
||
// Fetch settings early — needed for worktree naming and later configuration
|
||
const settings = await this.store.getSettings();
|
||
|
||
// Read execution mode to determine whether to skip review and workflow steps
|
||
const executionMode = task.executionMode ?? "standard";
|
||
|
||
// Construct run context for mutation correlation
|
||
// Use a synthetic correlation ID: task ID + timestamp + random suffix
|
||
const syntheticRunId = generateSyntheticRunId("exec", task.id);
|
||
this.currentRunContext = {
|
||
runId: syntheticRunId,
|
||
agentId: task.assignedAgentId ?? "executor",
|
||
};
|
||
|
||
// Build engine run context for audit instrumentation (FN-1404)
|
||
const engineRunContext: EngineRunContext = {
|
||
runId: syntheticRunId,
|
||
agentId: task.assignedAgentId ?? "executor",
|
||
taskId: task.id,
|
||
phase: "execute",
|
||
};
|
||
|
||
// Create run auditor for TaskStore-backed audit emission (no-ops if store doesn't support it)
|
||
const audit = createRunAuditor(this.store, engineRunContext);
|
||
|
||
// Stale spec enforcement: check if PROMPT.md has aged beyond the configured threshold.
|
||
// When enabled, stale tasks are moved back to triage with status "needs-replan"
|
||
// so they receive fresh specification before execution. This guard runs early in
|
||
// execute() to prevent stale tasks from entering worktree creation or agent sessions.
|
||
// If timestamp evaluation is skipped (missing/unreadable file), continue with execution
|
||
// so existing filesystem validation paths remain authoritative.
|
||
// Skip for tasks that are already in-progress, in-review, merging, or done —
|
||
// these should not be interrupted and sent back to triage for re-planning.
|
||
const activeColumns = new Set(["in-progress", "in-review", "done"]);
|
||
const activeMergeStatuses = new Set(["merging", "merging-pr", "merging-fix"]);
|
||
const isActiveTask = activeColumns.has(task.column) || activeMergeStatuses.has(task.status ?? "");
|
||
if (!isActiveTask) {
|
||
const tasksDir = join(this.store.getFusionDir(), "tasks");
|
||
const promptPath = getPromptPath(tasksDir, task.id);
|
||
const staleness = await evaluateSpecStaleness({ settings, promptPath });
|
||
if (staleness.isStale) {
|
||
executorLog.warn(`Task ${task.id} specification is stale — ${staleness.reason}`);
|
||
// Move to triage first, then set status so the task enters triage with needs-replan
|
||
await this.store.moveTask(task.id, "triage");
|
||
await this.store.updateTask(task.id, { status: "needs-replan" });
|
||
await this.store.logEntry(task.id, staleness.reason, undefined, this.currentRunContext);
|
||
return;
|
||
}
|
||
}
|
||
|
||
// Drift detection: a task that is already in-progress (i.e. we're not
|
||
// dispatching it fresh from todo) should always carry a `worktree`. If it
|
||
// doesn't, some prior update — most likely a partial pause/abort sequence
|
||
// where updateTask({ worktree: null }) succeeded but the subsequent
|
||
// moveTask()/status write failed — left the row in a half-state. The
|
||
// executor can still recover by falling through to the fresh-worktree
|
||
// path below, but we emit a loud audit record so these states stop being
|
||
// silent.
|
||
if (task.column === "in-progress" && task.mergeDetails) {
|
||
executorLog.warn(`${task.id}: stale mergeDetails found while executing in-progress task — resetting merge state before continuing`);
|
||
task = await this.cleanupMergeStateForReverification(
|
||
task,
|
||
"Executor detected stale merge state while task was in-progress — reset verification steps and merge metadata before resuming",
|
||
);
|
||
}
|
||
|
||
if (task.column === "in-progress" && !task.worktree) {
|
||
executorLog.error(
|
||
`${task.id}: drift detected — task is in-progress with no worktree. ` +
|
||
`Recovering by creating a fresh worktree. This usually indicates a partial ` +
|
||
`updateTask/moveTask sequence failed somewhere upstream.`,
|
||
);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Drift detected: in-progress with no worktree — creating fresh worktree to recover",
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
}
|
||
|
||
// Hoist worktreePath so it's accessible in the catch block for dep-abort cleanup
|
||
// Determine worktree name based on settings
|
||
let worktreePath: string;
|
||
if (task.worktree) {
|
||
worktreePath = task.worktree;
|
||
} else {
|
||
const naming = settings.worktreeNaming || "random";
|
||
let worktreeName: string;
|
||
|
||
switch (naming) {
|
||
case "task-id":
|
||
worktreeName = task.id.toLowerCase();
|
||
break;
|
||
case "task-title":
|
||
worktreeName = slugify(task.title || task.description.slice(0, 60));
|
||
break;
|
||
case "random":
|
||
default:
|
||
worktreeName = generateWorktreeName(this.rootDir);
|
||
break;
|
||
}
|
||
worktreePath = join(this.rootDir, ".worktrees", worktreeName);
|
||
}
|
||
|
||
// Set by stuck-abort handlers; the actual moveTask("todo") is deferred to
|
||
// the finally block so this.executing is cleared first (prevents re-dispatch race).
|
||
// true = requeue to todo, false = budget exhausted (already marked failed).
|
||
let stuckRequeue: boolean | null = null;
|
||
let taskDone = false;
|
||
|
||
try {
|
||
// Check dependencies
|
||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const unmetDeps = task.dependencies.filter((depId) => {
|
||
const dep = allTasks.find((t) => t.id === depId);
|
||
return dep && dep.column !== "done" && dep.column !== "in-review" && dep.column !== "archived";
|
||
});
|
||
|
||
if (unmetDeps.length > 0) {
|
||
executorLog.log(`${task.id} blocked by: ${unmetDeps.join(", ")} — deferring`);
|
||
return;
|
||
}
|
||
|
||
if (!await isGitRepository(this.rootDir)) {
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Cannot execute task: project directory is not a Git repository. Fusion requires a Git repository for worktree-based task execution.",
|
||
);
|
||
throw new Error(
|
||
"Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.",
|
||
);
|
||
}
|
||
|
||
// Create or reuse worktree — try pool first when recycling is enabled
|
||
// Prefer the persisted branch from a prior run so the agent resumes on the
|
||
// same branch instead of creating a fresh fusion/fn-XXXX-2, -3, etc.
|
||
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
||
// Use generateWorktreeName for human-friendly directory names (adjective-noun pattern)
|
||
// instead of task.id, so worktrees are named like ".worktrees/swift-falcon"
|
||
let isResume = existsSync(worktreePath);
|
||
let acquiredFromPool = false;
|
||
|
||
// Resolve the base branch — set by the scheduler when a dep is in-review
|
||
const baseBranch = task.baseBranch || null;
|
||
|
||
if (task.worktree && isResume && !await isUsableTaskWorktree(this.rootDir, worktreePath)) {
|
||
const invalidWorktreePath = worktreePath;
|
||
executorLog.log(`${task.id}: assigned worktree is not usable; creating a fresh worktree instead: ${invalidWorktreePath}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Assigned worktree is not a registered, usable git worktree; creating a fresh worktree instead`,
|
||
invalidWorktreePath,
|
||
this.currentRunContext,
|
||
);
|
||
await this.store.updateTask(task.id, { worktree: null, branch: null });
|
||
worktreePath = join(this.rootDir, ".worktrees", generateWorktreeName(this.rootDir));
|
||
isResume = existsSync(worktreePath);
|
||
}
|
||
|
||
if (!isResume) {
|
||
|
||
// Try acquiring a warm worktree from the pool
|
||
if (this.options.pool && settings.recycleWorktrees) {
|
||
const pooled = this.options.pool.acquire();
|
||
if (pooled) {
|
||
try {
|
||
const actualBranch = await this.options.pool.prepareForTask(pooled, branchName, baseBranch ?? undefined);
|
||
worktreePath = pooled;
|
||
acquiredFromPool = true;
|
||
executorLog.log(`Acquired worktree from pool: ${pooled}`);
|
||
await this.store.updateTask(task.id, { worktree: worktreePath, branch: actualBranch });
|
||
// Audit trail: record worktree reuse (FN-1404)
|
||
await audit.git({ type: "worktree:reuse", target: worktreePath, metadata: { branch: actualBranch } });
|
||
if (actualBranch !== branchName) {
|
||
executorLog.log(`Branch conflict resolved: using ${actualBranch} instead of ${branchName}`);
|
||
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath} (branch conflict: using ${actualBranch})`, undefined, this.currentRunContext);
|
||
} else {
|
||
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath}`, undefined, this.currentRunContext);
|
||
}
|
||
} catch (poolErr: unknown) {
|
||
// Pool preparation failed — release the worktree back and fall through
|
||
// to fresh worktree creation
|
||
const poolErrMessage = poolErr instanceof Error ? poolErr.message : String(poolErr);
|
||
this.options.pool.release(pooled);
|
||
executorLog.log(`Pool prepareForTask failed, falling through to fresh worktree: ${poolErrMessage}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Pool worktree preparation failed (${poolErrMessage}), creating fresh worktree`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Fall through to fresh worktree creation if pool had nothing
|
||
if (!acquiredFromPool) {
|
||
const created = await this.createWorktree(branchName, worktreePath, task.id, baseBranch ?? undefined);
|
||
worktreePath = created.path;
|
||
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
||
// Audit trail: record worktree creation and branch creation (FN-1404)
|
||
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
||
await audit.git({ type: "branch:create", target: created.branch });
|
||
if (created.branch !== branchName) {
|
||
executorLog.log(`Branch conflict resolved: using ${created.branch} instead of ${branchName}`);
|
||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (branch conflict: using ${created.branch})`, undefined, this.currentRunContext);
|
||
} else if (baseBranch) {
|
||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (based on ${baseBranch})`, undefined, this.currentRunContext);
|
||
} else {
|
||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath}`, undefined, this.currentRunContext);
|
||
}
|
||
|
||
// Run worktree init command for fresh worktrees (skip for pooled — caches are warm).
|
||
// The init command should deterministically install the full dependency
|
||
// graph required by test/typecheck/build commands (for pnpm workspaces,
|
||
// prefer `pnpm install --frozen-lockfile`) so transitive modules and
|
||
// declarations like @vitest/runner, loupe, debug, @types/express, and
|
||
// node-pty are present after bootstrap.
|
||
//
|
||
// NOTE: This is distinct from the separate workspace-export failure
|
||
// class where internal packages fail to resolve because exports point
|
||
// to missing dist/* outputs (e.g. "@fusion/core entry not found").
|
||
// 5-minute timeout accommodates larger dependency installs.
|
||
if (settings.worktreeInitCommand) {
|
||
const initStartedAt = Date.now();
|
||
try {
|
||
const initResult = await runConfiguredCommand(settings.worktreeInitCommand, worktreePath, 300_000);
|
||
if (initResult.spawnError || initResult.timedOut || initResult.exitCode !== 0) {
|
||
throw new Error(configuredCommandErrorMessage(initResult));
|
||
}
|
||
await this.store.logEntry(task.id, `[timing] Worktree init command completed in ${Date.now() - initStartedAt}ms`, settings.worktreeInitCommand, this.currentRunContext);
|
||
} catch (err: unknown) {
|
||
await this.store.logEntry(task.id, `[timing] Worktree init command failed after ${Date.now() - initStartedAt}ms`, undefined, this.currentRunContext);
|
||
const execError = err instanceof Error ? err : new Error(String(err));
|
||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
||
? String((execError as Record<string, unknown>).stderr)
|
||
: execError.message;
|
||
executorLog.error(`${task.id}: worktree init command failed — first test run will likely fail: ${message}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Worktree init command failed (first test run will likely fail): ${message}`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
}
|
||
}
|
||
|
||
// Run setup script for fresh worktrees (after worktreeInitCommand)
|
||
if (settings.setupScript) {
|
||
const scriptCommand = settings.scripts?.[settings.setupScript];
|
||
if (scriptCommand) {
|
||
const setupStartedAt = Date.now();
|
||
try {
|
||
const setupResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000);
|
||
if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) {
|
||
throw new Error(configuredCommandErrorMessage(setupResult));
|
||
}
|
||
await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.currentRunContext);
|
||
} catch (err: unknown) {
|
||
const execError = err instanceof Error ? err : new Error(String(err));
|
||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
||
? String((execError as Record<string, unknown>).stderr)
|
||
: execError.message;
|
||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.currentRunContext);
|
||
}
|
||
} else {
|
||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.currentRunContext);
|
||
}
|
||
}
|
||
}
|
||
} else if (task.worktree) {
|
||
// Task already had a worktree assigned and it exists on disk — reuse it
|
||
executorLog.log(`Reusing existing worktree: ${worktreePath}`);
|
||
} else {
|
||
// Directory exists at generated path but task has no worktree — create via normal flow
|
||
const created = await this.createWorktree(branchName, worktreePath, task.id);
|
||
worktreePath = created.path;
|
||
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
||
// Audit trail: record worktree creation and branch creation (FN-1404)
|
||
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
||
await audit.git({ type: "branch:create", target: created.branch });
|
||
}
|
||
|
||
// Capture the base commit SHA for diff computation whenever a task
|
||
// starts with a newly assigned worktree. Recycled worktrees must
|
||
// overwrite any prior task baseline instead of inheriting it.
|
||
if (!isResume) {
|
||
try {
|
||
const { stdout } = await execAsync("git rev-parse HEAD", {
|
||
cwd: worktreePath,
|
||
encoding: "utf-8",
|
||
});
|
||
const baseCommitSha = stdout.trim();
|
||
await this.store.updateTask(task.id, { baseCommitSha });
|
||
executorLog.log(`${task.id}: captured baseCommitSha ${baseCommitSha.slice(0, 7)}`);
|
||
// Audit trail: record base commit capture for later diff computation (FN-1404)
|
||
await audit.git({ type: "commit:create", target: baseCommitSha, metadata: { purpose: "base" } });
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.log(`Failed to capture baseCommitSha for ${task.id}: ${errorMessage}`);
|
||
// Non-fatal: task can continue without baseCommitSha
|
||
}
|
||
}
|
||
|
||
this.activeWorktrees.set(task.id, worktreePath);
|
||
executorLog.log(`${task.id}: worktree ready at ${worktreePath}`);
|
||
|
||
this.options.onStart?.(task, worktreePath);
|
||
|
||
const detail = await this.store.getTask(task.id);
|
||
executorLog.log(`${task.id}: fetched task detail (${detail.steps.length} steps, prompt length=${detail.prompt?.length ?? 0})`);
|
||
|
||
// Initialize steps from PROMPT.md if empty
|
||
if (detail.steps.length === 0) {
|
||
const steps = await this.store.parseStepsFromPrompt(task.id);
|
||
if (steps.length > 0) {
|
||
await this.store.updateStep(task.id, 0, "pending");
|
||
}
|
||
}
|
||
|
||
// On resume (task.branch already set from a prior run), reconcile step
|
||
// statuses from git history so the agent doesn't redo already-committed work.
|
||
if (isResume && task.branch && detail.steps.length > 0) {
|
||
await this.reconcileStepsFromGitHistory(task.id, detail, worktreePath);
|
||
}
|
||
|
||
// ── Step-Session vs Single-Session execution path ──
|
||
// When runStepsInNewSessions is enabled, each step runs in its own
|
||
// fresh agent session via StepSessionExecutor. Otherwise, the existing
|
||
// single-session flow runs all steps in one monolithic session.
|
||
|
||
// Build skill selection context early so it's available in both paths
|
||
const skillContext = await buildSessionSkillContext({
|
||
agentStore: this.options.agentStore!,
|
||
task: detail,
|
||
sessionPurpose: "executor",
|
||
projectRootDir: this.rootDir,
|
||
pluginRunner: this.options.pluginRunner,
|
||
});
|
||
|
||
if (settings.runStepsInNewSessions) {
|
||
// ── Step-Session Path ──────────────────────────────────────────
|
||
executorLog.log(`${task.id}: using step-session mode (maxParallel=${settings.maxParallelSteps ?? 2})`);
|
||
|
||
const stepSessionAgent = detail.assignedAgentId && this.options.agentStore
|
||
? await this.options.agentStore.getAgent(detail.assignedAgentId).catch(() => null)
|
||
: null;
|
||
const stepSessionRuntimeHint = extractRuntimeHint(stepSessionAgent?.runtimeConfig);
|
||
|
||
let accumulatedStepTokenUsage = detail.tokenUsage;
|
||
const tokenUsageRecordedSteps = new Set<number>();
|
||
|
||
const stepExecutor = new StepSessionExecutor({
|
||
store: this.store,
|
||
taskDetail: detail,
|
||
worktreePath,
|
||
rootDir: this.rootDir,
|
||
settings,
|
||
semaphore: this.options.semaphore,
|
||
stuckTaskDetector: this.options.stuckTaskDetector,
|
||
pluginRunner: this.options.pluginRunner,
|
||
runtimeHint: stepSessionRuntimeHint,
|
||
// Pass skill selection context from the main executor session
|
||
skillSelection: skillContext.skillSelectionContext,
|
||
// Pass agentStore and messageStore for delegation and messaging tools
|
||
agentStore: this.options.agentStore,
|
||
messageStore: this.options.messageStore,
|
||
onStepStart: (stepIndex) => {
|
||
this.options.stuckTaskDetector?.recordProgress(task.id);
|
||
try {
|
||
this.store.updateStep(task.id, stepIndex, "in-progress").catch((err) => {
|
||
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status to in-progress: ${err}`);
|
||
});
|
||
} catch (err) {
|
||
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status to in-progress: ${err}`);
|
||
}
|
||
},
|
||
onStepComplete: (stepIndex, result) => {
|
||
executorLog.log(`${task.id}: step ${stepIndex} ${result.success ? "succeeded" : "failed"} (${result.retries} retries)`);
|
||
try {
|
||
this.store.updateStep(task.id, stepIndex, result.success ? "done" : "skipped").catch((err) => {
|
||
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status: ${err}`);
|
||
});
|
||
} catch (err) {
|
||
executorLog.warn(`${task.id}: failed to update step ${stepIndex} status: ${err}`);
|
||
}
|
||
|
||
if (!result.tokenUsage) {
|
||
return;
|
||
}
|
||
|
||
accumulatedStepTokenUsage = this.accumulateTokenUsage(accumulatedStepTokenUsage, result.tokenUsage);
|
||
tokenUsageRecordedSteps.add(stepIndex);
|
||
if (!accumulatedStepTokenUsage) {
|
||
return;
|
||
}
|
||
|
||
this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage }).catch((err) => {
|
||
executorLog.warn(`${task.id}: failed to persist token usage on step ${stepIndex} complete: ${err}`);
|
||
});
|
||
},
|
||
});
|
||
this.activeStepExecutors.set(task.id, stepExecutor);
|
||
|
||
const stepWork = async () => {
|
||
const results = await stepExecutor.executeAll();
|
||
|
||
// Check abort conditions after execution completes
|
||
if (this.depAborted.has(task.id)) {
|
||
this.depAborted.delete(task.id);
|
||
await this.handleDepAbortCleanup(task.id, worktreePath);
|
||
return;
|
||
}
|
||
if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
await this.store.logEntry(task.id, "Execution paused — step sessions terminated, moved to todo", undefined, this.currentRunContext);
|
||
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
|
||
return;
|
||
}
|
||
if (this.stuckAborted.has(task.id)) {
|
||
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
|
||
this.stuckAborted.delete(task.id);
|
||
return;
|
||
}
|
||
|
||
for (const result of results) {
|
||
if (!result.tokenUsage || tokenUsageRecordedSteps.has(result.stepIndex)) {
|
||
continue;
|
||
}
|
||
accumulatedStepTokenUsage = this.accumulateTokenUsage(accumulatedStepTokenUsage, result.tokenUsage);
|
||
}
|
||
|
||
if (accumulatedStepTokenUsage) {
|
||
await this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage });
|
||
}
|
||
|
||
const allSuccess = results.every(r => r.success);
|
||
if (allSuccess) {
|
||
const updatedTask = await this.store.getTask(task.id);
|
||
const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha);
|
||
if (modifiedFiles.length > 0) {
|
||
await this.store.updateTask(task.id, { modifiedFiles });
|
||
executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`);
|
||
// Audit trail: record filesystem mutation (FN-1404)
|
||
await audit.filesystem({ type: "file:capture-modified", target: task.id, metadata: { files: modifiedFiles } });
|
||
}
|
||
|
||
this.scheduleCompletedTaskWatchdog(task.id, "step-session completion");
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after step-session completion")) {
|
||
return;
|
||
}
|
||
|
||
// ── Deterministic verification gate (FN-3345) ──────────
|
||
// Run testCommand/buildCommand after all steps succeed but BEFORE
|
||
// workflow steps and the in-review transition. Skipped in fast mode
|
||
// and when no verification commands are configured.
|
||
if (executionMode !== "fast") {
|
||
if (settings.testCommand?.trim() || settings.buildCommand?.trim()) {
|
||
const verificationResult = await this.runExecutorDeterministicVerification(task, worktreePath, settings);
|
||
|
||
if (!verificationResult.allPassed) {
|
||
const failedType = verificationResult.failedCommand === "testCommand" ? "test" : "build";
|
||
const failedResult = failedType === "test" ? verificationResult.testResult! : verificationResult.buildResult!;
|
||
const failedCommand = failedResult.command;
|
||
const failureOutput = failedResult.stderr || failedResult.stdout || "Unknown error";
|
||
const summary = summarizeVerificationOutput(failureOutput, failedType);
|
||
|
||
executorLog.log(`${task.id}: [verification] ${failedType} failed — attempting fix agent`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[verification] ${failedType} command failed (exit ${failedResult.exitCode}). Attempting fix agent...`,
|
||
summary,
|
||
this.currentRunContext,
|
||
);
|
||
|
||
const maxFixRetries = Math.min(settings.verificationFixRetries ?? 3, 3);
|
||
|
||
if (maxFixRetries === 0) {
|
||
executorLog.log(`${task.id}: [verification] fix retries set to 0 — sending task back immediately`);
|
||
await this.sendTaskBackForFix(
|
||
task, worktreePath,
|
||
`${failedType} command \`${failedCommand}\` failed (exit ${failedResult.exitCode}):\n${summary}`,
|
||
`Verification (${failedType})`,
|
||
`Deterministic verification failed (${failedType})`,
|
||
true,
|
||
true,
|
||
);
|
||
return;
|
||
}
|
||
|
||
let fixSucceeded = false;
|
||
for (let attempt = 1; attempt <= maxFixRetries; attempt++) {
|
||
const fixed = await this.attemptExecutorVerificationFix(
|
||
task, worktreePath,
|
||
{
|
||
command: failedCommand,
|
||
exitCode: failedResult.exitCode,
|
||
output: failureOutput,
|
||
type: failedType,
|
||
},
|
||
settings,
|
||
attempt,
|
||
maxFixRetries,
|
||
);
|
||
if (fixed) {
|
||
fixSucceeded = true;
|
||
executorLog.log(`${task.id}: [verification] fix agent succeeded on attempt ${attempt}/${maxFixRetries}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[verification] Fix agent succeeded on attempt ${attempt}/${maxFixRetries}. Verification now passing.`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
break;
|
||
}
|
||
executorLog.log(`${task.id}: [verification] fix agent attempt ${attempt}/${maxFixRetries} failed`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[verification] Fix agent attempt ${attempt}/${maxFixRetries} failed`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
}
|
||
|
||
if (!fixSucceeded) {
|
||
executorLog.log(`${task.id}: [verification] all fix attempts exhausted (${maxFixRetries}/${maxFixRetries}) — sending task back`);
|
||
await this.sendTaskBackForFix(
|
||
task, worktreePath,
|
||
`${failedType} command \`${failedCommand}\` failed (exit ${failedResult.exitCode}) after ${maxFixRetries} fix attempts:\n${summary}`,
|
||
`Verification (${failedType})`,
|
||
`Deterministic verification failed after ${maxFixRetries} fix attempts`,
|
||
true,
|
||
true,
|
||
);
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Run workflow steps before moving to in-review — skip in fast mode
|
||
if (executionMode !== "fast") {
|
||
const workflowResult = await this.runWorkflowSteps(task, worktreePath, settings);
|
||
if (workflowResult === "deferred-paused") {
|
||
if (await this.parkTaskAfterWorkflowStepPause(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
return;
|
||
}
|
||
if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
}
|
||
return;
|
||
}
|
||
if (!workflowResult.allPassed) {
|
||
// Check if revision was requested
|
||
if (workflowResult.revisionRequested) {
|
||
await this.handleWorkflowRevisionRequest(task, worktreePath, workflowResult.feedback, workflowResult.stepName);
|
||
return;
|
||
}
|
||
// Try to fix workflow step failures with retries
|
||
const retried = await this.handleWorkflowStepFailure(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown");
|
||
if (retried) {
|
||
return; // Retry scheduled
|
||
}
|
||
// Retries exhausted - send back to in-progress for remediation
|
||
await this.sendTaskBackForFix(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed");
|
||
return;
|
||
}
|
||
} else {
|
||
executorLog.log(`${task.id}: fast mode — skipping pre-merge workflow steps`);
|
||
await this.store.logEntry(task.id, "Fast mode — pre-merge workflow steps skipped", undefined, this.currentRunContext);
|
||
}
|
||
|
||
// Reset retry counters on success
|
||
await this.store.updateTask(task.id, { workflowStepRetries: undefined, taskDoneRetryCount: null });
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after step-session completion")) {
|
||
return;
|
||
}
|
||
|
||
await this.store.moveTask(task.id, "in-review");
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
// Audit trail: record task move (FN-1404)
|
||
await audit.database({ type: "task:move", target: task.id, metadata: { to: "in-review" } });
|
||
executorLog.log(`✓ ${task.id} completed (step-session) → in-review`);
|
||
this.options.onComplete?.(task);
|
||
} else {
|
||
const failedSteps = results.filter(r => !r.success);
|
||
const errorSummary = failedSteps.map(r => `Step ${r.stepIndex}: ${r.error || "unknown error"}`).join("; ");
|
||
await this.store.updateTask(task.id, { status: "failed", error: errorSummary });
|
||
await this.store.moveTask(task.id, "in-review");
|
||
executorLog.log(`✗ ${task.id} step-session failed → in-review: ${errorSummary}`);
|
||
this.options.onError?.(task, new Error(errorSummary));
|
||
}
|
||
};
|
||
|
||
const retryableStepWork = () => withRateLimitRetry(stepWork, {
|
||
onRetry: (attempt, delayMs, error) => {
|
||
const delaySec = Math.round(delayMs / 1000);
|
||
executorLog.warn(`⏳ ${task.id} rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`);
|
||
this.store.logEntry(task.id, `Rate limited — retry ${attempt} in ${delaySec}s`, undefined, this.currentRunContext).catch((err: unknown) => {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id} failed to log rate-limit retry: ${msg}`);
|
||
});
|
||
},
|
||
});
|
||
|
||
try {
|
||
if (this.options.semaphore) {
|
||
await this.options.semaphore.run(retryableStepWork, PRIORITY_EXECUTE);
|
||
} else {
|
||
await retryableStepWork();
|
||
}
|
||
} catch (err: unknown) {
|
||
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
||
if (this.depAborted.has(task.id)) {
|
||
this.depAborted.delete(task.id);
|
||
await this.handleDepAbortCleanup(task.id, worktreePath);
|
||
} else if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
await this.store.logEntry(task.id, "Execution paused during step-session", undefined, this.currentRunContext);
|
||
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
|
||
} else if (this.stuckAborted.has(task.id)) {
|
||
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
|
||
this.stuckAborted.delete(task.id);
|
||
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
|
||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
|
||
} else if (isTransientError(errorMessage)) {
|
||
const decision = computeRecoveryDecision({
|
||
recoveryRetryCount: task.recoveryRetryCount,
|
||
nextRecoveryAt: task.nextRecoveryAt,
|
||
});
|
||
|
||
if (decision.shouldRetry) {
|
||
const attempt = decision.nextState.recoveryRetryCount;
|
||
const delay = formatDelay(decision.delayMs);
|
||
if (!isSilentTransientError(errorMessage)) {
|
||
executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`);
|
||
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
|
||
}
|
||
if (worktreePath && existsSync(worktreePath)) {
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
// Audit trail: record worktree removal (FN-1404)
|
||
await audit.git({ type: "worktree:remove", target: worktreePath });
|
||
} catch (wtErr: unknown) {
|
||
const msg = wtErr instanceof Error ? wtErr.message : String(wtErr);
|
||
executorLog.warn(`${task.id}: worktree removal failed during transient-error retry cleanup (${worktreePath}): ${msg}`);
|
||
}
|
||
}
|
||
await this.store.updateTask(task.id, {
|
||
recoveryRetryCount: decision.nextState.recoveryRetryCount,
|
||
nextRecoveryAt: decision.nextState.nextRecoveryAt,
|
||
worktree: null,
|
||
branch: null,
|
||
});
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
stuckRequeue = null; // Prevent outer finally from re-processing
|
||
return;
|
||
}
|
||
|
||
executorLog.error(`✗ ${task.id} transient error retries exhausted: ${errorDetail}`);
|
||
if (errorStack) {
|
||
await this.store.logEntry(task.id, `Transient error retries exhausted: ${errorMessage}`, errorStack, this.currentRunContext);
|
||
}
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: errorMessage,
|
||
recoveryRetryCount: null,
|
||
nextRecoveryAt: null,
|
||
});
|
||
if (accumulatedStepTokenUsage) {
|
||
await this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage });
|
||
}
|
||
await this.store.moveTask(task.id, "in-review");
|
||
executorLog.log(`✗ ${task.id} transient retries exhausted → in-review`);
|
||
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
|
||
} else {
|
||
executorLog.error(`✗ ${task.id} step-session execution failed:`, errorDetail);
|
||
await this.store.logEntry(task.id, `Step-session execution failed: ${errorMessage}`, errorStack ?? errorDetail, this.currentRunContext);
|
||
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
|
||
if (accumulatedStepTokenUsage) {
|
||
await this.store.updateTask(task.id, { tokenUsage: accumulatedStepTokenUsage });
|
||
}
|
||
await this.store.moveTask(task.id, "in-review");
|
||
executorLog.log(`✗ ${task.id} step-session execution failed → in-review`);
|
||
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
|
||
}
|
||
} finally {
|
||
this.executing.delete(task.id);
|
||
this.loopRecoveryState.delete(task.id);
|
||
// Wrap cleanup in try/catch so activeStepExecutors.delete() always runs.
|
||
// If cleanup() throws, the executor continues to clean up the in-memory map
|
||
// and requeue logic without leaking the reference.
|
||
try {
|
||
await stepExecutor.cleanup();
|
||
} catch (cleanupErr) {
|
||
executorLog.warn(`StepSessionExecutor cleanup failed for ${task.id}: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
|
||
}
|
||
this.activeStepExecutors.delete(task.id);
|
||
|
||
// Stuck-requeue: clean up worktree and move to todo
|
||
if (stuckRequeue === true) {
|
||
try {
|
||
// Reset steps whose work was never committed before destroying the worktree
|
||
const latestTask = await this.store.getTask(task.id);
|
||
await this.resetStepsIfWorkLost(latestTask);
|
||
|
||
if (worktreePath && existsSync(worktreePath)) {
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
} catch (wtErr: unknown) {
|
||
const msg = wtErr instanceof Error ? wtErr.message : String(wtErr);
|
||
executorLog.warn(`${task.id}: worktree removal failed during stuck-requeue cleanup (${worktreePath}): ${msg}`);
|
||
}
|
||
}
|
||
await this.store.updateTask(task.id, { status: "stuck-killed", worktree: null, branch: null });
|
||
if (task.column !== "todo") {
|
||
await this.store.moveTask(task.id, "todo");
|
||
executorLog.log(`${task.id} moved to todo for retry after stuck kill`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to requeue stuck task ${task.id}: ${errorMessage}`);
|
||
}
|
||
stuckRequeue = null; // Prevent outer finally from re-processing
|
||
}
|
||
}
|
||
// Step-session path handled completely — return before outer catch/finally
|
||
return;
|
||
}
|
||
|
||
// ── Single-Session Path (default) ────────────────────────────────
|
||
// Build custom tools for the worker
|
||
// Track the last code review verdict per step so we can enforce REVISE
|
||
// (block fn_task_update status="done" until the agent re-reviews and gets APPROVE).
|
||
const codeReviewVerdicts = new Map<number, ReviewVerdict>();
|
||
|
||
let wasPaused = false;
|
||
// Mutable ref — populated after createFnAgent, tools access lazily via closure
|
||
const sessionRef: { current: AgentSession | null } = { current: null };
|
||
const stepCheckpoints = new Map<number, string>();
|
||
|
||
const stuckDetector = this.options.stuckTaskDetector;
|
||
const assignedAgentId = detail.assignedAgentId?.trim();
|
||
const reflectionTools = this.options.reflectionService && settings.reflectionEnabled && assignedAgentId
|
||
? [createReflectOnPerformanceTool(this.options.reflectionService, assignedAgentId)]
|
||
: [];
|
||
const assignedAgent = assignedAgentId && this.options.agentStore
|
||
? await this.options.agentStore.getAgent(assignedAgentId).catch(() => null)
|
||
: null;
|
||
const executorRuntimeHint = extractRuntimeHint(assignedAgent?.runtimeConfig);
|
||
|
||
// Log fast mode status
|
||
if (executionMode === "fast") {
|
||
executorLog.log(`${task.id}: fast mode — fn_review_step tool not injected`);
|
||
}
|
||
|
||
const customTools = [
|
||
this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stepCheckpoints, stuckDetector),
|
||
this.createTaskLogTool(task.id),
|
||
this.createTaskCreateTool(),
|
||
this.createTaskAddDepTool(task.id),
|
||
this.createTaskDoneTool(task.id, () => { taskDone = true; }),
|
||
createRunVerificationTool({
|
||
worktreePath,
|
||
rootDir: this.rootDir,
|
||
taskId: task.id,
|
||
recordActivity: () => stuckDetector?.recordActivity(task.id),
|
||
log: {
|
||
info: (s) => executorLog.log(s),
|
||
warn: (s) => executorLog.warn(s),
|
||
error: (s) => executorLog.warn(s),
|
||
},
|
||
}),
|
||
// Skip fn_review_step tool in fast mode — fast mode bypasses automated review gates
|
||
...(executionMode !== "fast" ? [
|
||
this.createReviewStepTool(task.id, worktreePath, detail.prompt, codeReviewVerdicts, sessionRef, stepCheckpoints, detail, stuckDetector),
|
||
] : []),
|
||
this.createSpawnAgentTool(task.id, worktreePath, settings),
|
||
this.createTaskDocumentWriteTool(task.id),
|
||
this.createTaskDocumentReadTool(task.id),
|
||
...createResearchTools({
|
||
store: this.store,
|
||
rootDir: this.rootDir,
|
||
getSettings: async () => this.store.getSettings(),
|
||
}),
|
||
...createMemoryTools(this.rootDir, settings, assignedAgent ? {
|
||
agentMemory: {
|
||
agentId: assignedAgent.id,
|
||
agentName: assignedAgent.name,
|
||
memory: assignedAgent.memory,
|
||
},
|
||
} : undefined),
|
||
// Conditionally add agent self-reflection when enabled and task has an assigned agent.
|
||
...reflectionTools,
|
||
// Agent delegation tools — discover and delegate work to other agents.
|
||
...(this.options.agentStore ? [
|
||
createListAgentsTool(this.options.agentStore),
|
||
createDelegateTaskTool(this.options.agentStore, this.store, { rootDir: this.rootDir }),
|
||
...(assignedAgentId ? [
|
||
createGetAgentConfigTool(this.options.agentStore, assignedAgentId),
|
||
createUpdateAgentConfigTool(this.options.agentStore, assignedAgentId),
|
||
] : []),
|
||
] : []),
|
||
// Messaging tools — allows executor agents to send and receive messages.
|
||
...(this.options.messageStore && assignedAgentId ? [
|
||
createSendMessageTool(this.options.messageStore, assignedAgentId),
|
||
createReadMessagesTool(this.options.messageStore, assignedAgentId),
|
||
] : []),
|
||
// Add plugin tools from PluginRunner
|
||
...(this.options.pluginRunner?.getPluginTools() ?? []),
|
||
];
|
||
|
||
// Accumulates the full assistant text output for the most recent session.
|
||
// Reset to "" each time a new session begins so detectPseudoPause only
|
||
// sees the last session's output, not the entire conversation history.
|
||
let lastAssistantText = "";
|
||
|
||
const agentLogger = new AgentLogger({
|
||
store: this.store,
|
||
taskId: task.id,
|
||
agent: "executor",
|
||
persistAgentToolOutput: settings.persistAgentToolOutput,
|
||
onAgentText: (taskId, delta) => {
|
||
lastAssistantText += delta;
|
||
stuckDetector?.recordActivity(taskId);
|
||
this.options.onAgentText?.(taskId, delta);
|
||
},
|
||
onAgentTool: (taskId, toolName) => {
|
||
stuckDetector?.recordActivity(taskId);
|
||
this.options.onAgentTool?.(taskId, toolName);
|
||
},
|
||
});
|
||
|
||
const agentWork = async () => {
|
||
// Resolve model settings using canonical lane hierarchy:
|
||
// 1. Task override pair (modelProvider + modelId)
|
||
// 2. Project execution lane pair (executionProvider + executionModelId)
|
||
// 3. Global execution lane pair (executionGlobalProvider + executionGlobalModelId)
|
||
// 4. Project default override pair (defaultProviderOverride + defaultModelIdOverride)
|
||
// 5. Global default pair (defaultProvider + defaultModelId)
|
||
const { provider: executorProvider, modelId: executorModelId } = resolveExecutorModelPair(
|
||
detail.modelProvider,
|
||
detail.modelId,
|
||
settings,
|
||
);
|
||
const executorFallbackProvider = settings.fallbackProvider;
|
||
const executorFallbackModelId = settings.fallbackModelId;
|
||
const executorThinkingLevel = detail.thinkingLevel ?? settings.defaultThinkingLevel;
|
||
|
||
// Determine whether we're resuming a previous session (pause/resume)
|
||
// or starting fresh. Use file-based sessions so conversation state
|
||
// persists across pause/unpause cycles.
|
||
const isResuming = !!task.sessionFile && existsSync(task.sessionFile);
|
||
const sessionManager = isResuming
|
||
? SessionManager.open(task.sessionFile!)
|
||
: SessionManager.create(worktreePath);
|
||
|
||
executorLog.log(`${task.id}: creating agent session (provider=${executorProvider ?? "default"}, model=${executorModelId ?? "default"}, resuming=${isResuming})`);
|
||
|
||
// Resolve per-agent custom instructions for the executor role
|
||
const executorInstructions = await this.resolveInstructionsForRole("executor");
|
||
const executorSystemPrompt = buildSystemPromptWithInstructions(
|
||
getExecutorSystemPrompt(settings),
|
||
executorInstructions,
|
||
);
|
||
|
||
// sessionFile must be let because it's destructured alongside session which is reassigned
|
||
// eslint-disable-next-line prefer-const
|
||
let { session, sessionFile } = await createResolvedAgentSession({
|
||
sessionPurpose: "executor",
|
||
runtimeHint: executorRuntimeHint,
|
||
pluginRunner: this.options.pluginRunner,
|
||
cwd: worktreePath,
|
||
systemPrompt: executorSystemPrompt,
|
||
tools: "coding",
|
||
customTools,
|
||
onText: agentLogger.onText,
|
||
onThinking: agentLogger.onThinking,
|
||
onToolStart: agentLogger.onToolStart,
|
||
onToolEnd: agentLogger.onToolEnd,
|
||
defaultProvider: executorProvider,
|
||
defaultModelId: executorModelId,
|
||
fallbackProvider: executorFallbackProvider,
|
||
fallbackModelId: executorFallbackModelId,
|
||
defaultThinkingLevel: executorThinkingLevel,
|
||
sessionManager,
|
||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||
taskId: task.id,
|
||
taskTitle: detail.title,
|
||
onFallbackModelUsed: createFallbackModelObserver({
|
||
agent: "executor",
|
||
label: "executor",
|
||
store: this.store,
|
||
taskId: task.id,
|
||
taskTitle: detail.title,
|
||
}),
|
||
});
|
||
|
||
if (isResuming) {
|
||
executorLog.log(`${task.id}: resumed session from ${task.sessionFile}`);
|
||
await this.store.logEntry(task.id, `Resumed agent session after unpause (model: ${describeModel(session)})`, undefined, this.currentRunContext);
|
||
} else {
|
||
executorLog.log(`${task.id}: using model ${describeModel(session)}`);
|
||
await this.store.logEntry(task.id, `Executor using model: ${describeModel(session)}`, undefined, this.currentRunContext);
|
||
// Persist session file path so pause/resume can reopen it
|
||
if (sessionFile) {
|
||
await this.store.updateTask(task.id, { sessionFile });
|
||
}
|
||
}
|
||
|
||
// Make session available to custom tools (fn_task_update checkpoint capture, fn_review_step rewind)
|
||
sessionRef.current = session;
|
||
|
||
// Register session so the pause listener can terminate it
|
||
// Initialize with empty set of seen comments
|
||
const seenSteeringIds = new Set<string>();
|
||
if (detail.comments) {
|
||
for (const comment of detail.comments) {
|
||
seenSteeringIds.add(comment.id);
|
||
}
|
||
}
|
||
this.activeSessions.set(task.id, {
|
||
session,
|
||
seenSteeringIds,
|
||
lastModelProvider: detail.modelProvider,
|
||
lastModelId: detail.modelId,
|
||
});
|
||
|
||
// Register with stuck task detector for heartbeat monitoring
|
||
stuckDetector?.trackTask(task.id, session);
|
||
executorLog.log(`${task.id}: session registered (model=${describeModel(session)}, stuckDetector=${!!stuckDetector})`);
|
||
|
||
// Invoke plugin onAgentRunStart hook (fire-and-forget)
|
||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||
void (this.options.pluginRunner as any)?.invokeHook("onAgentRunStart", task.id);
|
||
|
||
try {
|
||
// Record activity on prompt start (heartbeat for stuck detection)
|
||
stuckDetector?.recordActivity(task.id);
|
||
|
||
executorLog.log(`${task.id}: calling promptWithFallback()...`);
|
||
if (isResuming) {
|
||
// Session already has full conversation history — just tell the
|
||
// agent it was paused and should pick up where it left off.
|
||
await promptWithFallback(session, [
|
||
"Your session was paused and has now been resumed.",
|
||
"Continue working on the task from where you left off.",
|
||
"Review the current state of your worktree and proceed with the next pending step.",
|
||
].join("\n"));
|
||
} else {
|
||
const agentPrompt = buildExecutionPrompt(detail, this.rootDir, settings, worktreePath);
|
||
await promptWithFallback(session, agentPrompt);
|
||
}
|
||
|
||
// Re-raise errors that pi-coding-agent swallowed after exhausting retries.
|
||
// session.prompt() resolves normally even when retries are exhausted —
|
||
// the error is stored on session.state.error instead of being thrown.
|
||
checkSessionError(session);
|
||
await accumulateSessionTokenUsage(this.store, task.id, session);
|
||
|
||
// Check if proactive context compaction is needed based on token cap setting.
|
||
// This runs after the main prompt completes to avoid interrupting active work.
|
||
try {
|
||
const capResult = await this.tokenCapDetector.checkAndCompact(
|
||
session,
|
||
task.id,
|
||
settings.tokenCap,
|
||
async (s) => {
|
||
const compactResult = await compactSessionContext(s);
|
||
if (compactResult) {
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Context compacted at ${compactResult.tokensBefore} tokens (token cap: ${settings.tokenCap})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
}
|
||
return compactResult;
|
||
},
|
||
);
|
||
if (capResult.triggered) {
|
||
executorLog.log(`${task.id} token cap check: ${capResult.message}`);
|
||
}
|
||
} catch (err) {
|
||
executorLog.log(`${task.id} token cap check failed (non-fatal): ${err}`);
|
||
}
|
||
|
||
// If loop recovery is pending (compact-and-resume was triggered by
|
||
// handleLoopDetected), consume the pending state and resume with a
|
||
// deterministic prompt. The session has already been compacted, so
|
||
// we just need to send a fresh prompt to continue execution.
|
||
const loopState = this.loopRecoveryState.get(task.id);
|
||
if (loopState?.pending) {
|
||
loopState.pending = false;
|
||
executorLog.log(`${task.id} consuming loop recovery — resuming with fresh context`);
|
||
await this.store.logEntry(task.id, "Resuming execution after context compaction — taking a different approach", undefined, this.currentRunContext);
|
||
|
||
// Reset activity tracking so the detector doesn't immediately re-trigger
|
||
stuckDetector?.recordProgress(task.id);
|
||
|
||
const resumePrompt = [
|
||
"Your conversation was compacted because you were looping without making progress.",
|
||
"Review the current state of the worktree carefully:",
|
||
"1. Check `git log --oneline` to see what's already been committed",
|
||
"2. Read the files you were working on to understand current state",
|
||
"3. Review the PROMPT.md steps to see which are still pending",
|
||
"",
|
||
"Take a DIFFERENT approach from what you were doing before.",
|
||
"If the current step is complete, call fn_task_update to mark it done and move to the next step.",
|
||
"If you're stuck on a problem, try a simpler or alternative solution.",
|
||
"",
|
||
"Continue the task from where you left off.",
|
||
].join("\n");
|
||
|
||
await promptWithFallback(session, resumePrompt);
|
||
checkSessionError(session);
|
||
await accumulateSessionTokenUsage(this.store, task.id, session);
|
||
}
|
||
|
||
// If dependency was added during execution, discard worktree and move to triage
|
||
if (this.depAborted.has(task.id)) {
|
||
this.depAborted.delete(task.id);
|
||
await this.handleDepAbortCleanup(task.id, worktreePath);
|
||
return;
|
||
}
|
||
|
||
// If paused during execution, move to todo so the scheduler can resume
|
||
// after unpause. This path fires when session.dispose() causes the
|
||
// prompt to resolve gracefully instead of throwing.
|
||
if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
wasPaused = true;
|
||
if (await this.shouldFinalizeCompletedTask(task.id, taskDone)) {
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "paused after completion")) {
|
||
return;
|
||
}
|
||
executorLog.log(`${task.id} paused after completion (graceful session exit) — finalizing to in-review`);
|
||
await this.store.logEntry(task.id, "Execution paused after completion — finalizing to in-review");
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
this.options.onComplete?.(task);
|
||
} else {
|
||
executorLog.log(`${task.id} paused (graceful session exit) — moving to todo`);
|
||
await this.store.logEntry(task.id, "Execution paused — session preserved for resume, moved to todo");
|
||
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
|
||
}
|
||
return;
|
||
}
|
||
|
||
// If the stuck task detector disposed the session and the agent exited
|
||
// cleanly, stop here. The requeue is deferred to the finally block
|
||
// (after this.executing is cleared) to prevent a race where the
|
||
// scheduler re-dispatches while the old execution guard is still set.
|
||
if (this.stuckAborted.has(task.id)) {
|
||
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
|
||
this.stuckAborted.delete(task.id);
|
||
executorLog.log(`${task.id} terminated by stuck task detector (graceful session exit)`);
|
||
return;
|
||
}
|
||
|
||
// If the agent didn't explicitly call fn_task_done, check whether
|
||
// all steps are already complete — treat as implicit done to avoid
|
||
// unnecessary retry sessions for context-overflow / compaction cases.
|
||
if (!taskDone) {
|
||
const implicitCheck = await this.store.getTask(task.id);
|
||
if (implicitCheck.steps.length > 0 &&
|
||
implicitCheck.steps.every((s) => s.status === "done" || s.status === "skipped")) {
|
||
taskDone = true;
|
||
executorLog.log(`${task.id} all steps done — treating as implicit fn_task_done`);
|
||
await this.store.logEntry(task.id, "All steps complete — implicit fn_task_done (agent did not call tool explicitly)", undefined, this.currentRunContext);
|
||
this.scheduleCompletedTaskWatchdog(task.id, "implicit fn_task_done");
|
||
}
|
||
}
|
||
|
||
if (taskDone) {
|
||
// Capture modified files before running workflow steps
|
||
const updatedTask = await this.store.getTask(task.id);
|
||
const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha);
|
||
if (modifiedFiles.length > 0) {
|
||
await this.store.updateTask(task.id, { modifiedFiles });
|
||
executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`);
|
||
}
|
||
|
||
this.scheduleCompletedTaskWatchdog(task.id, "task completion");
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after task completion")) {
|
||
return;
|
||
}
|
||
|
||
// Run workflow steps before moving to in-review — skip in fast mode
|
||
if (executionMode !== "fast") {
|
||
const workflowResult = await this.runWorkflowSteps(task, worktreePath, settings);
|
||
if (workflowResult === "deferred-paused") {
|
||
if (await this.parkTaskAfterWorkflowStepPause(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
wasPaused = true;
|
||
return;
|
||
}
|
||
if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
wasPaused = true;
|
||
}
|
||
return;
|
||
}
|
||
if (!workflowResult.allPassed) {
|
||
// Check if revision was requested
|
||
if (workflowResult.revisionRequested) {
|
||
await this.handleWorkflowRevisionRequest(task, worktreePath, workflowResult.feedback, workflowResult.stepName);
|
||
return;
|
||
}
|
||
// Try to fix workflow step failures with retries
|
||
const retried = await this.handleWorkflowStepFailure(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown");
|
||
if (retried) {
|
||
return; // Retry scheduled
|
||
}
|
||
// Retries exhausted - send back to in-progress for remediation
|
||
await this.sendTaskBackForFix(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed");
|
||
return;
|
||
}
|
||
} else {
|
||
executorLog.log(`${task.id}: fast mode — skipping pre-merge workflow steps`);
|
||
await this.store.logEntry(task.id, "Fast mode — pre-merge workflow steps skipped", undefined, this.currentRunContext);
|
||
}
|
||
|
||
// Reset retry counters on success
|
||
await this.store.updateTask(task.id, { workflowStepRetries: undefined, taskDoneRetryCount: null });
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after task completion")) {
|
||
return;
|
||
}
|
||
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
executorLog.log(`✓ ${task.id} completed → in-review`);
|
||
this.options.onComplete?.(task);
|
||
} else {
|
||
let taskDoneSessionRetries = 0;
|
||
while (!taskDone && taskDoneSessionRetries < MAX_TASK_DONE_SESSION_RETRIES) {
|
||
taskDoneSessionRetries++;
|
||
executorLog.log(
|
||
`⚠ ${task.id} finished without fn_task_done — retrying with new session (${taskDoneSessionRetries}/${MAX_TASK_DONE_SESSION_RETRIES})`,
|
||
);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Agent finished without calling fn_task_done — retrying with new session (${taskDoneSessionRetries}/${MAX_TASK_DONE_SESSION_RETRIES})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
|
||
// Capture and analyse the previous session's text before resetting.
|
||
const previousSessionText = lastAssistantText;
|
||
const pseudoPause = detectPseudoPause(previousSessionText);
|
||
|
||
if (pseudoPause.kind !== "none") {
|
||
const shortMatch = (pseudoPause.matched ?? "").slice(0, 120);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Pseudo-pause detected (kind=${pseudoPause.kind}, matched='${shortMatch}')`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
executorLog.log(`${task.id} pseudo-pause detected (kind=${pseudoPause.kind}): ${shortMatch}`);
|
||
}
|
||
|
||
// Dispose old session and create a fresh one.
|
||
// Reset lastAssistantText so the new session's text is tracked cleanly.
|
||
lastAssistantText = "";
|
||
this.activeSessions.delete(task.id);
|
||
this.tokenUsageBaselines.delete(task.id);
|
||
session.dispose();
|
||
|
||
const { session: retrySession, sessionFile: retrySessionFile } = await createResolvedAgentSession({
|
||
sessionPurpose: "executor",
|
||
runtimeHint: executorRuntimeHint,
|
||
pluginRunner: this.options.pluginRunner,
|
||
cwd: worktreePath,
|
||
systemPrompt: executorSystemPrompt,
|
||
tools: "coding",
|
||
customTools,
|
||
onText: agentLogger.onText,
|
||
onThinking: agentLogger.onThinking,
|
||
onToolStart: agentLogger.onToolStart,
|
||
onToolEnd: agentLogger.onToolEnd,
|
||
defaultProvider: executorProvider,
|
||
defaultModelId: executorModelId,
|
||
fallbackProvider: executorFallbackProvider,
|
||
fallbackModelId: executorFallbackModelId,
|
||
defaultThinkingLevel: executorThinkingLevel,
|
||
sessionManager: SessionManager.create(worktreePath),
|
||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||
});
|
||
if (retrySessionFile) {
|
||
this.store.updateTask(task.id, { sessionFile: retrySessionFile }).catch((err: unknown) => {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id} failed to persist retry sessionFile: ${msg}`);
|
||
});
|
||
}
|
||
|
||
session = retrySession;
|
||
sessionRef.current = retrySession;
|
||
this.activeSessions.set(task.id, {
|
||
session: retrySession,
|
||
seenSteeringIds,
|
||
lastModelProvider: detail.modelProvider,
|
||
lastModelId: detail.modelId,
|
||
});
|
||
stuckDetector?.trackTask(task.id, retrySession);
|
||
|
||
let retryPrompt: string;
|
||
if (pseudoPause.kind !== "none") {
|
||
const shortMatch = (pseudoPause.matched ?? "").slice(0, 120);
|
||
retryPrompt = [
|
||
`Your previous turn ended with a pseudo-pause: "${shortMatch}". This is forbidden.`,
|
||
"",
|
||
"Turn-ending rules you violated:",
|
||
"- You MUST NOT end a turn by asking the user a question, summarizing progress, or requesting permission to continue.",
|
||
"- Phrases like 'If you want, I can continue', 'Should I proceed?', 'Let me know if...' are FORBIDDEN turn-endings.",
|
||
"- The user is not watching this conversation. Questions written as prose are ignored.",
|
||
"- If you genuinely cannot proceed, call fn_task_done with a clear explanation — never write the blocker as plain prose.",
|
||
"",
|
||
"What you must do now:",
|
||
"1. Review the PROMPT.md steps and identify the next pending step.",
|
||
"2. Do the work for that step immediately — call fn_task_update, write code, run tests.",
|
||
"3. Continue until all steps are done, then call fn_task_done.",
|
||
"Do NOT ask for permission. Do NOT write a summary. Just call a tool and keep working.",
|
||
"",
|
||
"Original task:",
|
||
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath),
|
||
].join("\n");
|
||
} else {
|
||
retryPrompt = [
|
||
"Your previous session ended without calling the fn_task_done tool.",
|
||
"The task may already be complete — review the current state of the worktree and either:",
|
||
"1. If the work is done, call fn_task_done with a summary of what was accomplished.",
|
||
"2. If there is remaining work, finish it and then call fn_task_done.",
|
||
"",
|
||
"Original task:",
|
||
buildExecutionPrompt(detail, this.rootDir, settings, worktreePath),
|
||
].join("\n");
|
||
}
|
||
|
||
stuckDetector?.recordActivity(task.id);
|
||
await promptWithFallback(retrySession, retryPrompt);
|
||
checkSessionError(retrySession);
|
||
await accumulateSessionTokenUsage(this.store, task.id, retrySession);
|
||
|
||
if (!taskDone) {
|
||
const implicitCheck = await this.store.getTask(task.id);
|
||
if (implicitCheck.steps.length > 0 &&
|
||
implicitCheck.steps.every((s) => s.status === "done" || s.status === "skipped")) {
|
||
taskDone = true;
|
||
executorLog.log(`${task.id} all steps done — treating as implicit fn_task_done`);
|
||
await this.store.logEntry(task.id, "All steps complete — implicit fn_task_done (agent did not call tool explicitly)", undefined, this.currentRunContext);
|
||
this.scheduleCompletedTaskWatchdog(task.id, "implicit fn_task_done");
|
||
}
|
||
}
|
||
}
|
||
|
||
if (taskDone) {
|
||
const updatedTask = await this.store.getTask(task.id);
|
||
const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha);
|
||
if (modifiedFiles.length > 0) {
|
||
await this.store.updateTask(task.id, { modifiedFiles });
|
||
executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`);
|
||
}
|
||
|
||
this.scheduleCompletedTaskWatchdog(task.id, "task completion retry");
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after task completion retry")) {
|
||
return;
|
||
}
|
||
|
||
// Run workflow steps before moving to in-review — skip in fast mode
|
||
if (executionMode !== "fast") {
|
||
const workflowResult = await this.runWorkflowSteps(task, worktreePath, settings);
|
||
if (workflowResult === "deferred-paused") {
|
||
if (await this.parkTaskAfterWorkflowStepPause(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
wasPaused = true;
|
||
return;
|
||
}
|
||
if (this.pausedAborted.has(task.id)) {
|
||
this.pausedAborted.delete(task.id);
|
||
wasPaused = true;
|
||
}
|
||
return;
|
||
}
|
||
if (!workflowResult.allPassed) {
|
||
if (workflowResult.revisionRequested) {
|
||
await this.handleWorkflowRevisionRequest(task, worktreePath, workflowResult.feedback, workflowResult.stepName);
|
||
return;
|
||
}
|
||
await this.sendTaskBackForFix(task, worktreePath, workflowResult.feedback, workflowResult.stepName || "Unknown", "Workflow step failed on retry");
|
||
return;
|
||
}
|
||
} else {
|
||
executorLog.log(`${task.id}: fast mode — skipping pre-merge workflow steps`);
|
||
await this.store.logEntry(task.id, "Fast mode — pre-merge workflow steps skipped", undefined, this.currentRunContext);
|
||
}
|
||
|
||
await this.store.updateTask(task.id, { workflowStepRetries: undefined, taskDoneRetryCount: null });
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after task completion retry")) {
|
||
return;
|
||
}
|
||
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
executorLog.log(`✓ ${task.id} completed on retry → in-review`);
|
||
this.options.onComplete?.(task);
|
||
} else {
|
||
const priorRequeues = task.taskDoneRetryCount ?? 0;
|
||
const nextRequeueCount = priorRequeues + 1;
|
||
const errorMessage = `Agent finished without calling fn_task_done (after ${MAX_TASK_DONE_SESSION_RETRIES} retries)`;
|
||
|
||
if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: errorMessage,
|
||
taskDoneRetryCount: nextRequeueCount,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`${errorMessage} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
executorLog.log(`✗ ${task.id} failed after ${MAX_TASK_DONE_SESSION_RETRIES} retries — requeued to todo (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`);
|
||
} else {
|
||
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
|
||
await this.store.logEntry(task.id, `${errorMessage} — moved to in-review for inspection`, undefined, this.currentRunContext);
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
executorLog.log(`✗ ${task.id} failed after ${MAX_TASK_DONE_SESSION_RETRIES} retries — no fn_task_done → in-review`);
|
||
}
|
||
this.options.onError?.(task, new Error(errorMessage));
|
||
}
|
||
}
|
||
} finally {
|
||
this.activeSessions.delete(task.id);
|
||
stuckDetector?.untrackTask(task.id);
|
||
await agentLogger.flush();
|
||
await this.persistTokenUsage(task.id, session).catch((err: unknown) => {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id}: failed to persist final single-session token usage before dispose: ${msg}`);
|
||
});
|
||
session.dispose();
|
||
// Terminate all spawned child agents when parent session ends
|
||
await this.terminateAllChildren(task.id);
|
||
// Clear session file when task completes or fails (not when paused —
|
||
// the file is preserved so unpause can resume the conversation).
|
||
// Check both the local flag (graceful exit) and the instance set
|
||
// (error path where dispose caused prompt to throw).
|
||
if (!wasPaused && !this.pausedAborted.has(task.id)) {
|
||
this.store.updateTask(task.id, { sessionFile: null }).catch((err: unknown) => {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id} failed to clear sessionFile: ${msg}`);
|
||
});
|
||
}
|
||
// Invoke plugin onAgentRunEnd hook (fire-and-forget)
|
||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||
void (this.options.pluginRunner as any)?.invokeHook("onAgentRunEnd", task.id);
|
||
}
|
||
};
|
||
|
||
const retryableWork = () => withRateLimitRetry(agentWork, {
|
||
onRetry: (attempt, delayMs, error) => {
|
||
const delaySec = Math.round(delayMs / 1000);
|
||
executorLog.warn(`⏳ ${task.id} rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`);
|
||
this.store.logEntry(task.id, `Rate limited — retry ${attempt} in ${delaySec}s`, undefined, this.currentRunContext).catch((err: unknown) => {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id} failed to log rate-limit retry: ${msg}`);
|
||
});
|
||
},
|
||
});
|
||
|
||
if (this.options.semaphore) {
|
||
await this.options.semaphore.run(retryableWork, PRIORITY_EXECUTE);
|
||
} else {
|
||
await retryableWork();
|
||
}
|
||
} catch (err: unknown) {
|
||
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
||
if (this.depAborted.has(task.id)) {
|
||
// Dependency added mid-execution — discard worktree and move to triage
|
||
this.depAborted.delete(task.id);
|
||
await this.handleDepAbortCleanup(task.id, worktreePath);
|
||
} else if (errorMessage.includes("Invalid transition")) {
|
||
// Task was moved by user/process while executor was running — already in desired state
|
||
// This check must come before pausedAborted since it's more specific
|
||
const transitionMatch = errorMessage.match(/Invalid transition: '([^']+)' → '([^']+)'/);
|
||
const fromColumn = transitionMatch?.[1] ?? "unknown";
|
||
const toColumn = transitionMatch?.[2] ?? "unknown";
|
||
const logMessage = `Task already moved from '${fromColumn}' — skipping transition to '${toColumn}'`;
|
||
executorLog.log(`${task.id} ${logMessage}`);
|
||
await this.store.logEntry(task.id, logMessage, errorMessage, this.currentRunContext);
|
||
if (fromColumn === "in-review" && toColumn === "in-review") {
|
||
try {
|
||
const finalizeResult = await this.finalizeAlreadyReviewedTask(task.id);
|
||
executorLog.log(`${task.id} duplicate in-review finalization result: ${finalizeResult}`);
|
||
} catch (finalizeErr: unknown) {
|
||
const finalizeErrMessage = finalizeErr instanceof Error ? finalizeErr.message : String(finalizeErr);
|
||
executorLog.warn(`${task.id} failed to finalize duplicate in-review transition: ${finalizeErrMessage}`);
|
||
}
|
||
}
|
||
// Task finished successfully (just already moved), so call onComplete
|
||
this.options.onComplete?.(task);
|
||
} else if (this.pausedAborted.has(task.id)) {
|
||
// Task was paused mid-execution — clean up worktree and move to todo
|
||
this.pausedAborted.delete(task.id);
|
||
if (await this.shouldFinalizeCompletedTask(task.id, taskDone)) {
|
||
if (await this.shouldDeferCompletionForGlobalPause(task.id, "paused after completion")) {
|
||
return;
|
||
}
|
||
executorLog.log(`${task.id} paused after completion — finalizing to in-review`);
|
||
await this.store.logEntry(task.id, "Execution paused after completion — finalizing to in-review", undefined, this.currentRunContext);
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
this.options.onComplete?.(task);
|
||
} else {
|
||
executorLog.log(`${task.id} paused — moving to todo`);
|
||
if (worktreePath && existsSync(worktreePath)) {
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
executorLog.log(`Removed old worktree for paused task: ${worktreePath}`);
|
||
// Audit trail: record worktree removal (FN-1404)
|
||
await audit.git({ type: "worktree:remove", target: worktreePath });
|
||
} catch (cleanupErr: unknown) {
|
||
const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr);
|
||
executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`);
|
||
}
|
||
}
|
||
await this.store.updateTask(task.id, { worktree: undefined, branch: undefined });
|
||
await this.store.logEntry(task.id, "Execution paused — agent terminated, moved to todo", undefined, this.currentRunContext);
|
||
await this.store.moveTask(task.id, "todo");
|
||
}
|
||
} else if (this.stuckAborted.has(task.id)) {
|
||
// Task was killed by stuck task detector — defer requeue to finally block
|
||
// (after this.executing is cleared) to prevent re-dispatch race.
|
||
stuckRequeue = this.stuckAborted.get(task.id) ?? true;
|
||
this.stuckAborted.delete(task.id);
|
||
executorLog.log(`${task.id} terminated by stuck task detector — will ${stuckRequeue ? "retry" : "not retry (budget exhausted)"}`);
|
||
} else {
|
||
// Context-limit error reached the executor after promptWithFallback's auto-compaction
|
||
// already attempted to recover. Recovery strategy (in order):
|
||
// 1. Reduced-prompt retry in the same session (up to MAX_REDUCED_PROMPT_ATTEMPTS)
|
||
// 2. Fresh-session requeue — terminate the saturated session and move the task
|
||
// back to "todo" so the next dispatch gets a clean session (bounded by
|
||
// recoveryRetryCount / MAX_RECOVERY_RETRIES).
|
||
// FN-2182 class: Step 7 overflow after earlier compaction used to hit the
|
||
// loopAttempts<1 guard and fail permanently; the requeue path below recovers
|
||
// by restarting with a fresh session against the already-written step output.
|
||
const MAX_REDUCED_PROMPT_ATTEMPTS = 3;
|
||
const loopState = this.loopRecoveryState.get(task.id);
|
||
const loopAttempts = loopState?.attempts ?? 0;
|
||
const isContextError = isContextLimitError(errorMessage);
|
||
|
||
if (isContextError && loopAttempts < MAX_REDUCED_PROMPT_ATTEMPTS) {
|
||
const activeEntry = this.activeSessions.get(task.id);
|
||
if (activeEntry) {
|
||
executorLog.log(`${task.id} context limit error after auto-compaction — attempting reduced-prompt retry (${loopAttempts + 1}/${MAX_REDUCED_PROMPT_ATTEMPTS})`);
|
||
await this.store.logEntry(task.id, `Context limit error after auto-compaction — attempting reduced-prompt retry (${loopAttempts + 1}/${MAX_REDUCED_PROMPT_ATTEMPTS}): ${errorMessage}`, undefined, this.currentRunContext);
|
||
|
||
this.loopRecoveryState.set(task.id, { attempts: loopAttempts + 1, pending: false });
|
||
|
||
try {
|
||
this.options.stuckTaskDetector?.recordProgress(task.id);
|
||
// Build a reduced prompt that's simpler and shorter to avoid context overflow
|
||
const reducedPrompt = [
|
||
"Your previous attempt hit the context window limit.",
|
||
"Focus on completing the task efficiently with minimal context:",
|
||
"1. Review git status and git log to see what's been done",
|
||
"2. Identify the most critical remaining work",
|
||
"3. Complete it with a simpler, more focused approach",
|
||
"",
|
||
"Do not repeat what's already been done. Just complete the task and call fn_task_done.",
|
||
].join("\n");
|
||
|
||
await promptWithFallback(activeEntry.session, reducedPrompt);
|
||
checkSessionError(activeEntry.session);
|
||
await accumulateSessionTokenUsage(this.store, task.id, activeEntry.session);
|
||
|
||
// Reduced-prompt retry succeeded — return to let the finally block clean up
|
||
// without marking the task as failed.
|
||
executorLog.log(`${task.id} reduced-prompt recovery succeeded — continuing`);
|
||
await this.store.logEntry(task.id, "Reduced-prompt recovery succeeded — continuing execution", undefined, this.currentRunContext);
|
||
return;
|
||
} catch (reducedErr: unknown) {
|
||
const reducedErrorMessage = reducedErr instanceof Error ? reducedErr.message : String(reducedErr);
|
||
if (!isContextLimitError(reducedErrorMessage)) {
|
||
executorLog.error(`${task.id} reduced-prompt recovery also failed: ${reducedErrorMessage}`);
|
||
await this.store.logEntry(task.id, `Reduced-prompt recovery failed: ${reducedErrorMessage}`, undefined, this.currentRunContext);
|
||
// Non-context failure — fall through to mark task as failed
|
||
} else {
|
||
// Still a context error — the session is saturated beyond recovery.
|
||
// Fall through to the fresh-session requeue path below.
|
||
executorLog.warn(`${task.id} session still saturated after reduced-prompt retry — will attempt fresh-session requeue`);
|
||
await this.store.logEntry(task.id, `Reduced-prompt retry still over context — will attempt fresh-session requeue`, undefined, this.currentRunContext);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Fresh-session requeue for context-limit errors: the saturated session
|
||
// cannot be salvaged, but the task's git state is intact. Move the task
|
||
// back to todo so the next scheduling pass creates a new session.
|
||
if (isContextError) {
|
||
const decision = computeRecoveryDecision({
|
||
recoveryRetryCount: task.recoveryRetryCount,
|
||
nextRecoveryAt: task.nextRecoveryAt,
|
||
});
|
||
|
||
if (decision.shouldRetry) {
|
||
const attempt = decision.nextState.recoveryRetryCount;
|
||
const delay = formatDelay(decision.delayMs);
|
||
executorLog.warn(`⚡ ${task.id} context-overflow fresh-session requeue ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`);
|
||
await this.store.logEntry(task.id, `Context-overflow fresh-session requeue (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
|
||
// Retain the worktree and accumulated step progress so the fresh
|
||
// session resumes where the saturated one left off, but clear
|
||
// sessionFile synchronously here so the next dispatch is forced
|
||
// to spawn a brand-new session instead of reopening the
|
||
// over-context one. The session-end finally block also clears
|
||
// sessionFile, but it runs as fire-and-forget — if moveTask
|
||
// wins the task lock first, the next executor pass would
|
||
// observe a stale sessionFile and resume into the saturated
|
||
// session, looping on the same context-limit failure.
|
||
await this.store.updateTask(task.id, {
|
||
recoveryRetryCount: decision.nextState.recoveryRetryCount,
|
||
nextRecoveryAt: decision.nextState.nextRecoveryAt,
|
||
sessionFile: null,
|
||
});
|
||
await this.store.moveTask(task.id, "todo", { preserveResumeState: true });
|
||
return;
|
||
}
|
||
|
||
executorLog.error(`✗ ${task.id} context-overflow requeue budget exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`);
|
||
await this.store.logEntry(task.id, `Context-overflow requeues exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.currentRunContext);
|
||
// Reset so downstream failure path can persist cleanly
|
||
await this.store.updateTask(task.id, {
|
||
recoveryRetryCount: null,
|
||
nextRecoveryAt: null,
|
||
});
|
||
// Fall through to terminal failure marking
|
||
} else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) {
|
||
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage);
|
||
} else if (isTransientError(errorMessage)) {
|
||
// Transient network/infrastructure error — use bounded recovery policy
|
||
const decision = computeRecoveryDecision({
|
||
recoveryRetryCount: task.recoveryRetryCount,
|
||
nextRecoveryAt: task.nextRecoveryAt,
|
||
});
|
||
|
||
if (decision.shouldRetry) {
|
||
const attempt = decision.nextState.recoveryRetryCount;
|
||
const delay = formatDelay(decision.delayMs);
|
||
// Silent transient errors (e.g., "request was aborted") are noisy — skip logging
|
||
if (!isSilentTransientError(errorMessage)) {
|
||
executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`);
|
||
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.currentRunContext);
|
||
}
|
||
// Clean up the old worktree so the retry gets a fresh one
|
||
if (worktreePath && existsSync(worktreePath)) {
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
executorLog.log(`Removed old worktree for transient retry: ${worktreePath}`);
|
||
// Audit trail: record worktree removal (FN-1404)
|
||
await audit.git({ type: "worktree:remove", target: worktreePath });
|
||
} catch (cleanupErr: unknown) {
|
||
const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr);
|
||
executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`);
|
||
}
|
||
}
|
||
await this.store.updateTask(task.id, {
|
||
recoveryRetryCount: decision.nextState.recoveryRetryCount,
|
||
nextRecoveryAt: decision.nextState.nextRecoveryAt,
|
||
worktree: null,
|
||
branch: null,
|
||
});
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
return;
|
||
}
|
||
|
||
// Recovery budget exhausted — escalate to real failure
|
||
executorLog.error(`✗ ${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorDetail}`);
|
||
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, errorStack ?? errorDetail, this.currentRunContext);
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: errorMessage,
|
||
recoveryRetryCount: null,
|
||
nextRecoveryAt: null,
|
||
});
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
executorLog.log(`✗ ${task.id} transient retries exhausted → in-review`);
|
||
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
|
||
return;
|
||
}
|
||
executorLog.error(`✗ ${task.id} execution failed:`, errorDetail);
|
||
await this.store.logEntry(task.id, `Execution failed: ${errorMessage}`, errorStack ?? errorDetail, this.currentRunContext);
|
||
await this.store.updateTask(task.id, { status: "failed", error: errorMessage });
|
||
await this.persistTokenUsage(task.id);
|
||
await this.store.moveTask(task.id, "in-review");
|
||
executorLog.log(`✗ ${task.id} execution failed → in-review`);
|
||
this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage));
|
||
}
|
||
} finally {
|
||
this.executing.delete(task.id);
|
||
// Clear run context at end of execute() lifecycle
|
||
this.currentRunContext = undefined;
|
||
|
||
// Terminate all spawned child agents on ALL exit paths.
|
||
// This must run here (in the outer finally) rather than only in agentWork's
|
||
// finally block, because failures during worktree creation or before
|
||
// agentWork is entered leave children orphaned with no other cleanup path.
|
||
try {
|
||
await this.terminateAllChildren(task.id);
|
||
} catch (err) {
|
||
executorLog.warn(`terminateAllChildren failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
// Reset loop recovery state at end of execute() lifecycle.
|
||
// State is in-memory and per-run — should not persist across attempts.
|
||
this.loopRecoveryState.delete(task.id);
|
||
this.tokenUsageBaselines.delete(task.id);
|
||
|
||
// Requeue stuck-killed task AFTER this.executing is cleared.
|
||
// This prevents the race where the scheduler re-dispatches the task
|
||
// (via task:moved → execute()) while the old execution guard is still set,
|
||
// which caused the new execute() call to silently no-op, stranding the
|
||
// task in "in-progress" with no active session or worktree.
|
||
if (stuckRequeue === true) {
|
||
try {
|
||
// Reset steps whose work was never committed before destroying the worktree
|
||
const latestTask = await this.store.getTask(task.id);
|
||
await this.resetStepsIfWorkLost(latestTask);
|
||
|
||
// Clean up the old worktree so the retry gets a fresh one
|
||
if (worktreePath && existsSync(worktreePath)) {
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
executorLog.log(`Removed old worktree for stuck-killed retry: ${worktreePath}`);
|
||
// Audit trail: record worktree removal (FN-1404)
|
||
await audit.git({ type: "worktree:remove", target: worktreePath });
|
||
} catch (cleanupErr: unknown) {
|
||
const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr);
|
||
executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`);
|
||
}
|
||
}
|
||
await this.store.updateTask(task.id, { status: "stuck-killed", worktree: null, branch: null });
|
||
// Only move to todo if not already there. The task.column check uses the
|
||
// captured task object from execute() start — if the task was already in "todo"
|
||
// when execute() started (e.g., resumed orphan), we skip the redundant move.
|
||
if (task.column !== "todo") {
|
||
await this.store.moveTask(task.id, "todo");
|
||
// Audit trail: record task move (FN-1404)
|
||
await audit.database({ type: "task:move", target: task.id, metadata: { to: "todo" } });
|
||
executorLog.log(`${task.id} moved to todo for retry after stuck kill`);
|
||
} else {
|
||
executorLog.log(`${task.id} already in todo — skipping redundant move`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to requeue stuck task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── Custom tools for the worker agent ──────────────────────────────
|
||
|
||
private createTaskUpdateTool(
|
||
taskId: string,
|
||
codeReviewVerdicts: Map<number, ReviewVerdict>,
|
||
sessionRef: { current: AgentSession | null },
|
||
stepCheckpoints: Map<number, string>,
|
||
stuckDetector?: StuckTaskDetector,
|
||
): ToolDefinition {
|
||
const store = this.store;
|
||
return {
|
||
name: "fn_task_update",
|
||
label: "Update Step",
|
||
description:
|
||
"Update a step's status. Call before starting a step (in-progress), " +
|
||
"after completing it (done), or to skip it (skipped). " +
|
||
"The board updates in real-time.",
|
||
parameters: taskUpdateParams,
|
||
execute: async (_id: string, params: Static<typeof taskUpdateParams>) => {
|
||
const { step, status } = params;
|
||
|
||
// Record step progress for stuck task detection.
|
||
// Step transitions (in-progress, done, skipped) indicate real progress
|
||
// and reset the loop detection counter. Generic activity (text deltas,
|
||
// tool calls) is tracked separately via recordActivity in AgentLogger.
|
||
if (status === "in-progress" || status === "done" || status === "skipped") {
|
||
stuckDetector?.recordProgress(taskId);
|
||
}
|
||
|
||
// Enforce code review REVISE: block advancing to "done" when the last
|
||
// code review for this step returned REVISE. The agent must fix the
|
||
// issues and call fn_review_step(type="code") again before proceeding.
|
||
if (status === "done" && codeReviewVerdicts.get(step) === "REVISE") {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Cannot mark Step ${step} as done — the last code review returned REVISE. ` +
|
||
`Fix the issues from the code review, commit your changes, and call ` +
|
||
`fn_review_step(step=${step}, type="code") again. The step can only advance ` +
|
||
`after the code review passes.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
const stepIndex = step - 1;
|
||
if (!Number.isInteger(stepIndex) || stepIndex < 0) {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Invalid step number: ${step}. Steps are 1-indexed.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
const task = await store.updateStep(taskId, stepIndex, status as StepStatus);
|
||
const stepInfo = task.steps[stepIndex];
|
||
const persistedStatus = stepInfo.status;
|
||
const progress = task.steps.filter((s) => s.status === "done").length;
|
||
|
||
// Capture session checkpoint only when the store actually moved the
|
||
// step to in-progress, so RETHINK can rewind to it. Doing this AFTER
|
||
// updateStep means a regression that updateStep ignores (e.g. the
|
||
// agent re-marking an already-done step) cannot replace the
|
||
// pre-step leaf with a later one.
|
||
if (
|
||
status === "in-progress" &&
|
||
persistedStatus === "in-progress" &&
|
||
sessionRef.current
|
||
) {
|
||
const leafId = sessionRef.current.sessionManager.getLeafId();
|
||
if (leafId) {
|
||
stepCheckpoints.set(step, leafId);
|
||
}
|
||
}
|
||
|
||
// If the persisted status doesn't match the requested status, the
|
||
// store rejected the transition (currently: in-progress regression
|
||
// on a done/skipped step). Tell the agent honestly so it doesn't
|
||
// assume the step reopened.
|
||
if (persistedStatus !== status) {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Step ${step} (${stepInfo.name}) is already ${persistedStatus} — ${status} request ignored to preserve completed work. Progress: ${progress}/${task.steps.length} done.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Step ${step} (${stepInfo.name}) → ${persistedStatus}. Progress: ${progress}/${task.steps.length} done.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
},
|
||
};
|
||
}
|
||
|
||
private createTaskLogTool(taskId: string): ToolDefinition {
|
||
return sharedCreateTaskLogTool(this.store, taskId);
|
||
}
|
||
|
||
private createTaskCreateTool(): ToolDefinition {
|
||
return sharedCreateTaskCreateTool(this.store, { sourceType: "api" }, { rootDir: this.rootDir });
|
||
}
|
||
|
||
private createTaskDocumentWriteTool(taskId: string): ToolDefinition {
|
||
return sharedCreateTaskDocumentWriteTool(this.store, taskId);
|
||
}
|
||
|
||
private createTaskDocumentReadTool(taskId: string): ToolDefinition {
|
||
return sharedCreateTaskDocumentReadTool(this.store, taskId);
|
||
}
|
||
|
||
private createTaskAddDepTool(taskId: string): ToolDefinition {
|
||
const store = this.store;
|
||
return {
|
||
name: "fn_task_add_dep",
|
||
label: "Add Dependency",
|
||
description:
|
||
"Declare a dependency on an existing task. Use when you discover " +
|
||
"mid-execution that another task must be completed first. " +
|
||
"Adding a dependency to an in-progress task will stop execution " +
|
||
"and discard current work, so confirm=true is required. " +
|
||
"Without confirm=true, a warning is returned first.",
|
||
parameters: taskAddDepParams,
|
||
execute: async (_id: string, params: Static<typeof taskAddDepParams>) => {
|
||
const targetId = params.task_id;
|
||
|
||
// Prevent self-dependency
|
||
if (targetId === taskId) {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Cannot add self-dependency: ${taskId} cannot depend on itself.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
// Validate target task exists
|
||
try {
|
||
await store.getTask(targetId);
|
||
} catch {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Task ${targetId} not found. Cannot add dependency on a non-existent task.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
// Read current task to get existing dependencies
|
||
const currentTask = await store.getTask(taskId);
|
||
const existing = currentTask.dependencies;
|
||
|
||
// Dedup check
|
||
if (existing.includes(targetId)) {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `${targetId} is already a dependency of ${taskId}. No changes made.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
// Confirmation gate — destructive action for in-progress tasks
|
||
if (!params.confirm) {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Warning: adding a dependency to an in-progress task will stop execution and discard current work. Call with confirm=true to proceed.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
// Add the dependency
|
||
await store.updateTask(taskId, { dependencies: [...existing, targetId] });
|
||
await store.logEntry(taskId, `Added dependency on ${targetId} — stopping execution for re-planning`);
|
||
|
||
// Trigger abort flow (same pattern as pausedAborted)
|
||
this.depAborted.add(taskId);
|
||
const activeSession = this.activeSessions.get(taskId);
|
||
activeSession?.session.dispose();
|
||
|
||
// Also terminate step sessions if active
|
||
const stepExecutor = this.activeStepExecutors.get(taskId);
|
||
if (stepExecutor) {
|
||
stepExecutor.terminateAllSessions().catch(err =>
|
||
executorLog.warn(`Failed to terminate step sessions for dep-abort ${taskId}: ${err}`)
|
||
);
|
||
}
|
||
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Added dependency on ${targetId}. Stopping execution — task will move to triage for re-planning.`,
|
||
}],
|
||
details: {},
|
||
};
|
||
},
|
||
};
|
||
}
|
||
|
||
private createTaskDoneTool(taskId: string, onDone: () => void): ToolDefinition {
|
||
const store = this.store;
|
||
return {
|
||
name: "fn_task_done",
|
||
label: "Mark Task Done",
|
||
description:
|
||
"Signal that all steps are complete, tests pass, and documentation is updated. " +
|
||
"Call this as the final action after finishing all work. " +
|
||
"Automatically marks all remaining steps as done. " +
|
||
"Optionally provide a summary of what was changed/fixed.",
|
||
parameters: Type.Object({
|
||
summary: Type.Optional(Type.String({
|
||
description: "Optional summary of what was changed/fixed and what was verified (2-4 sentences)",
|
||
})),
|
||
}),
|
||
execute: async (_id: string, params: { summary?: string }) => {
|
||
const task = await store.getTask(taskId);
|
||
const completionBlocker = await this.getTaskCompletionBlocker(task);
|
||
if (completionBlocker) {
|
||
return {
|
||
content: [{
|
||
type: "text" as const,
|
||
text: `Cannot mark task done yet — ${completionBlocker}. Resolve the blocker before calling fn_task_done().`,
|
||
}],
|
||
details: {},
|
||
};
|
||
}
|
||
|
||
onDone();
|
||
|
||
// Mark all pending/in-progress steps as done
|
||
for (let i = 0; i < task.steps.length; i++) {
|
||
if (task.steps[i].status !== "done" && task.steps[i].status !== "skipped") {
|
||
await store.updateStep(taskId, i, "done");
|
||
}
|
||
}
|
||
// Save summary if provided
|
||
if (params.summary) {
|
||
await store.updateTask(taskId, { summary: params.summary });
|
||
}
|
||
const settings = await store.getSettings();
|
||
const hardPauseActive = Boolean(task.paused || settings.globalPause);
|
||
if (hardPauseActive) {
|
||
await store.updateTask(taskId, { status: null });
|
||
} else {
|
||
await store.updateTask(taskId, { paused: false, status: null });
|
||
}
|
||
await store.logEntry(taskId, "Task marked done by agent");
|
||
|
||
const latestTask = await store.getTask(taskId);
|
||
let latestColumn = latestTask.column;
|
||
if (latestColumn === "todo") {
|
||
await store.logEntry(
|
||
taskId,
|
||
hardPauseActive
|
||
? "fn_task_done called while task was in todo during pause — promoting to in-progress for deferred completion handoff"
|
||
: "fn_task_done called while task was in todo — promoting to in-progress before completion handoff",
|
||
);
|
||
await store.moveTask(taskId, "in-progress");
|
||
latestColumn = "in-progress";
|
||
}
|
||
|
||
if (latestColumn === "in-progress" && !hardPauseActive) {
|
||
this.scheduleCompletedTaskWatchdog(taskId, "fn_task_done");
|
||
}
|
||
|
||
const successMessage = hardPauseActive
|
||
? "Task marked complete. Completion handoff deferred until pause is cleared."
|
||
: params.summary
|
||
? "Task marked complete with summary. All steps done. Moving to in-review."
|
||
: "Task marked complete. All steps done. Moving to in-review.";
|
||
return {
|
||
content: [{ type: "text" as const, text: successMessage }],
|
||
details: {},
|
||
};
|
||
},
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Create the fn_review_step tool for the executor agent.
|
||
*
|
||
* When the reviewer returns a RETHINK verdict, this tool:
|
||
* 1. Runs `git reset --hard <baseline>` to revert file changes
|
||
* 2. Rewinds the conversation to the pre-step checkpoint via `session.navigateTree()`
|
||
* 3. Resets the step status to "pending"
|
||
* 4. Returns a re-prompt instructing the agent to take a different approach
|
||
*/
|
||
private createReviewStepTool(
|
||
taskId: string,
|
||
worktreePath: string,
|
||
promptContent: string,
|
||
codeReviewVerdicts: Map<number, ReviewVerdict>,
|
||
sessionRef: { current: AgentSession | null },
|
||
stepCheckpoints: Map<number, string>,
|
||
detail: TaskDetail,
|
||
stuckDetector?: StuckTaskDetector,
|
||
): ToolDefinition {
|
||
const store = this.store;
|
||
const options = this.options;
|
||
|
||
return {
|
||
name: "fn_review_step",
|
||
label: "Review Step",
|
||
description:
|
||
"Spawn a reviewer agent to evaluate your plan or code for a step. " +
|
||
"Returns APPROVE, REVISE, RETHINK, or UNAVAILABLE. " +
|
||
"Call at step boundaries based on the task's review level. " +
|
||
"Skip reviews for Step 0 (Preflight) and the final documentation step.",
|
||
parameters: reviewStepParams,
|
||
execute: async (_toolCallId: string, params: Static<typeof reviewStepParams>) => {
|
||
const { step, type: reviewType, step_name, baseline } = params;
|
||
|
||
reviewerLog.log(`${taskId}: ${reviewType} review for Step ${step} (${step_name})`);
|
||
await store.logEntry(taskId, `${reviewType} review requested for Step ${step} (${step_name})`);
|
||
|
||
try {
|
||
const settings = await store.getSettings();
|
||
// Run the reviewer via semaphore.runNested so its slot accounting
|
||
// is honest: activeCount transiently bumps to reflect the second
|
||
// agent session, but the reviewer doesn't enter the wait queue
|
||
// (avoiding a fairness regression where unrelated work could
|
||
// overtake this task at low maxConcurrent). The parent (this
|
||
// executor) makes no LLM calls while suspended awaiting the tool
|
||
// result, so the soft breach of `limit` does not push real
|
||
// LLM-active concurrency above the configured cap.
|
||
const sem = options.semaphore;
|
||
const invokeReviewer = () => reviewStep(
|
||
worktreePath, taskId, step, step_name,
|
||
reviewType, promptContent, baseline,
|
||
{
|
||
onText: (delta) => options.onAgentText?.(taskId, delta),
|
||
// Execution defaults as final fallback
|
||
defaultProvider: settings.defaultProvider,
|
||
defaultModelId: settings.defaultModelId,
|
||
fallbackProvider: settings.fallbackProvider,
|
||
fallbackModelId: settings.fallbackModelId,
|
||
defaultThinkingLevel: detail.thinkingLevel ?? settings.defaultThinkingLevel,
|
||
// Task-level validator override (from task)
|
||
taskValidatorProvider: detail.validatorModelProvider,
|
||
taskValidatorModelId: detail.validatorModelId,
|
||
// Project-level validator override
|
||
projectValidatorProvider: settings.validatorProvider,
|
||
projectValidatorModelId: settings.validatorModelId,
|
||
// Project-level validator fallback
|
||
projectValidatorFallbackProvider: settings.validatorFallbackProvider,
|
||
projectValidatorFallbackModelId: settings.validatorFallbackModelId,
|
||
// Global validator lane
|
||
globalValidatorProvider: settings.validatorGlobalProvider,
|
||
globalValidatorModelId: settings.validatorGlobalModelId,
|
||
// Project-level default override (fallback before execution defaults)
|
||
projectDefaultOverrideProvider: settings.defaultProviderOverride,
|
||
projectDefaultOverrideModelId: settings.defaultModelIdOverride,
|
||
store,
|
||
taskId,
|
||
task: detail,
|
||
agentPrompts: settings.agentPrompts,
|
||
agentStore: this.options.agentStore,
|
||
rootDir: this.rootDir,
|
||
settings,
|
||
// Track the reviewer's session under this task so it's disposed
|
||
// alongside the main session when the task moves out of
|
||
// in-progress, is paused, or the engine globally pauses.
|
||
onSessionCreated: (s) => this.registerSubagentSession(taskId, s),
|
||
onSessionEnded: (s) => this.unregisterSubagentSession(taskId, s),
|
||
},
|
||
);
|
||
const result = sem
|
||
? await sem.runNested(invokeReviewer)
|
||
: await invokeReviewer();
|
||
|
||
await store.logEntry(
|
||
taskId,
|
||
`${reviewType} review Step ${step}: ${result.verdict}`,
|
||
result.summary,
|
||
);
|
||
reviewerLog.log(`${taskId}: Step ${step} ${reviewType} → ${result.verdict}`);
|
||
stuckDetector?.recordProgress(taskId);
|
||
|
||
// Track code review verdicts for enforcement. Plan reviews remain
|
||
// advisory — only code reviews write to the verdict map.
|
||
if (reviewType === "code") {
|
||
if (result.verdict === "REVISE") {
|
||
codeReviewVerdicts.set(step, "REVISE");
|
||
} else if (result.verdict === "APPROVE") {
|
||
codeReviewVerdicts.delete(step);
|
||
}
|
||
}
|
||
|
||
let text: string;
|
||
switch (result.verdict) {
|
||
case "APPROVE": text = "APPROVE"; break;
|
||
case "REVISE":
|
||
if (reviewType === "code") {
|
||
text = `REVISE — this step cannot be marked done until the code review passes.\n\n` +
|
||
`Fix the issues below, commit your changes, and call fn_review_step(step=${step}, ` +
|
||
`type="code", step_name="${step_name}", baseline="<new SHA>") again.\n\n${result.review}`;
|
||
} else {
|
||
text = `REVISE\n\n${result.review}`;
|
||
}
|
||
break;
|
||
case "RETHINK": {
|
||
// For code reviews: git reset to baseline to revert file changes
|
||
// For plan reviews: skip git reset (no code has been written yet)
|
||
if (reviewType === "code" && baseline) {
|
||
try {
|
||
await execAsync(`git reset --hard ${baseline}`, { cwd: worktreePath });
|
||
executorLog.log(`${taskId}: RETHINK — git reset --hard ${baseline}`);
|
||
} catch (gitErr: unknown) {
|
||
const gitErrMessage = gitErr instanceof Error ? gitErr.message : String(gitErr);
|
||
executorLog.error(`${taskId}: RETHINK git reset failed: ${gitErrMessage}`);
|
||
}
|
||
} else if (reviewType === "code") {
|
||
executorLog.log(`${taskId}: RETHINK — no baseline SHA, skipping git reset`);
|
||
}
|
||
|
||
// Rewind conversation to pre-step checkpoint
|
||
const checkpointId = stepCheckpoints.get(step);
|
||
if (checkpointId && sessionRef.current) {
|
||
try {
|
||
await sessionRef.current.navigateTree(checkpointId, { summarize: false });
|
||
executorLog.log(`${taskId}: RETHINK — session rewound to checkpoint ${checkpointId}`);
|
||
} catch (rewindErr: unknown) {
|
||
const msg = rewindErr instanceof Error ? rewindErr.message : String(rewindErr);
|
||
executorLog.warn(`${taskId}: RETHINK navigateTree rewind failed, falling back to branchWithSummary: ${msg}`);
|
||
// Fallback to branchWithSummary
|
||
try {
|
||
sessionRef.current.sessionManager.branchWithSummary(
|
||
checkpointId,
|
||
`RETHINK: ${result.summary || "Approach rejected by reviewer"}`,
|
||
);
|
||
executorLog.log(`${taskId}: RETHINK — branched from checkpoint ${checkpointId}`);
|
||
} catch (branchErr: unknown) {
|
||
const branchErrMessage = branchErr instanceof Error ? branchErr.message : String(branchErr);
|
||
executorLog.error(`${taskId}: RETHINK session rewind failed: ${branchErrMessage}`);
|
||
}
|
||
}
|
||
} else {
|
||
executorLog.log(`${taskId}: RETHINK — no session checkpoint for step ${step}, skipping rewind`);
|
||
}
|
||
|
||
// Reset step status to pending
|
||
await store.updateStep(taskId, step, "pending");
|
||
|
||
if (reviewType === "plan") {
|
||
await store.logEntry(
|
||
taskId,
|
||
`RETHINK: Step ${step} plan rewound — session checkpoint ${checkpointId || "N/A"}`,
|
||
result.summary,
|
||
);
|
||
text = `RETHINK\n\nYour plan was rejected. Here is why:\n\n${result.review}\n\nTake a different approach to planning this step. Do NOT repeat the rejected strategy.`;
|
||
} else {
|
||
await store.logEntry(
|
||
taskId,
|
||
`RETHINK: Step ${step} rewound — git reset to ${baseline || "N/A"}, session checkpoint ${checkpointId || "N/A"}`,
|
||
result.summary,
|
||
);
|
||
text = `RETHINK\n\nYour previous approach was rejected. Here is why:\n\n${result.review}\n\nTake a different approach. Do NOT repeat the rejected strategy. Re-read the step requirements and find an alternative solution.`;
|
||
}
|
||
break;
|
||
}
|
||
default: text = "UNAVAILABLE — reviewer did not produce a usable verdict.";
|
||
}
|
||
|
||
return { content: [{ type: "text" as const, text }], details: {} };
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
reviewerLog.error(`${taskId}: review failed: ${errorMessage}`);
|
||
await store.logEntry(taskId, `${reviewType} review failed: ${errorMessage}`);
|
||
return {
|
||
content: [{ type: "text" as const, text: `UNAVAILABLE — reviewer error: ${errorMessage}` }],
|
||
details: {},
|
||
};
|
||
}
|
||
},
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Clean up after a dep-abort: remove worktree, delete branch, move task to triage.
|
||
* Shared between the try-block (graceful return) and catch-block (error) paths.
|
||
*/
|
||
private async handleDepAbortCleanup(taskId: string, worktreePath: string): Promise<void> {
|
||
executorLog.log(`${taskId} dependency added — work discarded, moved to triage for re-planning`);
|
||
|
||
// Remove worktree
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: failed to remove worktree during dep-abort cleanup (${worktreePath}): ${msg}`);
|
||
}
|
||
|
||
// Delete the branch — use stored branch name if available, fall back to convention
|
||
const task = await this.store.getTask(taskId);
|
||
const branch = task.branch || `fusion/${taskId.toLowerCase()}`;
|
||
let branchDeleted = false;
|
||
try {
|
||
await execAsync(`git branch -D "${branch}"`, { cwd: this.rootDir });
|
||
branchDeleted = true;
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: failed to delete branch during dep-abort cleanup (${branch}): ${msg}`);
|
||
}
|
||
if (branchDeleted) {
|
||
// FN-2165 regression guard: null baseBranch on any task that stored this branch
|
||
try { this.store.clearStaleBaseBranchReferences([branch], taskId); } catch { /* best-effort */ }
|
||
}
|
||
|
||
// Clear worktree tracking
|
||
this.activeWorktrees.delete(taskId);
|
||
|
||
// Update task: clear worktree and status, move to triage
|
||
await this.store.updateTask(taskId, { worktree: null, status: null });
|
||
await this.store.moveTask(taskId, "triage");
|
||
await this.store.logEntry(taskId, "Execution stopped — work discarded, moved to triage for re-planning");
|
||
}
|
||
|
||
/**
|
||
* Handle a workflow step revision request.
|
||
*
|
||
* Re-opens ONLY the last step so the executor has exactly one pending slot
|
||
* to re-enter through. All earlier done steps stay done — the agent reads
|
||
* the injected feedback from PROMPT.md and applies an in-place fix rather
|
||
* than redoing any completed step.
|
||
*/
|
||
private async handleWorkflowRevisionRequest(
|
||
task: Task,
|
||
worktreePath: string,
|
||
feedback: string,
|
||
stepName: string,
|
||
): Promise<void> {
|
||
executorLog.log(`${task.id}: workflow revision requested by step "${stepName}"`);
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
|
||
const updatedTask = await this.store.getTask(task.id);
|
||
const reopen = await this.reopenLastStepForRevision(task.id, updatedTask);
|
||
const reopenSummary = reopen
|
||
? `re-opening Step ${reopen.index + 1} ("${reopen.name}") for in-place fix`
|
||
: "no step to re-open (none were completed)";
|
||
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Workflow step "${stepName}" requested revision — ${reopenSummary}`,
|
||
feedback,
|
||
);
|
||
|
||
await this.injectWorkflowRevisionInstructions(task, feedback);
|
||
|
||
await this.store.updateTask(task.id, {
|
||
status: null,
|
||
sessionFile: null,
|
||
});
|
||
|
||
// 4. Schedule fresh execution after guard unwinds
|
||
// This prevents the race condition where the scheduler re-dispatches
|
||
// while the old execution guard is still set.
|
||
executorLog.log(`${task.id}: scheduling fresh execution after revision request`);
|
||
this.scheduleWorkflowRerun(
|
||
task.id,
|
||
worktreePath,
|
||
`${task.id}: revision rerun scheduled — moved to todo then in-progress`,
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Re-open the last non-pending step so a revision/failure handler gives the
|
||
* executor exactly one pending slot to re-enter through. Returns the index
|
||
* and name of the step that was flipped to `pending`, or null when there
|
||
* was nothing to re-open.
|
||
*/
|
||
private async reopenLastStepForRevision(
|
||
taskId: string,
|
||
task: Task,
|
||
): Promise<{ index: number; name: string } | null> {
|
||
const steps = task.steps;
|
||
if (steps.length === 0) return null;
|
||
|
||
let targetIndex = -1;
|
||
for (let i = steps.length - 1; i >= 0; i--) {
|
||
if (steps[i].status !== "pending") {
|
||
targetIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (targetIndex === -1) {
|
||
await this.store.updateTask(taskId, { currentStep: 0 });
|
||
return null;
|
||
}
|
||
|
||
await this.store.updateStep(taskId, targetIndex, "pending");
|
||
await this.store.updateTask(taskId, { currentStep: targetIndex });
|
||
return { index: targetIndex, name: steps[targetIndex].name };
|
||
}
|
||
|
||
/**
|
||
* Inject or update the "Workflow Revision Instructions" section in PROMPT.md.
|
||
* This section contains feedback from workflow steps that requested revisions.
|
||
* The section is replaced entirely to avoid accumulation of old feedback.
|
||
*/
|
||
private async injectWorkflowRevisionInstructions(
|
||
task: Task,
|
||
feedback: string,
|
||
): Promise<void> {
|
||
const promptPath = join(this.store.getFusionDir(), "tasks", task.id, "PROMPT.md");
|
||
|
||
// Read existing PROMPT.md
|
||
let content: string;
|
||
try {
|
||
content = await readFile(promptPath, "utf-8");
|
||
} catch {
|
||
executorLog.warn(`${task.id}: PROMPT.md not found at ${promptPath}, skipping revision injection`);
|
||
return;
|
||
}
|
||
|
||
// All prior steps stay done — agent applies the feedback as an in-place
|
||
// patch rather than re-planning or re-executing earlier steps.
|
||
const scopeLine = "All prior steps remain **done**. Apply the feedback above as an in-place fix (make the necessary code changes, commit, and call `fn_task_done()` when complete). Do **not** re-run or re-plan any earlier step unless the feedback explicitly calls it out.";
|
||
|
||
// Check for existing Workflow Revision Instructions section
|
||
const revisionSectionHeader = "## Workflow Revision Instructions";
|
||
const revisionSectionContent = `${revisionSectionHeader}
|
||
|
||
The following feedback was received from quality gates and requires implementation changes:
|
||
|
||
${feedback}
|
||
|
||
**Important:** ${scopeLine}
|
||
|
||
`;
|
||
|
||
let newContent: string;
|
||
if (content.includes(revisionSectionHeader)) {
|
||
// Replace existing section
|
||
const sectionRegex = new RegExp(
|
||
`${revisionSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`,
|
||
"i"
|
||
);
|
||
if (sectionRegex.test(content)) {
|
||
newContent = content.replace(sectionRegex, revisionSectionContent);
|
||
} else {
|
||
// Fallback: append at end
|
||
newContent = content + "\n" + revisionSectionContent;
|
||
}
|
||
} else {
|
||
// Append new section before any closing markers or at end
|
||
// Look for common markers like "## Acceptance Criteria" or just append
|
||
const acceptanceCriteriaMatch = content.match(/\n##\s+Acceptance Criteria\n/);
|
||
if (acceptanceCriteriaMatch) {
|
||
const insertIdx = acceptanceCriteriaMatch.index!;
|
||
newContent = content.slice(0, insertIdx) + "\n" + revisionSectionContent + content.slice(insertIdx);
|
||
} else {
|
||
newContent = content + "\n" + revisionSectionContent;
|
||
}
|
||
}
|
||
|
||
// Write updated content
|
||
try {
|
||
await writeFile(promptPath, newContent);
|
||
executorLog.log(`${task.id}: injected workflow revision instructions into PROMPT.md`);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`${task.id}: failed to inject revision instructions: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Handle workflow step hard failures by retrying execution up to MAX_WORKFLOW_STEP_RETRIES times.
|
||
* This gives the executor a chance to fix workflow step failures automatically before
|
||
* moving the task to in-review with failed status.
|
||
*
|
||
* @returns true if a retry was scheduled, false if retries are exhausted
|
||
*/
|
||
/**
|
||
* Run deterministic verification (test + build commands) in the task's worktree.
|
||
* Returns a structured result indicating whether all commands passed.
|
||
*/
|
||
private async runExecutorDeterministicVerification(
|
||
task: Task,
|
||
worktreePath: string,
|
||
settings: Settings,
|
||
): Promise<VerificationResult> {
|
||
const testCommand = settings.testCommand?.trim();
|
||
const buildCommand = settings.buildCommand?.trim();
|
||
|
||
if (!testCommand && !buildCommand) {
|
||
executorLog.log(`${task.id}: no test/build commands configured — skipping verification`);
|
||
return { allPassed: true };
|
||
}
|
||
|
||
const parts: string[] = [];
|
||
if (testCommand) parts.push(`test: ${testCommand}`);
|
||
if (buildCommand) parts.push(`build: ${buildCommand}`);
|
||
executorLog.log(`${task.id}: [verification] running deterministic verification (${parts.join(", ")})`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[verification] Running deterministic verification (${parts.join(", ")})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
|
||
const result: VerificationResult = { allPassed: true };
|
||
|
||
// Run test command first if configured
|
||
if (testCommand) {
|
||
const testResult = await runVerificationCommand(
|
||
this.store, worktreePath, task.id, testCommand, "test", undefined, executorLog, "executor",
|
||
);
|
||
result.testResult = testResult;
|
||
|
||
if (!testResult.success) {
|
||
result.allPassed = false;
|
||
result.failedCommand = "testCommand";
|
||
executorLog.log(`${task.id}: [verification] test failed (exit ${testResult.exitCode})`);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
// Run build command second if configured
|
||
if (buildCommand) {
|
||
const buildResult = await runVerificationCommand(
|
||
this.store, worktreePath, task.id, buildCommand, "build", undefined, executorLog, "executor",
|
||
);
|
||
result.buildResult = buildResult;
|
||
|
||
if (!buildResult.success) {
|
||
result.allPassed = false;
|
||
result.failedCommand = "buildCommand";
|
||
executorLog.log(`${task.id}: [verification] build failed (exit ${buildResult.exitCode})`);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
executorLog.log(`${task.id}: [verification] passed`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[verification] Deterministic verification passed`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* Attempt to fix verification failures by spawning a dedicated AI fix agent.
|
||
* Follows the pattern established by the merger's attemptInMergeVerificationFix.
|
||
* Returns true if verification passes after the fix attempt, false otherwise.
|
||
*/
|
||
private async attemptExecutorVerificationFix(
|
||
task: Task,
|
||
worktreePath: string,
|
||
failureContext: {
|
||
command: string;
|
||
exitCode: number | null;
|
||
output: string;
|
||
type: "test" | "build";
|
||
},
|
||
settings: Settings,
|
||
retryNumber: number,
|
||
maxRetries: number,
|
||
): Promise<boolean> {
|
||
try {
|
||
executorLog.log(`${task.id}: spawning executor verification fix agent (attempt ${retryNumber}/${maxRetries})`);
|
||
|
||
const logger = new AgentLogger({
|
||
store: this.store,
|
||
taskId: task.id,
|
||
agent: "executor",
|
||
persistAgentToolOutput: settings.persistAgentToolOutput,
|
||
onAgentText: this.options.onAgentText,
|
||
onAgentTool: this.options.onAgentTool,
|
||
});
|
||
|
||
// Build skill selection context
|
||
let skillContext: Awaited<ReturnType<typeof buildSessionSkillContext>> | undefined;
|
||
if (this.options.agentStore) {
|
||
try {
|
||
skillContext = await buildSessionSkillContext({
|
||
agentStore: this.options.agentStore,
|
||
task,
|
||
sessionPurpose: "executor",
|
||
projectRootDir: worktreePath,
|
||
pluginRunner: this.options.pluginRunner,
|
||
});
|
||
} catch {
|
||
// Graceful fallback - no skill selection
|
||
}
|
||
}
|
||
|
||
// Resolve model using the executor's model hierarchy
|
||
const { provider: executorProvider, modelId: executorModelId } = resolveExecutorModelPair(
|
||
task.modelProvider,
|
||
task.modelId,
|
||
settings,
|
||
);
|
||
|
||
// Create the fix agent session
|
||
const { session } = await createResolvedAgentSession({
|
||
sessionPurpose: "executor",
|
||
pluginRunner: this.options.pluginRunner,
|
||
cwd: worktreePath, // Run in the task's worktree
|
||
systemPrompt: `You are a verification fix agent running during task execution in a worktree.
|
||
|
||
All step-session steps completed successfully but the deterministic verification command failed. Your job is to fix the failing code directly in the working directory.
|
||
|
||
## Scope
|
||
Only fix what is required to make the failing verification pass.
|
||
Do not refactor, rename broadly, or make opportunistic improvements.
|
||
|
||
## Rules
|
||
1. Read the error output carefully to understand what is failing before editing anything
|
||
2. Before assuming a code fix is needed, check whether the failure is caused by stale/missing build artifacts in a sibling workspace package — typical signatures: \`Failed to resolve import "./X.js"\` pointing into another package's \`dist/\`, \`Cannot find module\`, or \`ERR_MODULE_NOT_FOUND\` referencing a workspace-internal path. In that case, rebuild the affected package(s) (e.g. \`pnpm --filter <pkg> build\`, or \`pnpm --filter "<scope>/*" build\` for a group) and re-run verification before editing source files.
|
||
3. Make targeted fixes to the failing code path
|
||
4. After fixing, run the verification command to confirm the fix works
|
||
5. Do NOT make any git commits — just fix the code
|
||
6. You MAY modify any files needed to make the verification pass, including files unrelated to this task's original change. Pre-existing build/test breakage is in scope: fix it. Prefer the smallest change that makes verification green.
|
||
7. If you cannot fix the issue within scope, explain why and what evidence indicates a deeper/root problem`,
|
||
tools: "coding",
|
||
onText: logger.onText,
|
||
onThinking: logger.onThinking,
|
||
onToolStart: logger.onToolStart,
|
||
onToolEnd: logger.onToolEnd,
|
||
defaultProvider: executorProvider,
|
||
defaultModelId: executorModelId,
|
||
defaultThinkingLevel: settings.defaultThinkingLevel,
|
||
...(skillContext?.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||
});
|
||
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Executor verification fix agent started (model: ${describeModel(session)}, attempt ${retryNumber}/${maxRetries})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
await this.store.appendAgentLog(
|
||
task.id,
|
||
`Fix agent started (model: ${describeModel(session)}, attempt ${retryNumber}/${maxRetries})`,
|
||
"text",
|
||
undefined,
|
||
"executor",
|
||
);
|
||
|
||
try {
|
||
// Build the fix prompt
|
||
const fixPrompt = `Fix the failing ${failureContext.type} verification for task ${task.id}.
|
||
|
||
## Failed command
|
||
Command: \`${failureContext.command}\`
|
||
Exit code: ${failureContext.exitCode}
|
||
|
||
## Error output
|
||
${failureContext.output.slice(0, VERIFICATION_LOG_MAX_CHARS)}
|
||
|
||
## Instructions
|
||
1. Read the error output and identify the root cause
|
||
2. Make targeted fixes to resolve the failure
|
||
3. Run the verification command \`${failureContext.command}\` to confirm your fix works
|
||
4. If the fix doesn't work, try a different approach
|
||
5. Do NOT make any git commits`;
|
||
|
||
// Run the agent with rate limit retry
|
||
await withRateLimitRetry(async () => {
|
||
await promptWithFallback(session, fixPrompt);
|
||
}, {
|
||
onRetry: (attempt, delayMs, error) => {
|
||
const delaySec = Math.round(delayMs / 1000);
|
||
executorLog.warn(`⏳ ${task.id} executor fix agent rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`);
|
||
},
|
||
});
|
||
await accumulateSessionTokenUsage(this.store, task.id, session);
|
||
|
||
// Re-run full deterministic verification (test AND build) after the fix attempt
|
||
executorLog.log(`${task.id}: re-running deterministic verification after fix attempt ${retryNumber}/${maxRetries}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Re-running deterministic verification (attempt ${retryNumber}/${maxRetries})`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
await this.store.appendAgentLog(
|
||
task.id,
|
||
`Re-running verification (attempt ${retryNumber}/${maxRetries})`,
|
||
"text",
|
||
undefined,
|
||
"executor",
|
||
);
|
||
const reRunResult = await this.runExecutorDeterministicVerification(task, worktreePath, settings);
|
||
|
||
return reRunResult.allPassed;
|
||
} finally {
|
||
await logger.flush();
|
||
await session.dispose();
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id}: executor verification fix agent error: ${errorMessage}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Executor verification fix agent encountered an error`,
|
||
errorMessage,
|
||
this.currentRunContext,
|
||
);
|
||
await this.store.appendAgentLog(
|
||
task.id,
|
||
"Fix agent encountered an error",
|
||
"tool_error",
|
||
errorMessage,
|
||
"executor",
|
||
);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
private async handleWorkflowStepFailure(
|
||
task: Task,
|
||
worktreePath: string,
|
||
failureFeedback: string,
|
||
stepName: string,
|
||
): Promise<boolean> {
|
||
this.clearCompletedTaskWatchdog(task.id);
|
||
const currentRetries = task.workflowStepRetries ?? 0;
|
||
|
||
if (currentRetries >= MAX_WORKFLOW_STEP_RETRIES) {
|
||
// Retries exhausted — caller should fall through to hard failure
|
||
executorLog.warn(`${task.id}: workflow step "${stepName}" failed — retries exhausted (${MAX_WORKFLOW_STEP_RETRIES}/${MAX_WORKFLOW_STEP_RETRIES})`);
|
||
return false;
|
||
}
|
||
|
||
const retryCount = currentRetries + 1;
|
||
executorLog.log(`${task.id}: workflow step "${stepName}" failed — retry ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES} (executor will attempt to fix)`);
|
||
|
||
// 1. Update the workflowStepRetries counter on the task
|
||
await this.store.updateTask(task.id, {
|
||
workflowStepRetries: retryCount,
|
||
});
|
||
|
||
// 2. Inject failure feedback into PROMPT.md
|
||
await this.injectWorkflowStepFailureInstructions(task, failureFeedback, stepName, retryCount);
|
||
|
||
// 3. Re-open only the last step so the executor has a single pending
|
||
// slot to re-enter. Earlier done steps stay done.
|
||
const updatedTask = await this.store.getTask(task.id);
|
||
await this.reopenLastStepForRevision(task.id, updatedTask);
|
||
|
||
// 4. Clear any session file so we get a fresh session
|
||
await this.store.updateTask(task.id, {
|
||
status: null,
|
||
sessionFile: null,
|
||
});
|
||
|
||
// 5. Schedule fresh execution after guard unwinds
|
||
executorLog.log(`${task.id}: scheduling fresh execution after workflow step failure (retry ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES})`);
|
||
this.scheduleWorkflowRerun(
|
||
task.id,
|
||
worktreePath,
|
||
`${task.id}: workflow step retry scheduled — moved to todo then in-progress`,
|
||
);
|
||
|
||
return true;
|
||
}
|
||
|
||
/**
|
||
* Send a task back to in-progress after verification failure.
|
||
* Injects failure feedback into PROMPT.md, resets steps, clears session,
|
||
* and schedules a move to todo → in-progress after the executing guard clears.
|
||
*/
|
||
private async sendTaskBackForFix(
|
||
task: Task,
|
||
worktreePath: string,
|
||
failureFeedback: string,
|
||
stepName: string,
|
||
reason: string,
|
||
preserveResumeState: boolean = true,
|
||
mergeVerificationFailure: boolean = false,
|
||
): Promise<void> {
|
||
const taskId = task.id;
|
||
this.clearCompletedTaskWatchdog(taskId);
|
||
|
||
// 1. Add a task comment explaining the failure
|
||
await this.store.addTaskComment(
|
||
taskId,
|
||
`${reason}. The failing workflow step was "${stepName}". ` +
|
||
`Feedback:\n${failureFeedback}\n\n` +
|
||
`Please fix the issues so the verification can pass on the next attempt.`,
|
||
"agent",
|
||
);
|
||
|
||
// 2. Log an entry explaining the task was sent back
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`${reason} — moved back to in-progress for remediation`,
|
||
);
|
||
|
||
// 3. Inject failure feedback into PROMPT.md using the existing method
|
||
// Pass MAX_WORKFLOW_STEP_RETRIES to indicate retries are exhausted (shows "3/3 (0 remaining)")
|
||
await this.injectWorkflowStepFailureInstructions(task, failureFeedback, stepName, MAX_WORKFLOW_STEP_RETRIES);
|
||
|
||
// 4. Re-open only the last step for a single in-place fix pass. Earlier
|
||
// done steps stay done so the executor doesn't redo finished work.
|
||
const updatedTask = await this.store.getTask(taskId);
|
||
await this.reopenLastStepForRevision(taskId, updatedTask);
|
||
|
||
// 5. Clear error/status/session fields and reset workflow step retries
|
||
await this.store.updateTask(taskId, {
|
||
status: mergeVerificationFailure ? "merging-fix" : null,
|
||
error: null,
|
||
sessionFile: null,
|
||
workflowStepRetries: 0,
|
||
});
|
||
|
||
// 6. Schedule the move after the guard unwinds (per guard-unwind requirement)
|
||
this.scheduleWorkflowRerun(
|
||
taskId,
|
||
worktreePath,
|
||
`${taskId}: sent back to in-progress for remediation`,
|
||
preserveResumeState,
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Inject or update the "Workflow Step Failure" section in PROMPT.md.
|
||
* This section contains failure feedback from workflow steps that hard-failed.
|
||
* The section is replaced entirely to avoid accumulation of old feedback.
|
||
*/
|
||
private async injectWorkflowStepFailureInstructions(
|
||
task: Task,
|
||
failureFeedback: string,
|
||
stepName: string,
|
||
retryCount: number,
|
||
): Promise<void> {
|
||
const promptPath = join(this.store.getFusionDir(), "tasks", task.id, "PROMPT.md");
|
||
|
||
// Read existing PROMPT.md
|
||
let content: string;
|
||
try {
|
||
content = await readFile(promptPath, "utf-8");
|
||
} catch {
|
||
executorLog.warn(`${task.id}: PROMPT.md not found at ${promptPath}, skipping workflow failure injection`);
|
||
return;
|
||
}
|
||
|
||
const remainingRetries = MAX_WORKFLOW_STEP_RETRIES - retryCount;
|
||
const failureSectionHeader = "## Workflow Step Failure";
|
||
const failureSectionContent = `${failureSectionHeader}
|
||
|
||
The following workflow step failed and requires implementation fixes:
|
||
|
||
**Step:** ${stepName}
|
||
|
||
**Failure Feedback:**
|
||
${failureFeedback}
|
||
|
||
**Retry:** ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES} (${remainingRetries} remaining)
|
||
|
||
**Important:** This is a workflow step failure — fix the issues above by making the necessary code changes. The task has been sent back to in-progress for remediation. The executor will attempt to fix the issues on the next pass.
|
||
|
||
`;
|
||
|
||
let newContent: string;
|
||
if (content.includes(failureSectionHeader)) {
|
||
// Replace existing section
|
||
const sectionRegex = new RegExp(
|
||
`${failureSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`,
|
||
"i"
|
||
);
|
||
if (sectionRegex.test(content)) {
|
||
newContent = content.replace(sectionRegex, failureSectionContent);
|
||
} else {
|
||
// Fallback: append at end
|
||
newContent = content + "\n" + failureSectionContent;
|
||
}
|
||
} else {
|
||
// Remove any existing Workflow Revision Instructions section first (conflicting state)
|
||
const revisionSectionHeader = "## Workflow Revision Instructions";
|
||
if (content.includes(revisionSectionHeader)) {
|
||
const revisionRegex = new RegExp(
|
||
`${revisionSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`,
|
||
"i"
|
||
);
|
||
content = content.replace(revisionRegex, "");
|
||
}
|
||
|
||
// Append new section before any closing markers or at end
|
||
const acceptanceCriteriaMatch = content.match(/\n##\s+Acceptance Criteria\n/);
|
||
if (acceptanceCriteriaMatch) {
|
||
const insertIdx = acceptanceCriteriaMatch.index!;
|
||
newContent = content.slice(0, insertIdx) + "\n" + failureSectionContent + content.slice(insertIdx);
|
||
} else {
|
||
newContent = content + "\n" + failureSectionContent;
|
||
}
|
||
}
|
||
|
||
// Write updated content
|
||
try {
|
||
await writeFile(promptPath, newContent);
|
||
executorLog.log(`${task.id}: injected workflow step failure instructions into PROMPT.md (retry ${retryCount}/${MAX_WORKFLOW_STEP_RETRIES})`);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`${task.id}: failed to inject workflow step failure instructions: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Capture the list of files modified during agent execution.
|
||
* Uses git diff against the stored baseCommitSha to determine what changed.
|
||
* Returns an empty array if no changes or if git commands fail.
|
||
*/
|
||
private async resolveDiffBaseRef(worktreePath: string, baseCommitSha?: string): Promise<string | undefined> {
|
||
if (baseCommitSha) return baseCommitSha;
|
||
|
||
try {
|
||
const { stdout } = await execAsync(
|
||
"git merge-base HEAD origin/main 2>/dev/null || git merge-base HEAD main",
|
||
{ cwd: worktreePath, encoding: "utf-8" },
|
||
);
|
||
const ref = stdout.trim();
|
||
if (ref) return ref;
|
||
} catch (mergeBaseErr: unknown) {
|
||
const mergeBaseMsg = mergeBaseErr instanceof Error ? mergeBaseErr.message : String(mergeBaseErr);
|
||
executorLog.warn(`Failed merge-base lookup for diff base in ${worktreePath}, trying HEAD~1 fallback: ${mergeBaseMsg}`);
|
||
}
|
||
|
||
try {
|
||
const { stdout } = await execAsync("git rev-parse HEAD~1", {
|
||
cwd: worktreePath,
|
||
encoding: "utf-8",
|
||
});
|
||
return stdout.trim() || undefined;
|
||
} catch {
|
||
executorLog.log(`Could not determine base commit for diff in ${worktreePath}`);
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
private async captureModifiedFiles(worktreePath: string, baseCommitSha?: string): Promise<string[]> {
|
||
try {
|
||
const baseRef = await this.resolveDiffBaseRef(worktreePath, baseCommitSha);
|
||
if (!baseRef) {
|
||
return [];
|
||
}
|
||
|
||
// Get list of modified files using git diff --name-only
|
||
const { stdout } = await execAsync(`git diff --name-only ${baseRef}..HEAD`, {
|
||
cwd: worktreePath,
|
||
encoding: "utf-8",
|
||
});
|
||
const output = stdout.trim();
|
||
|
||
if (!output) {
|
||
return [];
|
||
}
|
||
|
||
return output.split("\n").filter(Boolean);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.log(`Failed to capture modified files: ${errorMessage}`);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
// ── Worktree management ────────────────────────────────────────────
|
||
|
||
/**
|
||
* Create a git worktree at `path` on a new branch.
|
||
*
|
||
* @param branch — Branch name (e.g., `fusion/fn-042`)
|
||
* @param path — Absolute worktree directory path
|
||
* @param startPoint — Optional git ref to branch from (e.g., `fusion/fn-041`).
|
||
* When provided, the worktree starts from that ref instead of HEAD.
|
||
*/
|
||
/**
|
||
* Run workflow step agents sequentially after main task execution completes.
|
||
* Each workflow step spawns a separate agent with the step's prompt.
|
||
* Returns structured result: all passed, all passed (true), failed (false), or revision requested.
|
||
*/
|
||
private async runWorkflowSteps(
|
||
task: Task,
|
||
worktreePath: string,
|
||
settings: Settings,
|
||
): Promise<WorkflowStepResult | "deferred-paused"> {
|
||
// Check if task has enabled workflow steps
|
||
const currentTask = await this.store.getTask(task.id);
|
||
if (!currentTask.enabledWorkflowSteps?.length) return { allPassed: true };
|
||
|
||
const workflowStepIds = currentTask.enabledWorkflowSteps;
|
||
const results: import("@fusion/core").WorkflowStepResult[] = [];
|
||
|
||
for (const wsId of workflowStepIds) {
|
||
const ws = await this.store.getWorkflowStep(wsId);
|
||
if (!ws) {
|
||
await this.store.logEntry(task.id, `[pre-merge] Workflow step ${wsId} not found — skipping`);
|
||
results.push({
|
||
workflowStepId: wsId,
|
||
workflowStepName: "Unknown",
|
||
phase: "pre-merge",
|
||
status: "skipped",
|
||
output: "Workflow step definition not found",
|
||
});
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
continue;
|
||
}
|
||
|
||
// Normalize legacy steps: undefined phase → "pre-merge"
|
||
const stepPhase = ws.phase || "pre-merge";
|
||
|
||
// readonly review steps always run pre-merge to reuse the coding worktree — see FN-2185 post-mortem.
|
||
// Skip non-readonly post-merge steps — those run in the merger after merge.
|
||
if (stepPhase === "post-merge" && ws.toolMode !== "readonly") continue;
|
||
|
||
// Normalize legacy steps without mode to prompt-mode
|
||
const stepMode: "prompt" | "script" = ws.mode || "prompt";
|
||
|
||
// Skip validation per mode
|
||
if (stepMode === "prompt" && !ws.prompt?.trim()) {
|
||
await this.store.logEntry(task.id, `[pre-merge] Workflow step '${ws.name}' has no prompt — skipping`);
|
||
results.push({
|
||
workflowStepId: ws.id,
|
||
workflowStepName: ws.name,
|
||
phase: stepPhase,
|
||
status: "skipped",
|
||
output: "No prompt configured for this workflow step",
|
||
});
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
continue;
|
||
}
|
||
|
||
if (stepMode === "script" && !ws.scriptName?.trim()) {
|
||
await this.store.logEntry(task.id, `[pre-merge] Workflow step '${ws.name}' has no scriptName — skipping`);
|
||
results.push({
|
||
workflowStepId: ws.id,
|
||
workflowStepName: ws.name,
|
||
phase: stepPhase,
|
||
status: "skipped",
|
||
output: "No scriptName configured for this workflow step",
|
||
});
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
continue;
|
||
}
|
||
|
||
if (await this.shouldDeferWorkflowStepCompletion(task.id, `before workflow step '${ws.name}'`)) {
|
||
return "deferred-paused";
|
||
}
|
||
|
||
await this.store.logEntry(task.id, `[pre-merge] Starting workflow step: ${ws.name} (${stepMode} mode)`);
|
||
executorLog.log(`${task.id} — [pre-merge] running workflow step: ${ws.name} (${stepMode} mode)`);
|
||
|
||
const startedAt = new Date().toISOString();
|
||
const stepStartedAtMs = Date.now();
|
||
|
||
// Push pending entry BEFORE execution so dashboard can show live status
|
||
results.push({
|
||
workflowStepId: ws.id,
|
||
workflowStepName: ws.name,
|
||
phase: stepPhase,
|
||
status: "pending",
|
||
startedAt,
|
||
});
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
|
||
try {
|
||
const result: WorkflowStepOutcome = stepMode === "script"
|
||
? await this.executeScriptWorkflowStep(task, ws, worktreePath, settings)
|
||
: await this.executeWorkflowStep(task, ws, worktreePath, settings);
|
||
if (await this.shouldDeferWorkflowStepCompletion(task.id, `workflow step '${ws.name}'`)) {
|
||
return "deferred-paused";
|
||
}
|
||
const completedAt = new Date().toISOString();
|
||
|
||
if (result.success) {
|
||
await this.store.logEntry(task.id, `[timing] Workflow step '${ws.name}' completed in ${Date.now() - stepStartedAtMs}ms`);
|
||
await this.store.logEntry(task.id, `[pre-merge] Workflow step completed: ${ws.name}`);
|
||
executorLog.log(`${task.id} — [pre-merge] workflow step passed: ${ws.name}`);
|
||
// Update existing pending entry in place
|
||
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
|
||
if (existingIdx >= 0) {
|
||
results[existingIdx] = {
|
||
...results[existingIdx],
|
||
status: "passed",
|
||
output: result.output,
|
||
completedAt,
|
||
};
|
||
}
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
} else if (result.revisionRequested) {
|
||
// Revision requested — this is a structured outcome that routes back to executor
|
||
await this.store.logEntry(task.id, `[timing] Workflow step '${ws.name}' requested revision after ${Date.now() - stepStartedAtMs}ms`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[pre-merge] Workflow step requested revision: ${ws.name}`,
|
||
result.output,
|
||
);
|
||
executorLog.log(`${task.id} — [pre-merge] workflow step requested revision: ${ws.name}`);
|
||
// Update existing pending entry in place
|
||
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
|
||
if (existingIdx >= 0) {
|
||
results[existingIdx] = {
|
||
...results[existingIdx],
|
||
status: "failed",
|
||
output: result.output || "Revision requested",
|
||
completedAt,
|
||
};
|
||
}
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
return {
|
||
allPassed: false,
|
||
revisionRequested: true,
|
||
feedback: result.output || "Workflow step requested revision",
|
||
stepName: ws.name,
|
||
};
|
||
} else {
|
||
// Hard failure
|
||
await this.store.logEntry(task.id, `[timing] Workflow step '${ws.name}' failed after ${Date.now() - stepStartedAtMs}ms`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[pre-merge] Workflow step failed: ${ws.name}`,
|
||
result.error || "Unknown error",
|
||
);
|
||
executorLog.error(`${task.id} — [pre-merge] workflow step failed: ${ws.name}; output captured in task log`);
|
||
// Update existing pending entry in place
|
||
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
|
||
if (existingIdx >= 0) {
|
||
results[existingIdx] = {
|
||
...results[existingIdx],
|
||
status: "failed",
|
||
output: result.error || "Workflow step failed",
|
||
completedAt,
|
||
};
|
||
}
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
return {
|
||
allPassed: false,
|
||
revisionRequested: false,
|
||
feedback: result.error || "Workflow step failed",
|
||
stepName: ws.name,
|
||
};
|
||
}
|
||
} catch (err: unknown) {
|
||
if (await this.shouldDeferWorkflowStepCompletion(task.id, `workflow step '${ws.name}'`)) {
|
||
return "deferred-paused";
|
||
}
|
||
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
||
const completedAt = new Date().toISOString();
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[pre-merge] Workflow step failed: ${ws.name}`,
|
||
errorStack ?? errorDetail,
|
||
);
|
||
executorLog.error(`${task.id} — [pre-merge] workflow step error: ${ws.name} — ${errorDetail}`);
|
||
// Update existing pending entry in place
|
||
const existingIdx = results.findIndex(r => r.workflowStepId === ws.id);
|
||
if (existingIdx >= 0) {
|
||
results[existingIdx] = {
|
||
...results[existingIdx],
|
||
status: "failed",
|
||
output: errorMessage || "Workflow step error",
|
||
completedAt,
|
||
};
|
||
}
|
||
await this.store.updateTask(task.id, { workflowStepResults: results });
|
||
return {
|
||
allPassed: false,
|
||
revisionRequested: false,
|
||
feedback: errorMessage || "Workflow step error",
|
||
stepName: ws.name,
|
||
};
|
||
}
|
||
}
|
||
|
||
return { allPassed: true };
|
||
}
|
||
|
||
/**
|
||
* Execute a script-mode workflow step by resolving the scriptName to a command
|
||
* from project settings and running it in the task worktree.
|
||
*/
|
||
private async executeScriptWorkflowStep(
|
||
task: Task,
|
||
workflowStep: WorkflowStep,
|
||
worktreePath: string,
|
||
settings: Settings,
|
||
): Promise<{ success: boolean; output?: string; error?: string }> {
|
||
const scriptName = workflowStep.scriptName!.trim();
|
||
const scriptCommand = settings.scripts?.[scriptName];
|
||
|
||
if (!scriptCommand) {
|
||
const available = settings.scripts ? Object.keys(settings.scripts).join(", ") : "none";
|
||
const msg = `Script '${scriptName}' not found in project settings. Available scripts: ${available}`;
|
||
await this.store.logEntry(task.id, msg);
|
||
return { success: false, error: msg };
|
||
}
|
||
|
||
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' executing script '${scriptName}': ${scriptCommand}`);
|
||
await this.store.logEntry(task.id, `Workflow step '${workflowStep.name}' executing script '${scriptName}': ${scriptCommand}`);
|
||
|
||
try {
|
||
const scriptResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000);
|
||
if (scriptResult.spawnError || scriptResult.timedOut || scriptResult.exitCode !== 0) {
|
||
return { success: false, error: configuredCommandErrorMessage(scriptResult) };
|
||
}
|
||
return { success: true, output: `Script '${scriptName}' completed successfully` };
|
||
} catch (err: unknown) {
|
||
const execError = err instanceof Error ? err : new Error(String(err));
|
||
const stderr = "stderr" in execError && typeof execError.stderr === "string" ? execError.stderr.trim() : "";
|
||
const stdout = "stdout" in execError && typeof execError.stdout === "string" ? execError.stdout.trim() : "";
|
||
const exitCode = "code" in execError ? execError.code : ("status" in execError ? execError.status : undefined);
|
||
const parts: string[] = [];
|
||
if (exitCode !== undefined) parts.push(`Exit code: ${exitCode}`);
|
||
if (stdout) parts.push(`stdout: ${truncateWorkflowScriptOutput(stdout)}`);
|
||
if (stderr) parts.push(`stderr: ${truncateWorkflowScriptOutput(stderr)}`);
|
||
if (!parts.length) parts.push(execError.message || "Unknown error");
|
||
const errorOutput = parts.join("\n");
|
||
return { success: false, error: errorOutput };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Execute a single workflow step by spawning an agent with the step's prompt.
|
||
* Returns structured outcome with support for revision requests.
|
||
*/
|
||
private async executeWorkflowStep(
|
||
task: Task,
|
||
workflowStep: WorkflowStep,
|
||
worktreePath: string,
|
||
settings: Settings,
|
||
): Promise<WorkflowStepOutcome> {
|
||
const toolMode: "coding" | "readonly" = workflowStep.toolMode || "readonly";
|
||
|
||
// Compute the diff scope so the workflow step agent reviews only what THIS
|
||
// task changed — not unrelated files it might wander into. Without this,
|
||
// open-ended review prompts (e.g. "verify visual polish") have been
|
||
// observed to spend the entire timeout budget reading pre-existing files
|
||
// that match the task description's keywords. See FN-3327 post-mortem.
|
||
const scopedFiles = await this.captureModifiedFiles(worktreePath, task.baseCommitSha);
|
||
let diffShortstat: string | undefined;
|
||
try {
|
||
const baseRef = await this.resolveDiffBaseRef(worktreePath, task.baseCommitSha);
|
||
if (baseRef) {
|
||
const { stdout } = await execAsync(`git diff --shortstat ${baseRef}..HEAD`, {
|
||
cwd: worktreePath,
|
||
encoding: "utf-8",
|
||
});
|
||
diffShortstat = stdout.trim() || undefined;
|
||
}
|
||
} catch {
|
||
// best-effort — fall through with no shortstat
|
||
}
|
||
|
||
const MAX_SCOPE_FILES = 100;
|
||
const scopeFileBlock = scopedFiles.length === 0
|
||
? "(no modified files detected for this task — review the worktree directly, but do NOT browse unrelated files)"
|
||
: scopedFiles.length > MAX_SCOPE_FILES
|
||
? `${scopedFiles.slice(0, MAX_SCOPE_FILES).map((f) => `- ${f}`).join("\n")}\n- ... (${scopedFiles.length - MAX_SCOPE_FILES} more files truncated)`
|
||
: scopedFiles.map((f) => `- ${f}`).join("\n");
|
||
|
||
const scopeBlock = `Diff Scope (files changed by THIS task vs base):
|
||
${scopeFileBlock}${diffShortstat ? `\nDiff stat: ${diffShortstat}` : ""}
|
||
|
||
CRITICAL SCOPING RULES — read before doing anything else:
|
||
- Review ONLY the files listed above. Do NOT analyze unmodified files or unrelated parts of the codebase.
|
||
- If NONE of the files in the diff scope are relevant to your review category (e.g. a UX/design reviewer with no UI/CSS/component files in scope, a security reviewer with no auth/network code in scope, an a11y reviewer with no markup changes), respond IMMEDIATELY with a single short approval line such as "No relevant changes in scope — approved." and STOP. Do not start exploring the codebase.
|
||
- Your wall-clock budget is short. Spending it browsing unmodified files will cause this step to time out and block merge.`;
|
||
|
||
const systemPrompt = `You are a workflow step agent executing: ${workflowStep.name}
|
||
|
||
Task Context:
|
||
- Task ID: ${task.id}
|
||
- Task Description: ${task.description}
|
||
- Worktree: ${worktreePath}
|
||
|
||
${scopeBlock}
|
||
|
||
Your role:
|
||
- Execute this workflow step exactly as scoped.
|
||
- Prioritize high-impact correctness/risk findings over stylistic nits.
|
||
- Keep feedback actionable and directly tied to evidence in files/outputs.
|
||
|
||
Your Instructions:
|
||
${workflowStep.prompt}
|
||
|
||
You have access to the file system to review changes.
|
||
|
||
## Feedback Format
|
||
|
||
When your review is complete, you MUST use one of these exact formats:
|
||
|
||
**For PASS (no issues found):**
|
||
Simply state your findings and approval. No special formatting required.
|
||
|
||
**For REVISION REQUESTED (issues found that require code changes):**
|
||
Your response MUST start with the exact phrase:
|
||
\`REQUEST REVISION\`
|
||
|
||
Followed by a clear, actionable description of what needs to be fixed.
|
||
Be specific: reference exact files, line numbers, or functions that need changes.
|
||
|
||
Example:
|
||
\`REQUEST REVISION
|
||
|
||
The login function in src/auth.ts does not handle the case where the user
|
||
account is locked. Add proper error handling for the LOCKED_ACCOUNT error code
|
||
and show an appropriate message to the user.\`
|
||
|
||
**Important:**
|
||
- Only use "REQUEST REVISION" when the implementation needs code changes.
|
||
- If the code is correct and no changes are needed, just state your findings.
|
||
- Be constructive and actionable — vague feedback wastes the executor's time.`;
|
||
|
||
const agentLogger = new AgentLogger({
|
||
store: this.store,
|
||
taskId: task.id,
|
||
agent: "reviewer",
|
||
persistAgentToolOutput: settings.persistAgentToolOutput,
|
||
onAgentText: (taskId, delta) => {
|
||
this.options.onAgentText?.(taskId, delta);
|
||
},
|
||
onAgentTool: (taskId, toolName) => {
|
||
this.options.onAgentTool?.(taskId, toolName);
|
||
},
|
||
});
|
||
|
||
// Determine primary model and an explicit fallback. The workflow step's
|
||
// own override takes precedence; otherwise we use the project default
|
||
// override before falling through to the global default. The
|
||
// fallback is the per-step override's missing-counterpart settings, then
|
||
// the global validator/fallback pair, then the executor's `fallbackProvider`.
|
||
const defaultModel = resolveProjectDefaultModel(settings);
|
||
const primaryProvider = workflowStep.modelProvider || defaultModel.provider;
|
||
const primaryModelId = workflowStep.modelId || defaultModel.modelId;
|
||
const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId);
|
||
|
||
type ModelTuple = { provider?: string; modelId?: string };
|
||
const fallbackCandidates: Array<ModelTuple & { label: string }> = [
|
||
{ provider: settings.validatorFallbackProvider, modelId: settings.validatorFallbackModelId, label: "validatorFallback" },
|
||
{ provider: settings.fallbackProvider, modelId: settings.fallbackModelId, label: "globalFallback" },
|
||
];
|
||
const fallback = fallbackCandidates.find(
|
||
(c) => c.provider && c.modelId && (c.provider !== primaryProvider || c.modelId !== primaryModelId),
|
||
);
|
||
|
||
const timeoutMs = Math.max(60_000, settings.workflowStepTimeoutMs ?? 360_000);
|
||
|
||
const runOnce = async (
|
||
provider: string | undefined,
|
||
modelId: string | undefined,
|
||
attemptLabel: string,
|
||
): Promise<WorkflowStepOutcome> => {
|
||
// Workflow step agents inherit executor instructions
|
||
const stepInstructions = await this.resolveInstructionsForRole("executor");
|
||
const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions);
|
||
|
||
// Build skill selection context for workflow step session
|
||
const skillContext = await buildSessionSkillContext({
|
||
agentStore: this.options.agentStore!,
|
||
task,
|
||
sessionPurpose: "executor",
|
||
projectRootDir: this.rootDir,
|
||
pluginRunner: this.options.pluginRunner,
|
||
});
|
||
|
||
const workflowAgent = task.assignedAgentId && this.options.agentStore
|
||
? await this.options.agentStore.getAgent(task.assignedAgentId).catch(() => null)
|
||
: null;
|
||
const workflowRuntimeHint = extractRuntimeHint(workflowAgent?.runtimeConfig);
|
||
|
||
const { session } = await createResolvedAgentSession({
|
||
sessionPurpose: "executor",
|
||
runtimeHint: workflowRuntimeHint,
|
||
pluginRunner: this.options.pluginRunner,
|
||
cwd: worktreePath,
|
||
systemPrompt: stepSystemPrompt,
|
||
tools: toolMode,
|
||
defaultProvider: provider,
|
||
defaultModelId: modelId,
|
||
fallbackProvider: settings.fallbackProvider,
|
||
fallbackModelId: settings.fallbackModelId,
|
||
defaultThinkingLevel: settings.defaultThinkingLevel,
|
||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||
});
|
||
|
||
executorLog.log(`${task.id}: workflow step '${workflowStep.name}' using model ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Workflow step '${workflowStep.name}' using model: ${describeModel(session)}${useOverride && attemptLabel === "primary" ? " (workflow step override)" : ""}${attemptLabel === "fallback" ? " (fallback after timeout)" : ""}`,
|
||
);
|
||
this.activeWorkflowStepSessions.set(task.id, session);
|
||
|
||
let output = "";
|
||
session.subscribe((event) => {
|
||
if (event.type === "message_update") {
|
||
const msgEvent = event.assistantMessageEvent;
|
||
if (msgEvent.type === "text_delta") {
|
||
output += msgEvent.delta;
|
||
agentLogger.onText(msgEvent.delta);
|
||
} else if (msgEvent.type === "thinking_delta") {
|
||
agentLogger.onThinking(msgEvent.delta);
|
||
}
|
||
}
|
||
if (event.type === "tool_execution_start") {
|
||
agentLogger.onToolStart(event.toolName, event.args as Record<string, unknown> | undefined);
|
||
}
|
||
if (event.type === "tool_execution_end") {
|
||
agentLogger.onToolEnd(event.toolName, event.isError, event.result);
|
||
}
|
||
});
|
||
|
||
let timedOut = false;
|
||
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
|
||
const timeoutPromise = new Promise<"timeout">((resolveTimeout) => {
|
||
timeoutHandle = setTimeout(() => {
|
||
timedOut = true;
|
||
resolveTimeout("timeout");
|
||
}, timeoutMs);
|
||
});
|
||
|
||
try {
|
||
const promptPromise = promptWithFallback(
|
||
session,
|
||
`Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` +
|
||
`Review the work done in this worktree and evaluate it against the criteria in your instructions.`,
|
||
);
|
||
|
||
const outcome = await Promise.race([
|
||
promptPromise.then(() => "completed" as const),
|
||
timeoutPromise,
|
||
]);
|
||
|
||
if (outcome === "timeout") {
|
||
executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' (${attemptLabel}) timed out after ${timeoutMs}ms — disposing session`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Workflow step '${workflowStep.name}' ${attemptLabel === "primary" ? "primary" : "fallback"} model timed out after ${Math.round(timeoutMs / 1000)}s — aborting session`,
|
||
);
|
||
try { session.dispose(); } catch { /* best-effort */ }
|
||
await agentLogger.flush();
|
||
return { success: false, error: `workflow step timed out after ${timeoutMs}ms`, timedOut: true };
|
||
}
|
||
|
||
// Completed within the timeout — let any post-completion errors surface.
|
||
checkSessionError(session);
|
||
await accumulateSessionTokenUsage(this.store, task.id, session);
|
||
session.dispose();
|
||
await agentLogger.flush();
|
||
|
||
const trimmedOutput = output.trim();
|
||
const revisionMatch = trimmedOutput.match(/^REQUEST REVISION\s*\n*/i);
|
||
if (revisionMatch) {
|
||
const feedbackStart = revisionMatch[0].length;
|
||
const feedback = trimmedOutput.slice(feedbackStart).trim();
|
||
return { success: false, revisionRequested: true, output: feedback };
|
||
}
|
||
return { success: true, output };
|
||
} catch (err: unknown) {
|
||
await agentLogger.flush();
|
||
try { session.dispose(); } catch { /* best-effort */ }
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
return { success: false, error: errorMessage };
|
||
} finally {
|
||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||
const activeWorkflowStepSession = this.activeWorkflowStepSessions.get(task.id);
|
||
if (activeWorkflowStepSession === session) {
|
||
this.activeWorkflowStepSessions.delete(task.id);
|
||
}
|
||
// Suppress unused-variable warning; `timedOut` documents intent.
|
||
void timedOut;
|
||
}
|
||
};
|
||
|
||
const primaryOutcome = await runOnce(primaryProvider, primaryModelId, "primary");
|
||
if (!primaryOutcome.timedOut) return primaryOutcome;
|
||
|
||
if (!fallback) {
|
||
executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' timed out and no fallback model is configured`);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Workflow step '${workflowStep.name}' timed out — no fallback model configured (set settings.validatorFallbackProvider/Id or fallbackProvider/Id)`,
|
||
);
|
||
return primaryOutcome;
|
||
}
|
||
|
||
executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with fallback ${fallback.provider}/${fallback.modelId} (label=${fallback.label})`);
|
||
return runOnce(fallback.provider, fallback.modelId, "fallback");
|
||
}
|
||
|
||
private MAX_WORKTREE_RETRIES = 3;
|
||
private WORKTREE_RETRY_DELAYS = [100, 500, 1000]; // ms
|
||
|
||
/**
|
||
* Create a git worktree with automatic recovery from conflicts.
|
||
* Implements retry logic with exponential backoff for transient failures.
|
||
*
|
||
* @param branch - The branch name to create (e.g., "fusion/fn-123")
|
||
* @param path - The desired worktree path
|
||
* @param taskId - The task ID for logging
|
||
* @param startPoint - Optional base branch/commit for new branch
|
||
* @returns The actual worktree path (may differ if recovery generated new name)
|
||
*/
|
||
private async createWorktree(
|
||
branch: string,
|
||
path: string,
|
||
taskId: string,
|
||
startPoint?: string,
|
||
): Promise<{ path: string; branch: string }> {
|
||
// Track the worktree path we're attempting to use (may change during recovery)
|
||
const currentPath = path;
|
||
let resolvedStartPoint: string | undefined;
|
||
if (startPoint) {
|
||
const resolved = await this.resolveWorktreeStartPoint(startPoint, taskId);
|
||
if (resolved === null) {
|
||
// Stored baseBranch no longer exists (e.g., upstream dep merged and branch
|
||
// deleted while this task sat queued/stuck). Clear it on the task so any
|
||
// subsequent retry branches from the default base, and proceed from HEAD.
|
||
await this.store.updateTask(taskId, { baseBranch: null });
|
||
} else {
|
||
resolvedStartPoint = resolved;
|
||
}
|
||
}
|
||
|
||
// When the task declares a non-main base (a sibling task's branch), the
|
||
// legacy behavior was to fork the worktree from that branch's tip,
|
||
// inheriting all of its commits. That caused content leakage when the
|
||
// dep was later squash-merged to main: the dep's raw commits became
|
||
// orphans whose content already existed in main, blocking the
|
||
// dependent's own merge with phantom conflicts.
|
||
//
|
||
// Prevention: instead of forking from the dep's tip, fork from `main`
|
||
// (or the configured remote/main if rebase-from-remote is enabled) and
|
||
// then `git merge --squash` the dep's content into a single import
|
||
// commit. The dependent branch then carries main's history + 1 commit
|
||
// for the dep's content; if the dep is later squash-merged to main, the
|
||
// patch-id on that import commit will match main's squash and Layer 2
|
||
// recovery (or a clean rebase) handles it.
|
||
//
|
||
// Fall-soft: any failure in this path falls back to the legacy behavior
|
||
// so we don't break worktree creation for setups where the squash flow
|
||
// can't run (no main branch resolvable, network down, etc.).
|
||
const squashImport = resolvedStartPoint
|
||
? await this.planSquashImportFromDep(taskId, resolvedStartPoint, startPoint)
|
||
: null;
|
||
const initialStartPoint = squashImport ? squashImport.mainBase : resolvedStartPoint;
|
||
|
||
for (let attempt = 0; attempt < this.MAX_WORKTREE_RETRIES; attempt++) {
|
||
try {
|
||
const result = await this.tryCreateWorktree(branch, currentPath, taskId, initialStartPoint, attempt);
|
||
// Squash-import dep content into the freshly created worktree so the
|
||
// branch contains main's history + 1 import commit instead of the
|
||
// dep's raw commits.
|
||
if (squashImport) {
|
||
await this.squashImportDepIntoWorktree(
|
||
result.path,
|
||
taskId,
|
||
squashImport.depTip,
|
||
squashImport.label,
|
||
).catch((importErr: unknown) => {
|
||
executorLog.warn(
|
||
`Squash-import of ${squashImport.label} into ${result.branch} failed for ${taskId} (continuing without): ${importErr instanceof Error ? importErr.message : String(importErr)}`,
|
||
);
|
||
});
|
||
}
|
||
// Mirror the merge-time rebase behavior: when worktreeRebaseBeforeMerge
|
||
// is enabled, fetch the remote and rebase the just-created task branch
|
||
// onto the latest <remote>/<defaultBranch>. This makes the worktree
|
||
// start from origin/main + local main both, so divergence only matters
|
||
// if the user actively skips this setting. Best-effort: failures here
|
||
// don't abort task setup.
|
||
await this.rebaseNewWorktreeOntoRemote(result.path, result.branch, taskId).catch((err: unknown) => {
|
||
executorLog.warn(
|
||
`Post-create worktree rebase failed for ${taskId} (continuing): ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
});
|
||
return result;
|
||
} catch (error: unknown) {
|
||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||
const isLastAttempt = attempt === this.MAX_WORKTREE_RETRIES - 1;
|
||
const isTerminalWorktreeError = error instanceof NonRetryableWorktreeError;
|
||
|
||
if (isLastAttempt || isTerminalWorktreeError) {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Worktree creation failed after ${this.MAX_WORKTREE_RETRIES} attempts`,
|
||
errorMessage,
|
||
);
|
||
throw new Error(
|
||
`Failed to create worktree after ${this.MAX_WORKTREE_RETRIES} attempts: ${errorMessage}`,
|
||
);
|
||
}
|
||
|
||
// Wait before retry (exponential backoff)
|
||
const delay = this.WORKTREE_RETRY_DELAYS[attempt] || 1000;
|
||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||
}
|
||
}
|
||
|
||
// Should never reach here, but TypeScript needs a return
|
||
throw new Error("Unexpected exit from worktree creation retry loop");
|
||
}
|
||
|
||
private quoteShellArg(value: string): string {
|
||
return `'${value.replace(/'/g, "'\\''")}'`;
|
||
}
|
||
|
||
/**
|
||
* Decide whether a task's declared dep base should be squash-imported
|
||
* (instead of forked from). Returns the planned operation's data when the
|
||
* dep tip differs from the resolvable main base; returns null when no
|
||
* import is needed (dep is already at main) or when no main base is
|
||
* resolvable (caller falls back to legacy fork-from-dep).
|
||
*
|
||
* `originalStartPoint` is the user-facing label (typically the branch name
|
||
* like `fusion/fn-2729`) used purely for log messages. `depTip` is the
|
||
* resolved SHA of the dep's tip — that's what gets squash-merged.
|
||
*/
|
||
private async planSquashImportFromDep(
|
||
taskId: string,
|
||
depTip: string,
|
||
originalStartPoint: string | undefined,
|
||
): Promise<{ depTip: string; mainBase: string; label: string } | null> {
|
||
let settings;
|
||
try {
|
||
settings = await this.store.getSettings();
|
||
} catch {
|
||
return null;
|
||
}
|
||
|
||
// Resolve the main base. Preference order:
|
||
// 1. <remote>/<defaultBranch> when worktreeRebaseBeforeMerge is enabled
|
||
// and a remote is resolvable (settings.worktreeRebaseRemote wins;
|
||
// otherwise fall back to "origin" or the lone remote).
|
||
// 2. rootDir's HEAD (i.e., whatever local main is currently checked out
|
||
// to). Used when remote rebase is disabled or no remote exists.
|
||
let mainBase = "";
|
||
|
||
if (settings.worktreeRebaseBeforeMerge !== false) {
|
||
let remote = settings.worktreeRebaseRemote?.trim() || "";
|
||
if (!remote) {
|
||
try {
|
||
const { stdout } = await execAsync("git remote", { cwd: this.rootDir });
|
||
const remotes = stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
||
if (remotes.includes("origin")) remote = "origin";
|
||
else if (remotes.length === 1) remote = remotes[0];
|
||
} catch {
|
||
// No remote resolvable.
|
||
}
|
||
}
|
||
if (remote) {
|
||
let defaultBranch = "";
|
||
try {
|
||
const { stdout } = await execAsync(
|
||
`git rev-parse --abbrev-ref ${this.quoteShellArg(remote)}/HEAD`,
|
||
{ cwd: this.rootDir },
|
||
);
|
||
defaultBranch = stdout.trim().replace(new RegExp(`^${remote}/`), "");
|
||
} catch {
|
||
// origin/HEAD not set; will fall through to local HEAD below.
|
||
}
|
||
if (defaultBranch && defaultBranch !== "HEAD") {
|
||
// Fetch best-effort so the remote ref reflects upstream tip.
|
||
await execAsync(
|
||
`git fetch ${this.quoteShellArg(remote)} ${this.quoteShellArg(defaultBranch)}`,
|
||
{ cwd: this.rootDir },
|
||
).catch(() => undefined);
|
||
try {
|
||
const { stdout } = await execAsync(
|
||
`git rev-parse --verify "${remote}/${defaultBranch}^{commit}"`,
|
||
{ cwd: this.rootDir, encoding: "utf-8" },
|
||
);
|
||
mainBase = stdout.trim();
|
||
} catch {
|
||
// Couldn't resolve remote ref — fall through.
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!mainBase) {
|
||
try {
|
||
const { stdout } = await execAsync("git rev-parse HEAD", {
|
||
cwd: this.rootDir,
|
||
encoding: "utf-8",
|
||
});
|
||
mainBase = stdout.trim();
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
if (!mainBase) return null;
|
||
|
||
// If the dep tip is already an ancestor of main, no squash import is
|
||
// needed — the dep's content is already represented in main.
|
||
try {
|
||
await execAsync(
|
||
`git merge-base --is-ancestor ${this.quoteShellArg(depTip)} ${this.quoteShellArg(mainBase)}`,
|
||
{ cwd: this.rootDir },
|
||
);
|
||
// Exit code 0 → ancestor → no import needed; legacy fork-from-main is fine.
|
||
// Returning the plan with mainBase but signalling "no work" via dep===main.
|
||
if (depTip === mainBase) return null;
|
||
// Dep is ancestor of main but its tip SHA differs from main's tip; the
|
||
// worktree should still branch off main, no squash needed.
|
||
return { depTip: mainBase, mainBase, label: originalStartPoint || depTip.slice(0, 8) };
|
||
} catch {
|
||
// Not an ancestor — squash-import is the safer path.
|
||
}
|
||
|
||
return { depTip, mainBase, label: originalStartPoint || depTip.slice(0, 8) };
|
||
}
|
||
|
||
/**
|
||
* Squash-merge the dep's content into a worktree that's already branched
|
||
* off main. Produces one commit on the worktree branch carrying the dep's
|
||
* content, instead of inheriting the dep's individual commits. Best-effort:
|
||
* any failure (conflict, hooks, IO) leaves the worktree at main and the
|
||
* caller proceeds — the dependent task will then need to import the dep's
|
||
* content itself, but the worktree itself is still usable.
|
||
*/
|
||
private async squashImportDepIntoWorktree(
|
||
worktreePath: string,
|
||
taskId: string,
|
||
depTip: string,
|
||
label: string,
|
||
): Promise<void> {
|
||
// No-op when dep is already represented in the worktree's history.
|
||
try {
|
||
await execAsync(
|
||
`git merge-base --is-ancestor ${this.quoteShellArg(depTip)} HEAD`,
|
||
{ cwd: worktreePath },
|
||
);
|
||
return;
|
||
} catch {
|
||
// Not an ancestor — proceed.
|
||
}
|
||
|
||
// Try a squash-merge. `--no-commit` is implied by `--squash`; the merge
|
||
// either stages the dep's diff or fails (conflicts / unrelated histories).
|
||
try {
|
||
await execAsync(
|
||
`git merge --squash --allow-unrelated-histories ${this.quoteShellArg(depTip)}`,
|
||
{ cwd: worktreePath },
|
||
);
|
||
} catch (err) {
|
||
// Reset any partial state so the worktree stays usable, then rethrow
|
||
// so the caller can decide whether to log/fall-through.
|
||
await execAsync("git reset --hard HEAD", { cwd: worktreePath }).catch(
|
||
() => undefined,
|
||
);
|
||
throw err;
|
||
}
|
||
|
||
// If no diff was staged the dep is content-equivalent to main; nothing
|
||
// to commit.
|
||
try {
|
||
await execAsync("git diff --cached --quiet", { cwd: worktreePath });
|
||
return; // exit 0 → no staged changes, nothing to commit
|
||
} catch {
|
||
// exit non-zero → staged changes exist, proceed to commit.
|
||
}
|
||
|
||
// Always non-empty (subject + body via two -m args). Drop
|
||
// --allow-empty-message: we never want git to silently accept an empty
|
||
// message — a missing message here would make the commit hard to
|
||
// attribute / explain in `git log` and break downstream consumers that
|
||
// parse merge metadata from commit messages.
|
||
const subject = `chore(${taskId}): import dependency content from ${label}`;
|
||
const body =
|
||
`Squash-imported the working tree of ${label} as a single commit so this ` +
|
||
`branch carries the dep's content without inheriting its individual commits. ` +
|
||
`If the dep is later squash-merged to main, this commit's patch-id should ` +
|
||
`match the merge and rebase cleanly.`;
|
||
try {
|
||
await execAsync(
|
||
`git commit -m ${this.quoteShellArg(subject)} -m ${this.quoteShellArg(body)}`,
|
||
{ cwd: worktreePath },
|
||
);
|
||
} catch (commitErr) {
|
||
await execAsync("git reset --hard HEAD", { cwd: worktreePath }).catch(
|
||
() => undefined,
|
||
);
|
||
throw commitErr;
|
||
}
|
||
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Squash-imported dependency content from ${label} into worktree (single import commit instead of inheriting raw commits)`,
|
||
);
|
||
}
|
||
|
||
/**
|
||
* After creating a fresh task worktree, fetch the configured remote and
|
||
* rebase the task branch onto `<remote>/<defaultBranch>`. The result is a
|
||
* branch that contains origin's tip plus any local main commits, so the
|
||
* eventual merge has fewer surprises and the executor sees the freshest
|
||
* code its peers/CI may have published.
|
||
*
|
||
* No-op when `worktreeRebaseBeforeMerge` is disabled, no remote is
|
||
* configured/resolvable, or the rebase produces conflicts (we abort and
|
||
* leave the worktree as-is so the executor can still run).
|
||
*/
|
||
private async rebaseNewWorktreeOntoRemote(
|
||
worktreePath: string,
|
||
branch: string,
|
||
taskId: string,
|
||
): Promise<void> {
|
||
let settings;
|
||
try {
|
||
settings = await this.store.getSettings();
|
||
} catch {
|
||
return;
|
||
}
|
||
if (settings.worktreeRebaseBeforeMerge === false) return;
|
||
|
||
let remote = settings.worktreeRebaseRemote?.trim() || "";
|
||
if (!remote) {
|
||
try {
|
||
const { stdout } = await execAsync("git remote", { cwd: this.rootDir });
|
||
const remotes = stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
||
if (remotes.includes("origin")) remote = "origin";
|
||
else if (remotes.length === 1) remote = remotes[0];
|
||
} catch {
|
||
// No remote resolvable — nothing to rebase against.
|
||
}
|
||
}
|
||
if (!remote) return;
|
||
|
||
let defaultBranch = "";
|
||
try {
|
||
const { stdout } = await execAsync(`git rev-parse --abbrev-ref ${remote}/HEAD`, { cwd: this.rootDir });
|
||
defaultBranch = stdout.trim().replace(new RegExp(`^${remote}/`), "");
|
||
} catch {
|
||
// origin/HEAD not set — fall back to current branch in rootDir.
|
||
}
|
||
if (!defaultBranch) {
|
||
try {
|
||
const { stdout } = await execAsync("git rev-parse --abbrev-ref HEAD", { cwd: this.rootDir });
|
||
defaultBranch = stdout.trim();
|
||
} catch {
|
||
return;
|
||
}
|
||
}
|
||
if (!defaultBranch || defaultBranch === "HEAD") return;
|
||
|
||
const remoteRef = `${remote}/${defaultBranch}`;
|
||
|
||
try {
|
||
await execAsync(`git fetch ${this.quoteShellArg(remote)} ${this.quoteShellArg(defaultBranch)}`, { cwd: this.rootDir });
|
||
} catch (err) {
|
||
executorLog.warn(
|
||
`Worktree rebase: fetch ${remote} ${defaultBranch} failed for ${taskId}: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
return;
|
||
}
|
||
|
||
try {
|
||
await execAsync(`git rebase ${this.quoteShellArg(remoteRef)}`, { cwd: worktreePath });
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Rebased new worktree branch ${branch} onto ${remoteRef}`,
|
||
);
|
||
} catch (rebaseErr) {
|
||
const msg = rebaseErr instanceof Error ? rebaseErr.message : String(rebaseErr);
|
||
executorLog.warn(
|
||
`Worktree rebase: rebase onto ${remoteRef} failed for ${taskId} — aborting and leaving local base intact: ${msg}`,
|
||
);
|
||
try {
|
||
await execAsync("git rebase --abort", { cwd: worktreePath });
|
||
} catch {
|
||
// best-effort
|
||
}
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Could not rebase new worktree onto ${remoteRef} — kept local base. The merge-time rebase will retry with conflict resolution.`,
|
||
);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Resolve a stored baseBranch to a concrete commit SHA.
|
||
*
|
||
* Returns `null` (not throw) when the ref cannot be resolved — typically
|
||
* because the upstream dep's branch was merged and deleted while this task
|
||
* sat queued/stuck. Callers should treat null as "fall back to default base"
|
||
* rather than fail the task permanently.
|
||
*/
|
||
private async resolveWorktreeStartPoint(startPoint: string, taskId: string): Promise<string | null> {
|
||
const command = isAbsolute(startPoint) && existsSync(startPoint)
|
||
? `git -C "${startPoint}" rev-parse --verify HEAD^{commit}`
|
||
: `git rev-parse --verify "${startPoint}^{commit}"`;
|
||
|
||
try {
|
||
const { stdout } = await execAsync(command, { cwd: this.rootDir });
|
||
return stdout.trim() || startPoint;
|
||
} catch (error: unknown) {
|
||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Worktree base ref "${startPoint}" is missing — falling back to default base`,
|
||
errorMessage,
|
||
);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Single attempt to create a worktree with conflict detection and recovery.
|
||
* Returns the actual worktree path used (may differ from input if recovery generated new name).
|
||
*/
|
||
private async tryCreateWorktree(
|
||
branch: string,
|
||
path: string,
|
||
taskId: string,
|
||
startPoint?: string,
|
||
attemptNumber = 0,
|
||
recoveryDepth = 0,
|
||
): Promise<{ path: string; branch: string }> {
|
||
// Guard: refuse to create a worktree nested inside another worktree.
|
||
// Nested worktrees happen when the executor is launched with rootDir pointed
|
||
// at a worktree directory instead of the main repo — produces paths like
|
||
// `.worktrees/green-finch/.worktrees/amber-panda` that bloat the filesystem
|
||
// and confuse every tool that walks git state.
|
||
await this.assertWorktreePathNotNested(path, taskId);
|
||
|
||
// If directory exists but is not a registered worktree, remove it first
|
||
if (existsSync(path)) {
|
||
const isRegistered = await this.isRegisteredWorktree(path);
|
||
if (!isRegistered) {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Removing existing directory (not a registered worktree): ${path}`,
|
||
);
|
||
try {
|
||
await execAsync(`rm -rf "${path}"`, { cwd: this.rootDir });
|
||
} catch (e: unknown) {
|
||
const eMessage = e instanceof Error ? e.message : String(e);
|
||
throw new Error(`Failed to remove existing directory ${path}: ${eMessage}`);
|
||
}
|
||
} else {
|
||
executorLog.log(`Worktree already exists: ${path}`);
|
||
return { path, branch };
|
||
}
|
||
}
|
||
|
||
const createWithBranch = async (branchToCreate: string) => {
|
||
const cmd = startPoint
|
||
? `git worktree add -b "${branchToCreate}" "${path}" "${startPoint}"`
|
||
: `git worktree add -b "${branchToCreate}" "${path}"`;
|
||
try {
|
||
await execAsync(cmd, { cwd: this.rootDir });
|
||
} catch (err) {
|
||
// Remove any partial directory left behind so the invariant holds:
|
||
// "if .worktrees/<slug> exists on disk, it is a fully registered git worktree."
|
||
try {
|
||
await execAsync(`rm -rf "${path}"`, { cwd: this.rootDir });
|
||
} catch {
|
||
// best-effort cleanup; log but don't mask the original error
|
||
executorLog.log(`Warning: failed to remove partial worktree directory after creation failure: ${path}`);
|
||
}
|
||
throw err;
|
||
}
|
||
};
|
||
|
||
const createFromExistingBranch = async () => {
|
||
try {
|
||
await execAsync(`git worktree add "${path}" "${branch}"`, { cwd: this.rootDir });
|
||
} catch (err) {
|
||
// Remove any partial directory left behind so the invariant holds:
|
||
// "if .worktrees/<slug> exists on disk, it is a fully registered git worktree."
|
||
try {
|
||
await execAsync(`rm -rf "${path}"`, { cwd: this.rootDir });
|
||
} catch {
|
||
// best-effort cleanup; log but don't mask the original error
|
||
executorLog.log(`Warning: failed to remove partial worktree directory after creation failure: ${path}`);
|
||
}
|
||
throw err;
|
||
}
|
||
};
|
||
|
||
try {
|
||
await createWithBranch(branch);
|
||
executorLog.log(`Worktree created: ${path}${startPoint ? ` (from ${startPoint})` : ""}`);
|
||
if (attemptNumber > 0) {
|
||
await this.store.logEntry(taskId, `Worktree created on attempt ${attemptNumber + 1}`, path);
|
||
}
|
||
return { path, branch };
|
||
} catch (initialError: unknown) {
|
||
const conflictInfo = this.extractWorktreeConflictInfo(initialError);
|
||
|
||
if (conflictInfo.type === "not-git-repo") {
|
||
throw new NonRetryableWorktreeError(
|
||
"Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.",
|
||
);
|
||
}
|
||
|
||
// Handle "already used by worktree" conflict
|
||
if (conflictInfo.type === "already-used" && conflictInfo.path) {
|
||
const result = await this.handleWorktreeConflict(
|
||
conflictInfo.path,
|
||
branch,
|
||
path,
|
||
taskId,
|
||
startPoint,
|
||
attemptNumber,
|
||
);
|
||
if (result) {
|
||
return result;
|
||
}
|
||
throw new Error(
|
||
`Worktree conflict at ${conflictInfo.path}: automatic cleanup failed`,
|
||
);
|
||
}
|
||
|
||
// Handle "invalid reference" - stale branch that doesn't exist
|
||
if (conflictInfo.type === "invalid-reference") {
|
||
if (recoveryDepth >= this.MAX_WORKTREE_RETRIES - 1) {
|
||
throw new NonRetryableWorktreeError(
|
||
`Stale branch reference for ${branch} remained invalid after ${this.MAX_WORKTREE_RETRIES} cleanup attempts`,
|
||
);
|
||
}
|
||
const branchCleaned = await this.cleanupStaleBranch(branch, taskId);
|
||
if (branchCleaned) {
|
||
await this.store.logEntry(taskId, `Removed stale branch reference, retrying`);
|
||
return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, recoveryDepth + 1);
|
||
}
|
||
throw new Error(
|
||
`Invalid reference for branch ${branch}: unable to clean up stale reference`,
|
||
);
|
||
}
|
||
|
||
// Handle "could not create leading directories" - permission/path issues
|
||
if (conflictInfo.type === "leading-directories") {
|
||
throw new Error(
|
||
`Cannot create worktree at ${path}: permission or path issue. ` +
|
||
`Check that parent directories are writable.`,
|
||
);
|
||
}
|
||
|
||
// Try creating from existing branch (branch might already exist)
|
||
try {
|
||
await createFromExistingBranch();
|
||
executorLog.log(`Worktree created from existing branch: ${path}`);
|
||
return { path, branch };
|
||
} catch (fallbackError: unknown) {
|
||
const fallbackErrorMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
|
||
// Check if the fallback also hit an "already used" conflict
|
||
const fallbackConflictInfo = this.extractWorktreeConflictInfo(fallbackError);
|
||
if (fallbackConflictInfo.type === "not-git-repo") {
|
||
throw new NonRetryableWorktreeError(
|
||
"Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.",
|
||
);
|
||
}
|
||
|
||
if (fallbackConflictInfo.type === "already-used" && fallbackConflictInfo.path) {
|
||
const result = await this.handleWorktreeConflict(
|
||
fallbackConflictInfo.path,
|
||
branch,
|
||
path,
|
||
taskId,
|
||
startPoint,
|
||
attemptNumber,
|
||
);
|
||
if (result) {
|
||
return result;
|
||
}
|
||
throw new Error(
|
||
`Worktree conflict at ${fallbackConflictInfo.path}: automatic cleanup failed`,
|
||
);
|
||
}
|
||
|
||
// Handle stale reference in fallback path too
|
||
if (fallbackConflictInfo.type === "invalid-reference") {
|
||
if (recoveryDepth >= this.MAX_WORKTREE_RETRIES - 1) {
|
||
throw new NonRetryableWorktreeError(
|
||
`Stale branch reference for ${branch} remained invalid after ${this.MAX_WORKTREE_RETRIES} cleanup attempts`,
|
||
);
|
||
}
|
||
const branchCleaned = await this.cleanupStaleBranch(branch, taskId);
|
||
if (branchCleaned) {
|
||
await this.store.logEntry(taskId, `Cleaned up stale reference in fallback, retrying`);
|
||
return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, recoveryDepth + 1);
|
||
}
|
||
}
|
||
|
||
throw new Error(`Failed to create worktree: ${fallbackErrorMessage}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Handle "already used by worktree" conflict.
|
||
* Either generates a new worktree name (if conflicting worktree is in use by active task)
|
||
* or cleans up the conflicting worktree and retries.
|
||
*
|
||
* @returns The worktree path if recovery succeeded, null if recovery failed
|
||
*/
|
||
private async handleWorktreeConflict(
|
||
conflictPath: string,
|
||
branch: string,
|
||
path: string,
|
||
taskId: string,
|
||
startPoint?: string,
|
||
attemptNumber?: number,
|
||
): Promise<{ path: string; branch: string } | null> {
|
||
const shouldGenerateNewName = await this.shouldGenerateNewWorktreeName(
|
||
conflictPath,
|
||
taskId,
|
||
);
|
||
|
||
if (shouldGenerateNewName) {
|
||
// Conflicting worktree belongs to an active task — generate new path AND
|
||
// use a suffixed branch name so git doesn't conflict with the branch
|
||
// already checked out in the existing worktree. Branch conflicts here
|
||
// mean the original task branch already exists and is checked out
|
||
// elsewhere, so suffix retries must branch from that task branch tip
|
||
// rather than the stale base ref to preserve the task's commits.
|
||
const conflictStartPoint = branch;
|
||
const newPath = join(this.rootDir, ".worktrees", generateWorktreeName(this.rootDir));
|
||
for (let suffix = 2; suffix <= 6; suffix++) {
|
||
const suffixedBranch = `${branch}-${suffix}`;
|
||
try {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Conflicting worktree in use by active task, trying new path with branch ${suffixedBranch}`,
|
||
newPath,
|
||
);
|
||
return await this.tryCreateWorktree(suffixedBranch, newPath, taskId, conflictStartPoint, attemptNumber);
|
||
} catch (suffixErr: unknown) {
|
||
const info = this.extractWorktreeConflictInfo(suffixErr);
|
||
if (info.type === "already-used") {
|
||
// This suffixed branch is also in use — try next suffix
|
||
continue;
|
||
}
|
||
throw suffixErr;
|
||
}
|
||
}
|
||
throw new Error(
|
||
`Cannot create branch for task: "${branch}" and suffixes -2 through -6 are all in use by other worktrees`,
|
||
);
|
||
}
|
||
|
||
// Safe to clean up - conflicting worktree is not in use
|
||
const cleanupSuccess = await this.cleanupConflictingWorktree(conflictPath, branch, taskId);
|
||
if (cleanupSuccess) {
|
||
await this.store.logEntry(taskId, `Cleaned up conflicting worktree, retrying`, path);
|
||
return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber);
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* Check if a path is registered as a git worktree.
|
||
*/
|
||
private async isRegisteredWorktree(path: string): Promise<boolean> {
|
||
return isRegisteredGitWorktree(this.rootDir, path);
|
||
}
|
||
|
||
/**
|
||
* Throw if `path` lies inside an existing registered worktree other than the
|
||
* repo root. The repo root itself is a worktree (main branch) and must be
|
||
* allowed — we only reject paths strictly *inside* a non-root worktree.
|
||
*/
|
||
private async assertWorktreePathNotNested(path: string, taskId: string): Promise<void> {
|
||
const target = resolvePath(path);
|
||
const rootResolved = resolvePath(this.rootDir);
|
||
const registered = await getRegisteredWorktreePaths(this.rootDir);
|
||
|
||
for (const wt of registered) {
|
||
if (wt === rootResolved) continue; // root is allowed as ancestor
|
||
if (wt === target) continue; // exact match handled later as "already registered"
|
||
const rel = relative(wt, target);
|
||
if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Refusing to create nested worktree`,
|
||
`target ${target} is inside registered worktree ${wt}`,
|
||
);
|
||
throw new NonRetryableWorktreeError(
|
||
`Refusing to create worktree at ${target}: path is nested inside existing worktree ${wt}. ` +
|
||
`This usually means the executor was launched with rootDir pointing at a worktree instead of the main repo.`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Determine if we should generate a new worktree name instead of cleaning up.
|
||
* Returns true if the conflicting worktree is used by an active task.
|
||
*/
|
||
private async shouldGenerateNewWorktreeName(
|
||
conflictPath: string,
|
||
currentTaskId: string,
|
||
): Promise<boolean> {
|
||
// Check if conflicting worktree is in our active set
|
||
for (const [taskId, worktreePath] of this.activeWorktrees) {
|
||
if (taskId !== currentTaskId && worktreePath === conflictPath) {
|
||
return true;
|
||
}
|
||
}
|
||
|
||
// Check if another non-done task uses this worktree
|
||
const otherUser = await findWorktreeUser(this.store, conflictPath, currentTaskId);
|
||
return otherUser !== null;
|
||
}
|
||
|
||
/**
|
||
* Clean up a conflicting worktree and its branch.
|
||
* Handles locked worktrees by unlocking first.
|
||
* Returns true if cleanup succeeded.
|
||
*/
|
||
private async cleanupConflictingWorktree(
|
||
worktreePath: string,
|
||
branch: string,
|
||
taskId: string,
|
||
): Promise<boolean> {
|
||
try {
|
||
// Check if worktree is locked and unlock if needed
|
||
try {
|
||
await execAsync(`git worktree unlock "${worktreePath}"`, {
|
||
cwd: this.rootDir,
|
||
});
|
||
await this.store.logEntry(taskId, `Unlocked worktree`, worktreePath);
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: failed to unlock conflicting worktree ${worktreePath} before cleanup: ${msg}`);
|
||
}
|
||
|
||
// Remove the worktree
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, {
|
||
cwd: this.rootDir,
|
||
});
|
||
await this.store.logEntry(taskId, `Removed conflicting worktree`, worktreePath);
|
||
|
||
// Delete the branch if it exists
|
||
try {
|
||
await execAsync(`git branch -D "${branch}"`, {
|
||
cwd: this.rootDir,
|
||
});
|
||
await this.store.logEntry(taskId, `Deleted branch`, branch);
|
||
// FN-2165 regression guard: null baseBranch on any task that stored this branch
|
||
this.store.clearStaleBaseBranchReferences([branch], taskId);
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: failed to delete conflicting branch ${branch}: ${msg}`);
|
||
}
|
||
|
||
return true;
|
||
} catch (error: unknown) {
|
||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Failed to clean up conflicting worktree`,
|
||
`${worktreePath}: ${errorMessage}`,
|
||
);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clean up a stale branch that no longer has a valid reference.
|
||
*
|
||
* Recovery strategy (in order):
|
||
* 1. `git worktree prune` — remove stale worktree metadata that may
|
||
* hold a lock on the branch reference
|
||
* 2. `git branch -D` — delete the branch normally
|
||
* 3. `git update-ref -d refs/heads/<branch>` — force-remove a corrupted
|
||
* or dangling reference when `git branch -D` fails
|
||
*
|
||
* Each step is logged so operators can trace the recovery path.
|
||
* Returns true if the branch reference was successfully removed.
|
||
*/
|
||
private async cleanupStaleBranch(branch: string, taskId: string): Promise<boolean> {
|
||
// Step 1: Prune stale worktree metadata that may hold a lock on the branch
|
||
try {
|
||
await execAsync("git worktree prune", { cwd: this.rootDir });
|
||
await this.store.logEntry(taskId, `Pruned stale worktree metadata`, branch);
|
||
} catch {
|
||
// Prune is best-effort — continue even if it fails
|
||
}
|
||
|
||
// Step 2: Try normal branch deletion
|
||
try {
|
||
await execAsync(`git branch -D "${branch}"`, {
|
||
cwd: this.rootDir,
|
||
});
|
||
await this.store.logEntry(taskId, `Removed stale branch`, branch);
|
||
// FN-2165 regression guard: null baseBranch on any task that stored this branch
|
||
try { this.store.clearStaleBaseBranchReferences([branch], taskId); } catch { /* best-effort */ }
|
||
return true;
|
||
} catch (branchDeleteError: unknown) {
|
||
const branchDeleteErrorMessage = branchDeleteError instanceof Error ? branchDeleteError.message : String(branchDeleteError);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`git branch -D failed for stale branch, trying update-ref`,
|
||
`${branch}: ${branchDeleteErrorMessage}`,
|
||
);
|
||
}
|
||
|
||
// Step 3: Force-remove the reference directly
|
||
try {
|
||
const refPath = `refs/heads/${branch}`;
|
||
await execAsync(`git update-ref -d "${refPath}"`, {
|
||
cwd: this.rootDir,
|
||
});
|
||
await this.store.logEntry(taskId, `Force-removed stale branch reference via update-ref`, refPath);
|
||
// FN-2165 regression guard: null baseBranch on any task that stored this branch
|
||
try { this.store.clearStaleBaseBranchReferences([branch], taskId); } catch { /* best-effort */ }
|
||
return true;
|
||
} catch (updateRefError: unknown) {
|
||
const updateRefErrorMessage = updateRefError instanceof Error ? updateRefError.message : String(updateRefError);
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Failed to remove stale branch reference`,
|
||
`${branch}: ${updateRefErrorMessage}`,
|
||
);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Extract conflict information from git worktree error output.
|
||
* Handles multiple error patterns:
|
||
* - "already used by worktree at '...'"
|
||
* - "invalid reference" / "unable to resolve reference" / "stale file handle"
|
||
* - "could not create leading directories"
|
||
* - "working tree already exists"
|
||
*/
|
||
private extractWorktreeConflictInfo(error: unknown): {
|
||
type: "already-used" | "invalid-reference" | "leading-directories" | "already-exists" | "not-git-repo" | "unknown";
|
||
path?: string;
|
||
message?: string;
|
||
} {
|
||
const execError = error instanceof Error ? error : new Error(String(error));
|
||
const output = [
|
||
execError.message,
|
||
"stderr" in execError && typeof execError.stderr === "string" ? execError.stderr.toString() : undefined,
|
||
"stdout" in execError && typeof execError.stdout === "string" ? execError.stdout.toString() : undefined,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
// Pattern: already used by worktree at '/path/to/worktree'
|
||
const alreadyUsedMatch = output.match(/already used by worktree at '([^']+)'/);
|
||
if (alreadyUsedMatch) {
|
||
return { type: "already-used", path: alreadyUsedMatch[1], message: output };
|
||
}
|
||
|
||
// Pattern: already checked out at '/path/to/worktree'
|
||
const alreadyCheckedOutMatch = output.match(/is already checked out at '([^']+)'/);
|
||
if (alreadyCheckedOutMatch) {
|
||
return { type: "already-used", path: alreadyCheckedOutMatch[1], message: output };
|
||
}
|
||
|
||
// Pattern: invalid reference: 'branch-name'
|
||
// Also covers: unable to resolve reference, stale file handle, not a valid ref
|
||
if (
|
||
output.match(/invalid reference/i) ||
|
||
output.match(/unable to resolve reference/i) ||
|
||
output.match(/stale file handle/i) ||
|
||
output.match(/not a valid ref/i) ||
|
||
output.match(/unable to delete.*ref/i)
|
||
) {
|
||
return { type: "invalid-reference", message: output };
|
||
}
|
||
|
||
// Pattern: could not create leading directories
|
||
if (output.match(/could not create leading directories/i)) {
|
||
return { type: "leading-directories", message: output };
|
||
}
|
||
|
||
// Pattern: working tree already exists
|
||
if (output.match(/working tree already exists/i)) {
|
||
return { type: "already-exists", message: output };
|
||
}
|
||
|
||
// Pattern: not a git repository / not a git repo
|
||
if (output.match(/not a git repo(sitory)?/i)) {
|
||
return { type: "not-git-repo", message: output };
|
||
}
|
||
|
||
return { type: "unknown", message: output };
|
||
}
|
||
|
||
/**
|
||
* Remove a task's worktree, but only if no other in-progress or todo task
|
||
* shares the same worktree path (dependency-chain reuse). The branch is
|
||
* always cleaned up by the merger on a per-task basis.
|
||
*/
|
||
async cleanup(taskId: string): Promise<void> {
|
||
const worktreePath = this.activeWorktrees.get(taskId);
|
||
if (!worktreePath) return;
|
||
|
||
this.activeWorktrees.delete(taskId);
|
||
|
||
// Check if another task still needs this worktree
|
||
const otherUser = await findWorktreeUser(this.store, worktreePath, taskId);
|
||
if (otherUser) {
|
||
executorLog.log(`Worktree retained for ${taskId} — still needed by ${otherUser}`);
|
||
return;
|
||
}
|
||
|
||
try {
|
||
await execAsync(`git worktree remove "${worktreePath}" --force`, { cwd: this.rootDir });
|
||
executorLog.log(`Cleaned up worktree for ${taskId}`);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to clean up worktree for ${taskId}:`, errorMessage);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* When the engine restarts mid-step, an `in-progress` step may have already
|
||
* passed its code review (log: `code review Step N: APPROVE`) but not yet
|
||
* been flipped to `done` by the agent's next `fn_task_update` call. Without
|
||
* intervention, the next executor pass re-enters the step and replays plan
|
||
* + code review, which we've measured at 5–20 min of pure waste per restart.
|
||
*
|
||
* This reconciler scans the task log for any in-progress step whose most
|
||
* recent approved code review is newer than its most recent `→ pending`
|
||
* transition, and marks those steps `done`. Subsequent resume logic then
|
||
* advances to the next actually-pending step.
|
||
*/
|
||
private async recoverApprovedStepsOnResume(taskId: string): Promise<void> {
|
||
let detail: TaskDetail;
|
||
try {
|
||
detail = await this.store.getTask(taskId);
|
||
} catch (err) {
|
||
executorLog.warn(`${taskId}: recoverApprovedStepsOnResume getTask failed: ${err instanceof Error ? err.message : String(err)}`);
|
||
return;
|
||
}
|
||
const log = detail.log ?? [];
|
||
if (log.length === 0) return;
|
||
|
||
let recovered = 0;
|
||
for (let i = 0; i < detail.steps.length; i++) {
|
||
if (detail.steps[i].status !== "in-progress") continue;
|
||
|
||
let lastPendingAt = -1;
|
||
let lastApproveAt = -1;
|
||
const stepName = detail.steps[i].name;
|
||
// Matches "Step 3 (My Step) → pending"; name is user-controlled, so match
|
||
// on prefix rather than a regex built from the name.
|
||
const transitionPrefix = `Step ${i} (${stepName}) → `;
|
||
const approvePrefix = `code review Step ${i}:`;
|
||
for (let j = 0; j < log.length; j++) {
|
||
const action = log[j].action || "";
|
||
if (action.startsWith(transitionPrefix)) {
|
||
const status = action.slice(transitionPrefix.length).trim();
|
||
if (status === "pending") lastPendingAt = j;
|
||
} else if (action.startsWith(approvePrefix) && action.includes("APPROVE")) {
|
||
lastApproveAt = j;
|
||
}
|
||
}
|
||
|
||
if (lastApproveAt > lastPendingAt) {
|
||
executorLog.log(
|
||
`${taskId}: step ${i} ("${stepName}") already has an approved code review — marking done on resume (skipping review replay)`,
|
||
);
|
||
try {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Step ${i} (${stepName}) recovered as done on resume — code review had already approved before the engine stopped`,
|
||
);
|
||
await this.store.updateStep(taskId, i, "done");
|
||
recovered++;
|
||
} catch (err) {
|
||
executorLog.warn(
|
||
`${taskId}: failed to recover step ${i} on resume: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
executorLog.log(`${taskId}: recovered ${recovered} approved step(s) on resume`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* On resume (task already has a branch from a prior run), walk git history
|
||
* and mark steps as done when a commit matching the step-completion convention
|
||
* is found. This prevents the agent from redoing already-committed work after
|
||
* an auto-requeue.
|
||
*
|
||
* Commit message convention (case-insensitive):
|
||
* feat|chore|fix(FN-XXXX): complete Step N
|
||
*
|
||
* Called after the worktree is acquired and before the agent session starts.
|
||
*/
|
||
private async reconcileStepsFromGitHistory(taskId: string, detail: TaskDetail, worktreePath: string): Promise<void> {
|
||
const baseCommitSha = detail.baseCommitSha;
|
||
if (!baseCommitSha) return;
|
||
|
||
const pendingOrInProgressSteps = detail.steps.filter(
|
||
(s, i) => (s.status === "pending" || s.status === "in-progress") && i > 0,
|
||
);
|
||
if (pendingOrInProgressSteps.length === 0) return;
|
||
|
||
let logOutput: string;
|
||
try {
|
||
const { stdout } = await execAsync(
|
||
`git log "${baseCommitSha}..HEAD" --oneline`,
|
||
{ cwd: worktreePath },
|
||
);
|
||
logOutput = stdout;
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${taskId}: reconcileStepsFromGitHistory — git log failed: ${msg}`);
|
||
return;
|
||
}
|
||
|
||
if (!logOutput.trim()) return;
|
||
|
||
// Match: feat(FN-2978): complete Step 3 / chore(fn-2978)!: Complete step 3
|
||
const stepCommitRegex = /^(?:feat|chore|fix)\([Ff][Nn]-\d+\)(?:!)?:\s*complete\s+step\s+(\d+)/i;
|
||
const reconciledStepIndices = new Set<number>();
|
||
|
||
for (const line of logOutput.split("\n")) {
|
||
// git log --oneline format: "<sha> <message>"
|
||
const message = line.replace(/^[0-9a-f]+ /, "").trim();
|
||
const match = message.match(stepCommitRegex);
|
||
if (!match) continue;
|
||
const stepIndex = parseInt(match[1], 10);
|
||
if (Number.isNaN(stepIndex) || stepIndex < 0 || stepIndex >= detail.steps.length) continue;
|
||
const step = detail.steps[stepIndex];
|
||
if (step.status === "pending" || step.status === "in-progress") {
|
||
reconciledStepIndices.add(stepIndex);
|
||
}
|
||
}
|
||
|
||
for (const stepIndex of reconciledStepIndices) {
|
||
await this.store.updateStep(taskId, stepIndex, "done");
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Reconciled Step ${stepIndex} as done from git history (resume)`,
|
||
undefined,
|
||
this.currentRunContext,
|
||
);
|
||
executorLog.log(`${taskId}: reconciled Step ${stepIndex} as done from git history`);
|
||
}
|
||
|
||
if (reconciledStepIndices.size > 0) {
|
||
// Refresh task and update currentStep to the lowest pending index
|
||
const updated = await this.store.getTask(taskId);
|
||
const lowestPending = updated.steps.findIndex((s) => s.status === "pending" || s.status === "in-progress");
|
||
if (lowestPending >= 0 && lowestPending !== updated.currentStep) {
|
||
await this.store.updateTask(taskId, { currentStep: lowestPending });
|
||
executorLog.log(`${taskId}: set currentStep to ${lowestPending} after step reconciliation`);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check whether the task's branch has any unique commits compared to main.
|
||
* If the branch has no unique commits and the task has steps marked done,
|
||
* those steps represent lost uncommitted work — reset them to "pending"
|
||
* so the next execution doesn't skip them.
|
||
*
|
||
* Called during stuck-kill cleanup when the worktree is about to be destroyed.
|
||
*/
|
||
private async resetStepsIfWorkLost(task: Task): Promise<void> {
|
||
const completedSteps = task.steps.filter(
|
||
(s) => s.status === "done" || s.status === "in-progress",
|
||
);
|
||
if (completedSteps.length === 0) return;
|
||
|
||
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
||
|
||
try {
|
||
// Check if the branch has any unique commits vs main
|
||
const { stdout: mergeBaseStdout } = await execAsync(
|
||
`git merge-base "${branchName}" HEAD 2>/dev/null`,
|
||
{ cwd: this.rootDir, encoding: "utf-8" },
|
||
);
|
||
const { stdout: branchHeadStdout } = await execAsync(
|
||
`git rev-parse "${branchName}" 2>/dev/null`,
|
||
{ cwd: this.rootDir, encoding: "utf-8" },
|
||
);
|
||
const mergeBase = mergeBaseStdout.trim();
|
||
const branchHead = branchHeadStdout.trim();
|
||
|
||
if (mergeBase === branchHead) {
|
||
// Branch has no unique commits — all step work was lost
|
||
executorLog.warn(
|
||
`${task.id} branch has no unique commits — resetting ${completedSteps.length} step(s) to pending`,
|
||
);
|
||
|
||
for (let i = 0; i < task.steps.length; i++) {
|
||
if (task.steps[i].status === "done" || task.steps[i].status === "in-progress") {
|
||
await this.store.updateStep(task.id, i, "pending");
|
||
}
|
||
}
|
||
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Reset ${completedSteps.length} step(s) to pending — branch had no commits (uncommitted work lost with worktree)`,
|
||
);
|
||
}
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`${task.id}: step-reset-on-work-lost failed (non-fatal, steps keep current status): ${msg}`);
|
||
// Branch may not exist or git commands may fail — non-fatal.
|
||
// Steps keep their current status (safe default: agent can
|
||
// inspect the worktree and decide).
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Mark a task as stuck-aborted so the executor's error handling
|
||
* knows not to treat the disposed session as a genuine failure.
|
||
* Called by the stuck task detector's onStuck callback.
|
||
*
|
||
* @param shouldRequeue — true to move the task back to "todo" for retry,
|
||
* false if the stuck kill budget is exhausted (task already marked failed).
|
||
*/
|
||
markStuckAborted(taskId: string, shouldRequeue: boolean = true): void {
|
||
// Terminate step-session executor if active
|
||
const stepExecutor = this.activeStepExecutors.get(taskId);
|
||
if (stepExecutor) {
|
||
stepExecutor.terminateAllSessions().catch(err =>
|
||
executorLog.warn(`Failed to terminate step sessions for stuck task ${taskId}: ${err}`)
|
||
);
|
||
}
|
||
this.stuckAborted.set(taskId, shouldRequeue);
|
||
|
||
// Safety net: if the executor's Promise never resolves (e.g. a bash subprocess
|
||
// is blocking the agent session even after dispose()), force-requeue the task
|
||
// directly after a short grace period. Without this, a task with a hung tool
|
||
// call stays stranded in "in-progress" until the engine restarts.
|
||
if (shouldRequeue && this.executing.has(taskId)) {
|
||
const FORCE_REQUEUE_GRACE_MS = 60_000; // 60 s — generous, but bounded
|
||
setTimeout(async () => {
|
||
if (!this.executing.has(taskId)) return; // executor unwound normally — nothing to do
|
||
executorLog.warn(
|
||
`${taskId} still executing ${FORCE_REQUEUE_GRACE_MS / 1000}s after stuck-kill signal ` +
|
||
`(likely a hung subprocess) — force-requeueing`,
|
||
);
|
||
try {
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Force-requeued after stuck-kill: executor did not unwind within ${FORCE_REQUEUE_GRACE_MS / 1000}s (hung subprocess)`,
|
||
);
|
||
await this.store.updateTask(taskId, { status: "stuck-killed", worktree: null, branch: null });
|
||
await this.store.moveTask(taskId, "todo");
|
||
// Remove from executing so the scheduler can re-dispatch normally.
|
||
// The old Promise is still running but the executing guard is cleared so
|
||
// a fresh execute() call won't be blocked.
|
||
this.executing.delete(taskId);
|
||
this.stuckAborted.delete(taskId);
|
||
executorLog.log(`${taskId} force-requeued to todo`);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`Failed to force-requeue stuck task ${taskId}: ${errorMessage}`);
|
||
}
|
||
}, FORCE_REQUEUE_GRACE_MS);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Handle a loop-detected event from the stuck task detector.
|
||
* Attempts an in-process compact-and-resume before falling back to kill/requeue.
|
||
*
|
||
* This method is the `onLoopDetected` callback wired through the dashboard.
|
||
* It:
|
||
* 1. Checks if the task has an active session
|
||
* 2. Rejects if the one-attempt ceiling has been reached
|
||
* 3. Calls `compactSessionContext()` to compact the conversation
|
||
* 4. Sets recovery-pending state so the execution flow can resume
|
||
*
|
||
* @returns true if the executor accepted recovery ownership (detector skips kill),
|
||
* false if recovery should not be attempted (detector proceeds with kill/requeue)
|
||
*/
|
||
async handleLoopDetected(event: StuckTaskEvent): Promise<boolean> {
|
||
const { taskId } = event;
|
||
const activeEntry = this.activeSessions.get(taskId);
|
||
|
||
// No active session — can't compact, let detector kill/requeue
|
||
if (!activeEntry) {
|
||
executorLog.log(`${taskId} loop detected but no active session — falling back to kill/requeue`);
|
||
return false;
|
||
}
|
||
|
||
// Check attempt ceiling (max 1 compact-and-resume per execute() lifecycle)
|
||
const state = this.loopRecoveryState.get(taskId);
|
||
if (state && state.attempts >= 1) {
|
||
executorLog.log(`${taskId} loop detected but compact ceiling reached — falling back to kill/requeue`);
|
||
return false;
|
||
}
|
||
|
||
// Attempt compaction
|
||
const attempt = (state?.attempts ?? 0) + 1;
|
||
executorLog.log(`${taskId} loop detected (attempt ${attempt}) — attempting compact-and-resume`);
|
||
await this.store.logEntry(taskId, `Loop detected (${event.activitySinceProgress} events since last progress) — attempting compact-and-resume (attempt ${attempt})`);
|
||
|
||
const compactResult = await compactSessionContext(activeEntry.session);
|
||
if (!compactResult) {
|
||
executorLog.log(`${taskId} compaction failed or unavailable — falling back to kill/requeue`);
|
||
await this.store.logEntry(taskId, "Context compaction failed or unavailable — falling back to kill/requeue");
|
||
return false;
|
||
}
|
||
|
||
executorLog.log(`${taskId} compaction succeeded (freed ${compactResult.tokensBefore} tokens) — setting recovery-pending`);
|
||
await this.store.logEntry(taskId, `Context compacted successfully — will resume with fresh context`);
|
||
|
||
// Mark recovery-pending so the execution flow can consume it
|
||
this.loopRecoveryState.set(taskId, { attempts: attempt, pending: true });
|
||
|
||
// Steer the session with a resume prompt to break the loop
|
||
try {
|
||
await activeEntry.session.steer(
|
||
"⚠️ Loop detected: you were repeating actions without making progress. " +
|
||
"The conversation has been compacted. Review the current state carefully, " +
|
||
"check what's already been done (git log, file contents), and take a different " +
|
||
"approach. Do NOT repeat the same actions. Advance to the next step if the " +
|
||
"current work is complete.",
|
||
);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.error(`${taskId} failed to steer after compaction: ${errorMessage}`);
|
||
// Recovery-pending is still set — the execution flow will handle it
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
getWorktreePath(taskId: string): string | undefined {
|
||
return this.activeWorktrees.get(taskId);
|
||
}
|
||
|
||
// ── Agent Spawning ─────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Terminate all child agents spawned by a parent task.
|
||
* Called from the finally block of agentWork when the parent session ends.
|
||
*/
|
||
private async terminateAllChildren(parentTaskId: string): Promise<void> {
|
||
const childIds = this.spawnedAgents.get(parentTaskId);
|
||
if (!childIds || childIds.size === 0) return;
|
||
|
||
executorLog.log(`Terminating ${childIds.size} child agents for parent ${parentTaskId}`);
|
||
|
||
for (const childId of childIds) {
|
||
await this.terminateChildAgent(childId);
|
||
}
|
||
this.spawnedAgents.delete(parentTaskId);
|
||
}
|
||
|
||
/**
|
||
* Terminate a single child agent by ID.
|
||
* Disposes the session, updates AgentStore state, and cleans up tracking Maps.
|
||
*/
|
||
private async terminateChildAgent(childId: string): Promise<void> {
|
||
const childSession = this.childSessions.get(childId);
|
||
if (childSession) {
|
||
childSession.dispose();
|
||
this.childSessions.delete(childId);
|
||
}
|
||
|
||
try {
|
||
await this.options.agentStore?.updateAgentState(childId, "terminated");
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Failed to update spawned child ${childId} state to 'terminated' during cleanup: ${msg}`);
|
||
}
|
||
|
||
// Auto-delete the child agent after a short delay so the UI can observe
|
||
// the terminal state before the agent is removed.
|
||
this.pendingEphemeralDeletions.add(childId);
|
||
const timerId = setTimeout(async () => {
|
||
this.ephemeralCleanupTimers.delete(childId);
|
||
this.pendingEphemeralDeletions.delete(childId);
|
||
try {
|
||
await this.options.agentStore?.deleteAgent(childId);
|
||
} catch (err: unknown) {
|
||
if (this.isBenignEphemeralDeleteRaceError(childId, err)) {
|
||
return;
|
||
}
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Failed to delete spawned agent ${childId}: ${msg}`);
|
||
}
|
||
}, 5000);
|
||
this.ephemeralCleanupTimers.set(childId, timerId);
|
||
|
||
this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1);
|
||
}
|
||
|
||
/**
|
||
* Run a spawned child agent's task to completion.
|
||
* Handles state transitions and cleanup.
|
||
*/
|
||
private async runSpawnedChild(
|
||
agentId: string,
|
||
childSession: AgentSession,
|
||
taskPrompt: string,
|
||
): Promise<void> {
|
||
try {
|
||
await this.options.agentStore?.updateAgentState(agentId, "running");
|
||
} catch (err: unknown) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Failed to update spawned child ${agentId} state to 'running': ${msg}`);
|
||
}
|
||
|
||
try {
|
||
await promptWithFallback(childSession, taskPrompt);
|
||
// Normal completion — mark as active (available)
|
||
try {
|
||
await this.options.agentStore?.updateAgentState(agentId, "active");
|
||
} catch (markActiveErr) {
|
||
executorLog.warn(`Child agent ${agentId} updateAgentState(active) failed: ${markActiveErr instanceof Error ? markActiveErr.message : String(markActiveErr)}`);
|
||
}
|
||
} catch (err: unknown) {
|
||
// Error during execution — mark as error
|
||
try {
|
||
await this.options.agentStore?.updateAgentState(agentId, "error");
|
||
} catch (markErrorErr) {
|
||
executorLog.warn(`Child agent ${agentId} updateAgentState(error) failed: ${markErrorErr instanceof Error ? markErrorErr.message : String(markErrorErr)}`);
|
||
}
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Child agent ${agentId} failed: ${errorMessage}`);
|
||
} finally {
|
||
this.childSessions.delete(agentId);
|
||
this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Create the fn_spawn_agent tool definition.
|
||
* Allows the parent agent to spawn child agents with delegated tasks.
|
||
*/
|
||
private createSpawnAgentTool(taskId: string, worktreePath: string, settings: Settings): ToolDefinition {
|
||
return {
|
||
name: "fn_spawn_agent",
|
||
label: "Spawn Agent",
|
||
description:
|
||
"Spawn a child agent to handle parallel work or specialized sub-tasks. " +
|
||
"Each child runs in its own git worktree (branched from your worktree) and executes autonomously. " +
|
||
"When you end (fn_task_done), all spawned children are terminated.",
|
||
parameters: spawnAgentParams,
|
||
execute: async (_id: string, params: Static<typeof spawnAgentParams>) => {
|
||
const { name, role, task: taskPrompt } = params;
|
||
|
||
// Check if AgentStore is available
|
||
if (!this.options.agentStore) {
|
||
return {
|
||
content: [{ type: "text" as const, text: "Agent spawning is not available (no AgentStore configured)" }],
|
||
details: { agentId: "", state: "error" },
|
||
};
|
||
}
|
||
|
||
// Read spawn limits from settings
|
||
const maxPerParent = settings.maxSpawnedAgentsPerParent ?? 5;
|
||
const maxGlobal = settings.maxSpawnedAgentsGlobal ?? 20;
|
||
|
||
// Check per-parent limit
|
||
const currentPerParent = this.spawnedAgents.get(taskId)?.size ?? 0;
|
||
if (currentPerParent >= maxPerParent) {
|
||
return {
|
||
content: [{ type: "text" as const, text: `Per-parent spawn limit reached (${currentPerParent}/${maxPerParent}). Wait for children to finish or reduce parallelism.` }],
|
||
details: { agentId: "", state: "error" },
|
||
};
|
||
}
|
||
|
||
// Check global limit
|
||
if (this.totalSpawnedCount >= maxGlobal) {
|
||
return {
|
||
content: [{ type: "text" as const, text: `Global spawn limit reached (${this.totalSpawnedCount}/${maxGlobal}). Cannot spawn more agents.` }],
|
||
details: { agentId: "", state: "error" },
|
||
};
|
||
}
|
||
|
||
try {
|
||
// Create agent in AgentStore with reportsTo = parent task ID
|
||
const agent = await this.options.agentStore.createAgent({
|
||
name: name.trim(),
|
||
role: role as AgentCapability,
|
||
reportsTo: taskId,
|
||
metadata: { type: "spawned", parentTaskId: taskId },
|
||
});
|
||
|
||
// Create git worktree for child (branched from parent's worktree)
|
||
const childWorktreeName = generateWorktreeName(this.rootDir);
|
||
const childWorktreePath = join(this.rootDir, ".worktrees", childWorktreeName);
|
||
const childBranch = `fusion/spawn-${agent.id}`;
|
||
await this.createWorktree(childBranch, childWorktreePath, taskId, worktreePath);
|
||
|
||
// Transition agent to active state
|
||
await this.options.agentStore.updateAgentState(agent.id, "active");
|
||
|
||
// Child agents inherit executor instructions
|
||
const childInstructions = await this.resolveInstructionsForRole("executor");
|
||
const childBasePrompt = `You are a child agent spawned by a parent task executor.
|
||
|
||
Your role:
|
||
- Complete the delegated task in your own worktree.
|
||
- Work autonomously, but stay tightly scoped to the delegated request.
|
||
- Prefer existing project patterns over inventing new ones.
|
||
- Run relevant tests and report what you verified.
|
||
- Do not widen scope or refactor unrelated areas.
|
||
|
||
Output expectations:
|
||
- Provide a concise summary of what you changed.
|
||
- Call out files touched and validations run.
|
||
- Explicitly mention unresolved blockers if you could not finish.
|
||
|
||
Parent task: ${taskId}
|
||
Child agent: ${agent.id} (${name})`;
|
||
const childSystemPrompt = buildSystemPromptWithInstructions(childBasePrompt, childInstructions);
|
||
|
||
// Build skill selection context for child agent session
|
||
const childTask = await this.store.getTask(taskId);
|
||
const skillContext = await buildSessionSkillContext({
|
||
agentStore: this.options.agentStore!,
|
||
task: childTask,
|
||
sessionPurpose: "executor",
|
||
projectRootDir: this.rootDir,
|
||
pluginRunner: this.options.pluginRunner,
|
||
});
|
||
const parentAgent = childTask.assignedAgentId
|
||
? await this.options.agentStore.getAgent(childTask.assignedAgentId).catch(() => null)
|
||
: null;
|
||
const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig)
|
||
?? extractRuntimeHint(parentAgent?.runtimeConfig);
|
||
|
||
// Resolve executor model via canonical lane hierarchy so child agents
|
||
// honor project executionProvider/executionModelId overrides (parity
|
||
// with main executor at the top of agentWork()).
|
||
const { provider: childExecutorProvider, modelId: childExecutorModelId } =
|
||
resolveExecutorModelPair(undefined, undefined, settings);
|
||
|
||
// Create child agent session
|
||
const { session: childSession } = await createResolvedAgentSession({
|
||
sessionPurpose: "executor",
|
||
runtimeHint: childRuntimeHint,
|
||
pluginRunner: this.options.pluginRunner,
|
||
cwd: childWorktreePath,
|
||
systemPrompt: childSystemPrompt,
|
||
tools: "coding",
|
||
defaultProvider: childExecutorProvider,
|
||
defaultModelId: childExecutorModelId,
|
||
fallbackProvider: settings.fallbackProvider,
|
||
fallbackModelId: settings.fallbackModelId,
|
||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||
});
|
||
|
||
// Store tracking state
|
||
this.childSessions.set(agent.id, childSession);
|
||
if (!this.spawnedAgents.has(taskId)) {
|
||
this.spawnedAgents.set(taskId, new Set());
|
||
}
|
||
this.spawnedAgents.get(taskId)!.add(agent.id);
|
||
this.totalSpawnedCount++;
|
||
|
||
// Run child asynchronously (don't await — parent continues working)
|
||
this.runSpawnedChild(agent.id, childSession, taskPrompt).catch((err: unknown) => {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
executorLog.warn(`Child agent ${agent.id} async error: ${errorMessage}`);
|
||
});
|
||
|
||
const result: SpawnAgentResult = {
|
||
agentId: agent.id,
|
||
name: agent.name,
|
||
state: "running",
|
||
role: agent.role,
|
||
message: `Agent "${name}" spawned and executing task: ${taskPrompt.slice(0, 100)}${taskPrompt.length > 100 ? "..." : ""}`,
|
||
};
|
||
|
||
return {
|
||
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
||
details: result,
|
||
};
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
return {
|
||
content: [{ type: "text" as const, text: `Failed to spawn agent: ${errorMessage}` }],
|
||
details: { agentId: "", state: "error", message: errorMessage },
|
||
};
|
||
}
|
||
},
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Format a timestamp for display in steering comments.
|
||
* Returns relative time for recent comments, absolute date for older ones.
|
||
*/
|
||
function formatTimestamp(iso: string): string {
|
||
const date = new Date(iso);
|
||
const now = new Date();
|
||
const diffMs = now.getTime() - date.getTime();
|
||
const diffMin = Math.floor(diffMs / 60000);
|
||
const diffHr = Math.floor(diffMin / 60);
|
||
const diffDay = Math.floor(diffHr / 24);
|
||
|
||
if (diffMin < 1) return "just now";
|
||
if (diffMin < 60) return `${diffMin}m ago`;
|
||
if (diffHr < 24) return `${diffHr}h ago`;
|
||
if (diffDay < 7) return `${diffDay}d ago`;
|
||
return date.toLocaleDateString();
|
||
}
|
||
|
||
// Project commands are injected here (for reliability) and also in the PROMPT.md (by triage).
|
||
// This ensures the executor agent always sees the authoritative commands from settings,
|
||
// even if the PROMPT.md was written manually or before commands were configured.
|
||
function scopePromptToWorktree(prompt: string, rootDir?: string, worktreePath?: string): string {
|
||
if (!rootDir || !worktreePath || rootDir === worktreePath || !prompt.includes(rootDir)) {
|
||
return prompt;
|
||
}
|
||
|
||
return prompt
|
||
.replaceAll(`${rootDir}/`, `${worktreePath}/`)
|
||
.replaceAll(`${worktreePath}/.fusion/`, `${rootDir}/.fusion/`);
|
||
}
|
||
|
||
export function buildExecutionPrompt(task: TaskDetail, rootDir?: string, settings?: Settings, worktreePath?: string): string {
|
||
const prompt = scopePromptToWorktree(task.prompt, rootDir, worktreePath);
|
||
const reviewMatch = prompt.match(/##\s*Review Level[:\s]*(\d)/);
|
||
const reviewLevel = reviewMatch ? parseInt(reviewMatch[1], 10) : 0;
|
||
|
||
// Build author arg for git commits based on settings
|
||
const authorArg = settings?.commitAuthorEnabled !== false
|
||
? ` --author="${settings?.commitAuthorName || "Fusion"} <${settings?.commitAuthorEmail || "noreply@runfusion.ai"}>"`
|
||
: "";
|
||
|
||
const sourceIssueRef = task.sourceIssue?.provider === "github" && task.sourceIssue.repository && task.sourceIssue.issueNumber
|
||
? `${task.sourceIssue.repository}#${task.sourceIssue.issueNumber}`
|
||
: "";
|
||
|
||
// Build step progress for resume
|
||
const hasProgress = task.steps.length > 0 && task.steps.some((s) => s.status !== "pending");
|
||
let progressSection = "";
|
||
if (hasProgress) {
|
||
const doneSteps = task.steps
|
||
.map((s, i) => ({ ...s, index: i }))
|
||
.filter((s) => s.status === "done");
|
||
const currentStep = task.currentStep;
|
||
const currentStepInfo = task.steps[currentStep];
|
||
|
||
progressSection = `
|
||
## ⚠️ RESUMING — Previous progress exists
|
||
|
||
This task was already partially executed. DO NOT redo completed steps.
|
||
|
||
### Step status:
|
||
${task.steps.map((s, i) => `- Step ${i} (${s.name}): **${s.status}**`).join("\n")}
|
||
|
||
### Resume from: Step ${currentStep}${currentStepInfo ? ` (${currentStepInfo.name})` : ""}
|
||
|
||
${doneSteps.length > 0 ? `Steps ${doneSteps.map((s) => s.index).join(", ")} are already complete — skip them entirely.` : ""}
|
||
Check the git log to understand what was already implemented:
|
||
\`\`\`bash
|
||
git log --oneline
|
||
\`\`\`
|
||
`;
|
||
}
|
||
|
||
// Build attachments section
|
||
let attachmentsSection = "";
|
||
if (task.attachments && task.attachments.length > 0 && rootDir) {
|
||
const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
|
||
const lines = ["## Attachments", ""];
|
||
for (const att of task.attachments) {
|
||
const absPath = `${rootDir}/.fusion/tasks/${task.id}/attachments/${att.filename}`;
|
||
if (IMAGE_MIMES.has(att.mimeType)) {
|
||
lines.push(`- **${att.originalName}** (screenshot): \`${absPath}\``);
|
||
} else {
|
||
lines.push(`- **${att.originalName}** (${att.mimeType}): \`${absPath}\` — read for context`);
|
||
}
|
||
}
|
||
attachmentsSection = "\n" + lines.join("\n") + "\n";
|
||
}
|
||
|
||
// Build project commands section from settings
|
||
let commandsSection = "";
|
||
if (settings?.testCommand || settings?.buildCommand) {
|
||
const lines = ["## Project Commands"];
|
||
if (settings.testCommand) lines.push(`- **Test:** \`${settings.testCommand}\``);
|
||
if (settings.buildCommand) lines.push(`- **Build:** \`${settings.buildCommand}\``);
|
||
commandsSection = "\n" + lines.join("\n") + "\n";
|
||
}
|
||
|
||
// Build project memory section from settings
|
||
// When enabled, agents consult and update project memory for durable project learnings.
|
||
// Backend-aware: instructions branch based on memoryBackendType (file, readonly, qmd)
|
||
const memoryEnabled = settings?.memoryEnabled !== false;
|
||
let memorySection = "";
|
||
if (memoryEnabled && rootDir) {
|
||
memorySection = "\n" + buildExecutionMemoryInstructions(rootDir, settings);
|
||
}
|
||
|
||
// Build steering comments section (last 10 comments only to avoid context bloat)
|
||
let steeringSection = "";
|
||
if (task.steeringComments && task.steeringComments.length > 0) {
|
||
const recentComments = [...task.steeringComments].slice(-10);
|
||
const lines = [
|
||
"",
|
||
"## Steering Comments",
|
||
"",
|
||
"The following comments were added by the user during execution. Consider adjusting your approach or replanning remaining steps based on this feedback.",
|
||
"",
|
||
];
|
||
for (const comment of recentComments) {
|
||
const timestamp = formatTimestamp(comment.createdAt);
|
||
lines.push(`**${comment.author}** — ${timestamp}`);
|
||
lines.push(`> ${comment.text}`);
|
||
lines.push("");
|
||
}
|
||
steeringSection = lines.join("\n");
|
||
}
|
||
|
||
return `Execute this task.
|
||
|
||
## Task: ${task.id}
|
||
${task.title ? `**${task.title}**` : ""}
|
||
${task.dependencies.length > 0 ? `Dependencies: ${task.dependencies.join(", ")}` : ""}
|
||
|
||
## PROMPT.md
|
||
|
||
${prompt}
|
||
${attachmentsSection}${commandsSection}${memorySection}${progressSection}${steeringSection}
|
||
## Review level: ${reviewLevel}
|
||
|
||
${reviewLevel === 0 ? "No reviews required. Implement directly." : ""}
|
||
${reviewLevel >= 1 ? `Before implementing each step (except Step 0 and the final step), call:
|
||
\`fn_review_step(step=N, type="plan", step_name="...")\`` : ""}
|
||
${reviewLevel >= 2 ? `After implementing + committing each step, call:
|
||
\`fn_review_step(step=N, type="code", step_name="...", baseline="<SHA from before step>")\`` : ""}
|
||
${reviewLevel >= 3 ? `After tests, also call fn_review_step with type="code" for test review.` : ""}
|
||
|
||
## Worktree Boundaries
|
||
|
||
You are running in an **isolated git worktree**. This means:
|
||
|
||
- **All code changes must be made inside the current worktree directory.** Do not modify files outside the worktree.
|
||
- **Exception — Project memory:** You MAY read and write to files under \`.fusion/memory/\` at the project root to save durable project learnings.
|
||
- **Exception — Task attachments:** You MAY read files under \`.fusion/tasks/{taskId}/attachments/\` at the project root for context.
|
||
- **Exception — Sibling task specs:** You MAY read \`.fusion/tasks/{taskId}/PROMPT.md\` and \`.fusion/tasks/{taskId}/task.json\` at the project root (read-only) to consult dependency tasks' specifications.
|
||
- **Shell commands** run inside the worktree by default. Avoid using \`cd\` to navigate outside the worktree.
|
||
|
||
## Begin
|
||
|
||
${hasProgress
|
||
? `Resume from Step ${task.currentStep}. Do NOT redo completed steps.`
|
||
: "Start with Step 0 (Preflight). Work through each step in order."}
|
||
Use \`fn_task_update\` to report progress on every step transition.
|
||
Use \`fn_task_log\` for important actions and decisions.
|
||
Use \`fn_task_create\` for truly separate follow-up work, not for fixes required to get tests, build, or typecheck back to green.
|
||
Commit at step boundaries: \`git commit -m "feat(${task.id}): complete Step N — description"${sourceIssueRef ? ` -m "Ref: ${sourceIssueRef}"` : ""}${authorArg}\`
|
||
When all steps are complete: call \`fn_task_done()\`
|
||
|
||
If a build command is configured, run that exact command in this worktree before calling \`fn_task_done()\`.
|
||
Treat a non-zero exit code as a blocking failure. Do not claim success without a real passing run.
|
||
Run the configured/full test suite and fix failures even when that requires edits outside the original File Scope.
|
||
If the repo has a lint command (e.g. \`pnpm lint\`, \`npm run lint\`), run it before \`fn_task_done()\` and fix any failures it reports.
|
||
If the repo has a typecheck command, run it before \`fn_task_done()\` and fix any failures it reports.
|
||
Use \`fn_task_create\` for truly separate follow-up work, not for fixes required to get tests, build, or typecheck back to green.
|
||
If lint is configured and failing, fix that too before completion.
|
||
**CRITICAL: Resolve ALL test failures (and any lint/typecheck failures) before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.`;
|
||
}
|
||
|
||
/**
|
||
* Format a comment for injection into a running agent session.
|
||
* Used for real-time steering during task execution.
|
||
*/
|
||
function formatCommentForInjection(comment: import("@fusion/core").SteeringComment): string {
|
||
const timestamp = formatTimestamp(comment.createdAt);
|
||
return `📣 **New feedback** — ${timestamp} (${comment.author}):\n\n${comment.text}\n\nPlease adjust your approach based on this feedback.`;
|
||
}
|
||
|
||
/**
|
||
* Result of a pseudo-pause detection check.
|
||
*/
|
||
export interface PseudoPauseResult {
|
||
/** Detection method: "regex" if a regex pattern matched, "structural" for structural
|
||
* heuristics, or "none" if no pseudo-pause was detected. */
|
||
kind: "regex" | "structural" | "none";
|
||
/** The matched text or pattern description when kind is not "none". */
|
||
matched?: string;
|
||
}
|
||
|
||
/**
|
||
* Detect whether the last assistant text output looks like a "pseudo-pause" —
|
||
* where the agent ended a turn by asking for permission or summarizing progress
|
||
* instead of calling a tool.
|
||
*
|
||
* Returns a {@link PseudoPauseResult} describing the detection kind and the
|
||
* matched text/pattern. Returns `{ kind: "none" }` when no pseudo-pause is found.
|
||
*
|
||
* @param lastText - The last assistant text output from the session.
|
||
*/
|
||
export function detectPseudoPause(lastText: string): PseudoPauseResult {
|
||
if (!lastText || lastText.trim().length === 0) {
|
||
return { kind: "none" };
|
||
}
|
||
|
||
const regexPatterns: RegExp[] = [
|
||
/\bif you (?:want|wish|need|like|prefer|'?d like)\b/i,
|
||
/\bshould I (?:continue|proceed|go ahead|move on|start|begin)\b/i,
|
||
/\blet me know\b/i,
|
||
/\b(?:want|would you like) me to (?:continue|proceed|finish|complete|do)\b/i,
|
||
/\bready to (?:proceed|continue|move on|begin)\b/i,
|
||
/\bshall I\b/i,
|
||
/\b(?:awaiting|waiting for) (?:your )?(?:approval|confirmation|go-ahead|response)\b/i,
|
||
];
|
||
|
||
for (const pattern of regexPatterns) {
|
||
const match = pattern.exec(lastText);
|
||
if (match) {
|
||
// Capture surrounding context (up to 120 chars around the match)
|
||
const start = Math.max(0, match.index - 40);
|
||
const end = Math.min(lastText.length, match.index + match[0].length + 80);
|
||
const snippet = lastText.slice(start, end).replace(/\n+/g, " ").trim();
|
||
return { kind: "regex", matched: snippet };
|
||
}
|
||
}
|
||
|
||
// Structural fallback: long output that ends with a question or a markdown "next steps" heading
|
||
const trimmed = lastText.trimEnd();
|
||
if (trimmed.length > 200) {
|
||
if (trimmed.endsWith("?")) {
|
||
const lastLine = trimmed.split("\n").at(-1) ?? trimmed;
|
||
return { kind: "structural", matched: lastLine.trim() };
|
||
}
|
||
const nextStepsPattern = /(?:^|\n)#+\s*(?:notes?|next steps?|summary|what'?s? next)\s*:?\s*$/i;
|
||
if (nextStepsPattern.test(trimmed)) {
|
||
const lastLine = trimmed.split("\n").at(-1) ?? trimmed;
|
||
return { kind: "structural", matched: lastLine.trim() };
|
||
}
|
||
// Also catch plain "Next steps:" or "### Next steps" at the very end
|
||
if (/next steps?\s*:?\s*$/i.test(trimmed)) {
|
||
const lastLine = trimmed.split("\n").at(-1) ?? trimmed;
|
||
return { kind: "structural", matched: lastLine.trim() };
|
||
}
|
||
}
|
||
|
||
return { kind: "none" };
|
||
}
|
||
|
||
/**
|
||
* Detect if a steering comment contains a review handoff request.
|
||
* Matches common handoff phrases that agents can use to request
|
||
* human review of their work.
|
||
*/
|
||
export function detectReviewHandoffIntent(commentText: string): boolean {
|
||
const text = commentText.toLowerCase();
|
||
const handoffPhrases = [
|
||
"send it back to me",
|
||
"hand off to user",
|
||
"needs human review",
|
||
"assign to user",
|
||
"return to user",
|
||
"user review needed",
|
||
"requesting user review",
|
||
];
|
||
|
||
return handoffPhrases.some((phrase) => text.includes(phrase));
|
||
}
|