// port-4040-allowlist: this file embeds the "never kill port 4040" rule in the executor prompt. import { exec, execFile, execSync } from "node:child_process"; import { promisify } from "node:util"; import { setImmediate as setImmediateCb } from "node:timers"; // Internal git plumbing intentionally bypasses sandbox backends. const execAsync = promisify(exec); const execFileAsync = promisify(execFile); const WORKFLOW_THINKING_LEVEL_SET: ReadonlySet = new Set(THINKING_LEVELS); import { basename, delimiter, isAbsolute, join, relative, resolve as resolvePath } from "node:path"; import { existsSync, lstatSync, realpathSync } from "node:fs"; import { readFile, rm, writeFile } from "node:fs/promises"; import { DEFAULT_PROVIDER_INSTANCE_ID, type ProviderInstanceRef, type TaskStore, type Task, type TaskDetail, type TaskTokenUsage, type StepStatus, type Settings, type WorkflowStep, type MissionStore, type AsyncMissionStore, type Slice, type AgentState, type AgentCapability, type RunMutationContext, type AgentHeartbeatConfig, type Agent, type AgentMemoryInclusionMode, type ProjectSettings, type MergeResult, type WorkflowIrNode, type WorkflowIrNodeKind, type WorkflowStepResult as CoreWorkflowStepResult, type ThinkingLevel } from "@fusion/core"; import { getUnmetSchedulingDependencies } from "./scheduler.js"; import type { ImplementationExit, ImplementationExitReporter } from "./executor/implementation-exit.js"; import { emitWorkflowLifecycleEvent } from "@fusion/core"; import { resolveTaskLifecycleColumns, resolveProjectColumnsForRoles, resolveWipTargetForTask, resolveTerminalColumns, RetryStormError, serializeRetryStormError, evaluateCompletedPromotionFailureProvenance, evaluateSkipBypassTaint, resolveWorkflowIrForTask, columnsWithFlag, evaluateForeachMergeProof, resolveCompleteColumn, resolveMergeOrchestrationColumn, resolveReboundTarget, resolveLifecycleColumns, resolveColumnAgentBinding, resolveEffectiveAgent, instanceNodeId, getWorkflowExtensionRegistry, getBuiltinWorkflow, parseNoOpCompletionMarker, allowsAutoMergeProcessing, resolveEffectiveAutoMerge, isLiveSharedBranchGroupMemberIntegration, resolveMaxAutoMergeRetries, resolveMaxConsecutiveToolFailureRetries, resolveConsecutiveToolFailureRetryBackoffMs, resolveConsecutiveToolFailureThreshold, resolveExecutorEscalationTarget, resolveOptionalStepRevisionBudget, resolveOptionalReviewRevisionBudget, DEFAULT_MAX_POST_REVIEW_FIXES, COMPLETION_SUMMARY_NODE_ID, upsertWorkflowStepResult, AWAITING_APPROVAL_PAUSE_REASON, THINKING_LEVELS, ACTIVE_WORKFLOW_WORK_ITEM_STATES, AgentStore, resolveExecutorFallbackModel, resolveValidatorFallbackModel } from "@fusion/core"; import { finalizeProvenAutoMergeTask } from "./auto-merge-finalization.js"; import { mergeEffectiveSettings } from "./effective-settings.js"; import { generateFeatureVideo, type GenerateFeatureVideoOptions } from "./review-artifacts/feature-video.js"; import { moveTaskToReplanColumn, resolvePlannerLanes, resolvePlannerLanesForTaskAsync, resolveReplanTargetColumn } from "./replan-target.js"; import type { TaskStep, WorkflowIr, WorkflowFieldDefinition, WorkflowColumnAgent, EffectiveAgentInput, WorkflowWorkEngineDispatchResult, WorkflowWorkItem, TaskMoveLanes } from "@fusion/core"; import { WorkflowGraphTaskRunner, type WorkflowGraphTaskRunResult, type WorkflowColumnBoundaryHooks } from "./workflow-graph-task-runner.js"; import { createExecutorColumnBoundaryHooks } from "./workflow-column-boundary-hooks.js"; import { ensureWorkflowCompletionSummary } from "./workflow-completion-summary.js"; import { createCodeNodeRunner } from "./code-node-runner.js"; import { getTaskReviewCheckoutPath, resolveReviewCheckoutCwd } from "./review-checkout.js"; import { getActiveNotificationService } from "./notifier.js"; import type { ParseStepsHandlerDeps, CodeNodeRunner } from "./workflow-node-handlers.js"; import type { WorkflowBranchPersistence, WorkflowBranchRunState } from "./workflow-graph-branches.js"; import type { WorkflowStepInstancePersistence, WorkflowStepInstanceState, } from "./workflow-graph-foreach.js"; import { FOREACH_ACTIVE_CONTEXT_KEY, SEAM_GOVERNING_NODE_CONTEXT_KEY, SEAM_SKILL_NAME_CONTEXT_KEY, SEAM_THINKING_LEVEL_CONTEXT_KEY, SPLIT_ACTIVE_CONTEXT_KEY, type ForeachActiveContext, type WorkflowLegacySeams, } from "./workflow-node-handlers.js"; import { MERGE_REGION_KINDS, PLAN_REVIEW_PROVIDER_FAILURE_HOLD_VALUE, SESSION_CONTENTION_HOLD_VALUE, WORKFLOW_DRIFT_PARK_CONTEXT_KEY, WORKFLOW_NODE_ENGINE_PAUSE_ABORT_KIND, WORKFLOW_OPTIONAL_GROUP_CONTEXT_KEY, } from "./workflow-graph-executor.js"; import type { WorkflowNodePreparationRequirement, WorkflowNodeResult } from "./workflow-graph-executor.js"; import { workflowNodeRequiresWorktree } from "./workflow-node-execution-needs.js"; import type { AuditPrimitiveInput, PreparedWorktree, WorkflowPrimitiveContext, WorkflowRuntimePrimitives, } from "./runtime-primitives.js"; import { createWorkflowRuntimePrimitiveProvider } from "./workflow-runtime-primitive-provider.js"; import { WorkflowCustomNodeExecutionService } from "./workflow-custom-node-execution.js"; import { WorkflowReviewService } from "./workflow-review-service.js"; import { WorkflowPlanningService } from "./workflow-planning-service.js"; import { buildPlanVerifiedMessage, buildReviewUnavailableMessage, buildReviewRollbackFailureMessage, buildReviewVerdictMessage, buildStepFailureMessage, emitProactiveStatus, sanitizeFailureReason, } from "./proactive-status.js"; import { ApprovalRequestStore, buildExecutionMemoryInstructions, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, resolveAgentPrompt, resolvePersistAgentThinkingLog, resolveEffectiveAgentPermissionPolicy, resolveAgentMemoryInclusionMode, loadWorkspaceConfig, type WorkspaceConfig, type RunCommandResult, FUSION_RUNTIME_SELF_AWARENESS, } from "@fusion/core"; import { findWorktreeUser, getConflictedFiles } from "./merger.js"; import { runVerificationCommand, summarizeVerificationOutput, VERIFICATION_LOG_MAX_CHARS, type VerificationResult, } from "./verification-utils.js"; import { canonicalFusionBranchName, canonicalStepInstanceBranchName, generateWorktreeName, resolveTaskWorkingBranch } from "./worktree-names.js"; import { resolveTaskWorktreePath, resolveWorktreesDir } from "./worktree-paths.js"; import { Type, type Static } from "@earendil-works/pi-ai"; import { describeModel, formatModelMarkerDetails, promptWithFallback, compactSessionContext } from "./pi.js"; import { buildAgentGatedActionSummary } from "./permanent-agent-gating.js"; import { accumulateSessionTokenUsage, captureSessionTokenBaseline, mergeTokenUsagePerModel, resetSessionTokenBaseline } from "./session-token-usage.js"; import { finalizePlanningSegment, startPlanningSegment, resolveEphemeralTaskCreationPolicy } from "@fusion/core"; import { enforceTaskTokenBudgetForPersist } from "./token-budget-enforcer.js"; import { createResolvedAgentSession, extractRuntimeHint, resolveExecutorSessionModel, resolveValidatorSessionModel, resolveExecutorThinkingLevel, resolveExecutorFallbackThinkingLevel, resolveValidatorThinkingLevel, resolveValidatorFallbackThinkingLevel, } from "./agent-session-helpers.js"; import { buildSessionSkillContext } from "./session-skill-context.js"; import { resolveMcpServersForStore } from "./mcp-resolution.js"; import { proseSignalsClearApproval, extractJsonObjectCandidates, type ReviewVerdict, type ReviewResult } from "./reviewer.js"; import { buildUserCommentsPromptSection, selectUserCommentsForAgentContext } from "./agent-user-comments.js"; import { resolveSandboxBackend } from "./sandbox/index.js"; import type { SandboxBackend } from "./sandbox/types.js"; import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@earendil-works/pi-coding-agent"; import { PRIORITY_EXECUTE, computeTopLevelConcurrencyClaimedFromStore, dropPreHeldExecutorSlot, takePreHeldExecutorSlot, type AgentSemaphore, } from "./concurrency.js"; // FNXC:Workspace 2026-06-21-15:00: F5/F8 — wire in the previously dead workspace-path helpers. // `normalizeRepoRelPath` is the single shared scope-path normalizer (F8); `deriveRepoScopeSubset` // maps the task's repo-prefixed declared File Scope to a repo-LOCAL subset so the per-repo scope-leak // filter reuses the SAME always-allowed/scope-match surface as the non-workspace path (F5). One-way // executor→workspace-paths edge (workspace-paths imports nothing). import { deriveRepoScopeSubset, normalizeRepoRelPath } from "./workspace-paths.js"; import { preservedWorktreeTargetPathForTask } from "./worktree-pinning.js"; import { RemovalReason, classifyTaskWorktree, describeRegisteredWorktrees, detectGitRepository, detectNestedWorktreeRoot, getRegisteredWorktreePaths, isInsideWorktreesDir, isRegisteredGitWorktree, relocateReclaimableWorktreeIntoRoot, removeWorktree, type GitRepoDetection, type WorktreePool } from "./worktree-pool.js"; import { attemptBranchAutocorrect } from "./branch-autocorrect.js"; import { ActiveSessionWorktreeRemovalError } from "./worktree-backend.js"; import {canonicalizeWorktreePath, registerArchiveWorkspaceWorktreeDisposer, registerArchiveWorktreeDisposer, registerTaskMoveDisposer} from "@fusion/core"; import { ActiveSessionPathHeldByForeignTaskError, acquireActiveSessionPath, activeSessionRegistry, executingTaskLock, reconcileSelfOwnedActiveSessionForRemoval, type ActiveSessionKind, } from "./active-session-registry.js"; // CLI Agent Executor (U7): task ↔ CLI session orchestration seam. import { CliTaskSession, launchCliTaskSession, killLiveTaskSessions, type CliTaskOutcome, type ResolvedCliExecutorConfig, } from "./cli-agent/task-session.js"; import type { CliSessionManager } from "./cli-agent/session-manager.js"; import { CliConcurrencyLimitError } from "./cli-agent/session-manager.js"; import type { TelemetryHub } from "./cli-agent/telemetry-hub.js"; import type { CliAdapterRegistry } from "./cli-agent/adapter.js"; import type { CliSessionStore } from "@fusion/core"; import { StaleWorktreeIndexLockError, classifyStaleLock, parseIndexLockPath, tryRemoveStaleLock, } from "./worktree-stale-lock.js"; import { parseStaleRegistrationPath, recoverStaleRegistration } from "./worktree-stale-registration.js"; import { BranchConflictError, BranchCrossContaminationError, assertCleanBranchAtBase, autoRecoverCrossContamination, classifyBootstrapMisbinding, classifyForeignCommits, classifyForeignOnlyContamination, classifyMisroutedForeignCommit, isBranchConflictError, reanchorBranchToBase, inspectBranchConflict, reportBranchAttribution, } from "./branch-conflicts.js"; import { classifyOrphanOurAdvance, rehomeOrphanOntoIntegration, } from "./merger-orphan-rehome.js"; import { BranchAttributionError, filterFilesToOwnTaskCommits } from "./branch-attribution.js"; import { resolveIntegrationBranch } from "./integration-branch.js"; import { AgentLogger } from "./agent-logger.js"; import { createLogger, executorLog, reviewerLog, formatError } from "./logger.js"; import { TokenCapDetector } from "./token-cap-detector.js"; import { isUsageLimitError, checkSessionError, type UsageLimitPauser } from "./usage-limit-detector.js"; import { isNonContinuableSessionError, isNonPlanDefectPlanReviewFailure, isSessionContentionError, isTransientError, isSilentTransientError } from "./transient-error-detector.js"; import { withRateLimitRetry } from "./rate-limit-retry.js"; import type { CredentialInstanceRotator } from "./credential-instance-rotation.js"; import { detectExternalIntegrationEvidenceGaps, formatExternalIntegrationEvidenceDiagnostic, } from "./spec-validation/external-integration-evidence.js"; import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js"; import { isRequiredArtifactReadFailedValue, parseRequiredArtifactMissingValue, requiredArtifactMissingValue, requiredArtifactReadFailedValue, workflowEntryArtifacts, } from "./required-workflow-artifacts.js"; import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js"; import type { PluginRunner } from "./plugin-runner.js"; import { isContextLimitError } from "./context-limit-detector.js"; import { StepSessionExecutor } from "./step-session-executor.js"; import { isUsableWorktreeDirectory, makeAncestryBlastRadiusGuard, resetStepToBaseline, runTaskStep, type RunTaskStepResult, } from "./step-runner.js"; // FNXC:MergerUnification 2026-06-21-19:05: the foundation branch imported `acquireWorkspaceRepoWorktree` here but never used it in executor.ts (the agent tool wraps it via agent-tools.ts), which fails lint on the inherited base. Removed until master-plan U1 re-adds it together with its per-repo acquisition usage. import { acquireTaskWorktree, type AcquireTaskWorktreeResult } from "./worktree-acquisition.js"; import { resolveCapturedBaseCommitSha } from "./base-commit-capture.js"; import { installTaskWorktreeIdentityGuard } from "./worktree-hooks.js"; import { resolveAgentInstructions, buildSystemPromptWithInstructions, buildPluginPromptSection, } from "./agent-instructions.js"; import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js"; import { resolveAndEmitGoalContext } from "./goal-injection-diagnostics.js"; import type { AgentReflectionService } from "./agent-reflection.js"; import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js"; import { AutoRecoveryDispatcher } from "./auto-recovery.js"; import { classifyMissingWorktreeSessionStartFailure, extractMissingWorktreePathFromSessionStartFailure, isMissingWorktreeSessionStartFailure, } from "./restart-recovery-coordinator.js"; import { BranchWorktreeAutoRecoveryHandler } from "./auto-recovery-handlers/branch-worktree.js"; import { autoRecoverWorktreeSessionStartFailure, COMPLETED_BLOCKED_PAUSE_REASON, MAX_WORKTREE_SESSION_RETRIES, PAUSE_ABORT_PARK_ERROR_MARKER, PAUSE_ABORT_PARK_OPERATOR_MARKER } from "./self-healing.js"; import { ContaminationAutoRecoveryHandler } from "./auto-recovery-handlers/contamination.js"; import { createFileScopeAutoRecoveryHandler } from "./auto-recovery-handlers/file-scope.js"; import { ReadonlyViolationError, filterCustomToolsForReadonly } from "./workflow-step-tool-policy.js"; import { evaluateSpecStaleness, getPromptPath } from "./spec-staleness.js"; import { resolveDedicatedPlannerColumnsForTask } from "./planner-lane-resolution.js"; import { createAgentCreateTool, createAgentDeleteTool, createDelegateTaskTool, createTaskAssignTool, createGetAgentConfigTool, createListAgentsTool, createMemoryTools, createGoalRetrievalTools, createMissionTools, createIdeationTools, createWebFetchTool, createReadMessagesTool, createReflectOnPerformanceTool, createUpdateAgentConfigTool, createResearchTools, createSendMessageTool, createArtifactListTool as sharedCreateArtifactListTool, createArtifactRegisterTool as sharedCreateArtifactRegisterTool, createArtifactViewTool as sharedCreateArtifactViewTool, createTaskCreateTool as sharedCreateTaskCreateTool, isAgentTaskCreateToolAvailable, isAgentDelegateTaskToolAvailable, createTaskDocumentReadTool as sharedCreateTaskDocumentReadTool, createTaskDocumentWriteTool as sharedCreateTaskDocumentWriteTool, createTaskPromptWriteTool as sharedCreateTaskPromptWriteTool, createTaskFileScopeAddTool as sharedCreateTaskFileScopeAddTool, createTaskLogTool as sharedCreateTaskLogTool, createTaskLogsReadTool as sharedCreateTaskLogsReadTool, createWorkflowListTool as sharedCreateWorkflowListTool, createWorkflowGetTool as sharedCreateWorkflowGetTool, createWorkflowValidateTool as sharedCreateWorkflowValidateTool, createWorkflowSelectTool as sharedCreateWorkflowSelectTool, createTaskPromoteTool as sharedCreateTaskPromoteTool, createWorkflowCreateTool as sharedCreateWorkflowCreateTool, createWorkflowUpdateTool as sharedCreateWorkflowUpdateTool, createWorkflowDeleteTool as sharedCreateWorkflowDeleteTool, createWorkflowSettingsTool as sharedCreateWorkflowSettingsTool, createTraitListTool as sharedCreateTraitListTool, createAcquireRepoWorktreeTool, } from "./agent-tools.js"; import { getTaskCompletionBlockerForStore } from "./task-completion.js"; import { createStreamingDeltaNormalizer } from "./streaming-delta.js"; import { getEnabledPluginTools, getResearchGuidanceForSurface, isResearchToolSurfaceEnabled, } from "./tool-availability.js"; import { createFusionAuthStorage, createFusionModelRegistry } from "./auth-storage.js"; import { createRunVerificationTool, runVerificationCommand as runTaskVerificationCommand } from "./run-verification-tool.js"; import { createFallbackModelObserver } from "./fallback-model-observer.js"; import { recordRetry } from "./retry-burned-logger.js"; import type { AgentActionGateContext } from "./agent-action-gate.js"; /* FNXC:WorkflowLifecycle 2026-07-26-11:20: KB-PROV: Provenance of a pause/abort marker, in one named union so the ~10 signatures that pass it around cannot drift apart. - `hard-cancel` — OPERATOR withdrawal only. AGENTS.md "Move-Task contract": user `moveTask(in-progress -> todo)`, task soft-delete, and a user-sourced move out of a planning lane. These carry `userCanceled: true` into `awaitAbortInFlightTaskWork`. - `engine-abort` — ENGINE/lifecycle teardown with no operator intent: workflow rerun bounces, archive disposal, approval-gate suspension, engine-sourced moves, `abortAllInFlight` (shutdown/global stop), stuck-kill force-requeue. Before KB-PROV these were mislabeled `hard-cancel`. - `global-pause` / `merge-seam` / `completion-finalize` — unchanged FN-6568/FN-6625 seams. `hard-cancel` and `engine-abort` are the two "generic" aborts; test them together with `isGenericAbortProvenance()`. */ export type PausedAbortProvenance = "global-pause" | "merge-seam" | "hard-cancel" | "engine-abort" | "completion-finalize"; /* FNXC:WorkflowLifecycle 2026-07-26-11:20: KB-PROV: The benign-abort classifiers in handleGraphFailure were written against the pre-split `hard-cancel` catch-all and exist PRECISELY to recover engine-initiated aborts (FN-6796, FN-6735, FN-7143, FN-7214, FN-7749). Splitting the label must not narrow them, so every former `=== "hard-cancel"` test routes through this predicate. Operator intent is still discriminated where it matters by `userCanceledTaskIds` / `live.userPaused`, never by the label alone. */ function isGenericAbortProvenance(provenance: PausedAbortProvenance | undefined): boolean { return provenance === "hard-cancel" || provenance === "engine-abort"; } // Re-export for backward compatibility (tests import from executor.ts) export { summarizeToolArgs } from "./agent-logger.js"; export { createAgentCreateTool, createAgentDeleteTool, createDelegateTaskTool, createTaskAssignTool, createGetAgentConfigTool, createListAgentsTool, createReadMessagesTool, createUpdateAgentConfigTool, createSendMessageTool, createTaskCreateTool, createTaskDocumentReadTool, createTaskDocumentWriteTool, createTaskLogTool, delegateTaskParams, listAgentsParams, memoryAppendParams, memoryGetParams, memorySearchParams, readMessagesParams, sendMessageParams, taskCreateParams, taskLogParams, } from "./agent-tools.js"; export { AGENT_BROWSER_NAVIGATION_SKILL_ID, probeAgentBrowserAvailability, augmentSessionSkillsForBrowserStep, formatAgentBrowserAvailabilityLog, } from "./executor/browser-probe.js"; export type { AgentBrowserAvailabilityProbeResult } from "./executor/browser-probe.js"; import { probeAgentBrowserAvailability, augmentSessionSkillsForBrowserStep, formatAgentBrowserAvailabilityLog, } from "./executor/browser-probe.js"; import type { AgentBrowserExec } from "./executor/browser-probe.js"; function mergeAdditionalSkillPaths(...pathGroups: Array): string[] | undefined { const merged = Array.from(new Set(pathGroups.flatMap((paths) => paths ?? []))); return merged.length > 0 ? merged : undefined; } /** * FNXC:WorkflowSteps 2026-07-30-21:40: * FN-8461 / GitHub #2388 require workflow skill-load warnings to describe a true * named-skill delivery failure, not an optional Compound Engineering source being * absent. Plugin body directories are paired with their parent discovery roots, * so check the requested bare name against each merged source; unrelated paths * must never hide a missing requested skill. */ function isWorkflowStepSkillDiscoverable( skillName: string, additionalSkillPaths: string[] | undefined, ceSkillsDir: string | undefined, ): boolean { // A configured CE root remains a viable source by contract: deployments can // inject a synthetic install root before its skill tree is materialized locally. if (ceSkillsDir) return true; const bareSkillName = skillName.includes(":") ? skillName.slice(skillName.lastIndexOf(":") + 1) : skillName; if (!bareSkillName || basename(bareSkillName) !== bareSkillName || bareSkillName === "." || bareSkillName === "..") { return false; } return (additionalSkillPaths ?? []).some((skillPath) => (basename(skillPath) === bareSkillName && existsSync(join(skillPath, "SKILL.md"))) || existsSync(join(skillPath, bareSkillName, "SKILL.md")), ); } const yieldEventLoop = (): Promise => new Promise((resolve) => setImmediateCb(resolve)); function getPromptSection(prompt: string, heading: string): string { const escapedHeading = heading.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const match = prompt.match(new RegExp(`^##\\s+${escapedHeading}\\s*$([\\s\\S]*?)(?=^##\\s+|$(?![\\s\\S]))`, "im")); return match?.[1]?.trim() ?? ""; } function promptDeclaresReviewLevelOnePlanOnly(prompt: string): boolean { return /^##\s+Review Level:\s*1\b[^\n]*\bPlan Only\b/im.test(prompt); } function promptDeclaresNoSourceChangeIntent(prompt: string): boolean { const normalized = prompt.toLowerCase(); return [ /should\s+not\s+change\s+(?:product\s+)?source/, /do\s+not\s+(?:edit|modify|change)\s+(?:product\s+)?source/, /no\s+(?:source|code)\s+changes?\s+(?:are\s+)?(?:expected|required|needed|allowed)/, /must\s+not\s+(?:edit|modify|change)\s+(?:product\s+)?(?:source|code)/, ].some((pattern) => pattern.test(normalized)); } function promptLooksCoordinationOnly(prompt: string): boolean { const titleMatch = prompt.match(/^#\s+Task:\s+[^\n]+/im)?.[0] ?? ""; const mission = getPromptSection(prompt, "Mission"); const assessment = prompt.match(/^\*\*Assessment:\*\*\s*([^\n]+)/im)?.[1] ?? ""; const coordinationText = `${titleMatch}\n${mission}\n${assessment}`.toLowerCase(); const hasCoordinationIntent = /\b(coordination|routing|route|handoff|assign(?:ment)?|owner|triage|select exactly one|record (?:the )?intentional block)\b/.test(coordinationText); const missionLower = mission.toLowerCase() .replace(/do\s+not\s+(?:edit|modify|change)\s+(?:product\s+)?source/g, "") .replace(/should\s+not\s+change\s+(?:product\s+)?source/g, "") .replace(/must\s+not\s+(?:edit|modify|change)\s+(?:product\s+)?(?:source|code)/g, ""); const hasImplementationDirective = /\b(implement|fix|add|change|modify|refactor|build|create|delete|remove)\b/.test(missionLower); return hasCoordinationIntent && !hasImplementationDirective; } function promptFileScopeIsBoardOnly(prompt: string): boolean { const fileScope = getPromptSection(prompt, "File Scope"); if (!fileScope.trim()) return false; const normalized = fileScope.toLowerCase(); const sourcePathPattern = /(?:^|[\s`'"(])(?:packages|src|source|sources|app|apps|lib|libs|components|scripts|docs|\.github|config|test|tests|__tests__)\//m; const sourceExtensionPattern = /\.(?:ts|tsx|js|jsx|mjs|cjs|swift|kt|java|py|go|rs|rb|php|cs|cpp|c|h|hpp|json|ya?ml|toml|mdx?|css|scss|html|sql|sh)\b/m; if (sourcePathPattern.test(normalized) || sourceExtensionPattern.test(normalized)) return false; const allowedBoardOnlyPattern = /(?:^|[^\w/])(?:task[- ]?board|board task|task document|task documents|task metadata|task logs|fusion task tools|fn_task_[\w-]*|\.fusion\/tasks|attachments?)(?=$|[^\w/-])/; return allowedBoardOnlyPattern.test(normalized); } function getNoCommitEligibilityReason(task: Task): "explicit noCommitsExpected=true" | "prompt-derived coordination-only no-source scope" | null { if (task.noCommitsExpected === true) return "explicit noCommitsExpected=true"; const rawPrompt = task.prompt; const prompt = typeof rawPrompt === "string" ? rawPrompt : ""; if (!prompt.trim()) return null; if ( promptDeclaresReviewLevelOnePlanOnly(prompt) && promptLooksCoordinationOnly(prompt) && promptDeclaresNoSourceChangeIntent(prompt) && promptFileScopeIsBoardOnly(prompt) ) { return "prompt-derived coordination-only no-source scope"; } return null; } /** * How long to wait after engine startup before spawning AI agent sessions for * orphaned in-progress tasks. The work itself (worktree setup, pi-coding-agent * session creation, child process spawn) is heavy and saturates the event * loop, which makes the dashboard unresponsive during cold start when there * are orphaned tasks from a prior run. Pushing this work past the initial * load window keeps the UI snappy; the tasks still resume — just after the * user has had time to see the board. * * Override via FUSION_RESUME_ORPHAN_DELAY_MS. Defaults to 0 under Vitest so * existing tests that expect immediate resumption keep passing without * needing per-test plumbing. * * Read lazily so an env-var change between module load and resumeOrphaned() * call (e.g. set in a test setup file) is observed. */ function getResumeOrphanDelayMs(): number { const raw = process.env.FUSION_RESUME_ORPHAN_DELAY_MS; if (raw !== undefined) { const parsed = Number.parseInt(raw, 10); if (Number.isFinite(parsed) && parsed >= 0) return parsed; } if (process.env.VITEST || process.env.NODE_ENV === "test") return 0; return 30_000; } const tokenCacheMetricsLog = createLogger("token-cache-metrics"); const OPTIONAL_STEP_REVISION_KEY_MARKER = "Workflow revision key:"; function normalizeOptionalStepRevisionKey(value: string | undefined): string { return (value ?? "").trim().toLowerCase(); } function optionalStepRevisionKey(nodeId: string | undefined, stepName: string | undefined): string { return normalizeOptionalStepRevisionKey(nodeId) || normalizeOptionalStepRevisionKey(stepName) || "pre-merge-optional-step"; } function countOptionalStepRevisionAttempts(task: Pick, key: string, stepName: string | undefined): number { const normalizedKey = normalizeOptionalStepRevisionKey(key); const normalizedStepName = normalizeOptionalStepRevisionKey(stepName); return (task.log ?? []).filter((entry) => { const action = entry.action ?? ""; const outcome = entry.outcome ?? ""; if (!/attempt \d+\//.test(action)) return false; const markerIndex = outcome.indexOf(OPTIONAL_STEP_REVISION_KEY_MARKER); if (markerIndex >= 0) { const markerValue = outcome.slice(markerIndex + OPTIONAL_STEP_REVISION_KEY_MARKER.length).split(/\r?\n/, 1)[0]?.trim(); return normalizeOptionalStepRevisionKey(markerValue) === normalizedKey; } if (!normalizedStepName) return false; return normalizeOptionalStepRevisionKey(outcome).includes(`step: ${normalizedStepName}`); }).length; } function optionalStepRevisionLogOutcome(details: string, key: string): string { return `${details}\n${OPTIONAL_STEP_REVISION_KEY_MARKER} ${key}`; } const STEP_STATUSES: StepStatus[] = ["pending", "in-progress", "done", "skipped"]; function canonicalizePath(path: string): string { try { return realpathSync(path); } catch { return resolvePath(path); } } /** Maximum retry attempts for workflow step hard failures before giving up */ const MAX_WORKFLOW_STEP_RETRIES = 3; /** Maximum in-session retries when an agent exits without calling fn_task_done(). */ const MAX_TASK_DONE_SESSION_RETRIES = 3; /** Maximum todo requeues after exhausting in-session fn_task_done retries. */ const MAX_TASK_DONE_REQUEUE_RETRIES = 3; export { MAX_EXECUTE_REQUEUE_LOOP_CYCLES, EXECUTE_REQUEUE_LOOP_VISIBLE_THRESHOLD, buildExecuteRequeueLoopSignature, isTransientMissingTaskJsonError, } from "./executor/requeue-loop.js"; import { MAX_EXECUTE_REQUEUE_LOOP_CYCLES, EXECUTE_REQUEUE_LOOP_VISIBLE_THRESHOLD, buildExecuteRequeueLoopHighWaterSignature, isInvalidAssistantContinuationErrorMessage, isTransientMissingTaskJsonError, TRANSIENT_WORKTREE_TASK_JSON_ENOENT_PATTERN, } from "./executor/requeue-loop.js"; const MAX_TRANSIENT_GRAPH_RESUME_RETRIES = 2; const TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS = process.env.VITEST || process.env.NODE_ENV === "test" ? 0 : 1_000; /* FNXC:SessionContention 2026-07-25-21:30: The contention ladder is deliberately long and slow compared with the provider-failure budget (2 fast retries): a lease is held for as long as the holder's own work takes — minutes, not milliseconds. Ten attempts backing off 5s→60s covers ~8 minutes of waiting, after which the task is left queued for ordinary re-dispatch rather than parked. */ const MAX_SESSION_CONTENTION_HOLD_RETRIES = 10; const SESSION_CONTENTION_HOLD_BACKOFF_MS = process.env.VITEST || process.env.NODE_ENV === "test" ? 0 : 5_000; const SESSION_CONTENTION_HOLD_MAX_BACKOFF_MS = 60_000; /** How long to wait before recovering a completed task still stuck in in-progress. */ const COMPLETED_TASK_WATCHDOG_MS = 60_000; /** How long to wait before retrying a workflow rerun handoff that never reached in-progress. */ const WORKFLOW_RERUN_WATCHDOG_MS = 15_000; /** Upper bound for in-process loop recovery before falling through to kill/requeue. */ const LOOP_COMPACTION_TIMEOUT_MS = 60_000; const TASK_DONE_REFUSAL_SUFFIX = "Either finish the work and resubmit, or do not call fn_task_done — exit the session and the engine will requeue."; type TaskDoneRefusalClass = | "bulk-step-completion-without-review" | "pending-code-review-revise"; type TaskDoneRefusalResult = | { ok: true } | { ok: false; refusalClass: TaskDoneRefusalClass; message: string; reason: string; }; type PendingReviewBlockResult = | { blocked: true; reason: | "review-request-without-verdict" | "code-review-rethink-or-unavailable-outstanding" | "code-review-unavailable-blocking"; stepIndex: number; } | { blocked: false }; function detectPendingReviewBlock( task: Task, _codeReviewVerdicts: Map, ): PendingReviewBlockResult { const inProgressStepIndices: number[] = []; for (let stepIndex = 0; stepIndex < task.steps.length; stepIndex++) { if (task.steps[stepIndex]?.status === "in-progress") { inProgressStepIndices.push(stepIndex); } } if (inProgressStepIndices.length === 0) { return { blocked: false }; } const recentActions = (task.log ?? []) .slice(-30) .map((entry) => entry.action?.trim()) .filter((action): action is string => Boolean(action)); for (const stepIndex of inProgressStepIndices) { const stepDisplay = stepIndex; const codeRequest = `code review requested for Step ${stepDisplay}`; const planRequest = `plan review requested for Step ${stepDisplay}`; const codeVerdictPrefix = `code review Step ${stepDisplay}:`; const planVerdictPrefix = `plan review Step ${stepDisplay}:`; for (let i = recentActions.length - 1; i >= 0; i--) { const action = recentActions[i]; if (!action) { continue; } if (action.startsWith(codeRequest) || action.startsWith(planRequest)) { return { blocked: true, reason: "review-request-without-verdict", stepIndex }; } if (action.startsWith(`${codeVerdictPrefix} RETHINK`)) { return { blocked: true, reason: "code-review-rethink-or-unavailable-outstanding", stepIndex }; } if (action.startsWith(`${codeVerdictPrefix} UNAVAILABLE`) && action.includes("blocking until reviewer returns a usable verdict")) { return { blocked: true, reason: "code-review-unavailable-blocking", stepIndex }; } if (action.startsWith(codeVerdictPrefix) || action.startsWith(planVerdictPrefix)) { break; } } } return { blocked: false }; } function formatTaskDoneRefusal(refusalClass: TaskDoneRefusalClass, reason: string): string { /* FNXC:Lifecycle 2026-07-16-10:20: FN-8141 — when the bulk-completion gate refuses (steps lack APPROVE verdicts), the agent must NOT reach for skip-every-step-then-complete as the escape hatch (that is exactly how FN-8141 laundered a failure into `done`). Name the honest blocked exit in the refusal so the sanctioned path is the advertised one. */ const blockedHint = refusalClass === "bulk-step-completion-without-review" ? " If the work genuinely cannot proceed, do NOT skip the remaining steps to force completion — call fn_task_done(outcome=\"blocked\", reason=\"...\") instead." : ""; return `fn_task_done refused (${refusalClass}): ${reason}. ${TASK_DONE_REFUSAL_SUFFIX}${blockedHint}`; } export function evaluateTaskDoneRefusal( task: Task, _params: { summary?: string }, codeReviewVerdicts: Map, ): TaskDoneRefusalResult { const pendingSteps: number[] = []; for (let stepIndex = 0; stepIndex < task.steps.length; stepIndex++) { const step = task.steps[stepIndex]; if (!step || step.status === "done" || step.status === "skipped") { continue; } pendingSteps.push(stepIndex); if (codeReviewVerdicts.get(stepIndex) === "REVISE") { const reason = `Step ${stepIndex} (${step.name}) has a pending code review verdict of REVISE`; return { ok: false, refusalClass: "pending-code-review-revise", reason, message: formatTaskDoneRefusal("pending-code-review-revise", reason), }; } } if (pendingSteps.length >= 2) { const allPendingApproved = pendingSteps.every((stepIndex) => codeReviewVerdicts.get(stepIndex) === "APPROVE"); if (!allPendingApproved) { const reason = `attempted to auto-complete ${pendingSteps.length} pending steps without APPROVE verdicts on all of them`; return { ok: false, refusalClass: "bulk-step-completion-without-review", reason, message: formatTaskDoneRefusal("bulk-step-completion-without-review", reason), }; } } return { ok: true }; } /* FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — synthesize a refusal for an IMPLICIT (agent-exited, no explicit fn_task_done) completion whose skipped steps are skip-bypass tainted. Only the implicit/auto paths consult this; an explicit accepted fn_task_done stays the honest exit that clears the taint. Reuses the bulk-step-completion class so the existing refusal budget/park machinery applies unchanged. */ function buildSkipBypassTaintRefusal( evaluation: ReturnType, ): Extract { const reason = evaluation.reason ?? "skipped steps after a bulk-step-completion refusal cannot auto-complete the task"; return { ok: false, refusalClass: "bulk-step-completion-without-review", reason, message: formatTaskDoneRefusal("bulk-step-completion-without-review", reason), }; } /** * Determines the step index from which revision should restart given a set of * completed steps and user feedback. Exported for unit tests; no longer called * from the executor (revision is now handled via `reopenLastStepForRevision`). */ export function determineRevisionResetStart( steps: ReadonlyArray<{ name: string }>, feedback: string, ): number { const total = steps.length; if (total === 0) return 0; const skipPreflight = /preflight/i.test(steps[0].name); const firstCandidate = skipPreflight ? 1 : 0; if (firstCandidate >= total) return total; const fb = feedback.toLowerCase(); for (let i = firstCandidate; i < total; i++) { const tokens = steps[i].name.toLowerCase().match(/[a-z][a-z]{4,}/g) ?? []; if (tokens.some((t) => fb.includes(t))) return i; } return firstCandidate; } export interface WorkflowRevisionFeedbackPartition { inScopeFeedback: string; outOfScopeFeedback: string; inScopeSegments: string[]; outOfScopeSegments: string[]; detectedPaths: string[]; } const WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS = 4_000; const WORKFLOW_FEEDBACK_PATH_REGEX = /`([^`\n]+)`|(?(); for (const match of feedback.matchAll(WORKFLOW_FEEDBACK_PATH_REGEX)) { const candidate = stripTrailingPathPunctuation(match[1] ?? match[2] ?? ""); const normalized = normalizeWorkflowScopePath(candidate); if (!normalized.includes("/") || !normalized) continue; if (seen.has(normalized)) continue; seen.add(normalized); extracted.push(normalized); } return extracted; } /** * FN-4811 follow-up: paths the scope-leak guard never flags, regardless of declared * scope. These are file types every task may legitimately touch as part of standard * delivery (e.g., `.changeset/` per AGENTS.md's "Finalizing Changes" section). * Cross-task contamination of these paths is caught by stronger guards downstream * (file-scope invariant at squash commit, branch-tip checks, post-merge audit). */ export function isAlwaysAllowedScopeLeakPath(filePath: string): boolean { const normalizedPath = normalizeWorkflowScopePath(filePath); return normalizedPath.startsWith(".changeset/"); } export function workflowPathMatchesDeclaredScope(filePath: string, scopePatterns: readonly string[]): boolean { const normalizedPath = normalizeWorkflowScopePath(filePath); for (const rawPattern of scopePatterns) { const pattern = normalizeWorkflowScopePath(rawPattern); if (!pattern) continue; if (/\/\*+$/.test(pattern)) { const directory = pattern.replace(/\/\*+$/, ""); if (normalizedPath === directory || normalizedPath.startsWith(`${directory}/`)) return true; continue; } if (pattern.endsWith("/")) { if (normalizedPath.startsWith(pattern)) return true; continue; } if (normalizedPath === pattern) return true; } return false; } export function parseReviewLevelFromPrompt(prompt: string): number { const reviewMatch = prompt.match(/##\s*Review Level[:\s]*(\d)/); return reviewMatch ? parseInt(reviewMatch[1], 10) : 0; } function extractPromptSection(prompt: string, heading: string): string { const escaped = heading.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const headingPattern = new RegExp(`^##\\s+${escaped}\\s*:?\\s*$`, "i"); const nextHeadingPattern = /^##\s+/; const lines = prompt.split(/\r?\n/); const start = lines.findIndex((line) => headingPattern.test(line.trim())); if (start === -1) return ""; const sectionLines: string[] = []; for (let i = start + 1; i < lines.length; i++) { const line = lines[i]; if (nextHeadingPattern.test(line.trim())) break; sectionLines.push(line); } return sectionLines.join("\n").trim(); } function extractPromptListEntries(section: string): string[] { return section .split(/\r?\n/) .map((line) => line.trim()) .map((line) => line.replace(/^[-*]\s+/, "").replace(/^`([^`]+)`.*$/, "$1").trim()) .filter(Boolean); } function isFusionTaskArtifactScopeEntry(entry: string): boolean { const normalized = entry.trim().toLowerCase().replace(/^\//, "").replace(/^\.\//, ""); return normalized.startsWith(".fusion/tasks/"); } function isNoSourceScopeEntry(entry: string): boolean { const normalized = entry.toLowerCase(); return ( normalized.includes("no source") || normalized.includes("no product-source") || normalized.includes("no code") || normalized.includes("no file mutations") || normalized.includes("task document") || normalized.includes("task documents") || normalized.includes("task metadata") || normalized.includes("task log") || normalized.includes("agent log") || normalized.includes("task artifacts") || normalized.includes("read-only evidence") || isFusionTaskArtifactScopeEntry(normalized) ); } function hasSourceChangingScopeEntry(entry: string): boolean { const normalized = entry.toLowerCase(); if (!normalized) return false; if (isFusionTaskArtifactScopeEntry(normalized)) return false; const sourcePathPattern = /(?:^|[\s`'"(])(?:packages|src|source|sources|app|apps|lib|libs|components|scripts|docs|\.github|config|test|tests|__tests__|\.changeset)\//m; if (sourcePathPattern.test(normalized)) return true; if (/\.(ts|tsx|js|jsx|mjs|cjs|swift|kt|java|py|rs|go|rb|md|json|ya?ml|toml|css|scss|html)\b/.test(normalized)) return true; if (normalized.includes("read-only") || isNoSourceScopeEntry(normalized)) return false; return false; } function promptDeclaresSourceFreeTaskArtifactContract(combinedText: string): boolean { const forbidsForceAddingFusionArtifacts = /(?:do not|don't|never|must not)\s+(?:force[- ]?add|git add -f)[^\n]*(?:\.fusion|gitignored)/.test(combinedText) || /(?:\.fusion|gitignored)[^\n]*(?:do not|don't|never|must not)\s+(?:force[- ]?add|git add -f)/.test(combinedText); const forbidsFabricatedCommits = /(?:do not|don't|never|must not)\s+(?:create|make|fabricate|manufacture)[^\n]*(?:empty|fabricated|zero[- ]diff)[^\n]*commits?/.test(combinedText) || /(?:empty|fabricated|zero[- ]diff)[^\n]*commits?[^\n]*(?:do not|don't|never|must not|forbidden)/.test(combinedText); const declaresOnlySourceFreeArtifacts = /(?:source[- ]free|gitignored)[^\n]*(?:task[- ]artifact|task artifact|\.fusion\/tasks|deliver(?:y|able)|artifact)/.test(combinedText) || /(?:only|limited to)[^\n]*(?:source[- ]free|gitignored)[^\n]*(?:task[- ]artifact|task artifact|\.fusion\/tasks)/.test(combinedText); return (forbidsForceAddingFusionArtifacts && forbidsFabricatedCommits) || declaresOnlySourceFreeArtifacts; } function promptScopeIsSourceFreeTaskArtifacts(promptScopeEntries: string[], declaredScope: string[]): boolean { if (promptScopeEntries.length === 0 || declaredScope.length === 0) return false; if (declaredScope.some(hasSourceChangingScopeEntry)) return false; return declaredScope.every((entry) => isFusionTaskArtifactScopeEntry(entry) || isNoSourceScopeEntry(entry)); } function getTaskTextForNoCommitEligibility(task: Task, promptContent: string): string { const logText = (task.log ?? []) .map((entry) => `${entry.action ?? ""}\n${entry.outcome ?? ""}`) .join("\n"); const sourceMetadata = task.sourceMetadata ? JSON.stringify(task.sourceMetadata) : ""; return [task.title, task.description, promptContent, sourceMetadata, logText] .filter((part): part is string => typeof part === "string" && part.length > 0) .join("\n"); } function evaluatePromptDerivedNoCommitEligibility(task: Task, promptContent: string): { eligible: boolean; reason?: string } { const combined = getTaskTextForNoCommitEligibility(task, promptContent).toLowerCase(); const promptScopeEntries = extractPromptListEntries(extractPromptSection(promptContent, "File Scope")); const metadataScope = Array.isArray(task.sourceMetadata?.fileScope) ? task.sourceMetadata.fileScope.filter((entry): entry is string => typeof entry === "string") : []; const declaredScope = [...promptScopeEntries, ...metadataScope]; const stepsComplete = Array.isArray(task.steps) && task.steps.length > 0 ? task.steps.every((step) => step.status === "done" || step.status === "skipped") : false; /* FNXC:TaskDoneCompletion 2026-07-03-00:00: Source-free deliveries that only write gitignored `.fusion/tasks/...` task artifacts must not fabricate empty commits or force-add ignored evidence just to satisfy fn_task_done. This exemption is intentionally narrower than Review Level 0/1: the PROMPT must declare a source-free task-artifact contract, every declared scope entry must be board/task artifact only, and any tracked source/docs/config/test/changeset path keeps the no_commits refusal intact. */ if ( stepsComplete && promptDeclaresSourceFreeTaskArtifactContract(combined) && promptScopeIsSourceFreeTaskArtifacts(promptScopeEntries, declaredScope) ) { return { eligible: true, reason: "prompt-derived source-free task-artifact contract" }; } /* FNXC:ReviewLevelPreset 2026-07-19-10:35 (U8 / R6): reviewLevel is a CREATION-TIME preset (it writes enabledWorkflowSteps at create), so the runtime no longer reads `task.reviewLevel`. The plan-only (level-1) eligibility signal here is derived from the PROMPT contract, not the row field — removing the last `task.reviewLevel` runtime read (R6 tombstone). The explicit preset-set field re-key lands with U9's schema (reviewLevel backfill + field adds). */ const reviewLevel = parseReviewLevelFromPrompt(promptContent); const isPlanOnly = reviewLevel === 1 && (/plan\s*only/.test(combined) || combined.includes("plan-only")); if (!isPlanOnly) return { eligible: false }; const explicitNoSourceIntent = [ "no expected product-source changes", "no product-source changes", "no source changes expected", "no source files expected", "no code changes expected", "no expected source changes", "no file mutations", "no source/config/file mutations", ].some((phrase) => combined.includes(phrase)); if (!explicitNoSourceIntent) return { eligible: false }; const excludedImplementationIntent = /\b(investigate and fix|fix if needed|implement|source-changing|code change|docs\/tests changes|documentation change|bug[- ]fix|feature)\b/.test(combined); const operationalIntent = /\b(operational|routing|route|assign|assignment|owner|handoff|coordination|coordinate|no-route|triage)\b/.test(combined); if (!operationalIntent || excludedImplementationIntent) return { eligible: false }; if (declaredScope.length === 0) return { eligible: false }; if (declaredScope.some(hasSourceChangingScopeEntry)) return { eligible: false }; if (!declaredScope.every(isNoSourceScopeEntry)) return { eligible: false }; const logText = (task.log ?? []) .map((entry) => `${entry.action ?? ""}\n${entry.outcome ?? ""}`) .join("\n") .toLowerCase(); const hasOperationalEvidence = /\b(evidence|recorded|documented|no-route|routed|assigned|handoff|decision)\b/.test(logText); if (!stepsComplete && !hasOperationalEvidence) return { eligible: false }; return { eligible: true, reason: "prompt/source metadata derived operational no-commit contract" }; } class NonRetryableWorktreeError extends Error {} function formatGitRepositoryDetectionError(rootDir: string, detection: Extract): string { const stderr = detection.stderr.trim() || "git rev-parse --git-dir failed without stderr"; const remedy = detection.reason === "dubious-ownership" ? ` Resolve Git safe-directory ownership with: git config --global --add safe.directory "${rootDir}"` : ""; return `Git repository detection failed for project directory "${rootDir}". Fusion could not verify worktree support because git reported: ${stderr}.${remedy}`; } function buildSessionWorktreePathRegex(rootDir: string, settings: Partial): RegExp { const configuredBase = resolveWorktreesDir(rootDir, settings).split(/[\\/]/).filter(Boolean).pop() ?? ".worktrees"; const escapedBase = configuredBase.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return new RegExp(`([A-Za-z]:)?[^"'\\s]*(?:\\.worktrees|${escapedBase})[\\\\/][^"'\\s]+`, "g"); } function normalizeWorktreePath(pathValue: string): string { return resolvePath(pathValue).replace(/\\/g, "/").replace(/\/+$/, ""); } async function extractPersistedSessionWorktreePath( sessionFile: string, rootDir: string, settings: Partial, ): Promise { try { const content = await readFile(sessionFile, "utf-8"); const matches = content.match(buildSessionWorktreePathRegex(rootDir, settings)) ?? []; if (matches.length === 0) return null; const normalizedCounts = new Map(); for (const match of matches) { const normalized = normalizeWorktreePath(match); normalizedCounts.set(normalized, (normalizedCounts.get(normalized) ?? 0) + 1); } let best: { path: string; count: number } | null = null; for (const [path, count] of normalizedCounts.entries()) { if (!best || count > best.count) best = { path, count }; } return best?.path ?? null; } catch { return null; } } function isSessionWorktreeCompatible( persistedWorktreePath: string | null, currentWorktreePath: string, ): boolean { if (!persistedWorktreePath) return true; return persistedWorktreePath === normalizeWorktreePath(currentWorktreePath); } function truncateWorkflowScriptOutput(output: string): string { if (output.length <= WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS) return output; return `... output truncated to last ${WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS} characters ...\n${output.slice(-WORKFLOW_SCRIPT_OUTPUT_MAX_CHARS)}`; } function configuredCommandErrorMessage(result: RunCommandResult): string { if (result.spawnError) return result.spawnError.message; const parts: string[] = []; if (result.timedOut) parts.push("Timed out"); if (result.exitCode !== null) parts.push(`Exit code: ${result.exitCode}`); if (result.signal) parts.push(`Signal: ${result.signal}`); const stdout = result.stdout.trim(); const stderr = result.stderr.trim(); if (stdout) parts.push(`stdout: ${truncateWorkflowScriptOutput(stdout)}`); if (stderr) parts.push(`stderr: ${truncateWorkflowScriptOutput(stderr)}`); return parts.length ? parts.join("\n") : "Command failed"; } function getConfiguredCommandSandboxBackend(auditor?: RunAuditor): SandboxBackend { return resolveSandboxBackend({ auditor }); } async function runConfiguredCommand( command: string, cwd: string, timeoutMs: number, extraEnv?: NodeJS.ProcessEnv, auditor?: RunAuditor, signal?: AbortSignal, ): Promise { const backend = getConfiguredCommandSandboxBackend(auditor); const result = await backend.run(command, { cwd, timeoutMs, maxBuffer: 10 * 1024 * 1024, encoding: "utf-8", ...(extraEnv !== undefined && { env: extraEnv }), ...(signal !== undefined && { signal }), }); return { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, signal: result.signal, bufferExceeded: result.bufferExceeded, timedOut: result.timedOut, spawnError: result.spawnError, }; } export async function __runConfiguredCommandForTests( command: string, cwd: string, timeoutMs: number, extraEnv?: NodeJS.ProcessEnv, auditor?: RunAuditor, signal?: AbortSignal, ): Promise { return runConfiguredCommand(command, cwd, timeoutMs, extraEnv, auditor, signal); } // ── Tool parameter schemas (module-level for reuse in ToolDefinition generics) ── const taskUpdateParams = Type.Object({ step: Type.Optional(Type.Number({ description: "Step number (0-indexed; matches the `### Step N:` numbers in PROMPT.md — Step 0 is Preflight). Omit when updating only custom_fields/dependencies." })), status: Type.Optional(Type.Union( STEP_STATUSES.map((s) => Type.Literal(s)), { description: "New status: pending, in-progress, done, or skipped. Required when step is set." }, )), dependencies: Type.Optional(Type.Array(Type.String(), { description: "Optional task dependency array. Replaces existing dependencies. Pass ['FN-001', 'FN-002'] to set dependencies. Pass [] to clear all dependencies. Omit parameter to preserve existing dependencies.", })), custom_fields: Type.Optional(Type.Record(Type.String(), Type.Unknown(), { description: "Optional patch of workflow-defined custom field values, keyed by field id. " + "Values are validated against the task's workflow field schema (type/enum membership); " + "pass null for a field to clear it. Rejected writes return the offending field id and reason. " + "Only fields declared by the task's workflow may be written.", })), }); // taskLogParams and taskCreateParams are imported from agent-tools.ts const taskAddDepParams = Type.Object({ task_id: Type.String({ description: "The ID of the task to depend on (e.g. \"KB-001\")" }), confirm: Type.Optional(Type.Boolean({ description: "Set to true to confirm adding the dependency. Required because adding a dep to an in-progress task will stop execution and discard current work." })), }); const spawnAgentParams = Type.Object({ name: Type.String({ description: "Name for the child agent" }), role: Type.Union([ Type.Literal("triage"), Type.Literal("executor"), Type.Literal("reviewer"), Type.Literal("merger"), Type.Literal("engineer"), Type.Literal("custom"), ], { description: "Role for the child agent" }), task: Type.String({ description: "Task description for the child agent to execute" }), systemPromptOverride: Type.Optional( Type.String({ description: "Optional persona/system-prompt for the child agent. When provided (non-empty), it replaces the generic child base prompt so the child runs as a specific persona (e.g. a compound-engineering reviewer). Executor instructions are still appended.", }), ), }); /** * Sentinel a skill running in a Fusion workflow step emits when it needs to ask * the user a blocking question (it has no synchronous question tool — see the CE * skills' "Running inside Fusion" sections). The executor detects this in the * step's output and parks the task `awaiting-user-input`, reusing the same * pause/resume machinery as an `awaitInput` node (U6). Returns the question text, * or null when no well-formed sentinel is present. */ export function parseAwaitInputSentinel(output: string | undefined): string | null { if (!output) return null; const m = output.match(/===FUSION_AWAIT_INPUT===\s*([\s\S]*?)\s*===END_FUSION_AWAIT_INPUT===/); const question = m?.[1]?.trim(); return question ? question : null; } const USER_QUESTION_TOOL_NAMES = new Set([ "askuserquestion", "ask_user", "ask_followup_question", "request_user_input", "elicit", "ask_question", "fn_ask_question", ]); /** * Normalize a question-tool invocation into the same durable await-input * contract used by skill sentinels. Some runtimes expose an interactive * question tool even though Fusion workflow-step sessions have no synchronous * listener; detecting the call at the session event boundary prevents the * task from continuing after the unanswered question is rendered. */ export function parseAwaitInputQuestionToolCall( toolName: string, args: Record | undefined, ): string | null { if (!USER_QUESTION_TOOL_NAMES.has(toolName.trim().toLowerCase()) || !args) return null; const records = Array.isArray(args.questions) ? args.questions : [args]; const questions = records.flatMap((value) => { if (!value || typeof value !== "object" || Array.isArray(value)) return []; const record = value as Record; const question = [record.question, record.prompt, record.message, record.text, record.title] .find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0) ?.trim(); return question ? [question] : []; }); return questions.length > 0 ? questions.join("\n\n") : null; } /** * (U2 / KTD-2) Fusion workflow-step conventions preamble, prepended to a skill * step's prompt at the skill-prompt build path (runGraphCustomNode). It teaches * any bundled skill the conventions Fusion needs — in ONE engine-side place, so * the skills stay byte-for-byte upstream. The block is skill-agnostic and rides * on the node prompt; it deliberately overrides the upstream skill bodies that * still say "call AskUserQuestion" / "Task ce-*". Stable text — the await-input * grammar here must match `parseAwaitInputSentinel` and the persona-override * contract (fn_spawn_agent's `systemPromptOverride` param) verbatim. * * (U9 / KTD-7) The persona-fan-out instruction is path-confined: the skill must * resolve `.md` strictly within `$FUSION_CE_AGENTS_DIR` and reject any * `../` traversal before reading, since the file body is injected verbatim into a * child's system prompt (a filesystem prompt-injection surface otherwise). */ export const FUSION_WORKFLOW_STEP_CONVENTIONS_PREAMBLE = `## Fusion workflow-step conventions You are running as a Fusion autonomous workflow step — NOT an interactive Claude Code session. Follow these conventions; they override any contrary instruction in the skill body below. 1. Asking the user: there is no interactive listener here. \`AskUserQuestion\` / \`request_user_input\` go into the void. When you must ask the user a question, emit EXACTLY ONE block of the form: ===FUSION_AWAIT_INPUT=== ===END_FUSION_AWAIT_INPUT=== and then STOP. Fusion parks the task awaiting the user's answer and re-runs this step with their reply. 2. Headless runs: when the environment variable \`FUSION_HEADLESS=1\` is set, do NOT ask the user anything. Record a reasonable assumption explicitly in your output and proceed — never emit the await-input block in this mode. 3. Dispatching a \`ce-\` subagent: do NOT use a raw \`Task ce-*(...)\` call. Instead, read the persona definition from \`$FUSION_CE_AGENTS_DIR/.md\`, strip its YAML frontmatter, and pass the remaining body as the \`systemPromptOverride\` argument to the \`fn_spawn_agent\` tool. Resolve the path strictly inside \`$FUSION_CE_AGENTS_DIR\` — reject any \`\` containing \`/\` or \`..\` (path traversal), and skip a def whose body is empty or implausibly large. If \`fn_spawn_agent\` is not available (a readonly step), do the persona's work inline yourself instead of spawning. `; /** Result returned from fn_spawn_agent tool */ interface SpawnAgentResult { agentId: string; name: string; state: AgentState; role: AgentCapability; message: string; } /** * Outcome of a single workflow step execution. * Supports three states: pass, hard failure, or revision requested with feedback. */ export interface WorkflowStepOutcome { success: boolean; revisionRequested?: boolean; output?: string; error?: string; /** Machine-readable verdict extracted from structured JSON output. */ verdict?: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; /** Notes extracted from structured JSON output (distinct from raw output). */ notes?: string; /** Set when the call exceeded `settings.workflowStepTimeoutMs`. Signals the * caller to escalate to the fallback model rather than treat the failure * as a generic revision request. */ timedOut?: boolean; /** True when no structured or prose verdict could be inferred. */ malformed?: boolean; /** Machine-readable graph failure used for deterministic recovery routing. */ failureValue?: string; } /** * Result of running all pre-merge workflow steps. * Returns true if all passed, false if any hard failure, or a structured * revision result if a revision was requested. */ export type WorkflowStepResult = | { allPassed: true } | { allPassed: false; revisionRequested: false; feedback: string; stepName: string } | { allPassed: false; revisionRequested: true; feedback: string; stepName: string }; export function parseWorkflowStepVerdict(rawOutput: string): { verdict: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; notes: string } | null { const trimmed = rawOutput.trim(); const candidates: string[] = []; const fencedMatches = [...trimmed.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)]; for (const match of fencedMatches) { candidates.push(match[1].trim()); } /* FNXC:ReviewLeniency 2026-07-01-23:30: Prefer a balanced, string-aware object scan over a greedy `\{[\s\S]*\}` match: models that emit reasoning PROSE (which may itself contain braces) followed by a trailing `{"verdict":...}` payload broke the greedy span into invalid JSON. extractJsonObjectCandidates returns each top-level object in document order; iterating last→first prefers the trailing verdict payload. */ candidates.push(...extractJsonObjectCandidates(trimmed)); for (let i = candidates.length - 1; i >= 0; i -= 1) { try { const parsed = JSON.parse(candidates[i]) as { verdict?: unknown; notes?: unknown }; if (!parsed || typeof parsed.verdict !== "string") continue; /* FNXC:ReviewLeniency 2026-07-01-23:30: "Any approved" — accept approval-family verdict variants (APPROVE, APPROVED, APPROVE_WITH_NOTES, approve_with_verdict, …), not just the exact WORKFLOW_STEP_VERDICTS strings. A token starting with APPROVE maps to APPROVE_WITH_NOTES when it mentions notes, else APPROVE; REVISE-family → REVISE; anything else (e.g. "PASS") is not a verdict and the candidate is skipped. */ const token = parsed.verdict.trim().toUpperCase(); let verdict: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE" | null = null; if (token.startsWith("APPROVE") || token.startsWith("APPROVAL")) { verdict = token.includes("NOTE") ? "APPROVE_WITH_NOTES" : "APPROVE"; } else if (token.startsWith("REVISE") || token.startsWith("REQUEST_REVISION") || token.startsWith("REJECT")) { verdict = "REVISE"; } if (!verdict) continue; return { verdict, notes: typeof parsed.notes === "string" ? parsed.notes : "", }; } catch { // continue } } return null; } export function inferWorkflowStepVerdictFromProse(rawOutput: string): { verdict: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; notes: string } | null { const trimmed = rawOutput.trim(); const revisionMatch = trimmed.match(/^REQUEST REVISION\s*\n*/i); if (revisionMatch) { return { verdict: "REVISE", notes: trimmed.slice(revisionMatch[0].length).trim() || "Revision requested" }; } /* * FNXC:PlanReview 2026-06-29-02:05: * Plan Review runs through reviewer-style agents that often emit a markdown * section such as `### Verdict: APPROVE` even when the prompt asks for trailing * JSON. Treat that explicit verdict as authoritative so a real approval does * not collapse into a synthetic pre-execution plan failure loop. */ const explicitVerdictMatch = trimmed.match(/(?:^|\n)\s*(?:#{1,6}\s*)?(?:verdict|status)\s*:\s*(APPROVE_WITH_NOTES|APPROVE|REVISE)\b/i); if (explicitVerdictMatch) { return { verdict: explicitVerdictMatch[1].toUpperCase() as "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE", notes: "", }; } /* FNXC:ReviewLeniency 2026-07-01-22:15: A gate review (code-review, browser-verification) whose text clearly approves must PASS even when it is not perfectly structured. Delegate to the shared proseSignalsClearApproval detector so this parser and the reviewer/plan-review parser agree on what "clearly approved" means, and so a prose rejection ("not approved", "please revise", "reject") is never promoted to APPROVE. Replaces the prior narrow approve/approved/looks good/no issues/out of scope regex (now a subset of the shared detector). */ if (proseSignalsClearApproval(trimmed)) { return { verdict: "APPROVE", notes: "" }; } return null; } /** * FNXC:WorkflowGates 2026-06-17-18:22: * Gate-class workflow steps must emit a parseable JSON or prose verdict before they can approve pre-merge completion. A fully malformed response is surfaced explicitly so blocking gates fail while advisory gates can record a non-blocking advisory failure. */ export function parseWorkflowStepOutput(rawOutput: string): { output: string; verdict?: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; notes?: string; malformed?: boolean; }; export function parseWorkflowStepOutput(rawOutput: string, options: { requireVerdict: false }): { output: string; verdict?: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; notes?: string; malformed?: boolean; }; export function parseWorkflowStepOutput(rawOutput: string, options: { requireVerdict?: boolean } = {}): { output: string; verdict?: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; notes?: string; malformed?: boolean; } { const trimmed = rawOutput.trim(); const parsed = parseWorkflowStepVerdict(trimmed); if (parsed) { return { output: parsed.notes || "", verdict: parsed.verdict, notes: parsed.notes, }; } const inferred = inferWorkflowStepVerdictFromProse(trimmed); if (inferred) { return { output: inferred.notes || trimmed, verdict: inferred.verdict, notes: inferred.notes, }; } if (options.requireVerdict === false) { return { output: trimmed }; } return { output: trimmed, malformed: true }; } /* FNXC:ExecutorPrompt 2026-06-21-03:59: Agents must not run the full/workspace-wide test suite by default; targeted/package-scoped verification is the norm, full runs require explicit task/workflow opt-in. FNXC:ExecutorPrompt 2026-07-05-00:35: FN-7608: a `require-approval` gate previously only parked the single tool call (soft rejection + task/agent paused in the store) while the turn-ending rules below forbade ending a turn without another tool call, so the model was effectively instructed to hunt for ungated workarounds (re-issuing the same bash, probing read-only equivalents, fn_web_fetch/fn_task_attach bypasses) instead of stopping. The engine now actually suspends the in-flight session when a gate resolves to wait-for-approval (see executor.ts buildActionGateContext.pauseForApproval), so the prompt must carve out waiting on a pending approval as a legitimate turn end and explicitly forbid probing for alternatives. This clause must stay byte-identical with EXECUTOR_PROMPT_TEXT in packages/core/src/agent-prompts.ts. */ const EXECUTOR_SYSTEM_PROMPT = `${FUSION_RUNTIME_SELF_AWARENESS} You are a task execution agent for "fn", an AI-orchestrated task board. You are working in a git worktree isolated from the main branch. Your job is to implement the task described in the PROMPT.md specification you're given. ## Your Role in the System You are the primary implementation agent in Fusion. You execute task specs in isolated worktrees, produce production-quality changes, and hand off work that can pass independent review and merge. ## Turn-ending rules — read carefully You MUST end every turn by either: - (a) calling another tool to make progress, OR - (b) calling \`fn_task_done\` if the entire task is complete, OR - (c) calling \`fn_task_done(outcome="blocked", reason="...")\` if the work genuinely cannot proceed (see "Cannot proceed" below) You MUST NOT end a turn by writing prose that asks the user a question, summarizes progress, or requests permission to continue. The following are FORBIDDEN turn-endings: - "If you want, I can continue with..." - "Should I proceed with...?" - "Let me know if you'd like me to..." - "Ready to move on to step N. Want me to continue?" - Any markdown progress summary at the end of a turn instead of a tool call **Exception — pending approval.** If a tool call reports that the action requires approval (a permission gate) and the task has been paused awaiting a decision, STOP. Waiting on a pending approval IS a legitimate turn end: the engine suspends this session automatically once the gate fires, so ending the turn here is expected, not a violation of the rule above. Do NOT re-issue the same gated call, probe for a read-only or "equivalent" alternative, fetch the gated resource through another tool (e.g. \`fn_web_fetch\`, \`fn_task_attach\`), or otherwise search for an ungated path around the blocked action — an approval gate is fully blocking, not something to route around or "make progress another way" against. Execution resumes on its own once the request is approved or denied. If you have just finished a step's work, immediately call \`fn_task_update\` to mark the step done and continue with the next pending step in the SAME turn. Do not pause to summarize. The user is not watching this conversation in real-time. They will read the final result. Asking permission wastes a full retry cycle and may orphan committed work. **Cannot proceed — the honest blocked exit.** If the work genuinely cannot be finished (an upstream API break, a missing prerequisite task, an unresolvable external error), call \`fn_task_done(outcome="blocked", reason="", blockedBy=["FN-XXXX"])\`. This parks the task as failed WITHOUT any completion claim, leaves your steps in their true statuses, preserves your worktree/branch, and records \`blockedBy\` task IDs as dependencies so the task requeues once the blocker completes. This is THE correct action when you are stuck — do NOT instead mark the remaining steps \`skipped\` and call \`fn_task_done\` to make the task look finished. Skipping steps to escape a blocker launders a failure into \`done\` and is never the right move. (\`skipped\` remains valid only for the stale-premise path below, when the requested work is already present on HEAD.) Never write the blocker as plain prose. ## How to work 1. Read the PROMPT.md carefully — it contains your mission, steps, file scope, acceptance criteria, and Do NOT constraints 2. Before touching code, read all files listed in "Context to Read First" and understand the full step outcome 3. Check existing patterns in the codebase before introducing new structure, naming, or APIs 4. Work through each step in order 5. Write clean, production-quality code 6. Test your changes continuously 7. Commit at meaningful boundaries (step completion) ## Reporting progress via tools You have tools to report progress. The board updates in real-time. **Step lifecycle:** The \`step\` argument is 0-based and equals the literal \`### Step N:\` number in PROMPT.md (Step 0 is Preflight). - Before starting a step: \`fn_task_update(step=N, status="in-progress")\` - After completing a step: \`fn_task_update(step=N, status="done")\` - If skipping a step: \`fn_task_update(step=N, status="skipped")\` **Preflight escape hatch — stale premise.** PROMPT.md is captured at task-creation time; HEAD may have moved on since then. During Preflight (Step 0), reproduce the failure or symptom described in the PROMPT. If reproduction shows the work is **already done or the premise no longer matches HEAD** — for example, the test that PROMPT claims is failing already passes on the current base, or the file PROMPT says to change already contains the described change — do NOT march through the remaining steps producing empty commits. Instead: 1. Call \`fn_task_log\` with a clear premise-stale finding: what PROMPT.md claimed vs. what HEAD actually shows (include the exact reproduction command + its result). 2. Mark Step 0 done: \`fn_task_update(step=0, status="done")\`. 3. Mark every remaining step skipped with a one-line reason: \`fn_task_update(step=N, status="skipped")\`. 4. Call \`fn_task_done\` with a summary that begins \`PREMISE STALE:\` followed by the concrete reason (e.g. \`PREMISE STALE: targeted reproduction passes unchanged on HEAD; PROMPT claimed MOBILE_MEDIA_QUERY had been expanded but useViewportMode.ts:9 still exports the legacy value\`). This path exists specifically to prevent the executor from looping when PROMPT.md is out of sync with HEAD. Use it only after running the actual reproduction — do not invoke it to dodge real work. If a task is verified as a no-op, duplicate, or redundant for the same reason (the requested behavior is already present on HEAD), \`fn_task_done\` may also use a leading sentinel summary of \`NO-OP:\`, \`NOOP:\`, \`DUPLICATE: FN-NNNN ...\`, or \`REDUNDANT:\`. These sentinels are audit-logged and allow a verified zero-commit completion; ordinary zero-commit implementation completions without a recognized leading sentinel are still refused. **Stale premise vs. blocked — do not confuse them.** Skipping remaining steps is ONLY for the stale-premise case above, where the requested work is already present on HEAD so there is nothing left to do. If the work is real but you CANNOT do it (upstream broke, a prerequisite task is missing, an external error is unresolvable), that is NOT a stale premise — do NOT skip steps to fake completion. Use \`fn_task_done(outcome="blocked", reason="...", blockedBy=[...])\` instead (see "Cannot proceed" above). **Logging important actions:** \`fn_task_log(message="what happened")\` **Out-of-scope work found during execution:** \`fn_task_create(description="what needs doing")\` When creating multiple related tasks, declare dependencies between them: \`fn_task_create(description="load door sounds", dependencies=[])\` → returns KB-050 \`fn_task_create(description="play sound on door open/close", dependencies=["KB-050"])\` **Discovered a dependency:** \`fn_task_add_dep(task_id="KB-XXX")\` — use when you discover mid-execution that another task must be completed first. This will return a warning first — you must call again with \`confirm=true\` to proceed. Adding a dependency stops execution, discards current work, and moves the task to triage for re-planning. ## Task Documents You can save and retrieve named documents for this task. Use these to store planning notes, research findings, or any persistent data that should survive across sessions. - **Save a document:** \`fn_task_document_write(key="plan", content="...")\` - **Read a document:** \`fn_task_document_read(key="plan")\` - **List all documents:** \`fn_task_document_read()\` (no key) Documents are versioned — each write creates a new revision. Use meaningful keys like "plan", "notes", "research", "architecture". ## Artifact Registry Use \`fn_artifact_register\` to register multi-type artifacts for discovery across agents and tasks, \`fn_artifact_list\` to find registered artifacts by type/author/task/search, and \`fn_artifact_view\` to inspect artifact metadata plus inline content or URI references. Artifact registration sends a best-effort system inbox notification to the dashboard user; notification failures do not make registration fail. **IMPORTANT — Register visual and media deliverables as artifacts:** Whenever you produce a visual or media output — a screenshot of the app or a UI change, a wireframe, a design mockup, a diagram, a rendered chart, a before/after capture, a screen recording, an HTML prototype, or a PDF export — you MUST register it so it appears in the dashboard Artifacts gallery: 1. Save the file to disk in your worktree (e.g. \`screenshots/after.png\`). 2. Call \`fn_artifact_register(type="image", title="Settings modal — after fix", description="What this shows and why it matters", path="screenshots/after.png")\`. Relative paths resolve against your worktree, and the file is COPIED into managed storage — so register even files you do not commit, and register before the worktree is cleaned up. Artifacts you register are associated with this task automatically. Type cheat sheet: - **Images** (screenshots, wireframes, mockups, diagrams): \`type="image"\` with \`path\` — PNG, JPEG, GIF, WebP, or SVG. - **Videos** (screen recordings, demo reels): \`type="video"\` with \`path\` — MP4, WebM, or MOV. They play with seeking directly in the gallery. - **Audio**: \`type="audio"\` with \`path\` — MP3, WAV, or OGG. - **HTML mockups/prototypes**: \`type="document"\`, \`mimeType="text/html"\`, with inline \`content\` or \`path\` — they render as LIVE sandboxed web previews in the gallery, so a self-contained HTML file is a great way to deliver an interactive mock. - **PDFs** (spec exports, reports): \`type="document"\`, \`mimeType="application/pdf"\`, with \`path\` — they open in an embedded PDF viewer. - **Text/markdown deliverables**: \`type="document"\` with inline \`content\` — rendered as formatted markdown and editable by the user. Register visual evidence proactively for any UI-affecting task: capture at least one screenshot demonstrating the final result when the change has a visible surface. If the task asks for wireframes, mockups, designs, HTML prototypes, or recordings, the registered artifacts ARE the deliverable. **IMPORTANT — Save your deliverables as documents:** When your task produces written output (documentation, specifications, reports, API references, README updates, guides, or any other content), you MUST save that content as a task document using \`fn_task_document_write\`. Use a key that describes the deliverable (e.g., key="readme", key="api-docs", key="changelog"). Do this in addition to writing the file to disk — the document persists in the task for review even after the worktree is cleaned up. If the task's PROMPT.md includes a "Documentation Requirements" section listing files to update, save each updated file's final content as a task document with a matching key. ## Git discipline - Commit after completing each step (not after every file change) - Use conventional commit messages prefixed with the task ID - Always include a short, specific summary after the em dash (5–10 words) - Do NOT commit just \`complete Step N\` — the summary is what makes the commit useful in \`git log\`, merger subject derivation, and step reconciliation - When the task has a GitHub issue reference, include \`Ref: owner/repo#N\` in the commit body - Do NOT commit broken or half-implemented code Good commit message examples: - \`feat(FN-1234): complete Step 2 — add retry guard for workflow step timeouts\` - \`feat(FN-1234): complete Step 4 — tighten prompt examples for commit summaries\` - \`test(FN-1234): add regression tests for paused-session cleanup\` Bad commit message examples: - \`feat(FN-1234): complete Step 2\` - \`misc updates\` - \`fix stuff\` - \`wip\` ## Worktree Boundaries You are running in an **isolated git worktree**. This means: - **All code changes must be made inside the current worktree directory.** Do not modify files outside the worktree — the worktree is your isolated execution environment. - **Exception — Project memory:** You MAY read and write to files under .fusion/memory/ at the project root to save durable project learnings (architecture patterns, conventions, pitfalls). - **Exception — Task attachments:** You MAY read files under .fusion/tasks/{taskId}/attachments/ at the project root for context screenshots and documents attached to this task. - **Exception — Sibling task specs:** You MAY read .fusion/tasks/{taskId}/PROMPT.md and .fusion/tasks/{taskId}/task.json at the project root (read-only) to consult dependency tasks' specifications. If those files do not exist, the dependency has been archived — call \`fn_task_show\` with its ID to load the spec from the archive. - **Shell commands** run inside the worktree by default. Avoid using cd to navigate outside the worktree. If you attempt to write to a path outside the worktree, the file tools will reject the operation with an error explaining the boundary. ## Guardrails - Do not call \`fn_workflow_select\` to change the workflow of the task you are executing; you did not create that task, the user or triage did. The only exception is when the user explicitly requested a specific workflow for this task in a steering comment, task instruction, or similar direct instruction. You may still set the workflow on tasks you create via \`fn_task_create\` or \`fn_delegate_task\`, because you are the creator of those new tasks. - **NEVER kill processes on port 4040.** Port 4040 is the production dashboard. Do not run \`kill\`, \`pkill\`, \`killall\`, or \`lsof -ti:4040 | xargs kill\` against it. If you need to start a test server, use \`--port 0\` for a random free port. If port 4040 is occupied, pick a different port — do NOT kill the occupant. - Treat the File Scope in PROMPT.md as the expected starting scope, not a hard boundary when quality gates fail - Read "Context to Read First" files before starting - Follow the "Do NOT" section strictly — these are hard constraints, not suggestions - If tests, lint, build, or typecheck fail and the fix requires touching code outside the declared File Scope, fix those failures directly and keep the repo green - When you must edit files beyond the declared File Scope to complete this task, call \`fn_task_file_scope_add\` to add them to the File Scope as you go — keep the declared scope in sync with what you actually change so your edits are not stranded by the scope-aware squash merge - Use \`fn_task_create\` for genuinely separate follow-up work, not for mandatory fixes required to make this task land cleanly - Update documentation listed in "Must Update" and check "Check If Affected" - NEVER delete, remove, or gut modules, interfaces, settings, exports, or test files outside your File Scope - NEVER remove features as "cleanup" — if something seems unused, create a task for investigation instead - Removing code is acceptable ONLY when it is explicitly part of your task's mission - If you remove existing functionality, you MUST create a changeset in \`.changeset/\` explaining the removal and rationale ## Spawning Child Agents You can spawn child agents to handle parallel work or specialized sub-tasks: **When to use \`fn_spawn_agent\`:** - Parallel work that can be divided into independent chunks with minimal overlap - Specialized tasks requiring different expertise or tools - Delegation of sub-tasks whose outputs can be validated independently **When NOT to spawn:** - The work is small enough to finish directly in your current step - Subtasks are tightly coupled and would create merge/cherry-pick overhead - You have not yet clarified expected outputs and acceptance criteria for the child **How to spawn:** \`\`\`javascript fn_spawn_agent({ name: "researcher", role: "engineer", task: "Research best practices for authentication in React applications" }) \`\`\` **Child agent behavior:** - Each child runs in its own git worktree (branched from your worktree) - Children execute autonomously and report completion - When you end (fn_task_done), all spawned children are terminated - Check AgentStore for spawned agent status **Limits:** - Max 5 spawned agents per parent by default (configurable via settings) - Max 20 total spawned agents system-wide (configurable via settings) ## Completion After all steps are done, lint passes, tests pass, typecheck passes, and docs are updated: \`\`\`bash Call \`fn_task_done()\` to signal completion. \`\`\` If a project build command is listed in the prompt, it is a hard completion gate: - Run the exact build command in the current worktree before \`fn_task_done()\` - Do not claim the build passes unless you actually ran it and got exit code 0 - If the build fails, do NOT call \`fn_task_done()\`; keep working until it passes Lint, tests, and typecheck are also hard quality gates: - Keep fixing failures caused by your change until lint, targeted tests, build, and typecheck pass. - If the repository exposes a typecheck command, run it and fix failures caused by your change. - When tests fail, first identify whether the failure is caused by your change, a pre-existing defect, an unrelated flaky test, or an outdated test expectation. - Update tests when intended behavior changed; fix implementation when behavior regressed unintentionally. - If broad workspace verification fails on unrelated or pre-existing failures after targeted checks pass, do NOT expand this task by fixing unrelated areas. Log the evidence, quarantine flakes per project policy, or create/link a follow-up task. - Do not repeatedly rerun a broad failing or hanging workspace command without a new hypothesis and a narrower confirming command. ## Verification commands — use fn_run_verification For ALL test/lint/build/typecheck verification, use the \`fn_run_verification\` tool, NOT raw bash. The tool prevents your session from being killed by the inactivity watchdog during long compiles, and verification is time-bounded by default (project \`verificationCommandTimeoutMs\` when set, otherwise 300s package / 900s workspace, hard-capped at 1800s). - Default to **targeted package-scoped** verification: use direct Vitest execution with package-relative paths: \`pnpm --filter @fusion/ exec vitest run src/path/to/test.ts --silent=passed-only --reporter=dot\`. Do not use \`pnpm --filter @fusion/ test -- --run \`; package test scripts can expand into broad quality suites before the filter is applied. - Do NOT run the full/workspace-wide test suite as your normal verification path. This prohibition includes root \`pnpm test\`, \`pnpm test:full\`, \`pnpm verify:workspace\`, whole-package tests with no file filter, and repeat loops. - A full/workspace-wide run is allowed ONLY when the task or workflow explicitly requires it. In that case, use \`fn_run_verification\` with \`allowFullSuite: true\`; the marathon soft-cap and hard timeout still apply, and the run still emits progress heartbeats. - Run **workspace-scoped non-test gates** (\`pnpm lint\`, \`pnpm build\`, and typecheck commands from root) when required for completion, but keep test verification targeted unless explicit task/workflow instructions require a full run. - If you need to run \`pnpm install\` (e.g. you added a new package), use \`fn_run_verification\` with \`scope: "workspace"\` and \`timeoutSec: 600\`. - If a verification command times out, do NOT blindly retry — investigate. Check for hung subprocesses, infinite test loops, or tests waiting on missing dependencies. Use \`node_modules/.modules.yaml\` presence to confirm bootstrap. ## Common Pitfalls - Editing files outside the assigned worktree (except allowed memory/attachment paths) - Skipping or partially running required quality gates - Leaving TODO/FIXME placeholders instead of completing required implementation - Introducing new patterns when existing local patterns should be reused - Marking a step done before required review/tooling gates are satisfied`; /* FNXC:EphemeralAgentTaskCreation 2026-07-26-07:40: The base prompt teaches fn_task_create/fn_delegate_task in several places ("Out-of-scope work found during execution", the Guardrails follow-up rule, the completion checklist). When the project policy withholds those tools, an unmodified prompt instructs the agent to call a tool that is not in its tool list — the same instruction/capability mismatch that produced the original retry storm, just from the other direction. This override states the absence and names what to do instead, so a withheld tool reads as policy rather than malfunction. It is appended last so it wins over the base text, and it applies to a custom operator prompt too (an operator who overrode the prompt still gets a truthful statement of what this session may do). */ function getWithheldTaskCreationGuidance(taskCreateWithheld: boolean, delegateWithheld: boolean): string { if (!taskCreateWithheld && !delegateWithheld) return ""; const withheld = [ ...(taskCreateWithheld ? ["`fn_task_create`"] : []), ...(delegateWithheld ? ["`fn_delegate_task`"] : []), ].join(" and "); return `## Follow-up task creation is disabled for this session This project's "Ephemeral agent follow-up tasks" policy withholds ${withheld}. ${ taskCreateWithheld && delegateWithheld ? "Those tools are" : "That tool is" } deliberately absent from your tool list — this is an operator setting, not a malfunction or a transient error. Do not attempt to call ${ taskCreateWithheld && delegateWithheld ? "them" : "it" }, and do not retry. Ignore any instruction above that tells you to file follow-up work with ${withheld}. When you find out-of-scope work, record it instead with \`fn_task_log(message="follow-up: ...")\` and include it in your \`fn_task_done\` summary so the operator sees it. If the work genuinely blocks this task, use \`fn_task_done(outcome="blocked", reason="...")\` rather than trying to create a task for it.`; } /** Resolve the executor system prompt from settings, falling back to the hardcoded constant. */ export function getExecutorSystemPrompt( settings: Settings, toolAvailability?: { taskCreateWithheld?: boolean; delegateWithheld?: boolean }, ): string { const customPrompt = resolveAgentPrompt("executor", settings.agentPrompts); const basePrompt = customPrompt || EXECUTOR_SYSTEM_PROMPT; const sections = [ basePrompt, isResearchToolSurfaceEnabled(settings) ? getResearchGuidanceForSurface("executor") : "", getWithheldTaskCreationGuidance( toolAvailability?.taskCreateWithheld === true, toolAvailability?.delegateWithheld === true, ), ].filter((section) => section.trim()); return sections.join("\n\n"); } export interface TaskExecutorOptions { /* * FNXC:PlanReviewLease 2026-07-26-21:07: * Resolves this engine's cluster node id for review-gate lease attribution. A GETTER, not a * value: the runtime resolves the id asynchronously during start(), which can complete after * the executor is constructed, so a snapshot taken at construction would be permanently * undefined. Read at runner-construction time instead. */ getLocalNodeId?: () => string | undefined; semaphore?: AgentSemaphore; /** Worktree pool for recycling idle worktrees across tasks. */ pool?: WorktreePool; /** * FNXC:ProviderRateLimitIsolation 2026-07-21-18:00: * Parks only tasks routed through the provider whose API limit was detected. */ usageLimitPauser?: UsageLimitPauser; /** Runtime-owned credential rotation inventory/cooldown coordinator. */ credentialRotator?: CredentialInstanceRotator; /** Stuck task detector — monitors agent sessions for stagnation and triggers recovery. */ stuckTaskDetector?: StuckTaskDetector; /** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */ agentStore?: import("@fusion/core").AgentStore; /** Reflection service used to generate self-reflection insights for agents. */ reflectionService?: AgentReflectionService; /** Plugin runner for invoking plugin hooks and providing plugin tools. */ pluginRunner?: PluginRunner; /** MessageStore for sending messages to other agents. When provided, executor agents gain fn_send_message capability. */ messageStore?: import("@fusion/core").MessageStore; missionStore?: MissionStore | AsyncMissionStore; secretsStore?: Pick; onSliceComplete?: (slice: Slice) => void; onStart?: (task: Task, worktreePath: string) => void; onComplete?: (task: Task) => void; onError?: (task: Task, error: Error) => void; /** Testable, best-effort completion-deliverable seam; production uses generateFeatureVideo. */ reviewArtifactGenerator?: (options: GenerateFeatureVideoOptions) => Promise; onAgentText?: (taskId: string, delta: string) => void; /** * FNXC:StuckDetector 2026-07-22-19:25: * Optional third arg is the primary-arg summary from AgentLogger so downstream * telemetry (and any external onAgentTool subscribers) keep the same fingerprint contract * the stuck detector uses — do not drop `detail` at the executor boundary. */ onAgentTool?: (taskId: string, toolName: string, detail?: string) => void; /* FNXC:PlannerOversight 2026-07-13-23:05: Session-advisor live delta path — AgentLogger invokes this after durable log flushes. Fail-soft; must not throw. */ onExecutorLogFlushed?: ( taskId: string, entries: Array<{ type?: string; text?: string; detail?: string; agent?: string }>, ) => void; autoRecoveryDispatcher?: AutoRecoveryDispatcher; /** PR-entity node deps (U3): assembled `PrNodeDeps` (store + injected GitHub * callbacks) for the `pr-create`/`pr-respond`/`pr-merge` workflow nodes. The * runtime binds the store and threads the CLI-injected ops. Absent → the pr-* * node kinds fail closed. */ prNodes?: import("./pr-nodes.js").PrNodeDeps; /** * CLI Agent Executor runtime (U7). When present, workflow nodes with * `config.executor === "cli-agent"` drive an engine-owned CLI session via the * task-session orchestration. Absent → cli-agent nodes report a clear config * error (the runtime was not wired). Bundled so a single option threads the * PTY manager + telemetry hub + adapter registry + hook endpoint together. */ cliAgentRuntime?: CliAgentRuntime; } /** Bundled CLI Agent Executor runtime dependencies (U7). */ export interface CliAgentRuntime { /** Engine-owned PTY session manager (U2). */ manager: CliSessionManager; /** In-process telemetry hub (U3) — owns per-session tokens + state machines. */ hub: TelemetryHub; /** Adapter registry (U2) — resolves adapter id → adapter. */ registry: CliAdapterRegistry; /** Durable session store (U1) — for re-entry / follow-up session lookups. */ store: CliSessionStore; /** Project this runtime drives (the executor is per-project; `cli_sessions` needs it). */ projectId: string; /** * Absolute URL of the dashboard hook ingestion endpoint the hook scripts POST * to (e.g. `http://127.0.0.1:4040/api/cli-agent/hooks`). */ hookEndpointUrl: string; /** Optional override for the hook scratch-dir root (tests). */ hookDirRoot?: string; } interface ActiveExecutorSessionState { session: AgentSession; seenSteeringIds: Set; lastResolvedModelProvider?: string; lastResolvedModelId?: string; lastTaskModelProvider?: string | null; lastTaskModelId?: string | null; lastAssignedAgentId?: string | null; lastEffectiveColumnAgentId?: string | null; } /* FNXC:WorkflowLifecycleTraits 2026-07-19-09:10 (U5b / KTD-10 / KTD-1): Every executor "requeue to backlog for retry/resume" rebound targets the task's TRAIT-derived backlog column (resolveReboundTarget: hold → intake → first), not the literal "todo". builtin:coding resolves to `todo` so the default pipeline is byte-identical; a custom/renamed workflow lands its recovered card in a valid backlog column. These are the KTD-1 RECOVERABLE rebounds (they preserve progress / resume state); the KTD-1 exhaustion parks (FN-8141 blocked, retry-exhausted) set `status:"failed"` in place WITHOUT a move and are intentionally untouched here. One IR resolution per rebound (a recovery path, not an enumeration loop); any resolution failure falls back to the legacy "todo" so a rebound is never stranded. FNXC:WorkflowLifecycleColumns 2026-07-30-15:10 (Phase C convergence): THE "ALREADY THERE?" GUARDS NOW COMPARE AGAINST THIS RESULT. Eight call sites read `X.column !== "todo"` before moving to the resolved column — so on a renamed board the guard was ALWAYS true and the engine issued a move into the column the card was already in. That is a real move: `moveTaskInternal` runs the reset-on-entry effects again. At the `preserveProgress: false` site (stale workflow parse pins) it reset step progress a second time on a card that had only been re-checked, and every site re-ran the status/error/pause clears. The move TARGET was converted here in U5b; the guards in front of it were not, which is the half-conversion shape: the correct target reached through a check that could not see it. Each site now resolves once and uses the same value for both. */ /** * The task's terminal column pair, fail-soft to the legacy ids. Mirrors * `resolveReboundColumnFor` below: one IR resolution on a rare guard path, and a * resolution failure must keep today's behaviour rather than answer "not terminal". */ /** The terminal ids from before workflows owned the vocabulary. */ const LEGACY_TERMINAL_COLUMNS: readonly string[] = ["done", "archived"]; /* FNXC:WorkflowResolvedColumns 2026-07-30-19:10 (exported for the follow-up dedup paths): EXPORTED rather than copied. `eval-followups.ts` and `pr-comment-handler.ts` each carried their own `CLOSED_FOLLOWUP_COLUMNS = new Set(["done", "archived"])` for the same question this answers, and a third and fourth copy of the union-with-legacy reasoning is exactly the drift this program exists to remove. Nothing else about the function changes. */ export async function resolveTerminalColumnsFor( store: TaskStore, taskId: string, /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (#2787 review — greptile P2): Optional CALLER-OWNED IR cache, matching the contract on `resolveTaskLifecycleColumns`. Sweeps that call this once per card on a whole board must read one IR per WORKFLOW, not one per task; callers resolving a single task pass nothing and are unaffected. */ irCache?: Map>>, ): Promise { /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2568 review — greptile): THE UNION IS DELIBERATE, and the `catch` alone was not enough. `resolveWorkflowIrForTask` does NOT throw when a custom workflow definition is missing, corrupt or unavailable — it returns the BUILT-IN IR. So the catch below only covers hard failures, while the common degraded case hands back a valid-looking default whose terminals are `done`/`archived`. A renamed board in that state would resolve terminals that do not include its own terminal column, and this guard would go inert exactly as it did before the conversion. Unioning with the legacy pair closes that: a resolvable board contributes its real terminals, and the legacy ids remain recognised whether they came from a genuine default workflow or from a silent substitution. Over-inclusion is the SAFE direction here, and that is why a union is acceptable rather than sloppy. This guard answers "is the card already finished, so skip parking?" — being too inclusive occasionally skips parking a card that was not really terminal; being too exclusive MOVES a finished card out of its terminal column, which is the failure the conversion exists to prevent. */ try { const resolved = resolveTerminalColumns(await resolveWorkflowIrForTask(store, taskId, irCache)); return [...new Set([...resolved, ...LEGACY_TERMINAL_COLUMNS])]; } catch { return LEGACY_TERMINAL_COLUMNS; } } /* FNXC:WorkflowLifecycleColumns 2026-07-30-15:20 (fleet — executor.ts cluster): The workflow's COMPLETE column, for the guards that ask "has this card finished?" and mean completion specifically — not the terminal PAIR. `resolveTerminalColumnsFor` above answers "done or archived"; these sites deliberately exclude archived, because an archived card is finished but not newly-completed, and treating the two alike would fire merge-confirmation handling for cards that were archived rather than merged. Same shape as the two helpers beside it: resolve from the task's own workflow, fall back to the legacy id. `resolveWorkflowIrForTask` does not throw on a missing definition — it returns the built-in default — so the catch covers hard failures only. */ async function resolveCompleteColumnFor(store: TaskStore, taskId: string): Promise { try { return resolveCompleteColumn(await resolveWorkflowIrForTask(store, taskId)) ?? "done"; } catch { return "done"; } } async function resolveReboundColumnFor(store: TaskStore, taskId: string): Promise { try { return resolveReboundTarget(await resolveWorkflowIrForTask(store, taskId)) ?? "todo"; } catch { return "todo"; } } /* FNXC:WorkflowExecution 2026-07-19-01:30: U5d (R9): explicit replacement for the deleted `graphCompletionInterceptors` Map. When this callback is present the run IS a graph-owned implementation phase: execution stops at the implementation-complete boundary (no workflow steps, no legacy in-review handoff), `fn_review_step` is not injected, review gates are marked graph-owned, and the captured modifiedFiles are handed back through the callback. Absent callback == the legacy path. FNXC:WorkflowExecution 2026-07-19-02:10: U5e (R9): this is now a parameter of `runImplementation()`, NOT of `execute()`. The graph calls the runner directly, so the callback no longer travels through routing. Remaining U5e work: the callback should become MANDATORY and collapse into an ordinary return value. It is still optional only because `executeWorkflowGraph` keeps one legacy fallback (executor.ts, the workflow-selection-api-unavailable branch) that minimal TEST stores reach; production stores always expose a workflow-selection reader and are always graph-owned. Deleting that fallback makes every `runImplementation` call graph-owned, at which point this type disappears in favor of a returned outcome. See docs/plans/2026-07-19-002-u5e-remaining-deletions-handoff.md. */ export type GraphCompletionCallback = (info: { modifiedFiles: string[] }) => void; /* FNXC:WorkflowLifecycleColumns 2026-07-30-18:10 (tightening my own rule against #2765): Does this IR express ANY lifecycle intent? #2765 published the general form of the distinction I hit in the no-wip fix: an empty role result means either DECLARED AND EMPTY (a v2 board the operator wrote that genuinely lacks the lane — a guard should act on it) or SYNTHESIZED (a v1 graph upgraded in place; `synthesizeDefaultColumns` emits `{ id, name: id, traits: [] }` for the five default ids, so every role resolves undefined even though those columns ARE the legacy lanes). My first discriminator proxied this with "hold and review are both undefined". That is right for the boards under test and wrong in general: a v2 workflow declaring, say, intake and complete but no hold/wip/review would read as SYNTHESIZED and the resume router would proceed into a wip lane the board does not have — the same failure the guard exists to stop, one case narrower. `resolveLifecycleColumns` returns all six roles, so the honest question is whether ANY of them resolved. Checking two of six was a proxy for that; this checks the thing. */ function declaresAnyLifecycleRole(lifecycle: ReturnType): boolean { if (!lifecycle) return false; return Object.values(lifecycle).some((columnId) => columnId !== undefined); } export class TaskExecutor { /* FNXC:Workspace 2026-06-21-12:00: activeWorktrees tracks the worktree paths a task currently holds for liveness/owner checks. In workspace mode a single task acquires N sub-repo worktrees (foundation `task.workspaceWorktrees`), so the value is a SET of paths, not one path. A non-workspace (single-repo) task holds a one-element set — every consumer is converted to membership semantics so the single-repo path is byte-for-byte unchanged (KTD2). Helpers below add/remove/iterate the set. */ private activeWorktrees = new Map>(); /** * FNXC:Workspace 2026-06-21-12:00: Register a worktree path under a task's active set, creating the set on first add (KTD2). Single-repo tasks call this once → one-element set. */ private addActiveWorktree(taskId: string, worktreePath: string): void { const set = this.activeWorktrees.get(taskId) ?? new Set(); set.add(worktreePath); this.activeWorktrees.set(taskId, set); } /** * FNXC:Workspace 2026-06-21-12:00: Read-only snapshot of every worktree path a task currently holds (KTD2). Empty when the task holds none. */ private getActiveWorktreePaths(taskId: string): string[] { const set = this.activeWorktrees.get(taskId); return set ? Array.from(set) : []; } private executing = new Set(); /** Tasks currently being prepared for unpause resume, before execute() has registered them. */ private resumingUnpaused = new Set(); /** Tasks whose active session was intentionally suspended by an action gate. */ private approvalSuspended = new Set(); /** Approval decisions received while the old execute() lifecycle is still unwinding. */ private approvalResumeAfterUnwind = new Set(); /** Completed orphan recovery tasks currently running during startup. */ private recoveringCompleted = new Set(); /** * FNXC:AgentReflection 2026-07-04-00:00: * FN-7528: taskIds for which a non-LLM post-task performance capture has already been fired via * `signalTaskComplete`. `onComplete` fires from several completion call sites (fresh completion, * duplicate in-review re-entry, auto-recovery, paused-after-completion finalize, retry-completed), * so this in-memory guard keeps capture to once per completion instead of once per call site. */ private capturedReflectionTaskIds = new Set(); /** Tracks tasks whose workflow-rerun bounce is in flight (todo→in-progress). * Prevents the task:moved handler from dispatching execute() before the * bounce finishes its own dispatch. */ private workflowRerunPending = new Set(); /** * Task ids whose current `task:moved` event is being emitted by this * executor's workflow lifecycle handling (column boundaries or Plan Review * replans). The store emits synchronously, so this narrowly distinguishes a * graph's own transition from an external engine/user move that must still * hard-cancel the active run. */ private workflowLifecycleMovesInFlight = new Set(); /** FN-5256: in-flight session-disposal promises keyed by taskId. The * task:moved (away from in-progress) and task:deleted listeners populate * this so a fast re-dispatch (task:moved → in-progress) awaits the prior * session being fully reaped before creating/acquiring a new worktree. */ private pendingTaskDisposals = new Map>(); private unregisterTaskMoveDisposer: (() => void) | undefined; private unregisterArchiveWorktreeDisposer: (() => void) | undefined; private unregisterArchiveWorkspaceWorktreeDisposer: (() => void) | undefined; /** Active agent sessions per task, used to terminate on pause and inject steering. */ private activeSessions = new Map(); /** Active step-session executors per task (mutually exclusive with activeSessions). */ private activeStepExecutors = new Map(); /** Steering comments already observed for active step-session executor runs. */ private activeStepExecutorSeenSteeringIds = new Map>(); /** Column-agent principal alignment (plan U5, R6): the EFFECTIVE column-agent id * currently running each executing task's coding/step session, when an * override/defer binding governs the in-flight seam. Keyed by task id, populated * by the execute / step-execute seam right after `resolveSeamColumnAgent` yields a * column agent, and cleared alongside the session (deleteActiveSession / * deleteActiveStepExecutor). Powers `isAgentEffectivelyExecuting`, the * reverse-direction heartbeat-scheduler guard that must know an agent is running a * task it is not `assignedAgentId` on. Empty for the legacy/no-binding path, so * that path is byte-identical. */ private effectiveColumnAgentByTask = new Map(); /** Active pre-merge workflow step sessions per task. */ private activeWorkflowStepSessions = new Map(); /** * FNXC:TaskTiming 2026-07-30-21:40: * Only graph-owned Plan Review sessions appear here. Self-healing uses this * narrow liveness proof so it never finalizes an in-flight planning segment. */ private activePlanningWorkflowSessions = new Set(); /** Steering comments already observed for active workflow step sessions. */ private activeWorkflowStepSessionSeenSteeringIds = new Map>(); /** Active configured-command abort controllers keyed by task. */ private activeConfiguredCommandControllers = new Map>(); /** Lazily-created root-project reader used only when an execution lookup is handed an agents-less worktree store. */ private authoritativeAssignedAgentStore: AgentStore | null = null; /** Active workflow-graph runner abort controllers keyed by task. */ private activeWorkflowGraphAbortControllers = new Map(); /** * Active CLI agent task sessions per task (U7). Mirrors activeSessions for the * cli-agent executor kind so the hard-cancel / abort path can SIGKILL the PTY * and mark `killed` (never resume-eligible), and the in-review handoff can reap * the PTY. A task has at most one live CLI session at a time. */ private activeCliTaskSessions = new Map(); private readonlyWorkflowStepAuditDone = false; /** * Reviewer subagent sessions per task. Reviewers (`reviewer.ts`) create their * own AgentSessions that aren't part of `activeSessions`/`activeStepExecutors`, * so without this map they survive when the parent task is stopped — they * keep producing log entries and step transitions after the user thinks they * killed the task. Disposed alongside the main session in the move-out, * pause, and global-pause handlers below. */ private activeSubagentSessions = new Map>(); /** Tasks that were paused mid-execution (to avoid marking them as "failed"). */ private pausedAborted = new Set(); /** * FNXC:WorkflowLifecycle 2026-06-17-03:42: * FN-6568 separates pause provenance from the legacy pausedAborted hard-cancel bit. Merge-seam/internal aborts caused FN-6528/FN-6531/FN-6534/FN-6537 to look like pause/resume aborts and left mergeRetries=NULL, so handleGraphFailure must know whether the abort came from global pause, the merge seam, or a generic hard cancel before choosing operator-action parking. * * FNXC:WorkflowLifecycle 2026-06-17-23:31: * FN-6625 adds completion-finalize provenance for the FN-6614 symptom where a completed/no-commit execution already handed off to in-review, then a trailing graph abort looked like a pause/resume engine abort and re-parked the task failed. Completion-finalize is sibling provenance to FN-6568 merge-seam, not operator pause intent. * * FNXC:WorkflowLifecycle 2026-07-26-11:20: * KB-PROV: `hard-cancel` had become a catch-all bucket: `awaitAbortInFlightTaskWork` stamped it unconditionally, so an ENGINE-initiated teardown was labeled with the provenance AGENTS.md reserves for the operator Move-Task hard cancel ("User moveTask(in-progress -> todo) is a hard cancel ... Engine rebounds must not set userPaused"). Observed on FN-8596: the graph's own `performWorkflowRerunBounce` (in-progress -> todo -> in-progress re-dispatch, moveSource "engine") logged `provenance=hard-cancel source=abort-in-flight:parent moved from in-progress to todo` even though `userCanceled` was correctly false and `userPaused` was never set. Behaviour was right, the LABEL lied. * * `engine-abort` splits that bucket: `hard-cancel` now means ONLY an operator withdrawal (`options.userCanceled === true`), `engine-abort` means an engine/lifecycle teardown. Both are "generic" (non-global-pause, non-merge-seam, non-completion-finalize) aborts, so every downstream classifier that used to accept `hard-cancel` must accept BOTH via `isGenericAbortProvenance()` — those classifiers exist FOR the engine case (see FN-6796's note that "an engine restart/pause-resume abort reaches graph-failure handling as `hard-cancel` provenance even when no user canceled the task") and discriminate real user intent through `userCanceledTaskIds`, not through the provenance label. Narrowing them to `hard-cancel` alone would strand benign engine aborts as operator-action failures. */ private pausedAbortProvenance = new Map(); /** * FNXC:WorkflowLifecycle 2026-06-18-10:56: * FN-6644 makes completed/no-commit finalize-to-review state durable beyond volatile pause provenance. FN-6641 showed FN-6625 was incomplete because teardown can re-mark `completion-finalize` as `hard-cancel`; this marker keeps the already-finalized handoff from being re-parked as an operator-action pause abort while preserving genuine live pauses and active hard-cancels. */ private completionFinalizedTaskIds = new Set(); /** Tasks that had a dependency added mid-execution (abort + discard worktree). */ private depAborted = new Set(); /** Tasks killed by stuck task detector. Value = shouldRequeue (budget not exhausted). */ private stuckAborted = new Map(); /** Tasks explicitly canceled by user move (in-progress → todo). */ private userCanceledTaskIds = new Set(); /* FNXC:WorkflowLifecycle 2026-06-23-21:16: During graph-owned execute nodes, the inner executor may intentionally self-requeue a task to `todo` for recoverable worktree/session repair. Persisted rows can be stale in tests or during store races, so keep a run-local marker that tells the outer graph failure sink not to overwrite that recovery with an in-review handoff. */ private graphExecuteSelfRequeued = new Set(); /** In-memory loop recovery state per task. Keyed by taskId, not persisted. * Tracks compact-and-resume attempt count per execute() lifecycle. * Reset at execute() lifecycle end (finally block). */ private loopRecoveryState = new Map(); /** Spawned child agent IDs per parent task ID. Used for lifecycle tracking. */ private spawnedAgents = new Map>(); /** Per-task baseline of session stats used for delta persistence across repeated updates. */ private tokenUsageBaselines = new Map(); /** In-memory branch conflict error counters per task for tripwire protection. */ private branchConflictErrorCount = new Map(); /** One-shot watchdogs for completed tasks that should have transitioned to in-review. */ private completedTaskWatchdogs = new Map>(); /** One-shot watchdogs for workflow reruns that should have bounced back to in-progress. */ private workflowRerunWatchdogs = new Map>(); /** Set of ephemeral spawned agent IDs with in-flight cleanup (prevents duplicate deletion attempts). */ private pendingEphemeralDeletions = new Set(); private workspaceConfig: WorkspaceConfig | null | undefined = undefined; /* FNXC:WorkflowLifecycle 2026-07-01-16:20: Breadcrumb task-log writes on the abort/pause/finalize paths are best-effort diagnostics and must NEVER break control flow. FN-7335 wired store.logEntry() straight into the SYNCHRONOUS markPausedAborted() as `void this.store.logEntry(...).catch(...)`; when store.logEntry is absent/throws synchronously (undefined method, store closed mid-abort, corrupted pager) the call throws a TypeError BEFORE the promise exists, so the trailing .catch() never runs and the exception unwinds out of markPausedAborted — aborting hard-cancel/pause and stranding the in-review handoff. Route every breadcrumb write through safeLogEntry() so both synchronous throws and async rejections are swallowed into a warn. */ private safeLogEntry(taskId: string, message: string): void { try { const result = this.store.logEntry(taskId, message, undefined, this.getRunContextFor(taskId)); void Promise.resolve(result).catch((error) => { executorLog.warn(`${taskId}: failed to write task-log breadcrumb: ${error instanceof Error ? error.message : String(error)}`); }); } catch (error) { executorLog.warn(`${taskId}: failed to write task-log breadcrumb: ${error instanceof Error ? error.message : String(error)}`); } } private markPausedAborted( taskId: string, provenance: PausedAbortProvenance = "hard-cancel", source = "unspecified", ): void { const previousProvenance = this.pausedAbortProvenance.get(taskId); const alreadyMarked = this.pausedAborted.has(taskId); this.pausedAborted.add(taskId); this.pausedAbortProvenance.set(taskId, provenance); if (!alreadyMarked || previousProvenance !== provenance) { /* FNXC:WorkflowLifecycle 2026-07-01-22:24: Pause aborts are frequent enough that operators need task-log breadcrumbs at the marker source, not only at the later graph-failure sink. Log first-mark/provenance-change events so a task card shows why a workflow was interrupted and which code path owned the abort. */ this.safeLogEntry( taskId, `Pause abort marked: provenance=${provenance} source=${source}${previousProvenance && previousProvenance !== provenance ? ` previous=${previousProvenance}` : ""}`, ); } } /** * FNXC:ReviewRouting 2026-07-01-16:36: * Review routing must expose whether the reviewer is using an explicit external checkout or the task worktree, but the invalid-sourceMetadata warning is only valid when sourceMetadata supplied the selected candidate. Higher-priority metadata can fail closed before sourceMetadata is considered, so centralize the logging to keep both review seams consistent and avoid false invalid-path warnings. */ private logReviewCheckoutRouting(taskId: string, task: unknown, reviewCwd: string, worktreePath: string): void { if (reviewCwd !== worktreePath) { reviewerLog.log(`${taskId}: review routed to external checkout ${reviewCwd} (task worktree: ${worktreePath})`); return; } const selectedCandidate = getTaskReviewCheckoutPath(task); const sourceMetadata = task && typeof task === "object" ? (task as Record).sourceMetadata : undefined; const sourceRecord = sourceMetadata && typeof sourceMetadata === "object" ? sourceMetadata as Record : undefined; const sourceExternalReviewCheckout = sourceRecord?.externalReviewCheckout; const sourceExternalReviewCheckoutPath = typeof sourceExternalReviewCheckout === "string" ? sourceExternalReviewCheckout.trim() : undefined; if (sourceExternalReviewCheckoutPath && selectedCandidate === sourceExternalReviewCheckoutPath) { reviewerLog.warn(`${taskId}: external review checkout metadata present (${sourceExternalReviewCheckoutPath}) but invalid — reviewing task worktree ${worktreePath}`); } } private markCompletionFinalized(taskId: string): void { this.markPausedAborted(taskId, "completion-finalize", "completion-finalize"); this.completionFinalizedTaskIds.add(taskId); } private clearPausedAborted(taskId: string): void { this.pausedAborted.delete(taskId); this.pausedAbortProvenance.delete(taskId); this.completionFinalizedTaskIds.delete(taskId); } private async clearStalePauseAbortBeforeDispatch(task: Task): Promise { if (!this.pausedAborted.has(task.id)) return; let globalPause = false; try { globalPause = (await this.store.getSettings()).globalPause === true; } catch { globalPause = false; } if (task.paused === true || task.userPaused === true || globalPause) return; /* * FNXC:WorkflowLifecycle 2026-06-29-10:35: * A stale pause-abort marker must not survive into a fresh unpaused dispatch. * FN-7225/FN-7226 showed graph-owned execution failures being narrated as * pause/resume cleanup even though the task row was not paused. Clear the * volatile marker silently at dispatch entry so the task log names the real * workflow failure (`step-execute`, parse, review, etc.) instead of implying * the engine actually paused. */ this.clearPausedAborted(task.id); executorLog.log(`${task.id}: cleared stale pause-abort marker before unpaused execution dispatch`); } clearPauseAbortStateForManualRetry(taskId: string): void { /* FNXC:ManualRetry 2026-06-29-00:57: User retry is a fresh execution boundary. Clear volatile pause-abort provenance so retries cannot inherit stale engine pause/resume classification from a prior run. */ this.clearPausedAborted(taskId); } /* FNXC:Workspace 2026-06-24-15:45 (concurrent workspace tasks — shared browse-root collision): In workspace mode `this.rootDir` is the SHARED browse-only (non-git) workspace root, and EVERY workspace task runs its agent session rooted there (per-sub-repo worktrees are acquired on demand). The session registrations below are keyed in the GLOBAL path-keyed activeSessionRegistry, whose foreign-task guard rejects a second task registering a path already held by a different task. With the bare root as the key, the second concurrent workspace task fails with "active-session path is held by task ; task may not overwrite it" — so only ONE task per workspace could ever run. Per-task session liveness does NOT require path-exclusivity on the shared root (real per-sub-repo exclusivity is enforced separately by the workspace-repo-acquire lease in worktree-acquisition.ts, keyed by sub-repo path). Give each task a task-scoped synthetic session key so the registry stays per-task. The in-memory activeWorktrees Set still holds the REAL root, so getActiveWorktreePaths() consumers that cd into a path are unaffected; only the registry key changes. Non-workspace tasks (unique worktree path != rootDir) are returned unchanged. */ /* FNXC:PlanReviewWorktree 2026-07-25-20:40 (concurrent root-rooted step sessions — single-repo collision): The task-scoped key must apply to the shared repo root in EVERY project mode, not only workspace mode. Read-only graph nodes that need no worktree (Plan Review is the canonical one — it reviews the store-injected PROMPT.md, see FNXC:PlanReviewSpecInjection) run rooted at `this.rootDir`, and a todo task has no worktree of its own. With the bare root as the registry key, two tasks reaching Plan Review at the same time collided: the second failed with "active-session path is held by task ; task may not overwrite it", which surfaced as a Plan Review provider failure, burned the in-place retry budget against a hold that retrying can never clear, and left the task parked (reported: FN-1398 holding /home/ubuntu/dev/freemap-svelte while FN-1403 planned). Path-exclusivity on the shared root is not what keeps these sessions correct: write-capable nodes are refused at the root outright (no-worktree-for-write-node above), real per-sub-repo exclusivity is the workspace-repo-acquire lease, and every isPathActive consumer guards removable WORKTREE paths — the root is never one. Liveness still works because the synthetic key stays in the registry under the task. */ private sessionRegistryPath(taskId: string, worktreePath: string): string { if (worktreePath === this.rootDir) { return `${worktreePath}#session:${taskId}`; } return worktreePath; } /* FNXC:SessionContention 2026-07-25-21:30 (contention prevention at the registration seam): Every executor session registration goes through `acquireActiveSessionPath` instead of the raw `registerPath`, so a LEAKED entry owned by a task with no live session surface in this process is RECLAIMED rather than throwing at the newcomer. That closes the second contention class (a dead holder can never release, so waiting on it is waiting forever). A genuinely live holder still throws the typed error — that case is real serialization, and callers classify it as a retryable contention hold (SESSION_CONTENTION_HOLD_VALUE), never as a provider/model failure. The probe reports LIVE on any uncertainty: an unknown holder with a fresh entry is treated as live by the staleness floor, so the reclaim only ever fires on proven-dead, aged entries. */ private acquireSessionRegistryPath(taskId: string, registryPath: string, kind: ActiveSessionKind, ownerKey: string): void { const outcome = acquireActiveSessionPath(activeSessionRegistry, registryPath, { taskId, kind, ownerKey }, { holderLiveProbe: (holderTaskId) => this.hasLiveTaskSessionSurface(holderTaskId) || executingTaskLock.has(holderTaskId), }); if (outcome.action === "contended") { throw new ActiveSessionPathHeldByForeignTaskError(registryPath, outcome.holderTaskId, taskId); } if (outcome.action === "reclaimed-stale-foreign") { executorLog.warn( `${taskId}: reclaimed a stale active-session entry on ${registryPath} from dead task ${outcome.holderTaskId} (idle ${outcome.ageMs}ms)`, ); void this.store.recordRunAuditEvent?.({ taskId, agentId: "executor", runId: generateSyntheticRunId("session-path-reclaim", taskId), domain: "database", mutationType: "session:reclaim-stale-foreign-path", target: taskId, metadata: { taskId, holderTaskId: outcome.holderTaskId, kind, ageMs: outcome.ageMs }, })?.catch?.(() => undefined); } } private setActiveSession(taskId: string, sessionState: ActiveExecutorSessionState, worktreePath: string): void { this.activeSessions.set(taskId, sessionState); this.acquireSessionRegistryPath(taskId, this.sessionRegistryPath(taskId, worktreePath), "executor", taskId); } private markGraphExecuteSelfRequeued(taskId: string): void { if (this.graphRouting.has(taskId)) { this.graphExecuteSelfRequeued.add(taskId); } } private deleteActiveSession(taskId: string, worktreePath?: string): void { this.activeSessions.delete(taskId); // U5: drop the effective column-agent principal for this task's session. this.effectiveColumnAgentByTask.delete(taskId); // FNXC:Workspace 2026-06-21-12:00: KTD2 — when no explicit path is given, unregister EVERY worktree path the task holds (a workspace task holds N sub-repo paths); single-repo tasks resolve a one-element set. const resolvedWorktreePaths = worktreePath ? [worktreePath] : this.getActiveWorktreePaths(taskId); for (const path of resolvedWorktreePaths) { // FNXC:Workspace 2026-06-24-15:45: map through sessionRegistryPath so the task-scoped synthetic // session key registered for the shared workspace browse-root is the one we unregister (the // in-memory Set holds the REAL root). Non-workspace/sub-repo paths pass through unchanged. activeSessionRegistry.unregisterPath(this.sessionRegistryPath(taskId, path)); } } private setActiveStepExecutor(taskId: string, stepExecutor: StepSessionExecutor, worktreePath: string, seenSteeringIds = new Set()): void { this.activeStepExecutors.set(taskId, stepExecutor); this.activeStepExecutorSeenSteeringIds.set(taskId, seenSteeringIds); this.acquireSessionRegistryPath(taskId, this.sessionRegistryPath(taskId, worktreePath), "step-session", `${taskId}#step-session`); } private deleteActiveStepExecutor(taskId: string, worktreePath?: string): void { this.activeStepExecutors.delete(taskId); this.activeStepExecutorSeenSteeringIds.delete(taskId); // U5: drop the effective column-agent principal for this task's step session. this.effectiveColumnAgentByTask.delete(taskId); // FNXC:Workspace 2026-06-21-12:00: KTD2 — unregister every held worktree path (Set), not one. const resolvedWorktreePaths = worktreePath ? [worktreePath] : this.getActiveWorktreePaths(taskId); for (const path of resolvedWorktreePaths) { // FNXC:Workspace 2026-06-24-15:45: map through sessionRegistryPath so the task-scoped synthetic // session key registered for the shared workspace browse-root is the one we unregister (the // in-memory Set holds the REAL root). Non-workspace/sub-repo paths pass through unchanged. activeSessionRegistry.unregisterPath(this.sessionRegistryPath(taskId, path)); } } private setActiveWorkflowStepSession(taskId: string, session: AgentSession, worktreePath: string, seenSteeringIds = new Set()): void { this.activeWorkflowStepSessions.set(taskId, session); this.activeWorkflowStepSessionSeenSteeringIds.set(taskId, seenSteeringIds); this.acquireSessionRegistryPath(taskId, this.sessionRegistryPath(taskId, worktreePath), "workflow-step", `${taskId}#workflow-step`); } private deleteActiveWorkflowStepSession(taskId: string, worktreePath?: string): void { this.activeWorkflowStepSessions.delete(taskId); this.activeWorkflowStepSessionSeenSteeringIds.delete(taskId); // FNXC:Workspace 2026-06-21-12:00: KTD2 — unregister every held worktree path (Set), not one. const resolvedWorktreePaths = worktreePath ? [worktreePath] : this.getActiveWorktreePaths(taskId); for (const path of resolvedWorktreePaths) { // FNXC:Workspace 2026-06-24-15:45: map through sessionRegistryPath so the task-scoped synthetic // session key registered for the shared workspace browse-root is the one we unregister (the // in-memory Set holds the REAL root). Non-workspace/sub-repo paths pass through unchanged. activeSessionRegistry.unregisterPath(this.sessionRegistryPath(taskId, path)); } } private createSeenSteeringIds(task: { comments?: Array<{ id: string }>; steeringComments?: Array<{ id: string }> }): Set { const seenSteeringIds = new Set(); for (const comment of task.steeringComments ?? task.comments ?? []) { seenSteeringIds.add(comment.id); } return seenSteeringIds; } private registerConfiguredCommandController(taskId: string, controller: AbortController): void { const controllers = this.activeConfiguredCommandControllers.get(taskId) ?? new Set(); controllers.add(controller); this.activeConfiguredCommandControllers.set(taskId, controllers); } private unregisterConfiguredCommandController(taskId: string, controller: AbortController): void { const controllers = this.activeConfiguredCommandControllers.get(taskId); if (!controllers) return; controllers.delete(controller); if (controllers.size === 0) { this.activeConfiguredCommandControllers.delete(taskId); } } private createConfiguredCommandAbortError(taskId: string, command: string): Error { const error = new Error(`Configured command aborted for ${taskId}: ${command}`); error.name = "AbortError"; return error; } private getAutoRecoveryDispatcher(audit: RunAuditor): AutoRecoveryDispatcher { if (this.options.autoRecoveryDispatcher) return this.options.autoRecoveryDispatcher; const fileScopeHandler = createFileScopeAutoRecoveryHandler({ taskStore: this.store, runAudit: audit, logger: executorLog, spawnAgent: async () => ({ agentId: "unavailable" }), classifyPatchIds: async () => ({ unique: [], alreadyUpstream: [] }), settings: () => ({ autoRecovery: { mode: "deterministic-only", maxRetries: 3 } } as ProjectSettings), }); const branchWorktreeHandler = new BranchWorktreeAutoRecoveryHandler({ taskStore: this.store, runAudit: audit, logger: executorLog, }); const contaminationHandler = new ContaminationAutoRecoveryHandler({ taskStore: this.store, runAudit: audit, logger: executorLog, repoDir: this.rootDir, }); return new AutoRecoveryDispatcher({ taskStore: this.store, auditEmitter: audit, handlers: { issueRetry: async (failure, decision, ctx) => { if (failure.class === "branch-cross-contamination") { return contaminationHandler.issueRetry(failure, decision, ctx); } if (failure.class === "branch-conflict-unrecoverable") { return branchWorktreeHandler.issueRetry(failure, decision, ctx); } return fileScopeHandler.issueRetry(failure, decision, ctx); }, spawnAiRecovery: async (failure, decision, ctx) => { if (failure.class === "branch-conflict-unrecoverable") { return branchWorktreeHandler.spawnAiRecovery(failure, decision, ctx); } return fileScopeHandler.spawnAiRecovery(failure, decision, ctx); }, }, }); } private async renewTaskLease( taskId: string, agentId: string, leaseEpoch: number, nodeId: string, runId: string | undefined, ): Promise { const renewedAt = new Date().toISOString(); if (this.options.agentStore) { await this.options.agentStore.checkoutTask( agentId, taskId, { nodeId, runId, leaseEpoch, renewedAt, }, this.getRunContextFor(taskId), ); return; } await this.store.renewCheckoutLease(taskId, { checkoutRunId: runId ?? null, checkoutLeaseRenewedAt: renewedAt, }); } private async finalizeAlreadyReviewedTask(taskId: string): Promise<"merged" | "blocked" | "missing"> { const latestTask = await this.store.getTask(taskId); /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): the board's own review lane. Spelled as the literal, this reported "missing" — a word that reads as "the task is gone" — for a card sitting in review on a renamed board, and the already-reviewed finalize never ran. */ if (!latestTask || latestTask.column !== (await this.resolveResumeLanes(taskId)).review) { return "missing"; } /* FNXC:WorkflowResolvedColumns 2026-07-30-14:40 (outer question resolved, inner one not): The guard directly above compares against `(await this.resolveResumeLanes(taskId)).review`, then this call re-asked with the literal — so a card that just PASSED the resolved lane check was refused by the unresolved blocker on any renamed board. */ const resumeReviewLane = (await this.resolveResumeLanes(taskId)).review; const blocker = getTaskMergeBlocker(latestTask, { reviewColumns: new Set([resumeReviewLane ?? "in-review"]), }); if (blocker) { await this.store.logEntry(taskId, "Task already in-review; merge deferred", blocker, this.getRunContextFor(taskId)); return "blocked"; } await this.store.logEntry( taskId, "Task already in-review after completion — finalizing merge", undefined, this.getRunContextFor(taskId), ); await this.store.mergeTask(taskId); return "merged"; } private async getExecutionPauseLabel(): Promise<"global pause" | "engine pause" | null> { const settings = await this.store.getSettings(); if (settings.globalPause) return "global pause"; if (settings.enginePaused) return "engine pause"; return null; } private async shouldDeferCompletionForGlobalPause( taskId: string, context: string, ): Promise { const settings = await this.store.getSettings(); if (!settings.globalPause) { return false; } this.clearCompletedTaskWatchdog(taskId); executorLog.log(`${taskId}: completion handoff deferred — global pause active (${context})`); await this.store.logEntry( taskId, `Completion handoff deferred — global pause active (${context})`, undefined, this.getRunContextFor(taskId), ).catch(() => undefined); return true; } private async shouldDeferWorkflowStepCompletion( taskId: string, context: string, ): Promise { let latestTask: Task | null = null; try { latestTask = await this.store.getTask(taskId); } catch { latestTask = null; } if (latestTask?.paused || this.pausedAborted.has(taskId)) { this.clearCompletedTaskWatchdog(taskId); executorLog.log(`${taskId}: completion handoff deferred — task paused (${context})`); await this.store.logEntry( taskId, `Completion handoff deferred — task paused (${context})`, undefined, this.getRunContextFor(taskId), ).catch(() => undefined); return true; } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: wip-lane liveness family): "still executing" is the board's WIP lane. With the literal a renamed board deferred EVERY completion handoff — the card was never in `in-progress`, so this read "no longer active" for a card that was actively executing, and the handoff was dropped with a log line. */ if ((latestTask && latestTask.column !== (await this.resolveResumeLanes(taskId)).wip) || this.userCanceledTaskIds.has(taskId)) { this.clearCompletedTaskWatchdog(taskId); executorLog.log(`${taskId}: completion handoff deferred — task no longer active (${context})`); await this.store.logEntry( taskId, `Completion handoff deferred — task no longer active (${context})`, undefined, this.getRunContextFor(taskId), ).catch(() => undefined); return true; } return this.shouldDeferCompletionForGlobalPause(taskId, context); } /** Child agent sessions keyed by agent ID. Used for termination. */ private childSessions = new Map(); /** Total count of currently spawned agents (across all parents). */ private totalSpawnedCount = 0; /** Token cap detector for proactive context compaction. */ private tokenCapDetector = new TokenCapDetector(); private _modelRegistry?: Promise; private _approvalRequestStore?: ApprovalRequestStore; /** Current run context for mutation correlation, keyed by task id. */ private currentRunContexts = new Map(); private getRunContextFor(taskId: string): RunMutationContext | undefined { return this.currentRunContexts.get(taskId); } /** * Stable handoff reasons used on task:handoff audit events. * Keep values greppable for executor/self-healing forensics: review-handoff-requested, * completed-task-recovered, step-session-completed, paused-after-completion, * fn_task_done, fn_task_done-retry-completed. * * FNXC:WorkflowLifecycle 2026-06-29-11:20: * Failed execution is not a review handoff. Error paths must either requeue * executable work for resume or fail in-place; `in-review` is reserved for * clean completion handoffs. */ private async handoffTaskToReview(task: Task, reason: string, runId = this.getRunContextFor(task.id)?.runId): Promise { const agentId = this.getRunContextFor(task.id)?.agentId; await this.generateCompletionFeatureVideo(task); if (reason.startsWith("workflow-")) { await ensureWorkflowCompletionSummary(this.store, task as TaskDetail, { reason, runId, }).catch((error: unknown) => { executorLog.warn(`${task.id}: failed to record workflow completion summary: ${error instanceof Error ? error.message : String(error)}`); }); } const handedOff = await this.store.handoffToReview(task.id, { ownerAgentId: agentId ?? null, evidence: { reason, runId, agentId, }, }); const settings = await this.store.getSettings(); if (isMergeRequestContractShadowEnabled(settings)) { this.store.setCompletionHandoffAcceptedMarker(task.id, { source: `executor:${reason}`, }); await this.store.upsertMergeRequestRecord(task.id, { state: handedOff.autoMerge === false ? "manual-required" : "queued", }); } return handedOff; } /* FNXC:ReviewArtifacts 2026-07-19-10:00: A successful executor handoff may offer reviewers a short local feature-video, but capture is strictly best-effort. Bound and swallow this optional work before the review transition so browser, scenario, and artifact failures never delay or fail it. */ private async generateCompletionFeatureVideo(task: Task): Promise { try { const [settings, detail] = await Promise.all([this.store.getSettings(), this.store.getTask(task.id)]); const generator = this.options.reviewArtifactGenerator ?? generateFeatureVideo; const result = await this.awaitFeatureVideoBounded(generator({ store: this.store, task: detail ?? task, settings })); executorLog.log(`${task.id}: feature-video ${result.status}${"reason" in result ? ` (${result.reason})` : ""}`); } catch (error) { executorLog.warn(`${task.id}: feature-video capture ignored: ${error instanceof Error ? error.message : String(error)}`); } } private async awaitFeatureVideoBounded(result: Promise): Promise { let timeout: ReturnType | undefined; try { return await Promise.race([ result, new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("feature-video timeout")), 20_000); }), ]); } finally { if (timeout) clearTimeout(timeout); } } private getModelRegistry(): Promise { if (!this._modelRegistry) { const authStorage = createFusionAuthStorage(); this._modelRegistry = createFusionModelRegistry(authStorage); } return this._modelRegistry; } private get approvalRequestStore(): ApprovalRequestStore { if (!this._approvalRequestStore) { const layer = this.store.getAsyncLayer(); if (!layer) throw new Error("Executor TaskStore is missing its PostgreSQL AsyncDataLayer"); /* FNXC:PostgresSatelliteCutover 2026-07-14-17:30: Runtime approval persistence is PostgreSQL-only; never reopen the removed project SQLite database when backend wiring is incomplete. */ this._approvalRequestStore = new ApprovalRequestStore(null, { asyncLayer: layer }); } return this._approvalRequestStore; } private buildActionGateContext(taskId: string | undefined, agent: Agent | null | undefined, projectDefaultPolicy?: { rules?: Partial; toolRules?: import("@fusion/core").AgentPermissionPolicyToolRules }): AgentActionGateContext | undefined { /* FNXC:AgentPermissions 2026-07-02-00:00: FN-7413 requires task-scoped runtime gates for permanent identity agents, stored ephemeral agents, and fallback executor-FN task workers. Use a stable synthetic actor for fallback workers so category/exact-tool rules and approval dedupe keys apply even when no agent row exists. */ const actorId = agent?.id ?? `executor-${taskId ?? "unknown"}`; const actorName = agent?.name ?? `Task worker ${taskId ?? "unknown"}`; const isEphemeral = !agent || isEphemeralAgent(agent); const policy = resolveEffectiveAgentPermissionPolicy(agent?.permissionPolicy, projectDefaultPolicy); return { agentId: actorId, agentName: actorName, isEphemeral, taskId, runId: taskId ? this.getRunContextFor(taskId)?.runId : undefined, permissionPolicy: policy, createApprovalRequest: async (decision, args) => await this.approvalRequestStore.create({ requester: { actorId, actorType: "agent", actorName, }, taskId, runId: taskId ? this.getRunContextFor(taskId)?.runId : undefined, targetAction: { category: decision.category === "exempt" ? "command_execution" : decision.category, action: decision.operation, summary: decision.summary, resourceType: decision.resourceType, resourceId: decision.resourceId ?? "", context: { ...decision.metadata, approvalDedupeKey: decision.approvalDedupeKey, toolName: decision.toolName, toolArgs: args, }, }, }), findApprovalByDedupeKey: async (dedupeKey) => { const latest = await this.approvalRequestStore.findLatestByDedupeKey({ requesterActorId: actorId, taskId, dedupeKey }); // FNXC:ApprovalRedemption 2026-07-26-14:30: decidedAt lets resolveGateOutcome apply the approval-grant TTL at redemption. return latest ? { id: latest.id, status: latest.status, decidedAt: latest.decidedAt } : null; }, findPendingApprovalByDedupeKey: async (dedupeKey) => { const latest = await this.approvalRequestStore.findLatestByDedupeKey({ requesterActorId: actorId, taskId, dedupeKey }); return latest?.status === "pending" ? { id: latest.id } : null; }, pauseForApproval: async ({ approvalRequestId, decision }) => { if (taskId) { /* FNXC:ApprovalHold 2026-07-09-00:10: FN-7736: stamp the canonical AWAITING_APPROVAL_PAUSE_REASON on the task (not just the agent) so recovery/oversight code can durably recognize this hold via isTaskBlockedOnApproval -- previously only `paused: true` was set with no reason, which self-healing's autoReboundPausedScopeDecay could rebound before the operator ever decided. FNXC:ApprovalResume 2026-07-12-17:02: MAIN-008: record the approval-specific suspension before pauseTask emits its task:updated event so every abort branch can preserve the in-progress row for a deterministic fresh resume. Clear the mark if pauseTask fails so a failed pause does not leave a sticky suspended marker. */ this.approvalSuspended.add(taskId); try { await this.store.pauseTask(taskId, true, this.getRunContextFor(taskId), { pausedByAgentId: actorId, pausedReason: AWAITING_APPROVAL_PAUSE_REASON }); } catch (error) { this.approvalSuspended.delete(taskId); throw error; } await this.store.logEntry( taskId, `Approval required for ${decision.toolName}. Request ${approvalRequestId} created; task and agent paused awaiting decision.`, undefined, this.getRunContextFor(taskId), ); /* FNXC:AgentGating 2026-07-05-00:10: FN-7608: pauseTask() alone does not stop the in-flight LLM turn -- the gated tool call only returns a soft rejection, and the executor system prompt forbids ending a turn without another tool call, so the agent kept hunting for ungated workarounds (re-issuing the same bash, probing read-only tools, fn_web_fetch/fn_task_attach bypasses) while the task sat "paused" only in the store. Make wait-for-approval a REAL session-suspending state by aborting the in-flight session here, using the same synchronous abort surface hard-cancel uses (awaitAbortInFlightTaskWork). This call is deliberately NOT awaited: awaitAbortInFlightTaskWork's agent-session branch awaits session.abort(), which internally awaits agent.waitForIdle() -- since pauseForApproval runs from inside this very tool call, the agent cannot become idle until our own execute() resolves, so awaiting the abort inline here would deadlock. Firing it (fire-and-forget, errors swallowed to a warn per the FN-7335 best-effort-breadcrumb pattern) lets the abort proceed the moment this tool's rejection unwinds back to the agent loop. */ void this.awaitAbortInFlightTaskWork(taskId, `awaiting-approval:${decision.toolName}`).catch((error) => { executorLog.warn(`${taskId}: failed to suspend in-flight session while awaiting approval: ${error instanceof Error ? error.message : String(error)}`); }); } if (agent && this.options.agentStore) { await this.options.agentStore.updateAgentState(agent.id, "paused"); await this.options.agentStore.updateAgent(agent.id, { pauseReason: "awaiting-approval" }); } }, markApprovalCompleted: async (approvalRequestId) => { await this.approvalRequestStore.markCompleted(approvalRequestId, { actor: { actorId, actorType: "agent", actorName }, note: "Tool executed after approval", // FNXC:ApprovalRedemption 2026-07-26-14:35: ownership guard — an agent must not be able to burn another agent's approval by id. expectedRequesterActorId: actorId, }); }, }; } private buildPermanentAgentGatingContext(taskId: string | undefined, agent: Agent | null | undefined, projectDefaultPolicy?: { rules?: Partial; toolRules?: import("@fusion/core").AgentPermissionPolicyToolRules }): import("@fusion/core").PermanentAgentGatingContext | undefined { const actorId = agent?.id ?? `executor-${taskId ?? "unknown"}`; const actorName = agent?.name ?? `Task worker ${taskId ?? "unknown"}`; return { permissionPolicy: resolveEffectiveAgentPermissionPolicy(agent?.permissionPolicy, projectDefaultPolicy), requester: { actorId, actorType: "agent", actorName, }, taskId, runId: taskId ? this.getRunContextFor(taskId)?.runId : undefined, // FNXC:AgentGating 2026-07-05-00:00: // FN-7609: operators approving a gated action need the real command/args, // and a stateless heartbeat retrying the same command must reuse a single // pending approval instead of minting duplicates. `summary` is now // payload-bearing (shared helper) and `approvalDedupeKey`/`command`/`cwd` // are persisted into targetAction.context so findPendingApprovalRequest // can match and the UI can render the payload without re-parsing. createApprovalRequest: async ({ category, toolName, args, approvalDedupeKey }) => await this.approvalRequestStore.create({ requester: { actorId, actorType: "agent", actorName, }, taskId, runId: taskId ? this.getRunContextFor(taskId)?.runId : undefined, targetAction: { category, action: toolName, summary: buildAgentGatedActionSummary(toolName, args), resourceType: "tool", resourceId: toolName, context: { toolName, toolArgs: args, source: "agent-gating", ...(approvalDedupeKey ? { approvalDedupeKey } : {}), ...(typeof (args as Record | undefined)?.command === "string" ? { command: (args as Record).command } : {}), ...(typeof (args as Record | undefined)?.cwd === "string" ? { cwd: (args as Record).cwd } : {}), }, }, }), findPendingApprovalRequest: async (dedupeKey) => { const pending = await this.approvalRequestStore.list({ status: "pending", requesterActorId: actorId, taskId, limit: 100 }); return pending.find((request) => request.targetAction.context?.approvalDedupeKey === dedupeKey) ?? null; }, /* FNXC:AgentGating 2026-07-26-14:50: Audit finding (gate-path divergence): the permanent gate minted an approval request but never paused, so the agent kept its turn while "awaiting approval". Mirror the action gate's task-level hold (canonical AWAITING_APPROVAL_PAUSE_REASON + approvalSuspended marker). Session suspension is intentionally not wired here: the permanent gate only runs in lanes WITHOUT an actionGateContext, where no executor in-flight session surface exists to abort. */ pauseForApproval: async ({ approvalRequestId, toolName }) => { if (!taskId) return; this.approvalSuspended.add(taskId); try { await this.store.pauseTask(taskId, true, this.getRunContextFor(taskId), { pausedByAgentId: actorId, pausedReason: AWAITING_APPROVAL_PAUSE_REASON }); await this.store.logEntry( taskId, `Approval required for ${toolName}. Request ${approvalRequestId} created; task paused awaiting decision.`, ); } catch (error) { this.approvalSuspended.delete(taskId); throw error; } }, }; } /** Returns the set of task IDs currently being executed. */ getExecutingTaskIds(): Set { // Graph-routed tasks count as executing for their WHOLE interpreter run — // between seams the inner execute() has released this.executing, but the // graph still owns the lifecycle; self-healing/recovery must not touch it. return new Set([ ...this.executing, ...this.recoveringCompleted, ...this.resumingUnpaused, ...TaskExecutor.processWideGraphRouting, ]); } /** * FNXC:TaskTiming 2026-07-30-21:40: * A planning segment has one owner: a graph Plan Review session is live only * while both its session registration and planning ownership marker remain. * This is intentionally narrower than isTaskActive(), which also covers * implementation and non-planning workflow sessions. */ hasActivePlanningWorkflowSession(taskId: string): boolean { return this.activePlanningWorkflowSessions.has(taskId) && this.activeWorkflowStepSessions.has(taskId); } isTaskActive(taskId: string): boolean { return ( this.executing.has(taskId) || this.activeSessions.has(taskId) || this.recoveringCompleted.has(taskId) || TaskExecutor.processWideGraphRouting.has(taskId) ); } /* FNXC:PlannerOversight 2026-07-21-22:56: Overseer retry_step must not hard-cancel a live agent (FN-8471 thrash: status=failed from a raced graph park while step-execute still held a session, then overseer moveTask→todo aborted the live work three times). True when any in-process graph claim, coding/step/CLI session, or unpause-resume handoff still owns the task — broader than isTaskActive so step/workflow/CLI surfaces are covered. */ isTaskLiveForOverseerRetry(taskId: string): boolean { // isTaskActive covers executing/graphRouting/coding session/recoveringCompleted; // hasLiveTaskSessionSurface adds step/workflow/CLI surfaces; resumingUnpaused is the unpause handoff gap. return ( this.isTaskActive(taskId) || this.hasLiveTaskSessionSurface(taskId) || this.resumingUnpaused.has(taskId) ); } /** * FNXC:ExecutorBinding 2026-06-19-00:00: * FN-6736 gives self-healing a narrow escape hatch for phantom in-memory executor bindings after the liveness gate proves the owner is dead. Never use this as a general task stopper: it refuses to detach observable live session surfaces, then clears only stale bookkeeping (`executing`, resume/recovery sets, process-wide graph routing, activeWorktrees, activeSessionRegistry paths, and executingTaskLock) so the scheduler can re-dispatch the preserved worktree. * * FNXC:ExecutorBinding 2026-06-30-00:00: * `preserveWorktrees: true` is the FN-6736 self-healing path. When the caller has already committed to `moveTask(..., { preserveWorktree: true })`, unregistering the held worktree path from `activeSessionRegistry` defeats the preserve: re-dispatch then sees the path as free and re-acquires a brand-new worktree (observed on FN-7249: gentle-peach orphaned, rosy-thorn rebuilt ~20s after reclaim). The preserve variant clears only the in-memory executor/lock bookkeeping and leaves the session-registry path entry intact so the re-dispatch reattaches to the same worktree. Non-self-healing callers (leaked-slot reaper, pause-abort recovery) keep the default full-clear behavior. */ /* FNXC:NodeWorktreeIsolation 2026-07-29-06:05 (FN-6756 — one liveness predicate, PR #2531 review): READ-ONLY liveness probe, extracted so callers can ASK before they mutate. `clearPhantomExecutorBinding` both answers "is this live?" and performs a destructive release, which forced every caller into a false choice: check first and release ownership before their own fallible writes (a torn write — ownership gone, task un-repaired, nobody owning the repair), or write first and discover the refusal too late. Splitting the question from the act lets a caller gate on liveness with no side effect and release only after its writes have committed. Deliberately the SAME expression the destructive path uses, not a copy: a probe that could disagree with the guard it stands in for is worse than no probe, and independent re-derivation of "liveness" at each call site is precisely how this bug reached users three times (reclaim sweep -> leaked-slot reaper -> pause-abort). Registry paths count. A triage PLANNING session is owned by TriageProcessor and appears in NONE of the four executor-owned maps; it registers here instead. */ hasLiveSessionSurface(taskId: string): boolean { return this.activeSessions.has(taskId) || this.activeStepExecutors.has(taskId) || this.activeWorkflowStepSessions.has(taskId) || this.activeCliTaskSessions.has(taskId) || activeSessionRegistry.pathsForTask(taskId).length > 0; } clearPhantomExecutorBinding(taskId: string, options: { preserveWorktrees?: boolean } = {}): boolean { /* FNXC:NodeWorktreeIsolation 2026-07-29-02:10 (FN-6756 — planner worktrees reaped from under live planners): THE REGISTRY IS PART OF THE LIVENESS SIGNAL, not just something this method tears down. This is documented as "the last line of defense against pulling a worktree out from under a running agent" (see `reapLeakedConcurrencySlots`). It was blind to an entire class of agent. The four sets below are all TaskExecutor-owned; a triage PLANNING session is owned by `TriageProcessor` and lives in ITS OWN `activeSessions` map, so a live planner matched none of them. The consequence was not theoretical — it is FN-8600 recurring through a second door. Under plan-in-place a card is specified while it sits in `todo`/`triage`, both of which `reapLeakedConcurrencySlots` treats as reapable, and planning routinely outlives that sweep's 60s grace. Every earlier gate passes for a planner (not in the executor's `executing` set, reapable column, past grace), so this method decided alone — and returned true, releasing the slot and then UNREGISTERING the planner's own registry paths below. It destroyed the very evidence that proves the planner alive. FN-8600 fixed the self-owned-branch reclaim sweep by registering planning paths here (`triage.ts` acquireActiveSessionPath, and see the "planning" kind note in active-session-registry.ts). That fix landed at ONE surface. This is the second, which is what the AGENTS.md Surface Enumeration rule exists to prevent. Deliberately keyed on ANY registered path for the task, not on kind: the point is that a registered session surface of any kind means someone is working in that worktree. A leaked entry now blocks THIS sweep rather than a live planner losing its worktree — the strictly safer failure, and the one the "last line of defense" wording already promises. The registry is process-local and in-memory, so a leak cannot outlive the process; stale entries have their own reconciler (`reconcileStaleSelfOwned`) and the reclaim-aware `acquireActiveSessionPath`. NOT fixed by raising the grace period: a longer timeout only makes this rarer and harder to reproduce. The liveness gate is the bug. */ if (this.hasLiveSessionSurface(taskId)) { executorLog.warn(`${taskId}: refusing to clear phantom executor binding because a live session surface is still registered`); return false; } // FNXC:Workspace 2026-06-21-12:00: KTD2 — collect every worktree path the task holds (a workspace task holds N) before clearing the binding, so the registry sweep below unregisters all of them, not just one. const heldWorktreePaths = this.getActiveWorktreePaths(taskId); this.activeWorktrees.delete(taskId); this.executing.delete(taskId); this.recoveringCompleted.delete(taskId); this.resumingUnpaused.delete(taskId); this.approvalSuspended.delete(taskId); this.approvalResumeAfterUnwind.delete(taskId); TaskExecutor.processWideGraphRouting.delete(taskId); executingTaskLock.release(taskId); this.effectiveColumnAgentByTask.delete(taskId); if (options.preserveWorktrees) { executorLog.warn(`${taskId}: cleared phantom executor binding for self-healing re-dispatch (worktree session-registry entries preserved)`); return true; } const registeredPaths = new Set(activeSessionRegistry.pathsForTask(taskId)); for (const path of heldWorktreePaths) { registeredPaths.add(path); } for (const path of registeredPaths) { activeSessionRegistry.unregisterPath(path); } executorLog.warn(`${taskId}: cleared phantom executor binding for self-healing re-dispatch`); return true; } isEphemeralDeletionPending(agentId: string): boolean { return this.pendingEphemeralDeletions.has(agentId); } disposeEphemeralTimers(): void { this.pendingEphemeralDeletions.clear(); } private isBenignEphemeralDeleteRaceError(agentId: string, err: unknown): boolean { const msg = err instanceof Error ? err.message : String(err); const lower = msg.toLowerCase(); if (lower.includes("not found") || lower.includes("already deleted") || lower.includes("does not exist")) { executorLog.debug(`Skip spawned-agent cleanup for ${agentId}: already deleted by another pathway`); return true; } return false; } /** * Abort the in-flight bash subprocess (if any) on every active agent session. * * Invoked at runtime shutdown so detached subprocess trees spawned by agent * bash tools — including grandchildren like vitest workers — are killed via * pi-coding-agent's killProcessTree. Without this, when the worker is killed * those process groups are orphaned because they're detached. * * Sessions are not disposed here so any near-complete agent loop still has a * chance to wrap up during the runtime's graceful drain window. */ /** * Register a subagent session (e.g. reviewer) under its parent task ID so it * can be disposed when the parent stops. Used as the `onSessionCreated` * callback passed to `reviewStep`. */ private registerSubagentSession(taskId: string, session: AgentSession): void { let set = this.activeSubagentSessions.get(taskId); if (!set) { set = new Set(); this.activeSubagentSessions.set(taskId, set); } set.add(session); } /** * Deregister a subagent session that has finished naturally. The reviewer's * own `finally` block disposes the session — this just removes it from the * map. */ private unregisterSubagentSession(taskId: string, session: AgentSession): void { const set = this.activeSubagentSessions.get(taskId); if (!set) return; set.delete(session); if (set.size === 0) this.activeSubagentSessions.delete(taskId); } /** * Dispose all subagent sessions for a task and remove them from the map. * Called by the kill paths (move-out-of-in-progress, pause, global pause) * so subagents stop alongside the main session. */ private disposeSubagentsForTask(taskId: string, reason: string): void { const set = this.activeSubagentSessions.get(taskId); if (!set || set.size === 0) return; executorLog.log(`${taskId}: disposing ${set.size} subagent session(s) — ${reason}`); for (const session of set) { try { session.dispose(); } catch (err) { executorLog.warn(`${taskId}: failed to dispose subagent session: ${err}`); } } this.activeSubagentSessions.delete(taskId); } /* FNXC:WorkflowResolvedColumns 2026-07-31-23:59 — `isPlannerColumnFor` DELETED, and the deletion is the whole fix for its two guards. It was a private method with ZERO production callers. `tsc` reported it unused ("'isPlannerColumnFor' is declared but its value is never read"); the only things reaching it were two tests going through `executor as unknown as { isPlannerColumnFor: … }`, which is why nothing noticed. Its doc comment described the planning-evacuation branch of the `task:moved` handler — but that branch calls `isBackwardMoveOutOfPlanning` below, never this. So its two sync-resolved lane reads were counted as inert conversions in code that cannot run. Converting them would have "fixed" a guard with no behaviour behind it and produced two more sites to maintain; deleting is the honest reduction. The tests that only exercised it went with it — a test whose subject has no caller pins nothing. */ /** * Was this card pulled BACKWARD out of a planner lane — as opposed to advancing forward * out of it? * * FNXC:WorkflowLifecycleColumns 2026-07-30-16:55 (PR #2628 review, greptile P1): * THE FORWARD EXCLUSIONS MUST RESOLVE TOO, and leaving them literal made this branch WORSE * than before I touched it. With a role-aware source check and name-matched destinations, a * renamed board's ordinary FORWARD move (planning -> building) passed the source test and * matched none of the exclusions, so the evacuation fired on a card that was simply * advancing: it aborted live planning work and deleted the valid pre-execution worktree. * Before the conversion the source check failed and nothing happened; a half-conversion * turned a missed rescue into active damage. Third time this program has produced that * shape — gates converted, destinations left literal. * * Forward means the workflow's own wip, review, or complete lane. When a role is not * declared it cannot be a forward target, so it is simply not excluded. * * FNXC:WorkflowResolvedColumns 2026-07-31-23:59 (LANES COME FROM THE EMITTER — the sync resolver * is gone): * This took its lanes from `resolvePlannerLanes`, whose selection reader returns `undefined` * unconditionally under PostgreSQL, so it answered with the DEFAULT board for every task and both * its guards were INERT — counted by `check-inert-sync-lanes`, invisible to the census because * they already read as converted. * * The comment above said it had to be synchronous because the `task:moved` emitter is. That was * true and is no longer binding: the emitter now resolves the lanes ONCE, asynchronously * (`moves.ts` -> `resolveWorkflowIrForTask`), and hands them down on the payload. Reading a * parameter is as synchronous as reading `from`, so nothing is reordered and no listener resolves. * * `lanes` is REQUIRED rather than optional, deliberately. An optional parameter that the one * production caller happens to pass is the "seam with no supplier" shape this program keeps * finding — required means a future caller fails typecheck instead of silently falling back to a * default board. When the emitter itself could not resolve (`lanes` undefined on the payload), the * legacy ids answer, which is exactly what `resolvePlannerLanes` degraded to anyway. */ private isBackwardMoveOutOfPlanning(taskId: string, from: string, to: string, moveLanes: TaskMoveLanes | undefined): boolean { /* FNXC:WorkflowResolvedColumns 2026-07-31-23:59 (fallback CHANGED — adopting the better argument from the duplicate PR #3140): The payload is the real path and is preferred. The FALLBACK, for the case where the emitter could not resolve, is the SYNC resolver rather than the legacy literals. I had it the other way round. Falling back to literals reads cleaner and drops these guards off `check-inert-sync-lanes` — but it makes the NO-PAYLOAD path strictly WORSE, because `resolvePlannerLanes` is best-effort (it answers correctly under legacy SQLite, and only degrades to the default board under PostgreSQL) whereas a literal can never be right on a renamed board. Optimising the guard off a ratchet at the cost of the degraded path is scoring the number. THESE TWO GUARDS STAY COUNTED by `check-inert-sync-lanes`, which is the honest state: the sync call is still here, so the ratchet should still point at it. `executor.ts` goes 4 -> 2, from the `isPlannerColumnFor` deletion below, not from these. That took two corrections to get right, recorded because the intermediate state was wrong in a way that looked authoritative. I predicted "stays counted", the gate reported ZERO, and I wrote the under-reporting down as fact. It was a gate defect, not a property of this code: the scan registered a sync local only from a direct call initializer and did not follow one through a conditional (#3169) or through the object literal these lanes are rebuilt into (#3170). With both hops followed the gate reports 2 here — the original prediction. The shape was deliberately NOT rewritten to whatever form the scanner recognised. Payload-first with a sync fallback is correct on the merits, and a guard that pushes authors toward a worse degraded path to keep its own count tidy is a guard doing harm — so the scanner was fixed instead. */ const sync = moveLanes ? undefined : resolvePlannerLanes(this.store, taskId); const lanes = { hold: moveLanes?.hold ?? sync?.hold ?? "todo", intake: moveLanes?.intake ?? sync?.intake ?? "triage", wip: moveLanes?.wip ?? sync?.wip ?? "in-progress", review: moveLanes?.review ?? sync?.review ?? "in-review", complete: moveLanes?.complete ?? sync?.complete ?? "done", }; if (from !== lanes.hold && from !== lanes.intake) return false; const forwardTargets = [lanes.wip, lanes.review, lanes.complete].filter( (column): column is string => typeof column === "string", ); /* DELIBERATELY NOT ALSO EXCLUDING planner-to-planner moves. The literal version fired the evacuation on `todo -> triage` (a replan rebound), and whether that is right is a separate question from this review fix — the replan path is engine-initiated, so aborting the planning session there may be exactly wrong, but changing it is a behavior change with its own surfaces to enumerate. This conversion keeps that case behaving as it does today. */ return !forwardTargets.includes(to); } /** * FN-5256: register an in-flight disposal so a subsequent dispatch (task:moved * → in-progress) can await it before acquiring/creating a worktree. Swallows * errors so a failed disposal doesn't poison the map; surfaces them via the * executor log instead. */ private trackTaskDisposal(taskId: string, disposal: Promise): void { const wrapped = disposal .catch((err) => { executorLog.warn(`${taskId}: tracked disposal failed: ${err}`); }) .finally(() => { if (this.pendingTaskDisposals.get(taskId) === wrapped) { this.pendingTaskDisposals.delete(taskId); } }); this.pendingTaskDisposals.set(taskId, wrapped); } /** * FN-5256: synchronously await session disposal so callers (e.g. pause-before-park) * can rely on the worktree-bound shells being reaped before they return. Mirrors * `abortInFlightTaskWork`, but awaits the async `abort()` / `terminateAllSessions()` * calls instead of fire-and-forget. */ async awaitAbortInFlightTaskWork(taskId: string, reason: string, options: { userCanceled?: boolean } = {}): Promise { let hadActiveSurface = false; const abortedSurfaces: string[] = []; if (options.userCanceled) { this.userCanceledTaskIds.add(taskId); } /* FNXC:WorkflowLifecycle 2026-07-26-11:20: KB-PROV: Stamp the provenance the caller actually reported instead of a blanket `hard-cancel`. `options.userCanceled` is already the truthful operator-intent signal every caller computes (`source === "user"`, soft-delete, the registered move disposer), so derive the label from it: operator withdrawal keeps `hard-cancel`, everything else is an `engine-abort`. Without this, the FN-8596 engine rerun bounce told the operator `provenance=hard-cancel` for work the engine itself re-dispatched, and any future consumer branching on `hard-cancel` would read an engine bounce as an operator withdrawal. Behaviour is unchanged: `userPaused` is still never set by engine rebounds, and the downstream classifiers accept both labels via `isGenericAbortProvenance()`. */ this.markPausedAborted(taskId, options.userCanceled ? "hard-cancel" : "engine-abort", `abort-in-flight:${reason}`); this.options.stuckTaskDetector?.untrackTask(taskId); this.clearWorkflowRerunWatchdog(taskId); this.clearCompletedTaskWatchdog(taskId); // Defensive graph-interpreter cleanup: a pause/abort mid-graph must not leave a // stale routing claim behind. The graph runner's own finally blocks also clear // this; double-delete is harmless. // FNXC:WorkflowExecution 2026-07-19-01:30: U5d — there is no completion-interceptor // entry to clear anymore. The graph-owned signal is now a call-scoped callback // parameter (see GraphCompletionCallback), so it cannot outlive the run that created // it and needs no abort-time cleanup. TaskExecutor.processWideGraphRouting.delete(taskId); // FN-5256: claim each surface synchronously BEFORE awaiting any async // abort. Without this, two concurrent disposal calls for the same task // (e.g., task:moved-away followed immediately by task:deleted) both pass // the `has(taskId)` guards and double-call abort/dispose. const claimedSession = this.activeSessions.get(taskId); if (claimedSession) { hadActiveSurface = true; abortedSurfaces.push("agent-session"); this.deleteActiveSession(taskId); } const claimedStepExecutor = this.activeStepExecutors.get(taskId); if (claimedStepExecutor) { hadActiveSurface = true; abortedSurfaces.push("step-session"); this.deleteActiveStepExecutor(taskId); } const claimedWorkflowSession = this.activeWorkflowStepSessions.get(taskId); if (claimedWorkflowSession) { hadActiveSurface = true; abortedSurfaces.push("workflow-step-session"); this.deleteActiveWorkflowStepSession(taskId); } const claimedConfiguredCommands = this.activeConfiguredCommandControllers.get(taskId); if (claimedConfiguredCommands && claimedConfiguredCommands.size > 0) { hadActiveSurface = true; abortedSurfaces.push(`configured-command:${claimedConfiguredCommands.size}`); this.activeConfiguredCommandControllers.delete(taskId); for (const controller of claimedConfiguredCommands) { controller.abort(); } } const claimedWorkflowGraphController = this.activeWorkflowGraphAbortControllers.get(taskId); if (claimedWorkflowGraphController) { hadActiveSurface = true; abortedSurfaces.push("workflow-graph"); this.activeWorkflowGraphAbortControllers.delete(taskId); claimedWorkflowGraphController.abort(); } const claimedSubagents = this.activeSubagentSessions.has(taskId); if (claimedSubagents) { hadActiveSurface = true; abortedSurfaces.push("subagent-session"); this.disposeSubagentsForTask(taskId, reason); } // CLI Agent Executor (U7): a cli-agent session is a hard-cancel surface like // any API session. Claim it synchronously, then SIGKILL the PTY and mark // `killed` (never resume-eligible) — the same dispose/abort contract API // sessions honor. moveTask(in-progress→todo) routes here (AGENTS.md hard // cancel), so this is what guarantees the PTY tree is reaped on column exit. const claimedCliSession = this.activeCliTaskSessions.get(taskId); if (claimedCliSession) { hadActiveSurface = true; abortedSurfaces.push("cli-agent-session"); this.activeCliTaskSessions.delete(taskId); } if (claimedSession) { const { session } = claimedSession; const sessionWithAbort = session as AgentSession & { abort?: () => Promise }; if (typeof sessionWithAbort.abort === "function") { await sessionWithAbort.abort().catch((err) => { executorLog.warn(`Failed to abort agent session for ${taskId}: ${err}`); }); } try { session.dispose(); } catch (err) { executorLog.warn(`Failed to dispose agent session for ${taskId}: ${err}`); } } if (claimedStepExecutor) { const stepExecutorWithAbort = claimedStepExecutor as StepSessionExecutor & { abortAllSessionBash?: () => void }; if (typeof stepExecutorWithAbort.abortAllSessionBash === "function") { try { stepExecutorWithAbort.abortAllSessionBash(); } catch (err) { executorLog.warn(`Failed to abort step-session bash for ${taskId}: ${err}`); } } await claimedStepExecutor.terminateAllSessions().catch((err) => executorLog.error(`Failed to terminate step sessions for ${taskId}:`, err), ); } if (claimedWorkflowSession) { const sessionWithAbort = claimedWorkflowSession as AgentSession & { abort?: () => Promise }; if (typeof sessionWithAbort.abort === "function") { await sessionWithAbort.abort().catch((err) => { executorLog.warn(`Failed to abort workflow step session for ${taskId}: ${err}`); }); } try { claimedWorkflowSession.dispose(); } catch (err) { executorLog.warn(`Failed to dispose workflow step session for ${taskId}: ${err}`); } } if (claimedCliSession) { await claimedCliSession.kill("killed").catch((err) => { executorLog.warn(`Failed to kill CLI agent session for ${taskId}: ${err}`); }); } this.loopRecoveryState.delete(taskId); this.stuckAborted.delete(taskId); if (hadActiveSurface) { executorLog.log(`${taskId}: awaited abort of in-flight work — ${reason}`); this.safeLogEntry( taskId, `Pause abort cleanup completed: reason=${reason}; surfaces=${abortedSurfaces.join(", ") || "none"}`, ); } } async abortAllInFlight(reason: string): Promise { const taskIds = new Set([ ...this.activeSessions.keys(), ...this.activeStepExecutors.keys(), ...this.activeWorkflowStepSessions.keys(), ...this.activeConfiguredCommandControllers.keys(), ...this.activeWorkflowGraphAbortControllers.keys(), ...this.activeSubagentSessions.keys(), ...this.activeCliTaskSessions.keys(), ]); for (const taskId of taskIds) { try { await this.awaitAbortInFlightTaskWork(taskId, reason); } catch (err) { executorLog.warn(`abortAllInFlight: failed to abort task ${taskId} — ${reason}: ${err}`); } } for (const [agentId, session] of this.childSessions) { try { const sessionWithAbort = session as AgentSession & { abort?: () => Promise }; if (typeof sessionWithAbort.abort === "function") { await sessionWithAbort.abort(); } } catch (err) { executorLog.warn(`abortAllInFlight: failed to abort child session ${agentId} — ${reason}: ${err}`); } try { session.dispose(); } catch (err) { executorLog.warn(`abortAllInFlight: failed to dispose child session ${agentId} — ${reason}: ${err}`); } } this.childSessions.clear(); executorLog.log(`abortAllInFlight: aborted ${taskIds.size} task surface(s) — ${reason}`); } abortAllSessionBash(): void { for (const [taskId, { session }] of this.activeSessions) { try { session.abortBash(); } catch (err) { executorLog.warn(`abortAllSessionBash: failed for task ${taskId}: ${err}`); } } for (const [agentId, session] of this.childSessions) { try { session.abortBash(); } catch (err) { executorLog.warn(`abortAllSessionBash: failed for child agent ${agentId}: ${err}`); } } for (const [taskId, stepExecutor] of this.activeStepExecutors) { try { stepExecutor.abortAllSessionBash(); } catch (err) { executorLog.warn(`abortAllSessionBash: failed for step executor ${taskId}: ${err}`); } } } /** * @param store — Task store instance (also used to listen for events) * @param rootDir — Project root directory * @param options — Executor configuration * * Listens for `task:moved` to auto-execute tasks moved to `in-progress`, * `task:updated` to terminate agent sessions when individual tasks are paused, * and `settings:updated` to terminate **all** active agent sessions when * `globalPause` transitions from `false` to `true`. `enginePaused` only * prevents new work dispatch — running sessions continue to completion. * Paused tasks are moved back to `todo` rather than marked as `failed`. */ private async parkApprovalSuspension(taskId: string, surface: string): Promise { if (!this.approvalSuspended.has(taskId)) return false; this.clearPausedAborted(taskId); await this.store.logEntry( taskId, `Execution suspended for approval — ${surface} disposed; task remains in progress for decision resume`, undefined, this.getRunContextFor(taskId), ); executorLog.log(`${taskId}: approval suspension parked after ${surface} disposal`); return true; } private async dispatchUnpauseResume(task: Task): Promise { /* FNXC:ExecutorResume 2026-07-14-15:31: A terminal failed in-progress task must not be resurrected by an unrelated `task:updated` event. Planner oversight steering comments emit that event; treating it as an unpause cleared the failure and restarted the same missing-credential execution every 45 seconds. Explicit Retry/Unpause routes clear `status` before emitting their update, while startup orphan recovery has its own bounded path, so keep failed rows parked here for operator action. FNXC:ExecutorResume 2026-07-21-22:56: Claim resumingUnpaused BEFORE any await so concurrent task:updated handlers cannot both pass the gate, both await getExecutionPauseLabel, and both log "Resuming execution after unpause" (FN-8471 multi-resume race). Also treat process-wide graphRouting as already-owned work. */ if (task.status === "failed") { return false; } if ( this.executing.has(task.id) || this.resumingUnpaused.has(task.id) || this.recoveringCompleted.has(task.id) || this.activeSessions.has(task.id) || this.activeStepExecutors.has(task.id) || this.activeWorkflowStepSessions.has(task.id) || this.graphRouting.has(task.id) ) { return false; } // Synchronous single-flight claim before any await (TOCTOU fix). this.resumingUnpaused.add(task.id); let handoffOwnsClaim = false; try { const pauseLabel = await this.getExecutionPauseLabel(); if (pauseLabel) { executorLog.debug(`Skipping unpause resume for ${task.id} — ${pauseLabel} active`); return false; } // Re-check after await: a concurrent graph claim may have won meanwhile. if ( this.executing.has(task.id) || this.recoveringCompleted.has(task.id) || this.activeSessions.has(task.id) || this.activeStepExecutors.has(task.id) || this.activeWorkflowStepSessions.has(task.id) || this.graphRouting.has(task.id) ) { return false; } this.approvalSuspended.delete(task.id); if (this.isTaskWorkComplete(task) && !task.mergeDetails) { /* FNXC:ExecutorResume 2026-07-21-23:06: recoverCompletedTask refuses when resumingUnpaused still holds the id. Transfer ownership: clear the unpause claim before the recovery path runs, then own the flight via recoveringCompleted (FN-8471 early-claim fix). */ this.resumingUnpaused.delete(task.id); this.recoveringCompleted.add(task.id); handoffOwnsClaim = true; // prevent finally from double-deleting a already-cleared claim executorLog.log(`${task.id} unpaused with completed work and no session — recovering directly to in-review`); void this.recoverCompletedTask(task) .catch((err) => executorLog.error(`Failed to recover completed unpaused task ${task.id}:`, err)) .finally(() => this.recoveringCompleted.delete(task.id)); return true; } executorLog.log(`Unpaused ${task.id} in-progress with no session — resuming execution`); try { await this.clearResumeFailureState(task); await this.store.updateTask(task.id, { resumeLimboCount: 0, resumeLimboTipSha: null, resumeLimboStepSignature: null, }); await this.store.logEntry(task.id, "Resuming execution after unpause", undefined, this.getRunContextFor(task.id)); await this.recoverApprovedStepsOnResume(task.id); } catch (clearErr) { executorLog.warn(`${task.id} clearResumeFailureState failed during unpause: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`); } handoffOwnsClaim = true; this.execute(task) .catch((err) => executorLog.error(`Failed to resume unpaused ${task.id}:`, err)) .finally(() => this.resumingUnpaused.delete(task.id)); // execute().finally owns resumingUnpaused release from here. return true; } finally { if (!handoffOwnsClaim) { this.resumingUnpaused.delete(task.id); } } } private async resumeApprovalAfterUnwindIfNeeded(taskId: string): Promise { /* FNXC:ApprovalResume 2026-07-12-18:35: MAIN-008 review: this runs from execute()'s outer finally. A getTask throw (hard-deleted task between deferral and consume) must not escape finally and mask the original execute outcome — treat unreadable tasks as no deferred resume. */ if (!this.approvalResumeAfterUnwind.delete(taskId)) return false; let latestTask; try { latestTask = await this.store.getTask(taskId); } catch (error) { executorLog.warn(`${taskId}: failed to read latest task state for deferred approval resume: ${error instanceof Error ? error.message : String(error)}`); return false; } if (latestTask.paused || latestTask.userPaused || latestTask.column !== (await this.resolveResumeLanes(taskId)).wip) return false; return this.dispatchUnpauseResume(latestTask); } private async resolveMcpServers(agentId?: string | null) { /* * FNXC:McpConfig 2026-06-25-22:20: * Executor-owned lanes (main execution, retry, workflow model nodes, self-fix, and spawned child sessions) resolve the same trusted MCP server set from the task store immediately before session creation so secret material is never persisted in task state. * * FNXC:McpConfig 2026-07-12-17:02: * Secret-resolution failures remain content-free and observable. The * resolver excludes each affected server so it cannot connect with missing * credentials, while healthy MCP servers and task execution continue. */ const resolved = await resolveMcpServersForStore(this.store, { agentId: agentId ?? undefined }); if (resolved.errors.length > 0) { const serverNames = [...new Set(resolved.errors.map((error) => error.serverName))].sort(); executorLog.warn(`MCP executor resolution failed: servers=${serverNames.join(",")} count=${serverNames.length} reason=secret-materialization`); } return resolved.servers; } /** * Tasks whose graph run already owns a top-level concurrency slot (scheduler pre-held handoff). * Seam re-entry under that graph must not acquire a second slot. */ private outerConcurrencyClaims = new Set(); /* FNXC:GlobalConcurrencyControls 2026-07-14-18:30: Prefer a scheduler pre-held global slot when present so the hold/release tryAcquire and the executor share one top-level claim. Without this handoff the executor would acquire a second slot (or leave a gap if the pre-held slot were dropped) and live running counts could drift above the global cap again. While this outer claim is active, seam/step sessions must not acquire again — a second top-level acquire under a full global cap deadlocks (parent holds the last slot, child waits forever). */ private async runWithExecutorSemaphore(taskId: string, work: () => Promise): Promise { const sem = this.options.semaphore; if (!sem) { takePreHeldExecutorSlot(taskId); return work(); } if (this.outerConcurrencyClaims.has(taskId)) { return work(); } const runUnderOuterClaim = async (): Promise => { this.outerConcurrencyClaims.add(taskId); try { return await work(); } finally { this.outerConcurrencyClaims.delete(taskId); } }; if (takePreHeldExecutorSlot(taskId)) { try { return await runUnderOuterClaim(); } finally { sem.release(); } } return sem.run(runUnderOuterClaim, PRIORITY_EXECUTE); } /** * FNXC:PlannerOversight 2026-07-13-23:05: * Wire session-advisor live log flush after ProjectEngine starts (options are * captured at TaskExecutor construction time; this setter updates the callback). */ setOnExecutorLogFlushed(cb: TaskExecutorOptions["onExecutorLogFlushed"]): void { this.options = { ...this.options, onExecutorLogFlushed: cb }; } constructor( private store: TaskStore, private rootDir: string, private options: TaskExecutorOptions = {}, ) { /* FNXC:EngineDiagnostics 2026-07-26-09:39: Executor bookkeeping that fires on every dispatch/session (construct, execute() entry, worktree ready, session create/register, prompt start, graph event stream, column-boundary warns-as-info, model/plugin setup, skip/duplicate/no-op guards) is debug-only (FUSION_DEBUG=executor). Keep log/warn/error for lifecycle outcomes operators act on: Starting task, ✓/✗ completion, failures, requeues, handoffs, stuck kills, verification failures, real moves. */ executorLog.debug(`TaskExecutor constructed (rootDir=${rootDir}, hasSemaphore=${!!options.semaphore}, hasStuckDetector=${!!options.stuckTaskDetector})`); this.unregisterTaskMoveDisposer = registerTaskMoveDisposer(store, async (task) => { // Start both paths without awaiting between them. Each synchronously // detaches its current targets before its first await, fencing late // cleanup from a replacement execution after the move timeout expires. const children = this.terminateAllChildren(task.id); const activeWork = this.awaitAbortInFlightTaskWork(task.id, "user moved task from in-progress to todo", { userCanceled: true, }); await Promise.all([children, activeWork]); }); /* FNXC:WorkflowLifecycle 2026-07-16-10:00: Executor replaces the baseline only for its own TaskStore, so archive awaits abort/sweep/removal before branch deletion without cross-store coupling. */ this.unregisterArchiveWorktreeDisposer = registerArchiveWorktreeDisposer(store, async (task) => { if (!task.worktree || await canonicalizeWorktreePath(task.worktree) === await canonicalizeWorktreePath(this.rootDir)) return; await this.awaitAbortInFlightTaskWork(task.id, "task archived"); for (const path of activeSessionRegistry.pathsForTask(task.id)) activeSessionRegistry.unregisterPath(path); await this.removeOwnWorktreeWithReconcile({worktreePath: task.worktree, settings: await store.getSettings(), taskId: task.id, reason: RemovalReason.ExecutorDispose}); task.worktree = undefined; }); this.unregisterArchiveWorkspaceWorktreeDisposer = registerArchiveWorkspaceWorktreeDisposer(store, async (task, plan) => { const removed: string[] = []; const failed: {repoRel: string; error: unknown}[] = []; await this.awaitAbortInFlightTaskWork(task.id, "workspace task archived"); for (const entry of plan) { try { if (await canonicalizeWorktreePath(entry.worktreePath) === await canonicalizeWorktreePath(entry.repoRootDir)) throw new Error("Refusing to remove workspace repository root"); activeSessionRegistry.unregisterPath(entry.worktreePath); await removeWorktree({worktreePath: entry.worktreePath, rootDir: entry.repoRootDir, settings: await store.getSettings(), taskId: task.id, reason: RemovalReason.ExecutorDispose, force: true}); /* FNXC:WorkflowLifecycle 2026-07-16-16:00: Archive metadata can contain valid Git refs with shell metacharacters. Pass the ref as an argv value so cleanup never evaluates it as shell code. */ await execFileAsync("git", ["branch", "-D", entry.branch], {cwd: entry.repoRootDir, timeout: 120_000, maxBuffer: 10 * 1024 * 1024}); if (task.workspaceWorktrees) for (const repoRel of [entry.repoRel, ...entry.aliasRepoRels]) delete task.workspaceWorktrees[repoRel]; removed.push(entry.repoRel); } catch (error) { failed.push({repoRel: entry.repoRel, error}); } } return {removed, failed}; }); /* FNXC:WorkflowResolvedColumns 2026-07-31-23:20 (was FLAGGED AND LEFT COUNTED; RESOLVED below — still do NOT convert with `resolveTaskWorkflowIrSync` / `resolvePlannerLanes`): Four lifecycle literals live in this listener and they are genuinely wrong on a renamed board: execution never starts on a move INTO the board's own wip lane, terminal session release never runs on a move into its archive lane, and the two `from` guards never fire, so in-flight work is not aborted when a card leaves implementation. Nothing errors; the engine simply stops reacting. THE OBVIOUS FIX IS INERT, AND THAT IS NOW PROVED RATHER THAN ARGUED. `task:moved` is emitted synchronously, so an await here reorders this handler against every other subscriber — which points at the sync IR path. That path cannot answer for a renamed board, for TWO independent reasons (`sync-workflow-ir-second-blocker.test.ts`): 1. `getTaskWorkflowSelectionImpl` returns `undefined` unconditionally under PostgreSQL, so `resolveTaskWorkflowIrSync` always takes its `!workflowId` branch; 2. even with a selection, the CUSTOM-workflow branch loads its IR through `store.db`, whose implementation is an unconditional throw — so it falls into the catch and returns the DEFAULT IR anyway. A renamed lane IS a custom workflow, so (2) alone is decisive: the sync path can never serve this listener's case. `check-inert-sync-lane-conversions` already baselines twenty guards in exactly that state in `scheduler.ts`; these four must not join them. They stay literal and COUNTED, which is the honest state — an unconverted literal is visible to the census, while an inert conversion leaves the backlog and takes the evidence with it. THE CRITERION IS NARROWER THAN "THE LISTENER IS SYNC", and I got this wrong first time elsewhere: what blocks a guard is whether ITS ANSWER IS CONSUMED SYNCHRONOUSLY, not whether it happens to sit inside a synchronous function. In `self-healing.ts`'s fan-out, three of four guards only gated work the listener already `void`s, so they were reachable by the async resolver all along and are now converted. These four are NOT that case, for two independent reasons: A. `trackTaskDisposal` writes `pendingTaskDisposals` in THIS tick, and the `to === wip` branch above READS that map to serialise a fast bounce (in-progress -> todo -> in-progress; the FN-5256 note it carries). Deferring the branch selection to a microtask lets the second event's prologue read the map before the first event's write lands — which reopens exactly the race that comment exists to close. B. This is an if / else-if CHAIN, so the guards are entangled: converting one changes which branch a move falls into. They convert together or not at all, and (A) blocks the set. UNBLOCKING therefore needs the async resolver reachable from a SYNCHRONOUS consumer, which means either a sync reader that answers for custom workflows AND survives a writer on another node, or restructuring the disposal bookkeeping so nothing is read in-tick — the constraints are written up in `sync-workflow-ir-second-blocker.test.ts`. FNXC:WorkflowResolvedColumns 2026-07-31-23:55 — RESOLVED BY A THIRD ROUTE, and the analysis above is kept because it is what rules the other two out. The block reduces to "no resolver can be CALLED here". It never required that the answer be unavailable — only that this listener cannot go and fetch it. So the lanes are resolved ONCE by the emitter, which is already async, and ride along on the event payload (`moves.ts`). Every objection above is about calling a resolver in-tick, so none of them survive the move: - (2)/the PostgreSQL sync-IR dead end: no sync resolver is used, so neither blocker applies. - (A) the in-tick `pendingTaskDisposals` race: NO await is introduced. Destructuring one more field is as synchronous as reading `to`, so branch selection still happens in this tick and the FN-5256 fast-bounce serialisation is untouched. - (B) the entangled if / else-if chain: satisfied rather than dodged — all four convert in this one commit, so no move can fall into a different branch than before. THE RESIDUAL RISK MOVES TO THE EMITTER, AND IT IS NOT YET CLOSED — stated plainly because the tempting version of this note is the false one. `lanes` is OPTIONAL on the payload (`store.ts`: `lanes?: TaskMoveLanes`) and the fallback below is the LEGACY LITERAL, so a `task:moved` published without it leaves these four guards exactly as inert as before, on a renamed board, with nothing failing. The conversion is only as good as the emitters. That is a strictly better position than the flagged state — the fallback is reached on one path instead of every path, and `moves.ts` (the move path these branches actually serve) does pass lanes — but it is NOT the compile-time guarantee it would be if the field were required. Requiring it is the right end state and is deliberately NOT done here: it retypes every `task:moved` emitter, which is its own change with its own blast radius, and bundling it would put a mechanical retype in the same commit as this behavior change. FOLLOW-UP, tracked with the emitter-side work: either make `lanes` required, or add a gate that asserts every `task:moved` emit site supplies it. Until one of those lands, treat the fallback as a live inertness path rather than defensive dead code. */ store.on("task:moved", ({ task, from, to, source, lanes }) => { executorLog.log(`[event:task:moved] ${task.id}: ${from} → ${to}`); /* FNXC:WorkflowResolvedColumns 2026-07-31-21:30 (fleet): Lanes come from the EMITTER (see `moves.ts`), not from a resolver called here. This listener is synchronous and its branches start execution, dispose worktrees and release sessions, so its prologue is load-bearing — an await ahead of those branches would defer the `execute()` dispatch itself. The sync IR resolver is not an option either: it answers with the DEFAULT workflow under PostgreSQL, so a guard written through it is inert. Fail-soft to the legacy ids when the emit path could not resolve, matching every other consumer of this payload. `wipLane`/`archivedLane`/`holdLane` are read as SINGLE ids rather than sets because each branch below is a lane-identity test on one column, which is what the literals were. */ const wipLane = lanes?.wip ?? "in-progress"; const archivedLane = lanes?.archived ?? "archived"; const holdLane = lanes?.hold ?? "todo"; if (to === wipLane) { this.userCanceledTaskIds.delete(task.id); if (this.recoveringCompleted.has(task.id)) { executorLog.debug(`[event:task:moved] Skipping execute() for ${task.id} — completed-task recovery in progress`); return; } this.clearWorkflowRerunWatchdog(task.id); executorLog.log(`[event:task:moved] Initiating execute() for ${task.id}`); void (async () => { // FN-5256: if the prior session is still being torn down (because the // task was just moved away from in-progress), wait for the worktree- // bound shells to reap before we acquire/create a new worktree. Without // this, a fast bounce (in-progress → todo → in-progress) races the // executor's own conflict cleanup against a still-live shell. const pending = this.pendingTaskDisposals.get(task.id); if (pending) { executorLog.log(`[event:task:moved] Awaiting pending disposal for ${task.id} before dispatch`); await pending; } const taskForExecution = await this.resetMergeStateIfNeeded(task, from); await this.execute(taskForExecution); })().catch((err) => executorLog.error(`Failed to start ${task.id}:`, err), ); } else if (to === archivedLane) { /* FNXC:WorkflowLifecycle 2026-07-09-00:05: Archived is terminal, so it must release every active-session registry entry the task holds. Plan Review / other workflow-step and step-session sessions run while the task is in triage/planning/todo (not in-progress), so the old `from === "in-progress"`-only disposal branch below never fired for them — the registry entry (activeSessions / activeStepExecutors / activeWorkflowStepSessions, keyed on the shared project browse root) leaked past archive and blocked a successor task from acquiring the same session path with ActiveSessionPathHeldByForeignTaskError (FN-7717 / NEXT-508 -> NEXT-433). We deliberately do NOT do this for to === "done" / "in-review": those columns legitimately hold ai-merge / workspace-repo-land merge leases that must survive the transition (FN-6736 / Phase C/D merge-lease guarantees). This branch is checked BEFORE `from === "in-progress"` (and handles it too — a task can be archived directly from in-progress via fn_task_archive, a single `task:moved` event with no intermediate todo hop). Ordering the plain `from === "in-progress"`-only branch first would let that direct in-progress → archived transition fall into the narrower branch and skip the leaked-entry sweep below, re-opening the exact class of leak this fix closes for that one origin column. `awaitAbortInFlightTaskWork` here is the same call the in-progress branch makes (superset of its cleanup), so no case regresses. */ this.trackTaskDisposal( task.id, this.awaitAbortInFlightTaskWork(task.id, "task archived").then(() => { // Belt-and-suspenders sweep: clear any registry entry that survived the // abort above because its in-memory session map was already empty // (a leaked entry with no live session to abort). for (const path of activeSessionRegistry.pathsForTask(task.id)) { activeSessionRegistry.unregisterPath(path); } }), ); } else if (this.isBackwardMoveOutOfPlanning(task.id, from, to, lanes)) { /* FNXC:PlanningEvacuation 2026-07-25-23:00: A card pulled BACKWARD out of a planner lane (the reported case: todo → Ideas) must stop all engine work on it, not just its planning session. Plan Review and other pre-execution graph nodes run while the card sits in todo/triage, so without this branch the reviewer kept streaming against a card the operator had withdrawn. Forward transitions are excluded — those are the card advancing, and their own lanes own the handoff. Also release the pre-execution worktree acquired at planning time so a withdrawn card leaves nothing behind on disk. */ this.trackTaskDisposal( task.id, this.awaitAbortInFlightTaskWork(task.id, `task moved out of planning to ${to}`, { userCanceled: source === "user", }).then(async () => { await this.releasePreExecutionWorktree(task.id, `moved to ${to}`); }), ); } else if (from === wipLane) { if (this.workflowLifecycleMovesInFlight.has(task.id) && this.graphRouting.has(task.id)) { executorLog.log( `[event:task:moved] Preserving graph run for ${task.id} across its own ${from} → ${to} boundary`, ); return; } this.trackTaskDisposal( task.id, this.awaitAbortInFlightTaskWork(task.id, `parent moved from in-progress to ${to}`, { userCanceled: source === "user" && to === holdLane, }), ); } }); store.on("task:deleted", (task) => { this.approvalSuspended.delete(task.id); this.approvalResumeAfterUnwind.delete(task.id); this.trackTaskDisposal( task.id, this.awaitAbortInFlightTaskWork(task.id, "task soft-deleted", { userCanceled: true }), ); }); // When a task is paused while executing, terminate the agent session. // When steering comments are added during execution, inject them into the running session. // // Real-time steering comment injection mechanism: // 1. When execution starts, we initialize seenSteeringIds with all existing comment IDs // 2. On each task:updated event, we check if there are new comments not in seenSteeringIds // 3. New comments are injected via session.steer() which queues them for delivery // after the current assistant turn completes (before the next LLM call) // 4. Comments are marked as seen BEFORE injection to prevent retry loops on failure // 5. Each injection is logged to the task for user visibility store.on("task:updated", async (task) => { try { // FN-5256: handle pause by synchronously reaping every active session // surface in one shot. Awaiting the abort ensures spawned shells are // disposed before any re-dispatch can race the worktree. if ( task.paused && ( this.activeSessions.has(task.id) || this.activeStepExecutors.has(task.id) || this.activeWorkflowStepSessions.has(task.id) || this.activeConfiguredCommandControllers.has(task.id) ) ) { executorLog.log(`Pausing ${task.id} — awaiting in-flight session disposal`); await this.awaitAbortInFlightTaskWork(task.id, "task paused"); return; } // Handle unpause of an in-progress task with no active session. // Approval can be decided while the old session is still unwinding; // remember that edge instead of losing the only task:updated event. /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): both checks in this listener ask "is this card still in the wip lane?"; one snapshot for the pair. With the literal neither fired on a renamed board — an unpaused card with no active session was never resumed. */ const unpauseWipLane = (await this.resolveResumeLanes(task.id)).wip; if (!task.paused && task.column === unpauseWipLane && this.approvalSuspended.has(task.id)) { if ( this.executing.has(task.id) || this.activeSessions.has(task.id) || this.activeStepExecutors.has(task.id) || this.activeWorkflowStepSessions.has(task.id) ) { this.approvalResumeAfterUnwind.add(task.id); executorLog.log(`${task.id}: approval decision received during session unwind — deferred one resume`); return; } } // Explicit unpause updates and non-failed orphan updates can resume here; // startup failed-orphan recovery is owned by resumeOrphaned(). // dispatchUnpauseResume owns the terminal-failure and duplicate guards. if ( !task.paused && task.column === unpauseWipLane && !this.activeSessions.has(task.id) && !this.activeStepExecutors.has(task.id) && !this.activeWorkflowStepSessions.has(task.id) ) { await this.dispatchUnpauseResume(task); return; } // Column-agent restart-invalidation (plan U5, R7/KTD-4). A workflow- // definition edit (re-pointing a column's agent) or an agent runtimeConfig // change mutates NOTHING the task-field diff below observes — the watcher // would never see it. KTD-4's primary mechanism is event-driven invalidation, // but no `workflow:updated`/`agent:updated` store event exists on TaskStore // today (only task:/settings: events). Per the unit's documented fallback, we // re-resolve the column-effective agent/model on each `task:updated` tick for // GRAPH-MODE active entries ONLY (those whose session adopted a column agent — // `lastEffectiveColumnAgentId != null`). This is bounded by the active session // count, and only graph runs with a real column binding pay any cost. The // weaker guarantee (vs an arbitrary-time diff) is that a stale session // restarts on the next tick, not instantly — acceptable per the Risks note. // // agent-DELETED → fall back per R8 (no restart; the running session finishes // on its current model). agent-CHANGED (different effective agent OR same // agent with a new runtimeConfig model) → hot-swap, same path as a // task.modelProvider change. if ( this.activeSessions.has(task.id) && !task.paused && (this.activeSessions.get(task.id)!.lastEffectiveColumnAgentId ?? null) !== null && this.graphSeamGoverningNodeId.has(task.id) && this.graphColumnAgentResolver.has(task.id) ) { const activeEntry = this.activeSessions.get(task.id)!; const governingNodeId = this.graphSeamGoverningNodeId.get(task.id)!; const resolveBinding = this.graphColumnAgentResolver.get(task.id)!; const binding = resolveBinding(governingNodeId); const effective = binding ? resolveEffectiveAgent({ binding, ...this.extractOwnSettings(task) }) : undefined; if (!effective || effective.source !== "column-agent") { // Binding RELEASED (PR #1432 review): a workflow edit removed the // binding, or `defer` now resolves to the task's own settings. Hand the // session back to normal resolution: hot-swap to the assigned/task // model (the same resolution the legacy block below owns), clear the // column-agent tracking, and release the reverse heartbeat guard so // isAgentEffectivelyExecuting() stops blocking the OLD agent. executorLog.log(`${task.id}: column-agent binding released — reverting session to own-settings resolution`); activeEntry.lastEffectiveColumnAgentId = null; this.effectiveColumnAgentByTask.delete(task.id); // Fire-and-forget audit (matches the deletion-fallback posture above). this.store.logEntry( task.id, "Column-agent binding released — session reverts to its own model/agent resolution", undefined, this.getRunContextFor(task.id), ).catch((err: unknown) => executorLog.warn(`${task.id}: failed to log column-agent release: ${err instanceof Error ? err.message : String(err)}`)); const settings = await this.store.getSettings(); const assignedRuntimeConfig = await this.getAssignedAgentRuntimeConfig(task.assignedAgentId); const { provider: ownProvider, modelId: ownModelId } = resolveExecutorSessionModel( task.modelProvider, task.modelId, settings, assignedRuntimeConfig, ); const providerChanged = ownProvider !== activeEntry.lastResolvedModelProvider; const modelIdChanged = ownModelId !== activeEntry.lastResolvedModelId; if ((providerChanged || modelIdChanged) && ownProvider && ownModelId) { activeEntry.lastResolvedModelProvider = ownProvider; activeEntry.lastResolvedModelId = ownModelId; try { const model = (await this.getModelRegistry()).find(ownProvider, ownModelId); if (model) { await activeEntry.session.setModel(model); executorLog.log(`${task.id}: binding released — model reverted to ${ownProvider}/${ownModelId}`); } } catch (err: unknown) { executorLog.error(`${task.id}: failed to revert model after binding release: ${err instanceof Error ? err.message : String(err)}`); } } } else { { // Fetch the (possibly changed) effective column agent, best-effort. const newAgent = await this.options.agentStore?.getAgent(effective.agentId).catch(() => null) ?? null; if (!newAgent) { // agent-DELETED (R8): fall back, NO restart. The running session // keeps its current model; the NEXT resolution falls back. Update the // tracked id so we stop probing for the missing agent every tick. if (activeEntry.lastEffectiveColumnAgentId !== null) { executorLog.log(`${task.id}: column agent '${effective.agentId}' deleted mid-session — falling back, no restart (R8)`); // Fire-and-forget audit (matches the rework-log posture at ~3582): // a logEntry failure must not abort this task:updated tick and skip // the model-change detection below. this.store.logEntry( task.id, `Column agent '${effective.agentId}' deleted mid-session — falling back to current model, no restart (R8)`, undefined, this.getRunContextFor(task.id), ).catch((err: unknown) => executorLog.warn(`${task.id}: failed to log column-agent deletion fallback: ${err instanceof Error ? err.message : String(err)}`)); activeEntry.lastEffectiveColumnAgentId = null; // Release the reverse heartbeat guard for the deleted agent // (PR #1432 review): isAgentEffectivelyExecuting() must not keep // blocking an agent that no longer governs this session. this.effectiveColumnAgentByTask.delete(task.id); } } else { const settings = await this.store.getSettings(); /* FNXC:ColumnAgentModel 2026-06-27-10:05: Override column agents own the active session model even when a mid-flight task edit adds its own modelProvider/modelId; ignore task-level model fields during column-agent re-resolution so the watcher cannot clobber the governing agent's runtime model. */ const overrideColumnGoverns = binding!.mode === "override"; const { provider: newProvider, modelId: newModelId } = resolveExecutorSessionModel( overrideColumnGoverns ? undefined : task.modelProvider, overrideColumnGoverns ? undefined : task.modelId, settings, (newAgent.runtimeConfig ?? undefined) as Record | undefined, ); const agentChanged = (activeEntry.lastEffectiveColumnAgentId ?? null) !== newAgent.id; const providerChanged = newProvider !== activeEntry.lastResolvedModelProvider; const modelIdChanged = newModelId !== activeEntry.lastResolvedModelId; if (agentChanged || providerChanged || modelIdChanged) { activeEntry.lastEffectiveColumnAgentId = newAgent.id; // Re-key the reverse heartbeat guard to the NEW agent (PR #1432 // review): the old agent stops being blocked, the new one starts. this.effectiveColumnAgentByTask.set(task.id, newAgent.id); activeEntry.lastResolvedModelProvider = newProvider; activeEntry.lastResolvedModelId = newModelId; if (newProvider && newModelId) { try { const model = (await this.getModelRegistry()).find(newProvider, newModelId); if (model) { await activeEntry.session.setModel(model); executorLog.log(`${task.id}: column-agent hot-swap → agent '${newAgent.id}' model ${newProvider}/${newModelId}`); await this.store.logEntry(task.id, `Column agent changed — model now ${newProvider}/${newModelId} (agent ${newAgent.id})`, undefined, this.getRunContextFor(task.id)); } else { executorLog.log(`${task.id}: column-agent model ${newProvider}/${newModelId} not found in registry for hot-swap`); } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`${task.id}: failed to column-agent hot-swap: ${errorMessage}`); // Fire-and-forget audit (see ~3582): a logEntry failure here must // not abort the tick and skip later model-change detection. this.store.logEntry(task.id, `Column-agent change failed: ${errorMessage}`, undefined, this.getRunContextFor(task.id)) .catch((logErr: unknown) => executorLog.warn(`${task.id}: failed to log column-agent change failure: ${logErr instanceof Error ? logErr.message : String(logErr)}`)); } } } } } } } // Handle executor model hot-swap on active single-session executions if (this.activeSessions.has(task.id) && !task.paused) { const activeEntry = this.activeSessions.get(task.id)!; // R3 guard: when an OVERRIDE column agent governs this running session, the // column-agent watcher block above OWNS the model (override supersedes the // task's own model/assigned-agent settings). The legacy task-model hot-swap // would otherwise resolve a model from task.assignedAgentId's runtimeConfig // and clobber the column agent's model on a mid-flight task edit. Skip it // entirely when override governs; defer-resolved-to-own-settings (or no // binding) keeps the legacy behavior identical. let overrideColumnGoverns = false; if ((activeEntry.lastEffectiveColumnAgentId ?? null) !== null) { const governingNodeId = this.graphSeamGoverningNodeId.get(task.id); const resolveBinding = this.graphColumnAgentResolver.get(task.id); if (governingNodeId && resolveBinding) { const binding = resolveBinding(governingNodeId); if (binding?.mode === "override") overrideColumnGoverns = true; } } const taskModelProviderChanged = task.modelProvider !== activeEntry.lastTaskModelProvider; const taskModelIdChanged = task.modelId !== activeEntry.lastTaskModelId; const assignedAgentChanged = (task.assignedAgentId ?? null) !== (activeEntry.lastAssignedAgentId ?? null); if (!overrideColumnGoverns && (taskModelProviderChanged || taskModelIdChanged || assignedAgentChanged)) { activeEntry.lastTaskModelProvider = task.modelProvider; activeEntry.lastTaskModelId = task.modelId; activeEntry.lastAssignedAgentId = task.assignedAgentId ?? null; const settings = await this.store.getSettings(); const assignedRuntimeConfig = await this.getAssignedAgentRuntimeConfig(task.assignedAgentId); const { provider: newProvider, modelId: newModelId } = resolveExecutorSessionModel( task.modelProvider, task.modelId, settings, assignedRuntimeConfig, ); const providerChanged = newProvider !== activeEntry.lastResolvedModelProvider; const modelIdChanged = newModelId !== activeEntry.lastResolvedModelId; if (!providerChanged && !modelIdChanged) { return; } activeEntry.lastResolvedModelProvider = newProvider; activeEntry.lastResolvedModelId = newModelId; if (newProvider && newModelId) { try { const model = (await this.getModelRegistry()).find(newProvider, newModelId); if (model) { await activeEntry.session.setModel(model); executorLog.log(`${task.id}: executor model hot-swapped to ${newProvider}/${newModelId}`); await this.store.logEntry(task.id, `Model changed to ${newProvider}/${newModelId}`, undefined, this.getRunContextFor(task.id)); } else { executorLog.log(`${task.id}: model ${newProvider}/${newModelId} not found in registry for hot-swap`); } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`${task.id}: failed to hot-swap model: ${errorMessage}`); await this.store.logEntry(task.id, `Model change failed: ${errorMessage}`, undefined, this.getRunContextFor(task.id)); } } } } // Handle steering comments - inject new ones into whichever execution // surface currently owns the task: legacy single-session, step-session // executor (including graph-pinned/workflow stepwise runs), or an // individual workflow step AgentSession. if (task.steeringComments) { const injectionTargets: Array<{ kind: "legacy" | "step-session" | "workflow-step"; seenSteeringIds: Set; inject: (message: string, comment: import("@fusion/core").SteeringComment) => Promise<"injected" | "queued">; legacySession?: AgentSession; legacyState?: ActiveExecutorSessionState; }> = []; const activeSession = this.activeSessions.get(task.id); if (activeSession) { injectionTargets.push({ kind: "legacy", seenSteeringIds: activeSession.seenSteeringIds, inject: async (message) => { await activeSession.session.steer(message); return "injected"; }, legacySession: activeSession.session, legacyState: activeSession, }); } const stepExecutor = this.activeStepExecutors.get(task.id); if (stepExecutor) { /* FNXC:TaskDetailChat 2026-06-17-13:24: Task-detail chat comments must reach the running LLM thread immediately across legacy, step-session, and workflow-step surfaces. Step-session runs can be between per-step AgentSessions when a comment arrives, so keep the executor's task snapshot current and treat zero-session fan-out as a next-prompt fallback while preserving seenSteeringIds exactly-once delivery. */ stepExecutor.updateSteeringComments?.(task.steeringComments); const seenSteeringIds = this.activeStepExecutorSeenSteeringIds.get(task.id) ?? this.createSeenSteeringIds(task); this.activeStepExecutorSeenSteeringIds.set(task.id, seenSteeringIds); injectionTargets.push({ kind: "step-session", seenSteeringIds, inject: async (message, comment) => { const steeredSessionCount = await stepExecutor.steerActiveSessions(message); if (steeredSessionCount > 0) { stepExecutor.markSteeringCommentsDelivered?.([comment.id]); return "injected"; } return "queued"; }, }); } const workflowSession = this.activeWorkflowStepSessions.get(task.id); if (workflowSession) { const seenSteeringIds = this.activeWorkflowStepSessionSeenSteeringIds.get(task.id) ?? this.createSeenSteeringIds(task); this.activeWorkflowStepSessionSeenSteeringIds.set(task.id, seenSteeringIds); injectionTargets.push({ kind: "workflow-step", seenSteeringIds, inject: async (message) => { await workflowSession.steer(message); return "injected"; }, }); } const loggedCommentIds = new Set(); let legacyReviewHandoff: { comments: import("@fusion/core").SteeringComment[]; session: AgentSession; state: ActiveExecutorSessionState; } | undefined; for (const target of injectionTargets) { // Find new steering comments that haven't been seen by this running surface yet. const newComments = task.steeringComments.filter(c => !target.seenSteeringIds.has(c.id)); if (newComments.length === 0) continue; for (const comment of newComments) { const summary = comment.text.length > 80 ? comment.text.slice(0, 80) + "..." : comment.text; // Mark as seen BEFORE attempting injection to prevent retry loops on failure. target.seenSteeringIds.add(comment.id); const commentMessage = formatCommentForInjection(comment); try { executorLog.log(`Injecting comment into ${task.id} (${target.kind}): ${summary}`); const delivery = await target.inject(commentMessage, comment); if (delivery === "queued") { executorLog.log(`Queued comment for next ${target.kind} prompt in ${task.id}`); } else { executorLog.log(`Successfully injected comment into ${task.id} (${target.kind})`); } // Log to the task once per comment/tick even if multiple active surfaces exist. if (!loggedCommentIds.has(comment.id)) { await this.store.logEntry( task.id, `Comment received mid-execution: ${summary}`, `by ${comment.author}` ); loggedCommentIds.add(comment.id); } } catch (err) { executorLog.error(`Failed to inject comment for ${task.id} (${target.kind}):`, err); // Comment is already marked as seen - we won't retry to avoid spamming // the agent with failed injections. The error is logged for debugging. } } if (target.kind === "legacy" && target.legacySession && target.legacyState) { legacyReviewHandoff = { comments: newComments, session: target.legacySession, state: target.legacyState, }; } } // After injecting comments, check for review handoff intent on the legacy // session path. Step-session/workflow-step runs do not have the legacy // review handoff state required by executeReviewHandoff. if (legacyReviewHandoff) { // Only detect handoff in agent-authored comments when policy is enabled. // Merge per-task effective workflow settings (U3, KTD-3) so // reviewHandoffPolicy resolves from the workflow. Behavior-inert by default. const settings = await mergeEffectiveSettings(this.store, task, await this.store.getSettings()); if (settings.reviewHandoffPolicy === "comment-triggered") { const agentComments = legacyReviewHandoff.comments.filter(c => c.author !== "user"); for (const comment of agentComments) { if (detectReviewHandoffIntent(comment.text)) { executorLog.log(`Review handoff detected in ${task.id}: ${comment.text.slice(0, 50)}...`); await this.executeReviewHandoff(task, legacyReviewHandoff.session, legacyReviewHandoff.state); return; // Exit early - handoff handles session disposal } } } } } } catch (err) { executorLog.error("Uncaught error in task:updated listener:", err); } }); // When globalPause transitions from false → true, terminate all active agent sessions. store.on("settings:updated", ({ settings, previous }) => { if (settings.globalPause && !previous.globalPause) { for (const [taskId, controllers] of this.activeConfiguredCommandControllers) { executorLog.log(`Global pause — aborting configured command(s) for ${taskId}`); this.markPausedAborted(taskId, "global-pause", "global-pause:configured-command"); this.options.stuckTaskDetector?.untrackTask(taskId); for (const controller of controllers) { controller.abort(); } this.activeConfiguredCommandControllers.delete(taskId); this.loopRecoveryState.delete(taskId); this.spawnedAgents.delete(taskId); this.stuckAborted.delete(taskId); } // Dispose every reviewer subagent across every task. The per-task loops // below handle main + step sessions; reviewers live in their own map // and would otherwise outlive the global pause. for (const taskId of [...this.activeSubagentSessions.keys()]) { this.disposeSubagentsForTask(taskId, "global pause"); } for (const [taskId, { session }] of this.activeSessions) { executorLog.log(`Global pause — terminating agent session for ${taskId}`); this.markPausedAborted(taskId, "global-pause", "global-pause:agent-session"); this.options.stuckTaskDetector?.untrackTask(taskId); // abort() interrupts any in-flight LLM stream / tool call; // dispose() then releases session resources. const sessionWithAbort = session as unknown as { abort?: () => Promise }; if (typeof sessionWithAbort.abort === "function") { void sessionWithAbort.abort().catch((err) => { executorLog.warn(`Failed to abort agent session for ${taskId}: ${err}`); }); } session.dispose(); // Clean up all in-memory state so nothing leaks when tasks are later unpaused this.loopRecoveryState.delete(taskId); this.spawnedAgents.delete(taskId); this.stuckAborted.delete(taskId); } for (const [taskId, stepExecutor] of this.activeStepExecutors) { executorLog.log(`Global pause — terminating step sessions for ${taskId}`); this.markPausedAborted(taskId, "global-pause", "global-pause:step-session"); this.options.stuckTaskDetector?.untrackTask(taskId); stepExecutor.terminateAllSessions().catch(err => executorLog.warn(`Failed to terminate step sessions for global pause ${taskId}: ${err}`) ); // Clean up all in-memory state so nothing leaks when tasks are later unpaused this.loopRecoveryState.delete(taskId); this.spawnedAgents.delete(taskId); this.stuckAborted.delete(taskId); } for (const [taskId, workflowSession] of this.activeWorkflowStepSessions) { executorLog.log(`Global pause — terminating workflow step session for ${taskId}`); this.markPausedAborted(taskId, "global-pause", "global-pause:workflow-step-session"); this.options.stuckTaskDetector?.untrackTask(taskId); const sessionWithAbort = workflowSession as AgentSession & { abort?: () => Promise }; if (typeof sessionWithAbort.abort === "function") { void sessionWithAbort.abort().catch((err) => { executorLog.warn(`Failed to abort workflow step session for ${taskId}: ${err}`); }); } workflowSession.dispose(); this.deleteActiveWorkflowStepSession(taskId); this.loopRecoveryState.delete(taskId); this.spawnedAgents.delete(taskId); this.stuckAborted.delete(taskId); } for (const [taskId, controller] of this.activeWorkflowGraphAbortControllers) { executorLog.log(`Global pause — aborting workflow graph runner for ${taskId}`); this.markPausedAborted(taskId, "global-pause", "global-pause:workflow-graph"); this.options.stuckTaskDetector?.untrackTask(taskId); controller.abort(); this.activeWorkflowGraphAbortControllers.delete(taskId); this.loopRecoveryState.delete(taskId); this.spawnedAgents.delete(taskId); this.stuckAborted.delete(taskId); } } }); } /** * Check whether a task's work is complete — all steps are done or skipped. * Used to detect tasks that called fn_task_done() but never transitioned to in-review * (e.g., killed by stuck detector after fn_task_done but before moveTask). */ private isTaskWorkComplete(task: Task): boolean { if (task.steps.length === 0) return false; return task.steps.every((s) => s.status === "done" || s.status === "skipped"); } private async resetMergeStateIfNeeded(task: Task, from: Task["column"]): Promise { /* FNXC:WorkflowResolvedColumns 2026-07-30-16:40 (executor): Merge state is reset when a card leaves a lane where a merge could have been recorded — the REVIEW and COMPLETE roles, not the two ids. On a renamed board neither comparison matched, so a card re-entering execution carried STALE mergeDetails from its previous pass. `review` is not a trait: the role is carried by mergeOrchestration/mergeBlocker/humanReview, the same five-flag set the dependency gates in this file use. Unioned with the legacy pair because `resolveWorkflowIrForTask` degrades to the BUILT-IN IR rather than throwing. */ const mergeBearingColumns = new Set(["in-review", "done"]); try { const ir = await resolveWorkflowIrForTask(this.store, task.id); if (ir) { for (const flag of ["complete", "mergeOrchestration", "mergeBlocker", "humanReview"] as const) { for (const id of columnsWithFlag(ir, flag)) mergeBearingColumns.add(id); } } } catch { /* degraded: legacy pair only */ } if (!mergeBearingColumns.has(from)) { return task; } const hasMergeEvidence = Boolean(task.mergeDetails) || (task.mergeRetries ?? 0) > 0 || (task.verificationFailureCount ?? 0) > 0 || task.status === "merging" || task.status === "merging-pr" || task.status === "merging-fix"; if (!hasMergeEvidence) { return task; } return this.cleanupMergeStateForReverification( task, `Task returned to in-progress from ${from} column — resetting verification steps and merge state for re-verification`, { // Keep deterministic merge-verification bounce budget across remediation // cycles. Status may be cleared by intermediate paths, so the counter is // the canonical signal once a bounce has started. preserveVerificationFailureCount: (task.verificationFailureCount ?? 0) > 0, }, ); } private async cleanupMergeStateForReverification( task: Task, logMessage: string, options?: { preserveVerificationFailureCount?: boolean }, ): Promise { const preservedWorkflowStepResults = preservePreExecutionWorkflowStepResults(task); await this.store.updateTask(task.id, { mergeDetails: null, mergeRetries: 0, status: null, error: null, verificationFailureCount: options?.preserveVerificationFailureCount ? task.verificationFailureCount ?? 0 : 0, workflowStepResults: preservedWorkflowStepResults, }); const refreshedTask = await this.store.getTask(task.id); const steps = refreshedTask.steps ?? []; if (steps.length > 0) { const allStepsComplete = this.isTaskWorkComplete(refreshedTask); if (allStepsComplete) { await this.reopenLastStepForRevision(task.id, refreshedTask); } else { const resetIndexes = new Set(); for (let i = 0; i < steps.length; i++) { const name = steps[i].name.toLowerCase(); if (/testing|verification/.test(name) || /documentation|delivery/.test(name)) { resetIndexes.add(i); } } if (resetIndexes.size === 0) { const reopened = await this.reopenLastStepForRevision(task.id, refreshedTask); if (reopened) { resetIndexes.add(reopened.index); } } else { for (const index of resetIndexes) { if (steps[index].status !== "pending") { await this.store.updateStep(task.id, index, "pending"); } } const earliestIndex = Math.min(...Array.from(resetIndexes)); await this.store.updateTask(task.id, { currentStep: earliestIndex }); } } } await this.store.logEntry(task.id, logMessage, undefined, this.getRunContextFor(task.id)); return this.store.getTask(task.id); } private isNoProgressNoTaskDoneFailure(task: Task): boolean { return task.status === "failed" && task.error?.includes("without calling fn_task_done") === true && task.steps.every((step) => step.status === "pending"); } private async clearResumeFailureState(task: Task): Promise { const updates: { status?: null; error?: null; blockedBy?: null } = {}; if (task.status === "failed" || task.error) { updates.status = null; updates.error = null; } // Pre-dispatch gating state must not survive into a resumed in-progress run. // The scheduler sets status="queued" + blockedBy on dep/file-scope conflicts // (scheduler.ts:618, 660) and clears them on the todo→in-progress transition // (scheduler.ts:696). Resume paths (unpause, drift recovery, engine restart) // bypass that clear, so a task can end up actively executing while still // labeled "queued" in the UI. if (task.status === "queued") { updates.status = null; } if (task.blockedBy) { updates.blockedBy = null; } if (Object.keys(updates).length > 0) { await this.store.updateTask(task.id, updates); } } private clearCompletedTaskWatchdog(taskId: string): void { const handle = this.completedTaskWatchdogs.get(taskId); if (!handle) return; clearTimeout(handle); this.completedTaskWatchdogs.delete(taskId); } /** * FNXC:AgentReflection 2026-07-04-00:00: * FN-7528: single seam for every `onComplete` call site. Fires the deterministic, non-LLM * post-task performance capture (best-effort, fire-and-forget — a capture failure must never * block or fail task completion) before forwarding to the configured `onComplete` callback. * Capture is completion-gated: only runs once per taskId (see `capturedReflectionTaskIds`), * guarded by `reflectionService` presence, `settings.reflectionEnabled`, and an assigned agent id * mirroring the existing in-session reflection-tool guard. */ private signalTaskComplete(task: Task): void { this.triggerPostTaskReflectionCapture(task); this.options.onComplete?.(task); } private triggerPostTaskReflectionCapture(task: Task): void { const reflectionService = this.options.reflectionService; if (!reflectionService) return; const assignedAgentId = task.assignedAgentId?.trim(); if (!assignedAgentId) return; if (this.capturedReflectionTaskIds.has(task.id)) return; this.capturedReflectionTaskIds.add(task.id); void (async () => { try { const settings = await this.store.getSettings(); if (!settings.reflectionEnabled) return; await reflectionService.captureTaskPerformance(assignedAgentId, task.id); } catch (error) { executorLog.warn( `${task.id}: post-task performance capture failed (best-effort, non-blocking): ${error instanceof Error ? error.message : String(error)}`, ); } })(); } private clearWorkflowRerunWatchdog(taskId: string): void { const handle = this.workflowRerunWatchdogs.get(taskId); if (!handle) return; clearTimeout(handle); this.workflowRerunWatchdogs.delete(taskId); } private scheduleCompletedTaskWatchdog(taskId: string, trigger: string): void { this.clearCompletedTaskWatchdog(taskId); const handle = setTimeout(async () => { this.completedTaskWatchdogs.delete(taskId); // Claim recovery slot atomically (synchronously) before any async work. // Without this, two paths can pass the in-flight guards on the same // event-loop turn and both call recoverCompletedTask() concurrently. if ( this.recoveringCompleted.has(taskId) || this.executing.has(taskId) || this.activeSessions.has(taskId) || this.activeStepExecutors.has(taskId) || this.activeWorkflowStepSessions.has(taskId) || this.resumingUnpaused.has(taskId) ) { return; } this.recoveringCompleted.add(taskId); try { const pauseLabel = await this.getExecutionPauseLabel(); if (pauseLabel) { return; } let currentTask: Task | null = null; try { currentTask = await this.store.getTask(taskId); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: completed-task watchdog could not read latest task state: ${errorMessage}`); return; } if (!currentTask || currentTask.paused || currentTask.column !== (await this.resolveResumeLanes(taskId)).wip) { return; } if (!this.isTaskWorkComplete(currentTask)) { return; } executorLog.warn( `${taskId}: completed-task watchdog fired after ${COMPLETED_TASK_WATCHDOG_MS / 1000}s ` + `(${trigger}) — attempting direct recovery to in-review`, ); await this.store.logEntry( taskId, `Watchdog: task remained in-progress ${COMPLETED_TASK_WATCHDOG_MS / 1000}s after ${trigger} — attempting direct recovery to in-review`, ).catch(() => undefined); const recovered = await this.recoverCompletedTask(currentTask); if (!recovered) { await this.store.logEntry( taskId, "Watchdog recovery attempt could not finalize completed task — leaving for follow-up recovery", ).catch(() => undefined); } } finally { this.recoveringCompleted.delete(taskId); } }, COMPLETED_TASK_WATCHDOG_MS); this.completedTaskWatchdogs.set(taskId, handle); } /** * Result of a workflow-rerun bounce attempt. * * - `bounced` — the move sequence completed successfully and the task is * back in `in-progress` ready for re-execution. * - `skipped-pending` — another bounce for the same task is mid-flight; * this attempt is a no-op. Callers (notably the watchdog) must NOT log * this as a successful retry, since the original bounce may itself be * stuck. */ /* FNXC:ReviewLeniency 2026-07-02-02:10: Clear prior terminal failure results (failed/advisory_failure — incl. optional gate nodes like code-review) so a retry starts clean. Call this ONLY once the task has left the mergeable in-review column (i.e. it is in `todo`): clearing while still in-review drops the merge blocker during the rerun-bounce window and could let a concurrent auto-merge sweep merge an empty-`steps` graph-native task with its gate failure unaddressed. `moveTask(in-review→todo)` already clears ALL results (applyReopenFieldClears), so this is chiefly for the in-progress→todo bounce path where the move does not. Passed/skipped/pending evidence is kept. */ private async clearTerminalStepFailuresForRetry(taskId: string): Promise { const live = await this.store.getTask(taskId).catch(() => null); if (!live) return; const cleared = clearTerminalWorkflowStepFailures(live.workflowStepResults); if (cleared !== live.workflowStepResults) { await this.store.updateTask(taskId, { workflowStepResults: cleared }, this.getRunContextFor(taskId)); } } private async performWorkflowRerunBounce( taskId: string, worktreePath: string, preserveResumeState: boolean = true, ): Promise<"bounced" | "skipped-pending" | "deferred-paused"> { const pauseLabel = await this.getExecutionPauseLabel(); if (pauseLabel) { executorLog.log(`${taskId}: workflow rerun deferred — ${pauseLabel} active`); return "deferred-paused"; } // Re-entry guard: if a previous bounce for the same task is still // mid-flight (e.g., the watchdog fired before the original sequence // completed), skip rather than racing two concurrent moveTask sequences. if (this.workflowRerunPending.has(taskId)) { executorLog.warn(`${taskId}: workflow rerun bounce already in flight — skipping re-entry`); return "skipped-pending"; } this.workflowRerunPending.add(taskId); try { // moveTask(in-progress → todo) clears `task.worktree`; restore it before // the return trip so the dashboard never renders the task under // "Unassigned" and self-healing can't reclaim the worktree as idle. const latestTask = await this.store.getTask(taskId); if (!latestTask) { throw new Error("task missing during workflow rerun bounce"); } if (latestTask.paused) { executorLog.log(`${taskId}: workflow rerun deferred — task is paused`); return "deferred-paused"; } /* FNXC:WorkflowOptionalStepFix 2026-06-27-13:30: A pre-merge optional step REVISE (Code Review / Browser Verification) schedules this bounce via sendTaskBackForFix AFTER reopening the last plan step to `pending`. The graph run that hosted that step reports `disposition: "completed"`, so the outer completion flow can route the task to `in-review` BEFORE this setTimeout(0) bounce runs. Previously the bounce only handled `in-progress`/`todo` and THREW on `in-review` ("cannot bounce to in-progress"), leaving the task stranded in-review with a `pending` step: the merge gate blocks forever on the incomplete step while self-healing only re-runs the workflow graph (re-passing the advisory step) and never re-launches the executor to finish the reopened step — a permanent deadlock (observed on FN-7122). The bounce's ONLY caller is sendTaskBackForFix, which unconditionally intends to send the task back for remediation, so `in-review` must bounce back exactly like `in-progress` regardless of the column the completion race left it in. */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): both lanes from ONE snapshot — the comment above says in-review must bounce EXACTLY like in-progress, so resolving them separately is how the bounce ends up handling one lane and throwing on the other, which is the bug that comment is about. */ const bounceLanes = await this.resolveResumeLanes(taskId); if (latestTask.column === bounceLanes.wip || latestTask.column === bounceLanes.review) { const originalExecutionStartedAt = latestTask.executionStartedAt; // Preserve step progress across the in-progress/in-review → todo hop: // moveTask's default reopen-to-todo path resets every step to // pending and rewrites PROMPT.md checkboxes, which would discard // the partial progress this bounce is supposed to retry on top of. // `preserveWorktree` keeps the same checkout assigned across the // hop so listeners never observe an interim `worktree=null` state // — this bounce immediately re-promotes the task on the same // directory, so releasing it would publish a misleading snapshot // and could let self-healing reclaim the worktree as idle. if (preserveResumeState) { await this.store.moveTask(taskId, await resolveReboundColumnFor(this.store, taskId), { preserveResumeState: true, preserveWorktree: true, }); } else { await this.store.moveTask(taskId, await resolveReboundColumnFor(this.store, taskId), { preserveWorktree: true }); } // Restore worktree + executionStartedAt unconditionally to match // the original bounce contract: even with preserveWorktree the // worktree pointer could have been cleared by an in-flight // updateTask, and executionStartedAt is reset by moveTask when // preserveResumeState is false. Keep the writes so callers and // tests can observe the restoration deterministically. await this.store.updateTask(taskId, { worktree: worktreePath, executionStartedAt: originalExecutionStartedAt ?? null, }); const pauseLabelAfterTodo = await this.getExecutionPauseLabel(); if (pauseLabelAfterTodo) { executorLog.log(`${taskId}: workflow rerun parked in todo — ${pauseLabelAfterTodo} became active during bounce`); return "deferred-paused"; } // Now in `todo` (non-mergeable) — safe to clear prior gate failures. await this.clearTerminalStepFailuresForRetry(taskId); /* FNXC:WorkflowResolvedColumns 2026-07-30-21:40: census-invisible moveTask DESTINATION — a call argument, not a comparison. The SOURCE guard four lines up already resolves via resolveReboundColumnFor; leaving the destination literal is a split brain inside one function. */ await this.store.moveTask(taskId, await resolveWipTargetForTask(this.store, taskId)); return "bounced"; } if (latestTask.column === await resolveReboundColumnFor(this.store, taskId)) { await this.store.updateTask(taskId, { worktree: worktreePath }); const pauseLabelBeforeResume = await this.getExecutionPauseLabel(); if (pauseLabelBeforeResume) { executorLog.log(`${taskId}: workflow rerun parked in todo — ${pauseLabelBeforeResume} became active before resume`); return "deferred-paused"; } // Already in `todo` (non-mergeable) — safe to clear prior gate failures. await this.clearTerminalStepFailuresForRetry(taskId); /* FNXC:WorkflowResolvedColumns 2026-07-30-21:40: census-invisible moveTask DESTINATION — a call argument, not a comparison. The SOURCE guard four lines up already resolves via resolveReboundColumnFor; leaving the destination literal is a split brain inside one function. */ await this.store.moveTask(taskId, await resolveWipTargetForTask(this.store, taskId)); return "bounced"; } throw new Error(`task is in '${latestTask.column}', cannot bounce to in-progress`); } finally { this.workflowRerunPending.delete(taskId); } } private scheduleWorkflowRerun( taskId: string, worktreePath: string, successMessage: string, preserveResumeState: boolean = true, ): void { this.clearWorkflowRerunWatchdog(taskId); setTimeout(async () => { try { const outcome = await this.performWorkflowRerunBounce(taskId, worktreePath, preserveResumeState); if (outcome === "bounced") { executorLog.log(successMessage); } else if (outcome === "skipped-pending") { executorLog.warn(`${taskId}: rerun bounce skipped — another bounce already in flight`); } else { executorLog.log(`${taskId}: rerun bounce deferred while pause is active`); } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`${taskId}: failed to schedule rerun bounce: ${errorMessage}`); } }, 0); const watchdog = setTimeout(async () => { this.workflowRerunWatchdogs.delete(taskId); const pauseLabel = await this.getExecutionPauseLabel(); if (pauseLabel) { executorLog.log(`${taskId}: workflow rerun watchdog skipped — ${pauseLabel} active`); return; } let currentTask: Task | null = null; try { currentTask = await this.store.getTask(taskId); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: workflow rerun watchdog could not read latest task state: ${errorMessage}`); return; } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): the INVERSE of the guard above — this one SKIPS a card that is still executing. Note the direction: with the literal on a renamed board it never matched, so a rerun could fire on a card mid-execution. A mechanical sweep of every `!== "in-progress"` would fix the refusals and leave this admission in place. */ if (!currentTask || currentTask.paused || currentTask.column === (await this.resolveResumeLanes(taskId)).wip) { return; } executorLog.warn( `${taskId}: workflow rerun watchdog fired after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s ` + `— task is still ${currentTask.column}; retrying handoff once`, ); await this.store.logEntry( taskId, `Watchdog: workflow rerun handoff stalled for ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s ` + `(still ${currentTask.column}) — retrying once`, ).catch(() => undefined); try { const outcome = await this.performWorkflowRerunBounce(taskId, worktreePath, preserveResumeState); if (outcome === "bounced") { executorLog.warn(`${taskId}: workflow rerun watchdog retry succeeded`); } else if (outcome === "skipped-pending") { // The original bounce is still mid-flight, which means *it* is the // one that's hung — not us. Log honestly so operators don't see a // false "succeeded" message while the task is actually stranded. executorLog.error( `${taskId}: workflow rerun watchdog retry skipped — original bounce still in flight after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s; task may be stuck`, ); await this.store.logEntry( taskId, `Workflow rerun watchdog retry skipped — original bounce still in flight after ${WORKFLOW_RERUN_WATCHDOG_MS / 1000}s; task may be stuck`, ).catch(() => undefined); } else { executorLog.log(`${taskId}: workflow rerun watchdog retry deferred while pause is active`); } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`${taskId}: workflow rerun watchdog retry failed: ${errorMessage}`); } }, WORKFLOW_RERUN_WATCHDOG_MS); this.workflowRerunWatchdogs.set(taskId, watchdog); } private async parkCompletedBlockedTask(task: Task, completionBlocker: string, source: string, workComplete = this.isTaskWorkComplete(task)): Promise { if (task.paused === true || task.userPaused === true) return false; /* FNXC:WorkflowLifecycleColumns 2026-07-29-13:10: Was the raw literal pair `column === "done" || column === "archived"`. On a renamed board neither matched, so this "already finished, nothing to park" guard was INERT and a completed card resting in the workflow's own terminal column fell through — and the `column !== "todo"` branch below would then have MOVED it back out of that terminal column. Resolved through core's shared `resolveTerminalColumns`, which owns the per-role fallback (a partially-declared workflow keeps the legacy id for the half it did not declare). */ const terminalColumns = await resolveTerminalColumnsFor(this.store, task.id); /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2568 review — greptile): RE-READ AFTER THE AWAIT. The pause and column guards above ran against the `task` snapshot the caller passed, and this conversion introduced the first `await` between those guards and the writes below. Another dispatch or an operator action can move or pause the card while the IR resolution is in flight, and the stale snapshot would then let this method move a now-terminal task out of its terminal column, or overwrite a pause an operator just applied. Re-reading is cheap next to the resolution that precedes it, and it is the pause check that matters most: a user pause landing during the await is precisely the case where proceeding is least forgivable. Falling back to the passed snapshot on a read failure keeps this no worse than before the await existed. */ const liveTask = await this.store.getTask(task.id).catch(() => undefined) ?? task; if (liveTask.paused === true || liveTask.userPaused === true) return false; if (terminalColumns.includes(liveTask.column)) return false; if (!workComplete) return false; const message = `Completed work held — ${completionBlocker}; will advance to review when blocker clears`; /* FNXC:WorkflowLifecycle 2026-07-12-23:13: FN-7926: completed work with a persistent `getTaskCompletionBlocker` result must not self-requeue through the execute node. Re-running implementation cannot clear dependency/blockedBy state, so it only feeds FN-7863's generic no-progress backstop and misclassifies good work as `EXECUTION_DISPATCH_LOOP_EXHAUSTED`. Park in a scheduler-skipped todo state, preserve worktree/branch/steps, and reset the FN-7863 signature so the backstop remains reserved for genuinely incomplete no-progress loops. */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (rebase merge, both sides kept): main (#2644) resolved the literal `todo` into `reboundColumn`; this branch added the post-await `liveTask` re-read. Taking either side alone loses the other — the literal comes back, or the stale snapshot does. */ const reboundColumn = await resolveReboundColumnFor(this.store, task.id); if (liveTask.column !== reboundColumn) { await this.store.moveTask(task.id, reboundColumn, { preserveProgress: true, preserveResumeState: true, preserveWorktree: true, moveSource: "engine", recoveryRehome: true, }); } await this.store.updateTask(task.id, { paused: true, pausedReason: COMPLETED_BLOCKED_PAUSE_REASON, status: "queued", error: null, executeRequeueLoopCount: null, executeRequeueLoopSignature: null, }, this.getRunContextFor(task.id)); executorLog.log(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("completed-blocked-park", task.id), domain: "database", mutationType: "task:completed-blocked-parked", target: task.id, metadata: { taskId: task.id, blocker: completionBlocker, source, priorColumn: task.column, priorStatus: task.status ?? null, }, }); return true; } private async getCompletedTaskFinalizationDecision(taskId: string, taskDone: boolean): Promise<"finalize" | "blocked" | "incomplete"> { const task = await this.store.getTask(taskId); const completionBlocker = await this.getTaskCompletionBlocker(task); /* FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — `taskDone` means an ACCEPTED fn_task_done (explicit or a non-tainted implicit completion), which is the honest exit and always finalizes. Only the step-status-derived `isTaskWorkComplete` path can be laundered by skip-bypass, so the taint guard gates that path alone; a genuine no-op/PREMISE-STALE accepted done is never blocked. */ const workComplete = taskDone || (this.isTaskWorkComplete(task) && !evaluateSkipBypassTaint(task).blocked); if (completionBlocker) { executorLog.log(`${taskId} completion blocked — ${completionBlocker}`); if (workComplete && await this.parkCompletedBlockedTask(task, completionBlocker, "finalization", workComplete)) { return "blocked"; } return "incomplete"; } if (workComplete) return "finalize"; return "incomplete"; } private async shouldFinalizeCompletedTask(taskId: string, taskDone: boolean): Promise { return await this.getCompletedTaskFinalizationDecision(taskId, taskDone) === "finalize"; } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2703 review — greptile P1): The review lane arrives from the caller for the reason documented on `isBenignInReviewPauseAbort`: the synchronous resolver returns the default workflow in PostgreSQL mode, so resolving it here would have been a conversion that changes the census and not the behaviour. */ private isTaskAlreadyCompleteForNonContinuableSession(task: Task, taskDone: boolean, reviewLane: string): boolean { // FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — the step-status "already complete" branch // must not treat skip-bypass-tainted skips as completion; an accepted done / in-review // column are honest completion signals and stay unaffected. return taskDone || task.column === reviewLane || (this.isTaskWorkComplete(task) && !evaluateSkipBypassTaint(task).blocked); } private async handleNonContinuableSessionError(task: Task, taskDone: boolean, errorMessage: string): Promise { if (!isNonContinuableSessionError(errorMessage)) { return false; } const liveTask = await this.store.getTask(task.id); const nonContinuableLanes = await this.resolveResumeLanes(task.id); if (!liveTask || !this.isTaskAlreadyCompleteForNonContinuableSession(liveTask, taskDone, nonContinuableLanes.review)) { return false; } const diagnosticMessage = "Post-done session continuation suppressed — session not continuable (last role assistant); task work already complete, leaving clean in-review"; executorLog.warn(`${task.id} ${diagnosticMessage}`); await this.store.logEntry(task.id, diagnosticMessage, errorMessage, this.getRunContextFor(task.id)); if (liveTask.status === "failed" || liveTask.error) { await this.store.updateTask(task.id, { status: null, error: null }); } await this.persistTokenUsage(task.id); /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2703 review — greptile P1, and it is the same split I have been fixing all day, in code I wrote an hour earlier): ONE SNAPSHOT. The eligibility check above already resolved this task's lanes (`nonContinuableLanes`), and this branch resolved them AGAIN. A workflow selection or review-column edit between the two makes eligibility accept the card on the old board while this branch reads the new one — the card is then handed to `handoffTaskToReview`, reprocessing a row already in review. Writing the second resolution was not carelessness about the rule; it is that the rule is invisible at the call site. That is the argument for the structural ratchet in `executor-graph-failure-lanes-resolved.test.ts` rather than for trying harder. */ if (liveTask.column === nonContinuableLanes.review) { this.clearCompletedTaskWatchdog(task.id); this.signalTaskComplete(liveTask); return true; } const refreshedTask = await this.store.getTask(task.id); await this.handoffTaskToReview(refreshedTask ?? liveTask, "post-done-noncontinuable"); this.clearCompletedTaskWatchdog(task.id); this.signalTaskComplete(refreshedTask ?? liveTask); return true; } private async handleNonContinuableSessionRetry(task: Task, errorMessage: string): Promise { if (!isNonContinuableSessionError(errorMessage)) { return false; } const liveTask = await this.store.getTask(task.id); if (!liveTask) { return false; } const decision = computeRecoveryDecision({ recoveryRetryCount: liveTask.recoveryRetryCount, nextRecoveryAt: liveTask.nextRecoveryAt, }); if (decision.shouldRetry) { const attempt = decision.nextState.recoveryRetryCount; const delay = formatDelay(decision.delayMs); executorLog.warn(`⚡ ${task.id} non-continuable session — fresh-session retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`); await this.store.logEntry(task.id, `Non-continuable session — fresh-session retry (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { recoveryRetryCount: decision.nextState.recoveryRetryCount, nextRecoveryAt: decision.nextState.nextRecoveryAt, sessionFile: null, }); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: true }); return true; } executorLog.error(`✗ ${task.id} non-continuable session fresh-session retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`); await this.store.logEntry(task.id, `Non-continuable session fresh-session retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { recoveryRetryCount: null, nextRecoveryAt: null, }); return false; } private async getTaskCompletionBlocker(task: Task): Promise { return getTaskCompletionBlockerForStore(this.store, task); } private accumulateTokenUsage( existing: TaskTokenUsage | undefined, delta: Pick | undefined, timestamp = new Date().toISOString(), ): TaskTokenUsage | undefined { if (!delta) return existing; const merged: TaskTokenUsage = { inputTokens: (existing?.inputTokens ?? 0) + delta.inputTokens, outputTokens: (existing?.outputTokens ?? 0) + delta.outputTokens, cachedTokens: (existing?.cachedTokens ?? 0) + delta.cachedTokens, cacheWriteTokens: (existing?.cacheWriteTokens ?? 0) + delta.cacheWriteTokens, totalTokens: (existing?.totalTokens ?? 0) + delta.totalTokens, firstUsedAt: existing?.firstUsedAt ?? timestamp, lastUsedAt: timestamp, perModel: existing?.perModel, }; return merged; } private tokenUsageWithModelSnapshot( tokenUsage: TaskTokenUsage, session: AgentSession | undefined, existing: TaskTokenUsage | undefined, delta?: Pick, timestamp = tokenUsage.lastUsedAt, modelOverride?: { provider?: string; id?: string }, ): TaskTokenUsage { const model = modelOverride ?? (session as { model?: { provider?: string; id?: string } } | undefined)?.model; return { ...tokenUsage, /* * FNXC:TokenAnalytics 2026-06-18-16:23: * Persist the actually-used session model as an analytics snapshot while leaving task.modelProvider/task.modelId untouched so normal model-resolution hierarchy is not pinned by usage bookkeeping. * * FNXC:TokenAnalytics 2026-06-19-15:53: * Per-model buckets must merge only the just-produced delta. The sum of buckets stays equal to the task aggregate, while analytics grand totals and nTasks remain based on the task row rather than expanded buckets. */ modelProvider: model?.provider ?? existing?.modelProvider, modelId: model?.id ?? existing?.modelId, perModel: delta ? mergeTokenUsagePerModel(existing?.perModel, delta, model, timestamp) : tokenUsage.perModel, }; } private async extractSessionTokenUsage( session: AgentSession | undefined, ): Promise | undefined> { if (!session) return undefined; try { const statsResult = (session as AgentSession & { getSessionStats?: () => | { tokens?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number; total?: number; }; } | Promise<{ tokens?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number; total?: number; }; }>; }).getSessionStats?.(); const stats = await Promise.resolve(statsResult); const tokens = stats?.tokens; if (!tokens) return undefined; const inputTokens = tokens.input ?? 0; const outputTokens = tokens.output ?? 0; const cachedTokens = tokens.cacheRead ?? 0; const cacheWriteTokens = tokens.cacheWrite ?? 0; const totalTokens = tokens.total ?? (inputTokens + outputTokens + cachedTokens + cacheWriteTokens); return { inputTokens, outputTokens, cachedTokens, cacheWriteTokens, totalTokens, }; } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to read session stats for token usage: ${message}`); return undefined; } } /** * FNXC:TokenBudget 2026-07-16-00:00: * Step-session token usage bypasses the shared session helper, so all executor * writes use this seam to retain the required persist-time budget enforcement. */ private async persistTaskTokenUsage(taskId: string, tokenUsage: TaskTokenUsage): Promise { const runContext = this.getRunContextFor(taskId); await this.store.updateTask(taskId, { tokenUsage }, runContext); await enforceTaskTokenBudgetForPersist(this.store, taskId, runContext); } /* * FNXC:TokenAnalytics 2026-07-17-14:00: * `persistTokenUsage` is the sole writer for a central executor session. Prompt paths call this same delta seam rather than `accumulateSessionTokenUsage`, preventing independently-baselined helper and finalization writes from crediting the same cumulative tokens twice. */ private async captureExecutorTokenUsageBaseline(taskId: string, session: AgentSession): Promise { this.tokenUsageBaselines.set(taskId, (await this.extractSessionTokenUsage(session)) ?? { inputTokens: 0, outputTokens: 0, cachedTokens: 0, cacheWriteTokens: 0, totalTokens: 0, }); } private async persistTokenUsage(taskId: string, session?: AgentSession): Promise { const activeSession = session ?? this.activeSessions.get(taskId)?.session; const currentUsage = await this.extractSessionTokenUsage(activeSession); if (!currentUsage) return; const baseline = this.tokenUsageBaselines.get(taskId); this.tokenUsageBaselines.set(taskId, currentUsage); const delta = baseline ? { inputTokens: Math.max(0, currentUsage.inputTokens - baseline.inputTokens), outputTokens: Math.max(0, currentUsage.outputTokens - baseline.outputTokens), cachedTokens: Math.max(0, currentUsage.cachedTokens - baseline.cachedTokens), cacheWriteTokens: Math.max(0, currentUsage.cacheWriteTokens - baseline.cacheWriteTokens), totalTokens: Math.max(0, currentUsage.totalTokens - baseline.totalTokens), } : currentUsage; if ( delta.inputTokens === 0 && delta.outputTokens === 0 && delta.cachedTokens === 0 && delta.cacheWriteTokens === 0 && delta.totalTokens === 0 ) { return; } const task = await this.store.getTask(taskId); const merged = this.accumulateTokenUsage(task.tokenUsage, delta); if (!merged) return; const tokenUsage = this.tokenUsageWithModelSnapshot(merged, activeSession, task.tokenUsage, delta); tokenCacheMetricsLog.log(JSON.stringify({ taskId, agentId: task.assignedAgentId ?? undefined, role: "executor", inputTokens: tokenUsage.inputTokens, cachedTokens: tokenUsage.cachedTokens, cacheWriteTokens: tokenUsage.cacheWriteTokens, hitRatio: tokenUsage.inputTokens + tokenUsage.cachedTokens > 0 ? tokenUsage.cachedTokens / (tokenUsage.inputTokens + tokenUsage.cachedTokens) : 0, })); await this.persistTaskTokenUsage(taskId, tokenUsage); } /** * Execute a review handoff: move the task to in-review column with * awaiting-user-review status, assign the requesting user, and dispose * the agent session. */ private async executeReviewHandoff( task: Task, _session: AgentSession, _sessionEntry: { session: AgentSession; seenSteeringIds: Set; lastResolvedModelProvider?: string; lastResolvedModelId?: string; lastTaskModelProvider?: string | null; lastTaskModelId?: string | null; lastAssignedAgentId?: string | null }, ): Promise { try { executorLog.log(`Executing review handoff for ${task.id}`); // Log the handoff event await this.store.logEntry( task.id, "Review handoff requested by agent — moving to in-review for user review", undefined, this.getRunContextFor(task.id) ); // Update task with awaiting-user-review status and assignee // Use a single updateTask call for atomicity await this.store.updateTask( task.id, { status: "awaiting-user-review", assigneeUserId: "requesting-user", }, this.getRunContextFor(task.id) ); // Move the task to in-review column (this will also emit task:moved event) // The task:moved handler will clean up activeSessions await this.persistTokenUsage(task.id); await this.handoffTaskToReview(task, "review-handoff-requested"); // Dispose the agent session (this may already be done by task:moved handler) // but we do it here to be explicit if (this.activeSessions.has(task.id)) { const { session: activeSession } = this.activeSessions.get(task.id)!; activeSession.dispose(); this.deleteActiveSession(task.id); } // Untrack from stuck detector this.options.stuckTaskDetector?.untrackTask(task.id); executorLog.log(`Review handoff complete for ${task.id} — task moved to in-review`); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to execute review handoff for ${task.id}: ${errorMessage}`); } } /** * Fast-path a completed task directly to in-review without spawning a new agent. * Captures modified files, runs workflow steps, and transitions the task. * * @returns true if the task was successfully transitioned, false otherwise. */ async recoverCompletedTask(task: Task): Promise { try { if ( this.executing.has(task.id) || this.activeSessions.has(task.id) || this.activeStepExecutors.has(task.id) || this.activeWorkflowStepSessions.has(task.id) || this.resumingUnpaused.has(task.id) || TaskExecutor.processWideGraphRouting.has(task.id) ) { executorLog.debug(`${task.id}: skipping recoverCompletedTask — task has active execution in flight`); return false; } /* FNXC:WorkflowOptionalStepFix 2026-06-28-12:00: A pre-merge optional/advisory step REVISE (Code Review / Browser Verification) reopens plan steps to `pending` and schedules a remediation bounce (sendTaskBackForFix → scheduleWorkflowRerun) that moves the task in-review → todo → in-progress so the executor can finish the reopened steps. Re-entering the workflow graph here while that bounce is still scheduled — or while the live task already carries incomplete plan steps — preempts the executor's single fix cycle: the re-run re-passes the advisory step (its fix budget is now exhausted), advances to the `merge` node, and the merge gate refuses with "task has incomplete steps" forever (observed on FN-7210; the FN-7122 bounce fix handled the column race but not this competing graph re-entry). recoverCompletedTask only owns tasks whose work is genuinely COMPLETE, so refuse re-entry when a remediation bounce is in flight or the live task has non-terminal steps, and let the bounce / stale-incomplete-review recovery re-launch execution instead. */ if (this.workflowRerunWatchdogs.has(task.id) || this.workflowRerunPending.has(task.id)) { executorLog.debug(`${task.id}: skipping recoverCompletedTask — workflow remediation bounce already scheduled`); return false; } const liveForCompletenessCheck = await this.store.getTask(task.id).catch(() => task); if ( liveForCompletenessCheck && (liveForCompletenessCheck.steps?.length ?? 0) > 0 && !this.isTaskWorkComplete(liveForCompletenessCheck) ) { executorLog.debug(`${task.id}: skipping recoverCompletedTask — task has incomplete steps awaiting executor remediation`); return false; } /* FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — recoverCompletedTask is the shared auto-promotion chokepoint for every "work looks complete → in-review" path (unpause resume, completed-task watchdog, orphan resume). Refuse to auto-promote a skip-bypass-tainted task: its steps were skipped after a bulk-step-completion refusal with no accepted fn_task_done, so the only honest exits are an accepted fn_task_done or operator intervention (both clear the taint). Leaving it unpromoted lets the bounded requeue/park machinery converge it to a human instead of laundering it to review. */ if (liveForCompletenessCheck && evaluateSkipBypassTaint(liveForCompletenessCheck).blocked) { executorLog.warn(`${task.id}: skipping recoverCompletedTask — skip-bypass taint active (steps skipped after a bulk-step-completion refusal)`); await this.store.logEntry( task.id, "Auto-promotion withheld: steps were skipped after a bulk-step-completion refusal with no accepted fn_task_done — requires reviewer or operator sign-off", undefined, this.getRunContextFor(task.id), ).catch(() => undefined); return false; } /* FNXC:Lifecycle 2026-07-16-10:30: FN-8141 defense-in-depth: recoverCompletedTask is the shared promotion chokepoint for BOTH self-healing sweeps AND the executor's own unpause / resumeOrphaned fast-paths. A task whose most recent execution-outcome in the durable log was a failure/refusal park must not be promoted to in-review by ANY route, even one that re-derived completion from all-steps-done/ skipped (skipped counts as complete, which is exactly how FN-8141 laundered a failed task). The self-healing sweeps additionally emit the deduped no-action audit event; here we simply refuse. Escape hatch: an operator retrying the task starts a fresh execution whose clean completion marker supersedes the failure park, clearing this block with no code change. */ const failureProvenance = evaluateCompletedPromotionFailureProvenance(liveForCompletenessCheck ?? task); if (failureProvenance.blocked) { executorLog.debug(`${task.id}: skipping recoverCompletedTask — most recent execution ended in a failure/refusal park (operator-decides)`); return false; } const settings = await this.store.getSettings(); if (settings.globalPause || settings.enginePaused) { executorLog.log( `${task.id}: skipping recoverCompletedTask — ${ settings.globalPause ? "global pause" : "engine pause" } active`, ); return false; } // Capture modified files if the worktree still exists if (task.worktree && existsSync(task.worktree)) { const modifiedFiles = await this.captureModifiedFiles(task.worktree, task.baseCommitSha, task.id, undefined, "recovery"); if (modifiedFiles.length > 0) { await this.store.updateTask(task.id, { modifiedFiles }); executorLog.log(`${task.id}: recovered ${modifiedFiles.length} modified files`); } const enabledWorkflowStepsAlreadySatisfied = task.executionMode === "fast" ? areExplicitEnabledWorkflowStepsSatisfied(liveForCompletenessCheck) : areEnabledPreMergeWorkflowStepsSatisfied(liveForCompletenessCheck); const shouldReenterWorkflowGraph = task.executionMode === "fast" ? hasUnsatisfiedExplicitEnabledWorkflowSteps(liveForCompletenessCheck) : !enabledWorkflowStepsAlreadySatisfied; // Run workflow steps before transitioning — fast mode still honors explicit optional-step selections. if (enabledWorkflowStepsAlreadySatisfied) { /* FNXC:WorkflowLifecycle 2026-06-29-04:37: Completed graph-owned tasks can be observed briefly as in-progress after the main graph already recorded every enabled pre-merge gate. Recovery must not restart the graph from parse in that state; foreach pins from the completed run make parse fail with pin-mismatch. Hand off to review instead, which is the same terminal seam the completed graph reached. */ executorLog.log(`${task.id}: completed recovery found satisfied workflow gates — skipping graph re-entry`); } else if (shouldReenterWorkflowGraph) { if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow-graph re-entry during completed-task recovery")) { return false; } /* FNXC:WorkflowExecution 2026-06-25-00:00: U4 (KTD-2) watchdog re-entry. The legacy `runWorkflowSteps` recovery path was deleted; the workflow graph is the sole executor. A stranded completed task is recovered by RE-ENTERING the graph via `executeWorkflowGraph` (the same entry execute() uses), which: (1) re-runs any pending optional-group / gate nodes, (2) records their outcomes into `task.workflowStepResults` (U2) and emits the `[pre-merge]` logs, and (3) OWNS the in-review vs back-for-fix transition. The graph's execute seam registers the normal completion interceptor, so a task whose implementation already completed resumes at the post-implementation nodes (it does not re-run the agent from scratch). RECOVERY POLICY mapping (per plan U4): the old "any failure including REVISE is hard" recovery rule now maps onto the graph's gate semantics — a GATE node REVISE/failure routes the task back for fix, while an ADVISORY REVISE is non-blocking and proceeds to review. KTD-5: for a store lacking `getTaskWorkflowSelection` that has enabled steps, `executeWorkflowGraph` itself fails closed (parks) rather than letting recovery silently skip the gates. */ /* FNXC:WorkflowExecution 2026-07-19-17:55 (U10b / R9): Re-entry is unconditional. This used to branch on a `graphOwned` boolean and, when the graph "declined", fall through to the legacy in-review handoff below. The graph can no longer decline — the fallback is deleted — so that fall-through was a path where recovery could reach review having skipped the gates it re-entered to run. The handoff below is still reachable, but now only via the two branches that have legitimately decided there is nothing left to gate: gates already satisfied, or fast mode with no unsatisfied explicit selection. */ await this.executeWorkflowGraph(task); this.clearCompletedTaskWatchdog(task.id); await this.store.logEntry( task.id, `Auto-recovered: stranded completed task re-dispatched through the workflow graph — the graph re-ran pending workflow steps (recording results) and owns the in-review / back-for-fix transition`, ).catch(() => undefined); executorLog.log(`✓ ${task.id} auto-recovered completed task via workflow-graph re-entry`); return true; } else if (task.executionMode === "fast") { /* FNXC:FastOptionalSteps 2026-06-30-12:00: Fast recovery can hand off completed implementation directly only when the operator did not explicitly enable optional workflow steps, or when those enabled steps already have passed pre-merge results. Explicit optional selections are stronger than the fast default, so completed-task recovery must re-enter the workflow graph before review when any selected optional group is still unsatisfied. */ executorLog.debug(`${task.id}: fast mode — no unsatisfied explicit workflow steps on auto-recovery`); } } if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition during completed-task recovery")) { return false; } await this.persistTokenUsage(task.id); const originColumn = task.column; /* FNXC:WorkflowLifecycleColumns 2026-07-30-09:30 (Phase C convergence): Resolved from the task's OWN workflow. On a renamed board the literals matched nothing, so completed work stranded in the planning lane was NOT recognised as needing promotion: the code fell through to `handoffTaskToReview` directly from the planning column, and role adjacency has no planning -> review edge, so the handoff move was rejected and the card stayed stranded with its work finished and nothing left to rescue it. This is the recovery of last resort — a literal here means the last resort does not exist off the default lineage. */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (the sync resolver never resolved): AWAITED, because this method is async and the sync twin is a no-op in production. The note above says a literal here "means the last resort does not exist off the default lineage". `resolvePlannerLanes` was that literal wearing a trait lookup: its selection reader returns undefined unconditionally in PostgreSQL mode, so it resolved the DEFAULT workflow for every card and `promotedFromPlannerColumn` was false on every renamed board — the exact stranding this recovery exists to fix, with the conversion in place and the census counting it. Same struct, same fallbacks, one await. `recoverCompletedTask` has already awaited store reads by this point, so this adds no ordering constraint it did not already have. */ const plannerLanes = await resolvePlannerLanesForTaskAsync(this.store, task.id); const promotedFromPlannerColumn = originColumn === plannerLanes.hold || originColumn === plannerLanes.intake; /* FNXC:WorkflowLifecycleColumns 2026-07-30-16:45 (PR #2628 review, greptile P1): REFUSE BEFORE THE FIRST MOVE when the workflow declares no WIP lane. The previous version let `resolvePlannerLanes` substitute the legacy `in-progress`, so the promotion targeted a column that board does not declare: `moveTask` rejects it, recovery reports failure — and because the intake -> hold re-home happens FIRST, the card could be left half-moved, which is worse than the stranding this recovery exists to fix. Checked here rather than at the move so no partial hop is issued. A workflow with planning lanes and no WIP lane has nowhere to promote completed work TO; that is an operator configuration question, not something to guess past. Logged so the card is not silently skipped — the whole point of this recovery is that nothing else owns this state. */ if (promotedFromPlannerColumn && plannerLanes.wip === undefined) { const message = `Auto-recovery withheld: completed work is in '${originColumn}' but this workflow declares no WIP column to promote it to`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message).catch(() => undefined); return false; } let completionTask = task; if (promotedFromPlannerColumn) { this.recoveringCompleted.add(task.id); /* FNXC:WorkflowLifecycle 2026-07-20-08:42: Advanced-triage recovery reaches this shared seam with completed work, a preserved worktree, and a durable merge pin. The workflow transition map deliberately rejects triage -> in-review, so re-home through the legal triage -> todo -> in-progress path while the recovery ownership set prevents scheduler/executor dispatch. Todo callers retain their existing single hop. */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-09:30: the two-hop is needed whenever the card sits in a DISTINCT intake lane, because role adjacency gives intake only hold/archived — never wip. Post-U11 the default lineage merges the two roles onto one column, so `hold === intake` and the hop correctly collapses to the single move below; a board that still separates them (pre-U11, or a custom lineage) keeps the re-home. */ if (originColumn === plannerLanes.intake && plannerLanes.hold !== plannerLanes.intake) { completionTask = await this.store.moveTask(task.id, plannerLanes.hold, { moveSource: "engine", recoveryRehome: true, bypassGuards: true, preserveProgress: true, preserveWorktree: true, preserveResumeState: true, }); } // Non-undefined: the guard above returned early when this workflow declares no WIP lane. completionTask = await this.store.moveTask(task.id, plannerLanes.wip as string); } await this.handoffTaskToReview(completionTask, "completed-task-recovered"); if (promotedFromPlannerColumn) { this.recoveringCompleted.delete(task.id); } this.clearCompletedTaskWatchdog(task.id); await this.store.logEntry(task.id, `Auto-recovered: task work was complete but stranded in ${originColumn} — moved to in-review`); executorLog.log(`✓ ${task.id} auto-recovered completed task → in-review`); this.signalTaskComplete(task); return true; } catch (err: unknown) { this.recoveringCompleted.delete(task.id); const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to recover completed task ${task.id}: ${errorMessage}`); return false; } } /* * FNXC:WorkflowOptionalStepFix 2026-06-26-16:35: * Inline graph optional-step remediation consumes `postReviewFixCount` BEFORE calling `sendTaskBackForFix`, matching self-healing's budget-first ordering. Persistent optional-step REVISE loops are bounded by the resolved optional-group budget; `"unbounded"` intentionally skips the ceiling check so the step cycles until it returns APPROVE/APPROVE_WITH_NOTES or a human intervenes. * * FNXC:WorkflowRevisionBudget 2026-06-30-20:48: * Live Plan Review/spec and Code Review remediation must honor explicit workflow setting values before node `maxRevisions`, and must treat unset values as unbounded for those two built-in review paths. Browser Verification keeps the existing `maxPostReviewFixes` fallback unless its node config explicitly changes it. * * FNXC:WorkflowRevisionBudget 2026-06-30-22:04: * Plan Review and Code Review caps are independent policy budgets, so attempts are counted by workflow step key instead of the legacy aggregate `postReviewFixCount`. The aggregate still increments for existing dashboard summaries, but it must not let a Plan Review replan consume a Code Review remediation slot. */ /* * FNXC:PlanReviewReplanCap 2026-07-19-00:10: * U3 — the graph is the sole Plan Review owner (triage's out-of-graph gate and * its blockAfterPlanReviewRevise cap-park are deleted). Re-own the replan-cap * escalation here: when the plan-review replan budget (node `maxRevisions` / * `planReviewReplanCap` setting, or the unbounded-default hard cap) is exhausted, * park the task at `awaiting-approval` with reason `plan-review-replan-cap` so a * persistent planner/reviewer disagreement surfaces to a human instead of looping * forever or silently sitting in place. The reason string is special-cased by the * dashboard + notifications, so it must be preserved verbatim. */ private async parkPlanReviewReplanCapExhausted( taskId: string, capLabel: string, currentCount: number, feedback: string, ): Promise { await this.store.logEntry( taskId, "Plan Review replan cap reached — escalating to manual approval", `The Plan Review gate requested a planning revision ${currentCount} times without converging (cap ${capLabel}). To avoid an endless plan → Plan Review REVISE → replan loop, the task is routed to awaiting-approval for a human decision instead of replanning again. Latest Plan Review feedback:\n${feedback}`, this.getRunContextFor(taskId), ); // awaitingApprovalReason is written through a Record (matching // the manual plan-approval hold + the deleted triage cap-park) so the distinct // reason survives the update path. const escalationUpdates: Record = { status: "awaiting-approval", awaitingApprovalReason: "plan-review-replan-cap", error: null, recoveryRetryCount: null, nextRecoveryAt: null, }; await this.store.updateTask(taskId, escalationUpdates as Partial, this.getRunContextFor(taskId)); executorLog.warn( `${taskId}: Plan Review replan cap (${capLabel}) reached after ${currentCount} attempts — escalating to awaiting-approval`, ); } private async requestPreMergeOptionalStepFix( taskId: string, fallbackTask: Task, info: { stepName: string; feedback: string; phase: CoreWorkflowStepResult["phase"]; status: CoreWorkflowStepResult["status"]; verdict?: string; /** Raw graph node result when no reviewer verdict was produced. */ failureValue?: string; nodeId?: string; maxRevisions?: unknown; }, ): Promise { if (info.phase !== "pre-merge") return false; if (info.status !== "advisory_failure" && info.status !== "failed") return false; const liveTask = await this.store.getTask(taskId).catch(() => fallbackTask); const missingArtifactKeys = parseRequiredArtifactMissingValue(info.failureValue); if (missingArtifactKeys) { await this.recoverMissingRequiredArtifacts(liveTask, missingArtifactKeys, { source: "workflow-step", nodeId: info.nodeId, }); return true; } const isPlanReview = info.nodeId === "plan-review" || info.stepName === "Plan Review"; if (isPlanReview) { /* * FNXC:PlanReviewReplan 2026-07-05-17:32: * FN-7561: a malformed reviewer response arrives as `advisory_failure` with NO parsed verdict. That is an infra/formatting failure (e.g. the reviewer could not locate the spec, or fumbled its trailing JSON), not a plan defect — it must NEVER bounce the task to a triage replan. The graph already excludes malformed advisories from the fix handoff (shouldRequestPreMergeFix); this guard defends the explicit remediation-node path and any future caller so a malformed advisory can never drive the replan loop. A genuine REVISE (verdict === "REVISE", also carried as advisory_failure) still replans below. */ if (info.status === "advisory_failure" && info.verdict !== "REVISE") return false; if (info.verdict !== undefined && info.verdict !== "REVISE") return false; /* * FNXC:PlanReviewReplan 2026-07-15-12:00: * FN-7977 / issue #2124: graph traversal is the primary guard, but this * compatibility seam also receives explicit remediation edges and future * callers. A provider/model/transport failure without a genuine REVISE must * be logged and left in its current execution column, never sent to replan. */ if (isNonPlanDefectPlanReviewFailure({ verdict: info.verdict, errorMessage: info.feedback, failureValue: info.failureValue, })) { await this.store.logEntry( taskId, "Plan Review provider failure — task kept in place", `Plan Review failed without a REVISE verdict due to a provider, model, transport, or abort condition. The task remains in ${liveTask.column}; no automatic replan was scheduled.\n\nDiagnostic:\n${info.feedback}`, this.getRunContextFor(taskId), ); return false; } /* * FNXC:PlanReviewReplan 2026-06-29-00:41: * Plan Review is pre-execution spec validation, so a failed/revision result * must repair PROMPT.md through triage instead of reopening implementation * steps. Triage already advances an approved `needs-replan` task to `todo`, * which lets the scheduler continue execution after the planner fixes it. */ const feedback = info.feedback?.trim() || "Plan Review failed before execution. Revise the task plan, then continue execution."; const settings = await mergeEffectiveSettings(this.store, liveTask, await this.store.getSettings()); const maxRevisions = resolveOptionalReviewRevisionBudget({ optionalGroupId: info.nodeId ?? "plan-review", workflowSettings: settings as Record, nodeMaxRevisions: info.maxRevisions, fallbackMaxRevisions: settings.maxPostReviewFixes ?? DEFAULT_MAX_POST_REVIEW_FIXES, }); const budget = resolveOptionalStepRevisionBudget(maxRevisions, settings.maxPostReviewFixes ?? DEFAULT_MAX_POST_REVIEW_FIXES); if (!budget.unbounded && (!Number.isFinite(budget.max) || budget.max <= 0)) { // FNXC:RemediationVisibility 2026-07-26-19:20 (FN-8596 follow-up): returning false here // makes the graph's plan-replan node fail with `remediation-not-scheduled` and leaves the // card parked in place with nothing scheduled to fix it. Never let that be silent. executorLog.warn( `${taskId}: plan-review remediation NOT scheduled — revision budget is zero/invalid (max=${String(budget.max)}). Card left parked.`, ); return false; } const revisionKey = optionalStepRevisionKey(info.nodeId ?? "plan-review", info.stepName); const currentCount = countOptionalStepRevisionAttempts(liveTask, revisionKey, info.stepName); if (!budget.unbounded && currentCount >= budget.max) { // U3: finite replan budget exhausted → park awaiting-approval (cap park // re-owned from the deleted triage gate), not a silent leave-in-place. const feedbackForPark = info.feedback?.trim() || "Plan Review requested another planning revision but the replan budget is exhausted."; await this.parkPlanReviewReplanCapExhausted(taskId, String(budget.max), currentCount, feedbackForPark); return true; } /* * FNXC:PlanReviewReplanCap 2026-07-05-17:28: * FN-7561: an unset Plan Review revision budget resolves to "unbounded" (see FNXC:WorkflowRevisionBudget above), which by design skips the ceiling check — so a task whose planner and reviewer persistently disagree, or whose reviewer keeps hard-failing, replans triage↔plan-review forever, silently burning a triage + review LLM call every cycle (FN-7525 ran 13+ attempts overnight with zero operator visibility). Enforce a finite safety ceiling even when unbounded: once hit, emit a loud halting log entry and STOP replanning (return false) so the gate falls through to a visible failed/parked state a human can act on, instead of looping indefinitely. Explicit numeric operator budgets are still honored as-is above; this only backstops the unbounded DEFAULT. */ const PLAN_REVIEW_REPLAN_HARD_CAP = 15; if (budget.unbounded && currentCount >= PLAN_REVIEW_REPLAN_HARD_CAP) { // U3: the unbounded-default safety ceiling now parks awaiting-approval with // the replan-cap reason (re-owned from the deleted triage gate) so the // non-convergence surfaces to a human instead of silently sitting in place. await this.parkPlanReviewReplanCapExhausted( taskId, String(PLAN_REVIEW_REPLAN_HARD_CAP), currentCount, feedback, ); return true; } const nextCount = currentCount + 1; const totalFixCount = (liveTask.postReviewFixCount ?? 0) + 1; const budgetLabel = budget.unbounded ? "unbounded" : String(budget.max); await this.store.updateTask(taskId, { postReviewFixCount: totalFixCount }, this.getRunContextFor(taskId)); this.clearPausedAborted(taskId); await this.store.logEntry( taskId, "AI spec revision requested", `Plan Review requested a planning revision before execution.\n\nStatus: ${info.status}\nFeedback:\n${feedback}`, this.getRunContextFor(taskId), ); /* FNXC:PlanReviewReplan 2026-07-12-23:20: The replan rebound is workflow-aware: workflows without a "triage" column (Coding (Ideas)) replan in place in their planner column ("todo") instead of being orphaned in an undeclared "triage" column, which the board rendered back in the intake lane. */ const replanColumn = await resolveReplanTargetColumn(this.store, taskId); await this.store.logEntry( taskId, `Plan Review failed — moved to ${replanColumn} for automatic replan (attempt ${nextCount}/${budgetLabel})`, optionalStepRevisionLogOutcome(feedback, revisionKey), this.getRunContextFor(taskId), ); this.workflowLifecycleMovesInFlight.add(taskId); try { await moveTaskToReplanColumn(this.store, { id: taskId, column: liveTask.column }, replanColumn); } finally { this.workflowLifecycleMovesInFlight.delete(taskId); } await this.store.updateTask(taskId, { status: "needs-replan", error: null, recoveryRetryCount: null, nextRecoveryAt: null, graphResumeRetryCount: 0, }, this.getRunContextFor(taskId)); return true; } if (info.verdict !== "REVISE") { // FNXC:RemediationVisibility 2026-07-26-19:20: a hard-failed gate with no parsed REVISE // verdict schedules nothing, so the remediation node fails and the card parks. Say so. executorLog.warn( `${taskId}: pre-merge remediation NOT scheduled for step "${info.stepName}" — status=${info.status}, verdict=${info.verdict ?? "none"}. Card left parked.`, ); return false; } const settings = await mergeEffectiveSettings(this.store, liveTask, await this.store.getSettings()); const maxRevisions = resolveOptionalReviewRevisionBudget({ optionalGroupId: info.nodeId ?? "", workflowSettings: settings as Record, nodeMaxRevisions: info.maxRevisions, fallbackMaxRevisions: settings.maxPostReviewFixes ?? DEFAULT_MAX_POST_REVIEW_FIXES, }); const budget = resolveOptionalStepRevisionBudget(maxRevisions, settings.maxPostReviewFixes ?? DEFAULT_MAX_POST_REVIEW_FIXES); if (!budget.unbounded && (!Number.isFinite(budget.max) || budget.max <= 0)) { executorLog.warn( `${taskId}: pre-merge remediation NOT scheduled for step "${info.stepName}" — revision budget is zero/invalid (max=${String(budget.max)}). Card left parked.`, ); return false; } const revisionKey = optionalStepRevisionKey(info.nodeId, info.stepName); const currentCount = countOptionalStepRevisionAttempts(liveTask, revisionKey, info.stepName); if (!budget.unbounded && currentCount >= budget.max) { // Budget exhaustion is a legitimate terminal outcome, but it must be visible: the card stays // in place with a failed pre-merge step and only an operator bypass clears it. executorLog.warn( `${taskId}: pre-merge remediation budget EXHAUSTED for step "${info.stepName}" (${currentCount}/${String(budget.max)}). Card left parked for operator action.`, ); return false; } const nextCount = currentCount + 1; const totalFixCount = (liveTask.postReviewFixCount ?? 0) + 1; const budgetLabel = budget.unbounded ? "unbounded" : String(budget.max); await this.store.updateTask(taskId, { postReviewFixCount: totalFixCount }, this.getRunContextFor(taskId)); await this.store.logEntry( taskId, `Pre-merge optional workflow step requested executor fixes (attempt ${nextCount}/${budgetLabel})`, optionalStepRevisionLogOutcome(`Step: ${info.stepName}\nStatus: ${info.status}\nFeedback:\n${info.feedback}`, revisionKey), this.getRunContextFor(taskId), ); await this.sendTaskBackForFix( liveTask, liveTask.worktree ?? "", info.feedback, info.stepName, `Pre-merge optional workflow step "${info.stepName}" requested revision`, true, false, { attempt: nextCount, max: budget.unbounded ? undefined : budget.max }, ); return true; } private async recoverMissingRequiredArtifacts( task: Task, artifactKeys: string[], source: { source: "graph-entry" | "workflow-step"; nodeId?: string }, ): Promise { const currentTask = await this.store.getTask(task.id).catch(() => null); if (!currentTask || await this.isRequiredArtifactRecoveryProtected(currentTask)) return; task = currentTask; const decision = computeRecoveryDecision({ recoveryRetryCount: task.recoveryRetryCount, nextRecoveryAt: task.nextRecoveryAt, }); const attempt = decision.nextState.recoveryRetryCount ?? MAX_RECOVERY_RETRIES; const context = this.getRunContextFor(task.id); const action = decision.shouldRetry ? "replan" : "park-failed"; await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: context?.runId ?? generateSyntheticRunId("required-artifact-missing", task.id), domain: "database", mutationType: "task:required-artifact-missing", target: task.id, metadata: { taskId: task.id, artifactKeys, owner: "planning", source: source.source, action, attempt, maxAttempts: MAX_RECOVERY_RETRIES, ...(source.nodeId ? { nodeId: source.nodeId } : {}), }, }); if (!decision.shouldRetry) { const liveTask = await this.store.getTask(task.id).catch(() => null); if (!liveTask || await this.isRequiredArtifactRecoveryProtected(liveTask)) return; const error = `REQUIRED_ARTIFACT_RECOVERY_EXHAUSTED: ${artifactKeys.join(", ")} remained missing after ${MAX_RECOVERY_RETRIES} automatic planning retries.`; await this.store.logEntry(task.id, error, undefined, context); await this.store.updateTask(task.id, { status: "failed", error, recoveryRetryCount: null, nextRecoveryAt: null, }, context); return; } const replanColumn = await resolveReplanTargetColumn(this.store, task.id); await this.store.logEntry( task.id, `Required workflow artifact missing — moved to ${replanColumn} for automatic planning recovery (attempt ${attempt}/${MAX_RECOVERY_RETRIES} in ${formatDelay(decision.delayMs)})`, `Missing artifact keys: ${artifactKeys.join(", ")}`, context, ); this.workflowLifecycleMovesInFlight.add(task.id); try { const liveTask = await this.store.getTask(task.id).catch(() => null); if (!liveTask || await this.isRequiredArtifactRecoveryProtected(liveTask)) return; await moveTaskToReplanColumn(this.store, { id: task.id, column: liveTask.column }, replanColumn); } finally { this.workflowLifecycleMovesInFlight.delete(task.id); } await this.store.updateTask(task.id, { status: "needs-replan", error: null, recoveryRetryCount: decision.nextState.recoveryRetryCount, nextRecoveryAt: decision.nextState.nextRecoveryAt, graphResumeRetryCount: 0, }, context); } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: made ASYNC to own its resolution): This predicate protects a card from artifact-recovery replanning, and three of its conditions are lifecycle columns: the terminal pair, and a review row whose auto-merge is off (a human owns it). As literals they all read false on a renamed board — so a FINISHED card, or a review row a human was holding, could be moved to the replan column and have its status rewritten to needs-replan. ASYNC rather than lane parameters: all four callers already `await store.getTask` immediately before calling this, so there is no new I/O ordering, and a parameter list would put the resolution in four places that must agree. The archived half is why the SYNC planner-lane resolver was not an option — it exposes no archived lane — and widening a shared resolver from inside a call-site sweep is scope creep that makes a conversion unreviewable. */ private async isRequiredArtifactRecoveryProtected(task: Task): Promise { const terminalColumns = await resolveTerminalColumnsFor(this.store, task.id); const protectionReviewLane = (await this.resolveResumeLanes(task.id)).review; return Boolean( task.deletedAt || task.paused || task.userPaused === true || terminalColumns.includes(task.column) || task.mergeDetails?.mergeConfirmed === true || (task.column === protectionReviewLane && task.autoMerge === false), ); } /** * Auto-revive an `in-review` task whose pre-merge workflow step(s) failed, by * replaying the same send-back-for-fix flow the executor uses during a live * run. Invoked by SelfHealingManager's `recoverReviewTasksWithFailedPreMergeSteps` * scan when a task is parked in review with a failed pre-merge step and no * active session. * * Picks the latest failed pre-merge workflow step result (there is usually only * one, but if several ran we want the most recent), injects its feedback into * `PROMPT.md`, resets steps, and schedules todo → in-progress. The caller may * account for a scheduled retry, but this method independently enforces the * effective finite-or-unlimited revision budget before it can reopen work. * * @returns true when the task was sent back, false when no eligible failed * step exists (caller should skip). */ async recoverFailedPreMergeWorkflowStep(task: Task): Promise { try { /* FNXC:WorkflowPostMerge 2026-06-26-14:00: U7c: gate-ness is now sourced from the recorded `WorkflowStepResult.status`, NOT a `workflow_steps` table read. The graph executor (workflow-graph-executor.ts) maps a group outcome to status by gate semantics: a GATE REVISE / hard failure records `status: "failed"` (blocking), while an ADVISORY REVISE records `status: "advisory_failure"` (non-blocking). So a pre-merge result with `status === "failed"` IS by construction a blocking gate failure — the prior `getWorkflowStep(id).gateMode` lookup was redundant (and after the table drop it returned undefined for graph node ids anyway). Recovery revives the task from the latest blocking pre-merge failure. */ const failed = (task.workflowStepResults ?? []) .filter((r) => (r.phase || "pre-merge") === "pre-merge" && r.status === "failed") .sort((a, b) => { const aTs = Date.parse(a.completedAt || a.startedAt || ""); const bTs = Date.parse(b.completedAt || b.startedAt || ""); return (Number.isFinite(bTs) ? bTs : 0) - (Number.isFinite(aTs) ? aTs : 0); }); const target = failed[0]; if (!target) { executorLog.warn(`${task.id}: no failed pre-merge workflow step to recover from`); return false; } const feedback = target.output?.trim() || "(no feedback captured)"; const stepName = target.workflowStepName || target.workflowStepId || "Unknown"; const budget = await this.resolveFailedPreMergeWorkflowStepBudget(task, target); /* * FNXC:WorkflowRevisionBudget 2026-07-22-18:30: * Failed-step recovery is also a remediation entry point, not merely a * retry-label formatter. Enforce the same finite Code Review budget here * as live and restart-local graph remediation: an unset policy remains * unlimited, while zero or an exhausted explicit cap cannot silently send * work back for another fix. Progress-loop termination stays owned by the * graph executor's signature guard rather than this budget check. */ if (!budget.unbounded && (!Number.isFinite(budget.max) || budget.max <= 0)) return false; if (!budget.unbounded && budget.attempts >= budget.max) return false; await this.sendTaskBackForFix( task, task.worktree ?? "", feedback, stepName, `Auto-revived from in-review: pre-merge workflow step "${stepName}" had failed`, true, false, { attempt: budget.attempts + 1, max: budget.unbounded ? undefined : budget.max }, ); return true; } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to recover failed pre-merge workflow step for ${task.id}: ${errorMessage}`); return false; } } /** * Returns true when execute() should be deferred because the agent bound to * this task has an active heartbeat run and allowParallelExecution=false. * * Only applies to permanent (non-ephemeral) agents. Always returns false * when agentStore is unavailable or the agent cannot be resolved. */ private async shouldDeferForHeartbeat(agentId: string): Promise { if (!this.options.agentStore) return false; const agent = await this.options.agentStore.getAgent(agentId).catch(() => null); if (!agent) return false; if (isEphemeralAgent(agent)) return false; const rc = (agent.runtimeConfig ?? {}) as AgentHeartbeatConfig; if (rc.allowParallelExecution !== false) return false; const activeRun = await this.options.agentStore.getActiveHeartbeatRun(agentId).catch(() => null); return activeRun !== null; } private async getAuthoritativeAssignedAgent( assignedAgentId: string | null | undefined, ): Promise { const normalizedId = assignedAgentId?.trim(); if (!normalizedId) return null; const configuredAgent = await this.options.agentStore?.getAgent(normalizedId).catch(() => null) ?? null; if (configuredAgent) return configuredAgent; /* FNXC:ModelResolution 2026-07-10-00:00: Task execution sessions must honor the assigned permanent agent's runtimeConfig like chat sessions do. If the live executor was handed an agents-less worktree AgentStore, fall back to the authoritative project `.fusion` AgentStore instead of letting `resolveExecutorSessionModel` see an empty runtimeConfig and silently drift to the pi built-in model. */ try { /* FNXC:PostgresOnlyDataAccess 2026-07-17-14:20: The authoritative-agent fallback AgentStore MUST inherit the TaskStore's AsyncDataLayer so it runs in PostgreSQL backend mode. AgentStore does not derive `asyncLayer` from `taskStore`, so omitting it left this store in legacy-SQLite mode; in a PG deployment `init()`/`getAgent()` then hit the removed SQLite stub, the throw was swallowed by the catch below, and this method silently returned null — reintroducing the exact model-drift to the pi built-in that this fallback exists to prevent. Pass the layer (mirrors the canonical site in agent-tools.ts). */ const authoritativeAgentLayer = this.store.getAsyncLayer(); /* FNXC:PostgresOnlyDataAccess 2026-07-17-16:10: Do NOT memoize a layer-less AgentStore. If the very first lookup runs before the TaskStore's AsyncDataLayer is attached, a plain `??=` would cache a legacy-SQLite-mode store forever, so every later call keeps failing through the removed SQLite path even after the layer arrives. Rebuild when a layer is now available but the cached store is not in backend mode. */ if (!this.authoritativeAssignedAgentStore || (authoritativeAgentLayer && !this.authoritativeAssignedAgentStore.backendMode)) { this.authoritativeAssignedAgentStore = new AgentStore({ rootDir: join(this.rootDir, ".fusion"), taskStore: this.store, ...(authoritativeAgentLayer ? { asyncLayer: authoritativeAgentLayer } : {}), }); } await this.authoritativeAssignedAgentStore.init(); return await this.authoritativeAssignedAgentStore.getAgent(normalizedId).catch(() => null); } catch (err: unknown) { executorLog.warn(`Failed to read assigned agent ${normalizedId} from authoritative project AgentStore: ${err instanceof Error ? err.message : String(err)}`); return null; } } private async getAssignedAgentRuntimeConfig( assignedAgentId: string | null | undefined, ): Promise | undefined> { const agent = await this.getAuthoritativeAssignedAgent(assignedAgentId); return (agent?.runtimeConfig ?? undefined) as Record | undefined; } /** * Re-dispatch execute() for any unstarted in-progress task whose EFFECTIVE * principal is the given agent. Called after a heartbeat run completes to unblock * tasks that were deferred by the allowParallelExecution=false gate. * * TWO-PASS (plan U5, R6) — the `assignedAgentId`-only filter alone misses tasks an * override/defer column binding re-keys to the column agent: * 1. Tasks directly `assignedAgentId === agentId` (legacy, byte-identical). * 2. Tasks whose effective column agent resolves to `agentId` for their * governing execute / step-execute seam — resolved per candidate via the core * column-agent resolver against the task's workflow IR. Bounded: only * not-already-executing in-progress tasks are probed, and the IR resolution is * best-effort (failure → skip, never strands resume). * A task re-dispatched by pass 1 is not re-dispatched by pass 2 (dedupe set). */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40: The wip-lane read for the two resume sweeps, resolved at PROJECT level. `listTasks`' `column` option filters in the store, so both sweeps returned an EMPTY array on a renamed board and neither resume ran: - `resumeTaskForAgent` — a durable agent coming back up adopted nothing, so its in-flight task stayed orphaned; - `resumeOrphaned` — the engine-wide sweep found no orphans to re-dispatch after a restart. Both are recovery paths, which is the expensive place to be silently inert: the failure only shows up after a crash or a restart, when the operator is already looking at something else. The census cannot see either — it scores comparisons, and a query filter is not one. Project-level because a read has no task in hand, legacy ids unioned so a board mid-rename still finds rows under the old one, deduped by id because one column can carry two roles. */ private async listWipLaneTasks(): Promise { const columns = await resolveProjectColumnsForRoles(this.store, ["countsTowardWip"]); const byId = new Map(); for (const column of columns) { for (const task of await this.store.listTasks({ slim: true, column })) byId.set(task.id, task as Task); } return [...byId.values()]; } async resumeTaskForAgent(agentId: string): Promise { const settings = await this.store.getSettings(); if (settings.globalPause || settings.enginePaused) return; const tasks = await this.listWipLaneTasks(); const dispatched = new Set(); const isDispatchable = (task: Task): boolean => !task.deletedAt && !task.paused && !this.executing.has(task.id) && !this.activeSessions.has(task.id) && !this.activeStepExecutors.has(task.id) && !this.activeWorkflowStepSessions.has(task.id); const dispatch = (task: Task, reason: string): void => { if (dispatched.has(task.id)) return; dispatched.add(task.id); executorLog.log(`${task.id}: re-dispatching execute() after heartbeat completion for agent ${agentId} (${reason})`); this.execute(task).catch((err) => executorLog.error(`Failed to resume ${task.id} after heartbeat completion:`, err), ); }; // Pass 1: directly-assigned tasks (legacy behavior, byte-identical). for (const task of tasks) { if (task.assignedAgentId === agentId && isDispatchable(task)) { dispatch(task, "assigned"); } } // Pass 2: tasks whose EFFECTIVE column agent resolves to `agentId`. The graph // engine is the default runtime; the IR resolve is best-effort and skipped // for tasks already dispatched/executing. for (const task of tasks) { if (dispatched.has(task.id) || !isDispatchable(task)) continue; // Skip tasks the assigned-agent filter already covers — a redundant column // binding to the same agent would only re-confirm pass 1. if (task.assignedAgentId === agentId) continue; let matches = false; try { matches = await this.taskEffectiveAgentMatches(task, agentId); } catch { matches = false; } if (matches) dispatch(task, "effective-column-agent"); } } /** Column-agent principal alignment (plan U5, R6). True when the EFFECTIVE agent * governing `task`'s execute or step-execute seam — resolved through the shared * core resolver against the task's workflow IR — is `agentId`. Used by the * `resumeTaskForAgent` second pass to re-dispatch column-bound tasks the * `assignedAgentId` filter misses. Best-effort: an unresolvable IR yields false. */ private async taskEffectiveAgentMatches(task: Task, agentId: string): Promise { /* FNXC:WorkflowColumns 2026-06-22-18:00: Workflow columns are the default runtime, so resume pass 2 always resolves the task workflow IR. Persisted experimentalFeatures.workflowColumns=false values must not make column-agent dispatch inert. */ const ir = await resolveWorkflowIrForTask(this.store, task.id); if (!ir || ir.version !== "v2") return false; const ownSettings = this.extractOwnSettings(task); const matchesNodeId = (nodeId: string): boolean => { const binding = resolveColumnAgentBinding(ir, nodeId); if (!binding) return false; const effective = resolveEffectiveAgent({ binding, ...ownSettings }); return effective.source === "column-agent" && effective.agentId === agentId; }; // Governing seam nodes: the execute-seam prompt node lives at the top level. for (const node of ir.nodes) { const seam = node.kind === "prompt" ? node.config?.seam : undefined; if (seam !== "execute" && seam !== "step-execute") continue; if (matchesNodeId(node.id)) return true; } // step-execute seam nodes are legal ONLY inside a foreach template // (workflow-ir.ts), so they never appear in ir.nodes above. Walk each foreach // node's template subgraph and resolve the binding via a synthesized instance // node id. Step index 0 is sufficient — column resolution is index-independent // (all instances share the same template node and thus the same binding, R4). for (const node of ir.nodes) { if (node.kind !== "foreach") continue; const templateNodes = (node.config as { template?: { nodes?: WorkflowIrNode[] } } | undefined)?.template?.nodes ?? []; for (const templateNode of templateNodes) { const seam = templateNode.kind === "prompt" ? templateNode.config?.seam : undefined; if (seam !== "step-execute") continue; if (matchesNodeId(instanceNodeId(node.id, 0, templateNode.id))) return true; } } return false; } /** * Resume orphaned in-progress tasks (e.g., after crash/restart). * Call once after engine startup. * * Tasks that are already complete (all steps done/skipped) are fast-pathed * directly to in-review without spawning a new agent session. */ async resumeOrphaned(): Promise { const settings = await this.store.getSettings(); if (settings.globalPause || settings.enginePaused) { executorLog.log( `resumeOrphaned skipped — ${ settings.globalPause ? "global pause" : "engine pause" } is active`, ); return; } /* FNXC:WorkflowResolvedColumns 2026-07-30-21:40 (a MISSED PAIR, the class #2879 ratcheted): `listWipLaneTasks()` above already resolves the wip lane by role. This filter did not — it re-asserted the literal `in-progress` on the rows that read returned, so on a renamed board the read found the orphans and the filter dropped every one. That is the worse half of the pattern: the read looks converted, the census counts only the comparison, and the sweep silently does nothing. Here it means orphaned tasks are NEVER resumed after a crash or restart — the one path that recovers them. The rows come from a `listTasks({ column })` per resolved column, so a row is in that column by definition; the re-assert only ever had value as a stale-snapshot guard, which membership preserves. */ const wipColumns = await resolveProjectColumnsForRoles(this.store, ["countsTowardWip"]); const tasks = await this.listWipLaneTasks(); const inProgress = tasks.filter( (t) => wipColumns.has(t.column) && !t.deletedAt && !this.executing.has(t.id) && !t.paused, ); if (inProgress.length === 0) return; executorLog.log(`Found ${inProgress.length} orphaned in-progress task(s)`); const resumeDelayMs = getResumeOrphanDelayMs(); if (resumeDelayMs > 0) { executorLog.log( `Deferring orphan task resumption for ${resumeDelayMs}ms to keep dashboard responsive during cold start`, ); } // When the delay is zero (default in tests and when explicitly disabled), // skip the setTimeout indirection so the spawn happens on the current // microtask — matching the legacy behavior callers may rely on. const scheduleResume = resumeDelayMs > 0 ? (fn: () => void) => { setTimeout(fn, resumeDelayMs); } : (fn: () => void) => { fn(); }; let yieldNext = false; for (const task of inProgress) { if (yieldNext) await yieldEventLoop(); yieldNext = true; // Fast-path: if the task already completed its work (all steps done), // move it directly to in-review instead of re-executing from scratch. if (this.isTaskWorkComplete(task) && !task.mergeDetails) { if (this.recoveringCompleted.has(task.id)) { executorLog.debug(`${task.id} completed-task recovery already running - skipping duplicate startup recovery`); continue; } if (TaskExecutor.processWideGraphRouting.has(task.id)) { executorLog.debug(`${task.id} owned by the workflow graph interpreter — skipping completed-task fast-path`); continue; } executorLog.log(`${task.id} is already complete — fast-pathing to in-review`); this.recoveringCompleted.add(task.id); scheduleResume(() => { void this.recoverCompletedTask(task) .catch((err) => executorLog.error(`Failed to recover completed orphan ${task.id}:`, err), ) .finally(() => { this.recoveringCompleted.delete(task.id); }); }); continue; } if (this.isNoProgressNoTaskDoneFailure(task)) { executorLog.log(`${task.id} failed without fn_task_done and has no step progress — leaving for self-healing requeue`); continue; } executorLog.log(`Resuming ${task.id}: ${task.title || task.description.slice(0, 60)}`); try { await this.clearResumeFailureState(task); await this.store.logEntry(task.id, "Resumed after engine restart"); await this.recoverApprovedStepsOnResume(task.id); } catch (err) { executorLog.error(`Failed to write resume log for ${task.id}:`, err); } scheduleResume(() => { this.execute(task).catch((err) => executorLog.error(`Failed to resume ${task.id}:`, err), ); }); } } /** * Execute a task in an isolated git worktree. * * Worktree acquisition flow: * 1. If the worktree already exists on disk (resume after crash), reuse it. * 2. If a {@link WorktreePool} is provided and `recycleWorktrees` is enabled, * attempt to acquire a warm worktree from the pool. Pooled worktrees skip * the `worktreeInitCommand` since their build caches are already warm. * 3. Otherwise, create a fresh worktree via `git worktree add` and run the * `worktreeInitCommand` if configured. */ /** * Resolve custom instructions for a given agent role by looking up agents * in the AgentStore that have instructions configured. * Returns an empty string if no instructions are found. */ private async resolveInstructionsForRole(role: string, settings?: Settings): Promise { if (!this.options.agentStore) return ""; try { const agents = await this.options.agentStore.listAgents({ role: role as AgentCapability }); for (const agent of agents) { if (agent.instructionsText || agent.instructionsPath) { try { const ratingSummary = await this.options.agentStore.getRatingSummary(agent.id); const mode = resolveAgentMemoryInclusionMode({ agent, globalSettings: settings }).mode; return await resolveAgentInstructions(agent, this.rootDir, ratingSummary, mode); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${agent.id}: failed to load rating summary for instruction resolution, falling back to default instructions: ${msg}`); const mode = resolveAgentMemoryInclusionMode({ agent, globalSettings: settings }).mode; return await resolveAgentInstructions(agent, this.rootDir, undefined, mode); } } } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to resolve instructions for role '${role}', continuing without custom instructions: ${msg}`); } return ""; } /** * Execute a task in an isolated git worktree. * * **Worktree assignment:** New worktrees get humanized random names * (e.g., `.worktrees/swift-falcon/`) via `generateWorktreeName()` rather * than being named after the task ID. This decouples directory names from * tasks, enabling worktree reuse across dependency chains. When resuming * a task that already has `task.worktree` set, the existing path is used * as-is. Branches remain task-scoped (`fusion/{task-id}`). */ // ── Workflow graph interpreter (cutover M-B/M-C) ───────────────────────── // // The workflow graph runner owns lifecycle SEQUENCING for every task: // custom prompt/script/gate nodes run via the WorkflowStep machinery, and the // planning/execute/review/merge seam nodes delegate to the engine primitives. // Interpreter-level failure parks the task as a workflow failure rather than // falling through to a second runtime path. /* FNXC:WorkflowExecution 2026-07-19-01:30: U5d (R9) — the `graphCompletionInterceptors` Map is DELETED. It was shared per-task mutable state used to signal "this execute() call is a graph implementation phase": the graph set an entry, re-entered execute(), and execute() read the Map at ~12 sites to decide whether to stop at the implementation-complete boundary, skip outer routing, suppress `fn_review_step`, and mark review gates graph-owned. Signalling through a shared Map made the graph/legacy split invisible at the call site and left stale entries to clean up on abort. It is replaced by an EXPLICIT optional `graphCompletion` callback: presence of the callback IS the "graph-owned implementation phase" signal, and invoking it hands the captured modifiedFiles back to the graph runner. FNXC:WorkflowExecution 2026-07-19-02:10: U5e (R9) — the RE-ENTRY is now gone too. `executeCore`'s implementation body was lifted into `runImplementation()`, which the graph seam calls DIRECTLY; `executeCore` is routing only and `execute()` no longer carries a completion parameter. There is no longer any path by which the graph runner calls back into `execute()`. */ /** Per graph-run agent-log boundary; passed to failure handling rather than trusting stale task snapshots. */ private graphToolFailureRunCursors = new Map(); /** Step-inversion (KTD-2/KTD-8, U6/U8): graph-owned step-execute can pin * step-session physics for workflows that need a hard per-step boundary * before step-review. Default final-review coding does not pin here and * therefore respects `runStepsInNewSessions` (reuse one session when false, * fresh per-step sessions when true). Cleared when the graph run ends * (executeWorkflowGraph finally). */ private graphStepSessionPinned = new Set(); /** Step-inversion (U6/U8): caches the per-run implementation-phase result for a * graph-owned task so the foreach sub-walk's per-step `runTaskStep` driver runs * the (step-session) implementation exactly once per run and lets later step * instances observe the projection rather than re-running execute() per step. * Keyed by task id; cleared alongside the pin. */ private graphStepRunOnce = new Map>(); /** Step-inversion (KTD-4): the foreach instance the step-execute seam is * currently driving for a graph-owned task, so `runGraphTaskStep` can honor * `deferDoneToReview` when deciding whether a non-terminal step is a success * (review will author done) or a failure (implementation left it incomplete). * Stamped by the stepExecute seam around the runTaskStep call; cleared with the * per-run pins. Keyed by `${task.id}:${instanceId}` so parallel foreach * instances of the same task cannot clobber each other's active context * (the read path threads the same instanceId through `runGraphTaskStep`). */ private graphStepActiveContext = new Map(); /** * FNXC:ProactiveChatStatus 2026-07-16-12:30: * Keep a graph RETHINK summary until its rework reset succeeds. The status wording says the step * was rolled back, so it must not reach the task chat before resetStepToBaseline completes. */ private graphRethinkNarrations = new Map(); /** Composite key for graph-owned per-instance state: never share parallel foreach instances. */ private graphActiveContextKey(taskId: string, instanceId: string): string { return `${taskId}:${instanceId}`; } /** Column-agent seam wiring (column-agent plan U4, R2/R3/R4). Per-run binding * resolver keyed by task id: maps a governing node id to its column-agent * binding (if any), computed once per run in executeWorkflowGraph from the * resolved IR. The execute / step-execute seams consume it to decide whether the * coding/step session runs as a column agent. Cleared in the run's finally. */ private graphColumnAgentResolver = new Map WorkflowColumnAgent | undefined>(); /** (U3) Task ids whose current graph run is genuinely unattended (LFG / * pipeline / disable-model-invocation — no human will ever answer). Set only * by an explicit `unattended` workflow-run option; default-absent means a * board run. runGraphCustomNode reads this to set FUSION_HEADLESS on skill * steps. Cleared in executeWorkflowGraph's finally alongside the resolver. */ private graphUnattendedRuns = new Set(); /** Column-agent seam wiring (column-agent plan U4). The governing graph node id * for the implementation pass currently in flight for a task — the execute-seam * prompt node's id (execute seam), or the foreach instance node id (step-execute * seam, which the core resolver maps through template inheritance). Stamped by * the seam from the reserved {@link SEAM_GOVERNING_NODE_CONTEXT_KEY} context key * right before it drives the implementation phase, read inside execute()'s * session build, and cleared by the seam afterward. Keyed by task id. */ private graphSeamGoverningNodeId = new Map(); /** * FNXC:Settings-ThinkingLevel 2026-07-10-00:00: * Execute and step-execute seam nodes can pin reasoning effort for the implementation session; keep it per graph run so session creation applies node/step > task > settings precedence. */ private graphSeamThinkingLevel = new Map(); /** * FNXC:WorkflowStepSkills 2026-07-22-00:00: * FN-8490 pins the canonical `config.executor: "skill"` + trimmed * `config.skillName` request only for the pass-initiating foreach instance. * The implementation pass is shared across instances, so this template-constant * value must settle with the same lifecycle as governing-node and thinking pins. */ private graphSeamSkillName = new Map(); /** Tasks currently being orchestrated by the graph runner. Process-wide for * the same reason as executingTaskLock (FN-4811): duplicate execute() * invocations can arrive from different TaskExecutor instances in one * process (engine restart race, hybrid runtimes), and the graph runner does * not hold the executing-task lock between seams. */ private get graphRouting(): Set { return TaskExecutor.processWideGraphRouting; } private static processWideGraphRouting = new Set(); /** Wired by the runtime to ProjectEngine.onMerge — resolves with the merge outcome. */ private mergeRequester?: (taskId: string, options?: { signal?: AbortSignal }) => Promise; setMergeRequester(requestMerge: (taskId: string, options?: { signal?: AbortSignal }) => Promise): void { this.mergeRequester = requestMerge; } /** * Route a task through the workflow graph interpreter when eligible. * Returns true when the graph owned the task to a terminal disposition * (completed or failed); false when the legacy pipeline should run. */ private async executeWorkflowGraph(task: Task, opts?: { alreadyClaimed?: boolean }): Promise { // Claim synchronously before any await so concurrent execute() calls for // the same task cannot both enter graph routing (mirrors executingTaskLock). // executeCore may already have claimed before its pre-graph awaits (FN-8471). if (!opts?.alreadyClaimed) { this.graphRouting.add(task.id); } let graphAbortController: AbortController | undefined; /* FNXC:GlobalConcurrencyControls 2026-07-14-18:30: The hold/release sweep may have already tryAcquired a global slot for this card before moving it to in-progress. Claim that pre-held slot for the full graph run so utilization stays honest between workflow nodes and triage cannot overfill the cap while this task is still graph-owned. */ const hadPreHeldExecutorSlot = takePreHeldExecutorSlot(task.id); if (hadPreHeldExecutorSlot) { this.outerConcurrencyClaims.add(task.id); } try { let settings: Settings; try { settings = await this.store.getSettings(); } catch (err) { await this.handleGraphFailure(task, { disposition: "failed", outcome: "failure", reason: `settings-load-failed: ${err instanceof Error ? err.message : String(err)}`, visitedNodeIds: [], }); return; } /* FNXC:WorkflowExecution 2026-06-22-18:00: workflowGraphExecutor graduated from Experimental. Every task routes through the graph runner by default, and stale persisted experimentalFeatures.workflowGraphExecutor=false values are ignored so the product no longer has a user-facing or runtime graph-engine kill switch. */ settings = { ...settings }; /* * FNXC:ExecutorToolFailureRetry 2026-07-16-12:00: * Capture a count cursor without reading the task log. Failure handling receives this * execution-local boundary, so a stale task snapshot cannot accidentally qualify an old run. * * FNXC:ExecutorToolFailureRetry 2026-07-17-06:30: * Minimal/test TaskStore adapters may omit getAgentLogCount (same optional pattern as * project-engine). Treat a missing method as cursor 0 so graph entry does not throw * "is not a function" and still records a durable detector boundary when updateTask exists. */ if (resolveMaxConsecutiveToolFailureRetries(settings) > 0) { const cursor = typeof this.store.getAgentLogCount === "function" ? await this.store.getAgentLogCount(task.id).catch(() => 0) : 0; this.graphToolFailureRunCursors.set(task.id, cursor); if (typeof this.store.updateTask === "function") { await this.store.updateTask(task.id, { toolFailureDetectorLogCursor: cursor }, this.getRunContextFor(task.id)); } } let selection: { workflowId: string; stepIds: string[] } | undefined; /* FNXC:WorkflowExecution 2026-07-19-17:30 (U10b / R9): The legacy fallback is DELETED. It used to return `false` here — handing the run to a legacy execute path — when the store exposed neither workflow-selection reader. That escape hatch is gone: graph ownership is now UNCONDITIONAL, which is what lets `graphCompletion` be a required callback rather than an optional one and collapses the three completion boundaries in `runImplementation` to plain returns. A store that cannot resolve a workflow now ALWAYS fails closed, not only when the task has enabled pre-merge steps. The old "no enabled steps means nothing to gate, so the legacy path is safe" carve-out died with the path it protected: there is no second executor left to fall back to, so returning `false` would silently run nothing. */ if ( typeof this.store.getTaskWorkflowSelectionAsync !== "function" && typeof this.store.getTaskWorkflowSelection !== "function" ) { /* FNXC:FastOptionalSteps 2026-06-30-09:45: Fast mode only clears optional workflow steps by default; explicit `enabledWorkflowSteps` remains operator intent. Minimal or older stores that cannot resolve the graph must fail closed, even in fast mode, rather than falling through and silently skipping the selected optional-group body. */ await this.handleGraphFailure(task, { disposition: "failed", outcome: "failure", reason: "workflow-selection-api-unavailable: store lacks a workflow-selection reader so the workflow graph cannot run; " + "the legacy execute fallback was removed (U10b) and the graph is the only executor. Failing closed rather than running nothing (KTD-5).", visitedNodeIds: [], }); return; } try { selection = typeof this.store.getTaskWorkflowSelectionAsync === "function" ? await this.store.getTaskWorkflowSelectionAsync(task.id) : this.store.getTaskWorkflowSelection(task.id); } catch (err) { await this.handleGraphFailure(task, { disposition: "failed", outcome: "failure", reason: `workflow-selection-failed: ${err instanceof Error ? err.message : String(err)}`, visitedNodeIds: [], }); return; } selection ??= { workflowId: "builtin:coding", stepIds: [] }; // Resolve the production run id ONCE, here, so it is the single source of // truth shared by the runner AND the executor-side persistence deps // (parse-steps pin probe, foreach instance-row flips, resume reconcile). The // runner derives `${task.id}:${definition.id}`; we mirror that derivation // from the resolved definition and thread it everywhere. Best-effort: if the // definition cannot be resolved (older store), the runner falls back to its // own derivation and the deps fall back to the legacy `:run` literal — the // prior behavior — so this never strands a task. let resolvedRunId: string | undefined; try { const definition = selection.workflowId === "builtin:coding" ? { id: "builtin:coding" } : await this.store.getWorkflowDefinition?.(selection.workflowId); if (definition) resolvedRunId = `${task.id}:${definition.id}`; } catch { // Definition load failure — leave undefined; deps/runner use fallbacks. } // Column-agent binding (plan U3): the IR is NOT in scope inside // runGraphCustomNode, so resolve it here (the seam wiring) where the // selection is known, and thread a per-node binding lookup into the custom // node callback. Resolve the IR ONCE per run (never an uncached per-node // fetch — mirrors the hold-release.ts irCache posture); best-effort, so a // resolution failure simply yields no bindings (R8 graceful degradation). /* FNXC:WorkflowColumns 2026-06-22-18:00: Column-agent binding now participates in every graph run. The former workflowColumns kill switch was removed, so stale persisted false values cannot silently disable custom-node, seam, or watcher bindings. */ let columnAgentIr: WorkflowIr | undefined; try { columnAgentIr = await resolveWorkflowIrForTask(this.store, task.id); } catch { columnAgentIr = undefined; } if (columnAgentIr) { const missingEntryArtifacts: string[] = []; for (const artifact of workflowEntryArtifacts(columnAgentIr)) { let content: string | undefined; try { content = await this.readTaskArtifact(task.id, artifact.key); } catch (error) { const failureValue = requiredArtifactReadFailedValue(artifact.key); await this.handleGraphFailure(task, { disposition: "failed", outcome: "failure", reason: `workflow-required-artifact-read-failed:${artifact.key}:${error instanceof Error ? error.message : String(error)}`, visitedNodeIds: ["workflow-entry-artifact"], context: { "node:workflow-entry-artifact:value": failureValue }, }); return; } if (typeof content !== "string" || !content.trim()) missingEntryArtifacts.push(artifact.key); } if (missingEntryArtifacts.length > 0) { const liveTask = await this.store.getTask(task.id).catch(() => task); await this.recoverMissingRequiredArtifacts(liveTask, missingEntryArtifacts, { source: "graph-entry" }); return; } } const resolveBindingForNode = (nodeId: string): WorkflowColumnAgent | undefined => columnAgentIr ? resolveColumnAgentBinding(columnAgentIr, nodeId) : undefined; // Column-agent seam wiring (U4): expose the same per-run resolver to the // execute / step-execute seams (which key off a governing node id stamped // into context), so the coding/step session runs as the column agent under // the SAME binding lookup the custom-node seam uses (KTD-2 single resolver). this.graphColumnAgentResolver.set(task.id, resolveBindingForNode); // (U3) Genuinely-unattended run signal. This is an EXPLICIT opt-in, not an // inferred heuristic: a run is unattended only when an entrypoint that // knows no human will ever answer (LFG / pipeline / disable-model-invocation) // marks it so. No such marker reaches this executor path today (verified — // KTD-3), so this resolves to false (board run) for every current run, and // the safe default is preserved: absence of the explicit flag ALWAYS yields // no FUSION_HEADLESS, so a board task can only ever park (a human can answer // via the await-input card button), never silently skip approval. When such // an entrypoint is added, it sets `unattended` here. // No entrypoint sets this today, so clear any stale entry; a board run never // sets FUSION_HEADLESS. When an LFG/pipeline/disable-model-invocation // entrypoint is added, call `this.graphUnattendedRuns.add(task.id)` here and // the finally below clears it. this.graphUnattendedRuns.delete(task.id); graphAbortController = new AbortController(); this.activeWorkflowGraphAbortControllers.set(task.id, graphAbortController); const customNodeExecution = new WorkflowCustomNodeExecutionService({ execute: (node, nodeTask, nodeSettings, columnBinding, context) => this.runGraphCustomNode(node, nodeTask, nodeSettings, columnBinding, context), resolveColumnBinding: resolveBindingForNode, }); const runner = new WorkflowGraphTaskRunner({ localNodeId: this.options.getLocalNodeId?.(), store: { ...this.store, /* FNXC:WorkflowSelection 2026-07-14-17:06: Graph execution must reuse the asynchronously resolved selection. A PostgreSQL TaskStore cannot provide that selection through the synchronous compatibility method, and substituting builtin:coding here would silently execute the wrong graph. */ getTaskWorkflowSelection: () => selection, getTaskWorkflowSelectionAsync: async () => selection, getWorkflowDefinition: async (id: string) => (await this.store.getWorkflowDefinition?.(id)) ?? (id === "builtin:coding" ? getBuiltinWorkflow("builtin:coding") : undefined), getTask: (taskId: string) => this.store.getTask(taskId), }, runId: resolvedRunId, primitives: this.createAuthoritativeWorkflowPrimitives(settings), seams: this.createAuthoritativeWorkflowSeams(settings), prepareNodeExecution: (node, nodeTask, requirement) => this.prepareGraphNodeExecution(node, nodeTask, settings, requirement), runCustomNode: customNodeExecution.runner(settings), publishTaskProjection: async (taskId, patch) => { await this.store.updateTaskAtomic(taskId, (liveTask) => { const update: Parameters[1] = {}; if (patch.modifiedFiles) { const merged = [...new Set([...(liveTask.modifiedFiles ?? []), ...patch.modifiedFiles])].sort(); if (merged.length > 0) update.modifiedFiles = merged; } if (patch.mergeDetails) { update.mergeDetails = { ...(liveTask.mergeDetails ?? {}), ...patch.mergeDetails }; } if (patch.summary !== undefined) update.summary = patch.summary; return update; }); }, onEvent: (event) => executorLog.debug(`[workflow-graph] ${event.type} ${event.taskId}: ${event.detail}`), signal: graphAbortController.signal, // Wire SQLite-backed per-branch persistence in production (#1407): the // executor writes each branch's currentNodeId/status to // workflow_run_branches so fan-out crash-resume and the U9 badges have // real data, and prunes stale runs (#1412). Adapter degrades to no-op // when the store predates these methods (additive guard). branchPersistence: this.buildBranchPersistence(), // Step-inversion (KTD-6, U3/U4): per-instance run-state persistence. stepInstancePersistence: this.buildStepInstancePersistence(), // Step-inversion (KTD-4, U5): RETHINK reset-on-rework — when the foreach // sub-walk traverses a rework edge triggered by `outcome:rethink`, reset // the active instance's step to its persisted per-step baseline (git reset // + session rewind + step→pending) before re-entering step-execute. onReworkReset: (active) => this.applyGraphRethinkReset(task.id, active), // Step-inversion (KTD-12, U12): parse-steps node handler deps — artifact // read (through task-documents with PROMPT.md fallback), step-list write // (graph-source projection), pin-protection probe, and audit. parseStepsDeps: this.buildParseStepsDeps(resolvedRunId), // Step-inversion (KTD-15, U14): code node runner — esbuild compile + // child-process execution with the harness contract. runCode: this.buildCodeNodeRunner(), notifyDispatch: (event, payload) => getActiveNotificationService()?.dispatch(event, payload), // PR-entity nodes (U3): pr-create/pr-respond/pr-merge handler deps — // engine-owned store + CLI-injected GitHub callbacks. Absent → fail closed. prNodes: this.options.prNodes, // Step-inversion (KTD-11, U10): worktree isolation + ordered integration + // parallel scheduling. Per-instance worktrees branched off the task's main // branch tip; integration rebases each branch in step order; the projection // flips done-iff-integrated. Shared isolation never invokes these. ...this.buildForeachWorktreeDeps(task, resolvedRunId), // FIX 4 (context gap): task-level log sink so an integration-conflict // rework writes a visible "reworking on updated base (files: ...)" entry // the re-running agent can read. Best-effort; logging failures swallowed. logTaskEntry: (summary: string, detail?: string) => { void this.store .logEntry(task.id, summary, detail, this.getRunContextFor(task.id)) .catch(() => {}); }, /* FNXC:WorkflowStepResults 2026-06-25-12:00: Plan U2 (KTD-1/KTD-2): persistence adapter for an ENABLED optional-group node's outcome. The graph records each enabled group's WorkflowStepResult into the EXISTING `task.workflowStepResults` field keyed by `node.id` so the unified progress bar (getUnifiedTaskProgress) reflects graph-run steps — NO new table/type/store method. Upsert by `workflowStepId === node.id` (replace-if-present else append) through the existing `store.updateTask({workflowStepResults})` path. Fail-soft: degrade to a no-op when the store lacks updateTask, and swallow read/write errors (the executor wrapper also swallows) so result recording never affects the run. */ recordWorkflowStepResult: async (taskId: string, result: CoreWorkflowStepResult) => { if (typeof this.store.updateTask !== "function") return; try { const live = await this.store.getTask(taskId); /* FNXC:WorkflowStepResults 2026-07-09-00:25: FN-7727: route through the shared, pure upsert helper instead of a bare `existing[idx] = result` replace-in-place — a self-healing recovery re-run of this same node (e.g. code-review sent back for fix) must preserve the prior `status:"failed"` entry's history in `priorAttempts` rather than silently overwriting it. */ const existing = upsertWorkflowStepResult(live?.workflowStepResults, result); await this.store.updateTask(taskId, { workflowStepResults: existing }, this.getRunContextFor(taskId)); } catch { // Result recording is additive visibility — never affect the run. } }, requestPreMergeOptionalStepFix: (taskId, info) => this.requestPreMergeOptionalStepFix(taskId, task, info), // U5c (U1 KTD-1/2/3/12): wire the production lifecycle-move hooks so the // graph interpreter owns the card's column moves (was reverted in U5a // pending U6/U7 trait re-key; safe now). Absent → the graph performs no // lifecycle moves (pre-cutover byte-identical); present → the controller // moves the card on each node-column boundary with all move-safety. columnBoundaryHooks: this.buildColumnBoundaryHooks(task, resolvedRunId), }); let result: WorkflowGraphTaskRunResult; let continuation: WorkflowWorkItem | undefined; try { const loadedDetail = await this.store.getTask(task.id); /* FNXC:WorkflowExecution 2026-06-23-11:36: Graph dispatch must preserve the row identity that entered execute(). Minimal test stores and stale adapters can return an unrelated fallback task from getTask(); trusting that row would run the workflow under the wrong task id and bypass executor invariants. Use the refreshed row only when it matches the dispatch task. */ const detail: TaskDetail = loadedDetail?.id === task.id ? loadedDetail : { ...task, prompt: task.prompt ?? task.description ?? "" }; const workItems = await this.store.listWorkflowWorkItemsForTask?.(task.id, { kinds: ["task"] }) ?? []; for (let index = workItems.length - 1; index >= 0; index -= 1) { const candidate = workItems[index]; if (ACTIVE_WORKFLOW_WORK_ITEM_STATES.includes(candidate.state)) { continuation = candidate; break; } } if (continuation && continuation.state !== "running") { continuation = await this.store.transitionWorkflowWorkItem(continuation.id, "running", { leaseOwner: `executor:${task.id}`, leaseExpiresAt: null, lastError: null, }); } result = await runner.run(detail, settings, continuation?.nodeId); } catch (err) { if (continuation) { await this.store.transitionWorkflowWorkItem(continuation.id, "failed", { leaseOwner: null, leaseExpiresAt: null, lastError: "workflow-continuation-dispatch-failed", }).catch(() => undefined); } executorLog.error( `[workflow-graph] ${task.id} interpreter threw — parking task as workflow failure: ${err instanceof Error ? err.message : String(err)}`, ); await this.handleGraphFailure(task, { disposition: "failed", outcome: "failure", reason: `interpreter-error: ${err instanceof Error ? err.message : String(err)}`, visitedNodeIds: [], }); return; } if (result.disposition === "fell-back") { executorLog.warn(`[workflow-graph] ${task.id} could not resolve workflow — parking task instead of legacy fallback: ${result.reason}`); await this.handleGraphFailure(task, { ...result, disposition: "failed", outcome: "failure", reason: result.reason ?? "workflow-resolution-failed", }); return; } if (result.disposition === "suspended") { return; } if (result.disposition === "failed") { if (continuation) { await this.store.transitionWorkflowWorkItem(continuation.id, "failed", { leaseOwner: null, leaseExpiresAt: null, lastError: "workflow-continuation-failed", }); } await this.handleGraphFailure(task, result); } else if (result.disposition === "completed") { if (continuation) { await this.store.transitionWorkflowWorkItem(continuation.id, "succeeded", { leaseOwner: null, leaseExpiresAt: null, lastError: null, }); } const live = await this.store.getTask(task.id).catch(() => task); if ((live as TaskDetail).mergeDetails?.mergeConfirmed === true && (live as TaskDetail).column !== await resolveCompleteColumnFor(this.store, task.id)) { await this.finalizeMergeConfirmedWorkflowGraphTask(task.id, "graph-completed"); } await this.advanceNoMergeWorkflowToCompleteColumn(live as TaskDetail); if ((live.graphResumeRetryCount ?? 0) !== 0 || (live.consecutiveToolFailureRetryCount ?? 0) !== 0) { await this.store.updateTask(task.id, { graphResumeRetryCount: 0, consecutiveToolFailureRetryCount: 0, executorEscalationAttempted: false, toolFailureDetectorLogCursor: null, toolFailureRetryExhaustedAuditEmitted: false }, this.getRunContextFor(task.id)); } } return; } finally { // FNXC:WorkflowGraph 2026-06-20-23:35: // Terminate child agents spawned by this graph run's coding-mode skill steps. // U8 registered fn_spawn_agent for coding-mode steps, but the graph path // returns from execute() at the graphOwned early-return — BEFORE execute()'s // outer finally that calls terminateAllChildren. Without this, graph-step // children orphan their sessions/worktrees, and their ids accumulate in the // per-parent spawn budget (spawnedAgents[taskId]), starving later steps' // fan-out (e.g. ce-code-review's reviewer panel). Mirror the non-graph // cleanup; run it before the per-run graph bookkeeping below. try { await this.terminateAllChildren(task.id); } catch (err) { executorLog.warn(`terminateAllChildren failed for graph task ${task.id}: ${err instanceof Error ? err.message : String(err)}`); } if (hadPreHeldExecutorSlot) { this.outerConcurrencyClaims.delete(task.id); /* FNXC:GlobalConcurrencyControls 2026-07-19-17:40 (U10b): Always release. The `transferPreHeldToLegacy` branch — which re-registered the reserved global slot for a legacy execute path to pick up — died with that path: the graph can no longer decline ownership, so there is no second executor to hand the slot to. Holding the registration with nothing left to claim it would permanently reduce global capacity. */ this.options.semaphore?.release(); } if (graphAbortController && this.activeWorkflowGraphAbortControllers.get(task.id) === graphAbortController) { this.activeWorkflowGraphAbortControllers.delete(task.id); } this.graphRouting.delete(task.id); this.graphToolFailureRunCursors.delete(task.id); // Clear per-run step-inversion pins (KTD-8: pinned only for the run's life). this.graphStepSessionPinned.delete(task.id); this.graphStepRunOnce.delete(task.id); // Clear per-run column-agent seam wiring (U4): the resolver and any dangling // governing-node-id are scoped to this run only. this.graphColumnAgentResolver.delete(task.id); this.graphUnattendedRuns.delete(task.id); this.graphSeamGoverningNodeId.delete(task.id); this.graphSeamThinkingLevel.delete(task.id); this.graphSeamSkillName.delete(task.id); this.graphExecuteSelfRequeued.delete(task.id); // Per-instance keys: clear every instance slot owned by this task. const ctxPrefix = `${task.id}:`; for (const key of this.graphStepActiveContext.keys()) { if (key.startsWith(ctxPrefix)) this.graphStepActiveContext.delete(key); } for (const key of this.graphRethinkNarrations.keys()) { if (key.startsWith(ctxPrefix)) this.graphRethinkNarrations.delete(key); } } } /** * Build the store-backed WorkflowBranchPersistence wired into production * fan-out runs (#1407/#1412). Returns undefined when the store predates the * persistence methods (older embedded DBs) so the runner stays fully * in-memory — purely additive. Each adapter method is itself guarded so a * mixed/partial store never throws into the run. */ private buildBranchPersistence(): WorkflowBranchPersistence | undefined { // FNXC:PostgresOnlyDataAccess 2026-07-16-12:40: the store methods are now // async (PostgreSQL routing); the persistence interfaces already accept // Promise-returning impls and await them. const store = this.store as unknown as { saveWorkflowRunBranch?: (state: WorkflowBranchRunState) => void | Promise; loadWorkflowRunBranches?: (taskId: string, runId: string) => WorkflowBranchRunState[] | Promise; clearWorkflowRunBranches?: (taskId: string, keepRunId: string) => void | Promise; }; if (typeof store.saveWorkflowRunBranch !== "function") return undefined; return { saveBranchState: (state) => store.saveWorkflowRunBranch?.(state), loadBranchStates: async (taskId, runId) => (await store.loadWorkflowRunBranches?.(taskId, runId)) ?? [], clearStaleBranchStates: (taskId, keepRunId) => store.clearWorkflowRunBranches?.(taskId, keepRunId), }; } /** * Build the store-backed WorkflowStepInstancePersistence for graph-owned * foreach runs (KTD-6, U3/U4 seam). Returns undefined when the store predates * the instance CRUD methods (the SQLite migration is U4) so the sub-walk stays * fully in-memory — purely additive, same posture as buildBranchPersistence. */ private buildStepInstancePersistence(): WorkflowStepInstancePersistence | undefined { // FNXC:PostgresOnlyDataAccess 2026-07-16-12:40: async store methods; the // persistence interface awaits Promise-returning impls. const store = this.store as unknown as { saveWorkflowRunStepInstanceAsync?: (state: WorkflowStepInstanceState) => Promise; loadWorkflowRunStepInstancesAsync?: (taskId: string, runId: string) => Promise; clearWorkflowRunStepInstancesAsync?: (taskId: string, keepRunId: string) => Promise; saveWorkflowRunStepInstance?: (state: WorkflowStepInstanceState) => void; loadWorkflowRunStepInstances?: (taskId: string, runId: string) => WorkflowStepInstanceState[]; clearWorkflowRunStepInstances?: (taskId: string, keepRunId: string) => void; }; if (typeof store.saveWorkflowRunStepInstanceAsync !== "function" && typeof store.saveWorkflowRunStepInstance !== "function") return undefined; return { saveInstanceState: (state) => store.saveWorkflowRunStepInstanceAsync?.(state) ?? store.saveWorkflowRunStepInstance?.(state), loadInstanceStates: async (taskId, runId) => await store.loadWorkflowRunStepInstancesAsync?.(taskId, runId) ?? store.loadWorkflowRunStepInstances?.(taskId, runId) ?? [], clearStaleInstanceStates: (taskId, keepRunId) => store.clearWorkflowRunStepInstancesAsync?.(taskId, keepRunId) ?? store.clearWorkflowRunStepInstances?.(taskId, keepRunId), }; } /* FNXC:WorkflowLifecycle 2026-07-18-14:20 (U5c / U1 KTD-1/2/3/12): Build the PRODUCTION column-boundary hooks for one graph run. This is the piece that makes the graph the single source of truth for lifecycle MOVES: as the interpreter enters each node, the controller (createWorkflowColumnBoundary) moves the card to the node's trait column via these hooks. All the move-safety lives in the controller (same-column no-op, KTD-2 hold→wip parked for the scheduler, rejected-move leaves the card in place), so the executor only supplies the raw seams: - moveTask → real store.moveTask, engine-sourced with workflowMoveSource so the move is attributed to the graph; bypassGuards (KTD-9: the graph IS the lifecycle owner, so its own moves must not be re-vetoed by the same trait guards it implements — capacity KTD-10 is still enforced by moveTask). - emitAudit → ids/counts-only run-audit (KTD-12) for column-transition/drift. - onWarn → executor log sink; diagnostics never affect the run. FNXC:WorkflowIrPin 2026-07-19-18:30 (KTD-3 / U9b): The KTD-3 durable IR pin is WIRED: the U9b store schema landed the pin as task-row fields (workflowIrPin/workflowIrPinNodeId/workflowIrPinColumnId, migration 0026), so pinNodeEntry/loadPriorPin bind to that row via createStoreIrPinPersistence. Each real node entry persists the resolved IR's content hash (change-only writes); on restart/ re-entry the runner loads the prior pin and detectDrift parks the run with task:reconcile-workflow-drift when the pinned node/column is gone or the hash no longer resolves, instead of traversing a mutated graph. Stores without the fields (in-memory fakes, pre-U9b DBs) degrade to the previous inert no-pin posture. */ /* FNXC:WorkflowNoMergeCompletion 2026-07-19-12:40: A workflow with NO merge region had no way to reach its `complete` column. `end` is a graph terminal, never a column destination (KTD-1 — the boundary deliberately does not fire on it), so a card only lands in the complete column when a REAL node lives there. Every merge-bearing built-in gets that for free from `post-merge-verification`; a no-merge workflow does not. The two existing movers to the complete column (merger.completeTask, finalizeProvenAutoMergeTask) are both merge-proof-gated and unreachable without a merge, and the merge queue is only fed on entry to `in-review` — a column a no-merge workflow need not even declare. Net effect before this: a `builtin:lead-generation` card completed its whole graph and then sat in `outreach` forever, and its dependents never released because `complete` never became true for it. This is the trait-keyed completion mover for exactly that class. It is deliberately narrow: - it fires ONLY when the IR declares no merge-orchestration column, so every merge-bearing workflow (builtin:coding included) is byte-identical — the merge path keeps sole ownership of complete-column entry and the done-only-on-confirmed-merge invariant is untouched; - it does NOT reintroduce a move on `end`; KTD-1 stands; - an IR with no complete-trait column is a legal shape and no-ops here; - a task with no worktree (the normal case for a no-merge workflow) is not an error — nothing about the move depends on one. */ private async advanceNoMergeWorkflowToCompleteColumn(task: TaskDetail): Promise { let ir: WorkflowIr; try { ir = await resolveWorkflowIrForTask(this.store, task.id); } catch { // IR resolution is best-effort here: a card that already finished its graph // must never be failed by a bookkeeping lookup. return; } // Merge-bearing workflow → the merge path owns the complete column. Return // before reading anything else so this branch is provably inert for them. if (resolveMergeOrchestrationColumn(ir) !== undefined) return; const completeColumn = resolveCompleteColumn(ir); if (!completeColumn || completeColumn === task.column) return; try { /* * The normal move path first. A no-merge workflow's last real node sits in * the column immediately before the complete column, but the graph's own * column adjacency is derived from node placement — and NOTHING is placed * in the complete column (that is the whole gap), so `resolveAllowedColumns` * cannot see the edge and the shared validator rejects it. `bypassGuards` * is therefore required for adjacency alone; every other guard the flag * relaxes (merge-blocker in particular) is vacuous here because this branch * only runs for a workflow with no merge region at all. */ await this.store.moveTask(task.id, completeColumn, { moveSource: "engine", workflowMoveSource: "workflow-graph", bypassGuards: true, preserveProgress: true, workflowMoveMetadata: { fromColumn: task.column, reason: "no-merge-workflow-completed" }, }); } catch (err) { executorLog.warn( `[workflow-graph] ${task.id} completed a no-merge workflow but could not advance to '${completeColumn}': ${err instanceof Error ? err.message : String(err)}`, ); return; } // ids/outcomes-only metadata — no prose, no node/run internals. await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("workflow-no-merge-completion", task.id), domain: "database", mutationType: "task:workflow-complete-column-advanced", target: task.id, metadata: { taskId: task.id, fromColumn: task.column, toColumn: completeColumn, reason: "no-merge-workflow-completed" }, }); } /* FNXC:WorkflowColumnBoundary 2026-07-27-16:40 (PR #2475 review, P2): The wiring itself now lives in `createExecutorColumnBoundaryHooks` so the E2E suite can drive the REAL hooks instead of rebuilding them (a hand copy had already diverged in three places). What stays here is only genuine Executor state: the in-flight graph-move marker and the logger. */ private buildColumnBoundaryHooks(task: Pick, workflowRunId?: string): WorkflowColumnBoundaryHooks { return createExecutorColumnBoundaryHooks({ store: this.store, task, workflowRunId, markMoveInFlight: (taskId) => this.workflowLifecycleMovesInFlight.add(taskId), clearMoveInFlight: (taskId) => this.workflowLifecycleMovesInFlight.delete(taskId), onWarn: (message, detail) => { executorLog.debug(`[workflow-column-boundary] ${task.id}: ${message} ${JSON.stringify(detail)}`); }, }); } /** * Resolve which artifact/parser governs a graph-owned task's step list from its * workflow's `parse-steps` declaration (KTD-12). Returns undefined for legacy * tasks (no parse-steps node) so reconcile/resume keep their unchanged behavior. * Used by reconcile read-through to know which artifact backs the step source. */ private resolveTaskStepSource(ir: WorkflowIr | undefined): { artifact: string; parser: string } | undefined { if (!ir) return undefined; for (const node of ir.nodes) { if (node.kind !== "parse-steps") continue; const cfg = (node.config ?? {}) as { artifact?: unknown; parser?: unknown }; const parser = typeof cfg.parser === "string" ? cfg.parser : undefined; if (!parser) continue; const artifact = typeof cfg.artifact === "string" && cfg.artifact.trim() !== "" ? cfg.artifact : "PROMPT.md"; return { artifact, parser }; } return undefined; } /** * Resolve the custom field definitions declared by a task's selected workflow * (KTD-13) so the executor prompt can surface the schema and current values to * the agent. Pure read; degrades to undefined on any resolution failure (no * selection, missing/corrupt definition, older store) so prompt-building never * throws and legacy tasks see no custom-fields section. */ private async resolveTaskCustomFieldDefs(taskId: string): Promise { try { const ir = await resolveWorkflowIrForTask(this.store, taskId); const fields = ir.version === "v2" ? ir.fields : undefined; return fields && fields.length > 0 ? fields : undefined; } catch { return undefined; } } /** * Build the parse-steps node handler deps (KTD-12, U12): artifact read through * the task-documents machinery (PROMPT.md falls back to the task's own PROMPT * content the way step-init does), step-list write through the graph-source * projection (`updateTask({ steps })`), pin-protection probe (persisted instance * rows exist → re-parse illegal, KTD-3), and a logEntry-backed audit sink. */ /** * Read a task artifact by key through the task-documents layer, falling back to * the task's own PROMPT content for the default `PROMPT.md` step-source artifact * (the same source the legacy step-init reads). Shared by the parse-steps and * code-node deps (FIX 7: one source of truth for the fallback). */ private async readTaskArtifact(taskId: string, key: string): Promise { // Declared artifacts ride the task-documents layer. let documentReadError: unknown; try { const doc = await this.store.getTaskDocument(taskId, key); if (doc) return doc.content; } catch (error) { documentReadError = error; } if (key === "PROMPT.md") { try { const detail = await this.store.getTask(taskId); if (typeof detail.prompt === "string") return detail.prompt; return undefined; } catch (error) { throw new Error( `Unable to read required artifact ${key} from task documents or task storage: ${error instanceof Error ? error.message : String(error)}`, { cause: documentReadError ?? error }, ); } } if (documentReadError) throw documentReadError; return undefined; } private buildParseStepsDeps(runId?: string): ParseStepsHandlerDeps { return { readArtifact: (task, key): Promise => this.readTaskArtifact(task.id, key), writeSteps: async (task, steps: TaskStep[]): Promise => { await this.store.updateTask(task.id, { steps }); }, hasExpandedForeach: async (task): Promise => { const store = this.store as unknown as { loadWorkflowRunStepInstancesAsync?: (taskId: string, runId: string) => Promise; loadWorkflowRunStepInstances?: (taskId: string, runId: string) => WorkflowStepInstanceState[]; }; if (typeof store.loadWorkflowRunStepInstancesAsync !== "function" && typeof store.loadWorkflowRunStepInstances !== "function") return false; try { // Any persisted instance row for THIS run means a foreach has expanded — // re-parsing would desynchronize the pinned instance set (KTD-3). Probe // under the REAL run id (threaded from executeWorkflowGraph) so the // pin protection actually fires; fall back to the legacy literal only when // the run id was not threaded (older store / no definition). const rows = await store.loadWorkflowRunStepInstancesAsync?.(task.id, runId ?? `${task.id}:run`) ?? store.loadWorkflowRunStepInstances?.(task.id, runId ?? `${task.id}:run`) ?? []; return rows.length > 0; } catch { return false; } }, audit: (reason, detail) => { // The detail string carries the task id (handler convention); emit on the // engine log so the routable failure is auditable without a taskId arg. executorLog.warn(`[parse-steps] ${reason}: ${detail}`); }, }; } /** * Build the code node runner (KTD-15, U14): worktree cwd resolution, pre-read of * declared artifacts into the harness ctx, and customFields writes through the * U11 validation authority. Drives the esbuild-compile + child-process runner * in code-node-runner.ts. */ private buildCodeNodeRunner(): CodeNodeRunner { return createCodeNodeRunner({ resolveCwd: async (task): Promise => { try { return (await this.store.getTask(task.id)).worktree || this.rootDir; } catch { return this.rootDir; } }, readArtifacts: async (task): Promise> => { const out: Record = {}; try { const docs = await this.store.getTaskDocuments(task.id); for (const doc of docs) out[doc.key] = doc.content; } catch { // No documents — pass an empty artifact map. } // Surface PROMPT.md from the task prompt when not already a document // (shared artifact-read fallback — FIX 7). if (out["PROMPT.md"] === undefined) { const prompt = await this.readTaskArtifact(task.id, "PROMPT.md"); if (typeof prompt === "string") out["PROMPT.md"] = prompt; } return out; }, writeCustomFields: async (task, patch) => { if (typeof this.store.updateTaskCustomFields !== "function") { return { ok: false as const, rejection: { code: "no-fields-defined" as const, fieldId: "", detail: "custom fields unsupported by store" }, }; } const result = await this.store.updateTaskCustomFields(task.id, patch); return result.ok ? { ok: true as const } : { ok: false as const, rejection: result.rejection }; }, audit: (reason, detail) => { executorLog.warn(`[code-node] ${reason}: ${detail}`); }, }); } /** * Build the worktree-isolation + ordered-integration + parallel-scheduling deps * for a graph-owned foreach (KTD-11, U10). Returns the additive set the * WorkflowGraphTaskRunner forwards to the foreach sub-walk: * * - `allocateInstanceWorktree(i, base)` — a per-instance worktree on a * canonical `fusion/-step-` branch off `base` (the main tip), * created via the existing `createWorktree` path (the file-scope guard the * session machinery installs applies unchanged to anything the instance * session commits in this worktree — we do NOT bypass it); * - `resolveIntegrationBase()` — the task's main branch tip, re-read before each * (re)allocation so a rework lands on the UPDATED base; * - `integrationGitOps` — rebase the instance branch onto the main branch in * the task's MAIN worktree, fast-forward main on success; on conflict reuse * merger.ts `getConflictedFiles` (NOT reimplemented) and abort the rebase so * the next instance can integrate; `discardBranch` deletes the branch + frees * the instance worktree (pool hygiene); * - `integrationProjection` — projection-first ordering (KTD-7): `markStepDone` * flips the step `done` via `updateStep(source:"graph")` (the dependency-order * guard admits it), THEN `markInstanceIntegrated` flips the persisted row; * - `semaphoreAvailability` — the live free-slot count so parallel scheduling * clamps without hold-and-wait. * * Best-effort throughout: a git failure routes the foreach to a clean failure * (parked for human review) rather than crashing the run. */ private buildForeachWorktreeDeps(task: Task, runId?: string): { allocateInstanceWorktree: ( stepIndex: number, base: string | undefined, ) => Promise<{ worktreePath: string; branchName: string }>; resolveIntegrationBase: () => Promise; integrationGitOps: import("./step-integration.js").IntegrationGitOps; integrationProjection: import("./step-integration.js").IntegrationProjection; semaphoreAvailability: () => number; resumeReconcile: ( pinned: number, ) => Promise>; } { const taskId = task.id; // Per-instance worktree paths, so discard can free them. const instancePaths = new Map(); const mainWorktree = async (): Promise => { try { return (await this.store.getTask(taskId)).worktree || this.rootDir; } catch { return this.rootDir; } }; const mainBranch = async (): Promise => { try { const detail = await this.store.getTask(taskId); return resolveTaskWorkingBranch(detail); } catch { return resolveTaskWorkingBranch(task); } }; return { resolveIntegrationBase: async (): Promise => { // The main branch tip (HEAD of the task's working branch in its worktree). try { const { stdout } = await execAsync("git rev-parse HEAD", { cwd: await mainWorktree() }); const sha = stdout.trim(); return sha.length > 0 ? sha : await mainBranch(); } catch { return await mainBranch(); } }, allocateInstanceWorktree: async (stepIndex, base): Promise<{ worktreePath: string; branchName: string }> => { const branchName = canonicalStepInstanceBranchName(taskId, stepIndex); const worktreePath = resolveTaskWorktreePath( this.rootDir, undefined, `${taskId.toLowerCase()}-step-${stepIndex}`, ); // createWorktree installs the file-scope guard (session machinery, // unchanged) and branches off `base` (the integration base / updated tip). const created = await this.createWorktree(branchName, worktreePath, taskId, base); instancePaths.set(stepIndex, created.path); return { worktreePath: created.path, branchName: created.branch }; }, integrationGitOps: { integrate: async (branchName, stepIndex): Promise => { const cwd = await mainWorktree(); const target = await mainBranch(); // The instance branch is checked out in its OWN worktree, so the rebase // (which checks out `branchName`) must run THERE — running it from the // main worktree fails with "branch is already checked out in another // worktree". The final fast-forward merge still runs from the main // worktree (it only advances `target`, which is checked out there). const instanceCwd = instancePaths.get(stepIndex) ?? cwd; try { // Rebase the instance branch onto the current main tip (in its own // worktree), then ff main from the main worktree. await execAsync(`git rebase ${target} ${branchName}`, { cwd: instanceCwd }); await execAsync(`git checkout ${target}`, { cwd }); await execAsync(`git merge --ff-only ${branchName}`, { cwd }); return { kind: "integrated", integratedAt: new Date().toISOString() }; } catch (err) { // Conflict (or other rebase failure): classify via merger helper, abort. // The rebase ran in the instance worktree, so conflicts live there and // the abort must target that same cwd. const conflictedFiles = await getConflictedFiles(instanceCwd); try { await execAsync("git rebase --abort", { cwd: instanceCwd }); } catch { // best-effort; leave the worktree recoverable. } // Restore main checkout so the next instance integrates cleanly. try { await execAsync(`git checkout ${target}`, { cwd }); } catch { // best-effort. } executorLog.warn( `[step-integration] ${taskId} step ${stepIndex} branch ${branchName} conflict: ${err instanceof Error ? err.message : String(err)}`, ); return { kind: "conflict", conflictedFiles }; } }, discardBranch: async (branchName, stepIndex): Promise => { const cwd = await mainWorktree(); const path = instancePaths.get(stepIndex); if (path) { // Remove the instance worktree (pool hygiene). Best-effort; force so a // dirty/conflicting tree is still cleaned up. try { await execAsync(`git worktree remove --force "${path}"`, { cwd: this.rootDir }); } catch { // best-effort cleanup. } instancePaths.delete(stepIndex); } // Delete the (now-merged or conflicting) branch. try { await execAsync(`git branch -D ${branchName}`, { cwd }); } catch { // best-effort — the branch may already be gone. } }, }, integrationProjection: { markStepDone: async (stepIndex): Promise => { // Projection-first (KTD-7): graph-source write relaxes the guard to // dependency order; predecessors are integrated (done) by construction. await this.store.updateStep(taskId, stepIndex, "done", { source: "graph" }); }, markInstanceIntegrated: async (stepIndex, integratedAt, identity): Promise => { const store = this.store as unknown as { saveWorkflowRunStepInstanceAsync?: (state: WorkflowStepInstanceState) => Promise; loadWorkflowRunStepInstancesAsync?: (taskId: string, runId: string) => Promise; saveWorkflowRunStepInstance?: (state: WorkflowStepInstanceState) => void; loadWorkflowRunStepInstances?: (taskId: string, runId: string) => WorkflowStepInstanceState[]; }; if (typeof store.saveWorkflowRunStepInstanceAsync !== "function" && typeof store.saveWorkflowRunStepInstance !== "function") return; // The upsert is keyed by (taskId, runId, foreachNodeId, stepIndex). The // queue passes the REAL identity (the same runId + foreachNodeId the // foreach sub-walk persisted the row under) so this FLIPS the existing // row to completed/integratedAt instead of writing an orphan (FIX 1). // Load the current row to preserve its fields (currentNodeId, baseline, // reworkCount) we don't otherwise carry on the identity. let existing: WorkflowStepInstanceState | undefined; try { const rows = await store.loadWorkflowRunStepInstancesAsync?.(taskId, identity.runId) ?? store.loadWorkflowRunStepInstances?.(taskId, identity.runId) ?? []; existing = rows.find( (r) => r.foreachNodeId === identity.foreachNodeId && r.stepIndex === stepIndex, ); } catch { // Best-effort read; fall back to a minimal flip below. } try { await (store.saveWorkflowRunStepInstanceAsync?.({ ...(existing ?? {}), taskId, runId: identity.runId, foreachNodeId: identity.foreachNodeId, stepIndex, pinnedStepCount: identity.pinnedStepCount, currentNodeId: existing?.currentNodeId ?? "", status: "completed", reworkCount: existing?.reworkCount ?? 0, branchName: identity.branchName || canonicalStepInstanceBranchName(taskId, stepIndex), integratedAt, } as WorkflowStepInstanceState) ?? store.saveWorkflowRunStepInstance?.({ ...(existing ?? {}), taskId, runId: identity.runId, foreachNodeId: identity.foreachNodeId, stepIndex, pinnedStepCount: identity.pinnedStepCount, currentNodeId: existing?.currentNodeId ?? "", status: "completed", reworkCount: existing?.reworkCount ?? 0, branchName: identity.branchName || canonicalStepInstanceBranchName(taskId, stepIndex), integratedAt, } as WorkflowStepInstanceState)); } catch { // Persistence is additive bookkeeping — never fail the integration. } }, }, semaphoreAvailability: (): number => this.options.semaphore?.availableCount ?? 1, resumeReconcile: async ( pinned, ): Promise> => { // Crash-resume reconciliation (KTD-11): reconcile each persisted instance // row against branch existence. integrated → done; branch exists not // integrated → re-enter the integration queue; branch missing → re-run. // NOTE (handoff): this is the per-run resume seeding only; the full // self-healing sweep across stale runs (recoverStaleTransitionPending // analogue) is out of scope for U10. const store = this.store as unknown as { loadWorkflowRunStepInstancesAsync?: (taskId: string, runId: string) => Promise; loadWorkflowRunStepInstances?: (taskId: string, runId: string) => WorkflowStepInstanceState[]; }; if (typeof store.loadWorkflowRunStepInstancesAsync !== "function" && typeof store.loadWorkflowRunStepInstances !== "function") return []; let rows: WorkflowStepInstanceState[] = []; try { // Load under the REAL run id (threaded) so resume actually sees the rows // the sub-walk persisted; the legacy literal is the unthreaded fallback. rows = await store.loadWorkflowRunStepInstancesAsync?.(taskId, runId ?? `${taskId}:run`) ?? store.loadWorkflowRunStepInstances?.(taskId, runId ?? `${taskId}:run`) ?? []; } catch { return []; } const cwd = await mainWorktree(); const out: Array<{ stepIndex: number; disposition: "integrated" | "reintegrate" | "rerun"; branchName?: string }> = []; for (const row of rows) { if (row.stepIndex < 0 || row.stepIndex >= pinned) continue; if (row.status === "completed" || row.integratedAt) { out.push({ stepIndex: row.stepIndex, disposition: "integrated" }); continue; } const branchName = row.branchName || canonicalStepInstanceBranchName(taskId, row.stepIndex); let branchExists = false; try { await execAsync(`git rev-parse --verify --quiet ${branchName}`, { cwd }); branchExists = true; } catch { branchExists = false; } if (branchExists && row.status === "awaiting-integration") { out.push({ stepIndex: row.stepIndex, disposition: "reintegrate", branchName }); } else { out.push({ stepIndex: row.stepIndex, disposition: "rerun" }); } } return out; }, }; } /** * RETHINK reset-on-rework (KTD-4, U5): reset the active foreach instance's step * to its per-step baseline before the rework edge re-enters step-execute. Drives * the single extracted `resetStepToBaseline` (step-runner.ts) with the * instance's persisted `baselineSha`/`checkpointId`. Session rewind is best-effort * for graph-owned runs (the per-step session lives inside StepSessionExecutor and * is not exposed as a single ref here) — missing-checkpoint partial recovery is * the documented KTD-2 semantics; the git reset + step→pending are authoritative. */ private async applyGraphRethinkReset(taskId: string, active: ForeachActiveContext): Promise { // Clear the memoized implementation pass so the next `runGraphTaskStep` // re-executes (T9): the per-run pass is memoized in `graphStepRunOnce` keyed // by task id and is normally only cleared on REJECTION. A RETHINK fires AFTER // a SUCCESSFUL pass (a review verdict resets git/step state via this reset), // so without clearing the memo the rework re-awaits the already-resolved // promise and implementation never re-runs — leaving the instance permanently // pending or falsely successful under `deferDoneToReview`. Mirrors the // rejection-clear guard: only delete the memo when the stored promise is the // SETTLED pass (a fresh in-flight attempt another caller installed is left // untouched). At rethink time the pass under review has already resolved, so // checking settled-ness avoids clobbering a concurrent re-dispatch. const memo = this.graphStepRunOnce.get(taskId); if (memo) { let settled = false; await Promise.race([memo.then( () => { settled = true; }, () => { settled = true; }, ), Promise.resolve()]); if (settled && this.graphStepRunOnce.get(taskId) === memo) { this.graphStepRunOnce.delete(taskId); } } // Worktree isolation (KTD-11): reset the instance's OWN branch/worktree only — // sibling instances and the integration base are untouched, so the blast-radius // guard is STRUCTURAL (skipped) in this mode. Shared isolation resets the task's // main worktree and keeps the KTD-2 ancestry guard as written. const branchScoped = typeof active.worktreePath === "string" && active.worktreePath.length > 0; let worktreePath = active.worktreePath ?? this.rootDir; if (!branchScoped) { try { worktreePath = (await this.store.getTask(taskId)).worktree || this.rootDir; } catch { // Best-effort worktree resolution; fall back to rootDir. } } const liveSteps = await this.store.getTask(taskId).then((t) => t.steps).catch(() => []); const narrationKey = this.graphActiveContextKey(taskId, active.instanceId); const reviewSummary = this.graphRethinkNarrations.get(narrationKey); try { await resetStepToBaseline( { store: this.store, worktreePath, // No single session ref for graph-owned step-sessions — rewind is skipped // when checkpointId resolves but no session is current (KTD-2 partial path). sessionRef: { current: null }, reviewType: "code", // Branch-scoped RETHINK under worktree isolation makes the guard structural // (the reset can only touch the instance's own branch); shared isolation // keeps the defensive ancestry guard (KTD-2/KTD-11). blastRadiusGuard: branchScoped ? undefined : makeAncestryBlastRadiusGuard({ worktreePath, task: { id: taskId, steps: liveSteps }, stepIndex: active.stepIndex, }), }, { id: taskId, steps: liveSteps }, active.stepIndex, active.baselineSha, active.checkpointId, ); if (reviewSummary !== undefined) { const narration = buildReviewVerdictMessage("RETHINK", reviewSummary); void emitProactiveStatus(this.store, taskId, narration, "reviewer", sanitizeFailureReason(reviewSummary)); } } catch (error) { const safeReason = sanitizeFailureReason(error); void emitProactiveStatus( this.store, taskId, buildReviewRollbackFailureMessage(safeReason), "reviewer", safeReason, ); throw error; } finally { this.graphRethinkNarrations.delete(narrationKey); } } /** * Run ONLY the implementation phase for a graph-driven task — full setup plus the * agent session up to fn_task_done, stopping at the implementation-complete boundary * so the graph owns workflow gates, review, and merge. * * FNXC:WorkflowExecution 2026-07-19-02:10: * U5e (R9) — this now calls `runImplementation()` DIRECTLY. It used to re-enter * `execute()`, which meant every graph-driven implementation pass made a second trip * through routing (dependency/ephemeral gates, graph-routing duplicate check, * authoritative dispatch) that had to be suppressed by a signal. There is no re-entry * left: routing runs once, in `executeCore`, and the graph calls the runner. */ private async runImplementationPhase( task: Task, prepared?: PreparedWorktree, ): Promise<{ taskDone: boolean; modifiedFiles: string[]; exit?: ImplementationExit }> { let captured: { taskDone: boolean; modifiedFiles: string[]; exit?: ImplementationExit } = { taskDone: false, modifiedFiles: [] }; const graphCompletion: GraphCompletionCallback = (info) => { captured = { ...captured, taskDone: true, modifiedFiles: info.modifiedFiles }; }; /* Recorded independently of `graphCompletion`: the out-of-band exits never call it. */ const reportExit: ImplementationExitReporter = (exit) => { captured = { ...captured, exit }; }; const executionTask = prepared ? { ...task, worktree: prepared.worktreePath || task.worktree, branch: prepared.branchName || task.branch, } : task; await this.runImplementation(executionTask, graphCompletion, reportExit); return captured; } /** * Step-inversion per-step driver (KTD-2/KTD-8, closes the U3 interim gap). * * The U3 stand-in ran `runImplementationPhase` once per foreach instance, which * re-ran the whole implementation for every step. The real driver: * * 1. Pins step-session physics only when the workflow needs a discrete * per-step boundary before a step-review node. Final-review coding lets * `runStepsInNewSessions` choose between one reused executor session and * fresh per-step sessions. * 2. Drives the implementation phase exactly ONCE per run, memoized by task * id. Each foreach instance's `runTaskStep` observes projection truth for * its step rather than re-running the agent per step. * * Worktree/taskEnv/agent/semaphore state is threaded exactly the way * `runImplementationPhase` gets it — by re-entering `execute()` under a * completion interceptor — because that state is assembled inside `execute()` * and is not available standalone at createGraphSeams time (the plan's * documented threading approach for full step-session wiring). * * Returns whether the targeted step ended up `done`/`skipped` in the projection. */ private async runGraphTaskStep( task: Task, stepIndex: number, instanceId?: string, governingNodeId?: string, thinkingLevel?: ThinkingLevel, skillName?: string, ): Promise<{ success: boolean; error?: string; exit?: ImplementationExit }> { const active = this.foreachActiveForTask(task.id, instanceId); /* FNXC:WorkflowStepSessions 2026-06-30-00:00: Default Coding is graph-owned stepwise execution without per-step review. It should reuse the existing executor session when the workflow setting `runStepsInNewSessions` is false, and create fresh step sessions only when that setting is true. Workflows with a step-review node still pin StepSessionExecutor because review must run between step execution and done-marking. */ if (active?.deferDoneToReview === true) { this.graphStepSessionPinned.add(task.id); } // Single-flight per attempt (KTD-2/KTD-8): the implementation phase runs once // per run, memoized by task id, so each foreach instance's `runStep` observes // the projection rather than re-running the agent. A REJECTED phase must NOT // poison later attempts: a rework cycle re-enters `runStep` and would otherwise // re-await the same stored rejection forever, so the implementation is never // retried. On rejection we therefore clear the memo entry so the NEXT call // (the rework re-run) re-invokes the implementation phase. Concurrent // in-flight callers within a single attempt still share the one promise. let phase = this.graphStepRunOnce.get(task.id); if (!phase) { // Column-agent governing-node ownership (PR #1432 review): the slot is // written ONLY by the caller that CREATES the memoized pass, and cleared // when that pass settles. One step-session pass serves every foreach // instance, so the session-identity binding is the pass-INITIATING // instance's — deterministic, instead of concurrent seam invocations // racing set/delete on a shared per-task slot (parallel foreach could // otherwise stamp another instance's node mid-build or clear it before // the session resolved the binding). if (typeof governingNodeId === "string") { this.graphSeamGoverningNodeId.set(task.id, governingNodeId); } if (thinkingLevel) { this.graphSeamThinkingLevel.set(task.id, thinkingLevel); } if (skillName) { this.graphSeamSkillName.set(task.id, skillName); } phase = this.runImplementationPhase(task); this.graphStepRunOnce.set(task.id, phase); void phase .catch(() => undefined) .finally(() => { // Clear only our own stamp — a rework re-run may have installed a new one. if (typeof governingNodeId === "string" && this.graphSeamGoverningNodeId.get(task.id) === governingNodeId) { this.graphSeamGoverningNodeId.delete(task.id); } if (thinkingLevel && this.graphSeamThinkingLevel.get(task.id) === thinkingLevel) { this.graphSeamThinkingLevel.delete(task.id); } if (skillName && this.graphSeamSkillName.get(task.id) === skillName) { this.graphSeamSkillName.delete(task.id); } }); } /* FNXC:WorkflowExecutionOwnership 2026-07-29-11:20 (U8 / R4): The memoized pass's result was awaited and DISCARDED here — which is exactly where the implementation exit died. One pass serves every foreach instance, so the exit is a property of the pass, not of a step: each instance reports the same ending, which is correct because the ending is what stopped the whole session. */ let phaseResult: { taskDone: boolean; modifiedFiles: string[]; exit?: ImplementationExit } | undefined; try { phaseResult = await phase; } catch (err) { // Clear the poisoned memo so a rework cycle can retry the implementation // (only if it is still the same rejected promise — do not clobber a fresh // attempt another caller may have already installed). if (this.graphStepRunOnce.get(task.id) === phase) { this.graphStepRunOnce.delete(task.id); } /* FNXC:WorkflowExecution 2026-06-29-09:01: Stepwise graph execution is projection-driven: a shared implementation pass can complete every task step and pass deterministic verification without using the legacy monolithic `task_done` sentinel. If the target step is already terminal in Task.steps[], the workflow node succeeds and the graph continues to its review/merge nodes instead of converting stale legacy completion failure into `steps#N:step-execute`. */ try { const live = await this.store.getTask(task.id); const status = live.steps[stepIndex]?.status; if (status === "done" || status === "skipped") { executorLog.warn( `${task.id}: graph step ${stepIndex} completed in projection despite implementation-pass error; continuing workflow (${err instanceof Error ? err.message : String(err)})`, ); return { success: true }; } } catch { // Fall through to the original failure value below. } return { success: false, error: err instanceof Error ? err.message : String(err) }; } // Consult the projection (the single source of truth, KTD-7) for this step's // terminal state. The step-session pass marks each step done/skipped as it // completes; a step-review node (when present) decides done-ness instead. try { const live = await this.store.getTask(task.id); if (!live || live.id !== task.id) { return { success: false, error: `step ${stepIndex} live task unavailable after implementation pass`, }; } const status = live.steps[stepIndex]?.status; /* FNXC:WorkflowExecutionOwnership 2026-07-29-14:10 (U8 / R4, PR #2546 review — greptile P2): Carry the pass's ending on the SUCCESS returns too. One pass serves every foreach instance, so "this step completed" and "the pass stopped on a pending-review block" are independent facts and both can hold. Reporting only on failure made the exit branch-dependent: with `deferDoneToReview` every instance returns success, so the ending would never reach the seam and the graph-owned park would be unreachable for that shape. The seam still routes it only on FAILURE — a genuinely completed step must not be diverted to the park — so this is inert today and correct once the seam flip lands. */ if (status === "done" || status === "skipped") return { success: true, exit: phaseResult?.exit }; // Step not terminal after the pass: when a review will author done-ness // (deferDoneToReview), the pass having RUN is the success signal — the review // gates the projection write. Otherwise the implementation pass failed to // complete this step, so report failure rather than masking it (FIX 3: the // prior code returned success on both branches, hiding step-session failures). if (active?.deferDoneToReview === true) return { success: true, exit: phaseResult?.exit }; return { success: false, exit: phaseResult?.exit, error: `step ${stepIndex} not completed by implementation pass (status: ${status ?? "unknown"})`, }; } catch (err) { return { success: false, error: err instanceof Error ? err.message : String(err) }; } } /** Read the active foreach instance context for a graph-owned task (if any) so * the step driver can honor `deferDoneToReview`. The active context is threaded * through the foreach sub-walk; we surface it via a per-task slot the * step-execute seam stamps. Returns undefined outside a foreach instance. */ private foreachActiveForTask(taskId: string, instanceId?: string): ForeachActiveContext | undefined { if (typeof instanceId === "string") { const byInstance = this.graphStepActiveContext.get(this.graphActiveContextKey(taskId, instanceId)); if (byInstance) return byInstance; } // Fallback (single-instance / no instanceId threaded): return the sole slot // owned by this task if exactly one exists. const prefix = `${taskId}:`; let only: ForeachActiveContext | undefined; for (const [key, value] of this.graphStepActiveContext) { if (!key.startsWith(prefix)) continue; if (only) return undefined; // ambiguous: more than one instance active only = value; } return only; } /** * Project a graph-owned step only after it has a real worktree. * * A fresh task has no worktree until the authoritative implementation pass * acquires one. Projecting before that pass produces a false "step started" * event and captures the baseline from the project root. In that fresh path, * let the implementation pass own the first projection and reuse the base SHA * it captures during worktree acquisition. Resumed and isolated-step runs * already have a worktree, so they keep the normal per-step projection and * pre-work baseline behavior. */ private async runProjectedGraphTaskStep( task: Task, live: TaskDetail, stepIndex: number, active: ForeachActiveContext, governingNodeId?: string, thinkingLevel?: ThinkingLevel, skillName?: string, ): Promise { const worktreePath = active.worktreePath || live.worktree; const runStep = (idx: number) => this.runGraphTaskStep( task, idx, active.instanceId, governingNodeId, thinkingLevel, skillName, ); /* * FNXC:BaselineCwdGating 2026-07-21-19:21: * FN-8464 requires graph step projection to defer until this candidate is a real directory. * A stale, non-directory, or inaccessible truthy path must follow fresh-worktree ordering so * runTaskStep never spawns baseline git with an unusable cwd; acquisition supplies baseCommitSha. */ if (!worktreePath || !isUsableWorktreeDirectory(worktreePath)) { const result = await runStep(stepIndex); const refreshed = await this.store.getTask(task.id).catch(() => live); return { outcome: result.success ? "success" : "failure", baselineSha: refreshed.baseCommitSha, checkpointId: undefined, exit: result.exit, }; } return runTaskStep( { store: this.store, worktreePath, runStep, }, { id: task.id, steps: live.steps }, stepIndex, { markDoneOnSuccess: active.deferDoneToReview !== true, projectionSource: "graph" }, ); } /** Public authoritative-driver seam factory: exposes the same real lifecycle * seams the internal graph runner uses, without changing legacy behavior. */ public createAuthoritativeWorkflowPrimitives(settings: Settings): WorkflowRuntimePrimitives { return createWorkflowRuntimePrimitiveProvider((providerSettings) => this.createAuthoritativeWorkflowPrimitivesFromExecutor(providerSettings), ).create(settings); } private createAuthoritativeWorkflowPrimitivesFromExecutor(settings: Settings): WorkflowRuntimePrimitives { const logAudit = async (taskId: string | undefined, input: AuditPrimitiveInput): Promise => { if (!taskId) return; try { await this.store.logEntry(taskId, input.message, input.metadata ? JSON.stringify(input.metadata) : undefined); } catch { // Audit is diagnostic-only and must not affect workflow execution. } }; const planningService = new WorkflowPlanningService(); return { prepareWorktree: async (_ctx, task) => { const live = await this.store.getTask(task.id).catch(() => null); const liveTask = live?.id === task.id ? live : null; /* FNXC:WorkflowExecution 2026-06-23-11:49: The workflow execute node must not perform a second worktree acquisition ahead of the authoritative executor. Passing the repo root as a prepared worktree makes the inner execute() reject a valid fresh-worktree task as repo-root reuse; pass only an existing task worktree and let execute() acquire when none exists. FNXC:WorkflowExecution 2026-06-23-22:31: Upgrade safety requires the graph primitive to tolerate older or minimal stores that return null or a mismatched row during startup/cutover. Only trust the live row when it is for the requested task; otherwise fall back to the runner snapshot. */ const prepared: PreparedWorktree = { worktreePath: liveTask?.worktree || task.worktree || "", branchName: liveTask?.branch || task.branch, }; return { outcome: "success", value: "worktree-ready", data: prepared }; }, readArtifact: async (_ctx, task, key) => { const deps = this.buildParseStepsDeps(`${task.id}:artifact-read`); return deps.readArtifact(task, key); }, writeArtifact: async (ctx, task, key, content) => { const writer = (this.store as unknown as { writeTaskDocument?: (taskId: string, key: string, content: string) => Promise; }).writeTaskDocument; if (!writer) { await logAudit(task.id, { type: "artifact-write-unavailable", message: `Workflow node ${ctx.node.node.id} could not write artifact ${key}: store writer unavailable`, }); return { outcome: "failure", value: "artifact-write-unavailable" }; } await writer.call(this.store, task.id, key, content); return { outcome: "success", value: "artifact-written", data: { key } }; }, runPlanningSession: (ctx, task) => planningService.runPlanningSession(ctx, task), runCodingSession: async (ctx, task, prepared) => { const governingNodeId = ctx.node.context?.[SEAM_GOVERNING_NODE_CONTEXT_KEY]; if (typeof governingNodeId === "string") { this.graphSeamGoverningNodeId.set(task.id, governingNodeId); } let result: { taskDone: boolean; modifiedFiles: string[]; exit?: ImplementationExit }; try { result = await this.runImplementationPhase(task, prepared); } finally { this.graphSeamGoverningNodeId.delete(task.id); } /* FNXC:WorkflowExecutionOwnership 2026-07-29-16:20 (U8 / R4, R5): THIS is the live implementation node, not the identically-shaped `execute` entry in `createAuthoritativeWorkflowSeams`. `createDefaultNodeHandlers` prefers the PRIMITIVES handler whenever `deps.primitives` is set, and `executeWorkflowGraph` always sets it — so the legacy-seams prompt handler is unreachable for prompt nodes and anything wired only there never runs. The exit announcement was wired only there; it is announced here now. Measured, not assumed: instrumenting the seam and `createPromptLikeHandler` produced no output for a graph run that demonstrably visited `steps#0:step-execute`, while a module-load write from the same file appeared — so the negative was real and not swallowed output. */ emitWorkflowLifecycleEvent({ type: "NodeCompleted", taskId: task.id, at: new Date().toISOString(), runId: this.getRunContextFor(task.id)?.runId, nodeId: typeof governingNodeId === "string" ? governingNodeId : ctx.node.node.id, outcome: result.taskDone ? "success" : "failure", ...(result.exit ? { exit: result.exit } : {}), }); if (result.taskDone) { return { outcome: "success", value: "implemented", data: result }; } let paused = this.pausedAborted.has(task.id); if (!paused) { try { paused = Boolean((await this.store.getTask(task.id)).paused); } catch { // Best-effort pause probe; fall through to the failure value. } } /* FNXC:WorkflowExecutionOwnership 2026-07-29-18:45 (U8 / R4): THE PENDING-REVIEW ENDING IS A ROUTED OUTCOME, not a transition this phase performs. The implementation phase used to call `handoffTaskToReview` itself and let the graph discover the move afterwards; it now reports and stops, and this value routes the run to the workflow's `review-pending-handoff` node, which performs the handoff and ends the run — the same two effects in the same order, with the graph as the owner. Checked before the pause probe because a pending-review stop is not a pause. */ if (result.exit === "review-handoff-pending-review") { return { outcome: "failure", value: "review-pending", data: result }; } return { outcome: "failure", value: paused ? "implementation-paused" : "implementation-incomplete", data: result, }; }, runTaskStep: async (ctx, task, stepIndex) => { const context = ctx.node.context ?? {}; const active = context[FOREACH_ACTIVE_CONTEXT_KEY] as ForeachActiveContext | undefined; if (!active || typeof active.stepIndex !== "number") { return { outcome: "failure" }; } const live = await this.store.getTask(task.id); /* FNXC:WorkflowResume 2026-06-29-08:53: `step-execute` is a workflow node and must be idempotent on replay. If the live projection already says this foreach instance is terminal, return success before invoking the step runner so retries/restarts cannot fail a fully completed task on a stale step snapshot. */ const liveStatus = live.steps[stepIndex]?.status; if (liveStatus === "done" || liveStatus === "skipped") { return { outcome: "success", value: "step-already-terminal", data: { status: liveStatus }, }; } this.graphStepActiveContext.set(this.graphActiveContextKey(task.id, active.instanceId), active); const stepGoverningNodeId = context[SEAM_GOVERNING_NODE_CONTEXT_KEY]; const seamSkillName = context[SEAM_SKILL_NAME_CONTEXT_KEY]; return await this.runProjectedGraphTaskStep( task, live, stepIndex, active, typeof stepGoverningNodeId === "string" ? stepGoverningNodeId : undefined, undefined, typeof seamSkillName === "string" && seamSkillName.trim() ? seamSkillName.trim() : undefined, ); }, resetTaskStep: async (ctx, task, stepIndex, baselineSha, checkpointId) => { const active = ctx.node.context?.[FOREACH_ACTIVE_CONTEXT_KEY] as ForeachActiveContext | undefined; const branchScoped = typeof active?.worktreePath === "string" && active.worktreePath.length > 0; let worktreePath = active?.worktreePath ?? this.rootDir; if (!branchScoped) { try { worktreePath = (await this.store.getTask(task.id)).worktree || this.rootDir; } catch { // Best-effort worktree resolution; fall back to rootDir. } } const liveSteps = await this.store.getTask(task.id).then((t) => t.steps).catch(() => []); return await resetStepToBaseline( { store: this.store, worktreePath, sessionRef: { current: null }, reviewType: "code", blastRadiusGuard: branchScoped ? undefined : makeAncestryBlastRadiusGuard({ worktreePath, task: { id: task.id, steps: liveSteps }, stepIndex, }), }, { id: task.id, steps: liveSteps }, stepIndex, baselineSha, checkpointId, ); }, runReview: async (ctx, task, input) => { if (typeof input.stepIndex === "number") { const context = ctx.node.context ?? {}; const active = context[FOREACH_ACTIVE_CONTEXT_KEY] as ForeachActiveContext | undefined; if (!active || typeof active.stepIndex !== "number") { return { outcome: "success", value: "unavailable", data: { verdict: "UNAVAILABLE", review: "no active step instance" }, }; } const config = { type: input.type, advisory: context[SPLIT_ACTIVE_CONTEXT_KEY] === true, } as const; const seamResult = await this.createAuthoritativeWorkflowSeams(settings).stepReview?.( task, context, config, ); return { outcome: "success", value: seamResult?.verdict === "APPROVE" ? "approve" : seamResult?.verdict === "REVISE" ? "revise" : seamResult?.verdict === "RETHINK" ? "rethink" : "unavailable", data: seamResult ?? { verdict: "UNAVAILABLE", review: "step review unavailable" }, }; } const live = await this.store.getTask(task.id); await this.persistTokenUsage(task.id); await this.handoffTaskToReview(live, "workflow-graph-review"); return { outcome: "success", value: "in-review", data: { verdict: "APPROVE", summary: "Task handed off for merge review" }, }; }, runVerification: async () => ({ outcome: "success", value: "verification-skipped", data: { verdict: "skipped", } }), // FNXC:WorkflowExecution 2026-06-25-00:00: U4 (KTD-2) — the legacy // `runWorkflowStep` primitive + the `workflow-step` seam it served were // removed. Workflow quality gates run as the graph's own optional-group / // gate nodes (builtin:coding already routes through them), which record // results into `task.workflowStepResults` directly (U2). No `runWorkflowStep` // primitive remains in `WorkflowRuntimePrimitives`. updateSteps: async (_ctx, task, steps) => { await this.store.updateTask(task.id, { steps }); return { outcome: "success", value: "steps-updated", data: { count: steps.length } }; }, transitionTask: async (_ctx, task, input) => { const taskStore = this.store; const patch: Partial = {}; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40: Resolve a requested ROLE to this task's own column, because the seam that asks cannot. `workflow-node-handlers.ts`'s review-handoff seam is a pure function over an IR node and a task — no store — so it could only name `in-review`. Post-U12 `moveTask` REJECTS a destination the workflow does not declare, so on a renamed review lane that transition threw `TransitionRejectionError` and killed the walk mid-run. Not a silent wrong answer for once: a hard failure in the middle of a workflow, which is why it outranked the rest of the backlog. Resolved per task from its OWN selection, so there is one authority — the mistake that took #2843 five review rounds was answering one question with two reads. `column` still wins when both are supplied, and an unresolvable role falls back to the legacy id rather than failing the transition, which is exactly the behaviour callers had before. */ let targetColumn = input.column; if (targetColumn === undefined && input.columnRole === "review") { targetColumn = (await resolveTaskLifecycleColumns(taskStore, task.id))?.review ?? "in-review"; } /* FNXC:WorkflowNotifications 2026-06-29-08:50: Workflow graph lifecycle transitions must use TaskStore move semantics, not raw `updateTask({ column })`, because ntfy/webhook notification delivery is subscribed to `task:moved`. Direct column writes make graph-owned tasks invisible to in-review/done lifecycle notifications and bypass column hooks. */ if (targetColumn !== undefined) { const moveOptions = { preserveProgress: input.preserveProgress, moveSource: "engine" as const, workflowMoveSource: "workflow-graph", workflowMoveMetadata: { reason: input.reason, nodeId: _ctx.node.node.id, workflowId: _ctx.run.workflowId, runId: _ctx.run.runId, }, }; const storeWithMove = taskStore as typeof taskStore & { moveTask?: typeof taskStore.moveTask; }; if (typeof storeWithMove.moveTask === "function") { await storeWithMove.moveTask(task.id, targetColumn, moveOptions); } else { patch.column = targetColumn; } } if (input.status !== undefined && input.status !== null) patch.status = input.status; if (Object.keys(patch).length > 0) { await taskStore.updateTask(task.id, patch); } return { outcome: "success", value: input.reason }; }, requestMerge: async (ctx, task) => { if (!this.mergeRequester) { return { outcome: "failure", value: "merge-unavailable", data: { status: "failed", reason: "merge-unavailable" } }; } /* FNXC:WorkflowCancellation 2026-07-15-10:42: Fail fast on an already-cancelled walk BEFORE any side effect. `ensureWorkflowMergeBoundaryTask` mutates the task row and the requester enqueues a real merge; neither may run for a walk the engine has already abandoned. `merge-cancelled` is deliberately not `data.status: "failed"` — `classifyMergeFailure` would read an unknown reason as `merge-failed` and route a cancellation into bounded auto-merge retry. */ if (ctx.signal?.aborted) { return { outcome: "failure", value: "merge-cancelled" }; } const mergeTask = await this.ensureWorkflowMergeBoundaryTask(task, { reason: "workflow-merge-boundary", nodeId: ctx.node.node.id, workflowId: ctx.run.workflowId, runId: ctx.run.runId, }); /* FNXC:WorkflowMerge 2026-06-29-23:18: FN-7261 reached the merge node in fast mode with every legacy implementation step still pending, producing a no-op merge proof for work that never ran. A graph-native workflow may project its checklist at the merge boundary only when node workflow results prove implementation completed; otherwise incomplete legacy steps are authoritative and merge must fail before the merger can create stale no-op proof. FNXC:WorkflowMerge 2026-06-30-00:38: Fast default Coding tasks must still execute implementation work. FN-7260/FN-7271 reached merge with no parsed task steps, no foreach instances, and no implementation proof, then finalized through no-op merge. The workflow merge boundary must fail before requesting merge when a coding workflow has not produced implementation evidence; fast mode only bypasses review/verification gates. */ const missingImplementationProof = await this.getWorkflowMergeImplementationProofFailure(mergeTask); if (missingImplementationProof) { await this.store.logEntry( mergeTask.id, `Workflow merge blocked before requester: ${missingImplementationProof}`, undefined, this.getRunContextFor(mergeTask.id), ); return { outcome: "failure", value: "implementation-incomplete", data: { status: "failed", reason: "implementation-incomplete" }, }; } if (hasNonTerminalWorkflowSteps(mergeTask)) { await this.store.logEntry( mergeTask.id, "Workflow merge blocked before requester: implementation steps are incomplete", undefined, this.getRunContextFor(mergeTask.id), ); return { outcome: "failure", value: "implementation-incomplete", data: { status: "failed", reason: "implementation-incomplete" }, }; } /* FNXC:WorkflowCancellation 2026-07-15-10:42: The timeout bounds a wedged merge queue; it is NOT the cancellation path. `ctx.signal` (graph abort) is linked in via `AbortSignal.any` so a hard-cancel collapses the merge node immediately instead of after the full timeout, and is raced separately so the walk returns rather than waiting on a requester that may not settle on abort. Keep both signals live: dropping the timeout re-strands the walk behind a wedged queue, dropping the cancel link restores the 30-minute stall. */ const GRAPH_MERGE_TIMEOUT_MS = 30 * 60 * 1000; const controller = new AbortController(); const mergeSignal = ctx.signal ? AbortSignal.any([ctx.signal, controller.signal]) : controller.signal; let timeoutHandle: ReturnType | undefined; const timeout = new Promise<"timeout">((resolve) => { timeoutHandle = setTimeout(() => { controller.abort(); resolve("timeout"); }, GRAPH_MERGE_TIMEOUT_MS); timeoutHandle.unref?.(); }); let onGraphAbort: (() => void) | undefined; const cancelled = new Promise<"cancelled">((resolve) => { if (!ctx.signal) return; onGraphAbort = () => resolve("cancelled"); ctx.signal.addEventListener("abort", onGraphAbort, { once: true }); }); try { const result = await Promise.race([this.mergeRequester(mergeTask.id, { signal: mergeSignal }), timeout, cancelled]); if (result === "cancelled") { executorLog.warn(`${mergeTask.id}: workflow merge primitive cancelled by graph abort`); return { outcome: "failure", value: "merge-cancelled" }; } if (result === "timeout") { executorLog.warn(`${mergeTask.id}: workflow merge primitive timed out after ${GRAPH_MERGE_TIMEOUT_MS}ms`); return { outcome: "failure", value: "merge-timeout", data: { status: "timeout" } }; } if (result.merged || result.noOp) { /* FNXC:WorkflowMerge 2026-06-29-09:24: The workflow merge primitive owns the normal lifecycle transition after a graph merge node succeeds. Finalize the proven landed task here so `mergeConfirmed` cannot strand a card in `in-progress`; executor preflight recovery is only a fallback for rows already stranded by older runs. */ const finalization = await finalizeProvenAutoMergeTask({ store: this.store, taskId: mergeTask.id, result, rootDir: this.rootDir, audit: createRunAuditor(this.store, { runId: ctx.run.runId, agentId: "executor", taskId: mergeTask.id, taskLineageId: mergeTask.lineageId, phase: "workflow-merge", }), auditAgentId: "executor", auditPhase: "workflow-merge", source: "workflow-graph-merge-finalize", log: (message) => executorLog.warn(message), }); if (finalization.outcome === "blocked" || finalization.outcome === "missing") { return { outcome: "failure", value: `merge-finalize-${finalization.outcome}`, data: { status: "failed", reason: finalization.reason ?? finalization.outcome }, }; } return { outcome: "success", value: result.noOp ? "merge-noop" : "merged", data: { status: "merged", noOp: result.noOp }, }; } return { outcome: "failure", value: result.reason ?? result.error ?? "merge-failed", data: { status: "failed", reason: result.reason ?? result.error ?? "merge-failed" }, }; } finally { if (timeoutHandle) clearTimeout(timeoutHandle); // FNXC:WorkflowCancellation 2026-07-15-10:42: the graph signal outlives this node; leaving the listener attached leaks one per merge attempt across a retry loop. if (onGraphAbort) ctx.signal?.removeEventListener("abort", onGraphAbort); await logAudit(mergeTask.id, { type: "merge-requested", message: `Workflow node ${ctx.node.node.id} requested merge`, }); } }, abortRun: async (_ctx, task, input) => { if (input.hardCancel) { this.markPausedAborted(task.id, "merge-seam", "workflow-abort-run:merge-seam"); } await this.store.updateTask(task.id, { paused: true, pausedReason: input.reason, } as Partial); return { outcome: "success", value: "aborted" }; }, audit: async (ctx: WorkflowPrimitiveContext, input) => { await logAudit(ctx.run.taskId, input); }, }; } /* FNXC:WorkflowMerge 2026-07-19-04:30 (U5a / R1 / KTD-7): Resolve the merge boundary's target column from the merge NODE's own IR column. builtin:coding places its merge-class nodes in `in-review` (parity oracle); a user workflow (benchmark) places the merge node in `Merging`. Resolution failure falls back to `in-review` so a bad IR never strands the merge boundary. */ private async resolveMergeBoundaryColumn(taskId: string, nodeId: string): Promise { try { const ir = await resolveWorkflowIrForTask(this.store, taskId); // Prefer the named node's column when it is itself a merge-class node // (merge-gate/merge-attempt/…). Otherwise fall back to the FIRST merge-class // node's column — the boundary's caller may pass a synthetic id // ("legacy-merge-seam") or a non-merge node, so keying on merge-class kinds // (not an arbitrary node's column) is what reliably lands the card in the // workflow's merge column: `in-review` for builtin:coding (KTD-7 parity), // `Merging` for the benchmark. const named = ir.nodes.find((n) => n.id === nodeId); if (named && MERGE_REGION_KINDS.has(named.kind) && named.column) return named.column; const mergeNode = ir.nodes.find((n) => MERGE_REGION_KINDS.has(n.kind) && n.column); if (mergeNode?.column) return mergeNode.column; return "in-review"; } catch { return "in-review"; } } private async ensureWorkflowMergeBoundaryTask( task: TaskDetail, metadata: { reason: string; nodeId: string; workflowId: string; runId: string }, ): Promise { let live = await this.store.getTask(task.id); if (!live) return task; /* FNXC:WorkflowMerge 2026-07-19-04:10 (U5a / R1 / KTD-7): The merge NODE's OWN column drives the pre-merge handoff — not a hardcoded "in-review". builtin:coding places its merge-class nodes (merge-gate / merge-attempt / …) in `in-review`, so the default pipeline lands in `in-review` exactly as before (KTD-7 parity oracle). A user-authored workflow (the 6-column benchmark) places the merge node in `Merging`, so the card lands there because the IR says so — deleting the hardcoded-"in-review" + handoff-invariant-violation-allowlist assumption. Resolution failures fall back to `in-review` so a bad/unresolvable IR never strands the merge boundary. */ const targetColumn = await this.resolveMergeBoundaryColumn(task.id, metadata.nodeId); /* FNXC:WorkflowMerge 2026-07-26-22:59: A prior review handoff can move a graph-native workflow into its merge column before this boundary projects successful node results onto the legacy checklist. Preserve the no-move behavior, but do not return until the projection has run. */ const alreadyAtMergeColumn = live.column === targetColumn; if (live.column === await resolveCompleteColumnFor(this.store, live.id)) return live; if (live.paused || live.userPaused) return live; /* FNXC:WorkflowMerge 2026-06-29-10:15: User-authored workflows may legitimately route execution directly to a merge node without an explicit review node. Reaching that node is the workflow-owned merge boundary, so the engine must establish the durable in-review/merge lifecycle handoff before requesting merge instead of assuming a prior node already moved the card. FNXC:WorkflowMerge 2026-06-29-15:28: Compound Engineering and similar graph-native workflows execute skill nodes instead of legacy parsed task steps. The graph records those nodes as `workflowStepResults.source = "node"`; at the merge boundary, project a successful graph-native run onto the legacy checklist so `task has incomplete steps` cannot block a workflow that already completed its authoritative nodes. */ const mergeProof = await this.evaluateWorkflowMergeBoundary(live, metadata.runId); if (mergeProof.hasForeachStepExecute && !mergeProof.complete) { const reason = !mergeProof.hasRelevantNodeResult ? "no pre-merge node result recorded" : !mergeProof.allResultsTerminal ? `non-terminal pre-merge node result ${mergeProof.nonTerminalResult?.workflowStepId ?? "unknown"} (${mergeProof.nonTerminalResult?.status ?? "unknown"})` : `foreach step instances incomplete at merge boundary: missing ${mergeProof.missingInstanceIds.join(", ")}`; await this.store.logEntry(live.id, `Workflow merge boundary blocked: ${reason}`, undefined, this.getRunContextFor(live.id)); return live; } if (this.shouldCompleteChecklistAtWorkflowMerge(live, mergeProof)) { const completedSteps = live.steps.map((step) => step.status === "done" || step.status === "skipped" ? step : { ...step, status: "done" as const }, ); const updated = await this.store.updateTask( live.id, { steps: completedSteps, currentStep: Math.max(0, completedSteps.length - 1), } as Partial, this.getRunContextFor(live.id), ); live = (updated as TaskDetail | undefined) ?? { ...live, steps: completedSteps, currentStep: Math.max(0, completedSteps.length - 1) }; await this.store.logEntry( live.id, "Workflow merge boundary completed graph-native task checklist before requesting merge", undefined, this.getRunContextFor(live.id), ); } if (alreadyAtMergeColumn) return live; const moveOptions = { preserveProgress: true, moveSource: "engine" as const, workflowMoveSource: "workflow-graph", workflowMoveMetadata: metadata, }; const storeWithMove = this.store as typeof this.store & { moveTask?: (id: string, column: string, options?: unknown) => Promise; }; if (typeof storeWithMove.moveTask === "function") { const moved = await storeWithMove.moveTask(live.id, targetColumn, moveOptions); await this.store.logEntry(live.id, `Workflow merge boundary moved task to ${targetColumn} before requesting merge`, undefined, this.getRunContextFor(live.id)); return moved ?? { ...live, column: targetColumn }; } await this.store.updateTask(live.id, { column: targetColumn } as Partial, this.getRunContextFor(live.id)); await this.store.logEntry(live.id, `Workflow merge boundary moved task to ${targetColumn} before requesting merge`, undefined, this.getRunContextFor(live.id)); return { ...live, column: targetColumn }; } private async evaluateWorkflowMergeBoundary(task: TaskDetail, runId?: string): Promise<{ resolved: boolean; hasRelevantNodeResult: boolean; allResultsTerminal: boolean; coverageComplete: boolean; hasForeachStepExecute: boolean; missingInstanceIds: string[]; nonTerminalResult?: CoreWorkflowStepResult; complete: boolean; }> { const relevant = (task.workflowStepResults ?? []).filter((result) => result.source === "node" && (result.phase ?? "pre-merge") === "pre-merge", ); // FNXC:WorkflowMerge 2026-07-27-12:30: FN-8601 keeps required presence // independent from terminality: a failed node result proves execution occurred, // while allResultsTerminal separately rejects it at the merge boundary. const hasRelevantNodeResult = relevant.length > 0; const nonTerminalResult = relevant.find((result) => result.status !== "passed" && result.status !== "skipped"); const allResultsTerminal = nonTerminalResult === undefined; let ir: WorkflowIr | undefined; try { ir = await resolveWorkflowIrForTask(this.store, task.id); } catch { /* preserve legacy behavior for unresolved IRs */ } if (!ir) return { resolved: false, hasRelevantNodeResult, allResultsTerminal, coverageComplete: true, hasForeachStepExecute: false, missingInstanceIds: [], nonTerminalResult, complete: false }; let persistedInstances: Awaited> = []; try { persistedInstances = await this.loadMergeBoundaryInstances(task.id, runId); } catch { /* persistence is additive */ } const coverage = evaluateForeachMergeProof({ ir, steps: task.steps, workflowStepResults: task.workflowStepResults, persistedInstances }); const complete = hasRelevantNodeResult && allResultsTerminal && coverage.missingInstanceIds.length === 0; return { resolved: true, hasRelevantNodeResult, allResultsTerminal, coverageComplete: coverage.missingInstanceIds.length === 0, hasForeachStepExecute: coverage.hasForeachStepExecute, missingInstanceIds: coverage.missingInstanceIds, nonTerminalResult, complete }; } private async loadMergeBoundaryInstances(taskId: string, runId?: string): Promise> { if (!runId) return []; const store = this.store as typeof this.store & { loadWorkflowRunStepInstancesAsync?: (id: string, idRun: string) => Promise>; loadWorkflowRunStepInstances?: (id: string, idRun: string) => Array<{ foreachNodeId: string; stepIndex: number; pinnedStepCount: number }>; }; try { return await store.loadWorkflowRunStepInstancesAsync?.(taskId, runId) ?? store.loadWorkflowRunStepInstances?.(taskId, runId) ?? []; } catch { return []; } } private async getWorkflowMergeImplementationProofFailure(task: TaskDetail): Promise { /* FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — the graph merge boundary is another AUTO-promotion path. If the task is skip-bypass tainted (steps skipped after a bulk-step-completion refusal with no accepted fn_task_done), treat it as missing implementation proof so the merge is blocked with `implementation-incomplete` rather than laundered through a no-op merge. Runs before the noCommitsExpected exemption so a tainted task cannot slip past it. */ const taint = evaluateSkipBypassTaint(task); if (taint.blocked) return "implementation did not run: steps were skipped after a bulk-step-completion refusal without an accepted fn_task_done"; if (task.noCommitsExpected === true) return undefined; let ir: WorkflowIr | undefined; try { ir = await resolveWorkflowIrForTask(this.store, task.id); } catch { ir = undefined; } if (!ir) return undefined; const usesParsedSteps = ir.nodes.some((node) => node.kind === "parse-steps"); const usesExecuteSeam = ir.nodes.some((node) => node.kind === "prompt" && node.config?.seam === "execute"); if (!usesParsedSteps && !usesExecuteSeam) return undefined; const steps = Array.isArray(task.steps) ? task.steps : []; const hasTerminalParsedSteps = steps.length > 0 && steps.every((step) => step.status === "done" || step.status === "skipped"); const hasModifiedFiles = (task.modifiedFiles?.length ?? 0) > 0; const proof = await this.evaluateWorkflowMergeBoundary(task); const hasGraphNativeImplementationProof = proof.hasRelevantNodeResult && proof.allResultsTerminal && proof.coverageComplete; if (usesParsedSteps) { if (hasTerminalParsedSteps || hasGraphNativeImplementationProof) return undefined; return proof.hasForeachStepExecute && !proof.coverageComplete ? `implementation did not run: foreach step instances are incomplete (missing ${proof.missingInstanceIds.join(", ")})` : "implementation did not run: parsed coding steps are missing or incomplete"; } if (usesExecuteSeam) return hasTerminalParsedSteps || hasModifiedFiles || hasGraphNativeImplementationProof ? undefined : "implementation did not run: execute seam has no completion proof"; return undefined; } /* FNXC:WorkflowMerge 2026-07-27-12:00: FN-8601 gates checklist projection and foreach merge admission on required node-result presence, terminal status for every present result, and expanded-instance coverage. Non-foreach/no-seam coverage is vacuous and does not change legacy move behavior. */ private shouldCompleteChecklistAtWorkflowMerge(task: TaskDetail, proof?: { complete: boolean }): boolean { if (!Array.isArray(task.steps) || task.steps.length === 0) return false; if (task.steps.every((step) => step.status === "done" || step.status === "skipped")) return false; if (proof) return proof.complete; const graphNodeResults = (task.workflowStepResults ?? []).filter((result) => result.source === "node" && (result.phase ?? "pre-merge") === "pre-merge"); return graphNodeResults.length > 0 && graphNodeResults.every((result) => result.status === "passed" || result.status === "skipped"); } public createAuthoritativeWorkflowSeams(_settings: Settings): WorkflowLegacySeams { return { // Built-in triage/spec generation runs upstream of the interpreter today, // so planning is a no-op for already-specified tasks. Custom planning // behavior is expressed as a custom prompt node before the execute seam. planning: async () => ({ outcome: "success", value: "pre-specified" }), execute: async (seamTask, context) => { // Column-agent seam wiring (U4, R4): record the governing node id (the // execute-seam prompt node, stamped into context by createPromptLikeHandler) // so execute()'s session build can resolve the column-agent binding for the // node's DECLARED column. Cleared after the pass so a later seam without a // binding cannot inherit a stale node id. const governingNodeId = context?.[SEAM_GOVERNING_NODE_CONTEXT_KEY]; if (typeof governingNodeId === "string") { this.graphSeamGoverningNodeId.set(seamTask.id, governingNodeId); } const seamThinkingLevel = context?.[SEAM_THINKING_LEVEL_CONTEXT_KEY]; if (typeof seamThinkingLevel === "string" && WORKFLOW_THINKING_LEVEL_SET.has(seamThinkingLevel)) { this.graphSeamThinkingLevel.set(seamTask.id, seamThinkingLevel as ThinkingLevel); } let result: { taskDone: boolean; modifiedFiles: string[]; exit?: ImplementationExit }; try { result = await this.runImplementationPhase(seamTask); } finally { this.graphSeamGoverningNodeId.delete(seamTask.id); this.graphSeamThinkingLevel.delete(seamTask.id); } /* FNXC:WorkflowExecutionOwnership 2026-07-27-16:25 (U8 / R4): THIS BOOLEAN IS THE OWNERSHIP BOUNDARY, and it is too narrow. `runImplementation` has 28 measured ways of disposing of a task (16 column moves, 3 review handoffs, 9 terminal parks — counted by `executor-lifecycle-ownership-ledger.test.ts`) and exactly 3 ways of telling the graph anything, all of which collapse to `taskDone: true` here. The consequence is not a missing feature, it is a second lifecycle owner. Because the seam has no value for "the agent stopped because a step is blocked on a pending review" or "the session was paused after the work was already complete", the implementation phase performs those transitions ITSELF (`executor-exit-while-review-pending`, `paused-after-completion`) and the graph learns about them afterwards — which is why `handleGraphFailure` carries `alreadyFinalizedToReview` / `completionFinalized` classifiers whose whole job is to recognise a move the graph did not make. U8's direction: widen this vocabulary so a disposition is REPORTED here and the graph routes it, rather than performed upstream and compensated for downstream. The compensating classifiers are the acceptance test — they become unreachable, and then deletable, exactly when the last out-of-band transition is gone. */ /* FNXC:WorkflowExecutionOwnership 2026-07-28-20:25 (U8 / R4, R5): Announce the exit on the U3 lifecycle bus. Until this, the two out-of-band review handoffs left NO trace anywhere that the executor — not the graph — moved the card; they surfaced as an ordinary `implementation-incomplete` failure that `handleGraphFailure` then quietly compensated for. An operator could not tell the two apart, and neither could a test. Emission is deliberately AFTER the phase and BEFORE the return, and it changes nothing: the outcome/value below are byte-identical to what this seam returned before, for every exit, which `executor-implementation-exit-events.test.ts` pins by driving each exit and asserting the seam's return. Per R5 an exit id is a REACTION — dropping every subscriber must change no execution outcome, and that is asserted too. */ emitWorkflowLifecycleEvent({ type: "NodeCompleted", taskId: seamTask.id, at: new Date().toISOString(), runId: this.getRunContextFor(seamTask.id)?.runId, nodeId: typeof governingNodeId === "string" ? governingNodeId : "execute", outcome: result.taskDone ? "success" : "failure", ...(result.exit ? { exit: result.exit } : {}), }); if (result.taskDone) { return { outcome: "success", value: "implemented" }; } // Distinguish pause/abort from genuine implementation failure so the // failure handler can leave paused tasks to the pause machinery. let paused = this.pausedAborted.has(seamTask.id); if (!paused) { try { paused = Boolean((await this.store.getTask(seamTask.id)).paused); } catch { // Best-effort pause probe; fall through to the failure value. } } return { outcome: "failure", value: paused ? "implementation-paused" : "implementation-incomplete", }; }, // FNXC:WorkflowExecution 2026-06-25-00:00: U4 (KTD-2) — the legacy // `workflowStep` seam was removed. Workflow quality gates run as the graph's // own optional-group / gate nodes (builtin:coding replaced its `workflow-step` // seam node with optional-group nodes) which record into // `task.workflowStepResults` (U2). `WorkflowLegacySeams.workflowStep` no // longer exists, and `resolveSeamName` no longer recognizes the // `workflow-step` seam (an IR node still declaring it now fails loudly via // WorkflowIrError rather than silently no-opping). review: async (seamTask) => { // The legacy "review" stage is the in-review handoff: the in-review column is // the staging state the merge queue consumes. const live = await this.store.getTask(seamTask.id); await this.persistTokenUsage(seamTask.id); await this.handoffTaskToReview(live, "workflow-graph-review"); return { outcome: "success", value: "in-review" }; }, "review-handoff": async (seamTask) => { /* * FNXC:WorkflowPrPolicy 2026-06-29-16:42: * Compound Engineering can run an optional manual PR review lane after implementation. That lane must start from the review column without invoking the generic reviewer again; this seam is a pure lifecycle handoff so PR creation/feedback nodes run while the card is visibly in review. */ const live = await this.store.getTask(seamTask.id); await this.persistTokenUsage(seamTask.id); await this.handoffTaskToReview(live, "workflow-graph-review-handoff"); return { outcome: "success", value: "in-review" }; }, merge: async (seamTask, _context, signal) => { if (!this.mergeRequester) { return { outcome: "failure", value: "merge-unavailable" }; } // FNXC:WorkflowCancellation 2026-07-15-10:42: fail fast before the boundary-task mutation and the merge request — an abandoned walk must not enqueue a merge. Mirrors the `requestMerge` primitive. if (signal?.aborted) { return { outcome: "failure", value: "merge-cancelled" }; } const mergeTask = await this.ensureWorkflowMergeBoundaryTask(seamTask, { reason: "workflow-merge-boundary", nodeId: "legacy-merge-seam", workflowId: "legacy-seams", runId: this.getRunContextFor(seamTask.id)?.runId ?? "legacy-seam", }); const missingImplementationProof = await this.getWorkflowMergeImplementationProofFailure(mergeTask); if (missingImplementationProof) { await this.store.logEntry( mergeTask.id, `Workflow merge blocked before requester: ${missingImplementationProof}`, undefined, this.getRunContextFor(mergeTask.id), ); return { outcome: "failure", value: "implementation-incomplete" }; } // Bound the wait: a wedged merge queue must not strand the graph walk // holding the routing claim. On timeout the run fails cleanly and the // task is parked for human review; the queue can still finish later. // FNXC:WorkflowCancellation 2026-07-15-10:42: the timeout is the wedged-queue bound, `signal` is the cancellation path — both must stay live. See the `requestMerge` primitive for the stall this prevents. const GRAPH_MERGE_TIMEOUT_MS = 30 * 60 * 1000; let timeoutHandle: ReturnType | undefined; const timeout = new Promise<"timeout">((resolve) => { timeoutHandle = setTimeout(() => resolve("timeout"), GRAPH_MERGE_TIMEOUT_MS); timeoutHandle.unref?.(); }); let onGraphAbort: (() => void) | undefined; const cancelled = new Promise<"cancelled">((resolve) => { if (!signal) return; onGraphAbort = () => resolve("cancelled"); signal.addEventListener("abort", onGraphAbort, { once: true }); }); try { const result = await Promise.race([this.mergeRequester(mergeTask.id, signal ? { signal } : undefined), timeout, cancelled]); if (result === "cancelled") { executorLog.warn(`${mergeTask.id}: graph merge seam cancelled by graph abort`); return { outcome: "failure", value: "merge-cancelled" }; } if (result === "timeout") { executorLog.warn(`${mergeTask.id}: graph merge seam timed out after ${GRAPH_MERGE_TIMEOUT_MS}ms`); return { outcome: "failure", value: "merge-timeout" }; } if (result.merged || result.noOp) { return { outcome: "success", value: result.noOp ? "merge-noop" : "merged" }; } return { outcome: "failure", value: result.reason ?? result.error ?? "merge-failed" }; } finally { if (timeoutHandle) clearTimeout(timeoutHandle); if (onGraphAbort) signal?.removeEventListener("abort", onGraphAbort); } }, schedule: async () => ({ outcome: "success" }), // Step-inversion (KTD-2/KTD-4, U3): run exactly the foreach-active step. // The foreach sub-walk has set `foreach:active` with the step index; here // we drive runTaskStep (step-runner.ts) over the task's worktree, then // capture the per-step baselineSha/checkpointId back INTO the active // context object so a later RETHINK (U5) can reset the step. The full // single-step session physics (a StepSessionExecutor scoped to one step) // is U5/U7 territory; U3 wires the seam and the context capture, using the // existing implementation phase as the single-pass step driver. stepExecute: async (seamTask, context) => { const active = context[FOREACH_ACTIVE_CONTEXT_KEY] as ForeachActiveContext | undefined; if (!active || typeof active.stepIndex !== "number") { return { outcome: "failure", value: "no-active-step-instance" }; } const live = await this.store.getTask(seamTask.id); // Worktree isolation (KTD-11, U10): run the instance's session in ITS OWN // worktree when the foreach allocated one; otherwise the task's main // worktree (shared isolation — unchanged). The file-scope guard the session // machinery installs applies to either worktree unchanged (not bypassed). // Stamp the active instance so `runGraphTaskStep` can honor // `deferDoneToReview` when judging a non-terminal step (FIX 3). this.graphStepActiveContext.set(this.graphActiveContextKey(seamTask.id, active.instanceId), active); // Column-agent seam wiring (U4, R4): the governing node id — the foreach // INSTANCE node id (`#:`) stamped into // context by createPromptLikeHandler — threads INTO runGraphTaskStep, // which stamps the per-task slot only when it CREATES the memoized // implementation pass and clears it when that pass settles (PR #1432 // review). One step-session pass serves every instance, so the // session-identity binding is deterministically the pass-initiating // instance's; per-invocation set/delete here would race under parallel // foreach (overwrite mid-build, or clear while the shared pass is live). const stepGoverningNodeId = context[SEAM_GOVERNING_NODE_CONTEXT_KEY]; const seamThinkingLevel = context[SEAM_THINKING_LEVEL_CONTEXT_KEY]; const seamSkillName = context[SEAM_SKILL_NAME_CONTEXT_KEY]; const result = await this.runProjectedGraphTaskStep( seamTask, live, active.stepIndex, active, typeof stepGoverningNodeId === "string" ? stepGoverningNodeId : undefined, typeof seamThinkingLevel === "string" && WORKFLOW_THINKING_LEVEL_SET.has(seamThinkingLevel) ? (seamThinkingLevel as ThinkingLevel) : undefined, typeof seamSkillName === "string" && seamSkillName.trim() ? seamSkillName.trim() : undefined, ); // Capture baseline/checkpoint back into the reserved active context so the // foreach sub-walk threads them to later template nodes (step-review/reset). active.baselineSha = result.baselineSha; active.checkpointId = result.checkpointId; /* FNXC:WorkflowExecutionOwnership 2026-07-29-11:30 (U8 / R4): `step-done` / `step-failed` was a two-value flattening of every possible ending, and it is why the pending-review ending could never reach an edge on the stepwise shape. A blocked-on-pending-review pass is a WAIT, not a step defect: the outcome stays `failure` (the step genuinely did not complete) while the VALUE names the ending, which is what the foreach propagates upward — `runForeach` returns a failing instance's value as its own — so the `steps` node can carry an `outcome:review-pending` edge to the park node. Every other ending keeps `step-failed` exactly as before. */ const failureValue = result.exit === "review-handoff-pending-review" ? "review-pending" : "step-failed"; return { outcome: result.outcome, value: result.outcome === "success" ? "step-done" : failureValue, contextPatch: { [FOREACH_ACTIVE_CONTEXT_KEY]: active, }, }; }, // Step-inversion (KTD-4, U5): review the foreach-active step. Mirrors the // legacy in-session review call (deleted in U10): run // reviewStep under semaphore.runNested against the instance's step number/ // name and the task's PROMPT content. On an authoritative (non-advisory) // APPROVE, mark the step done through the projection (updateStep, KTD-7) — // the step-execute seam left it in-progress (markDoneOnSuccess:false) so the // review is the single done authority. The handler maps the returned verdict // to outcome edges and applies the UNAVAILABLE bounded-retry limiter. stepReview: async (seamTask, context, config) => { const active = context[FOREACH_ACTIVE_CONTEXT_KEY] as ForeachActiveContext | undefined; if (!active || typeof active.stepIndex !== "number") { // No active instance — surface UNAVAILABLE so the handler routes it // rather than fabricating an authoritative verdict. return { verdict: "UNAVAILABLE", review: "no active step instance" }; } const stepIndex = active.stepIndex; const detail = await this.store.getTask(seamTask.id); // Worktree isolation (KTD-11): review the instance's OWN worktree when set. const worktreePath = active.worktreePath || detail.worktree || this.rootDir; const reviewCwd = resolveReviewCheckoutCwd(detail, worktreePath); this.logReviewCheckoutRouting(seamTask.id, detail, reviewCwd, worktreePath); const stepName = detail.steps[stepIndex]?.name ?? `Step ${stepIndex}`; const promptContent = detail.prompt ?? ""; const userComments = selectUserCommentsForAgentContext(detail, { limit: null }); // Merge per-task effective workflow settings (U3, KTD-3) so the validator // model-lane reads below pick up workflow values. Behavior-inert by default. const settings = await mergeEffectiveSettings(this.store, detail, await this.store.getSettings()); /* FNXC:AgentSteering 2026-06-30-12:37: Workflow graph step-review nodes are optional or mandatory reviewer gates. Pass canonical user comments and legacy steering into each per-cwd reviewer so workspace aggregation never drops operator requirements. FNXC:AgentSteering 2026-06-30-13:20: Graph reviewer gates request uncapped comment context because every user-authored requirement can affect approval, including older steering retained on long-running tasks. */ const sem = this.options.semaphore; // FNXC:Workspace 2026-06-22-00:30: KTD3 — step-inversion review seam loops per sub-repo. // `reviewStep` stays single-cwd; THIS CALLER loops. Single-cwd by default reviews // `worktreePath`; in workspace mode that is the browse-only non-git root, so we instead spawn // one reviewer per acquired sub-repo (cwd = repo.worktreePath) via reviewWorkspacePerRepo and // aggregate as a conjunction. `invokeReviewerForCwd` is the per-cwd reviewStep call both modes share. const reviewService = new WorkflowReviewService(); const invokeReviewerForCwd = (cwd: string) => reviewService.reviewStep({ cwd, taskId: seamTask.id, stepIndex, stepName, type: config.type, promptContent, // Code reviews diff against the per-step baseline captured at // step-execute; plan reviews pass no baseline (advisory). baselineSha: config.type === "code" ? active.baselineSha : undefined, options: { defaultProvider: settings.defaultProvider, defaultModelId: settings.defaultModelId, fallbackProvider: settings.fallbackProvider, fallbackModelId: settings.fallbackModelId, /* * FNXC:Settings-ThinkingLevel 2026-07-13-00:27: * Step-review model sessions honor per-node `config.thinkingLevel` before the task validator override, then shared task thinking, validator workflow lane, global lane, and default thinking settings. */ defaultThinkingLevel: resolveValidatorThinkingLevel( typeof config.thinkingLevel === "string" && WORKFLOW_THINKING_LEVEL_SET.has(config.thinkingLevel) ? (config.thinkingLevel as ThinkingLevel) : detail.validatorThinkingLevel ?? detail.thinkingLevel, settings, ), fallbackThinkingLevel: resolveValidatorFallbackThinkingLevel( typeof config.thinkingLevel === "string" && WORKFLOW_THINKING_LEVEL_SET.has(config.thinkingLevel) ? (config.thinkingLevel as ThinkingLevel) : detail.validatorThinkingLevel ?? detail.thinkingLevel, settings, ), taskValidatorProvider: detail.validatorModelProvider, taskValidatorModelId: detail.validatorModelId, taskValidatorCredentialInstanceId: detail.validatorCredentialInstanceId, projectValidatorProvider: settings.validatorProvider, projectValidatorModelId: settings.validatorModelId, projectValidatorFallbackProvider: settings.validatorFallbackProvider, projectValidatorFallbackModelId: settings.validatorFallbackModelId, globalValidatorProvider: settings.validatorGlobalProvider, globalValidatorModelId: settings.validatorGlobalModelId, projectDefaultOverrideProvider: settings.defaultProviderOverride, projectDefaultOverrideModelId: settings.defaultModelIdOverride, store: this.store, taskId: seamTask.id, task: detail, userComments: userComments.length > 0 ? userComments : undefined, agentPrompts: settings.agentPrompts, agentStore: this.options.agentStore, rootDir: this.rootDir, settings, onSessionCreated: (s) => this.registerSubagentSession(seamTask.id, s), onSessionEnded: (s) => this.unregisterSubagentSession(seamTask.id, s), }, }); const runForCwd = (cwd: string) => { const invoke = () => invokeReviewerForCwd(cwd); return sem ? sem.runNested(invoke) : invoke(); }; const invokeReviewer = () => this.workspaceConfig && reviewCwd === worktreePath ? this.reviewWorkspacePerRepo(detail, (cwd) => runForCwd(cwd)) : runForCwd(reviewCwd); let review: { verdict: ReviewVerdict; review: string; summary: string }; try { review = await invokeReviewer(); } catch (err) { const message = err instanceof Error ? err.message : String(err); reviewerLog.error(`${seamTask.id}: step-review failed: ${message}`); const narration = buildReviewUnavailableMessage(err); void emitProactiveStatus(this.store, seamTask.id, narration, "reviewer", sanitizeFailureReason(err)); return { verdict: "UNAVAILABLE", review: `reviewer error: ${message}` }; } await this.store.logEntry( seamTask.id, `${config.type} step-review Step ${stepIndex}: ${review.verdict}${config.advisory ? " (advisory)" : ""}`, review.summary, ); const narration = config.type === "plan" && review.verdict === "APPROVE" ? buildPlanVerifiedMessage() : review.verdict === "UNAVAILABLE" ? buildReviewUnavailableMessage(review.summary) : buildReviewVerdictMessage(review.verdict, review.summary); if (review.verdict === "RETHINK") { // RETHINK's rollback claim is emitted by applyGraphRethinkReset only after reset succeeds. this.graphRethinkNarrations.set(this.graphActiveContextKey(seamTask.id, active.instanceId), review.summary); } else { void emitProactiveStatus(this.store, seamTask.id, narration, "reviewer", narration ? sanitizeFailureReason(review.summary) : undefined); } // Single-writer rule (KTD-4): advisory (split-branch) reviews never write // the projection — they are fan-out checks that cannot clobber the // authoritative verdict. Only an on-path APPROVE marks the step done. if (review.verdict === "APPROVE" && !config.advisory) { try { const cur = await this.store.getTask(seamTask.id); const status = cur.steps[stepIndex]?.status; if (stepIndex >= 0 && stepIndex < cur.steps.length && status !== "done" && status !== "skipped") { await this.updateStepGraph(seamTask.id, stepIndex, "done"); await this.store.logEntry( seamTask.id, `Step ${stepIndex} (${stepName}) marked done by step-review APPROVE (graph)`, ); } } catch (err) { reviewerLog.warn( `${seamTask.id}: failed to mark Step ${stepIndex} done after APPROVE: ${err instanceof Error ? err.message : String(err)}`, ); } } return { verdict: review.verdict, review: review.review, summary: review.summary }; }, }; } /** * Graph-source projection write (U6/KTD-7): a thin wrapper over * `store.updateStep` that tags the write with `source: "graph"` when the store * supports it (additive) so the out-of-order-done guard relaxes to dependency * order and a suppressed write audits loudly instead of silently. Falls back to * the legacy single-arg call on older stores. */ private async updateStepGraph( taskId: string, stepIndex: number, status: import("@fusion/core").StepStatus, ): Promise { const store = this.store as unknown as { updateStep: ( id: string, idx: number, status: import("@fusion/core").StepStatus, opts?: { source?: "graph" }, ) => Promise; }; await store.updateStep(taskId, stepIndex, status, { source: "graph" }); } /** * Pause the graph for user input: park the task paused with status * "awaiting-user-input" and the node's question as pausedReason. On a later * re-run (after the user unpauses), consume the newest steering comment as * the answer. Pre-execute placement is fully supported; post-execute * placement re-walks earlier read-only nodes until CU-U5 checkpoints land. */ private async runAwaitInputNode(node: WorkflowIrNode, live: TaskDetail): Promise { /* FNXC:WorkflowAskUser 2026-07-05-00:00: FN-7579's `ask-user` node is the first-class discoverable surface over this SAME park/resume plumbing that a `prompt` node with `config.awaitInput: true` already used. Question resolution order: `config.question` (the ask-user node's dedicated field) first, then `config.prompt` (back-compat with the original awaitInput alias), then the shared default string. Nothing below this line branches on node.kind — both node kinds share one pause/resume contract so behavior can never drift between them. */ const question = typeof node.config?.question === "string" && node.config.question.trim() ? node.config.question.trim() : typeof node.config?.prompt === "string" && node.config.prompt.trim() ? node.config.prompt.trim() : "This workflow is waiting for your input."; const marker = `workflow-input:${node.id}`; const steering = Array.isArray(live.steeringComments) ? live.steeringComments : []; // Resume only when THIS node previously paused the task (its marker is on // pausedReason). A pre-existing steering comment (e.g. one added at task // creation) must never short-circuit the pause on the node's first run — // otherwise the node consumes a stale comment and never asks the user. const pausedReason = live.pausedReason ?? ""; const pausedByThisNode = pausedReason.startsWith(marker); if (!live.paused && pausedByThisNode) { // Correlate the reply to THIS pause: the marker embeds a watermark // (`${marker}@${pauseEpochMs}: …`) recorded when the node paused. Only // count steering comments created at/after that watermark as the answer, // so an unpause-without-reply can't consume a comment that predates the // pause. The watermark is epoch milliseconds (colon-free) so it never // collides with the `:` that separates the marker from the question, nor // with the dashboard's colon-delimited question parser. const watermark = (() => { const m = pausedReason.slice(marker.length).match(/^@(\d+)/); const t = m ? Number(m[1]) : NaN; return Number.isFinite(t) ? t : undefined; })(); const replies = watermark === undefined ? steering : steering.filter((c) => { const created = Date.parse((c as { createdAt?: string }).createdAt ?? ""); return Number.isFinite(created) ? created >= watermark : false; }); if (replies.length > 0) { // Input has arrived (user replied and unpaused): consume the latest // post-pause comment and clear this node's marker so a future fresh // visit re-asks instead of silently consuming a stale comment. const latest = replies[replies.length - 1] as { text?: string; comment?: string }; const answer = (latest?.text ?? latest?.comment ?? "").toString(); await this.store.updateTask(live.id, { status: null, pausedReason: null }, this.getRunContextFor(live.id)); await this.store.logEntry(live.id, `Workflow input received for node '${node.id}'`, undefined, this.getRunContextFor(live.id)); return { outcome: "success", value: "input-received", contextPatch: { [`input:${node.id}`]: answer } }; } // Unpaused but no post-pause reply yet — re-park below and keep waiting. } await this.store.logEntry(live.id, `Workflow paused for user input: ${question}`, undefined, this.getRunContextFor(live.id)); await this.store.updateTask( live.id, { status: "awaiting-user-input", paused: true, pausedReason: `${marker}@${Date.now()}: ${question}` }, this.getRunContextFor(live.id), ); // Failure outcome ends the walk; handleGraphFailure leaves paused tasks // untouched, so the task sits awaiting input until the user responds. return { outcome: "failure", value: "awaiting-user-input" }; } /** Pause the task for explicit user approval of a raw CLI command. The user * approves via the dashboard, which records the command and unpauses; on the * next run isWorkflowCliCommandApproved returns true and the node executes. */ private async pauseForCliApproval(node: WorkflowIrNode, live: TaskDetail, command: string): Promise { const marker = `workflow-cli-approval:${node.id}`; await this.store.logEntry(live.id, `Workflow paused for CLI command approval: ${command}`, undefined, this.getRunContextFor(live.id)); await this.store.updateTask( live.id, { status: "awaiting-cli-approval", paused: true, pausedReason: `${marker}: ${command}` }, this.getRunContextFor(live.id), ); return { outcome: "failure", value: "awaiting-cli-approval" }; } /** Run an arbitrary (approved) CLI command in the task worktree, supervised. */ private async runRawCliCommand( task: TaskDetail, label: string, command: string, worktreePath: string, extraEnv?: NodeJS.ProcessEnv, ): Promise<{ success: boolean; output?: string; error?: string }> { executorLog.log(`${task.id}: workflow node '${label}' executing approved CLI command: ${command}`); await this.store.logEntry(task.id, `Workflow node '${label}' executing CLI command: ${command}`, undefined, this.getRunContextFor(task.id)); const abort = new AbortController(); this.registerConfiguredCommandController(task.id, abort); try { const result = await runConfiguredCommand( command, worktreePath, 120_000, extraEnv, createRunAuditor(this.store, { runId: this.getRunContextFor(task.id)?.runId ?? generateSyntheticRunId("exec-cli", task.id), agentId: this.getRunContextFor(task.id)?.agentId ?? (task.assignedAgentId ?? "executor"), taskId: task.id, phase: "execute", }), abort.signal, ); if (abort.signal.aborted) throw this.createConfiguredCommandAbortError(task.id, command); if (result.spawnError || result.timedOut || result.exitCode !== 0) { return { success: false, error: configuredCommandErrorMessage(result) }; } return { success: true, output: `CLI command completed successfully` }; } catch (err: unknown) { if (err instanceof Error && err.name === "AbortError") throw err; return { success: false, error: err instanceof Error ? err.message : String(err) }; } finally { this.unregisterConfiguredCommandController(task.id, abort); } } /** Build the persona prefix for an agent from its TYPED identity fields (KTD-6). * Reads `soul` and `instructionsText` — the fields the `Agent` type actually * exposes (`packages/core/src/types.ts`) — and joins them. The custom-node * `"agent"` branch historically read a non-existent `customInstructions` * field (silently undefined); this is the single consistent source used by * both the node-agent and column-agent paths. */ /** Extract a task's OWN settings for the effective-agent resolver: its assigned * agent identity (trimmed, non-empty) and a COMPLETE model pair (an incomplete * pair does not count — KTD-5, mirrors resolveExecutorSessionModel's both-present * rule). Centralizes the previously-duplicated extraction so the four call sites * (restart watcher, taskEffectiveAgentMatches, resolveSeamColumnAgent, * resolveEffectivePrincipalId) share one normalized idiom. */ private extractOwnSettings( task: Pick, ): Pick { const ownAgentId = typeof task.assignedAgentId === "string" && task.assignedAgentId.trim() ? task.assignedAgentId.trim() : undefined; const ownModelComplete = Boolean(task.modelProvider && task.modelId); return { ownAgentId, ownModelProvider: ownModelComplete ? task.modelProvider : undefined, ownModelId: ownModelComplete ? task.modelId : undefined, }; } private buildAgentPersona(agent: Agent): string | undefined { const parts = [agent.soul, agent.instructionsText] .map((p) => (typeof p === "string" ? p.trim() : "")) .filter((p) => p.length > 0); return parts.length > 0 ? parts.join("\n\n") : undefined; } /** Fetch the column agent and surface its model + persona for adoption by a * custom node (plan U3). Best-effort, mirroring the node-agent posture at the * `"agent"` branch: on null/throw, log and return undefined so the caller * falls back to the node's own/default resolution (R8). Emits a logEntry * naming the substitution and mode so the audit trail explains who ran. */ private async adoptColumnAgentForNode( node: WorkflowIrNode, live: TaskDetail, columnAgentId: string, mode: WorkflowColumnAgent["mode"] | undefined, ): Promise<{ modelProvider?: string; modelId?: string; persona?: string } | undefined> { try { const agent = await this.options.agentStore?.getAgent(columnAgentId); if (!agent) { await this.store.logEntry( live.id, `Workflow node '${node.id}': column agent '${columnAgentId}' not found — falling back to node/default resolution`, undefined, this.getRunContextFor(live.id), ); return undefined; } const rc = (agent.runtimeConfig ?? {}) as { executorProvider?: string; executorModelId?: string }; await this.store.logEntry( live.id, `Workflow node '${node.id}': running as column agent '${columnAgentId}' (${mode})`, undefined, this.getRunContextFor(live.id), ); return { modelProvider: rc.executorProvider, modelId: rc.executorModelId, persona: this.buildAgentPersona(agent), }; } catch { // Agent lookup is best-effort; fall back to node/default resolution (R8). // A secondary logEntry failure (DB locked / mid-recovery) must NOT propagate // out of this error handler and escalate the node to a hard failure. try { await this.store.logEntry( live.id, `Workflow node '${node.id}': column agent '${columnAgentId}' lookup failed — falling back to node/default resolution`, undefined, this.getRunContextFor(live.id), ); } catch (logErr: unknown) { executorLog.warn(`${live.id}: failed to log column-agent lookup failure: ${logErr instanceof Error ? logErr.message : String(logErr)}`); } return undefined; } } /** * Resolve the effective COLUMN AGENT governing the coding/step session currently * being built for a task (column-agent plan U4, R2/R3/R4/R8). * * Reads the governing node id stamped by the active seam ({@link * graphSeamGoverningNodeId}) and the per-run binding resolver ({@link * graphColumnAgentResolver}), both scoped to a graph-owned run. Feeds the task's * OWN settings (`assignedAgentId` + complete `modelProvider`/`modelId` pair) into * the shared core resolver (`resolveEffectiveAgent`, KTD-2/KTD-5) so defer/override * precedence is never reimplemented here. When the verdict is `column-agent`, * fetches the full Agent best-effort and audits the adoption; on a missing/deleted * agent it logs and returns undefined so the caller falls back to the * `assignedAgentId` path (R8). Returns undefined for the legacy/no-binding path so * the session build is byte-identical (characterization parity). * * Exposes the resolved Agent object (not just an id) so U5 can consume the same * effective principal for gating/heartbeat/restart without re-resolving. */ private async resolveSeamColumnAgent( task: Task, detail: TaskDetail, ): Promise<{ agent: Agent; mode: WorkflowColumnAgent["mode"] | undefined } | undefined> { const governingNodeId = this.graphSeamGoverningNodeId.get(task.id); const resolveBinding = this.graphColumnAgentResolver.get(task.id); if (!governingNodeId || !resolveBinding) return undefined; const binding = resolveBinding(governingNodeId); if (!binding) return undefined; // The task's OWN settings: its assigned agent identity and a COMPLETE model // pair (an incomplete pair does not count — KTD-5, mirrors // resolveExecutorSessionModel's both-present rule). const effective = resolveEffectiveAgent({ binding, ...this.extractOwnSettings(detail), }); if (effective.source !== "column-agent") return undefined; // Column agent governs: fetch the full Agent (best-effort, R8 fallback). let agent: Agent | null = null; try { agent = (await this.options.agentStore?.getAgent(effective.agentId)) ?? null; } catch { agent = null; } if (!agent) { // Best-effort audit: a logEntry failure (DB locked / mid-recovery) must NOT // escalate this graceful fallback into a hard session failure (R8). try { await this.store.logEntry( task.id, `Workflow seam node '${governingNodeId}': column agent '${effective.agentId}' not found — falling back to assigned-agent resolution`, undefined, this.getRunContextFor(task.id), ); } catch (logErr: unknown) { executorLog.warn(`${task.id}: failed to log column-agent fallback: ${logErr instanceof Error ? logErr.message : String(logErr)}`); } return undefined; } try { await this.store.logEntry( task.id, `Workflow seam node '${governingNodeId}': running as column agent '${effective.agentId}' (${binding.mode})`, undefined, this.getRunContextFor(task.id), ); } catch (logErr: unknown) { executorLog.warn(`${task.id}: failed to log column-agent adoption: ${logErr instanceof Error ? logErr.message : String(logErr)}`); } return { agent, mode: binding.mode }; } /** * Column-agent principal alignment (plan U5, R6). Resolve the EFFECTIVE * principal id for the in-flight seam WITHOUT fetching the full Agent or * emitting an adoption log — a light counterpart to {@link resolveSeamColumnAgent} * used by the heartbeat-deferral gate (which only needs the id to call * {@link shouldDeferForHeartbeat}, which itself loads the agent). * * Returns the column-agent id when a governing binding selects it via the shared * core resolver (`resolveEffectiveAgent`, KTD-2/KTD-5), else `task.assignedAgentId` * (the legacy principal). Returns `undefined` only when there is no principal at * all (no binding AND no assigned agent) — keeping the no-binding path * byte-identical to the prior `assignedAgentId` deferral behavior. */ private resolveEffectivePrincipalId( task: Task, detail: Task, ): string | undefined { const ownSettings = this.extractOwnSettings(detail); const assignedAgentId = ownSettings.ownAgentId; const governingNodeId = this.graphSeamGoverningNodeId.get(task.id); const resolveBinding = this.graphColumnAgentResolver.get(task.id); if (!governingNodeId || !resolveBinding) return assignedAgentId; const binding = resolveBinding(governingNodeId); if (!binding) return assignedAgentId; const effective = resolveEffectiveAgent({ binding, ...ownSettings }); if (effective.source === "column-agent") return effective.agentId; return assignedAgentId; } /** * Column-agent principal alignment (plan U5, R6). True when `agentId` is the * EFFECTIVE column-agent principal currently running some executing task's * coding/step session — i.e. an override/defer-bound column staffs it, even * though the agent is not the task's `assignedAgentId`. Injected into the * heartbeat scheduler's reverse-direction parallel-execution guards * (`agent-heartbeat.ts`) so an `allowParallelExecution=false` column agent does * not heartbeat concurrently with its own override session. Returns false for the * legacy/no-binding path (the map is empty), preserving prior behavior exactly. */ isAgentEffectivelyExecuting(agentId: string): boolean { if (!agentId) return false; for (const effectiveId of this.effectiveColumnAgentByTask.values()) { if (effectiveId === agentId) return true; } return false; } /** Build the task-scoped runtime env that carries plugin-injected keys * (e.g. compound-engineering `FUSION_CE_SKILLS_DIR` / `FUSION_CE_AGENTS_DIR`) * plus the plugin PATH contribution. Shared by the legacy single-session path * (agentWork, ~7434) and the graph-node skill-step path (runGraphCustomNode, * U8) so both deliver the same injected env to their sessions. We never mutate * process.env globally — this scoped env is threaded through taskEnv so session * subprocesses inherit it without leaking across concurrent tasks. */ private async buildInjectedRuntimeEnv( taskId: string, worktreePath: string, branch: string | undefined, ): Promise<{ env: NodeJS.ProcessEnv; injectedKeyCount: number; pathEntryCount: number }> { const runtimeEnvContribution = await this.options.pluginRunner?.collectExecutorRuntimeEnv({ taskId, worktreePath, rootDir: this.rootDir, branch, }); const pathPrepend = runtimeEnvContribution?.pathPrepend ?? []; const injectedEnv = runtimeEnvContribution?.env ?? {}; return { env: { ...process.env, ...injectedEnv, PATH: [...pathPrepend, process.env.PATH ?? ""].filter(Boolean).join(delimiter), }, injectedKeyCount: Object.keys(injectedEnv).length, pathEntryCount: pathPrepend.length, }; } private async ensureGraphCustomNodeWorktree( task: TaskDetail, settings: Settings, nodeId: string, ): Promise { /* FNXC:WorkflowExecution 2026-06-29-08:21: Custom graph nodes can be the first executable node in a workflow. If such a node is coding/script-capable, acquire the same task worktree the legacy executor would have acquired instead of failing with `no-worktree-for-write-node`; the node remains isolated from main and CE `plan` can run first. */ if (this.workspaceConfig === undefined) { this.workspaceConfig = await loadWorkspaceConfig(this.rootDir); } if (this.workspaceConfig && (this.workspaceConfig.repos.length ?? 0) > 0) { return task; } const syntheticRunId = generateSyntheticRunId("workflow-node-worktree", task.id); const audit = createRunAuditor(this.store, { runId: syntheticRunId, agentId: task.assignedAgentId ?? "executor", taskId: task.id, phase: "execute", }); const commandAbortController = new AbortController(); this.registerConfiguredCommandController(task.id, commandAbortController); try { await this.store.logEntry( task.id, `Workflow node '${nodeId}' requires a task worktree — acquiring worktree before node execution`, undefined, this.getRunContextFor(task.id), ); const acquisition = await acquireTaskWorktree({ task, rootDir: this.rootDir, store: this.store, settings, pool: this.options.pool, logger: executorLog, audit, runContext: this.getRunContextFor(task.id), runInitCommand: true, createWorktree: this.createWorktree.bind(this), runConfiguredCommand: (command, cwd, timeoutMs, env) => runConfiguredCommand( command, cwd, timeoutMs, env, audit, commandAbortController.signal, ).then((result) => { if (commandAbortController.signal.aborted) { throw this.createConfiguredCommandAbortError(task.id, command); } return result; }), taskEnv: process.env, secretsStore: this.options.secretsStore, }); this.addActiveWorktree(task.id, acquisition.worktreePath); if (!acquisition.isResume) { await this.captureBaseCommitSha(task, acquisition.worktreePath, audit, { isResume: false }); } this.options.onStart?.(task, acquisition.worktreePath); executorLog.log(`${task.id}: workflow node '${nodeId}' acquired worktree at ${acquisition.worktreePath}`); return await this.store.getTask(task.id); } catch (error) { const message = error instanceof Error ? error.message : String(error); await this.store.logEntry( task.id, `Workflow node '${nodeId}' failed to acquire task worktree: ${message}`, undefined, this.getRunContextFor(task.id), ); throw error; } finally { this.unregisterConfiguredCommandController(task.id, commandAbortController); } } /* FNXC:NodeWorktreeIsolation 2026-07-25-22:10 (planning acquires the task worktree): Public seam for the planning/triage lane. Specification runs a CODING-tool session; pointing it at the shared main checkout meant every planning agent had write tools in the operator's tree and every concurrent planner shared one path. Acquire the task's own worktree up front and let the whole lifecycle — planning, Plan Review, implementation, code review — reuse that single worktree. Returns null (caller falls back to the root, unchanged behavior) when the project is a workspace, or when acquisition fails: planning must never be blocked by a worktree problem. */ /* FNXC:PlanningEvacuation 2026-07-25-23:00 (pre-execution worktree release): Planning now acquires a worktree, so a card that never reaches execution — withdrawn to Ideas, archived from a planner lane, or parked pre-execution — would otherwise hold one forever. Release it. Safety conditions, all required: - the task never executed (`firstExecutionAt`/`executionStartedAt` unset): execution evidence means the worktree may hold real work, and only the normal merge/archive lifecycle may remove it; - no live session registered on the path (the same isPathActive guard the other sweeps use); - the branch carries no commits beyond its base — planning writes its spec to the task store, not the worktree, so a clean branch means there is genuinely nothing to lose. Metadata (`worktree`/`branch`) is cleared with it, so a later promotion re-acquires cleanly. Fail-soft throughout: a cleanup problem must never block the lifecycle move that triggered it. */ public async releasePreExecutionWorktree(taskId: string, reason: string): Promise { try { const live = await this.store.getTask(taskId); if (!live?.worktree) return false; if (live.firstExecutionAt || live.executionStartedAt) return false; if (activeSessionRegistry.isPathActive(live.worktree) || activeSessionRegistry.isPathActive(resolvePath(live.worktree))) return false; if (this.hasLiveTaskSessionSurface(taskId) || executingTaskLock.has(taskId)) return false; if (existsSync(live.worktree)) { if (await this.preExecutionWorktreeHasWork(live.worktree)) { executorLog.log(`${taskId}: keeping pre-execution worktree ${live.worktree} — it carries commits or uncommitted changes`); return false; } const settings = await this.store.getSettings(); await removeWorktree({ rootDir: this.rootDir, worktreePath: live.worktree, settings, taskId, reason: RemovalReason.SelfHealingReclaim, }); } this.activeWorktrees.get(taskId)?.delete(live.worktree); await this.store.updateTask(taskId, { worktree: null, branch: null, baseCommitSha: null, sessionFile: null }, this.getRunContextFor(taskId)); await this.store.logEntry(taskId, `Released the pre-execution worktree (${reason}) — it will be re-acquired when planning or execution resumes`, undefined, this.getRunContextFor(taskId)).catch(() => undefined); executorLog.log(`${taskId}: released pre-execution worktree ${live.worktree} (${reason})`); return true; } catch (error) { executorLog.warn(`${taskId}: could not release the pre-execution worktree: ${formatError(error).message}`); return false; } } /** True when a pre-execution worktree holds commits past its base or any uncommitted change. */ private async preExecutionWorktreeHasWork(worktreePath: string): Promise { try { const { stdout: dirty } = await execFileAsync("git", ["status", "--porcelain"], { cwd: worktreePath, timeout: 30_000 }); if (dirty.trim()) return true; const { stdout: ahead } = await execFileAsync("git", ["log", "--oneline", "@{upstream}..HEAD"], { cwd: worktreePath, timeout: 30_000 }) .catch(async () => await execFileAsync("git", ["log", "--oneline", "-1", "HEAD", "--not", "--remotes", "--branches=main", "--branches=master"], { cwd: worktreePath, timeout: 30_000 })); return Boolean(ahead.trim()); } catch { // Cannot prove the worktree is clean → treat it as holding work and keep it. return true; } } public async ensureTaskWorktreeForPlanning(taskId: string): Promise { try { if (this.workspaceConfig === undefined) { this.workspaceConfig = await loadWorkspaceConfig(this.rootDir); } if (this.workspaceConfig && (this.workspaceConfig.repos.length ?? 0) > 0) return null; const live = await this.store.getTask(taskId); if (live.worktree && existsSync(live.worktree)) return live.worktree; const settings = await this.store.getSettings(); const acquisitionTask = live.worktree ? ({ ...live, worktree: undefined, sessionFile: undefined } as TaskDetail) : live; const acquired = await this.ensureGraphCustomNodeWorktree(acquisitionTask, settings, "planning"); return acquired.worktree || null; } catch (error) { executorLog.warn(`${taskId}: could not acquire a planning worktree — planning falls back to the repo root: ${formatError(error)}`); return null; } } private async prepareGraphNodeExecution( node: WorkflowIrNode, nodeTask: TaskDetail, settings: Settings, requirement: WorkflowNodePreparationRequirement, ): Promise { if (!requirement.requiresWorktree) return; const live = await this.store.getTask(nodeTask.id); if (live.worktree && existsSync(live.worktree)) return; const taskForAcquisition = live.worktree ? ({ ...live, worktree: undefined, sessionFile: undefined } as TaskDetail) : live; if (live.worktree) { /* FNXC:WorkflowExecution 2026-06-29-15:28: A graph-native skill node such as Compound Engineering `plan` may be the first write-capable node. A stale task row can still point at a removed checkout after reset/retry/self-healing; a truthy `worktree` field is not proof of node readiness. Fall through to fresh acquisition when the directory is missing so the graph starts a session instead of failing immediately at the first CE node. */ await this.store.logEntry( live.id, `Workflow node '${node.id}' assigned worktree is missing — reacquiring before node execution`, live.worktree, this.getRunContextFor(live.id), ); } /* FNXC:WorkflowExecution 2026-06-29-09:50: The workflow graph decides which nodes require pre-execution lifecycle resources. This adapter only fulfills a graph-declared worktree requirement with executor-owned git mechanics; custom-node handlers remain ordinary node execution and no longer decide when to bootstrap task isolation. */ await this.ensureGraphCustomNodeWorktree(taskForAcquisition, settings, node.id); } private async finalizeMergeConfirmedWorkflowGraphTask(taskId: string, reason: string): Promise { const live = await this.store.getTask(taskId).catch(() => null); if (!live || live.mergeDetails?.mergeConfirmed !== true || live.column === await resolveCompleteColumnFor(this.store, live.id)) return false; /* FNXC:WorkflowMerge 2026-06-29-08:32: A workflow graph merge node can await a successful ProjectEngine merge request and return before the row reaches `done`. Merge confirmation is durable proof of landing; the executor must finalize that row from any non-terminal column instead of re-running parse or clearing mergeDetails. */ await this.store.logEntry( taskId, `Workflow graph observed confirmed merge while task was '${live.column}' — finalizing to done (${reason})`, undefined, this.getRunContextFor(taskId), ); const finalization = await finalizeProvenAutoMergeTask({ store: this.store, taskId, result: { task: live, ok: true, merged: true, commitSha: live.mergeDetails?.commitSha, noOp: live.mergeDetails?.noOpMerge === true, reason: live.mergeDetails?.noOpReason, mergeConfirmed: true, } as MergeResult, rootDir: this.rootDir, audit: createRunAuditor(this.store, { runId: generateSyntheticRunId("workflow-graph-merge-finalize", taskId), agentId: "executor", taskId, taskLineageId: live.lineageId, phase: "workflow-graph-merge-finalize", }), auditAgentId: "executor", auditPhase: "workflow-graph-merge-finalize", source: "workflow-graph-merge-finalize", log: (message) => executorLog.warn(message), }); if (finalization.outcome === "blocked") { executorLog.warn(`${taskId}: workflow graph merge-confirmed finalization blocked — ${finalization.reason ?? "unknown"}`); await this.store.logEntry( taskId, `Workflow graph merge-confirmed finalization blocked — ${finalization.reason ?? "unknown"}`, undefined, this.getRunContextFor(taskId), ); if (finalization.reason === "task has incomplete steps" && live.mergeDetails?.noOpMerge === true && !live.mergeDetails?.commitSha) { /* FNXC:WorkflowMerge 2026-06-29-23:12: FN-7261 exposed stale no-op proof as a re-execution blocker: a reopened task with incomplete implementation steps and only no-op merge proof must fall through to merge-state cleanup/reverification, not consume execute() by repeatedly trying blocked finalization. */ return false; } return true; } executorLog.log(`${taskId}: workflow graph merge-confirmed task finalized (${finalization.outcome})`); return true; } /** Run a custom (non-seam) graph node on the proven WorkflowStep machinery. * * `columnBinding` (plan U3) is the agent binding governing this node's * declared column, resolved by the seam wiring in executeWorkflowGraph * (the IR is not in scope here). When present, the core resolver decides * whether the column agent supersedes (override) or defers to the node's own * `cfg.agentId`/model pair — never a reimplemented precedence. */ private async runGraphCustomNode( node: WorkflowIrNode, nodeTask: TaskDetail, settings: Settings, columnBinding?: WorkflowColumnAgent, graphContext?: Record, ): Promise { const cfg = node.config ?? {}; let live = await this.store.getTask(nodeTask.id); const staleInput = await this.resolveWorkflowInputMarkerForGraphNode(live, node.id); if (staleInput === "waiting") return { outcome: "failure", value: "awaiting-user-input" }; if (staleInput === "clear") live = await this.store.getTask(nodeTask.id); // Await-input nodes never run a session — they pause for the user. // FNXC:WorkflowAskUser 2026-07-05-00:00: `ask-user` is the dedicated, // discoverable node kind for this same pause; `prompt` + `config.awaitInput: // true` remains a back-compat alias (both route to the identical runner). if (cfg.awaitInput === true || node.kind === "ask-user") { return this.runAwaitInputNode(node, live); } // Skill-emitted await-input resume (U6): a prior run of THIS node may have // paused the task because its skill asked the user a blocking question via // the ===FUSION_AWAIT_INPUT=== sentinel. Mirror runAwaitInputNode's resume: // when the user has replied (a steering comment at/after the pause // watermark), clear the marker and fall through to RE-RUN the skill so it // continues with the answer; otherwise keep the task parked and halt. const skillAwaitMarker = `workflow-input:${node.id}`; const skillPausedReason = live.pausedReason ?? ""; if (skillPausedReason.startsWith(skillAwaitMarker)) { // Mirror runAwaitInputNode: only inspect replies once the task is actually // unpaused. While `live.paused` is still true the user has added a comment // but not released the task — keep it parked and never consume that reply, // so a still-paused task can't short-circuit straight back into the skill. if (live.paused) { return { outcome: "failure", value: "awaiting-user-input" }; } const watermark = (() => { const mm = skillPausedReason.slice(skillAwaitMarker.length).match(/^@(\d+)/); const t = mm ? Number(mm[1]) : NaN; return Number.isFinite(t) ? t : undefined; })(); const steering = Array.isArray(live.steeringComments) ? live.steeringComments : []; const replies = watermark === undefined ? steering : steering.filter((c) => { const created = Date.parse((c as { createdAt?: string }).createdAt ?? ""); return Number.isFinite(created) ? created >= watermark : false; }); if (replies.length === 0) { // Unpaused without a post-watermark reply — re-park and keep waiting. await this.store.updateTask(live.id, { status: "awaiting-user-input", paused: true }, this.getRunContextFor(live.id)); return { outcome: "failure", value: "awaiting-user-input" }; } await this.store.updateTask(live.id, { status: null, pausedReason: null }, this.getRunContextFor(live.id)); await this.store.logEntry(live.id, `Workflow input received for step '${node.id}' — resuming`, undefined, this.getRunContextFor(live.id)); } const executorKind = typeof cfg.executor === "string" ? cfg.executor : "model"; // CLI Agent Executor (U7): a `cli-agent` node drives an engine-owned CLI // session through the task-session orchestration — NOT through the // executeWorkflowStep / model machinery. It is write-capable (the agent edits // the worktree), so it requires a task worktree like any coding node. if (executorKind === "cli-agent") { return this.runCliAgentNode(node, await this.store.getTask(live.id), cfg); } // Fast mode bypasses pre-merge automated review/validation gates. Custom // graph prompt/script/gate nodes are implemented by synthesizing pre-merge // WorkflowStep executions below, so skip them here before worktree or CLI // approval gates can fire. Human waits (`awaitInput`) and implementation // CLI-agent nodes are handled above and remain enforced. const optionalGroupId = typeof graphContext?.[WORKFLOW_OPTIONAL_GROUP_CONTEXT_KEY] === "string" ? graphContext[WORKFLOW_OPTIONAL_GROUP_CONTEXT_KEY] : undefined; /* FNXC:FastOptionalSteps 2026-06-30-09:14: Fast skips top-level custom prompt/script/gate review bodies by default, but an enabled optional-group template is explicit operator intent. The graph marks those template nodes so Browser Verification and custom optional groups still run under fast mode. */ const isCompletionSummaryNode = cfg.summaryTarget === "task" || node.id === "completion-summary"; /* FNXC:WorkflowCompletion 2026-07-01-18:42: Fast mode skips review/validation work, not the agent-authored completion summary. FN-7335 reached review with "Fast mode — custom graph node 'completion-summary' skipped"; keep summary nodes executable so fast tasks still produce the same review/done card summary as standard tasks. */ if (live.executionMode === "fast" && !isCompletionSummaryNode && !optionalGroupId && !cfg.seam && (node.kind === "prompt" || node.kind === "script" || node.kind === "gate")) { executorLog.debug(`${live.id}: fast mode — skipping custom graph node '${node.id}'`); await this.store.logEntry( live.id, `Fast mode — custom graph node '${node.id}' skipped`, undefined, this.getRunContextFor(live.id), ); return { outcome: "success", value: "workflow-step-skipped" }; } const scriptName = typeof cfg.scriptName === "string" && cfg.scriptName.trim() ? cfg.scriptName : undefined; const rawCliCommand = executorKind === "cli" && typeof cfg.cliCommand === "string" && cfg.cliCommand.trim() ? cfg.cliCommand.trim() : undefined; // Isolation guard: write-capable nodes must run inside a task worktree, not // the shared repo root. Before the execute seam runs, live.worktree is unset // — a coding/script/CLI node falling back to this.rootDir would mutate the // main checkout and cross-contaminate other tasks. Reject such nodes until a // worktree exists. Read-only nodes (default toolMode) are safe against root. /* FNXC:WorkflowReviewers 2026-07-15-00:00: Inline-fix Code Review, Browser Verification, and custom review nodes become write-capable even when their workflow definition says `toolMode: readonly`. Use the shared classifier consumed by graph preparation so issue #2075 cannot leave runtime requiring a worktree that preparation declined to acquire. Plan Review remains excluded because it uses the narrow PROMPT.md writer. */ const writeCapable = workflowNodeRequiresWorktree(node, { optionalGroupId, reviewerInlineFixes: (settings as Settings & { reviewerInlineFixes?: boolean }).reviewerInlineFixes, }); let executionTarget = writeCapable ? await this.store.getTask(live.id) : live; /* FNXC:NodeWorktreeIsolation 2026-07-25-22:10 (EVERY node runs in the task's own worktree): Operator requirement: Plan Review, Code Review — everything except merge — executes in the task-specific worktree, never in the shared main checkout. Read-only gates used to fall back to `this.rootDir` because a pre-execution task has no worktree yet, which is what made two tasks share a path in the first place (the reported FN-1398/FN-1403 Plan Review collision) and what let a reviewer read a main checkout that other tasks and the operator mutate underneath it. ACQUIRE the worktree at planning time instead: `ensureGraphCustomNodeWorktree` is the same acquisition the write-capable nodes already use, so the worktree/branch/baseCommitSha the implementation session later resumes into is created once, here, and reused. A recorded-but-missing worktree is RE-ACQUIRED (strip the stale metadata first, mirroring prepareGraphNodeExecution) rather than degraded to the root — this replaces FN-7996's run-Plan-Review-from-the-repo-root fallback, which is exactly the shared-path behavior being removed. Workspace projects are unchanged: `ensureGraphCustomNodeWorktree` returns the task untouched there, because workspace sessions are rooted at the browse-root by design and per-repo isolation comes from the sub-repo acquire lease. */ const nodeDisplayName = typeof cfg.name === "string" && cfg.name.trim() ? cfg.name.trim() : node.id; const isPlanReviewNode = node.id === "plan-review-step" || nodeDisplayName === "Plan Review" || optionalGroupId === "plan-review"; if (!this.workspaceConfig) { const recordedWorktreeMissing = Boolean(executionTarget.worktree) && !existsSync(executionTarget.worktree!); /* A node with NO recorded worktree is pre-execution (planning / Plan Review): acquire one. A node whose RECORDED worktree vanished is a different situation — for gates that review implementation output, the work is gone with it, and handing them a fresh empty worktree would let them review the wrong tree and pass. Those keep failing fast into the unusable-worktree recovery (FN-7996). Plan Review is the exception: it reviews the store-injected PROMPT.md, so it re-acquires rather than parking — this replaces its old "run from the repo root" degrade. */ const shouldAcquire = !executionTarget.worktree || (recordedWorktreeMissing && isPlanReviewNode); if (shouldAcquire) { if (recordedWorktreeMissing) { await this.store.logEntry( live.id, `Plan Review worktree ${executionTarget.worktree} is missing on disk — re-acquiring a task worktree instead of running in the shared checkout`, undefined, this.getRunContextFor(live.id), ); } const acquisitionTask = recordedWorktreeMissing ? ({ ...executionTarget, worktree: undefined, sessionFile: undefined } as TaskDetail) : executionTarget; executionTarget = await this.ensureGraphCustomNodeWorktree(acquisitionTask, settings, node.id); } } if (writeCapable && !executionTarget.worktree && !this.workspaceConfig) { return { outcome: "failure", value: "no-worktree-for-write-node" }; } const worktreePath = executionTarget.worktree || this.rootDir; let prompt = typeof cfg.prompt === "string" ? cfg.prompt : ""; let modelProvider = typeof cfg.modelProvider === "string" && cfg.modelProvider.trim() ? cfg.modelProvider : undefined; let modelId = typeof cfg.modelId === "string" && cfg.modelId.trim() ? cfg.modelId : undefined; // ── Column-agent binding (plan U3, KTD-2/KTD-3) ────────────────────────── // When the node's declared column names an agent, the CORE resolver decides // whether the column agent supersedes (override) or defers to the node's own // settings — we never reimplement precedence. The node's own `cfg.agentId` // and complete model pair feed the resolver as "own settings" (KTD-5). const ownModelComplete = Boolean(modelProvider && modelId); const effective = resolveEffectiveAgent({ binding: columnBinding, ownAgentId: typeof cfg.agentId === "string" && cfg.agentId.trim() ? cfg.agentId.trim() : undefined, ownModelProvider: ownModelComplete ? modelProvider : undefined, ownModelId: ownModelComplete ? modelId : undefined, }); // The effective executor identity: a column agent supersedes the node's own // `executor: "agent"` adoption wholesale (identity + model + persona). When // the resolver yields the column agent, we run the column-agent adoption // path below INSTEAD of the node's own agent branch. const columnAgentId = effective.source === "column-agent" ? effective.agentId : undefined; const columnAgentMode = columnBinding?.mode; if (columnAgentId) { // CLI executor with a raw command runs no session — the column agent // cannot contribute a model/persona to raw process execution, so it is a // no-op here. Log the skip so the audit trail explains why the column // agent did not apply (plan U3). Skill / model / script-via-session nodes // DO adopt the column agent below. if (executorKind === "cli" && rawCliCommand) { await this.store.logEntry( live.id, `Workflow node '${node.id}': column agent '${columnAgentId}' (${columnAgentMode}) not applied — raw CLI execution runs no session`, undefined, this.getRunContextFor(live.id), ); } else { const adopted = await this.adoptColumnAgentForNode(node, live, columnAgentId, columnAgentMode); if (adopted) { modelProvider = adopted.modelProvider ?? modelProvider; modelId = adopted.modelId ?? modelId; if (adopted.persona) prompt = `${adopted.persona}\n\n${prompt}`; } // Whether or not the agent resolved, the column agent SUPERSEDES the // node's own `executor: "agent"` adoption — skip that branch so we never // blend the column agent's model with the node agent's persona. } } // Executor kinds for prompt nodes: // - "model" (default): run the prompt on the configured/override model. // - "agent": run as a named agent — adopt its model and persona prompt. // - "skill": invoke a named skill with the prompt as its input. // - "cli": run a named project script with the prompt passed via env // (FUSION_NODE_PROMPT). Named scripts only — raw commands are // never accepted from node config. if (!columnAgentId && executorKind === "agent" && typeof cfg.agentId === "string" && cfg.agentId.trim()) { try { const agent = await this.options.agentStore?.getAgent(cfg.agentId); if (agent) { const rc = (agent.runtimeConfig ?? {}) as { executorProvider?: string; executorModelId?: string }; modelProvider = rc.executorProvider ?? modelProvider; modelId = rc.executorModelId ?? modelId; // KTD-6: read the TYPED persona fields (soul / instructionsText), not // the non-existent `customInstructions` (which was silently undefined, // so node-agent persona injection never actually fired). Same fields // the column-agent path uses — one consistent persona source. const persona = this.buildAgentPersona(agent); if (persona) prompt = `${persona}\n\n${prompt}`; } else { await this.store.logEntry(live.id, `Workflow node '${node.id}': agent '${cfg.agentId}' not found — using default model`, undefined, this.getRunContextFor(live.id)); } } catch { // Agent lookup is best-effort; fall back to the default model. } } else if (executorKind === "skill" && typeof cfg.skillName === "string" && cfg.skillName.trim()) { // (U2) Prepend the Fusion workflow-step conventions preamble BEFORE the // "Invoke the skill" line. A skill node always runs as a workflow step here // (graph path → executeWorkflowStep), so the conventions always apply. prompt = `${FUSION_WORKFLOW_STEP_CONVENTIONS_PREAMBLE}Invoke the "${cfg.skillName}" skill with the following input, following the skill's instructions exactly:\n\n${prompt}`; } else if (executorKind === "cli") { const rawCommand = rawCliCommand; if (rawCommand) { // Arbitrary command: gated by trust-on-first-use approval unless the // node explicitly opts out. Two node flags bypass the pause: // - cliSkipApproval: CLI-specific "skip first-run approval". // - autoApprove: the node's general "Auto-approve requests" // toggle. The only human-approval pause reachable from a custom // node is this CLI gate (review-style nodes run as ephemeral // readonly agents with no permission gate), so honoring it here is // what makes that toggle actually do something. // The exact command string must otherwise have been approved by the user. // // SECURITY: both flags are intentional project-owner-only escape hatches. // They are only reachable by someone who can author/edit a workflow // definition for this project through the trusted dashboard editor / // executor lane — the same trust boundary that already lets them add // named scripts. They are NOT enforced at the IR-validation layer. // Prompt-injectable surfaces strip these flags at the write boundary // before persisting: the import / AI-design routes (stripApprovalFlags // in register-workflow-routes.ts) and the chat/planning workflow // authoring tools (createWorkflowAuthoringTools(..., {stripApprovalFlags: // true}) in chat.ts / planning.ts) — all via stripApprovalBypassFlags in // @fusion/core. Only the executor lane keeps these flags intact. const skipApproval = cfg.cliSkipApproval === true || cfg.autoApprove === true; if (!skipApproval && !(await this.store.isWorkflowCliCommandApproved(rawCommand))) { return this.pauseForCliApproval(node, live, rawCommand); } // We are proceeding to execute. If this task was previously paused by // THIS node's CLI-approval gate, clear that status/pausedReason now — // otherwise the task keeps the "awaiting-cli-approval" status through // later graph nodes even though approval already happened (mirrors the // status reset in runAwaitInputNode). const approvalMarker = `workflow-cli-approval:${node.id}`; if ((live.pausedReason ?? "").startsWith(approvalMarker)) { await this.store.updateTask(live.id, { status: null, pausedReason: null }, this.getRunContextFor(live.id)); } const env = prompt ? { ...process.env, FUSION_NODE_PROMPT: prompt } : undefined; const out = await this.runRawCliCommand( live, typeof cfg.name === "string" && cfg.name.trim() ? cfg.name : node.id, rawCommand, worktreePath, env, ); const blocking = node.kind === "gate" || cfg.gateMode === "gate"; return { outcome: out.success || !blocking ? "success" : "failure", value: out.success ? "passed" : "failed" }; } // No raw command: fall back to a named script (still required). if (!scriptName) { return { outcome: "failure", value: "cli-command-missing" }; } } const mode: "prompt" | "script" = executorKind === "cli" || node.kind === "script" || (node.kind === "gate" && scriptName) ? "script" : "prompt"; const now = new Date().toISOString(); // (U1) Carry the node's skill name onto the synthesized step so the step // session can actually LOAD it (executeWorkflowStep merges it into the // resolved skillSelection). Without this, the named skill was only injected // as prompt text pointing at a skill the session never discovered. const stepSkillName = executorKind === "skill" && typeof cfg.skillName === "string" && cfg.skillName.trim() ? cfg.skillName.trim() : undefined; /* * FNXC:Settings-ThinkingLevel 2026-07-10-00:00: * Graph model nodes can pin reasoning effort independently from modelProvider/modelId; carry only validated THINKING_LEVELS into the synthesized WorkflowStep. */ const stepThinkingLevel = typeof cfg.thinkingLevel === "string" && WORKFLOW_THINKING_LEVEL_SET.has(cfg.thinkingLevel) ? cfg.thinkingLevel as ThinkingLevel : undefined; const step: WorkflowStep = { id: `graph:${node.id}`, name: typeof cfg.name === "string" && cfg.name.trim() ? cfg.name : node.id, description: typeof cfg.description === "string" ? cfg.description : "", mode, phase: "pre-merge", gateMode: node.kind === "gate" || cfg.gateMode === "gate" ? "gate" : "advisory", prompt, toolMode: cfg.toolMode === "coding" ? "coding" : "readonly", scriptName, enabled: true, createdAt: now, updatedAt: now, ...(stepSkillName ? { skillName: stepSkillName } : {}), ...(cfg.requiresBrowser === true ? { requiresBrowser: true } : {}), ...(modelProvider && modelId ? { modelProvider, modelId } : {}), ...(stepThinkingLevel ? { thinkingLevel: stepThinkingLevel } : {}), }; if (cfg.summaryTarget === "task") { (step as WorkflowStep & { summaryTarget?: "task" }).summaryTarget = "task"; } if (cfg.requireExternalIntegrationEvidence === true) { (step as WorkflowStep & { requireExternalIntegrationEvidence?: boolean }).requireExternalIntegrationEvidence = true; } if (optionalGroupId) { (step as WorkflowStep & { optionalGroupId?: string }).optionalGroupId = optionalGroupId; } if (cfg.reviewCanFixInline === true) { (step as WorkflowStep & { reviewCanFixInline?: boolean }).reviewCanFixInline = true; } // (U8a) Thread the plugin-injected runtime env (FUSION_CE_SKILLS_DIR / // FUSION_CE_AGENTS_DIR + PATH contribution) into prompt-mode skill/model // steps on the GRAPH path. The legacy single-session caller builds this in // agentWork; the graph path never did, so skill loading and persona fan-out // silently no-op'd here. CLI executor keeps its own FUSION_NODE_PROMPT env. let nodeEnv: NodeJS.ProcessEnv | undefined; if (executorKind === "cli" && prompt) { nodeEnv = { ...process.env, FUSION_NODE_PROMPT: prompt }; } else if (mode === "prompt") { const injected = await this.buildInjectedRuntimeEnv(live.id, worktreePath, executionTarget.branch ?? undefined); nodeEnv = injected.env; executorLog.log( `${live.id}: graph node '${node.id}' runtime env injected (${injected.pathEntryCount} PATH entries, ${injected.injectedKeyCount} env keys)`, ); } // (U3) Genuinely-unattended signal. `unattended` is an explicit opt-in // threaded from the workflow-run options (default false = board run, where a // human can still answer asynchronously via the await-input card button). // No origin heuristic — absence always yields a board run. executeWorkflowStep // sets FUSION_HEADLESS=1 only when this is explicitly true. const unattended = this.graphUnattendedRuns.has(live.id); const outcome = mode === "script" ? await this.executeScriptWorkflowStep(live, step, worktreePath, settings, nodeEnv) : await this.executeWorkflowStep(live, step, worktreePath, settings, nodeEnv, { unattended }); // Skill-emitted await-input (U6): if the skill asked the user a blocking // question via the ===FUSION_AWAIT_INPUT=== sentinel, park the task // awaiting-user-input with the question (dashboard / task card surfaces it) // and halt the walk. On resume this node re-runs and the resume check above // consumes the user's steering reply. const awaitQuestion = parseAwaitInputSentinel((outcome as { output?: string }).output); if (awaitQuestion) { await this.store.logEntry( live.id, `Workflow step '${node.id}' is waiting for your input: ${awaitQuestion}`, undefined, this.getRunContextFor(live.id), ); await this.store.updateTask( live.id, { status: "awaiting-user-input", paused: true, pausedReason: `${skillAwaitMarker}@${Date.now()}: ${awaitQuestion}` }, this.getRunContextFor(live.id), ); return { outcome: "failure", value: "awaiting-user-input" }; } const blocking = step.gateMode === "gate"; // Script-mode outcomes carry no structured verdict; prompt-mode may. const verdict = (outcome as { verdict?: string }).verdict; // FNXC:WorkflowSteps 2026-06-26-00:00: Surface the step agent's output text // and parsed verdict notes on the node result's contextPatch so the // optional-group exit record carries them through to the recorded // WorkflowStepResult (workflow-graph-loop exitStepRecord → // workflow-graph-executor recordOptionalGroupStepResult). Without this the // Workflow tab only shows a generic fallback and `[pre-merge]` revision logs // pass `undefined` detail. `notes` is only attached when the parsed verdict // produced notes; `output` carries the raw step output when present. const stepOutput = (outcome as { output?: string }).output; const stepNotes = (outcome as { notes?: string }).notes; const contextPatch: Record = {}; if (typeof stepOutput === "string") contextPatch.output = stepOutput; if (typeof stepNotes === "string" && stepNotes) contextPatch.notes = stepNotes; if (cfg.summaryTarget === "task" && typeof stepOutput === "string" && stepOutput.trim()) { /* * FNXC:WorkflowCompletion 2026-06-29-11:09: * Built-in completion-summary nodes are agent/model workflow steps. Persist * their generated text through the graph projection path so summaries are * authored during workflow execution, before review/merge, and not only * synthesized later by recovery fallback code. */ contextPatch.summary = stepOutput.trim(); } /* * FNXC:PlanReview 2026-06-29-02:05: * Advisory graph steps still need a distinct non-pass value when their * review output is malformed. Returning plain `failed` made optional-group * recovery synthesize a Plan Review REVISE even when no reviewer requested * one; `advisory_failure` preserves visibility without inventing feedback. */ const malformed = (outcome as { malformed?: boolean }).malformed === true; const advisoryFailureValue = malformed ? "advisory_failure" : "failed"; /* FNXC:ReviewLeniency 2026-07-02-00:30: Malformed review output (no parseable verdict, even after the fallback-model retry in executeWorkflowStep) is treated as a NON-BLOCKING advisory rather than a hard gate failure. Operators asked that an unparseable reviewer response not block a task in review — a genuine REVISE (parsed verdict) still blocks, and the advisory_failure value keeps the malformed result visible on the Workflow tab. Only `malformed` relaxes a gate; every parsed non-pass verdict continues to block exactly as before. */ return { outcome: outcome.success || !blocking || malformed ? "success" : "failure", value: (outcome as WorkflowStepOutcome).failureValue ?? verdict ?? (outcome.success ? "passed" : advisoryFailureValue), ...(Object.keys(contextPatch).length > 0 ? { contextPatch } : {}), }; } /** * Resolve the cli-agent executor config off a workflow node (U7), snapshotting * the launch-time values. A mid-run node-config edit therefore applies to the * NEXT run only. Per-task overrides follow the existing per-task settings * precedent: when reachable cheaply we read a task field; otherwise the node * config is authoritative (documented hook point — `task.cliAdapterId` etc. are * not modeled on TaskDetail in v1, so node config is the sole source here). */ private resolveCliExecutorConfig(cfg: Record): ResolvedCliExecutorConfig | null { const cliAdapterId = typeof cfg.cliAdapterId === "string" && cfg.cliAdapterId.trim() ? cfg.cliAdapterId.trim() : undefined; if (!cliAdapterId) return null; const cliAutonomy = cfg.cliAutonomy && typeof cfg.cliAutonomy === "object" ? (cfg.cliAutonomy as ResolvedCliExecutorConfig["cliAutonomy"]) : null; const cliNotify = cfg.cliNotify && typeof cfg.cliNotify === "object" ? (cfg.cliNotify as Record) : null; const settings = cfg.cliSettings && typeof cfg.cliSettings === "object" ? (cfg.cliSettings as Record) : undefined; return { cliAdapterId, cliAutonomy, cliNotify, settings }; } /** * CLI Agent Executor seam (U7): run a `cli-agent` workflow node by driving an * engine-owned CLI session through the task-session orchestration. * * Re-entry policy (KTD): a re-entry into execute launches a FRESH session — any * prior live session for the task is killed first (context reset). The resolved * config is snapshotted at launch. * * Outcome mapping (R20 positive-completion gating): * - success → node success (pipeline advances; PTY reaped at handoff * to in-review via reapCliTaskSessionForHandoff). * - needs-attention / user-exited / auth-failed → node failure (the graph * failure handler parks the task for a human — never a silent stall). * - killed → node failure value "cli-agent-killed" (hard cancel * already moved the task; this just unwinds the graph walk). * * A CliConcurrencyLimitError at spawn surfaces as a clear typed error value * ("cli-agent-at-capacity") rather than a hang. */ private async runCliAgentNode( node: WorkflowIrNode, live: TaskDetail, cfg: Record, ): Promise { const runtime = this.options.cliAgentRuntime; if (!runtime) { await this.store.logEntry( live.id, `Workflow node '${node.id}' uses the cli-agent executor but no CLI agent runtime is wired`, undefined, this.getRunContextFor(live.id), ); return { outcome: "failure", value: "cli-agent-runtime-unavailable" }; } if (!live.worktree) { await this.store.logEntry( live.id, `Workflow node '${node.id}' (cli-agent) is write-capable but no task worktree exists yet — place it after the execute seam`, undefined, this.getRunContextFor(live.id), ); return { outcome: "failure", value: "no-worktree-for-write-node" }; } const config = this.resolveCliExecutorConfig(cfg); if (!config) { await this.store.logEntry( live.id, `Workflow node '${node.id}' (cli-agent) is missing 'cliAdapterId'`, undefined, this.getRunContextFor(live.id), ); return { outcome: "failure", value: "cli-agent-adapter-missing" }; } const prompt = typeof cfg.prompt === "string" ? cfg.prompt : (live.prompt ?? ""); // Re-entry: kill any prior LIVE session for this task (RETHINK/replan context // reset) before launching fresh. killLiveTaskSessions(live.id, runtime.manager, runtime.store); let session: CliTaskSession; try { session = await launchCliTaskSession({ taskId: live.id, projectId: runtime.projectId, worktreePath: live.worktree, prompt, config, manager: runtime.manager, hub: runtime.hub, registry: runtime.registry, hookEndpointUrl: runtime.hookEndpointUrl, hookDirRoot: runtime.hookDirRoot, log: (msg) => executorLog.log(`[cli-agent] ${msg}`), }); } catch (err) { if (err instanceof CliConcurrencyLimitError) { await this.store.logEntry( live.id, `cli-agent session for node '${node.id}' rejected at PTY pool ceiling (${err.active}/${err.ceiling}) — queued`, undefined, this.getRunContextFor(live.id), ); // A typed, surfaced state — NOT a silent stall. The graph failure handler // parks the task; a later sweep / capacity opening re-runs it. return { outcome: "failure", value: "cli-agent-at-capacity" }; } throw err; } this.activeCliTaskSessions.set(live.id, session); let outcome: CliTaskOutcome; try { outcome = await session.result(); } finally { // Detach the live-session handle. Reaping (success) / killing (cancel) is // handled per-outcome below or by the abort path. if (this.activeCliTaskSessions.get(live.id) === session) { this.activeCliTaskSessions.delete(live.id); } } switch (outcome.kind) { case "success": // Reap the PTY at the execute→in-review handoff (autoMerge:false tasks // don't hold slots): graceful kill, record terminationReason "completed". await this.reapCliTaskSessionForHandoff(session, live.id); return { outcome: "success", value: "cli-agent-done" }; case "killed": // Hard cancel already moved the task + killed the PTY via the abort path; // just unwind the graph walk. return { outcome: "failure", value: "cli-agent-killed" }; case "auth-failed": return { outcome: "failure", value: "cli-agent-auth-failed" }; case "user-exited": return { outcome: "failure", value: "cli-agent-user-exited" }; case "needs-attention": default: return { outcome: "failure", value: "cli-agent-needs-attention" }; } } /** * Reap a CLI task session at the execute→in-review handoff (U7). Graceful PTY * kill recorded as `completed`. Best-effort: a reap failure must not block the * pipeline advancement that the positive done already authorized. */ private async reapCliTaskSessionForHandoff(session: CliTaskSession, taskId: string): Promise { try { await session.reap(); } catch (err) { executorLog.warn(`${taskId}: failed to reap cli-agent session at handoff: ${err}`); } } private isTransientResumeAfterRestartGraphFailure(live: Task, result: WorkflowGraphTaskRunResult): boolean { if ((result.reason ?? "").trim().length > 0) return false; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (failedNode !== undefined && failedNode !== "execute") return false; if (live.steps.some((step) => step.status === "done")) return false; const failureState = live as Task & { lastError?: unknown; failureReason?: unknown }; if (failureState.lastError != null || failureState.failureReason != null) return false; const latestAction = live.log.at(-1)?.action; return latestAction === "Resumed after engine restart" || latestAction === "Resuming execution after unpause"; } /* FNXC:SessionContention 2026-07-25-21:30: Two ways in, because contention must never slip through to a park: - the typed failure value the graph now publishes (SESSION_CONTENTION_HOLD_VALUE), and - a message-shape fallback over the run's `:error` context patches, so contention arriving from a path that has not been taught the typed value is still recognized. */ private graphFailureErrorTexts(result: WorkflowGraphTaskRunResult): string[] { if (!result.context) return []; const texts: string[] = []; for (const [key, value] of Object.entries(result.context)) { if (key.endsWith(":error") && typeof value === "string" && value.trim()) texts.push(value); } return texts; } private isSessionContentionGraphFailure(result: WorkflowGraphTaskRunResult): boolean { if (this.graphFailureValue(result) === SESSION_CONTENTION_HOLD_VALUE) return true; return this.graphFailureErrorTexts(result).some((text) => isSessionContentionError(text)); } /* FNXC:SessionContention 2026-07-25-21:30 (self-recovering wait — the task is never parked): Retry the graph in place on an exponential backoff while the holder finishes. The counter is IN-MEMORY on purpose: it needs no schema change, and an engine restart resetting it is the desired behavior (a restart also drops the in-process registry, so the contention is gone anyway). When the ladder is exhausted the task is left cleanly dispatchable — status/error cleared, progress untouched — so ordinary scheduling picks it up later with a fresh budget. There is no terminal branch here by design: lease contention always ends (the holder finishes, or self-healing sweeps it), so parking the task would only require a human to press Retry on a condition that fixed itself. */ private sessionContentionHoldAttempts = new Map(); private clearSessionContentionHold(taskId: string): void { this.sessionContentionHoldAttempts.delete(taskId); } private async holdForSessionContention( task: Task, live: TaskDetail, result: WorkflowGraphTaskRunResult, ): Promise { const detail = this.graphFailureErrorTexts(result).find((text) => isSessionContentionError(text)); const priorAttempts = this.sessionContentionHoldAttempts.get(task.id) ?? 0; const attempt = priorAttempts + 1; if (attempt > MAX_SESSION_CONTENTION_HOLD_RETRIES) { this.clearSessionContentionHold(task.id); const message = `Still waiting on another task to release a shared session path after ${MAX_SESSION_CONTENTION_HOLD_RETRIES} attempts — leaving the task queued for normal re-dispatch (not a failure)${detail ? `: ${detail}` : ""}`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); if (live.status != null || live.error != null) { await this.store.updateTask(task.id, { status: null, error: null }, this.getRunContextFor(task.id)); } return; } this.sessionContentionHoldAttempts.set(task.id, attempt); const message = `Waiting on another task to release a shared session path — retrying in place (${attempt}/${MAX_SESSION_CONTENTION_HOLD_RETRIES})${detail ? `: ${detail}` : ""}`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); // A contention hold is not a failure state: clear any stale park so the row never shows as failed // while it is simply waiting its turn. if (live.status != null || live.error != null) { await this.store.updateTask(task.id, { status: null, error: null }, this.getRunContextFor(task.id)); } const delayMs = SESSION_CONTENTION_HOLD_BACKOFF_MS === 0 ? 0 : Math.min(SESSION_CONTENTION_HOLD_MAX_BACKOFF_MS, SESSION_CONTENTION_HOLD_BACKOFF_MS * 2 ** (attempt - 1)); const scheduleRetry = () => { void (async () => { try { const resume = await this.store.getTask(task.id); if (!resume || resume.deletedAt || resume.paused || resume.userPaused) { this.clearSessionContentionHold(task.id); return; } await this.execute(resume); } catch (err) { executorLog.error(`Failed session-contention retry for ${task.id}:`, err); } })(); }; setTimeout(scheduleRetry, delayMs).unref?.(); } /* FNXC:WorkflowExecutionOwnership 2026-07-29-20:10 (U8 / R4, PR #2590 review — greptile): The compat classifier keyed on `graphFailureValue`, which reads only the LAST visited node's value. That is correct when the generic `failure` edge goes straight to `end` — the built-in shape — but a user-authored graph may route its generic failure THROUGH another node, and that node's value then becomes the terminal one. The classifier would miss the pending-review ending entirely and the card would fall to the terminal park: `status: failed` on work that was only WAITING for a reviewer, which is the deadlock the inline handoff existed to avoid. A guard that cannot fire for the exact shape it was written for. The ending is durable in the run context — the graph publishes `node::value` for every node it runs — so detect it there rather than trusting whichever node happened to end the walk. */ private graphRunReportedPendingReview( result: WorkflowGraphTaskRunResult, failureValue: string | undefined, ): boolean { if (failureValue === "review-pending") return true; const context = result.context; if (!context) return false; /* FNXC:WorkflowExecutionOwnership 2026-07-29-21:40 (U8 / R4, PR #2590 review — greptile, 2nd): Scanning EVERY `node:*:value` was too broad in the opposite direction. The run context is shared for the whole walk, so a graph that continues past a pending-review node and then dies on a genuine downstream failure still carries the earlier value — and a blanket scan would park that card in review, hiding a real failure behind a wait. Trading a guard that misses for one that over-claims is not a fix. The narrow rule: the pending-review ending counts only when nothing AFTER it produced its own verdict. Walk the visited nodes backwards and take the first recorded value — that is the run's actual last word. If it is `review-pending`, the ending stands; if a later node spoke, that node's outcome is the run's, and this classifier stays out of the way. */ for (let i = result.visitedNodeIds.length - 1; i >= 0; i--) { const value = this.recordedNodeValue(context, result.visitedNodeIds[i]); if (typeof value === "string") return value === "review-pending"; } return false; } /* FNXC:WorkflowExecutionOwnership 2026-07-30-10:10 (U8, PR #2599 review — coderabbit, major): A visited node id does NOT always name the context key its value is stored under, and the two shapes that differ are the ones this unit cares about most. A foreach instance (`steps#0:step-execute`) records under the CONTAINER key `node:steps:value`; an optional-group template (`group::template`) records under the group key, then the template key. Reading `node::value` directly therefore misses a foreach ending and walks on to some earlier node's value — and the default coding workflow IS a foreach, so the backward walk would have misread precisely the shape it was written for. Extracted from `graphFailureValue`, which already knew this, so the two cannot drift apart. */ private recordedNodeValue(context: Record, nodeId: string): string | undefined { const direct = context[`node:${nodeId}:value`]; if (typeof direct === "string") return direct; const groupDelimiter = nodeId.indexOf("::"); if (groupDelimiter !== -1) { const groupValue = context[`node:${nodeId.slice(0, groupDelimiter)}:value`]; if (typeof groupValue === "string") return groupValue; const templateValue = context[`node:${nodeId.slice(groupDelimiter + 2)}:value`]; return typeof templateValue === "string" ? templateValue : undefined; } const foreachDelimiter = nodeId.indexOf("#"); if (foreachDelimiter === -1) return undefined; const containerValue = context[`node:${nodeId.slice(0, foreachDelimiter)}:value`]; return typeof containerValue === "string" ? containerValue : undefined; } private graphFailureValue(result: WorkflowGraphTaskRunResult): string | undefined { const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (!failedNode || !result.context) return undefined; const value = result.context[`node:${failedNode}:value`]; if (typeof value === "string") return value; /* FNXC:WorkflowLifecycle 2026-07-16-18:20: Optional-group template failures record materialized `::` ids in visitedNodeIds, but runOptionalGroup publishes context values under the UNQUALIFIED template id, and the group wrapper publishes the group's FINAL routing value (e.g. FN-7977's plan-review provider-failure hold) under the group id. FN-7996 parked terminally because this lookup only understood `#` foreach ids, so every graph-failure router (provider hold, awaiting states) missed group-template failures. Prefer the group's own value (it carries post-classification routing intent), then the template's. */ const groupInstanceDelimiter = failedNode.indexOf("::"); if (groupInstanceDelimiter !== -1) { const groupNode = failedNode.slice(0, groupInstanceDelimiter); const groupValue = result.context[`node:${groupNode}:value`]; if (typeof groupValue === "string") return groupValue; const templateNode = failedNode.slice(groupInstanceDelimiter + 2); const templateValue = result.context[`node:${templateNode}:value`]; return typeof templateValue === "string" ? templateValue : undefined; } const foreachInstanceDelimiter = failedNode.indexOf("#"); if (foreachInstanceDelimiter === -1) return undefined; /* FNXC:WorkflowLifecycle 2026-06-15-03:23: Foreach step-execute failures record instance ids in visitedNodeIds, but the graph walk stores the failed value on the foreach container context key. Check that container key before classifying execute-node failures so awaiting operator states from step-execute are preserved instead of parked as terminal graph failures. */ const foreachContainerNode = failedNode.slice(0, foreachInstanceDelimiter); const containerValue = result.context[`node:${foreachContainerNode}:value`]; return typeof containerValue === "string" ? containerValue : undefined; } private isAwaitingGraphFailureValue(value: string | undefined): value is "awaiting-user-input" | "awaiting-cli-approval" { return value === "awaiting-user-input" || value === "awaiting-cli-approval"; } /* FNXC:MissingWorktreeRecovery 2026-07-16-18:25: FN-7996: a session-start unusable-worktree refusal (assertValidWorktreeSession in pi.ts) thrown inside ANY workflow graph node (Plan Review, code review, custom gates) surfaced as a generic node "exception" and fell through every graph-failure router into the terminal park, which also OVERWROTE task.error with a generic message — erasing the signature the in-review missing-worktree self-healing sweep classifies on. The overseer then blindly re-dispatched the same stale task.worktree all day. Extract the underlying node error from the graph context so handleGraphFailure can route these into the same bounded recovery the execute session-start path already uses (clear stale worktree/branch/session metadata, requeue to todo, budgeted by worktreeSessionRetryCount). */ private extractUnusableWorktreeGraphFailure(result: WorkflowGraphTaskRunResult): string | null { if (!result.context) return null; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (!failedNode) return null; /* FNXC:MissingWorktreeRecovery 2026-07-16-19:40: Detection is scoped to the FAILED node's error keys only (exact id, plus the `group::template` / `container#N:template` materialized-id derivations under which runOptionalGroup/foreach publish template context). A catch-all scan over every `node:*:error` entry would match a STALE error left by an earlier, already-handled node and misroute an unrelated later failure into worktree recovery (greptile PR#2231 P1). */ const candidateKeys: string[] = [`node:${failedNode}:error`]; const groupInstanceDelimiter = failedNode.indexOf("::"); if (groupInstanceDelimiter !== -1) { candidateKeys.push(`node:${failedNode.slice(groupInstanceDelimiter + 2)}:error`); candidateKeys.push(`node:${failedNode.slice(0, groupInstanceDelimiter)}:error`); } const foreachInstanceDelimiter = failedNode.indexOf("#"); if (foreachInstanceDelimiter !== -1) { candidateKeys.push(`node:${failedNode.slice(0, foreachInstanceDelimiter)}:error`); const instanceRest = failedNode.slice(foreachInstanceDelimiter + 1); const templateDelimiter = instanceRest.indexOf(":"); if (templateDelimiter !== -1) { candidateKeys.push(`node:${instanceRest.slice(templateDelimiter + 1)}:error`); } } for (const key of candidateKeys) { const value = result.context[key]; if (typeof value === "string" && isMissingWorktreeSessionStartFailure(value)) return value; } return null; } private async routeUnusableWorktreeGraphFailureToRecovery( task: Task, live: TaskDetail, result: WorkflowGraphTaskRunResult, /** Shared per-recovery lane snapshot — see `resolveResumeLanes`. */ resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { if (live.deletedAt) return false; if (live.paused || live.userPaused === true) return false; if ((await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) return false; // Pause/abort provenance owns aborted runs; a genuine abort never carries the // session-start refusal as its terminal node error in the same walk. if (this.pausedAborted.has(task.id)) return false; const errorText = this.extractUnusableWorktreeGraphFailure(result); if (!errorText) return false; /* FNXC:MissingWorktreeRecovery 2026-07-16-19:40: FN-5147: with auto-merge off, `in-review` is terminal-until-human-merged — recovery must not move those tasks backward or re-enqueue them. Mirrors the gating the in-review self-healing sweep (recoverMissingWorktreeReviewFailures) applies before the same recovery. */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): FN-5147 — with the literal, a renamed board skipped this auto-merge-off gate entirely, so an automatic recovery moved a human-review-terminal card backward. #2689 converted the terminal guard at the top of this method; this is the other half of the same decision. */ if (live.column === (await this.resolveResumeLanes(live.id, resumeLanesMemo)).review) { const settings = await this.store.getSettings(); if (!allowsAutoMergeProcessing(live, settings)) return false; } const stalePath = extractMissingWorktreePathFromSessionStartFailure(errorText) ?? live.worktree ?? ""; const audit = createRunAuditor(this.store, { runId: this.getRunContextFor(task.id)?.runId ?? generateSyntheticRunId("graph-worktree-recovery", task.id), agentId: this.getRunContextFor(task.id)?.agentId ?? (task.assignedAgentId ?? "executor"), taskId: task.id, phase: "execute", }); const outcome = await this.recoverMissingWorktreeSessionStartFailure(live, stalePath, new Error(errorText), audit); // escalate-exhausted intentionally returns false: the failure falls through to the // visible terminal park so a human inspects the task instead of it looping silently. return outcome === "requeue-todo"; } private isMergeGraphFailure(failedNode: string | undefined): boolean { /* FNXC:WorkflowLifecycle 2026-06-19-00:00: FN-6735 requires every workflow merge-region node id to classify as a merge-seam graph failure. A benign pause/resume abort can surface as the synthetic legacy `merge`, `requestMerge`, or a primitive merge-region id, and all must route through bounded merge retry rather than terminal operator-action parking. */ if (!failedNode) return false; if (failedNode === "merge" || failedNode === "requestMerge") return true; if (MERGE_REGION_KINDS.has(failedNode as WorkflowIrNodeKind)) return true; return failedNode === "merge-manual-hold" || failedNode === "merge-retry"; } /* FNXC:WorkflowRemediation 2026-07-01-23:40: A live agent session surface for a task proves the work is still executing, independent of the persisted column/pause/status row that handleGraphFailure re-fetches. This mirrors clearPhantomExecutorBinding's `hasLiveSessionSurface` (FN-6736) but deliberately EXCLUDES `this.executing` and graph-routing membership: those are still set for the graph run that is currently ending (graphRouting is cleared in executeWorkflowGraph's finally, AFTER handleGraphFailure returns), so including them would report every ending run as "still executing" and suppress all failures. Only a registered coding/step/CLI session surface means a SEPARATE, live agent is working the task. */ private hasLiveTaskSessionSurface(taskId: string): boolean { return ( this.activeSessions.has(taskId) || this.activeStepExecutors.has(taskId) || this.activeWorkflowStepSessions.has(taskId) || this.activeCliTaskSessions.has(taskId) ); } /* FNXC:WorkflowRemediation 2026-07-01-23:40: A `pre-merge-remediation` / `plan-replan` node (e.g. `code-review-remediation`) is a FIRE-AND-FORGET async scheduler, not a terminal work node: its job is to hand off an implementation fix (sendTaskBackForFix re-dispatches the coding session) and stop traversal. These nodes carry only a `success` rework edge back to their gate and NO `failure` out-edge, so when their schedule call cannot re-arm (missing rehydrated failureContext after a restart → `missing-remediation-context`, `remediation-not-scheduled`, or an exhausted rework budget) the failure bubbles out as the terminal graph outcome and handleGraphFailure would stamp `status:"failed"` — even while a previously-scheduled fix/reviewer session is still live. Classify these nodes so that terminal sink can preserve a still-executing task instead of flagging a spurious failure. Detection prefers the resolved IR `workflowAction` (covers custom workflows), with a node-id fallback for the built-in ids when the IR cannot be resolved. */ private async isRemediationGraphNode(taskId: string, failedNode: string | undefined): Promise { if (!failedNode) return false; try { const ir = await resolveWorkflowIrForTask(this.store, taskId); const node = ir?.nodes?.find((n) => n.id === failedNode); const action = node?.config?.workflowAction; if (action === "pre-merge-remediation" || action === "plan-replan") return true; if (node) return false; } catch { // Best-effort IR resolution; fall through to the built-in id fallback. } return ( failedNode === "code-review-remediation" || failedNode === "browser-verification-remediation" || failedNode === "plan-replan" ); } /* FNXC:WorkflowRemediation 2026-07-03-23:10: Retryable parked-remediation recovery is only for pre-merge optional-step remediation nodes. Plan Review `plan-replan` failures must stay on the existing replan/triage path instead of delegating to `recoverFailedPreMergeWorkflowStep`, which reopens implementation work. */ private async isPreMergeRemediationGraphNode(taskId: string, failedNode: string | undefined): Promise { if (!failedNode) return false; try { const ir = await resolveWorkflowIrForTask(this.store, taskId); const node = ir?.nodes?.find((n) => n.id === failedNode); const action = node?.config?.workflowAction; if (action === "pre-merge-remediation") return true; if (node) return false; } catch { // Best-effort IR resolution; fall through to the built-in id fallback. } return failedNode === "code-review-remediation" || failedNode === "browser-verification-remediation"; } private isTerminalMergeGraphFailureValue(value: string | undefined): boolean { if (!value) return false; const normalized = value.toLowerCase(); return normalized.includes("conflict") || normalized.includes("contamination") || normalized.includes("foreign") || normalized.includes("retry-exhausted") || normalized.includes("retries exhausted") || normalized.includes("max retries"); } private latestFailedPreMergeWorkflowStep(task: Pick): CoreWorkflowStepResult | undefined { return (task.workflowStepResults ?? []) .filter((r) => (r.phase || "pre-merge") === "pre-merge" && r.status === "failed") .sort((a, b) => { const aTs = Date.parse(a.completedAt || a.startedAt || ""); const bTs = Date.parse(b.completedAt || b.startedAt || ""); return (Number.isFinite(bTs) ? bTs : 0) - (Number.isFinite(aTs) ? aTs : 0); })[0]; } private async resolveFailedPreMergeWorkflowStepBudget( task: Task, target: CoreWorkflowStepResult, ): Promise<{ unbounded: boolean; max: number; label: string; key: string; stepName?: string; attempts: number }> { const settings = await mergeEffectiveSettings(this.store, task, await this.store.getSettings()); const fallback = settings.maxPostReviewFixes ?? DEFAULT_MAX_POST_REVIEW_FIXES; let rawMaxRevisions: unknown; try { const ir = await resolveWorkflowIrForTask(this.store, task.id); if (ir.version === "v2") { const node = ir.nodes.find((candidate) => candidate.id === target.workflowStepId && candidate.kind === "optional-group"); rawMaxRevisions = node?.config?.maxRevisions; } } catch { rawMaxRevisions = undefined; } const maxRevisions = resolveOptionalReviewRevisionBudget({ optionalGroupId: target.workflowStepId ?? "", workflowSettings: settings as Record, nodeMaxRevisions: rawMaxRevisions, fallbackMaxRevisions: fallback, }); const budget = resolveOptionalStepRevisionBudget(maxRevisions, fallback); const key = optionalStepRevisionKey(target.workflowStepId, target.workflowStepName); return { ...budget, key, stepName: target.workflowStepName, attempts: countOptionalStepRevisionAttempts(task, key, target.workflowStepName), label: budget.unbounded ? "unbounded" : String(budget.max), }; } private async isLiveSharedBranchGroupMember(live: Pick): Promise { const groupId = live.branchContext?.groupId?.trim(); // FNXC:PostgresCutover 2026-07-10: getBranchGroup is async on the PG branch. const branchGroup = groupId ? await this.store.getBranchGroup(groupId) : null; return isLiveSharedBranchGroupMemberIntegration(live, branchGroup); } private async routeRetryableRemediationGraphFailureToPreMergeFix( live: TaskDetail, failedNode: string | undefined, failureValue: string | undefined, ): Promise { /* FNXC:WorkflowRemediation 2026-07-03-20:10: A failed `pre-merge-remediation` node is retryable when the durable blocking Code Review/optional-step result is still present and its revision budget remains. Route that parked graph failure through the same pre-merge fix handoff as live review REVISE handling; manual retry remains an escape hatch, not the primary recovery. Built-in Code Review defaults to an unbounded budget, so do not apply the legacy `postReviewFixCount` cap unless workflow settings or node config provide a numeric cap. */ if (!await this.isPreMergeRemediationGraphNode(live.id, failedNode)) return false; if (live.deletedAt || live.paused || live.userPaused === true) return false; if ((await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) return false; if (!live.worktree) return false; const settings = await this.store.getSettings().catch(() => undefined); if (!settings || settings.globalPause === true || settings.enginePaused === true) return false; /* FNXC:AutoMergeHold 2026-07-09-17:04: FN-7750 requires retryable pre-merge remediation to treat stale shared-group members as standalone manual-hold rows when global auto-merge is off; only live/open groups retain the shared-member exemption. */ if (!allowsAutoMergeProcessing(live, settings) && !(await this.isLiveSharedBranchGroupMember(live))) return false; const target = this.latestFailedPreMergeWorkflowStep(live); if (!target) return false; const budget = await this.resolveFailedPreMergeWorkflowStepBudget(live, target); if (!budget.unbounded && (!Number.isFinite(budget.max) || budget.max <= 0)) return false; if (!budget.unbounded && budget.attempts >= budget.max) return false; const nextCount = budget.attempts + 1; const totalFixCount = (live.postReviewFixCount ?? 0) + 1; await this.store.updateTask(live.id, { postReviewFixCount: totalFixCount }, this.getRunContextFor(live.id)); await this.store.logEntry( live.id, `Auto-recovered retryable remediation node '${failedNode ?? "unknown"}' for failed pre-merge workflow step (attempt ${nextCount}/${budget.label})`, optionalStepRevisionLogOutcome(`Step: ${budget.stepName ?? budget.key}${failureValue ? `\nGraph value: ${failureValue}` : ""}`, budget.key), this.getRunContextFor(live.id), ); const sentBack = await this.recoverFailedPreMergeWorkflowStep(live); if (!sentBack) return false; await this.persistTokenUsage(live.id); return true; } private isRetryableMergePauseAbortStatus(status: string | null | undefined): boolean { /* FNXC:WorkflowMerge 2026-07-01-22:05: FN-7335 surfaced a merge-node pause/resume abort while the row was legitimately `in-review` with status="reviewing" from the AI merge reviewer. That status is merge activity, not a pre-existing terminal failure; keep the retry classifier strict on real errors while allowing transient merge/review statuses to re-enter bounded merge retry. */ return status == null || status === "reviewing" || status === "merging" || status === "merging-pr"; } private async isRetryableBenignMergePauseAbort( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, pausedAborted: boolean, /** Shared per-recovery lane snapshot — see `resolveResumeLanes`; a fresh resolution here could disagree * with the one the rest of `handleGraphFailure` uses. */ resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { /* FNXC:WorkflowLifecycle 2026-06-19-00:05: FN-6735 treats a generic engine pause/resume abort at the merge seam as transient only when the row is still a clean in-review auto-merge candidate: no user/global pause, no pre-existing failure, no merge-confirmed partial landing, no terminal conflict/contamination value, within mergeRetries budget, and still eligible for auto-merge or shared-branch local integration. Anything outside those guards keeps the existing terminal operator-action park. */ if (!pausedAborted) return false; if (abortProvenance === "global-pause" || live.userPaused === true) return false; if (abortProvenance === "completion-finalize") return false; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: executor.ts review-lane classifiers, on top of #2689): "IS THIS CARD IN THE REVIEW LANE?" from the task's own workflow. Five pause-abort classifiers asked it as the default lineage's literal, and each refusal drops the card through to the operator-action park these paths exist to avoid (FN-6796's benign in-review abort, the manual-merge-hold abort, the two stale-replay handlers, this retryable merge abort). The literal made the recovery inert, silently. */ if (live.column !== (await this.resolveResumeLanes(live.id, resumeLanesMemo)).review || !this.isRetryableMergePauseAbortStatus(live.status) || live.error != null) return false; if (live.mergeDetails?.mergeConfirmed === true) return false; const failureValue = this.graphFailureValue(result); if (this.isTerminalMergeGraphFailureValue(failureValue)) return false; /* FNXC:WorkflowMerge 2026-07-12-17:38: FN-1165 / Runfusion#1991 — missing implementation proof is not a transient merge pause. Let the implementation-incomplete classifier fail closed or requeue resumable parsed steps before any requester can mint a no-branch no-op merge proof. */ if (failureValue === "implementation-incomplete") return false; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (!this.isMergeGraphFailure(failedNode)) return false; let settings: Settings | undefined; try { settings = await this.store.getSettings(); } catch { return false; } const sharedBranchMember = await this.isLiveSharedBranchGroupMember(live); if (!sharedBranchMember && !allowsAutoMergeProcessing(live, settings)) return false; if (!sharedBranchMember && resolveEffectiveAutoMerge(live, settings) === false) return false; if ((live.mergeRetries ?? 0) >= resolveMaxAutoMergeRetries(settings)) return false; return true; } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2703 review — greptile P1, and it is the most important finding in this sweep): THE SYNCHRONOUS RESOLVER IS A NO-OP IN PRODUCTION. `resolvePlannerLanes` reads `store.resolveTaskWorkflowIrSync`, whose selection reader is `getTaskWorkflowSelectionImpl` — and in PostgreSQL mode that function returns `undefined` unconditionally ("Backend mode cannot synchronously read PostgreSQL"). PostgreSQL is the shipped backend, so every sync-resolved conversion resolves the DEFAULT workflow and answers with the legacy ids no matter what board the task is on. That makes a sync conversion cosmetic: the census counts it as converted, `--strict` goes down by one, and the guard behaves exactly as the literal did. Worse than leaving the literal, because the number says the site is done. THE FIX IS TO STOP BEING SYNCHRONOUS, not to keep the literal. Both of this file's sync classifiers are called from async methods that have already awaited a store read, so the lane can be threaded in from the caller's existing snapshot — no new I/O, no second resolution, and the two halves of the decision provably read the same board. */ private isBenignInReviewPauseAbort( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, pausedAborted: boolean, userCanceled: boolean, /** The caller's already-resolved review lane — see the note above on the sync resolver. */ reviewLane: string, ): boolean { /* FNXC:WorkflowLifecycle 2026-06-20-00:00: FNXC:WorkflowLifecycle 2026-07-26-11:20: KB-PROV: post-split the engine case arrives as `engine-abort` and an operator withdrawal as `hard-cancel`; this classifier still accepts BOTH (`isGenericAbortProvenance`) because the `userCanceled` guard below — not the label — is the load-bearing operator-intent discriminator FN-6796 designed. Narrowing to `engine-abort` would change behaviour for the operator path. FN-6796: an engine restart/pause-resume abort reaches graph-failure handling as `hard-cancel`/`engine-abort` provenance even when no user canceled the task. A clean completed `in-review` row in that shape is already handed off for review and must not be stranded with the operator-action pause-abort marker; the discriminator is the in-memory `userCanceledTaskIds` set plus the resting column and clean row state, while global/user pause, merge-seam, terminal merge values, merge-confirmed partial landings, and pre-existing status/error still park exactly as before. */ if (!pausedAborted) return false; if (!isGenericAbortProvenance(abortProvenance)) return false; if (userCanceled) return false; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2703 review — replaces my own earlier reasoning): This comparison used the SYNC `resolvePlannerLanes`, which I justified as the right resolver for a synchronous classifier. That justification was wrong in production: in PostgreSQL mode the sync selection reader always returns undefined, so the sync resolver hands back the DEFAULT workflow's lanes and the guard behaves exactly as the literal did. The lane now arrives from the caller's snapshot — see the note on this method. */ if (live.column !== reviewLane) return false; if (live.userPaused === true) return false; if (live.status != null || live.error != null) return false; if (live.mergeDetails?.mergeConfirmed === true) return false; if (this.isTerminalMergeGraphFailureValue(this.graphFailureValue(result))) return false; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (this.isMergeGraphFailure(failedNode)) return false; if (live.steps.length === 0) return false; if (!live.steps.every((step) => step.status === "done" || step.status === "skipped")) return false; return true; } private isStalePauseAbortParkFailure(live: TaskDetail, nodeId = "plan"): boolean { return live.status === "failed" && typeof live.error === "string" && live.error.includes(PAUSE_ABORT_PARK_ERROR_MARKER) && live.error.includes("engine abort during pause/resume") && live.error.includes(`at node '${nodeId}'`); } private async isBenignManualMergeHoldPauseAbort( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, pausedAborted: boolean, /** Shared per-recovery lane snapshot — see `resolveResumeLanes`; a fresh resolution here could disagree * with the one the rest of `handleGraphFailure` uses. */ resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { /* FNXC:WorkflowLifecycle 2026-07-09-14:54: FN-7749 / Runfusion#1979: with auto-merge off, a manual merge hold is the healthy `in-review` resting state for Merge & Close. A benign generic (`hard-cancel`/`engine-abort`, KB-PROV 2026-07-26) pause/resume abort at any merge-region node must not park the task failed; FN-5147 forbids moving, failing, or re-enqueueing the row, so this classifier only permits preserving `in-review` and clearing a stale pause-abort status/error. */ if (!pausedAborted) return false; if (!isGenericAbortProvenance(abortProvenance)) return false; if (live.paused || live.userPaused === true) return false; if (live.column !== (await this.resolveResumeLanes(live.id, resumeLanesMemo)).review) return false; if (live.mergeDetails?.mergeConfirmed === true) return false; if (this.isTerminalMergeGraphFailureValue(this.graphFailureValue(result))) return false; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (!this.isMergeGraphFailure(failedNode)) return false; const cleanRow = live.status == null && live.error == null; const staleParkedFailure = this.isStalePauseAbortParkFailure(live, failedNode); if (!cleanRow && !staleParkedFailure) return false; let settings: Settings | undefined; try { settings = await this.store.getSettings(); } catch { return false; } /* FNXC:AutoMergeHold 2026-07-09-17:07: FN-7749's benign manual-hold classifier must exclude only live shared-group integrations. FN-7750 stale shared-group members are standalone manual-hold rows and should not be stranded as pause-abort failures. */ if (await this.isLiveSharedBranchGroupMember(live)) return false; return !allowsAutoMergeProcessing(live, settings) || resolveEffectiveAutoMerge(live, settings) === false; } private async handleStaleInReviewPlanPauseAbortReplay( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, pausedAborted: boolean, userCanceled: boolean, /** Shared per-recovery lane snapshot — see `resolveResumeLanes`; a fresh resolution here could disagree * with the one the rest of `handleGraphFailure` uses. */ resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { /* FNXC:WorkflowLifecycle 2026-06-28-21:05: FN-7143 showed that a stale graph lifecycle replay can surface at `plan` after an in-review pause/resume even though planning is not actually running anymore. Plan is not a safe re-entry point for review rows, typed or generic, so this classifier is clear/log-only: preserve in-review, never route to triage/todo, and keep genuine user/global pauses plus real plan failures on the operator-action path. */ if (!pausedAborted) return false; if (!isGenericAbortProvenance(abortProvenance) && abortProvenance !== "global-pause") return false; if (userCanceled) return false; if (live.column !== (await this.resolveResumeLanes(live.id, resumeLanesMemo)).review) return false; if (live.paused || live.userPaused === true) return false; if (live.autoMerge === false) return false; if (live.mergeDetails?.mergeConfirmed === true) return false; if (result.interruptedAbortKind && result.interruptedAbortKind !== WORKFLOW_NODE_ENGINE_PAUSE_ABORT_KIND) return false; const failedNode = result.interruptedNodeId ?? result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (failedNode !== "plan") return false; if (this.isMergeGraphFailure(failedNode)) return false; const failureValue = typeof result.context?.[`node:${failedNode}:value`] === "string" ? result.context[`node:${failedNode}:value`] as string : this.graphFailureValue(result); if (failureValue !== "aborted") return false; if (this.isTerminalMergeGraphFailureValue(failureValue)) return false; const cleanRow = live.status == null && live.error == null; const staleParkedFailure = this.isStalePauseAbortParkFailure(live, "plan"); if (!cleanRow && !staleParkedFailure) return false; let settings: Settings; try { settings = await this.store.getSettings(); } catch { return false; } if (settings.globalPause === true || settings.enginePaused === true) return false; if (!allowsAutoMergeProcessing(live, settings) && !(await this.isLiveSharedBranchGroupMember(live))) return false; this.clearPausedAborted(live.id); this.activeWorktrees.delete(live.id); const message = "Workflow graph plan node pause/resume replay surfaced after task was already in-review — stale replay ignored, in-review state preserved"; executorLog.log(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); if (staleParkedFailure) { await this.store.updateTask(live.id, { status: null, error: null }, this.getRunContextFor(live.id)); await this.store.logEntry(live.id, "Auto-recovered: cleared stale in-review plan pause/resume replay failure — failure notification suppressed", undefined, this.getRunContextFor(live.id)); } try { await this.store.recordRunAuditEvent?.({ taskId: live.id, agentId: "executor", runId: generateSyntheticRunId("workflow-stale-plan-replay", live.id), domain: "database", mutationType: "task:classify-stale-in-review-plan-pause-abort-replay", target: live.id, metadata: { nodeId: failedNode, fromColumn: live.column, abortProvenance, clearedStaleFailure: staleParkedFailure, graphResumeRetryCount: live.graphResumeRetryCount ?? 0, mode: "preserved-in-review", }, }); } catch (error) { executorLog.warn(`${live.id}: failed to record stale plan replay audit: ${error instanceof Error ? error.message : String(error)}`); } await this.persistTokenUsage(live.id); return true; } private async handleStaleInReviewParsePauseAbortReplay( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, pausedAborted: boolean, userCanceled: boolean, /** Shared per-recovery lane snapshot — see `resolveResumeLanes`; a fresh resolution here could disagree * with the one the rest of `handleGraphFailure` uses. */ resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { /* FNXC:WorkflowLifecycle 2026-06-29-01:18: A stale in-review pause/resume replay at `parse` is not an operator action. Unlike `plan`, parse is a safe workflow re-entry point for review rows, so auto-retry the graph with the shared transient resume budget and suppress the parked failure notification. */ if (!pausedAborted) return false; if (!isGenericAbortProvenance(abortProvenance) && abortProvenance !== "global-pause") return false; if (userCanceled) return false; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): ONE SNAPSHOT for the entry gate AND the deferred recheck inside `scheduleRetry` below — the recheck is the second half of THIS decision ("is the card still where it was when we admitted it?"), so resolving the board again inside the timeout callback would let a workflow edit make the two halves disagree. */ const replayLanes = await this.resolveResumeLanes(live.id, resumeLanesMemo); if (live.column !== replayLanes.review) return false; if (live.paused || live.userPaused === true) return false; if (live.autoMerge === false) return false; if (live.mergeDetails?.mergeConfirmed === true) return false; if (result.interruptedAbortKind && result.interruptedAbortKind !== WORKFLOW_NODE_ENGINE_PAUSE_ABORT_KIND) return false; const failedNode = result.interruptedNodeId ?? result.visitedNodeIds[result.visitedNodeIds.length - 1]; if (failedNode !== "parse") return false; const failureValue = typeof result.context?.[`node:${failedNode}:value`] === "string" ? result.context[`node:${failedNode}:value`] as string : this.graphFailureValue(result); if (failureValue !== "aborted") return false; if (this.isTerminalMergeGraphFailureValue(failureValue)) return false; const cleanRow = live.status == null && live.error == null; const staleParkedFailure = this.isStalePauseAbortParkFailure(live, "parse"); if (!cleanRow && !staleParkedFailure) return false; const priorRetries = live.graphResumeRetryCount ?? 0; if (priorRetries >= MAX_TRANSIENT_GRAPH_RESUME_RETRIES) return false; let settings: Settings; try { settings = await this.store.getSettings(); } catch { return false; } if (settings.globalPause === true || settings.enginePaused === true) return false; if (!allowsAutoMergeProcessing(live, settings) && !(await this.isLiveSharedBranchGroupMember(live))) return false; const nextRetries = priorRetries + 1; this.clearPausedAborted(live.id); this.activeWorktrees.delete(live.id); const message = `Workflow graph parse node pause/resume replay surfaced after task was already in-review — auto-retrying workflow graph (${nextRetries}/${MAX_TRANSIENT_GRAPH_RESUME_RETRIES})`; executorLog.log(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); await this.store.logEntry(live.id, "Auto-recovered: retrying stale in-review parse pause/resume replay — failure notification suppressed", undefined, this.getRunContextFor(live.id)); await this.store.updateTask(live.id, { graphResumeRetryCount: nextRetries, status: null, error: null }, this.getRunContextFor(live.id)); try { await this.store.recordRunAuditEvent?.({ taskId: live.id, agentId: "executor", runId: generateSyntheticRunId("workflow-stale-parse-retry", live.id), domain: "database", mutationType: "task:retry-stale-in-review-parse-pause-abort-replay", target: live.id, metadata: { nodeId: failedNode, fromColumn: live.column, attempt: nextRetries, maxAttempts: MAX_TRANSIENT_GRAPH_RESUME_RETRIES, abortProvenance: abortProvenance ?? "unknown", clearedStaleFailure: staleParkedFailure, mode: "preserved-in-review-retry-graph", }, }); } catch (error) { executorLog.warn(`${live.id}: failed to record stale parse replay retry audit: ${error instanceof Error ? error.message : String(error)}`); } await this.persistTokenUsage(live.id); const scheduleRetry = () => { void (async () => { try { const resumeTask = await this.store.getTask(live.id); if ( resumeTask.deletedAt || resumeTask.paused || resumeTask.userPaused || resumeTask.status != null || resumeTask.error != null || resumeTask.column !== replayLanes.review || this.activeSessions.has(live.id) || this.activeStepExecutors.has(live.id) || this.activeWorkflowStepSessions.has(live.id) || this.activeWorkflowGraphAbortControllers.has(live.id) || TaskExecutor.processWideGraphRouting.has(live.id) ) { executorLog.debug(`${live.id}: skipping stale parse graph retry — task is no longer in a safe in-review resume state`); return; } await this.executeWorkflowGraph(resumeTask); } catch (err) { executorLog.error(`Failed stale parse graph retry for ${live.id}:`, err); } })(); }; if (TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS > 0) { const handle = setTimeout(scheduleRetry, TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS); handle.unref?.(); } else { setTimeout(scheduleRetry, 0).unref?.(); } return true; } private async isReentrantPausedAbortedInFlightNode( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, pausedAborted: boolean, userCanceled: boolean, resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { /* FNXC:WorkflowLifecycle 2026-06-28-18:32: FN-7214 makes engine-internal pause aborts re-entrant only when the workflow graph reports a typed in-flight node interruption. User pauses, active global pauses, merge/finalize aborts, genuine node failures, autoMerge:false review rows, and exhausted retry budgets must continue through the existing protected failure paths. FNXC:WorkflowLifecycle 2026-06-28-21:39: A global engine pause aborts active workflow graph controllers with `global-pause` provenance; after the global pause is lifted, the typed interrupted-node marker is sufficient to re-enter that node. Only active global-pause settings and explicit task/user pauses remain terminal so resume never runs behind an operator-controlled pause. */ if (!pausedAborted) return false; if (!isGenericAbortProvenance(abortProvenance) && abortProvenance !== "global-pause") return false; if (userCanceled) return false; if (live.paused || live.userPaused === true) return false; if (live.status != null || live.error != null) return false; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: executor.ts — the split-snapshot defect): THE LANES ARE RESOLVED HERE, AT THE TOP, because this method already resolved them — at the very END, for its return value — while every eligibility check below compared against the default lineage's literals. On a renamed board the four `in-review` gates all read false, so a card in review skipped the global-pause recheck, the `autoMerge === false` refusal, the shared-branch-member arbitration and the merge-confirmed refusal — and then the final line, which DOES resolve lanes, answered "re-entrant". FN-7214's comment above says an auto-merge-off review row must stay terminal. */ const resumeLanes = await this.resolveResumeLanes(live.id, resumeLanesMemo); if ((await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) return false; if (result.interruptedAbortKind !== WORKFLOW_NODE_ENGINE_PAUSE_ABORT_KIND) return false; if (!result.interruptedNodeId) return false; if (live.column === resumeLanes.review && result.interruptedNodeId === "plan") return false; if (this.isMergeGraphFailure(result.interruptedNodeId)) return false; if (this.isTerminalMergeGraphFailureValue(this.graphFailureValue(result))) return false; if ((live.graphResumeRetryCount ?? 0) >= MAX_TRANSIENT_GRAPH_RESUME_RETRIES) return false; let settings: Settings | undefined; if (abortProvenance === "global-pause" || live.column === resumeLanes.review) { try { settings = await this.store.getSettings(); } catch { return false; } if (settings.globalPause === true) return false; } if (live.column === resumeLanes.review) { if (live.autoMerge === false) return false; if (!settings) return false; const sharedBranchMember = await this.isLiveSharedBranchGroupMember(live); if (!sharedBranchMember && !allowsAutoMergeProcessing(live, settings)) return false; if (live.mergeDetails?.mergeConfirmed === true) return false; } return live.column === resumeLanes.hold || live.column === resumeLanes.review || live.column === resumeLanes.wip; } /* FNXC:WorkflowLifecycleColumns 2026-07-30-16:00 (Phase C convergence — resume eligibility): The columns a RESUME may legitimately start from, resolved from the task's own workflow: the hold (backlog) lane, the wip lane, and the review lane. These decisions were spelled as the default lineage's three names, so on a renamed board every resume-safety check answered "not a safe resume state" and the paused-node re-entry, the pause-abort auto-continue, and the benign-todo abort-marker clear all stopped firing. The last of those is the one that bites: FN-6478's benign path exists so a re-queued card clears its abort marker instead of being parked `failed` for an operator — and on a renamed board it took the operator-action branch instead, which is the retry storm that path was written to end. ASYNC on purpose: every call site here is already async (a store read precedes each one), so there is no listener-ordering hazard of the kind that forced the synchronous planner-lane resolver in `replan-target.ts`. Fail-soft to the legacy trio so an unresolvable or column-less workflow behaves as before. FOLLOW-UP, deliberately not done here: PR #2628 exports a synchronous `resolvePlannerLanes` (hold/intake/wip) from `replan-target.ts`. Once both land, this helper and that one should become one resolver returning the full lane set — two resolvers for the same question is the drift this program keeps paying for. Kept separate now only to avoid a cross-branch dependency. */ private async resolveResumeLanes( taskId: string, memo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise<{ hold: string; wip: string; review: string; wipDeclared: boolean }> { /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2640 review, greptile P2): ONE RESOLUTION PER RECOVERY, and the reason is correctness as much as I/O. Eligibility and re-entry ran this separately, so a workflow edit landing between the two calls would have the two halves of one decision reading DIFFERENT lane sets — the eligibility check admits a card in review, the re-entry then resolves a board where that column is not the review lane. The memo is caller-owned and per-recovery, which is the same shape as the IR caches elsewhere in the engine: one snapshot for one decision, never a process-lifetime cache that has to guess when a mid-flight workflow edit invalidates it. */ if (memo?.lanes) return memo.lanes; try { const lifecycle = resolveLifecycleColumns(await resolveWorkflowIrForTask(this.store, taskId)); const lanes = { hold: lifecycle?.hold ?? "todo", wip: lifecycle?.wip ?? "in-progress", review: lifecycle?.review ?? "in-review", /* FNXC:WorkflowLifecycleColumns 2026-07-30-15:30 (PR #2760 review — greptile P1): Whether the resolved IR actually DECLARES an implementation lane, which the `?? "in-progress"` default above destroys. Callers that must not act without a real implementation lane read this instead of comparing against the default. THREE states, not two, and conflating the last two is a regression: a. wip declared -> true b. lifecycle lanes declared, wip NOT -> FALSE; the workflow genuinely has no implementation lane, so there is nowhere to resume TO c. NO lifecycle lane declared at all -> true; this is a v1 workflow upgraded in place. Its synthesized columns carry `traits: []`, so `resolveLifecycleColumns` returns `{}` — measured, not assumed — and treating that as "no wip lane" would terminalize every legacy custom workflow's graph-failure recovery instead of resuming it. The discriminator is whether the IR expresses lifecycle intent AT ALL. An untraited legacy board expresses none, so the legacy trio is the honest answer and today's behaviour is preserved. */ wipDeclared: lifecycle?.wip !== undefined || !declaresAnyLifecycleRole(lifecycle), }; if (memo) memo.lanes = lanes; return lanes; } catch { // IR unavailable: we cannot know, so keep the legacy board's assumption and today's behaviour. const lanes = { hold: "todo", wip: "in-progress", review: "in-review", wipDeclared: true }; if (memo) memo.lanes = lanes; return lanes; } } private async reenterPausedAbortedWorkflowNode( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { const nodeId = result.interruptedNodeId ?? result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown"; const priorRetries = live.graphResumeRetryCount ?? 0; if (priorRetries >= MAX_TRANSIENT_GRAPH_RESUME_RETRIES) return false; const nextRetries = priorRetries + 1; /* FNXC:WorkflowLifecycleColumns 2026-07-30-16:05: resolved ONCE for the whole re-entry — `preservedInReview`, the audit `mode` label, the resume-safety recheck, and the branch that picks execute() vs executeWorkflowGraph() must all agree on which column is which. They were four independent literal comparisons, so on a renamed board `preservedInReview` was false for a card in review AND the recheck rejected it, and the re-entry silently never happened. */ const reentryLanes = await this.resolveResumeLanes(live.id, resumeLanesMemo); const preservedInReview = live.column === reentryLanes.review; this.clearPausedAborted(live.id); this.activeWorktrees.delete(live.id); const message = `Workflow graph node '${nodeId}' was interrupted by engine pause/resume — re-entering workflow graph (${nextRetries}/${MAX_TRANSIENT_GRAPH_RESUME_RETRIES})`; executorLog.log(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); await this.store.logEntry(live.id, `Auto-recovered: re-entering paused-aborted workflow graph node '${nodeId}' — failure notification suppressed`, undefined, this.getRunContextFor(live.id)); await this.store.updateTask(live.id, { graphResumeRetryCount: nextRetries, status: null, error: null }, this.getRunContextFor(live.id)); try { await this.store.recordRunAuditEvent?.({ taskId: live.id, agentId: "executor", runId: generateSyntheticRunId("workflow-node-reentry", live.id), domain: "database", mutationType: "task:reenter-paused-aborted-workflow-node", target: live.id, metadata: { nodeId, fromColumn: live.column, attempt: nextRetries, maxAttempts: MAX_TRANSIENT_GRAPH_RESUME_RETRIES, abortProvenance: abortProvenance ?? "unknown", preservedInReview, mode: preservedInReview ? "preserved-in-review" : live.column === reentryLanes.hold ? "reexecuted-from-todo" : "reentered-graph", }, }); } catch (error) { executorLog.warn(`${live.id}: failed to record paused-node graph re-entry audit: ${error instanceof Error ? error.message : String(error)}`); } await this.persistTokenUsage(live.id); const scheduleRetry = () => { void (async () => { try { const resumeTask = await this.store.getTask(live.id); if ( resumeTask.deletedAt || resumeTask.paused || resumeTask.userPaused || resumeTask.status != null || resumeTask.error != null || (preservedInReview ? resumeTask.column !== reentryLanes.review : resumeTask.column !== reentryLanes.hold && resumeTask.column !== reentryLanes.wip) || this.activeSessions.has(live.id) || this.activeStepExecutors.has(live.id) || this.activeWorkflowStepSessions.has(live.id) || this.activeWorkflowGraphAbortControllers.has(live.id) || TaskExecutor.processWideGraphRouting.has(live.id) ) { executorLog.debug(`${live.id}: skipping paused-node graph re-entry — task is no longer in a safe resume state`); return; } if (preservedInReview) { await this.executeWorkflowGraph(resumeTask); } else if (resumeTask.column === reentryLanes.hold) { await this.execute(resumeTask); } else { await this.executeWorkflowGraph(resumeTask); } } catch (err) { executorLog.error(`Failed paused-node graph re-entry for ${live.id}:`, err); } })(); }; if (TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS > 0) { const handle = setTimeout(scheduleRetry, TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS); handle.unref?.(); } else { setTimeout(scheduleRetry, 0).unref?.(); } return true; } private async routeGraphMergeFailureToRetry( live: TaskDetail, result: WorkflowGraphTaskRunResult, abortProvenance: PausedAbortProvenance | undefined, ): Promise { if (!this.mergeRequester) return false; /* FNXC:WorkflowMerge 2026-07-12-17:38: FN-1165 defense in depth — implementation-incomplete merge graph failures must never reach the merge requester, because a no-branch task can otherwise be finalized as an intentional no-op. */ if (this.graphFailureValue(result) === "implementation-incomplete") return false; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown"; const message = `Workflow graph merge failure at node '${failedNode}' routed to bounded auto-merge retry${abortProvenance === "merge-seam" ? " after merge-seam abort" : isGenericAbortProvenance(abortProvenance) || abortProvenance === undefined ? " after benign pause/resume abort" : ""}`; executorLog.warn(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); try { const mergeTask = await this.ensureWorkflowMergeBoundaryTask(live, { reason: "workflow-merge-retry-boundary", nodeId: failedNode, workflowId: result.context?.["workflow:id"] as string | undefined ?? "workflow-graph", runId: this.getRunContextFor(live.id)?.runId ?? "graph-merge-retry", }); await this.mergeRequester(mergeTask.id); } catch (error) { executorLog.warn(`${live.id}: bounded auto-merge retry request failed after graph merge failure: ${error instanceof Error ? error.message : String(error)}`); } await this.persistTokenUsage(live.id); return true; } private async routeImplementationIncompleteMergeGraphFailure(live: TaskDetail, failedNode: string): Promise { /* FNXC:WorkflowMerge 2026-07-14-18:20: FN-1165 greptile P1s: (1) system-paused implementation-incomplete merge failures must still classify — clear only non-user pause parks so incomplete steps can requeue; real global/user pauses never enter this method. (2) Do not drop activeWorktrees until we know the outcome is terminal fail-closed. Resumable requeue preserves progress (and often the persisted worktree); releasing tracking early leaves that worktree uncounted while a later dispatch can allocate a second one. Keep the active registration on the resumable path; release only on fail-closed. */ this.clearPausedAborted(live.id); let resumeLive = live; if (live.paused === true && live.userPaused !== true) { // FNXC:WorkflowMerge 2026-07-14-18:35: TaskDetail.pausedReason is string|undefined (not null). Persist clear via updateTask (store accepts null); in-memory resume snapshot uses undefined to satisfy the type. await this.store.updateTask(live.id, { paused: false, pausedReason: null, }, this.getRunContextFor(live.id)); resumeLive = { ...live, paused: false, pausedReason: undefined }; } if (hasNonTerminalWorkflowSteps(resumeLive) && await this.routeGraphFailureToExecutionResume(resumeLive, failedNode, "implementation-incomplete")) { return true; } // Fail-closed terminal path — release active worktree tracking now that no resume will reuse it. this.activeWorktrees.delete(live.id); const message = `Workflow graph merge blocked at node '${failedNode}': implementation incomplete with no executable proof to resume — failing instead of retrying merge`; executorLog.warn(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); if (!(await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column) && live.error == null) { await this.store.updateTask(live.id, { error: message, status: "failed" }, this.getRunContextFor(live.id)); } await this.persistTokenUsage(live.id); return true; } private async hasTrailingConsecutiveToolFailures(taskId: string, cursor: number | null | undefined, threshold: number): Promise { if (cursor == null) return false; /* FNXC:ExecutorToolFailureRetry 2026-07-17-06:30: Optional log APIs on minimal/test stores: missing getAgentLogCount/getAgentLogs cannot prove a trailing failure streak, so return false rather than throw mid-failure handling. */ if (typeof this.store.getAgentLogCount !== "function" || typeof this.store.getAgentLogs !== "function") { return false; } const currentCount = await this.store.getAgentLogCount(taskId).catch(() => cursor); if (currentCount <= cursor) return false; const entries = await this.store.getAgentLogs(taskId, { limit: currentCount - cursor }).catch(() => []); let failures = 0; for (let index = entries.length - 1; index >= 0; index -= 1) { const type = entries[index]!.type; if (type === "tool_result") return false; if (type === "tool_error") { failures += 1; if (failures >= threshold) return true; } // Invocation markers and non-completion entries intentionally do not reset the run. } return false; } /** Terminal failure of a graph run: record the error and park the task in * review so a human can act — never leave it invisible in in-progress. */ private async handleGraphFailure(task: Task, result: WorkflowGraphTaskRunResult): Promise { this.clearCompletedTaskWatchdog(task.id); this.options.stuckTaskDetector?.untrackTask(task.id); try { const loadedLive = await this.store.getTask(task.id); /* FNXC:WorkflowLifecycle 2026-06-23-12:01: Graph failure handling must never mutate a different task row than the one that entered execute(). Minimal stores can return fallback rows from getTask(); treat that as an unavailable live snapshot and leave the inner executor recovery result intact instead of handing off the wrong task. */ if (!loadedLive || loadedLive.id !== task.id) { executorLog.warn(`${task.id}: graph failure live-state refetch returned ${loadedLive?.id ?? "null"} — preserving inner executor result`); await this.persistTokenUsage(task.id); return; } const live = loadedLive; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: executor.ts handleGraphFailure): ONE LANE SNAPSHOT FOR THE WHOLE METHOD, declared where `live` first exists. The three wip comparisons below run BEFORE the re-entry classifiers' memo was created, so a snapshot declared beside that memo is used-before-declared — which is how the two halves came to read different boards in the first place. The memo is seeded from this snapshot so the classifiers still share it. */ const resumeLanesMemo: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } } = {}; const failureLanes = await this.resolveResumeLanes(live.id, resumeLanesMemo); /* FNXC:Lifecycle 2026-07-16-21:22: FN-8141 follow-up 1 — an honest `fn_task_done(outcome="blocked")` park (status="failed", error "BLOCKED: ", executor ~14657) must SURVIVE the same graph-teardown machinery that undid the original incident's failed park. Every downstream classifier in this method can wash the marker out: the genuine-pause-abort todo-rehome branch (~9504) clears status/error on a task the abort bounced back to `todo`; the execution-resume router and the terminal graph-failure sink (~9982) overwrite the distinctive `BLOCKED:` error with a generic "Workflow graph terminated with failure" string; and the engine-internal auto-continue (~9540) re-runs the doomed session. Self-healing (#2257/#2260) and dependency-gated scheduling key off this exact `BLOCKED:` error + the recorded blockedBy dependencies, so any of those would re-open the laundering hole. Detect the live blocked park BEFORE every other classifier and honor it exactly like the non-graph post-loop honor-park (executor ~12163): clear the in-memory pause-abort marker so `recoverPausedAbortFailures` has nothing to chase, RELEASE the worktree/concurrency slot (FN-6782 leaked-`maxWorktrees`-holder precedent; the graph finally does not delete `activeWorktrees`), and return WITHOUT touching status/error/column/ dependencies/steps — the park stays intact for the blocker/operator. Unblocking still works: the operator requeue (moveTask in-progress→todo, moves.ts ~628) and `buildManualRetryResetPatch` clear the `BLOCKED:` error, and the scheduler leaves the parked row untouched while blockedBy dependencies are unmet. */ if (live.status === "failed" && live.error?.startsWith("BLOCKED:")) { this.clearPausedAborted(task.id); this.activeWorktrees.delete(task.id); const blockedParkHonored = `Workflow graph run ended after an honest blocked park (${live.error}) — honoring park, not requeueing, retrying, or clearing state`; executorLog.log(`${task.id}: ${blockedParkHonored}`); await this.store.logEntry(task.id, blockedParkHonored, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } /* FNXC:WorkflowIrPin 2026-07-19-21:10 (KTD-3 drift park, PR #2342): A graph run that exited on the drift guard carries WORKFLOW_DRIFT_PARK_CONTEXT_KEY and visited no nodes. Before this branch existed the result fell through to the generic terminal sink as a misleading `failedNode: 'unknown'` failure — and, combined with the stale pin (now cleared by detectDrift itself), that made a permanent requeue→drift→fail loop. Park it here with an accurate drift reason instead: preserve worktree/branch/step progress untouched, do NOT re-emit the `task:reconcile-workflow-drift` audit (detectDrift already emitted the ids-only event once), and leave the row recoverable by ordinary requeue — which now succeeds because the cleared pin lets the next run re-resolve the CURRENT IR and adopt the changed workflow. */ if (result.context?.[WORKFLOW_DRIFT_PARK_CONTEXT_KEY] === true) { const driftMessage = "Workflow drift park: the workflow definition changed under this run (pinned node/column no longer in the current IR). Stale IR pin cleared — requeue the task to re-resolve the current workflow and continue."; executorLog.warn(`${task.id}: ${driftMessage}`); await this.store.logEntry(task.id, driftMessage, undefined, this.getRunContextFor(task.id)); if (live.status == null && live.error == null) { await this.store.updateTask(task.id, { error: driftMessage, status: "failed" }, this.getRunContextFor(task.id)); } await this.persistTokenUsage(task.id); return; } /* FNXC:SessionContention 2026-07-25-21:30: Classified BEFORE every other graph-failure router. A node that could not start because another task holds its session path or sub-repo lease is not a provider outage, not a plan defect, and not a terminal failure — it is a wait. Route it to the self-recovering backoff hold, which never parks the task and never consumes the provider/artifact retry budgets. */ if (this.isSessionContentionGraphFailure(result)) { await this.holdForSessionContention(task, live, result); await this.persistTokenUsage(task.id); return; } /* FNXC:MissingWorktreeRecovery 2026-07-16-18:25: An unusable-worktree session-start refusal inside a graph node must route to the bounded worktree-session recovery BEFORE any other classifier: FN-7977's provider-failure hold would otherwise retry the same stale worktree in place, and the terminal sink would park the task failed with the signature erased (FN-7996 looped dispatch→park all day). */ if (await this.routeUnusableWorktreeGraphFailureToRecovery(task, live, result, resumeLanesMemo)) { await this.persistTokenUsage(task.id); return; } if (isRequiredArtifactReadFailedValue(this.graphFailureValue(result))) { /* FNXC:WorkflowArtifacts 2026-07-21-17:00: A TaskStore read outage is not proof that an artifact is absent. Keep the task in place and use the bounded graph-resume budget instead of replanning or terminalizing a possibly healthy workflow contract. */ const priorRetries = live.graphResumeRetryCount ?? 0; if (priorRetries < MAX_TRANSIENT_GRAPH_RESUME_RETRIES) { const nextRetries = priorRetries + 1; const message = `Required workflow artifact could not be read — retrying in place (${nextRetries}/${MAX_TRANSIENT_GRAPH_RESUME_RETRIES})`; await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { graphResumeRetryCount: nextRetries }, this.getRunContextFor(task.id)); const scheduleRetry = () => { void (async () => { try { const resumeTask = await this.store.getTask(task.id); if (await this.isRequiredArtifactRecoveryProtected(resumeTask) || resumeTask.status === "failed") return; await this.execute(resumeTask); } catch (err) { executorLog.error(`Failed required-artifact read retry for ${task.id}:`, err); } })(); }; const handle = setTimeout(scheduleRetry, TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS); handle.unref?.(); } else { await this.store.logEntry( task.id, "Required workflow artifact read retry budget exhausted — task remains held in its current state", undefined, this.getRunContextFor(task.id), ); } await this.persistTokenUsage(task.id); return; } if (this.graphFailureValue(result) === PLAN_REVIEW_PROVIDER_FAILURE_HOLD_VALUE) { /* * FNXC:PlanReviewReplan 2026-07-15-16:35: * FN-7977: graph-native Plan Review provider failures are a bounded * in-place retry. They must not follow the built-in failure edge into * plan-replan or overwrite a progressed card's column, worktree, or steps. */ const priorRetries = live.graphResumeRetryCount ?? 0; if (priorRetries < MAX_TRANSIENT_GRAPH_RESUME_RETRIES) { const nextRetries = priorRetries + 1; const message = `Plan Review provider failure — retrying in place (${nextRetries}/${MAX_TRANSIENT_GRAPH_RESUME_RETRIES})`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { graphResumeRetryCount: nextRetries, }, this.getRunContextFor(task.id)); const scheduleRetry = () => { this.execute(live).catch((err) => executorLog.error(`Failed Plan Review provider retry for ${task.id}:`, err), ); }; const handle = setTimeout(scheduleRetry, TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS); handle.unref?.(); } else { const message = "Plan Review provider retry budget exhausted — task remains held in its current state"; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); } await this.persistTokenUsage(task.id); return; } if (live.mergeDetails?.mergeConfirmed === true && live.column !== await resolveCompleteColumnFor(this.store, live.id)) { if (await this.finalizeMergeConfirmedWorkflowGraphTask(live.id, "graph-failure")) { await this.persistTokenUsage(task.id); return; } } // A paused/aborted implementation is not a graph failure while the task // is still in-progress — leave the pause machinery in charge instead of // parking the task in review. const pausedAborted = this.pausedAborted.has(task.id); const abortProvenance = this.pausedAbortProvenance.get(task.id); const mergeSeamAborted = abortProvenance === "merge-seam"; const completionFinalizeAborted = abortProvenance === "completion-finalize"; const persistedCompletionFinalizeLog = live.log?.some((entry) => entry.action.includes("Execution paused after completion — finalizing to in-review")) === true; const persistedCompletedProgress = live.steps.length > 0 && live.steps.every((step) => step.status === "done" || step.status === "skipped"); /* FNXC:WorkflowLifecycle 2026-06-17-23:39: A real live pause still parks even if stale provenance says completion-finalize; completed handoff rows are expected to be unpaused. FNXC:WorkflowLifecycle 2026-06-18-10:57: FN-6644: a completed/no-commit execution that already finalized to in-review must not be re-parked as an operator-action pause abort when later teardown overwrites FN-6625 `completion-finalize` provenance with `hard-cancel` (FN-6641). Only suppress the pause-abort branch for already-finalized, non-in-progress rows with no live user/global pause; active execution hard-cancel and genuine pause/global-pause still park or preserve exactly as before. FNXC:WorkflowLifecycle 2026-06-18-12:00: FN-6647 closes the remaining durability gap by deriving already-finalized completion from the persisted task row: non-in-progress column, completed steps, no live pause/status/error, and the finalize-to-review log entry. The volatile `completionFinalizedTaskIds` marker still helps within one executor lifecycle, but teardown/restart loss must not reclassify a completed in-review row as a hard-cancel pause abort. */ /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): on a renamed board a completed, already-finalized row read as still-in-wip, so FN-6644/FN-6647's suppression never fired and the row was re-parked as an operator-action pause abort — the durability gap those tickets closed. */ const alreadyFinalizedToReview = Boolean( live.column !== failureLanes.wip && persistedCompletedProgress && live.status == null && live.error == null && live.userPaused !== true // FNXC:WorkflowLifecycle 2026-06-18-16:20: // FN-6648: do NOT require `paused !== true` here. The // paused-after-completion graceful-exit path (executor ~8748/8194) // finalizes a FULLY COMPLETED task to in-review while leaving a // NON-user `paused: true` flag set — handoffToReview / // applyInReviewEnterEffects clear status/blockedBy/overlapBlockedBy // but never `paused`. Requiring `paused !== true` made this clean // completion unrecognizable, so `genuinePauseAbort` parked it failed // with the spurious "engine abort during pause/resume" error // (FN-6638 recurrence). `userPaused`/global-pause are still excluded, // and `persistedCompletedProgress` + `persistedCompletionFinalizeLog` // + status/error == null keep this scoped to genuine completions. && abortProvenance !== "global-pause" && !mergeSeamAborted && persistedCompletionFinalizeLog, ); const completionFinalized = completionFinalizeAborted || this.completionFinalizedTaskIds.has(task.id) || alreadyFinalizedToReview; const suppressFinalizedCompletionAbort = Boolean( completionFinalized && live.column !== failureLanes.wip && !live.userPaused // FN-6648: `paused !== true` intentionally dropped here too — the // suppression is already gated on `completionFinalized` (completed // steps + finalize-to-review evidence) plus userPaused/global-pause // exclusions, so a lingering non-user post-completion pause flag must // not defeat it. See alreadyFinalizedToReview note above. && abortProvenance !== "global-pause" && !mergeSeamAborted, ); const genuinePauseAbort = Boolean( live.userPaused || abortProvenance === "global-pause" // FN-6648: gate the bare `paused` clause on the completion-finalize // suppression so a completed task carrying a non-user post-completion // pause flag is not parked as an operator-action failure. || (live.paused && !mergeSeamAborted && !suppressFinalizedCompletionAbort) || (pausedAborted && !mergeSeamAborted && !completionFinalizeAborted && !suppressFinalizedCompletionAbort), ); const failedNodeForLog = result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown"; const failureValueForLog = this.graphFailureValue(result) ?? "none"; if (pausedAborted || live.paused || live.userPaused || abortProvenance) { this.safeLogEntry( task.id, `Pause abort classified: provenance=${abortProvenance ?? "unknown"}; node=${failedNodeForLog}; interrupted=${result.interruptedNodeId ?? "none"}; abortKind=${result.interruptedAbortKind ?? "none"}; column=${live.column}; status=${live.status ?? "none"}; paused=${live.paused === true}; userPaused=${live.userPaused === true}; value=${failureValueForLog}; genuine=${genuinePauseAbort}; mergeSeam=${mergeSeamAborted}; completionSuppressed=${suppressFinalizedCompletionAbort}`, ); } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (PR #2640 review, greptile P2): one lane snapshot for one recovery decision — see `resolveResumeLanes`. Eligibility and re-entry are two halves of the SAME decision and must not read different boards. FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): the surrounding branches share it now too. This method asked "still in the wip lane?" in three more places as the default lineage's id while creating this memo for the classifiers — so the classifiers read the board and the branches around them read the default names. */ if (genuinePauseAbort && await this.isReentrantPausedAbortedInFlightNode(live, result, abortProvenance, pausedAborted, this.userCanceledTaskIds.has(task.id), resumeLanesMemo)) { if (await this.reenterPausedAbortedWorkflowNode(live, result, abortProvenance, resumeLanesMemo)) { return; } } /* FNXC:WorkflowMerge 2026-07-14-18:20: FN-1165 greptile P1: system pause (`live.paused` without userPaused/global-pause) must still enter the implementation-incomplete merge classifier. Requiring `live.paused !== true` let pause-abort parking win and skipped fail-closed/resumable routing for missing implementation proof. User pause and global-pause stay excluded. */ if ( genuinePauseAbort && abortProvenance !== "global-pause" && abortProvenance !== "completion-finalize" && live.userPaused !== true && this.isMergeGraphFailure(failedNodeForLog) && failureValueForLog === "implementation-incomplete" ) { if (await this.routeImplementationIncompleteMergeGraphFailure(live, failedNodeForLog)) { return; } } if (genuinePauseAbort && await this.isRetryableBenignMergePauseAbort(live, result, abortProvenance, pausedAborted, resumeLanesMemo)) { if (await this.routeGraphMergeFailureToRetry(live, result, abortProvenance)) { return; } } if (genuinePauseAbort && await this.isBenignManualMergeHoldPauseAbort(live, result, abortProvenance, pausedAborted, resumeLanesMemo)) { /* FNXC:WorkflowLifecycle 2026-07-09-14:56: FN-7749 / Runfusion#1979: auto-merge-off manual merge hold is terminal-until-human-merged, not an executor failure. Preserve the `in-review` row for Merge & Close, do not invoke merge retry, and clear only stale pause-abort status/error so FN-5147's no-backward-move/no-reenqueue contract stays intact. */ this.clearPausedAborted(task.id); this.activeWorktrees.delete(task.id); const manualHoldBenign = "Workflow graph run ended at manual merge hold with auto-merge off — benign, in-review manual-hold state preserved for Merge & Close"; executorLog.log(`${task.id}: ${manualHoldBenign}`); await this.store.logEntry(task.id, manualHoldBenign, undefined, this.getRunContextFor(task.id)); if (live.status != null || live.error != null) { await this.store.logEntry(task.id, "Auto-recovered: cleared stale auto-merge-off manual merge hold pause-abort failure — failure notification suppressed", undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { status: null, error: null }, this.getRunContextFor(task.id)); } await this.persistTokenUsage(task.id); return; } if (genuinePauseAbort && this.isBenignInReviewPauseAbort(live, result, abortProvenance, pausedAborted, this.userCanceledTaskIds.has(task.id), failureLanes.review)) { this.clearPausedAborted(task.id); this.activeWorktrees.delete(task.id); const inReviewBenign = "Workflow graph run ended during engine pause/resume while already in-review — benign, in-review state preserved"; executorLog.log(`${task.id}: ${inReviewBenign}`); await this.store.logEntry(task.id, inReviewBenign, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } if (genuinePauseAbort && await this.handleStaleInReviewParsePauseAbortReplay(live, result, abortProvenance, pausedAborted, this.userCanceledTaskIds.has(task.id), resumeLanesMemo)) { return; } if (genuinePauseAbort && await this.handleStaleInReviewPlanPauseAbortReplay(live, result, abortProvenance, pausedAborted, this.userCanceledTaskIds.has(task.id), resumeLanesMemo)) { return; } if (genuinePauseAbort) { /* FNXC:WorkflowLifecycle 2026-06-15-01:45: FN-6478: a graph exit during an in-progress pause is recoverable by explicit unpause, but the same exit after the task has already left in-progress strands the workflow graph. Preserve userPaused and autoMerge:false review parking; surface non-in-progress paused exits as operator-actionable failures without moving the task backward or re-enqueueing execution. FNXC:WorkflowLifecycle 2026-06-17-03:48: FN-6568: merge-seam aborts are not pause provenance. A non-paused merge-node failure must bypass this operator-action pause branch so FN-6528/FN-6531/FN-6534/FN-6537-style failures route to bounded auto-merge retry instead of being parked failed with mergeRetries=NULL. FNXC:WorkflowLifecycle 2026-06-17-23:32: FN-6625: completion-finalize aborts are teardown artifacts after a completed/no-commit execution has already advanced to in-review. Without excluding that provenance, the FN-6614 execute-node tail failure was mislabeled as an operator-action pause abort and re-parked failed. */ // FNXC:WorkflowLifecycle 2026-07-12-09:05: check `live.paused` BEFORE the // bare pausedAborted marker — a task-pause park that survived teardown // (preservePause, FN-7851) is operator intent, not an engine-internal // abort, and must be labeled as such so the benign re-queue log below // does not misreport it as engine churn. const pauseProvenance = live.userPaused ? "explicit user pause" : abortProvenance === "global-pause" ? "global pause" : live.paused ? "task pause" : pausedAborted ? "engine abort during pause/resume" : "task pause"; // Typed discriminant for the engine-internal abort case (mirrors the // `pauseProvenance === "engine abort during pause/resume"` arm above): // a generic (`hard-cancel`/`engine-abort`, KB-PROV 2026-07-26) teardown that is // NOT a user pause or global pause. Used // to gate the auto-continue branch so the gate cannot silently drift if // the human-readable provenance label is ever revised. const isEngineInternalAbort = pausedAborted && !live.paused && !live.userPaused && abortProvenance !== "global-pause"; if (live.column !== failureLanes.wip) { // FN-6782: a pause/resume abort that has left the task back in `todo` // is benign — the work is simply re-queued for a fresh dispatch, not // stranded. Parking it `status: "failed"` (operator action required) // here is what caused the retry storm: the scheduler re-dispatches the // todo task, this branch re-fires on the still-set pausedAborted // marker, and it re-parks instantly with no backoff. Treat `todo` like // the in-progress benign case: clear the abort marker so the next // dispatch starts clean, log, and return WITHOUT parking failed. The // operator-action failure is preserved only for genuinely stranded // non-todo columns (e.g. in-review), per FN-6478. if (live.column === await resolveReboundColumnFor(this.store, task.id)) { this.clearPausedAborted(task.id); // FNXC:WorkflowLifecycle 2026-06-20-00:00: FN-6782 leak fix — a task // parked back to `todo` must not keep pinning its in-memory worktree // slot. The execute() finally does not delete activeWorktrees on this // early-return path, so without this release the slot leaks — a `todo` // task stays a maxWorktrees holder and concurrency-blocks the whole // queue (the FN-6756 "in todo yet still a holder, maxWorktrees=3/3" // symptom). Mirror clearPhantomExecutorBinding's release semantics. // Safe here: handleGraphFailure is terminal for this run (no seam // re-entry), and the next dispatch re-acquires a fresh worktree. this.activeWorktrees.delete(task.id); // FNXC:WorkflowLifecycle 2026-06-20-22:42: FN-6782 follow-up — an // "engine abort during pause/resume" is NOT an operator action: the // engine tore down in-flight work (hard-cancel via // abortInFlightTaskWork) while the workflow graph run was ending and // the task got re-queued to todo. Bouncing it back through todo for // a fresh scheduler dispatch is observable churn and used to fire a // spurious failure notification. Instead, continue the agent session // automatically by re-executing in place, bounded by the same // graphResumeRetryCount budget + backoff as the transient-resume // path (and reset to 0 on the next clean graph completion, executor // ~4242) so a genuinely wedged task still falls through to the benign // re-queue after MAX retries rather than looping with no backoff. // Scoped strictly to the engine-internal abort provenance: an // explicit user pause / global pause / task pause that landed in todo // must still wait for an explicit resume (the benign re-queue below). // The graphResumeRetryCount budget is deliberately SHARED with the // transient-resume-after-restart path (executor ~6850): both are // "the graph run ended transiently, re-run it" recoveries, and a // single combined cap is the belt-and-suspenders guard the // executor-retry-storm tests assert against. The count is reset to 0 // only on a clean graph completion (~4242) — NOT on the benign // fallback below, so a still-wedged task that exhausts the budget // stops auto-continuing instead of looping (resetting here would // reintroduce a slower storm). if (isEngineInternalAbort) { const priorRetries = live.graphResumeRetryCount ?? 0; if (priorRetries < MAX_TRANSIENT_GRAPH_RESUME_RETRIES) { const nextRetries = priorRetries + 1; const retryMessage = `Workflow graph run ended during ${pauseProvenance} — auto-continuing the agent session (${nextRetries}/${MAX_TRANSIENT_GRAPH_RESUME_RETRIES}) instead of re-queueing to todo`; executorLog.log(`${task.id}: ${retryMessage}`); await this.store.logEntry(task.id, retryMessage, undefined, this.getRunContextFor(task.id)); // Emit the Auto-recovered marker BEFORE clearing status so the // status-clearing updateTask's task:updated event already carries // the recovery log — NotificationService.maybeSuppressTransientFailedNotification // (recoveredStatus path) then proactively cancels any pending // failure timer rather than relying on the race-contingent // fire-time re-check. await this.store.logEntry(task.id, "Auto-recovered: engine-internal pause/resume abort — retrying agent session, failure notification suppressed", undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { graphResumeRetryCount: nextRetries, status: null, error: null }, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); const scheduleRetry = () => { // Re-fetch at fire time: the snapshot is up to // TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS stale, and the direct // execute() bypasses the scheduler's pause filter (we cleared // pausedAborted at the top of this branch). If a user paused, // moved, or deleted the task during the backoff window, abort // the auto-continue and leave it to normal scheduling so we // never resume work the user just parked. void (async () => { try { const resumeTask = await this.store.getTask(task.id); if ( resumeTask.deletedAt || resumeTask.paused || resumeTask.userPaused || resumeTask.column !== await resolveReboundColumnFor(this.store, task.id) ) { executorLog.log( `${task.id}: skipping pause-abort auto-continue — task is now ${resumeTask.deletedAt ? "deleted" : resumeTask.paused || resumeTask.userPaused ? "paused" : `in '${resumeTask.column}'`} at retry fire time`, ); return; } await this.execute(resumeTask); } catch (err) { executorLog.error(`Failed pause-abort internal retry for ${task.id}:`, err); } })(); }; if (TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS > 0) { const handle = setTimeout(scheduleRetry, TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS); handle.unref?.(); } else { setTimeout(scheduleRetry, 0).unref?.(); } return; } // Note: the count is left at MAX (not reset here) deliberately, so // this task stops auto-continuing until a clean graph completion // resets it (~4242). Because the budget is SHARED with the // transient-resume-after-restart path (~6869), a task that already // burned retries there starts here with a smaller auto-continue // budget — and vice versa. That cross-draining is intentional: a // single combined cap across both transient-recovery paths is what // bounds runaway re-runs, even if it means a repeatedly // hard-cancelled task that never completes cleanly exhausts the // shared budget and falls back to plain todo re-queueing. executorLog.warn(`${task.id}: engine abort during pause/resume exhausted ${MAX_TRANSIENT_GRAPH_RESUME_RETRIES} internal retries — falling back to benign todo re-queue`); } // FNXC:WorkflowLifecycle 2026-07-12-09:05: a row still carrying a // pause park (paused/userPaused) is NOT "cleared for normal // scheduling" — the scheduler skips it until an explicit unpause. // Say so, or the log contradicts the board (FN-7851 misdiagnosis). const todoBenign = live.paused || live.userPaused ? `Workflow graph run ended during ${pauseProvenance} with task parked in todo — benign, paused awaiting explicit unpause` : `Workflow graph run ended during ${pauseProvenance} with task re-queued to todo — benign, cleared for normal scheduling`; executorLog.log(`${task.id}: ${todoBenign}`); await this.store.logEntry(task.id, todoBenign, undefined, this.getRunContextFor(task.id)); // FNXC:WorkflowLifecycle 2026-06-20-19:58: reconcile a stale // persisted failure with the benign reclassification. A pause-abort // parked `status:"failed"` on an earlier non-todo observation stays // dispatchable (scheduler.ts filters column+paused, NOT status) and // re-enters this branch in `todo`; `recoverPausedAbortFailures` that // would clear it is suppressed during global/engine pause // (self-healing.ts). Leaving the row failed contradicts the benign // log: the board shows it failed AND the deferred failure // notification fires (notification-service fire-time check sees // status === "failed"). Clear status/error here so the row matches // the log, then emit an `Auto-recovered:`-prefixed entry so // NotificationService.maybeSuppressTransientFailedNotification // PROACTIVELY cancels the pending failure timer on the task:updated // event (recoveredStatus path) — rather than relying only on the // fire-time re-check, which is race-contingent when // failureNotificationDelayMs is near 0. The prefix is the documented // contract for self-healing recovery logs (see self-healing.ts / // project-engine.ts). Scoped to the actual-clear path so the common // no-failure benign re-queue is not mislabeled as a recovery. if (live.status != null || live.error != null) { await this.store.updateTask(task.id, { status: null, error: null }, this.getRunContextFor(task.id)); await this.store.logEntry(task.id, "Auto-recovered: cleared stale pause-abort failure on todo re-queue — failure notification suppressed", undefined, this.getRunContextFor(task.id)); } await this.persistTokenUsage(task.id); return; } /* FNXC:WorkflowLifecycle 2026-07-12: A pause-abort whose task already reached a terminal SUCCESS column is benign teardown, not an operator problem. The live-acceptance repro: the workflow merge boundary hard-cancels the in-flight executor session when it moves the task in-progress → in-review (abort-in-flight provenance=engine-abort, formerly hard-cancel — KB-PROV 2026-07-26), the AI merge then lands and the task advances to done — and only afterwards does the aborted graph run reach this sink, where it logged "Workflow graph failure surfaced ... operator action required; retry or explicitly unpause/resume" on a task that finished perfectly. The `status: "failed"` write below was already guarded for done/archived, but the alarming operator-action log entry (and its warn) still fired on every auto-merged task. Treat done/archived like the todo benign case: clear the abort marker, release the worktree slot, log a benign completion note, and never emit the PAUSE_ABORT_PARK markers (so self-healing's recoverPausedAbortFailures has nothing to chase). */ if ((await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) { this.clearPausedAborted(task.id); this.activeWorktrees.delete(task.id); const doneBenign = `Workflow graph run ended during ${pauseProvenance} after the task already completed ('${live.column}') — benign, no action needed`; executorLog.log(`${task.id}: ${doneBenign}`); await this.store.logEntry(task.id, doneBenign, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown"; // FNXC:WorkflowLifecycle 2026-06-20-00:00: build the parked-failure // message from the shared markers so self-healing's recoverPausedAbortFailures // predicate cannot drift out of sync with this text (PR #1687 review). const message = `${PAUSE_ABORT_PARK_ERROR_MARKER} ${pauseProvenance} in '${live.column}' at node '${failedNode}' — ${PAUSE_ABORT_PARK_OPERATOR_MARKER}; retry or explicitly unpause/resume after inspecting the task`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); if (live.status == null && live.error == null) { await this.store.updateTask(task.id, { error: message, status: "failed" }, this.getRunContextFor(task.id)); } await this.persistTokenUsage(task.id); return; } const benignMessage = "Workflow graph run ended while task is paused — pause state preserved"; executorLog.log(`${task.id}: ${benignMessage} (${pauseProvenance})`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); return; } const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; const mergeGraphFailure = this.isMergeGraphFailure(failedNode); const failureValue = this.graphFailureValue(result); /* FNXC:WorkflowExecutionOwnership 2026-07-28-09:40 (U8 / R3): The execution-policy ladder below — the FN-7863/FN-7926 dispatch-loop gate, the FN-7996 tool-failure retry, and the FN-7998 escalation — decided the task's own lifecycle by naming `"todo"` and `"in-progress"` literally. Under any workflow that renames those columns the whole ladder was unreachable and its failure was SILENT in the worst direction: the `live.column !== wip` guard below classified a card sitting in its own implementation column as "already advanced — no further action needed", so the graph failure was swallowed, no status was written, and the scheduler re-dispatched the same doomed run. Nothing failed; the retry budgets, the escalation, and the bounded terminalization simply never ran. Resolve ONCE per failure and thread the pair through the ladder. One IR read per graph failure: this is a terminal recovery path, not an enumeration loop. FNXC:WorkflowExecutionOwnership 2026-07-28-14:05 (U8 / R3, PR #2497 review — greptile P1): THE FALLBACK IS PER-WORKFLOW, NEVER PER-ROLE. The first cut wrote `columns?.hold ?? "todo"`, which conflates two different situations: "no workflow could be resolved" and "this workflow resolved fine and simply declares no hold column". Only the first justifies the legacy literal. For the second, substituting `todo` invents a column the workflow does not declare — and node-target escalation then PERSISTS it, stranding the card somewhere the board cannot route and defeating the scheduler node re-resolution the escalation exists for. U1 returns `undefined` per missing role precisely so a caller cannot borrow an unrelated column; `?? "todo"` threw that guarantee away one line after asking for it. So: - IR unresolvable -> the legacy literals, i.e. exactly pre-conversion behavior. - IR resolved -> `resolveReboundTarget` (KTD-10: hold -> intake -> first column), which can only ever name a DECLARED column, and `undefined` for wip when the workflow declares none. A `wipColumn` of `undefined` is not a wildcard — every gate below treats "I cannot prove where the wip column is" as "do not take the shortcut", so an unprovable card terminalizes VISIBLY rather than being swallowed by the already-advanced branch. Fail closed toward the operator seeing the failure. The two literals that remain are ONLY the unresolvable-workflow fallback, and they are the same pre-conversion values `resolveReboundColumnFor` already falls back to at its ~16 executor call sites — this adds no new rule and no new reachable-by-a-valid-workflow literal. They are legacy-compat for a task whose workflow cannot be read at all, and they belong to the same sweep that retires `resolveReboundColumnFor`'s own `?? "todo"` when U11 removes the column; they are deliberately NOT a per-role default, which is what made the first cut wrong. */ let lifecycleIr: WorkflowIr | undefined; try { lifecycleIr = await resolveWorkflowIrForTask(this.store, task.id); } catch { lifecycleIr = undefined; } const wipColumn = lifecycleIr ? resolveLifecycleColumns(lifecycleIr)?.wip : "in-progress"; const holdColumn = lifecycleIr ? resolveReboundTarget(lifecycleIr) : "todo"; /* FNXC:WorkflowExecutionOwnership 2026-07-29-18:55 (U8 / R4): COMPAT PATH for user-authored graphs, deliberately named. Every BUILT-IN shape declares the `outcome:review-pending` edge, so a built-in run never reaches here — it routed to its park node and ended. A custom workflow without the edge falls through to its generic `failure` edge and lands here, where the handoff the implementation phase used to perform inline happens instead. For those graphs this is a relocation, not an elimination: the transition is still executor-performed. What changes is that it is one named classifier in the failure ladder rather than a call buried two thousand lines into a session loop. */ if (this.graphRunReportedPendingReview(result, failureValue)) { const compatMessage = "Implementation stopped on a pending review — parking in review (this workflow does not route the review-pending outcome)"; executorLog.log(`${task.id}: ${compatMessage}`); await this.store.logEntry(task.id, compatMessage, undefined, this.getRunContextFor(task.id)); await this.handoffTaskToReview(live, "executor-exit-while-review-pending"); await this.persistTokenUsage(task.id); return; } const executeNodeSelfRequeued = failedNode === "execute" && this.graphExecuteSelfRequeued.has(task.id); if (failedNode === "execute" && ((holdColumn !== undefined && live.column === holdColumn) || executeNodeSelfRequeued)) { /* FNXC:WorkflowLifecycle 2026-06-23-12:03: The graph execute node delegates to the authoritative executor. If that inner executor requeues the task to todo for self-heal/retry, the outer graph failure must not override it by parking the task in review. FNXC:WorkflowLifecycle 2026-06-23-21:19: Also honor the in-process self-requeue marker. Upgrade/restart races and minimal stores can return a stale `in-progress` live row even after the inner executor already moved the task to `todo`; stale reads must not strand progressing tasks in review. FNXC:WorkflowLifecycle 2026-07-12-00:00: FN-7863: the scheduler's wall-clock dispatchStormCount guard only increments when re-dispatches happen inside its short window; slow execute→pause-abort→todo loops reset that counter every cycle. Count this funnel by execution-progress signature instead, warn early for board-visible monitoring, and terminalize only non-paused live tasks after the bounded no-progress cap while preserving worktree/branch/step progress. FNXC:WorkflowLifecycle 2026-07-12-23:14: FN-7926 diverts completed-but-blocked rows before the FN-7863 counter increments. A stable all-done step signature plus unresolved dependency/blockedBy is a waiting state, not an implementation no-progress loop; park it with the specific blocker and let self-healing advance it when `getTaskCompletionBlocker` clears. */ const completionBlocker = await this.getTaskCompletionBlocker(live); if (completionBlocker && await this.parkCompletedBlockedTask(live, completionBlocker, "execute-requeue")) { await this.persistTokenUsage(task.id); return; } const { signature, madeForwardProgress } = buildExecuteRequeueLoopHighWaterSignature(live, live.executeRequeueLoopSignature); const nextCount = madeForwardProgress || live.executeRequeueLoopSignature == null ? 1 : (live.executeRequeueLoopCount ?? 0) + 1; if (live.executeRequeueLoopCount !== nextCount || live.executeRequeueLoopSignature !== signature) { await this.store.updateTask(task.id, { executeRequeueLoopCount: nextCount, executeRequeueLoopSignature: signature, }, this.getRunContextFor(task.id)); } if (nextCount === EXECUTE_REQUEUE_LOOP_VISIBLE_THRESHOLD) { const warningMessage = `Execution dispatch loop building: ${nextCount}/${MAX_EXECUTE_REQUEUE_LOOP_CYCLES} no-progress execute re-queues`; executorLog.warn(`${task.id}: ${warningMessage}`); await this.store.logEntry(task.id, warningMessage, undefined, this.getRunContextFor(task.id)); } const canTerminalizeExecuteLoop = live.userPaused !== true && live.paused !== true && !(await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column); if (nextCount >= MAX_EXECUTE_REQUEUE_LOOP_CYCLES && canTerminalizeExecuteLoop) { const terminalError = `EXECUTION_DISPATCH_LOOP_EXHAUSTED: execute node re-queued task to todo ${nextCount} times with no forward progress (last value=${failureValue ?? "no-value"}). No further automatic retries will run. Manually retry, decompose, or rescope the task.`; await this.store.updateTask(task.id, { status: "failed", error: terminalError, executeRequeueLoopCount: nextCount, executeRequeueLoopSignature: signature, }, this.getRunContextFor(task.id)); await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("execution-dispatch-loop", task.id), domain: "database", mutationType: "task:execution-dispatch-loop-terminalized", target: task.id, metadata: { taskId: task.id, cycleCount: nextCount, maxCycles: MAX_EXECUTE_REQUEUE_LOOP_CYCLES, progressSignature: signature, failureValue: failureValue ?? null, }, }); executorLog.warn(`${task.id}: ${terminalError}`); await this.store.logEntry(task.id, terminalError, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } const benignMessage = `Workflow graph execute node ended after executor re-queued task to todo (${failureValue ?? "no-value"}) — executor recovery preserved`; executorLog.log(`${task.id}: ${benignMessage}`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } if (mergeGraphFailure && failureValue === "implementation-incomplete") { if (await this.routeImplementationIncompleteMergeGraphFailure(live, failedNode ?? "unknown")) { return; } } if (mergeGraphFailure && !this.isTerminalMergeGraphFailureValue(failureValue) && await this.routeGraphMergeFailureToRetry(live, result, abortProvenance)) { return; } if (mergeGraphFailure && this.isTerminalMergeGraphFailureValue(failureValue) && !(await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) { const message = `Workflow graph terminal merge failure at node '${failedNode ?? "unknown"}' (${failureValue}) — operator action required`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); if (live.status == null && live.error == null) { await this.store.updateTask(task.id, { error: message, status: "failed" }, this.getRunContextFor(task.id)); } await this.persistTokenUsage(task.id); return; } if (failedNode === "parse" && failureValue === "pin-mismatch" && await this.routeResetParsePinMismatchToRetry(live)) { return; } if (await this.routeRetryableRemediationGraphFailureToPreMergeFix(live, failedNode, failureValue)) { return; } if (await this.routeGraphFailureToExecutionResume(live, failedNode ?? "unknown", failureValue, resumeLanesMemo)) { return; } /* FNXC:WorkflowExecutionOwnership 2026-07-28-14:10 (U8 / R3, PR #2497 review): `wipColumn === undefined` means the workflow declares no implementation column, so there is no evidence the card "already advanced" past one. Swallowing the failure on a guess is the exact silent-loss this conversion exists to remove — require a KNOWN wip column before taking the benign shortcut. */ if (wipColumn !== undefined && live.column !== wipColumn) { const benignMessage = `Workflow graph run ended after task already advanced to '${live.column}' — no further action needed`; executorLog.log(`${task.id}: ${benignMessage}`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); return; } if (this.isAwaitingGraphFailureValue(failureValue)) { /* FNXC:WorkflowLifecycle 2026-06-15-12:00: Awaiting-input and awaiting-CLI-approval workflow node values are resumable operator waits, not terminal execute failures. Classify the node value before the generic graph-failure sink so a stale or partially reloaded pause flag cannot park a legitimately runnable task in review with the execute-node symptom. */ const benignMessage = `Workflow graph run ended awaiting ${failureValue === "awaiting-cli-approval" ? "CLI approval" : "user input"} at node '${failedNode ?? "unknown"}' — awaiting state preserved`; executorLog.log(`${task.id}: ${benignMessage}`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); if (live.status !== failureValue || !live.paused) { await this.store.updateTask(task.id, { status: failureValue, paused: true }, this.getRunContextFor(task.id)); } return; } if (this.isTransientResumeAfterRestartGraphFailure(live, result)) { const priorRetries = live.graphResumeRetryCount ?? 0; if (priorRetries < MAX_TRANSIENT_GRAPH_RESUME_RETRIES) { const nextRetries = priorRetries + 1; const benignMessage = `Transient resume-after-restart graph failure — auto-retrying (${nextRetries}/${MAX_TRANSIENT_GRAPH_RESUME_RETRIES}) instead of parking`; executorLog.warn(`${task.id}: ${benignMessage}`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { graphResumeRetryCount: nextRetries, status: null, error: null, }, this.getRunContextFor(task.id)); const scheduleRetry = () => { this.execute(live).catch((err) => executorLog.error(`Failed transient graph resume retry for ${task.id}:`, err), ); }; if (TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS > 0) { const handle = setTimeout(scheduleRetry, TRANSIENT_GRAPH_RESUME_RETRY_BACKOFF_MS); handle.unref?.(); } else { setTimeout(scheduleRetry, 0).unref?.(); } return; } } /* FNXC:WorkflowRemediation 2026-07-01-23:40: Do NOT flag a still-executing task as failed. A `pre-merge-remediation` / `plan-replan` node (e.g. `code-review-remediation`) is a fire-and-forget async scheduler with no `failure` out-edge, so a failed re-arm (missing rehydrated failureContext after restart, remediation-not-scheduled, or an exhausted rework budget) bubbles out as the terminal graph outcome here. When a SEPARATE live agent session surface is still registered for this task, the previously-scheduled fix/reviewer is genuinely mid-flight — parking `status:"failed"` would surface a spurious "Task Failed" over live work. Preserve the row and let the live session drive its own terminal handoff instead. Scoped strictly to remediation nodes + a live session surface so genuine execute/merge terminal failures (and remediation failures with NO live session, e.g. a truly exhausted budget) still park exactly as before. FNXC:WorkflowRemediation 2026-07-21-22:56: Extend the same preserve rule to execute-family nodes when a SEPARATE live session surface exists. A losing raced graph (duplicate resume after plan-review) can terminate at steps#N:step-execute while a peer session still owns coding work; stamping status=failed arms overseer retry_step hard-cancels (FN-8471). Merge-region failures still park — they are not execute-family. */ const isExecuteFamilyNode = failedNode === "execute" || failedNode === "step-execute" || failedNode?.endsWith(":step-execute") === true; if (this.hasLiveTaskSessionSurface(task.id)) { const isRemediation = await this.isRemediationGraphNode(task.id, failedNode); if (isRemediation || isExecuteFamilyNode) { const kind = isRemediation ? "remediation" : "execute"; const benignMessage = `Workflow graph ended at ${kind} node '${failedNode ?? "unknown"}' while a live agent session is still executing — not flagging as failed; live session preserved`; executorLog.warn(`${task.id}: ${benignMessage}`); await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } } const message = `Workflow graph terminated with failure at node '${failedNode ?? "unknown"}'`; const settings = await this.store.getSettings(); const maxToolFailureRetries = resolveMaxConsecutiveToolFailureRetries(settings); if (maxToolFailureRetries > 0 && isExecuteFamilyNode && !live.paused && !live.userPaused && !live.deletedAt && live.column === wipColumn) { // Prefer the execution-local boundary; recovery paths refetch durable state rather than use the stale failure snapshot. const cursor = this.graphToolFailureRunCursors.get(task.id) ?? (await this.store.getTask(task.id))?.toolFailureDetectorLogCursor; const threshold = resolveConsecutiveToolFailureThreshold(settings); if (await this.hasTrailingConsecutiveToolFailures(task.id, cursor, threshold)) { const claim = await this.store.claimNextToolFailureRetry(task.id, cursor!, maxToolFailureRetries); if (claim.outcome === "claimed") { await this.store.updateTask(task.id, { status: null, error: null }, this.getRunContextFor(task.id)); await this.store.logEntry(task.id, `Consecutive tool-call failures — auto-retrying same model (${claim.attempt}/${maxToolFailureRetries}) instead of parking`, undefined, this.getRunContextFor(task.id)); await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("tool-failure-retry", task.id), domain: "database", mutationType: "task:execution-tool-failure-retry", target: task.id, metadata: { taskId: task.id, nodeId: failedNode ?? "unknown", attempt: claim.attempt, maxAttempts: maxToolFailureRetries, consecutiveToolFailures: threshold, mode: "same-model" } }); const schedule = () => { void (async () => { const resume = await this.store.getTask(task.id); if (resume && !resume.deletedAt && !resume.paused && !resume.userPaused && resume.column === wipColumn) await this.execute(resume); })().catch((error) => executorLog.error(`${task.id}: tool-failure retry failed`, error)); }; const delay = resolveConsecutiveToolFailureRetryBackoffMs(settings); setTimeout(schedule, delay).unref?.(); return; } if (claim.outcome === "already-claimed-for-run") { await this.store.getTask(task.id); return; } /* FNXC:ExecutorEscalation 2026-07-16-21:00: FN-7998 inserts exactly one opt-in recovery between FN-7996 exhaustion and the unchanged terminal park. Refetch before writing so a pause, deletion, or later run cannot inherit a costly model/node override from this stale graph result. */ const escalationTarget = resolveExecutorEscalationTarget(settings); const hasModelTarget = escalationTarget.provider !== undefined && escalationTarget.modelId !== undefined; /* FNXC:WorkflowExecutionOwnership 2026-07-28-14:15 (U8 / R3, PR #2497 review — greptile P1): A node escalation is a REQUEUE: it parks the card back in the hold lane so the scheduler re-resolves the effective node. Without a declared requeue target there is nowhere legal to put it, and persisting an invented column is worse than not escalating — the card lands where the board cannot route it and the node is never dispatched. Degrade to the no-node-target shape (in-place retry, which is already how an enabled escalation with no usable target behaves) rather than writing an undeclared column. */ const nodeTargetRequeueColumn = escalationTarget.nodeId !== undefined ? holdColumn : undefined; const hasNodeTarget = escalationTarget.nodeId !== undefined && nodeTargetRequeueColumn !== undefined; if (escalationTarget.nodeId !== undefined && nodeTargetRequeueColumn === undefined) { await this.store.logEntry(task.id, "Node escalation downgraded to an in-place retry — this task's workflow declares no column to requeue into", undefined, this.getRunContextFor(task.id)); } let claimedEscalation = false; let priorEscalationRetryCount = 0; /* FNXC:ExecutorEscalation 2026-07-16-22:30: The one-shot latch is claimed under the TaskStore lock. Concurrent exhausted graph handlers for the same detector cursor must not both schedule an alternate run; a loser leaves the winner's in-progress row untouched. */ await this.store.updateTaskAtomic(task.id, (current) => { const ownsFailureRun = current.toolFailureDetectorLogCursor === cursor && current.column === wipColumn && !current.paused && !current.userPaused && !current.deletedAt; if (!ownsFailureRun || current.executorEscalationAttempted === true || !escalationTarget.enabled) return null; claimedEscalation = true; priorEscalationRetryCount = current.consecutiveToolFailureRetryCount ?? 0; return { ...(hasModelTarget ? { modelProvider: escalationTarget.provider, modelId: escalationTarget.modelId } : {}), ...(hasNodeTarget ? { nodeId: escalationTarget.nodeId, column: nodeTargetRequeueColumn } : {}), executorEscalationAttempted: true, /* FNXC:ExecutorEscalation 2026-07-16-22:40: Invalidate the exhausted run cursor before releasing the claim so concurrent stale handlers cannot park or audit the alternate execution; the alternate captures its own cursor at startup. */ toolFailureDetectorLogCursor: null, status: null, error: null, }; }, this.getRunContextFor(task.id)); if (claimedEscalation) { await this.store.logEntry(task.id, "Same-model retries exhausted — escalating to alternate model/node (one attempt) instead of parking", undefined, this.getRunContextFor(task.id)); await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("escalation-retry", task.id), domain: "database", mutationType: "task:execution-escalation-retry", target: task.id, metadata: { taskId: task.id, nodeId: failedNode ?? "unknown", hasModelTarget, hasNodeTarget, priorConsecutiveToolFailureRetryCount: priorEscalationRetryCount } }); if (!hasNodeTarget) { const scheduleEscalation = () => { void (async () => { const resumeTask = await this.store.getTask(task.id); if (resumeTask && !resumeTask.deletedAt && !resumeTask.paused && !resumeTask.userPaused && resumeTask.column === wipColumn) await this.execute(resumeTask); })().catch((error) => executorLog.error(`${task.id}: escalation retry failed`, error)); }; const handle = setTimeout(scheduleEscalation, resolveConsecutiveToolFailureRetryBackoffMs(settings)); handle.unref?.(); } return; } /* FNXC:ExecutorToolFailureRetry 2026-07-16-20:45: Exhaustion belongs to the graph run that supplied `cursor`, not a later run that may have begun while this handler awaited its durable claim. Revalidate the cursor under TaskStore's per-task atomic lock while applying the terminal state; only that successful CAS may emit the exhaustion audit. This keeps an old terminal handler from parking a newer in-progress executor run. */ let cursorOwnedTerminalPark = false; let escalationAttemptFailed = false; let escalationHadModelTarget = false; let escalationHadNodeTarget = false; await this.store.updateTaskAtomic(task.id, (current) => { if ( current.toolFailureDetectorLogCursor !== cursor || current.column !== wipColumn || current.paused || current.userPaused || current.deletedAt || current.status !== null ) { return null; } cursorOwnedTerminalPark = true; escalationAttemptFailed = current.executorEscalationAttempted === true; escalationHadModelTarget = current.modelProvider != null && current.modelId != null; escalationHadNodeTarget = current.nodeId != null; return { error: message, status: "failed" }; }, this.getRunContextFor(task.id)); if (!cursorOwnedTerminalPark) return; if (await this.store.markToolFailureRetryExhaustedAudit(task.id)) { await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("tool-failure-retry-exhausted", task.id), domain: "database", mutationType: "task:execution-tool-failure-retry-exhausted", target: task.id, metadata: { taskId: task.id, nodeId: failedNode ?? "unknown", attempts: maxToolFailureRetries, limit: maxToolFailureRetries, outcome: "terminal-park" } }); } if (escalationAttemptFailed) { await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("escalation-exhausted", task.id), domain: "database", mutationType: "task:execution-escalation-exhausted", target: task.id, metadata: { taskId: task.id, nodeId: failedNode ?? "unknown", hadModelTarget: escalationHadModelTarget, hadNodeTarget: escalationHadNodeTarget } }); } executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); return; } } if (live.executorEscalationAttempted === true) { const failureCursor = task.toolFailureDetectorLogCursor; let escalationTerminalParked = false; let escalationHadModelTarget = false; let escalationHadNodeTarget = false; /* FNXC:ExecutorEscalation 2026-07-16-22:35: Once the durable escalation latch is set, every terminal failure of that alternate run emits the exhaustion audit even if an operator disables the setting mid-run. Cursor ownership prevents an old concurrent handler from parking the newly scheduled alternate execution. */ await this.store.updateTaskAtomic(task.id, (current) => { if ( current.toolFailureDetectorLogCursor !== failureCursor || current.column !== wipColumn || current.paused || current.userPaused || current.deletedAt || current.status !== null ) return null; escalationTerminalParked = true; escalationHadModelTarget = current.modelProvider != null && current.modelId != null; escalationHadNodeTarget = current.nodeId != null; return { error: message, status: "failed" }; }, this.getRunContextFor(task.id)); if (!escalationTerminalParked) return; await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "executor", runId: generateSyntheticRunId("escalation-exhausted", task.id), domain: "database", mutationType: "task:execution-escalation-exhausted", target: task.id, metadata: { taskId: task.id, nodeId: failedNode ?? "unknown", hadModelTarget: escalationHadModelTarget, hadNodeTarget: escalationHadNodeTarget } }); } else { // status "failed" doubles as the self-healing exemption: review-task // revival sweeps skip tasks carrying a non-null status, preventing the // FN-5704-style loop of re-running the graph from scratch. await this.store.updateTask(task.id, { error: message, status: "failed" }, this.getRunContextFor(task.id)); } executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); } catch (err) { executorLog.error( `${task.id}: failed to park graph-failed task: ${err instanceof Error ? err.message : String(err)}`, ); } } private async routeGraphFailureToExecutionResume( live: TaskDetail, failedNode: string, failureValue: string | undefined, /** Shared per-recovery lane snapshot — see `resolveResumeLanes`. */ resumeLanesMemo?: { lanes?: { hold: string; wip: string; review: string; wipDeclared: boolean } }, ): Promise { /* * FNXC:WorkflowLifecycle 2026-06-29-11:08: * A workflow graph failure is not a completion handoff. FN-7228/FN-7229 showed * restart-time parse failures and incomplete steps being parked in `in-review` * with errors, which blocks the engine from resuming the correct unfinished * step. Keep executable work in the executable queue: clear graph failure * markers and move review-column rows with unfinished work back to `todo` * preserving step progress. Generic graph failures that remain in-progress * are left failed in-place by the caller; they must never be handed to review. */ if (live.deletedAt) return false; if (live.paused || live.userPaused === true) return false; if ((await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) return false; /* * FNXC:WorkflowCompletion 2026-07-01-16:26: * Backstop for issue #1863. The advisory completion-summary node must never * drive the in-review→todo resume loop: it has no failure edge, so a failure * here would bounce the task back to execution every run and never stick. * The graph executor now degrades summary-node failures to success, so this * should be unreachable — but if a summary failure ever reaches this router, * let the caller park the task `failed` (a visible terminal state) instead of * looping it forever. */ if (failedNode === COMPLETION_SUMMARY_NODE_ID) return false; const incompleteSteps = hasNonTerminalWorkflowSteps(live); const implementationIncompleteMergeFailure = this.isMergeGraphFailure(failedNode) && failureValue === "implementation-incomplete"; if (implementationIncompleteMergeFailure && !incompleteSteps) return false; const prematureMergeWithIncompleteSteps = implementationIncompleteMergeFailure && incompleteSteps; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: executor.ts — the REVERSE half-conversion): THE DESTINATION WAS ALREADY RESOLVED HERE AND THE GATE WAS NOT. `resolveReboundColumnFor` below picks the board's rebound column (U7), but this gate compared against three default-lineage literals — so on a renamed board the router refused before ever reaching the resolved move. That is the mirror image of the dangerous half-conversion: instead of admitting a card and sending it nowhere, it refuses a card whose recovery was fully implemented, and nothing is logged as wrong. Same one-decision-two-boards defect, opposite direction, and the silent one. */ const resumeRouterLanes = await this.resolveResumeLanes(live.id, resumeLanesMemo); /* FNXC:WorkflowLifecycleColumns 2026-07-30-14:20: A workflow that declares NO implementation lane has nowhere to resume TO, so this router must not claim the card — the graph failure has to reach the terminalize branch and be visible. Without this, a card resting in such a workflow's HOLD lane with incomplete steps matched the second arm above (`incompleteSteps && live.column === lanes.hold`), the router rehomed it and returned true, and the failure was swallowed: `status` and `error` both stayed null. The operator saw a card that had silently stopped. That is the exact shape the sibling branch below already guards with `wipColumn !== undefined` before claiming a card "already advanced"; this is the same fail-closed rule on the opposite path, which was failing OPEN. */ if (!resumeRouterLanes.wipDeclared) return false; if (live.column !== resumeRouterLanes.review && !(incompleteSteps && live.column === resumeRouterLanes.hold) && !(prematureMergeWithIncompleteSteps && live.column === resumeRouterLanes.wip)) return false; const message = incompleteSteps ? `Workflow graph failed at node '${failedNode}'${failureValue ? ` (${failureValue})` : ""} with incomplete steps — moved back to todo for execution resume` : `Workflow graph failed at node '${failedNode}'${failureValue ? ` (${failureValue})` : ""} before a clean review handoff — moved back to todo for workflow retry`; executorLog.warn(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); await this.store.updateTask(live.id, { status: null, error: null, }, this.getRunContextFor(live.id)); const reboundColumn = await resolveReboundColumnFor(this.store, live.id); if (live.column !== reboundColumn) { await this.store.moveTask(live.id, reboundColumn, { preserveProgress: true, moveSource: "engine", recoveryRehome: true, }); } // FNXC:ReviewLeniency 2026-07-02-02:10: clear prior terminal failure results // (incl. optional gate nodes like code-review) AFTER the task is in `todo` // (non-mergeable) so the resumed run re-evaluates gates from a clean slate // without dropping the in-review merge blocker mid-flight. (in-review→todo // moveTask already clears all results; this covers the already-`todo` path.) await this.clearTerminalStepFailuresForRetry(live.id); await this.persistTokenUsage(live.id); return true; } private async routeResetParsePinMismatchToRetry(live: TaskDetail): Promise { /* FNXC:WorkflowReset 2026-06-29-10:04: A user reset/retry can race an aborting graph-owned foreach instance that persists after the route cleared pins. If the next run reaches parse and sees only stale foreach pins while the task has no implementation progress, recover by deleting all graph instance rows and requeueing to todo. Do not hand the task to in-review, because parse has not executed work or produced mergeable output. */ if (live.deletedAt) return false; if (live.paused || live.userPaused === true) return false; if ((await resolveTerminalColumnsFor(this.store, live.id)).includes(live.column)) return false; const hasImplementationProgress = (live.currentStep ?? 0) > 0 || (live.steps ?? []).some((step) => step.status === "done" || step.status === "in-progress" || step.status === "skipped"); if (hasImplementationProgress) return false; const maybeStore = this.store as unknown as { clearWorkflowRunStepInstancesAsync?: (taskId: string) => Promise; clearWorkflowRunStepInstances?: (taskId: string) => void; clearWorkflowRunBranches?: (taskId: string, keepRunId: string) => void; }; try { await (maybeStore.clearWorkflowRunStepInstancesAsync?.(live.id) ?? maybeStore.clearWorkflowRunStepInstances?.(live.id)); } catch { // Legacy stores may not persist graph step instances. } this.clearPausedAborted(live.id); this.activeWorktrees.delete(live.id); await this.store.updateTask(live.id, { status: null, error: null, graphResumeRetryCount: 0, }, this.getRunContextFor(live.id)); const reboundColumn = await resolveReboundColumnFor(this.store, live.id); if (live.column !== reboundColumn) { await this.store.moveTask(live.id, reboundColumn, { preserveProgress: false }); } const message = "Auto-recovered: cleared stale workflow parse pins after reset/retry — task requeued before execution"; executorLog.warn(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); await this.persistTokenUsage(live.id); return true; } private async maybeDispatchWorkflowWorkEngine(task: Task): Promise { let detail: TaskDetail; let workflow: WorkflowIr; try { detail = await this.store.getTask(task.id); workflow = await resolveWorkflowIrForTask(this.store, task.id); } catch (error) { executorLog.warn(`${task.id}: failed to resolve workflow work-engine bindings: ${error instanceof Error ? error.message : String(error)}`); return false; } if (workflow.version !== "v2") return false; const column = workflow.columns.find((candidate) => candidate.id === detail.column); const extensionEntries = Object.entries(column?.extensions ?? {}); if (extensionEntries.length === 0) return false; const registry = getWorkflowExtensionRegistry(); for (const [extensionId, metadata] of extensionEntries) { const definition = registry.get(extensionId); const extension = definition?.extension; if (!definition || definition.degraded || extension?.kind !== "work-engine" || !extension.dispatch) continue; let result: WorkflowWorkEngineDispatchResult; try { result = await extension.dispatch({ task: detail, workflow, columnId: detail.column, metadata, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); executorLog.warn(`${task.id}: workflow work-engine ${extensionId} failed: ${message}`); if (extension.fallback === "degradeToDefault") continue; await this.store.logEntry(task.id, `Workflow work engine ${extensionId} failed`, message); await this.store.updateTask(task.id, { status: extension.fallback === "parkNeedsAttention" ? "queued" : "failed", error: message, }); return true; } if (result.kind === "not-claimed") continue; if (result.kind === "degraded-to-default") { executorLog.warn(`${task.id}: workflow work-engine ${extensionId} degraded to default: ${result.reason}`); await this.store.logEntry(task.id, `Workflow work engine ${extensionId} degraded to default`, result.reason); continue; } if (result.kind === "parked") { await this.store.logEntry(task.id, result.message, result.reason); await this.store.updateTask(task.id, { status: "queued", error: result.reason }); return true; } await this.store.logEntry( task.id, result.message ?? `Workflow work engine ${extensionId} claimed execution`, ); try { await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: "workflow-work-engine", runId: result.runId ?? generateSyntheticRunId("workflow-work-engine", task.id), domain: "database", mutationType: "workflow:work-engine:claimed", target: task.id, metadata: { extensionId, columnId: detail.column, pluginId: definition.pluginId, }, }); } catch (error) { executorLog.warn(`${task.id}: failed to record workflow work-engine claim audit: ${error instanceof Error ? error.message : String(error)}`); } return true; } return false; } private async evaluateTaskVerdictProviders( task: TaskDetail, context: Record = {}, ): Promise<{ ok: true } | { ok: false; message: string }> { let workflow: WorkflowIr; try { workflow = await resolveWorkflowIrForTask(this.store, task.id); } catch (error) { executorLog.warn(`${task.id}: failed to resolve workflow for verdict providers: ${error instanceof Error ? error.message : String(error)}`); return { ok: true }; } const providers = getWorkflowExtensionRegistry().list("verdict-provider"); for (const definition of providers) { const extension = definition.extension; if (definition.degraded || extension.kind !== "verdict-provider" || !extension.evaluate) continue; try { const verdict = await extension.evaluate({ task, workflow, reworkRound: 0, metadata: context, }); if (verdict.status === "pass") continue; const reasons = verdict.failureReasons?.map((reason) => reason.message).filter(Boolean).join("; "); return { ok: false, message: `fn_task_done refused (verdict-provider): ${verdict.summary}${reasons ? ` — ${reasons}` : ""}`, }; } catch (error) { if (extension.fallback === "degradeToDefault") continue; const message = error instanceof Error ? error.message : String(error); return { ok: false, message: `fn_task_done refused (verdict-provider): provider '${definition.id}' failed — ${message}`, }; } } return { ok: true }; } private async blockOuterDispatchWhenDependenciesUnmet(task: Task): Promise { if (!task.dependencies || task.dependencies.length === 0) return false; const settings = await this.store.getSettings(); const tasks = await this.store.listTasks({ includeArchived: false, slim: true }); const liveTask = tasks.find((candidate) => candidate.id === task.id) ?? task; const markerAcceptedByTaskId = new Map(); if (settings.mergeRequestContractShadowEnabled === true) { for (const depId of liveTask.dependencies) { markerAcceptedByTaskId.set(depId, (await this.store.getCompletionHandoffAcceptedMarker(depId)) !== null); } } const unmetDeps = getUnmetSchedulingDependencies( liveTask, tasks, settings.mergeRequestContractShadowEnabled === true ? { markerAcceptedByTaskId } : undefined, ); if (unmetDeps.length === 0) return false; /* FNXC:DependencyGating 2026-06-20-07:30: Workflow-graph and workflow-authoritative executor dispatches can be invoked outside the classic scheduler loop, so they must re-apply the shared scheduling dependency gate before graph routing, column-agent seams, or review handoff can run. Requeue with blockedBy instead of executing so missing or soft-deleted dependency residue keeps the scheduler helper's non-blocking semantics while live todo/queued/in-progress/triage dependencies block every dispatch surface. */ const reboundColumn = await resolveReboundColumnFor(this.store, liveTask.id); if (liveTask.column !== reboundColumn) { await this.store.moveTask(liveTask.id, reboundColumn, { preserveProgress: true, preserveWorktree: true, preserveResumeState: true, moveSource: "engine", recoveryRehome: true, }); } await this.store.updateTask(liveTask.id, { status: "queued", blockedBy: unmetDeps[0] }, this.getRunContextFor(liveTask.id)); await this.store.logEntry( liveTask.id, `queued — unmet dependencies: ${unmetDeps.join(", ")}`, "Executor pre-dispatch dependency gate blocked workflow/authoritative execution.", this.getRunContextFor(liveTask.id), ); executorLog.log(`${liveTask.id}: executor dispatch blocked by unmet dependencies: ${unmetDeps.join(", ")}`); return true; } /* FNXC:EphemeralAgents 2026-07-01-00:00: `ephemeralAgentsEnabled: false` means "never spawn short-lived executor-FN-XXXX workers; only permanent agents run work" (see types.ts ephemeralAgentsEnabled). The legacy spawn refusal lives in EphemeralWorkerManager.onTaskStart (ephemeral-worker-manager.ts), but that runs as a fire-and-forget bookkeeping callback AFTER execution has already begun, so it cannot stop a run. The workflow-engine dispatch paths (executeWorkflowGraph, maybeDispatchWorkflowWorkEngine) execute tasks in-process without ever consulting the toggle. Any task that reaches execute() without a permanent assignment via a non-scheduler path (resume-after-restart, heartbeat re-entry, mission/autopilot, work-engine claim) therefore ran despite the operator disabling ephemeral agents. This guard is the executor's last line of defense, mirroring the scheduler cutover gate (scheduler.ts:2464) and the spawn refusal (ephemeral-worker-manager.ts:132). It runs once at the top of the outer dispatch — before all three workflow paths — so a single check covers every workflow dispatch entry point. A task explicitly assigned to a permanent (non-ephemeral) agent is exactly how ephemeral-off mode is meant to run, so those are allowed through; everything else is re-queued for the scheduler to auto-assign a permanent agent or hold. */ private async blockOuterDispatchWhenEphemeralDisabled(task: Task): Promise { const settings = await this.store.getSettings(); if (settings.ephemeralAgentsEnabled !== false) return false; // A permanent (non-ephemeral) assignment is the sanctioned executor when // ephemeral workers are off. `assignedAgentId` is only ever set by permanent // assignment — default ephemeral mode never sets it — so when we cannot // resolve the agent (no agentStore) we trust the presence of the id and allow // the run rather than starving a legitimately-assigned task. const assignedId = task.assignedAgentId?.trim(); if (assignedId) { if (!this.options.agentStore) return false; const agent = await this.options.agentStore.getAgent(assignedId).catch(() => null); if (agent && !isEphemeralAgent(agent)) return false; } const liveTask = (await this.store.getTask(task.id).catch(() => null)) ?? task; const reboundColumn = await resolveReboundColumnFor(this.store, liveTask.id); if (liveTask.column !== reboundColumn) { await this.store.moveTask(liveTask.id, reboundColumn, { preserveProgress: true, preserveWorktree: true, preserveResumeState: true, moveSource: "engine", recoveryRehome: true, }); } await this.store.updateTask(liveTask.id, { status: "queued" }, this.getRunContextFor(liveTask.id)); await this.store.logEntry( liveTask.id, "queued — ephemeral agents disabled; no permanent executor assigned", "Executor pre-dispatch ephemeral gate blocked workflow/authoritative execution.", this.getRunContextFor(liveTask.id), ); executorLog.log(`${liveTask.id}: executor dispatch blocked — ephemeralAgentsEnabled=false and no permanent agent assigned`); return true; } /* FNXC:GlobalConcurrencyControls 2026-07-15-03:50: Structural cleanup for scheduler pre-held global slots: every execute() exit path (early return, throw, graph-owned, legacy handoff) must leave no unclaimed registration. take() removes the registration so a successful claim+release is a no-op here; early returns that never take() release the underlying semaphore. New early-return paths cannot reintroduce permanent capacity leaks without bypassing this wrapper. */ async execute(task: Task): Promise { try { await this.executeCore(task); } finally { if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); } } /* FNXC:WorkflowExecution 2026-07-19-02:10: U5e (R9) — `executeCore` is ROUTING ONLY. It decides who owns the task (duplicate-dispatch drop, dependency/ephemeral gates, the workflow graph, authoritative dispatch) and, when no one else claims it, drives the implementation phase itself. The routing block used to be wrapped in `if (!graphCompletion)` because the graph re-ENTERED `execute()` to run the implementation phase, and that inner call had to skip routing or it would recurse. The graph now calls `runImplementation()` directly, so there is no inner invocation to exclude and the gates are unconditional. */ private async executeCore(task: Task): Promise { this.completionFinalizedTaskIds.delete(task.id); /* FNXC:ExecutorSoftDelete 2026-07-20-23:30: Soft-delete refuse belongs in routing, not only inside runImplementation. After U10b the graph owns every execute() call, so a deletedAt check that lives only under the implementation seam never fires for graph entry (cursor capture / selection / fail-closed parks run first). Refuse here before graph ownership so soft-deleted cards never start a workflow run; runImplementation keeps the same check as defense-in-depth for graph-owned re-entry that already holds the process lock. */ if (task.deletedAt) { executorLog.warn(`${task.id}: refusing execute — task is soft-deleted`); if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } /* FNXC:WorkflowExecution 2026-07-21-22:56: Claim graphRouting BEFORE any await. The previous check-then-await-then-claim window let concurrent execute() calls (task:moved + unpause resume after plan-review) both pass the graphRouting.has gate, both enter executeWorkflowGraph, and one park status=failed while the other still owned work (FN-8471 overseer thrash). */ if (this.graphRouting.has(task.id)) { // Duplicate dispatch while the graph runner owns this task — drop it, // mirroring the executingTaskLock duplicate-invocation behavior. executorLog.debug(`execute() called for ${task.id} while graph routing is active — skipping duplicate`); return; } this.graphRouting.add(task.id); let graphRunnerOwnsClaim = false; try { await this.clearStalePauseAbortBeforeDispatch(task); if (await this.blockOuterDispatchWhenDependenciesUnmet(task)) { // FNXC:GlobalConcurrencyControls 2026-07-14-18:30: release any scheduler pre-held slot when outer dispatch aborts before agent work starts. if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } // FNXC:EphemeralAgents 2026-07-01-00:00: gate ALL workflow dispatch paths // (graph/authoritative/work-engine) on ephemeralAgentsEnabled before any of // them can claim the task, so the single check covers all three entry points. if (await this.blockOuterDispatchWhenEphemeralDisabled(task)) { if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } /* FNXC:WorkflowExecution 2026-07-19-10:40: U10 (R9) — the `workflowAuthoritativeDispatch` branch is DELETED along with WorkflowAuthoritativeDriver. It was the pre-graph "authoritative" runtime: a second in-process execution path that could claim a task between the graph and the legacy implementation. The graph is now the sole orchestrator, so a second claimant is not a fallback, it is a race. FNXC:WorkflowExecution 2026-07-19-17:45 (U10b / R9): The trailing `await this.runImplementation(task)` is DELETED too, and `maybeExecuteWorkflowGraph` is now `executeWorkflowGraph` returning void. The old boolean meant "did the graph claim this task"; with the legacy fallback gone the answer is always yes, so a bare `runImplementation` call with NO `graphCompletion` — an implementation pass that nothing owns the completion of — is unreachable by construction rather than by convention. That is what makes `graphCompletion` a required parameter below. */ graphRunnerOwnsClaim = true; await this.executeWorkflowGraph(task, { alreadyClaimed: true }); } finally { // executeWorkflowGraph's finally releases the claim when it owns the run. if (!graphRunnerOwnsClaim) { this.graphRouting.delete(task.id); } } } /* FNXC:WorkflowExecution 2026-07-19-02:10: U5e (R9) — the implementation phase, lifted out of the dual-purpose `executeCore` into a standalone runner the workflow graph calls DIRECTLY. Before the lift the graph re-entered `execute()` under a completion signal, because worktree / taskEnv / agent / semaphore state is assembled here and was not available standalone at `createGraphSeams` time. Lifting the body moves that assembly behind an ordinary method call, so the graph gets the state it needs without a second trip through routing. Owns: the process-wide task lock, soft-delete refusal, work-engine dispatch, heartbeat deferral, settings merge, worktree acquisition, the agent session, and everything up to the implementation-complete boundary. It does NOT own workflow gates, review handoff, or merge — those are the graph's. */ private async runImplementation( task: Task, /* FNXC:WorkflowExecution 2026-07-19-17:50 (U10b / R9): REQUIRED, and an explicit parameter rather than an options bag. It was optional only to describe "a run the graph does not own" — the legacy fallback. That fallback is deleted, so every implementation pass is graph-owned and every completion boundary below is an unconditional handoff. Making it required is the type-level statement of that invariant: an implementation pass whose completion nothing owns can no longer be constructed. */ graphCompletion: GraphCompletionCallback, /* FNXC:WorkflowExecutionOwnership 2026-07-28-20:15 (U8 / R4, R5): Optional exit reporter. `graphCompletion` can only say "done"; the endings it cannot express are the ones the executor transitions itself (see `executor/implementation-exit.ts`). This names them so they are OBSERVABLE before they are moved — it changes no routing and nothing branches on it, by R5: an exit id is a reaction, and a dropped reaction must never cost a state change. Optional so the ~22 uninstrumented dispositions stay silent rather than forcing a 3k-line diff; the ownership ledger is the record of that gap, not this callback. */ reportImplementationExit?: ImplementationExitReporter, ): Promise { // FN-4811 follow-up (FN-4814/FN-4809/FN-4811 production failure): claim a // PROCESS-WIDE lock synchronously before any other work. Per-instance // `this.executing` was insufficient in production because two execute() // invocations for the same task ID still both reached "Executor detected // stale merge state" (executor.ts:2661) and both generated runIds — producing // duplicate "Worktree created at /..." log entries within the same second. // The only fully-reliable guard is a singleton lock shared across all // TaskExecutor instances in the same process (e.g., engine restart race, // multi-project hybrid runtime, etc.). This is `executingTaskLock` in // active-session-registry.ts, a module-level Set. const claimed = executingTaskLock.tryClaim(task.id); executorLog.debug(`execute() called for ${task.id} (claimed=${claimed}, perInstanceExecuting=${this.executing.has(task.id)})`); if (!claimed) { // FNXC:GlobalConcurrencyControls 2026-07-15-02:55: graph fallback may have re-registered a pre-held slot; drop it when this process cannot claim the executor lock. if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } // Maintain the per-instance Set too, for back-compat with all the existing // `this.executing.has()` checks throughout the file (handler gates, // stuck-detector, resumeTaskForAgent, etc.). Per-instance state stays // consistent with the process-wide lock. this.executing.add(task.id); if (task.deletedAt) { executorLog.warn(`${task.id}: refusing execute — task is soft-deleted`); this.executing.delete(task.id); executingTaskLock.release(task.id); if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } if (await this.maybeDispatchWorkflowWorkEngine(task)) { executorLog.log(`${task.id}: workflow work engine claimed execution`); this.executing.delete(task.id); executingTaskLock.release(task.id); // FNXC:GlobalConcurrencyControls 2026-07-15-02:55: work-engine ownership never take()s the legacy handoff registration — release the reserved global slot. if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } // Column-agent principal alignment (plan U5, R6): the heartbeat-deferral gate // must consult the EFFECTIVE principal, not blindly `assignedAgentId`. For a // graph-routed seam the binding context (governing node id + per-run resolver) // is already set by the time the seam re-enters execute() — so the effective // column agent (when an override/defer binding governs) is the principal whose // `allowParallelExecution=false` must serialize. For the legacy/no-binding path // `resolveEffectivePrincipalId` returns `assignedAgentId`, so the gate is // byte-identical to before. const deferralPrincipalId = this.resolveEffectivePrincipalId(task, task); if (deferralPrincipalId && await this.shouldDeferForHeartbeat(deferralPrincipalId)) { executorLog.debug(`${task.id}: skipping execute — agent ${deferralPrincipalId} has active heartbeat run (allowParallelExecution=false)`); // Release the slot we just claimed — we never actually ran. this.executing.delete(task.id); executingTaskLock.release(task.id); // FNXC:GlobalConcurrencyControls 2026-07-15-02:55: heartbeat defer must free any re-registered pre-held global slot so capacity is not stranded until the next dispatch. if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } executorLog.log(`Starting ${task.id}: ${task.title || task.description.slice(0, 60)}`); // Fetch settings early — needed for worktree naming and later configuration. // Merge per-task effective workflow settings (U3, KTD-3) OVER the project/global // base so the ~20 flat `settings.` read sites threaded from here (workflow // step timeout, scope enforcement, runStepsInNewSessions, model lanes, // reviewHandoffPolicy, …) pick up workflow values with zero read-site changes. // Behavior-inert when nothing is customized (declaration defaults === legacy // defaults; absent-default lanes never override). const settings = await mergeEffectiveSettings(this.store, task, await this.store.getSettings()); // Keep runtime plugin workflow step templates synchronized into TaskStore. // TaskStore resolves plugin-prefixed workflow IDs from this injected cache // to avoid a PluginLoader↔TaskStore circular dependency. const pluginWorkflowStepTemplates = this.options.pluginRunner?.getPluginWorkflowStepTemplates() ?? []; this.store.setPluginWorkflowStepTemplates(pluginWorkflowStepTemplates); // Read execution mode to determine whether to skip review and workflow steps const executionMode = task.executionMode ?? "standard"; // Construct run context for mutation correlation // Use a synthetic correlation ID: task ID + timestamp + random suffix const syntheticRunId = generateSyntheticRunId("exec", task.id); this.currentRunContexts.set(task.id, { runId: syntheticRunId, agentId: task.assignedAgentId ?? "executor", }); // Build engine run context for audit instrumentation (FN-1404) const engineRunContext: EngineRunContext = { runId: syntheticRunId, agentId: task.assignedAgentId ?? "executor", taskId: task.id, phase: "execute", }; // Create run auditor for TaskStore-backed audit emission (no-ops if store doesn't support it) const audit = createRunAuditor(this.store, engineRunContext); // Stale spec enforcement: check if PROMPT.md has aged beyond the configured threshold. // When enabled, stale tasks are moved back to triage with status "needs-replan" // so they receive fresh specification before execution. This guard runs early in // execute() to prevent stale tasks from entering worktree creation or agent sessions. // If timestamp evaluation is skipped (missing/unreadable file), continue with execution // so existing filesystem validation paths remain authoritative. // Skip for tasks that are already in-progress, in-review, merging, or done — // these should not be interrupted and sent back to triage for re-planning. /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40: THIS GUARD DID THE EXACT THING ITS OWN COMMENT SAYS IT MUST NOT. The comment directly above is explicit: skip for tasks already in-progress, in-review, merging or done, because "these should not be interrupted and sent back to triage for re-planning". Keyed on a hard-coded `Set`, a renamed board matched NOTHING, so `isActiveTask` was false for a card in a renamed wip/review/complete lane — the stale-spec guard then ran on a LIVE task and `moveTaskToReplanColumn` + `status: "needs-replan"` yanked it out of execution mid-flight. `activeMergeStatuses` still covers the merging states, so a merging card was protected by accident; a plain in-progress card was not. CENSUS-INVISIBLE: a `Set` literal is a definition, not a comparison, so nothing in the lifecycle backlog pointed here. Found by grepping for lane-shaped list literals. Resolved from the task's OWN workflow, unioned with the legacy trio for the reason documented on `resolveTerminalColumnsFor`: `resolveWorkflowIrForTask` returns the BUILT-IN IR rather than throwing when a definition is missing or corrupt, so a degraded resolution must not NARROW this set — narrowing it re-opens the interruption this fixes. */ /* FNXC:WorkflowResolvedColumns 2026-07-30-16:10 (the arity trap, seventh site): MEMBERSHIP, not first-per-role. `activeColumns` is a `.has()` test, but was filled from `resolveLifecycleColumns`, which returns the FIRST column carrying each trait — so a workflow with two wip lanes, or a review lane plus a second merge-blocking one, had only one of each recognised as active. A card in the second read as INACTIVE and its prompt file was treated as reclaimable. The IR is already in hand one line up; `columnsWithFlag` returns every column carrying the trait. The legacy trio stays unioned in — this predicate is about liveness, and under-reporting active is the destructive direction. */ const activeIr = await resolveWorkflowIrForTask(this.store, task.id); const activeColumns = new Set(["in-progress", "in-review", "done"]); if (activeIr) { for (const flag of ["countsTowardWip", "mergeOrchestration", "mergeBlocker", "humanReview", "complete"] as const) { for (const lane of columnsWithFlag(activeIr, flag)) activeColumns.add(lane); } } const activeMergeStatuses = new Set(["merging", "merging-pr", "merging-fix"]); const isActiveTask = activeColumns.has(task.column) || activeMergeStatuses.has(task.status ?? ""); if (!isActiveTask) { const tasksDir = join(this.store.getFusionDir(), "tasks"); const promptPath = getPromptPath(tasksDir, task.id); const staleness = await evaluateSpecStaleness({ settings, promptPath, task, /* FNXC:WorkflowLifecycleColumns 2026-07-30-12:40 (U11): one-line pass-through so the guard is driven rather than defaulted. Touches no executor logic. */ plannerColumns: await resolveDedicatedPlannerColumnsForTask(this.store, task.id), }); if (staleness.isStale) { executorLog.warn(`Task ${task.id} specification is stale — ${staleness.reason}`); // Move to the workflow-aware replan column first, then set status so the task // enters it with needs-replan (workflows without "triage" replan in place in todo). await moveTaskToReplanColumn(this.store, task); await this.store.updateTask(task.id, { status: "needs-replan" }); await this.store.logEntry(task.id, staleness.reason, undefined, this.getRunContextFor(task.id)); // FNXC:GlobalConcurrencyControls 2026-07-15-02:55: replan handoff never starts agent work — free any re-registered pre-held slot before leaving execute(). if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } } // Drift detection: a task that is already in-progress (i.e. we're not // dispatching it fresh from todo) should always carry a `worktree`. If it // doesn't, some prior update — most likely a partial pause/abort sequence // where updateTask({ worktree: null }) succeeded but the subsequent // moveTask()/status write failed — left the row in a half-state. The // executor can still recover by falling through to the fresh-worktree // path below, but we emit a loud audit record so these states stop being // silent. /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: execute() preflight): THREE DRIFT CHECKS, ONE SNAPSHOT — merge-confirmed while still executing, stale mergeDetails, and in-wip with no worktree. None fired on a renamed board, so every recovery they perform silently stopped happening. The third one's own message says it "usually indicates a partial updateTask/moveTask sequence failed" — a diagnostic that could never print on a renamed board. */ const preflightWipLane = (await this.resolveResumeLanes(task.id)).wip; if (task.column === preflightWipLane && task.mergeDetails?.mergeConfirmed === true) { if (await this.finalizeMergeConfirmedWorkflowGraphTask(task.id, "execute-preflight")) { this.executing.delete(task.id); executingTaskLock.release(task.id); if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); return; } } if (task.column === preflightWipLane && task.mergeDetails) { executorLog.warn(`${task.id}: stale mergeDetails found while executing in-progress task — resetting merge state before continuing`); task = await this.cleanupMergeStateForReverification( task, "Executor detected stale merge state while task was in-progress — reset verification steps and merge metadata before resuming", ); } if (task.column === preflightWipLane && !task.worktree) { executorLog.error( `${task.id}: drift detected — task is in-progress with no worktree. ` + `Recovering by creating a fresh worktree. This usually indicates a partial ` + `updateTask/moveTask sequence failed somewhere upstream.`, ); await this.store.logEntry( task.id, "Drift detected: in-progress with no worktree — creating fresh worktree to recover", undefined, this.getRunContextFor(task.id), ); } // Hoist worktreePath so it's accessible in the catch block for dep-abort cleanup let worktreePath = task.worktree ?? ""; // Set by stuck-abort handlers; the actual moveTask("todo") is deferred to // the finally block so this.executing is cleared first (prevents re-dispatch race). // true = requeue to todo, false = budget exhausted (already marked failed). let stuckRequeue: boolean | null = null; let staleAssistantContinuationRequeue = false; let taskDone = false; let reviewAddressingActivated = false; let taskEnv: NodeJS.ProcessEnv | undefined; try { await this.transitionReviewAddressing(task.id, ["queued"], "in-progress"); reviewAddressingActivated = true; // Check dependencies const allTasks = await this.store.listTasks({ slim: true, includeArchived: false }); /* FNXC:WorkflowResolvedColumns 2026-07-30-21:40 (batch-engine — dependency satisfaction, per DEPENDENCY): Resolved from each DEPENDENCY's own workflow, not this task's: dependencies routinely span workflows, so asking "is my blocker finished?" against the blocked task's vocabulary is the wrong question. That is the answer main settled on in `branch-group-ops.ts` (#2720) and it is reused here rather than re-derived. MEMBERSHIP and unioned with the legacy trio, because a workflow may declare more than one complete or review lane and `resolveWorkflowIrForTask` yields the BUILT-IN IR for a missing workflow rather than throwing — without the union a degraded renamed board treats a finished blocker as unmet and the dependent never runs. NOTE the set is wider than the terminal pair: this guard has always counted `in-review` as satisfying a dependency, so the review role is included. Narrowing it to terminal-only would be a behaviour change, not a conversion. */ const depIrCache = new Map>>(); const satisfiedByDep = new Map>(); for (const depId of task.dependencies) { if (satisfiedByDep.has(depId)) continue; const satisfied = new Set(["done", "in-review", "archived"]); try { const depIr = await resolveWorkflowIrForTask(this.store, depId, depIrCache); if (depIr) { for (const flag of ["complete", "archived", "mergeOrchestration", "mergeBlocker", "humanReview"] as const) { for (const id of columnsWithFlag(depIr, flag)) satisfied.add(id); } } } catch { /* degraded: the legacy trio */ } satisfiedByDep.set(depId, satisfied); } const unmetDeps = task.dependencies.filter((depId) => { const dep = allTasks.find((t) => t.id === depId); return dep !== undefined && !satisfiedByDep.get(depId)!.has(dep.column); }); if (unmetDeps.length > 0) { executorLog.log(`${task.id} blocked by: ${unmetDeps.join(", ")} — deferring`); return; } if (this.workspaceConfig === undefined) { this.workspaceConfig = await loadWorkspaceConfig(this.rootDir); } /* FNXC:Workspace 2026-06-22-00:00: Workspace mode is only meaningful with at least one usable sub-repo. An empty `{ repos: [] }` must NOT bypass the git-repository guard, inject workspace instructions, or expose the workspace tool — otherwise a non-git directory with an empty config would skip validation and enable a workspace with nothing to work on. Gate every workspace check on repos.length > 0. */ const hasWorkspaceRepos = (this.workspaceConfig?.repos.length ?? 0) > 0; if (!hasWorkspaceRepos) { const gitDetection = await detectGitRepository(this.rootDir); if (gitDetection.status === "not-repo") { await this.store.logEntry( task.id, "Cannot execute task: project directory is not a Git repository. Fusion requires a Git repository for worktree-based task execution.", ); throw new Error( "Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.", ); } if (gitDetection.status === "error") { /* FNXC:Worktree 2026-07-10-00:00: FN-7799 requires environmental Git probe failures in valid repos to surface the real cause instead of telling operators to run `git init`. Dubious ownership and similar persistent failures otherwise block every task across restarts with a false non-repo diagnosis. */ const message = formatGitRepositoryDetectionError(this.rootDir, gitDetection); await this.store.logEntry(task.id, message); throw new Error(message); } } const hadAssignedWorktree = Boolean(task.worktree); const taskCommandAbortController = new AbortController(); this.registerConfiguredCommandController(task.id, taskCommandAbortController); /* FNXC:Workspace 2026-06-21-12:00: KTD1 — in workspace mode `this.rootDir` is a NON-git parent. Acquiring a root worktree there fails. Skip root acquisition entirely and run the agent session rooted at the browse-only workspace root; the agent acquires per-sub-repo worktrees on demand via fn_acquire_repo_worktree. `task.worktree` stays unset. We synthesize a non-fresh, non-resume acquisition with an empty branch so the downstream env-injection/onStart bookkeeping runs unchanged while every rootDir git preflight (base capture, contamination, liveness) is gated off below. The non-workspace branch is byte-for-byte the original acquisition path. */ const acquisition: AcquireTaskWorktreeResult = this.workspaceConfig ? { worktreePath: this.rootDir, branch: "", source: "existing", hydrated: true, isResume: Boolean(task.sessionFile), } : await (async () => { try { return await acquireTaskWorktree({ task, rootDir: this.rootDir, store: this.store, settings, pool: this.options.pool, logger: executorLog, audit, runContext: this.getRunContextFor(task.id), runInitCommand: true, createWorktree: this.createWorktree.bind(this), runConfiguredCommand: (command, cwd, timeoutMs, env) => runConfiguredCommand( command, cwd, timeoutMs, env, audit, taskCommandAbortController.signal, ).then((result) => { if (taskCommandAbortController.signal.aborted) { throw this.createConfiguredCommandAbortError(task.id, command); } return result; }), taskEnv, secretsStore: this.options.secretsStore, }); } finally { this.unregisterConfiguredCommandController(task.id, taskCommandAbortController); } })(); worktreePath = acquisition.worktreePath; if (acquisition.reclaimed) { await audit.git({ type: "branch:auto-reclaim", target: acquisition.branch, metadata: { taskId: task.id, branch: acquisition.branch, worktreePath: acquisition.worktreePath, existingTipSha: acquisition.reclaimed.existingTipSha, strandedCommitCount: acquisition.reclaimed.strandedCommitCount ?? 0, trigger: "dispatch-preflight", }, }); } if (!acquisition.isResume && acquisition.source === "fresh" && settings.setupScript) { const scriptCommand = settings.scripts?.[settings.setupScript]; if (scriptCommand) { const setupStartedAt = Date.now(); const setupAbortController = new AbortController(); this.registerConfiguredCommandController(task.id, setupAbortController); try { const setupResult = await runConfiguredCommand( scriptCommand, worktreePath, 120_000, taskEnv, audit, setupAbortController.signal, ); if (setupAbortController.signal.aborted) { throw this.createConfiguredCommandAbortError(task.id, scriptCommand); } if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) { throw new Error(configuredCommandErrorMessage(setupResult)); } await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.getRunContextFor(task.id)); } catch (err: unknown) { if (err instanceof Error && err.name === "AbortError") { throw err; } const execError = err instanceof Error ? err : new Error(String(err)); const message = "stderr" in execError && typeof (execError as Record).stderr === "string" ? String((execError as Record).stderr) : execError.message; await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.getRunContextFor(task.id)); } finally { this.unregisterConfiguredCommandController(task.id, setupAbortController); } } else { await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.getRunContextFor(task.id)); } } /* FNXC:Workspace 2026-06-21-12:00: KTD1 — every preflight below (base-commit capture, contamination check, worktree-liveness gate) runs git against `worktreePath`, which equals the non-git workspace root in workspace mode. They would all fail. Gate the whole block off in workspace mode; the per-repo equivalents return in Phase B (master U3) against each acquired sub-repo worktree. The non-workspace branch is unchanged. */ if (!this.workspaceConfig) { // Capture the base commit SHA for diff computation whenever a task // starts with a newly assigned worktree. if (!acquisition.isResume) { await this.captureBaseCommitSha(task, worktreePath, audit, { isResume: false }); } // Contamination check must use a FRESH merge-base with the integration // branch — NOT task.baseCommitSha. baseCommitSha is intentionally // preserved across sessions for stable diff math, which makes it // potentially stale relative to main. Using it here would falsely flag // every legitimately-merged commit on main since that stale SHA as // "foreign contamination" (see FN-4417). The real signal we want is: // does the branch contain commits past its current merge-base with main // that are attributed to OTHER tasks? Compute the merge-base fresh. const contaminationBaseRef = await this.resolveContaminationBaseRef(worktreePath); if (contaminationBaseRef) { try { await assertCleanBranchAtBase(this.rootDir, acquisition.branch, contaminationBaseRef, task.id); } catch (contaminationError: unknown) { if (!(contaminationError instanceof BranchCrossContaminationError)) { throw contaminationError; } const recovered = await this.tryBootstrapMisbindingRecovery(task, contaminationError, audit); if (recovered) { return; } throw contaminationError; } } const expectedRoot = canonicalizePath(this.rootDir); let observedWorktreeRealpath: string; let livenessFailure: string | null = null; try { observedWorktreeRealpath = canonicalizePath(worktreePath); if (observedWorktreeRealpath === expectedRoot) { livenessFailure = "realpath_matches_repo_root"; } } catch (error) { observedWorktreeRealpath = `unresolvable:${worktreePath}`; livenessFailure = `unresolvable_worktree:${error instanceof Error ? error.message : String(error)}`; } if (!livenessFailure && !isInsideWorktreesDir(this.rootDir, worktreePath, settings)) { livenessFailure = "outside_worktrees_dir"; } let livenessFailureReason: string | null = null; let livenessClassification: string | null = null; const shouldGate = acquisition.isResume || (hadAssignedWorktree && !task.sessionFile && acquisition.source !== "fresh"); if (!livenessFailure && shouldGate) { const classification = await classifyTaskWorktree(this.rootDir, worktreePath); if (!classification.ok) { const reanchor = await detectNestedWorktreeRoot(this.rootDir, worktreePath, settings); if (reanchor.reanchored) { await this.store.updateTask(task.id, { worktree: reanchor.root }); await this.store.logEntry(task.id, `Re-anchored nested task.worktree from ${worktreePath} to ${reanchor.root}`, undefined, this.getRunContextFor(task.id)); await this.emitWorktreeReanchoredAudit(task.id, worktreePath, reanchor.root, "executor-liveness-gate"); worktreePath = reanchor.root; observedWorktreeRealpath = canonicalizePath(reanchor.root); } else { livenessClassification = classification.classification; livenessFailureReason = classification.reason; livenessFailure = `not_usable_task_worktree:${classification.classification}`; } } } if (livenessFailure) { const expected = `${resolveWorktreesDir(this.rootDir, settings)}/* (usable, registered)`; const observed = `${worktreePath} (${observedWorktreeRealpath})`; let registeredPaths: string[] = []; try { const registeredSnapshot = await describeRegisteredWorktrees(this.rootDir); registeredPaths = registeredSnapshot.canonicalized; } catch { registeredPaths = []; } const visibleRegistered = registeredPaths.slice(0, 10); const registeredSuffix = registeredPaths.length > 10 ? `, … +${registeredPaths.length - 10} more` : ""; const registeredSection = ` — registered=[${visibleRegistered.join(", ")}${registeredSuffix}]`; const reasonSection = livenessFailureReason ? ` (${livenessFailureReason})` : ""; const failureMessage = `worktree liveness assertion failed: ${livenessFailure}${reasonSection} — observed=${observed}, expected=${expected}${registeredSection}`; executorLog.error(`${task.id}: ${failureMessage}`); await this.store.logEntry(task.id, failureMessage, undefined, this.getRunContextFor(task.id)); const priorRequeues = task.taskDoneRetryCount ?? 0; const nextRequeueCount = priorRequeues + 1; const terminalAction = priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES ? "requeue-todo" : "park-in-review"; const isRepoRootCollision = livenessFailure === "realpath_matches_repo_root"; const auditClassification = livenessClassification ?? (isRepoRootCollision ? "repo-root" : null); const auditReason = livenessFailureReason ?? (isRepoRootCollision ? "worktree path realpath matches the project root, not a task worktree" : null); /* * FNXC:WorktreeLiveness 2026-06-21-11:10: * The executor still keeps the repo-root realpath check as defense in depth. If acquisition ever hands the root to this gate, emit structured evidence that separates the invalid checkout path from the normal git registered-worktree snapshot and the configured task-worktree pattern. */ if (auditClassification) { const registeredContainsObserved = registeredPaths.includes(observedWorktreeRealpath); await audit.git({ type: "worktree:incomplete-detected", target: worktreePath, metadata: { classification: auditClassification, reason: auditReason ?? undefined, source: "executor-liveness-gate", taskId: task.id, retryCount: nextRequeueCount, maxRetries: MAX_TASK_DONE_REQUEUE_RETRIES, terminalAction, observed: worktreePath, observedRealpath: observedWorktreeRealpath, expected, registered: visibleRegistered, registeredTotal: registeredPaths.length, registeredContainsObserved, invalidCheckoutPath: isRepoRootCollision ? "repo-root" : undefined, expectedPatternExcludesRepoRoot: isRepoRootCollision, }, }); } if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) { await this.store.updateTask(task.id, { status: "queued", error: null, worktree: null, branch: null, sessionFile: null, taskDoneRetryCount: nextRequeueCount, paused: false, pausedByAgentId: null, }); await this.store.logEntry( task.id, `${failureMessage} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`, undefined, this.getRunContextFor(task.id), ); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true }); executorLog.log(`✗ ${task.id} worktree liveness failed — requeued to todo (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`); } else { await this.store.updateTask(task.id, { status: "failed", error: failureMessage, worktree: null, branch: null, sessionFile: null, paused: false, pausedByAgentId: null, }); await this.store.logEntry(task.id, `${failureMessage} — execution failed after worktree liveness retry budget was exhausted`, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); executorLog.log(`✗ ${task.id} worktree liveness failed`); } this.options.onError?.(task, new Error(failureMessage)); return; } } // end !this.workspaceConfig preflight gate (FNXC:Workspace KTD1) // FNXC:Workspace 2026-06-21-12:00: KTD2 — register the worktree path under the task's Set. In workspace mode `worktreePath` is the browse-only root; per-repo sub-repo worktree paths ARE now added to the same Set as the agent acquires them (F2: fn_acquire_repo_worktree's onAcquired callback → addActiveWorktree), so the Set holds root + N sub-repo paths, not just the root. Non-workspace tasks add exactly one path → a one-element set (unchanged liveness/owner semantics). this.addActiveWorktree(task.id, worktreePath); executorLog.debug(`${task.id}: worktree ready at ${worktreePath}`); const injected = await this.buildInjectedRuntimeEnv(task.id, worktreePath, acquisition.branch ?? undefined); taskEnv = injected.env; executorLog.log( `${task.id}: executor runtime env injected (${injected.pathEntryCount} PATH entries, ${injected.injectedKeyCount} env keys)`, ); this.options.onStart?.(task, worktreePath); const detail = await this.store.getTask(task.id); executorLog.debug(`${task.id}: fetched task detail (${detail.steps.length} steps, prompt length=${detail.prompt?.length ?? 0})`); // Initialize steps from PROMPT.md if empty if (detail.steps.length === 0) { const steps = await this.store.parseStepsFromPrompt(task.id); if (steps.length > 0) { await this.store.updateStep(task.id, 0, "pending"); } } // On resume (task.branch already set from a prior run), reconcile step // statuses from git history so the agent doesn't redo already-committed work. if (acquisition.isResume && task.branch && detail.steps.length > 0) { await this.reconcileStepsFromGitHistory(task.id, detail, worktreePath); } // ── Step-Session vs Single-Session execution path ── // When runStepsInNewSessions is enabled, each step runs in its own // fresh agent session via StepSessionExecutor. Otherwise, the existing // single-session flow runs all steps in one monolithic session. // Build skill selection context early so it's available in both paths const skillContext = await buildSessionSkillContext({ agentStore: this.options.agentStore!, task: detail, sessionPurpose: "executor", projectRootDir: this.rootDir, pluginRunner: this.options.pluginRunner, }); const graphSeamSkillName = this.graphSeamSkillName.get(task.id); const ceSkillsDir = typeof taskEnv?.FUSION_CE_SKILLS_DIR === "string" && taskEnv.FUSION_CE_SKILLS_DIR.trim() ? taskEnv.FUSION_CE_SKILLS_DIR.trim() : typeof process.env.FUSION_CE_SKILLS_DIR === "string" && process.env.FUSION_CE_SKILLS_DIR.trim() ? process.env.FUSION_CE_SKILLS_DIR.trim() : undefined; let stepSessionSkillSelection = skillContext.skillSelectionContext; if (graphSeamSkillName) { const bare = graphSeamSkillName.includes(":") ? graphSeamSkillName.slice(graphSeamSkillName.lastIndexOf(":") + 1) : graphSeamSkillName; const existing = stepSessionSkillSelection?.requestedSkillNames ?? []; stepSessionSkillSelection = { projectRootDir: stepSessionSkillSelection?.projectRootDir ?? this.rootDir, ...(stepSessionSkillSelection?.sessionPurpose ? { sessionPurpose: stepSessionSkillSelection.sessionPurpose } : { sessionPurpose: "executor" }), requestedSkillNames: [...new Set([...existing, graphSeamSkillName, bare])], }; } const stepSessionAdditionalSkillPaths = mergeAdditionalSkillPaths( skillContext.additionalSkillPaths, graphSeamSkillName && ceSkillsDir ? [ceSkillsDir] : undefined, ); if ( graphSeamSkillName && !isWorkflowStepSkillDiscoverable(graphSeamSkillName, stepSessionAdditionalSkillPaths, ceSkillsDir) ) { await this.store.logEntry( task.id, `[skill-load] Foreach step-execute requests skill '${graphSeamSkillName}' but it cannot be discovered from configured plugin body directories or FUSION_CE_SKILLS_DIR; the step runs with role-fallback skills only.`, ); } // Graph-owned stepwise runs force step-session physics for the run (KTD-2/ // KTD-8): the discrete per-step boundary the foreach driver needs exists only // in StepSessionExecutor. Pinned per run so a mid-flight setting toggle never // selects the unsupported (graph ON × step-sessions OFF) combination. const forceStepSession = this.graphStepSessionPinned.has(task.id); if (settings.runStepsInNewSessions || forceStepSession) { // ── Step-Session Path ────────────────────────────────────────── executorLog.debug(`${task.id}: using step-session mode (maxParallel=${settings.maxParallelSteps ?? 2}${forceStepSession ? ", graph-pinned" : ""})`); const stepSessionAgent = await this.getAuthoritativeAssignedAgent(detail.assignedAgentId); // Column-agent SESSION IDENTITY (U4, R2/R3/R4/R8): when the governing // step-execute node's declared column binds an agent that supersedes the // task's assigned agent, the per-step session's MODEL, runtime hint, and // attribution adopt the column agent. The core resolver decides defer vs // override (KTD-2); a missing agent logs + falls back (R8). Principal // alignment (U5, R5/R6): the gating contexts below ALSO key off the // effective `stepIdentityAgent`, and the effective principal is tracked for // the reverse-direction heartbeat guard. const stepColumnAgent = await this.resolveSeamColumnAgent(task, detail); const stepIdentityAgent = stepColumnAgent?.agent ?? stepSessionAgent; // U5 (R6): track the effective column-agent principal so the heartbeat // scheduler's reverse guard knows this agent is executing a task it may not // be assigned to. Cleared in deleteActiveStepExecutor. if (stepColumnAgent?.agent) { this.effectiveColumnAgentByTask.set(task.id, stepColumnAgent.agent.id); } const stepSessionRuntimeHint = extractRuntimeHint(stepIdentityAgent?.runtimeConfig); let accumulatedStepTokenUsage = detail.tokenUsage; const tokenUsageRecordedSteps = new Set(); let stepRotationEvent: import("./credential-instance-rotation.js").RotationEvent | undefined; let stepRotationDeclined = false; let stepDispatchedRotation = false; const initialStepSessionModel = resolveExecutorSessionModel( detail.modelProvider, detail.modelId, settings, (stepIdentityAgent?.runtimeConfig ?? undefined) as Record | undefined, detail.credentialInstanceId ?? undefined, ); let activeStepInstanceRef: ProviderInstanceRef | undefined = initialStepSessionModel.provider ? { providerId: initialStepSessionModel.provider, instanceId: initialStepSessionModel.credentialInstanceId ?? DEFAULT_PROVIDER_INSTANCE_ID, } : undefined; const stepExecutorRef: { current?: StepSessionExecutor } = {}; const nextStepInstance = async (): Promise => { /* FNXC:CredentialInstanceRotation 2026-08-01-11:22: Executor-step retries refresh task and project pause state at the limit boundary, rather than trusting dispatch snapshots. A pause arriving while a session is in flight must prevent an autonomous billed-account switch. */ const [liveTask, liveSettings] = await Promise.all([ this.store.getTask(task.id).catch(() => undefined), this.store.getSettings().catch(() => settings), ]); if (stepRotationDeclined || this.pausedAborted.has(task.id) || !liveTask || liveTask.userPaused === true || liveTask.autoMerge === false || liveSettings.globalPause === true || liveSettings.enginePaused === true || !activeStepInstanceRef?.providerId) return undefined; stepRotationEvent ??= await this.options.credentialRotator?.beginEvent({ providerId: activeStepInstanceRef.providerId, startingInstanceId: activeStepInstanceRef.instanceId, lane: "executor-step", taskId: task.id, }); if (!stepRotationEvent) { stepRotationDeclined = true; return undefined; } // FNXC:CredentialInstanceRotation 2026-08-01-11:34: beginEvent awaits credential inventory, so repeat the human-control check after it resolves. A pause that races this await must prevent cooldown writes and credential dispatch. const [postInventoryTask, postInventorySettings] = await Promise.all([ this.store.getTask(task.id).catch(() => undefined), this.store.getSettings().catch(() => settings), ]); if (this.pausedAborted.has(task.id) || !postInventoryTask || postInventoryTask.userPaused === true || postInventoryTask.autoMerge === false || postInventorySettings.globalPause === true || postInventorySettings.enginePaused === true) return undefined; this.options.credentialRotator?.markLimited(activeStepInstanceRef); if (stepDispatchedRotation) stepRotationEvent.recordOutcome("rotation-failed-limit"); const next = await stepRotationEvent.next(); if (!next) { stepRotationEvent.finishExhausted(); return undefined; } activeStepInstanceRef = next; stepDispatchedRotation = true; await stepExecutorRef.current?.retargetCredentialInstance(next); return next; }; /* FNXC:WorkflowStepControl 2026-06-29-10:15: Graph-pinned step sessions are lifecycle-owned by the workflow graph, not by the legacy executor prompt/tools. Their callback projection must use source:"graph" so independent steps can finish out of index order and so duplicate graph runner writes do not trigger the legacy sequential fn_task_update guard. */ const stepProjectionOptions = forceStepSession ? { source: "graph" as const } : undefined; const stepExecutor = new StepSessionExecutor({ store: this.store, taskDetail: detail, worktreePath, rootDir: this.rootDir, settings, // FNXC:GlobalConcurrencyControls 2026-07-14-18:30: When the graph run already owns a top-level slot (outerConcurrencyClaims), do not pass the semaphore into per-step sessions — each step would acquire a second slot and can deadlock under a full global cap. semaphore: this.outerConcurrencyClaims.has(task.id) ? undefined : this.options.semaphore, stuckTaskDetector: this.options.stuckTaskDetector, pluginRunner: this.options.pluginRunner, runtimeHint: stepSessionRuntimeHint, assignedAgentRuntimeConfig: (stepIdentityAgent?.runtimeConfig ?? undefined) as Record | undefined, /* * FNXC:CredentialInstanceRotation 2026-08-01-10:41: * Step sessions must start on the task-selected account. On a usage-limit * retry, re-read the live selection and resolve its provider with the same * effective column-agent runtime config used to create the session. */ credentialInstanceId: detail.credentialInstanceId, resolveCredentialInstanceRetarget: nextStepInstance, // Attribute the per-step run auditor to the column agent when it governs // (U4); absent → StepSessionExecutor falls back to assignedAgentId. effectiveAgentId: stepColumnAgent?.agent.id, actionGateContext: this.buildActionGateContext(task.id, stepIdentityAgent, settings.defaultAgentPermissionPolicy), permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, stepIdentityAgent, settings.defaultAgentPermissionPolicy), // FNXC:McpConfig 2026-06-25-23:03: Per-step workflow sessions are an executor lane, so they inherit the task's resolved MCP set from the effective step identity agent and never re-read or log plaintext secret values. mcpServers: await this.resolveMcpServers(stepIdentityAgent?.id), workflowStepThinkingLevel: this.graphSeamThinkingLevel.get(task.id), // FNXC:PluginSkills 2026-07-12-00:00: Step sessions must forward plugin skill body dirs alongside requested names; otherwise plugin-provided SKILL.md bodies are invisible to the inner createFnAgent loader. skillSelection: stepSessionSkillSelection, additionalSkillPaths: stepSessionAdditionalSkillPaths, // Pass agentStore and messageStore for delegation and messaging tools agentStore: this.options.agentStore, messageStore: this.options.messageStore, callerIsEphemeral: !stepIdentityAgent || isEphemeralAgent(stepIdentityAgent), sourceTaskId: task.id, sourceAgentId: stepIdentityAgent?.id, taskEnv, // FNXC:StepLifecycle 2026-07-22-09:53: Await the dependency-aware store projection before session allocation so a rejected out-of-order start cannot execute while its persisted step remains pending. onStepStart: async (stepIndex) => { try { const startResult = await this.store.startStep( task.id, stepIndex, stepProjectionOptions, ); if (!startResult.accepted) { executorLog.warn( `${task.id}: step ${stepIndex} start was rejected (${startResult.disposition}); persisted status is ` + `${startResult.task.steps?.[stepIndex]?.status ?? "missing"}`, ); return false; } this.options.stuckTaskDetector?.recordProgress(task.id); } catch (err) { executorLog.warn(`${task.id}: failed to update step ${stepIndex} status to in-progress: ${err}`); return false; } }, onStepComplete: (stepIndex, result) => { // FNXC:EngineDiagnostics 2026-07-26-10:05: per-step success is expected bookkeeping (incl. foreach instances); failures stay at log. if (result.success) { executorLog.debug(`${task.id}: step ${stepIndex} succeeded (${result.retries} retries)`); } else { executorLog.log(`${task.id}: step ${stepIndex} failed (${result.retries} retries)`); } try { this.store.updateStep(task.id, stepIndex, result.success ? "done" : "skipped", stepProjectionOptions).catch((err) => { executorLog.warn(`${task.id}: failed to update step ${stepIndex} status: ${err}`); }); const safeReason = result.success ? undefined : sanitizeFailureReason(result.error); if (!result.success) { void emitProactiveStatus( this.store, task.id, buildStepFailureMessage(stepIndex, detail.steps[stepIndex]?.name, safeReason!), "executor", safeReason, ); } } catch (err) { executorLog.warn(`${task.id}: failed to update step ${stepIndex} status: ${err}`); } if (!result.tokenUsage) { return; } const previousStepTokenUsage = accumulatedStepTokenUsage; accumulatedStepTokenUsage = this.accumulateTokenUsage(accumulatedStepTokenUsage, result.tokenUsage); if (accumulatedStepTokenUsage) { // FNXC:TokenAnalytics 2026-06-19-15:55: Step-scoped token writes now carry the producing session model so workflow-step sessions contribute their exact deltas to per-model analytics instead of relying on the last central session snapshot. accumulatedStepTokenUsage = this.tokenUsageWithModelSnapshot(accumulatedStepTokenUsage, undefined, previousStepTokenUsage, result.tokenUsage, accumulatedStepTokenUsage.lastUsedAt, { provider: result.tokenUsage.modelProvider, id: result.tokenUsage.modelId }); } tokenUsageRecordedSteps.add(stepIndex); if (!accumulatedStepTokenUsage) { return; } this.persistTaskTokenUsage(task.id, accumulatedStepTokenUsage).catch((err) => { executorLog.warn(`${task.id}: failed to persist token usage on step ${stepIndex} complete: ${err}`); }); }, }); stepExecutorRef.current = stepExecutor; this.setActiveStepExecutor(task.id, stepExecutor, worktreePath, this.createSeenSteeringIds(detail)); const stepWork = async () => { const results = await stepExecutor.executeAll(); // Check abort conditions after execution completes if (this.depAborted.has(task.id)) { this.depAborted.delete(task.id); await this.handleDepAbortCleanup(task.id, worktreePath); return; } if (this.pausedAborted.has(task.id)) { if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); return; } if (await this.parkApprovalSuspension(task.id, "step sessions")) return; this.clearPausedAborted(task.id); await this.store.logEntry(task.id, "Execution paused — step sessions terminated, moved to todo", undefined, this.getRunContextFor(task.id)); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: true }); return; } if (this.stuckAborted.has(task.id)) { stuckRequeue = this.stuckAborted.get(task.id) ?? true; this.stuckAborted.delete(task.id); return; } for (const result of results) { if (!result.tokenUsage || tokenUsageRecordedSteps.has(result.stepIndex)) { continue; } const previousStepTokenUsage = accumulatedStepTokenUsage; accumulatedStepTokenUsage = this.accumulateTokenUsage(accumulatedStepTokenUsage, result.tokenUsage); if (accumulatedStepTokenUsage) { accumulatedStepTokenUsage = this.tokenUsageWithModelSnapshot(accumulatedStepTokenUsage, undefined, previousStepTokenUsage, result.tokenUsage, accumulatedStepTokenUsage.lastUsedAt, { provider: result.tokenUsage.modelProvider, id: result.tokenUsage.modelId }); } } if (accumulatedStepTokenUsage) { await this.persistTaskTokenUsage(task.id, accumulatedStepTokenUsage); } const allSuccess = results.every(r => r.success); if (allSuccess) { const updatedTask = await this.store.getTask(task.id); // FNXC:Workspace 2026-06-21-23:30: KTD1 — per-repo post-session capture. // The singular call below runs UNGATED with worktreePath = the browse-only non-git workspace root and silently returns [] (resolveDiffBaseRef swallows the git failure at the root). In workspace mode there is nothing to diff at the root; the real changes live in each acquired sub-repo worktree. So we ADD (not replace) a workspace branch that loops `task.workspaceWorktrees` and reuses the EXISTING captureModifiedFiles per repo — reusing it (rather than hand-building `git diff ..HEAD`) gives us the merge-base fallback for an undefined repo.baseCommitSha (resolveDiffBaseRef) AND restores the contamination/divergence audit (filterFilesToOwnTaskCommits) for free per repo. Returned files are repo-prefixed (e.g. `repo-a/src/foo.ts`) and aggregated into task.modifiedFiles. if (this.workspaceConfig) { const workspaceWorktrees = updatedTask.workspaceWorktrees ?? {}; const aggregated = await this.captureWorkspaceModifiedFiles(updatedTask, audit, "post-session"); for (const [repoRel, repo] of Object.entries(workspaceWorktrees)) { // Per-repo branch-attribution audit (cwd = sub-repo). Run against repo.worktreePath/repo.branch, NOT the non-git root (a root call would fail and surface nothing). The contamination signal already rides on captureWorkspaceModifiedFiles above; this is the supplementary commit-attribution surface (FN-5233 pattern). try { const attributionBase = await this.resolveContaminationBaseRef(repo.worktreePath); if (attributionBase && repo.branch) { const attribution = await reportBranchAttribution(repo.worktreePath, repo.branch, attributionBase, task.id); const hasAnomaly = attribution.foreign.length > 0 || attribution.unattributed.length > 0 || attribution.ownUntrailed.length > 0; if (hasAnomaly) { const summary = `branch-attribution anomalies on ${repoRel}@${repo.branch}: foreign=${attribution.foreign.length}, unattributed=${attribution.unattributed.length}, ownUntrailed=${attribution.ownUntrailed.length}, ownTrailed=${attribution.ownTrailed}`; executorLog.warn(`${task.id}: ${summary}`); await this.store.logEntry(task.id, `[branch-attribution] ${summary}`, undefined, this.getRunContextFor(task.id)); await audit.git({ type: "branch:attribution-anomaly", target: repo.branch, metadata: { taskId: task.id, repo: repoRel, baseSha: attributionBase, ownTrailed: attribution.ownTrailed, foreign: attribution.foreign, unattributed: attribution.unattributed, ownUntrailed: attribution.ownUntrailed, }, }); } } } catch (attributionErr: unknown) { executorLog.warn(`${task.id}: post-session per-repo branch-attribution audit failed for ${repoRel}: ${attributionErr instanceof Error ? attributionErr.message : String(attributionErr)}`); } } if (aggregated.length > 0) { await this.store.updateTask(task.id, { modifiedFiles: aggregated }); executorLog.log(`${task.id}: captured ${aggregated.length} modified files across ${Object.keys(workspaceWorktrees).length} sub-repo(s)`); await audit.filesystem({ type: "file:capture-modified", target: task.id, metadata: { files: aggregated } }); } } else { const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha, task.id, audit, "post-session"); if (modifiedFiles.length > 0) { await this.store.updateTask(task.id, { modifiedFiles }); executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`); // Audit trail: record filesystem mutation (FN-1404) await audit.filesystem({ type: "file:capture-modified", target: task.id, metadata: { files: modifiedFiles } }); } // Post-session branch attribution audit: walk base..branch and surface // any commit that's foreign (different FN-id), unattributed (no subject // tag AND no Fusion-Task-Id trailer), or own-but-untrailed (signals the // commit-msg hook didn't fire — typically a worktree without identity // guards or a plumbing-driven commit). Logged loudly so contamination // gets caught within minutes of happening rather than days later at // merge time (FN-5233 was this pattern). try { const attributionBase = await this.resolveContaminationBaseRef(worktreePath); if (attributionBase && updatedTask.branch) { const attribution = await reportBranchAttribution(this.rootDir, updatedTask.branch, attributionBase, task.id); const hasAnomaly = attribution.foreign.length > 0 || attribution.unattributed.length > 0 || attribution.ownUntrailed.length > 0; if (hasAnomaly) { const summary = `branch-attribution anomalies on ${updatedTask.branch}: foreign=${attribution.foreign.length}, unattributed=${attribution.unattributed.length}, ownUntrailed=${attribution.ownUntrailed.length}, ownTrailed=${attribution.ownTrailed}`; executorLog.warn(`${task.id}: ${summary}`); await this.store.logEntry(task.id, `[branch-attribution] ${summary}`, undefined, this.getRunContextFor(task.id)); await audit.git({ type: "branch:attribution-anomaly", target: updatedTask.branch, metadata: { taskId: task.id, baseSha: attributionBase, ownTrailed: attribution.ownTrailed, foreign: attribution.foreign, unattributed: attribution.unattributed, ownUntrailed: attribution.ownUntrailed, }, }); } } } catch (attributionErr: unknown) { executorLog.warn(`${task.id}: post-session branch-attribution audit failed: ${attributionErr instanceof Error ? attributionErr.message : String(attributionErr)}`); } } // end !this.workspaceConfig singular capture (FNXC:Workspace KTD1) this.scheduleCompletedTaskWatchdog(task.id, "step-session completion"); if (await this.shouldDeferCompletionForGlobalPause(task.id, "before workflow steps after step-session completion")) { return; } // ── Deterministic verification gate (FN-3345) ────────── // Run testCommand/buildCommand after all steps succeed but BEFORE // workflow steps and the in-review transition. Skipped in fast mode // and when no verification commands are configured. if (executionMode !== "fast") { if (settings.testCommand?.trim() || settings.buildCommand?.trim()) { const verificationResult = await this.runExecutorDeterministicVerification(task, worktreePath, settings, taskEnv); if (!verificationResult.allPassed) { const failedType = verificationResult.failedCommand === "testCommand" ? "test" : "build"; const failedResult = failedType === "test" ? verificationResult.testResult! : verificationResult.buildResult!; const failedCommand = failedResult.command; const failureOutput = failedResult.stderr || failedResult.stdout || "Unknown error"; const summary = summarizeVerificationOutput(failureOutput, failedType); executorLog.log(`${task.id}: [verification] ${failedType} failed — attempting fix agent`); await this.store.logEntry( task.id, `[verification] ${failedType} command failed (exit ${failedResult.exitCode}). Attempting fix agent...`, summary, this.getRunContextFor(task.id), ); const maxFixRetries = Math.min(settings.verificationFixRetries ?? 3, 3); if (maxFixRetries === 0) { executorLog.log(`${task.id}: [verification] fix retries set to 0 — sending task back immediately`); await this.sendTaskBackForFix( task, worktreePath, `${failedType} command \`${failedCommand}\` failed (exit ${failedResult.exitCode}):\n${summary}`, `Verification (${failedType})`, `Deterministic verification failed (${failedType})`, true, true, ); return; } let fixSucceeded = false; for (let attempt = 1; attempt <= maxFixRetries; attempt++) { const fixed = await this.attemptExecutorVerificationFix( task, worktreePath, { command: failedCommand, exitCode: failedResult.exitCode, output: failureOutput, type: failedType, }, settings, attempt, maxFixRetries, taskEnv, ); if (fixed) { fixSucceeded = true; executorLog.log(`${task.id}: [verification] fix agent succeeded on attempt ${attempt}/${maxFixRetries}`); await this.store.logEntry( task.id, `[verification] Fix agent succeeded on attempt ${attempt}/${maxFixRetries}. Verification now passing.`, undefined, this.getRunContextFor(task.id), ); break; } executorLog.log(`${task.id}: [verification] fix agent attempt ${attempt}/${maxFixRetries} failed`); await this.store.logEntry( task.id, `[verification] Fix agent attempt ${attempt}/${maxFixRetries} failed`, undefined, this.getRunContextFor(task.id), ); } if (!fixSucceeded) { executorLog.log(`${task.id}: [verification] all fix attempts exhausted (${maxFixRetries}/${maxFixRetries}) — sending task back`); await this.sendTaskBackForFix( task, worktreePath, `${failedType} command \`${failedCommand}\` failed (exit ${failedResult.exitCode}) after ${maxFixRetries} fix attempts:\n${summary}`, `Verification (${failedType})`, `Deterministic verification failed after ${maxFixRetries} fix attempts`, true, true, ); return; } } } } // FNXC:WorkflowExecution 2026-06-25-00:00: U4 (KTD-2/KTD-5) — workflow // steps are graph-owned. For a graph-driven run the execute seam // registered a completion interceptor; stop at the // implementation-complete boundary and hand the remaining lifecycle // (workflow gates → review → merge) back to the graph runner, which // records results into task.workflowStepResults (U2). The legacy // runWorkflowSteps loop was deleted. A NON-graph run reaching here has no // enabled workflow steps to run (a minimal store WITH enabled steps is // parked fail-closed inside executeWorkflowGraph, KTD-5), so there // is nothing to gate before the in-review handoff. this.clearCompletedTaskWatchdog(task.id); executorLog.log(`✓ ${task.id} implementation complete — graph interpreter owns the remaining lifecycle`); const liveModified = (await this.store.getTask(task.id).catch(() => task)).modifiedFiles ?? []; reportImplementationExit?.("complete-from-live-files"); graphCompletion({ modifiedFiles: liveModified }); return; } else { const failedSteps = results.filter(r => !r.success); const errorSummary = failedSteps.map(r => `Step ${r.stepIndex}: ${r.error || "unknown error"}`).join("; "); await this.store.updateTask(task.id, { status: null, error: null }); await this.store.logEntry(task.id, `Step-session failed — requeued for execution resume: ${errorSummary}`, undefined, this.getRunContextFor(task.id)); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true, moveSource: "engine", recoveryRehome: true }); executorLog.log(`✗ ${task.id} step-session failed → todo resume: ${errorSummary}`); this.options.onError?.(task, new Error(errorSummary)); } }; const retryableStepWork = () => withRateLimitRetry(stepWork, { signal: this.activeWorkflowGraphAbortControllers.get(task.id)?.signal, rotation: this.options.credentialRotator && activeStepInstanceRef ? { providerId: activeStepInstanceRef.providerId, nextInstance: nextStepInstance, } : undefined, onRetry: (attempt, delayMs, error) => { const delaySec = Math.round(delayMs / 1000); executorLog.warn(`⏳ ${task.id} rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`); this.store.logEntry(task.id, `Rate limited — retry ${attempt} in ${delaySec}s`, undefined, this.getRunContextFor(task.id)).catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${task.id} failed to log rate-limit retry: ${msg}`); }); }, }); try { await this.runWithExecutorSemaphore(task.id, retryableStepWork); if (stepDispatchedRotation) stepRotationEvent?.recordOutcome("rotation-succeeded"); } catch (err: unknown) { const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err); if (this.depAborted.has(task.id)) { this.depAborted.delete(task.id); await this.handleDepAbortCleanup(task.id, worktreePath); } else if (this.pausedAborted.has(task.id)) { if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); return; } if (await this.parkApprovalSuspension(task.id, "step session")) return; this.clearPausedAborted(task.id); await this.store.logEntry(task.id, "Execution paused during step-session", undefined, this.getRunContextFor(task.id)); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: true }); } else if (this.stuckAborted.has(task.id)) { stuckRequeue = this.stuckAborted.get(task.id) ?? true; this.stuckAborted.delete(task.id); } else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) { await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage); } else if (isTransientError(errorMessage)) { const decision = computeRecoveryDecision({ recoveryRetryCount: task.recoveryRetryCount, nextRecoveryAt: task.nextRecoveryAt, }); if (decision.shouldRetry) { const attempt = decision.nextState.recoveryRetryCount; const delay = formatDelay(decision.delayMs); if (!isSilentTransientError(errorMessage)) { executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`); await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id)); } if (worktreePath && existsSync(worktreePath)) { try { const settings = await this.store.getSettings(); await removeWorktree({ worktreePath, rootDir: this.rootDir, settings, taskId: task.id, audit, reason: RemovalReason.ExecutorTransientRetry, expectedOwnerTaskId: task.id, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); } catch (wtErr: unknown) { const msg = wtErr instanceof Error ? wtErr.message : String(wtErr); executorLog.warn(`${task.id}: worktree removal failed during transient-error retry cleanup (${worktreePath}): ${msg}`); } } await this.store.updateTask(task.id, { recoveryRetryCount: decision.nextState.recoveryRetryCount, nextRecoveryAt: decision.nextState.nextRecoveryAt, worktree: null, branch: null, }); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true }); stuckRequeue = null; // Prevent outer finally from re-processing return; } executorLog.error(`✗ ${task.id} transient error retries exhausted: ${errorDetail}`); if (errorStack) { await this.store.logEntry(task.id, `Transient error retries exhausted: ${errorMessage}`, errorStack, this.getRunContextFor(task.id)); } await this.store.updateTask(task.id, { status: "failed", error: errorMessage, recoveryRetryCount: null, nextRecoveryAt: null, }); if (accumulatedStepTokenUsage) { await this.persistTaskTokenUsage(task.id, accumulatedStepTokenUsage); } executorLog.log(`✗ ${task.id} transient retries exhausted — failed in execution`); this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage)); } else { if (accumulatedStepTokenUsage) { await this.persistTaskTokenUsage(task.id, accumulatedStepTokenUsage); } if (await this.handleNonContinuableSessionError(task, false, errorMessage)) { return; } executorLog.error(`✗ ${task.id} step-session execution failed:`, errorDetail); await this.store.logEntry(task.id, `Step-session execution failed: ${errorMessage}`, errorStack ?? errorDetail, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { status: null, error: null }); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true, moveSource: "engine", recoveryRehome: true }); executorLog.log(`✗ ${task.id} step-session execution failed → todo resume`); this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage)); } } finally { this.executing.delete(task.id); executingTaskLock.release(task.id); this.loopRecoveryState.delete(task.id); // Wrap cleanup in try/catch so activeStepExecutors.delete() always runs. // If cleanup() throws, the executor continues to clean up the in-memory map // and requeue logic without leaking the reference. try { await stepExecutor.cleanup(); } catch (cleanupErr) { executorLog.warn(`StepSessionExecutor cleanup failed for ${task.id}: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`); } this.deleteActiveStepExecutor(task.id); // Stuck-requeue: clean up worktree and move to todo if (stuckRequeue === true) { try { // Re-read latest task state. Self-healing may have already moved // the task out of in-progress while this step-session execution // was unwinding; continuing the cleanup would clobber a valid // recovery (see the analogous block in the outer finally for the // full reasoning). /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet: stuck-requeue family): "has a concurrent recovery already moved this card on?" — the pre-completion lanes are the board's wip and hold. With literals a renamed board always answered "moved on", the cleanup never ran, and the log line blamed a concurrent recovery that had not happened. */ const latestTask = await this.store.getTask(task.id); const requeueLanes = await this.resolveResumeLanes(task.id); if (latestTask.column !== requeueLanes.wip && latestTask.column !== requeueLanes.hold) { executorLog.log( `${task.id} stuck-requeue skipped — task is now in '${latestTask.column}' (recovered concurrently)`, ); } else { const settings = await this.store.getSettings(); const preserveProgress = settings.preserveProgressOnStuckRequeue !== false; /* FNXC:StuckRequeue 2026-06-27-23:15: Stuck requeue may destroy a checkout that contains only uncommitted step output. Always reconcile lost-work step state before worktree removal, even when preserve-progress is enabled, so a retry cannot skip code that no longer exists. */ await this.resetStepsIfWorkLost(latestTask); if (worktreePath && existsSync(worktreePath)) { try { await removeWorktree({ worktreePath, rootDir: this.rootDir, settings, taskId: task.id, reason: RemovalReason.ExecutorStuckKilled, expectedOwnerTaskId: task.id, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); } catch (wtErr: unknown) { const msg = wtErr instanceof Error ? wtErr.message : String(wtErr); executorLog.warn(`${task.id}: worktree removal failed during stuck-requeue cleanup (${worktreePath}): ${msg}`); } } await this.store.updateTask(task.id, { status: "queued", error: null, worktree: null, branch: null, }); const reboundColumn = await resolveReboundColumnFor(this.store, task.id); if (latestTask.column !== reboundColumn) { this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, reboundColumn, preserveProgress ? { preserveProgress: true } : undefined); executorLog.log(`${task.id} moved to ${reboundColumn} for retry after stuck kill${preserveProgress ? " (progress preserved)" : ""}`); } } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to requeue stuck task ${task.id}: ${errorMessage}`); } stuckRequeue = null; // Prevent outer finally from re-processing } } // Step-session path handled completely — return before outer catch/finally return; } // ── Single-Session Path (default) ──────────────────────────────── // Build custom tools for the worker // Track the last code review verdict per step so we can enforce REVISE // (block fn_task_update status="done" until the agent re-reviews and gets APPROVE). // Keyed by the canonical 0-indexed step number used by PROMPT.md headings. const codeReviewVerdicts = new Map(); let wasPaused = false; // Mutable ref — populated after createFnAgent, tools access lazily via closure const sessionRef: { current: AgentSession | null } = { current: null }; /* FNXC:ReviewerProviderErrors 2026-07-19-02:30: DELETED (U10/R9): the deferred provider-error re-raise channel (`reviewerFatalRef`) and the per-step conversation checkpoint map (`stepCheckpoints`, the RETHINK rewind target) existed only to serve the legacy in-session `fn_review_step` tool. Both die with it. Graph-owned review nodes run on their own session and can throw directly, and a RETHINK is a graph edge rather than an in-conversation `navigateTree` rewind — so neither mechanism has a caller. Do not re-introduce a tool-handler-deferred error channel here: it only ever existed because pi-agent-core converts a tool throw into a `tool_error` result the model reads and retries. */ const stuckDetector = this.options.stuckTaskDetector; const assignedAgentId = detail.assignedAgentId?.trim(); const reflectionTools = this.options.reflectionService && settings.reflectionEnabled && assignedAgentId ? [createReflectOnPerformanceTool(this.options.reflectionService, assignedAgentId)] : []; const assignedAgent = await this.getAuthoritativeAssignedAgent(assignedAgentId); // Column-agent SESSION IDENTITY (U4, R2/R3/R4/R8): when the governing execute // seam node's declared column binds an agent that supersedes the task's // assigned agent, the coding session's MODEL, runtime hint, persona, and // memory tools adopt the column agent. The core resolver decides defer vs // override (KTD-2); a missing agent logs + falls back (R8). No binding → // `columnAgentSeam` is undefined and every line below is byte-identical to the // assigned-agent path (characterization parity). Gating contexts key off // `identityAgent` — the effective column agent when a binding governs, else // the assigned agent (U5/KTD-3 principal substitution). const columnAgentSeam = await this.resolveSeamColumnAgent(task, detail); const identityAgent = columnAgentSeam?.agent ?? assignedAgent; const executorRuntimeHint = extractRuntimeHint(identityAgent?.runtimeConfig); // U5 (R6): track the effective column-agent principal so the heartbeat // scheduler's reverse guard knows this agent is executing a task it may not // be assigned to. Cleared in deleteActiveSession. if (columnAgentSeam?.agent) { this.effectiveColumnAgentByTask.set(task.id, columnAgentSeam.agent.id); } // Log fast mode status if (executionMode === "fast") { executorLog.debug(`${task.id}: fast mode`); } /* FNXC:TaskVerificationRequest 2026-07-30-00:00: Chat can only enqueue a server-resolved profile. The executor owns the live worktree, so it claims and runs that request here through the existing bounded runner (which acquires withVerificationSlot); no chat-side subprocess exists. */ let verificationRequestInFlight = false; const runPendingTaskVerification = async (): Promise => { if (verificationRequestInFlight) return; const pendingVerification = await this.store.getTaskVerificationRequestAsync(task.id); if (pendingVerification?.status !== "requested") return; verificationRequestInFlight = true; try { const claimedVerification = await this.store.claimTaskVerificationRequest(task.id, pendingVerification.requestId); if (!claimedVerification) return; const startedAt = Date.now(); try { const verificationResult = await runTaskVerificationCommand({ command: claimedVerification.command, cwd: worktreePath, timeoutMs: settings.verificationCommandTimeoutMs ?? 300_000, onHeartbeat: () => stuckDetector?.recordActivity(task.id), }); await this.store.finishTaskVerificationRequest(task.id, claimedVerification.requestId, verificationResult.success ? "passed" : "failed", { success: verificationResult.success, exitCode: verificationResult.exitCode, durationMs: Date.now() - startedAt, timedOut: verificationResult.timedOut ?? false, stdoutTail: verificationResult.stdout.slice(-8_000), stderrTail: verificationResult.stderr.slice(-8_000), }); } catch (error) { await this.store.finishTaskVerificationRequest(task.id, claimedVerification.requestId, "failed", undefined, error instanceof Error ? error.message.slice(0, 1_000) : "Verification runner failed"); } } finally { verificationRequestInFlight = false; } }; await runPendingTaskVerification(); /* FNXC:EphemeralAgentTaskCreation 2026-07-26-06:20: A `deny` project policy removes fn_task_create from the session's tool list instead of registering a tool that only refuses at execute time; see isAgentTaskCreateToolAvailable. FNXC:EphemeralAgentTaskCreation 2026-07-26-07:40: fn_delegate_task is withheld by the same policy (it creates a task through the same primitive), and the suppression emits a run-audit event. Without the event an operator cannot distinguish "the policy suppressed the tool" from "the agent had nothing to file" — every other policy decision in this engine leaves that trail. */ const executionCallerIsEphemeral = !identityAgent || isEphemeralAgent(identityAgent); const taskCreateWithheld = !isAgentTaskCreateToolAvailable(settings, executionCallerIsEphemeral); const delegateWithheld = !isAgentDelegateTaskToolAvailable(settings, executionCallerIsEphemeral); if (taskCreateWithheld || delegateWithheld) { await this.store.recordRunAuditEvent?.({ taskId: task.id, agentId: identityAgent?.id ?? "executor", runId: this.getRunContextFor(task.id)?.runId ?? generateSyntheticRunId("task-create-withheld", task.id), domain: "database", mutationType: "agent:task-create-withheld", target: task.id, metadata: { taskId: task.id, policy: resolveEphemeralTaskCreationPolicy(settings), withheldTaskCreate: taskCreateWithheld, withheldDelegateTask: delegateWithheld, lane: "execution-session", }, }).catch(() => undefined); } /* FNXC:AgentProvisioningGate 2026-07-26-13:20: fn_agent_create / fn_agent_delete previously received no options in the executor lane, which made the factory synthesize approvalMode "never" and disabled the provisioning approval gate in production. Pass a live settingsProvider plus the shared PostgreSQL-backed ApprovalRequestStore when the async layer exists; without a layer we pass no approval store so the factory fails CLOSED (require-approval => DENY). */ const provisioningApprovalLayer = typeof this.store.getAsyncLayer === "function" ? this.store.getAsyncLayer() : null; const agentProvisioningToolOptions = { settingsProvider: async () => await this.store.getSettings(), ...(provisioningApprovalLayer ? { approvalRequestStore: this.approvalRequestStore } : {}), }; const customTools = [ this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stuckDetector), this.createTaskLogTool(task.id), this.createTaskLogsReadTool(task.id), ...(taskCreateWithheld ? [] : [this.createTaskCreateTool(executionCallerIsEphemeral, task.id, identityAgent?.id)]), this.createTaskAddDepTool(task.id), this.createTaskDoneTool(task.id, worktreePath, detail.prompt ?? "", codeReviewVerdicts, () => { taskDone = true; }, audit), createRunVerificationTool({ worktreePath, rootDir: this.rootDir, taskId: task.id, recordActivity: () => stuckDetector?.recordActivity(task.id), verificationCommandTimeoutMs: settings.verificationCommandTimeoutMs, onVerificationStart: (timeoutMs) => stuckDetector?.beginVerification(task.id, timeoutMs), onVerificationEnd: () => stuckDetector?.endVerification(task.id), log: { info: (s) => executorLog.log(s), debug: (s) => executorLog.debug(s), warn: (s) => executorLog.warn(s), error: (s) => executorLog.warn(s), }, }), /* FNXC:WorkflowReviewGates 2026-07-19-02:30: U10 (R9): the legacy in-session `fn_review_step` tool is DELETED. Plan/code/browser review gates are owned exclusively by workflow-graph nodes, so an implementation session never spawns its own reviewer. Nothing is injected here; the entry is kept as a tombstone marker so a future reader does not re-add a second review authority. */ this.createSpawnAgentTool(task.id, worktreePath, settings, taskEnv), this.createTaskDocumentWriteTool(task.id), this.createTaskDocumentReadTool(task.id), // FNXC:FileScope 2026-07-08-22:40: let the coding agent extend its own declared ## File Scope at runtime (fn_task_file_scope_add) so edits beyond the initial scope are not stranded by the scope-aware squash merge. this.createTaskFileScopeAddTool(task.id), this.createArtifactListTool(), this.createArtifactViewTool(), /* FNXC:ArtifactRegistry 2026-07-10-14:30: fn_artifact_register was previously gated on assignedAgentId, but default ephemeral mode never sets assignedAgentId on in-progress tasks — so executor agents never had the register tool at all and agent-produced screenshots/wireframes could not reach the Artifacts gallery. Always expose it, attributing ephemeral runs to the established "executor" fallback author. */ this.createArtifactRegisterTool(assignedAgentId ?? "executor", task.id, worktreePath), this.createWorkflowListTool(), this.createWorkflowGetTool(), this.createWorkflowValidateTool(), this.createWorkflowSelectTool(task.id), this.createTaskPromoteTool(task.id), this.createWorkflowCreateTool(), this.createWorkflowUpdateTool(), this.createWorkflowDeleteTool(), this.createWorkflowSettingsTool(), this.createTraitListTool(), ...(isResearchToolSurfaceEnabled(settings) ? createResearchTools({ store: this.store, rootDir: this.rootDir, getSettings: async () => this.store.getSettings(), }) : []), ...createMissionTools(this.store, { agentId: engineRunContext.agentId, agentName: identityAgent?.name, }), ...createIdeationTools(this.store), ...createGoalRetrievalTools(this.store, { runContext: { runId: engineRunContext.runId, agentId: engineRunContext.agentId, }, taskId: task.id, }), createWebFetchTool(), ...createMemoryTools(this.rootDir, settings, identityAgent ? { agentMemory: { agentId: identityAgent.id, agentName: identityAgent.name, memory: identityAgent.memory, }, } : undefined), // Conditionally add agent self-reflection when enabled and task has an assigned agent. ...reflectionTools, // Agent delegation tools — discover and delegate work to other agents. ...(this.options.agentStore ? [ createListAgentsTool(this.options.agentStore), ...(delegateWithheld ? [] : [createDelegateTaskTool(this.options.agentStore, this.store, { rootDir: this.rootDir, sourceTaskId: task.id, sourceAgentId: assignedAgentId, callerIsEphemeral: executionCallerIsEphemeral })]), createTaskAssignTool(this.options.agentStore, this.store), ...(assignedAgentId ? [ createGetAgentConfigTool(this.options.agentStore, assignedAgentId), createUpdateAgentConfigTool(this.options.agentStore, assignedAgentId), createAgentCreateTool(this.options.agentStore, assignedAgentId, agentProvisioningToolOptions), createAgentDeleteTool(this.options.agentStore, assignedAgentId, agentProvisioningToolOptions), ] : []), ] : []), // Messaging tools — allows executor agents to send and receive messages. ...(this.options.messageStore && assignedAgentId ? [ createSendMessageTool(this.options.messageStore, assignedAgentId, { autoRecovery: settings.autoRecovery, runAudit: audit, taskStore: this.store, settings, agentStore: this.options.agentStore }), createReadMessagesTool(this.options.messageStore, assignedAgentId), ] : []), // Add plugin tools from PluginRunner ...getEnabledPluginTools(this.options.pluginRunner), ]; if (this.workspaceConfig && this.workspaceConfig.repos.length > 0) { customTools.push(createAcquireRepoWorktreeTool({ workspaceRootDir: this.rootDir, workspaceRepos: this.workspaceConfig.repos, task, store: this.store, settings, logger: executorLog, secretsStore: this.options.secretsStore, runContext: engineRunContext, audit, // FNXC:Workspace 2026-06-21-22:30: F2 — register each freshly-acquired sub-repo worktree path in this task's activeWorktrees Set (KTD2) so owner/liveness checks see live per-repo worktrees, not just the browse-only root. onAcquired: (worktreePath: string) => this.addActiveWorktree(task.id, worktreePath), taskEnv, // FNXC:Workspace 2026-06-22 — forward the configured worktree-init runner so sub-repo worktrees run configured setup. runConfiguredCommand: (command, cwd, timeoutMs, env) => runConfiguredCommand(command, cwd, timeoutMs, env, audit), })); } // Accumulates the full assistant text output for the most recent session. // Reset to "" each time a new session begins so detectPseudoPause only // sees the last session's output, not the entire conversation history. let lastAssistantText = ""; const agentLogger = new AgentLogger({ store: this.store, taskId: task.id, agent: "executor", persistAgentToolOutput: settings.persistAgentToolOutput, // Executor sessions are task-scoped ephemeral workers. persistAgentThinkingLog: resolvePersistAgentThinkingLog(settings, { ephemeral: true }), onAgentText: (taskId, delta) => { lastAssistantText += delta; stuckDetector?.recordActivity(taskId); this.options.onAgentText?.(taskId, delta); }, onAgentTool: (taskId, toolName, detail) => { /* FNXC:StuckDetector 2026-07-22-18:05: Tool heartbeats carry name+detail fingerprints so the stuck detector can distinguish legitimate iterative single-step work from repetitive thrash loops. FNXC:StuckDetector 2026-07-22-19:25: Forward `detail` to options.onAgentTool so external telemetry keeps the full fingerprint contract (CodeRabbit on PR #2404). */ stuckDetector?.recordActivity(taskId, { toolName, toolDetail: detail }); this.options.onAgentTool?.(taskId, toolName, detail); }, // FNXC:PlannerOversight 2026-07-13-23:05: live session-advisor delta path (fail-soft). onEntriesFlushed: (taskId, entries) => { try { this.options.onExecutorLogFlushed?.(taskId, entries); } catch { /* ignore */ } }, }); let agentRotationEvent: import("./credential-instance-rotation.js").RotationEvent | undefined; let agentRotationDeclined = false; let agentDispatchedRotation = false; let activeAgentInstanceRef: ProviderInstanceRef | undefined; const agentWork = async () => { // Resolve model settings using canonical lane hierarchy: // 1. Task override pair (modelProvider + modelId) // 2. Project execution lane pair (executionProvider + executionModelId) // 3. Global execution lane pair (executionGlobalProvider + executionGlobalModelId) // 4. Project default override pair (defaultProviderOverride + defaultModelIdOverride) // 5. Global default pair (defaultProvider + defaultModelId) // Column-agent session identity (U4): the model precedence input is the // EFFECTIVE identity agent's runtimeConfig (column agent when it governs, // else the assigned agent — byte-identical no-binding path). /* FNXC:ColumnAgentModel 2026-06-27-11:24: Override column agents own initial session model selection as well as mid-flight re-resolution. Ignore task-level modelProvider/modelId before resolveExecutorSessionModel so pre-existing task model pairs cannot run the column-agent identity on the task model. */ const overrideColumnGovernsInitialSession = columnAgentSeam?.mode === "override"; const executorSessionModel = resolveExecutorSessionModel( overrideColumnGovernsInitialSession ? undefined : detail.modelProvider, overrideColumnGovernsInitialSession ? undefined : detail.modelId, settings, (identityAgent?.runtimeConfig ?? undefined) as Record | undefined, overrideColumnGovernsInitialSession ? undefined : activeAgentInstanceRef?.instanceId ?? detail.credentialInstanceId, ); const { provider: executorProvider, modelId: executorModelId } = executorSessionModel; activeAgentInstanceRef ??= executorProvider ? { providerId: executorProvider, instanceId: executorSessionModel.credentialInstanceId ?? DEFAULT_PROVIDER_INSTANCE_ID } : undefined; const { provider: executorFallbackProvider, modelId: executorFallbackModelId } = resolveExecutorFallbackModel(settings); const executorSessionThinkingSource = this.graphSeamThinkingLevel.get(task.id) ?? detail.thinkingLevel; const executorThinkingLevel = resolveExecutorThinkingLevel(executorSessionThinkingSource, settings); const executorFallbackThinkingLevel = resolveExecutorFallbackThinkingLevel(executorSessionThinkingSource, settings); // U1 telemetry: now that the session model/provider/node are resolved, // give the agent logger the context it needs to emit usage_events tool // rows (KTD3). nodeId is sourced from the routed/effective node, null // when the task has no node context. agentLogger.setUsageContext({ model: executorModelId ?? null, provider: executorProvider ?? null, nodeId: detail.effectiveNodeId ?? detail.nodeId ?? null, agentId: engineRunContext.agentId ?? null, }); // Determine whether we're resuming a previous session (pause/resume) // or starting fresh. Use file-based sessions so conversation state // persists across pause/unpause cycles. Resume is allowed only when // persisted session metadata still matches the task's live worktree. let isResuming = !!task.sessionFile && existsSync(task.sessionFile); if (isResuming) { const persistedWorktreePath = await extractPersistedSessionWorktreePath(task.sessionFile!, this.rootDir, settings); if (!isSessionWorktreeCompatible(persistedWorktreePath, worktreePath)) { executorLog.warn( `${task.id}: stale sessionFile worktree mismatch (session=${persistedWorktreePath}, task=${worktreePath}); starting fresh session`, ); await this.store.logEntry( task.id, `Detected stale persisted session metadata (worktree mismatch: ${persistedWorktreePath} vs ${worktreePath}) — discarded resume state and started fresh session`, undefined, this.getRunContextFor(task.id), ); await this.store.updateTask(task.id, { sessionFile: null }); isResuming = false; } } const sessionManager = isResuming ? SessionManager.open(task.sessionFile!) : SessionManager.create(worktreePath); executorLog.debug(`${task.id}: creating agent session (provider=${executorProvider ?? "default"}, model=${executorModelId ?? "default"}, resuming=${isResuming})`); // Resolve per-agent custom instructions for the executor role. // Column-agent session identity (U4, R3/KTD-6): when a column agent governs, // its TYPED persona (soul/instructionsText, via buildAgentPersona — the same // source the custom-node path uses) supersedes the role-resolved executor // instructions, so the coding session speaks AS the column agent. No binding // → role instructions unchanged (characterization parity). const columnAgentPersona = columnAgentSeam ? this.buildAgentPersona(columnAgentSeam.agent) : undefined; const executorInstructions = columnAgentPersona ?? (await this.resolveInstructionsForRole("executor", settings)); // Build structured layers for cross-session prompt caching. const executorPluginContributions = await buildPluginPromptSection( "executor-system", this.options.pluginRunner, ); if (executorPluginContributions) { executorLog.debug(`${task.id}: applied plugin prompt contributions for executor-system surface`); } const executorGoalResolution = await resolveAndEmitGoalContext({ lane: "executor", store: this.store, audit, taskId: task.id, runContext: engineRunContext, }); const executorGoalContext = executorGoalResolution.goalContext; const executorLayers = buildPromptLayers({ basePrompt: getExecutorSystemPrompt(settings, { taskCreateWithheld, delegateWithheld }), goalContext: executorGoalContext, agentInstructions: executorInstructions, pluginContributions: executorPluginContributions, }); const executorSystemPromptFinal = collapsePromptLayers(executorLayers); // sessionFile must be let because it's assigned before downstream retry-session reassignment. let session: AgentSession; let sessionFile: string | null | undefined; try { const createdSession = await createResolvedAgentSession({ sessionPurpose: "executor", runtimeHint: executorRuntimeHint, pluginRunner: this.options.pluginRunner, cwd: worktreePath, systemPrompt: executorSystemPromptFinal, systemPromptLayers: executorLayers, tools: "coding", customTools, onText: agentLogger.onText, onThinking: agentLogger.onThinking, onToolStart: agentLogger.onToolStart, onToolEnd: agentLogger.onToolEnd, defaultProvider: executorProvider, defaultModelId: executorModelId, ...(activeAgentInstanceRef ? { credentialInstanceId: activeAgentInstanceRef.instanceId } : {}), fallbackProvider: executorFallbackProvider, fallbackModelId: executorFallbackModelId, fallbackThinkingLevel: executorFallbackThinkingLevel, defaultThinkingLevel: executorThinkingLevel, runAuditor: audit, settings, sessionManager, taskEnv, mcpServers: await this.resolveMcpServers(identityAgent?.id), // FNXC:PluginSkills 2026-07-12-00:00: Plugin skill session delivery requires forwarding both requested names and body directories so the pi loader can discover plugin-package SKILL.md files. ...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}), ...(skillContext.additionalSkillPaths.length > 0 ? { additionalSkillPaths: skillContext.additionalSkillPaths } : {}), // Column-agent principal alignment (plan U5, R5): action gating is // computed for the agent ACTUALLY RUNNING. When the governing execute // seam's column binds an agent that supersedes the assigned agent, // `identityAgent` is that column agent; otherwise it is `assignedAgent` // (byte-identical to before). The builders already accept an `Agent` // object, so this is a call-site object swap, not gating-internals surgery. actionGateContext: this.buildActionGateContext(task.id, identityAgent, settings.defaultAgentPermissionPolicy), permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, identityAgent, settings.defaultAgentPermissionPolicy), taskId: task.id, taskTitle: detail.title, onFallbackModelUsed: createFallbackModelObserver({ agent: "executor", label: "executor", store: this.store, taskId: task.id, taskTitle: detail.title, }), }); session = createdSession.session; sessionFile = createdSession.sessionFile; } catch (sessionStartError) { if (await this.recoverMissingWorktreeSessionStartFailure(task, worktreePath, sessionStartError, audit)) { return; } throw sessionStartError; } const executorModelDesc = describeModel(session); const executorModelDetails = formatModelMarkerDetails(executorModelDesc, executorThinkingLevel); const executorModelMarker = `Executor using model: ${executorModelDetails}`; if (isResuming) { executorLog.debug(`${task.id}: resumed session from ${task.sessionFile}`); await this.store.logEntry(task.id, `Resumed agent session after unpause (model: ${executorModelDesc})`, undefined, this.getRunContextFor(task.id)); } else { executorLog.debug(`${task.id}: using model ${executorModelDesc}`); await this.store.logEntry(task.id, executorModelMarker, undefined, this.getRunContextFor(task.id)); // Persist session file path so pause/resume can reopen it if (sessionFile) { await this.store.updateTask(task.id, { sessionFile }); } } await this.store.appendAgentLog(task.id, executorModelMarker, "status", undefined, "executor"); // Capture both executor and session-helper baselines before any task prompt consumes tokens. await this.captureExecutorTokenUsageBaseline(task.id, session); captureSessionTokenBaseline(session); // Make session available to custom tools sessionRef.current = session; // Register session so the pause listener can terminate it. // Initialize with all existing steering comments so only mid-flight // comments are injected into the running session. const seenSteeringIds = this.createSeenSteeringIds(detail); this.setActiveSession(task.id, { session, seenSteeringIds, lastResolvedModelProvider: executorProvider, lastResolvedModelId: executorModelId, lastTaskModelProvider: detail.modelProvider, lastTaskModelId: detail.modelId, lastAssignedAgentId: detail.assignedAgentId ?? null, // U5 (R7): the effective column-agent governing this session (null when no // binding governs — legacy path). The watcher re-resolves this for graph- // mode entries to detect a mid-flight workflow-edit / agent-config change. lastEffectiveColumnAgentId: columnAgentSeam?.agent.id ?? null, }, worktreePath); /* FNXC:TaskVerificationRequest 2026-07-30-17:40: A chat request can arrive after this executor session starts. Poll while this task retains the live worktree so requested records are claimed by their owner rather than waiting for an unrelated future dispatch. */ const verificationRequestTimer = setInterval(() => { void runPendingTaskVerification().catch((error) => { executorLog.warn(`${task.id}: verification request pickup failed: ${error instanceof Error ? error.message : String(error)}`); }); }, 1_000); let leaseRenewalTimer: ReturnType | undefined; if (detail.assignedAgentId && detail.checkedOutBy === detail.assignedAgentId) { const leaseEpoch = detail.checkoutLeaseEpoch ?? 0; const checkoutNodeId = detail.checkoutNodeId ?? detail.effectiveNodeId ?? detail.nodeId ?? "local"; const runId = this.getRunContextFor(task.id)?.runId; await this.renewTaskLease(task.id, detail.assignedAgentId, leaseEpoch, checkoutNodeId, runId).catch(() => {}); leaseRenewalTimer = setInterval(() => { void this.renewTaskLease(task.id, detail.assignedAgentId!, leaseEpoch, checkoutNodeId, runId).catch(() => {}); }, 30_000); } // Register with stuck task detector for heartbeat monitoring stuckDetector?.trackTask(task.id, session); executorLog.debug(`${task.id}: session registered (model=${describeModel(session)}, stuckDetector=${!!stuckDetector})`); // Invoke plugin onAgentRunStart hook (fire-and-forget) void this.options.pluginRunner?.invokeHookSafe("onAgentRunStart", task.id); try { // Record activity on prompt start (heartbeat for stuck detection) stuckDetector?.recordActivity(task.id); executorLog.debug(`${task.id}: calling promptWithFallback()...`); if (isResuming) { // Session already has full conversation history — just tell the // agent it was paused and should pick up where it left off. await promptWithFallback(session, [ "Your session was paused and has now been resumed.", "Continue working on the task from where you left off.", "Review the current state of your worktree and proceed with the next pending step.", ].join("\n")); } else { const customFieldDefs = await this.resolveTaskCustomFieldDefs(task.id); const pluginTaskContributions = await buildPluginPromptSection("executor-task", this.options.pluginRunner); const agentPrompt = buildExecutionPrompt( detail, this.rootDir, settings, worktreePath, this.options.pluginRunner, customFieldDefs, this.workspaceConfig, { pluginTaskContributions, }, ); await promptWithFallback(session, agentPrompt); } // Re-raise errors that pi-coding-agent swallowed after exhausting retries. // session.prompt() resolves normally even when retries are exhausted — // the error is stored on session.state.error instead of being thrown. checkSessionError(session); await this.persistTokenUsage(task.id, session); // Check if proactive context compaction is needed based on token cap setting. // This runs after the main prompt completes to avoid interrupting active work. try { const capResult = await this.tokenCapDetector.checkAndCompact( session, task.id, settings.tokenCap, async (s) => { const compactResult = await compactSessionContext(s); if (compactResult) { await this.store.logEntry( task.id, `Context compacted at ${compactResult.tokensBefore} tokens (token cap: ${settings.tokenCap})`, undefined, this.getRunContextFor(task.id), ); } return compactResult; }, ); if (capResult.triggered) { executorLog.debug(`${task.id} token cap check: ${capResult.message}`); } } catch (err) { executorLog.debug(`${task.id} token cap check failed (non-fatal): ${err}`); } // If loop recovery is pending (compact-and-resume was triggered by // handleLoopDetected), consume the pending state and resume with a // deterministic prompt. The session has already been compacted, so // we just need to send a fresh prompt to continue execution. const loopState = this.loopRecoveryState.get(task.id); if (loopState?.pending) { loopState.pending = false; executorLog.log(`${task.id} consuming loop recovery — resuming with fresh context`); await this.store.logEntry(task.id, "Resuming execution after context compaction — taking a different approach", undefined, this.getRunContextFor(task.id)); // Reset activity tracking so the detector doesn't immediately re-trigger stuckDetector?.recordProgress(task.id); const resumePrompt = [ "Your conversation was compacted because you were looping without making progress.", "Review the current state of the worktree carefully:", "1. Check `git log --oneline` to see what's already been committed", "2. Read the files you were working on to understand current state", "3. Review the PROMPT.md steps to see which are still pending", "", "Take a DIFFERENT approach from what you were doing before.", "If the current step is complete, call fn_task_update to mark it done and move to the next step.", "If you're stuck on a problem, try a simpler or alternative solution.", "", "Continue the task from where you left off.", ].join("\n"); await promptWithFallback(session, resumePrompt); checkSessionError(session); await this.persistTokenUsage(task.id, session); } // If dependency was added during execution, discard worktree and move to triage if (this.depAborted.has(task.id)) { this.depAborted.delete(task.id); await this.handleDepAbortCleanup(task.id, worktreePath); return; } // If paused during execution, move to todo so the scheduler can resume // after unpause. This path fires when session.dispose() causes the // prompt to resolve gracefully instead of throwing. if (this.pausedAborted.has(task.id)) { if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); return; } if (await this.parkApprovalSuspension(task.id, "agent session")) { wasPaused = true; return; } this.clearPausedAborted(task.id); wasPaused = true; const finalizationDecision = await this.getCompletedTaskFinalizationDecision(task.id, taskDone); if (finalizationDecision === "finalize") { if (await this.shouldDeferCompletionForGlobalPause(task.id, "paused after completion")) { return; } executorLog.log(`${task.id} paused after completion (graceful session exit) — finalizing to in-review`); await this.store.logEntry(task.id, "Execution paused after completion — finalizing to in-review"); await this.persistTokenUsage(task.id); /* FNXC:WorkflowLifecycle 2026-06-17-23:33: FN-6625: the completed/no-commit handoff may dispose graph execution after the task is already in-review. Mark that abort as completion-finalize so a trailing FN-6614-style graph failure resolves benignly instead of looking like a user/global pause; FN-6568 uses the same provenance seam for merge aborts. FNXC:WorkflowLifecycle 2026-06-18-10:58: FN-6644/FN-6641: the graceful-session-exit handoff must also record durable completed-finalize state because a later teardown can re-mark the abort as `hard-cancel`. The classifier uses that durable handoff marker, not the volatile provenance alone, to keep completed no-commit tasks from being re-parked failed. */ this.markCompletionFinalized(task.id); reportImplementationExit?.("review-handoff-paused-after-completion"); await this.handoffTaskToReview(task, "paused-after-completion"); this.clearCompletedTaskWatchdog(task.id); this.signalTaskComplete(task); } else if (finalizationDecision === "blocked") { await this.persistTokenUsage(task.id); return; } else { executorLog.log(`${task.id} paused (graceful session exit) — moving to todo`); await this.store.logEntry(task.id, "Execution paused — session preserved for resume, moved to todo"); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: true }); } return; } // If the stuck task detector disposed the session and the agent exited // cleanly, stop here. The requeue is deferred to the finally block // (after this.executing is cleared) to prevent a race where the // scheduler re-dispatches while the old execution guard is still set. if (this.stuckAborted.has(task.id)) { if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); return; } stuckRequeue = this.stuckAborted.get(task.id) ?? true; this.stuckAborted.delete(task.id); executorLog.log(`${task.id} terminated by stuck task detector (graceful session exit)`); return; } // If the agent didn't explicitly call fn_task_done, check whether // all steps are already complete — treat as implicit done to avoid // unnecessary retry sessions for context-overflow / compaction cases. if (!taskDone) { const implicitCheck = await this.store.getTask(task.id); if (implicitCheck.steps.length > 0 && implicitCheck.steps.every((s) => s.status === "done" || s.status === "skipped")) { // Implicit and explicit paths share the same structural pending-review and bulk-step-completion guards. const refusal = this.evaluateImplicitCompletionRefusal(implicitCheck, codeReviewVerdicts); if (!refusal.ok) { await this.handleImplicitTaskDoneRefusal(implicitCheck, refusal); return; } taskDone = true; executorLog.log(`${task.id} all steps done — treating as implicit fn_task_done`); await this.store.logEntry(task.id, "All steps complete — implicit fn_task_done (agent did not call tool explicitly)", undefined, this.getRunContextFor(task.id)); this.scheduleCompletedTaskWatchdog(task.id, "implicit fn_task_done"); } } if (taskDone) { // Capture modified files before running workflow steps const updatedTask = await this.store.getTask(task.id); const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha, task.id, audit, "workflow-fanout"); if (modifiedFiles.length > 0) { await this.store.updateTask(task.id, { modifiedFiles }); executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`); } // Graph-driven completion (interpreter cutover): the workflow graph // owns workflow steps, review handoff, and merge from here — stop // at the implementation-complete boundary and hand control back. this.clearCompletedTaskWatchdog(task.id); executorLog.log(`✓ ${task.id} implementation complete — graph interpreter owns the remaining lifecycle`); reportImplementationExit?.("complete"); graphCompletion({ modifiedFiles }); return; } else { let taskDoneSessionRetries = 0; let retryAbortedDueToReclaim = false; let refusalHandled = false; let pendingReviewParked = false; /* FNXC:ExecutorTaskDonePark 2026-07-15-16:10: FN-7965 — set when the row was terminally parked (status=failed) by the in-session fn_task_done refusal handler; suppresses both the retry and every post-loop completion/requeue branch so the park survives. */ let terminallyParked = false; while (!taskDone && taskDoneSessionRetries < MAX_TASK_DONE_SESSION_RETRIES) { const liveTask = await this.store.getTask(task.id); /* FNXC:ExecutorTaskDonePark 2026-07-15-16:10: FN-7965: the explicit `fn_task_done` tool handler parks the task terminally (status=failed, worktree/branch/sessionFile cleared) once the refusal retry budget is exhausted — but it runs INSIDE the agent session, so this loop never learned the row had been parked and spawned a retry session anyway. That session completed and marked the task done against a row with no worktree, so the pre-merge graph died on the first write-capable node with `no-worktree-for-write-node` and surfaced as a bogus "terminated at code-review-remediation" instead of the real refusal. Re-read state and honor the park. This deliberately does NOT reuse the FN-4806 reclaim branch below: that silently requeues to `todo`, which would clear the park and — with the refusal budget already exhausted — re-park on the next pickup, looping todo→execute→park. A terminal park is the agent's own failure and must stay parked for a human. Note the reclaim probes below cannot cover this: they test `liveTask.worktree === null`, but the store maps a cleared column to `undefined`, never `null` (`task-store/serialization.ts` — `row.worktree || undefined`). Tightening that probe is a separate change with real blast radius, so the park is detected by status here instead. */ if (liveTask.status === "failed") { const parkMessage = `${task.id}: task parked failed during no-fn_task_done retry — honoring park, not retrying`; executorLog.log(parkMessage); await this.store.logEntry(task.id, parkMessage, undefined, this.getRunContextFor(task.id)); this.deleteActiveSession(task.id); this.tokenUsageBaselines.delete(task.id); session.dispose(); terminallyParked = true; break; } const hasExplicitWorktreeBinding = typeof liveTask.worktree === "string" || liveTask.worktree === null; const hasExplicitBranchBinding = typeof liveTask.branch === "string" || liveTask.branch === null; /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): the contract holds while the card is in ITS board's wip lane; the literal made every renamed-board retry look reclaimed. */ const worktreeContractIntact = liveTask.column === (await this.resolveResumeLanes(task.id)).wip && !liveTask.paused && (!hasExplicitWorktreeBinding || liveTask.worktree === worktreePath) && (!hasExplicitBranchBinding || (typeof liveTask.branch === "string" && liveTask.branch.length > 0)); if (!worktreeContractIntact) { const reclaimMessage = `${task.id}: worktree/branch reclaimed during no-fn_task_done retry — aborting retry and requeueing`; executorLog.log(reclaimMessage); await this.store.logEntry(task.id, reclaimMessage, undefined, this.getRunContextFor(task.id)); this.deleteActiveSession(task.id); this.tokenUsageBaselines.delete(task.id); session.dispose(); retryAbortedDueToReclaim = true; break; } const pendingReviewBlock = detectPendingReviewBlock(liveTask, codeReviewVerdicts); if (pendingReviewBlock.blocked) { executorLog.log( `[executor] ${task.id}: fn_task_done not called but task is blocked on pending review (${pendingReviewBlock.reason}) — skipping retry session`, ); await this.store.logEntry( task.id, `Agent finished without calling fn_task_done but Step ${pendingReviewBlock.stepIndex} is blocked on pending review (${pendingReviewBlock.reason}) — skipping retry session`, undefined, this.getRunContextFor(task.id), ); this.deleteActiveSession(task.id); this.tokenUsageBaselines.delete(task.id); session.dispose(); await this.persistTokenUsage(task.id); // A pending-review block is not an execution failure. The executor // cannot continue until the reviewer decision is resolved, so park // the task in review without setting status=failed; otherwise the // merge/review queue deadlocks on a task that is both in-review and // failed. /* FNXC:WorkflowExecutionOwnership 2026-07-29-18:50 (U8 / R4): The `handoffTaskToReview` call that stood here is GONE — the graph performs it via the `review-pending-handoff` node the live primitive now routes to. What remains is a report and a stop, which is all an implementation phase should do. Why review and not `failed` (a pending-review block is a wait; status=failed on an in-review row deadlocks the merge queue) now lives with the node in the IR, where the routing decision is. */ reportImplementationExit?.("review-handoff-pending-review"); pendingReviewParked = true; break; } taskDoneSessionRetries++; executorLog.log( `⚠ ${task.id} finished without fn_task_done — retrying with new session (${taskDoneSessionRetries}/${MAX_TASK_DONE_SESSION_RETRIES})`, ); await this.store.logEntry( task.id, `Agent finished without calling fn_task_done — retrying with new session (${taskDoneSessionRetries}/${MAX_TASK_DONE_SESSION_RETRIES})`, undefined, this.getRunContextFor(task.id), ); // Capture and analyse the previous session's text before resetting. const previousSessionText = lastAssistantText; const pseudoPause = detectPseudoPause(previousSessionText); if (pseudoPause.kind !== "none") { const shortMatch = (pseudoPause.matched ?? "").slice(0, 120); await this.store.logEntry( task.id, `Pseudo-pause detected (kind=${pseudoPause.kind}, matched='${shortMatch}')`, undefined, this.getRunContextFor(task.id), ); executorLog.log(`${task.id} pseudo-pause detected (kind=${pseudoPause.kind}): ${shortMatch}`); } // Dispose old session and create a fresh one. // Reset lastAssistantText so the new session's text is tracked cleanly. lastAssistantText = ""; this.deleteActiveSession(task.id); this.tokenUsageBaselines.delete(task.id); session.dispose(); let retrySession: AgentSession | null = null; try { const createdRetrySession = await createResolvedAgentSession({ sessionPurpose: "executor", runtimeHint: executorRuntimeHint, pluginRunner: this.options.pluginRunner, cwd: worktreePath, systemPrompt: executorSystemPromptFinal, systemPromptLayers: executorLayers, tools: "coding", customTools, onText: agentLogger.onText, onThinking: agentLogger.onThinking, onToolStart: agentLogger.onToolStart, onToolEnd: agentLogger.onToolEnd, defaultProvider: executorProvider, defaultModelId: executorModelId, ...(executorSessionModel.credentialInstanceId ? { credentialInstanceId: executorSessionModel.credentialInstanceId } : {}), fallbackProvider: executorFallbackProvider, fallbackModelId: executorFallbackModelId, fallbackThinkingLevel: executorFallbackThinkingLevel, defaultThinkingLevel: executorThinkingLevel, runAuditor: audit, settings, sessionManager: SessionManager.create(worktreePath), taskEnv, mcpServers: await this.resolveMcpServers(identityAgent?.id), // FNXC:PluginSkills 2026-07-12-00:00: Retry executor sessions must keep the same plugin skill body discovery paths as the primary attempt so requested plugin skill names resolve to real bodies. ...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}), ...(skillContext.additionalSkillPaths.length > 0 ? { additionalSkillPaths: skillContext.additionalSkillPaths } : {}), // U5 (R5): retry session re-keys gating to the effective principal, // mirroring the primary execute-seam session above. actionGateContext: this.buildActionGateContext(task.id, identityAgent, settings.defaultAgentPermissionPolicy), permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, identityAgent, settings.defaultAgentPermissionPolicy), // FNXC:SessionRouting 2026-06-24-11:20: // #1675: propagate task id so retry-session requests carry the same // X-Session-Id/X-Session-Affinity as the primary session, keeping the // task's LLM requests grouped under one stable routing/observability id. taskId: task.id, }); retrySession = createdRetrySession.session; await this.captureExecutorTokenUsageBaseline(task.id, retrySession); captureSessionTokenBaseline(retrySession); if (createdRetrySession.sessionFile) { this.store.updateTask(task.id, { sessionFile: createdRetrySession.sessionFile }).catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${task.id} failed to persist retry sessionFile: ${msg}`); }); } session = retrySession; sessionRef.current = retrySession; this.setActiveSession(task.id, { session: retrySession, seenSteeringIds, lastResolvedModelProvider: executorProvider, lastResolvedModelId: executorModelId, lastTaskModelProvider: detail.modelProvider, lastTaskModelId: detail.modelId, lastAssignedAgentId: detail.assignedAgentId ?? null, // U5 (R7): preserve the effective column-agent across the retry. lastEffectiveColumnAgentId: columnAgentSeam?.agent.id ?? null, }, worktreePath); stuckDetector?.trackTask(task.id, retrySession); const retryCustomFieldDefs = await this.resolveTaskCustomFieldDefs(task.id); const retryPluginTaskContributions = await buildPluginPromptSection("executor-task", this.options.pluginRunner); let retryPrompt: string; if (pseudoPause.kind !== "none") { const shortMatch = (pseudoPause.matched ?? "").slice(0, 120); retryPrompt = [ `Your previous turn ended with a pseudo-pause: "${shortMatch}". This is forbidden.`, "", "Turn-ending rules you violated:", "- You MUST NOT end a turn by asking the user a question, summarizing progress, or requesting permission to continue.", "- Phrases like 'If you want, I can continue', 'Should I proceed?', 'Let me know if...' are FORBIDDEN turn-endings.", "- The user is not watching this conversation. Questions written as prose are ignored.", "- If you genuinely cannot proceed, call fn_task_done with a clear explanation — never write the blocker as plain prose.", "", "What you must do now:", "1. Review the PROMPT.md steps and identify the next pending step.", "2. Do the work for that step immediately — call fn_task_update, write code, run tests.", "3. Continue until all steps are done, then call fn_task_done.", "Do NOT ask for permission. Do NOT write a summary. Just call a tool and keep working.", "", "Original task:", buildExecutionPrompt( detail, this.rootDir, settings, worktreePath, this.options.pluginRunner, retryCustomFieldDefs, this.workspaceConfig, { pluginTaskContributions: retryPluginTaskContributions, }, ), ].join("\n"); } else { retryPrompt = [ "Your previous session ended without calling the fn_task_done tool.", "The task may already be complete — review the current state of the worktree and either:", "1. If the work is done, call fn_task_done with a summary of what was accomplished.", "2. If there is remaining work, finish it and then call fn_task_done.", "", "Original task:", buildExecutionPrompt( detail, this.rootDir, settings, worktreePath, this.options.pluginRunner, retryCustomFieldDefs, this.workspaceConfig, { pluginTaskContributions: retryPluginTaskContributions, }, ), ].join("\n"); } stuckDetector?.recordActivity(task.id); await promptWithFallback(retrySession, retryPrompt); checkSessionError(retrySession); await this.persistTokenUsage(task.id, retrySession); } catch (retryError) { this.deleteActiveSession(task.id); this.tokenUsageBaselines.delete(task.id); retrySession?.dispose(); if (await this.recoverMissingWorktreeSessionStartFailure(task, worktreePath, retryError, audit)) { return; } throw retryError; } if (!taskDone) { const implicitCheck = await this.store.getTask(task.id); if (implicitCheck.steps.length > 0 && implicitCheck.steps.every((s) => s.status === "done" || s.status === "skipped")) { // Implicit and explicit paths share the same structural pending-review and bulk-step-completion guards. const refusal = this.evaluateImplicitCompletionRefusal(implicitCheck, codeReviewVerdicts); if (!refusal.ok) { await this.handleImplicitTaskDoneRefusal(implicitCheck, refusal); retrySession?.dispose(); retrySession = null; retryAbortedDueToReclaim = false; refusalHandled = true; break; } taskDone = true; executorLog.log(`${task.id} all steps done — treating as implicit fn_task_done`); await this.store.logEntry(task.id, "All steps complete — implicit fn_task_done (agent did not call tool explicitly)", undefined, this.getRunContextFor(task.id)); this.scheduleCompletedTaskWatchdog(task.id, "implicit fn_task_done"); } } } if (taskDone) { const updatedTask = await this.store.getTask(task.id); const modifiedFiles = await this.captureModifiedFiles(worktreePath, updatedTask.baseCommitSha, task.id, audit, "no-task-done-retry"); if (modifiedFiles.length > 0) { await this.store.updateTask(task.id, { modifiedFiles }); executorLog.log(`${task.id}: captured ${modifiedFiles.length} modified files`); } this.scheduleCompletedTaskWatchdog(task.id, "task completion retry"); if (await this.shouldDeferCompletionForGlobalPause(task.id, "before in-review transition after task completion retry")) { return; } // FNXC:WorkflowExecution 2026-06-25-00:00: U4 (KTD-2/KTD-5) — workflow // gates are graph-owned (record into task.workflowStepResults, U2); the // legacy runWorkflowSteps loop was deleted. For a graph-driven run the // execute seam registered a completion interceptor, so stop at the // implementation boundary and let the graph own the remaining // lifecycle. A non-graph fallback reaching here has NO enabled workflow // steps (a minimal store WITH enabled steps is parked fail-closed in // executeWorkflowGraph, KTD-5) — nothing to gate before handoff. this.clearCompletedTaskWatchdog(task.id); executorLog.log(`✓ ${task.id} implementation complete (retry) — graph interpreter owns the remaining lifecycle`); reportImplementationExit?.("complete-after-retry"); graphCompletion({ modifiedFiles }); return; } else if (terminallyParked) { // FN-7965: the in-session refusal handler already wrote the terminal failure and cleared // the binding. Nothing further to do — requeueing or handing off to review here is exactly // the resurrection that stranded the pre-merge graph. await this.persistTokenUsage(task.id); return; } else if (retryAbortedDueToReclaim) { // FN-4806: Worktree/branch was reclaimed mid-retry by an engine-side housekeeping path // (e.g. FN-4546 stale-active-branch reclaim, FN-4742 self-healing removals). This is NOT // an agent failure — the agent never got a fair retry attempt. Silently requeue to todo // with preserved progress so a fresh worktree is created on next pickup. Do not mark // status=failed, do not surface onError, do not burn taskDoneRetryCount budget. const silentMessage = `${task.id}: worktree/branch reclaimed mid-retry — requeued to todo (engine self-heal, no failure)`; await this.store.logEntry( task.id, "Worktree/branch reclaimed mid-retry — requeued to todo (engine self-heal, no failure)", undefined, this.getRunContextFor(task.id), ); // Clear any stale binding so the next pickup creates a fresh worktree. // baseCommitSha is also cleared because it pinned to the now-reclaimed worktree; // the next pickup will re-anchor it on the fresh checkout. await this.store.updateTask(task.id, { worktree: null, branch: null, baseCommitSha: null }); await this.persistTokenUsage(task.id); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true }); executorLog.log(silentMessage); } else if (refusalHandled) { return; } else if (pendingReviewParked) { return; } else { // FN-4806: Genuine "agent finished without calling fn_task_done after N retries" // exhaustion. Not a reclaim/self-heal — the agent had a fair chance and failed to // signal completion. Mark failed, surface onError, and either requeue (budget // remaining) or escalate to in-review (budget exhausted). const priorRequeues = task.taskDoneRetryCount ?? 0; const nextRequeueCount = priorRequeues + 1; const errorMessage = `Agent finished without calling fn_task_done (after ${MAX_TASK_DONE_SESSION_RETRIES} retries)`; if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) { await this.store.updateTask(task.id, { status: "queued", error: null, taskDoneRetryCount: nextRequeueCount, }); await this.store.logEntry( task.id, `${errorMessage} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`, undefined, this.getRunContextFor(task.id), ); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true }); executorLog.log(`✗ ${task.id} failed after ${MAX_TASK_DONE_SESSION_RETRIES} retries — requeued to todo (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`); } else { await this.store.updateTask(task.id, { status: "failed", error: errorMessage }); await this.store.logEntry(task.id, `${errorMessage} — execution failed after task-done retry budget was exhausted`, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); executorLog.log(`✗ ${task.id} failed after ${MAX_TASK_DONE_SESSION_RETRIES} retries — no fn_task_done`); } this.options.onError?.(task, new Error(errorMessage)); } } } finally { clearInterval(verificationRequestTimer); if (leaseRenewalTimer) { clearInterval(leaseRenewalTimer); } this.deleteActiveSession(task.id); stuckDetector?.untrackTask(task.id); await agentLogger.flush(); await this.persistTokenUsage(task.id, session).catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${task.id}: failed to persist final single-session token usage before dispose: ${msg}`); }); this.tokenUsageBaselines.delete(task.id); resetSessionTokenBaseline(session); session.dispose(); // Terminate all spawned child agents when parent session ends await this.terminateAllChildren(task.id); // Clear session file when task completes or fails (not when paused — // the file is preserved so unpause can resume the conversation). // Check both the local flag (graceful exit) and the instance set // (error path where dispose caused prompt to throw). if (!wasPaused && !this.pausedAborted.has(task.id)) { this.store.updateTask(task.id, { sessionFile: null }).catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${task.id} failed to clear sessionFile: ${msg}`); }); } // Invoke plugin onAgentRunEnd hook (fire-and-forget) void this.options.pluginRunner?.invokeHookSafe("onAgentRunEnd", task.id); } }; const retryableWork = () => withRateLimitRetry(agentWork, { signal: this.activeWorkflowGraphAbortControllers.get(task.id)?.signal, rotation: this.options.credentialRotator ? { providerId: activeAgentInstanceRef?.providerId ?? detail.modelProvider ?? "", nextInstance: async () => { /* FNXC:CredentialInstanceRotation 2026-08-01-11:05: Executor agent runs rotate only after the shared retry helper classifies a usage limit. Live task/settings reads and the executor pause-abort marker bail before opening an event, because a pause arriving mid-run cannot authorize changing the billed credential. A successful offer causes agentWork to construct a fresh session; a non-limit failure intentionally leaves its attempt without an outcome row. */ const [liveTask, liveSettings] = await Promise.all([ this.store.getTask(task.id).catch(() => undefined), this.store.getSettings().catch(() => settings), ]); if (agentRotationDeclined || this.pausedAborted.has(task.id) || !liveTask || liveTask.userPaused === true || liveTask.autoMerge === false || liveSettings.globalPause === true || liveSettings.enginePaused === true || !activeAgentInstanceRef?.providerId) return undefined; agentRotationEvent ??= await this.options.credentialRotator!.beginEvent({ providerId: activeAgentInstanceRef.providerId, startingInstanceId: activeAgentInstanceRef.instanceId, lane: "executor-agent", taskId: task.id, }); if (!agentRotationEvent) { agentRotationDeclined = true; return undefined; } // FNXC:CredentialInstanceRotation 2026-08-01-11:34: Inventory lookup is asynchronous; re-check human control before this retry marks a credential limited or offers another billed account. const [postInventoryTask, postInventorySettings] = await Promise.all([ this.store.getTask(task.id).catch(() => undefined), this.store.getSettings().catch(() => settings), ]); if (this.pausedAborted.has(task.id) || !postInventoryTask || postInventoryTask.userPaused === true || postInventoryTask.autoMerge === false || postInventorySettings.globalPause === true || postInventorySettings.enginePaused === true) return undefined; this.options.credentialRotator!.markLimited(activeAgentInstanceRef); if (agentDispatchedRotation) agentRotationEvent.recordOutcome("rotation-failed-limit"); const next = await agentRotationEvent.next(); if (!next) { agentRotationEvent.finishExhausted(); return undefined; } activeAgentInstanceRef = next; agentDispatchedRotation = true; return next; }, } : undefined, onRetry: (attempt, delayMs, error) => { const delaySec = Math.round(delayMs / 1000); executorLog.warn(`⏳ ${task.id} rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`); this.store.logEntry(task.id, `Rate limited — retry ${attempt} in ${delaySec}s`, undefined, this.getRunContextFor(task.id)).catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${task.id} failed to log rate-limit retry: ${msg}`); }); }, }); await this.runWithExecutorSemaphore(task.id, retryableWork); if (agentDispatchedRotation) agentRotationEvent?.recordOutcome("rotation-succeeded"); } catch (err: unknown) { const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err); if (this.depAborted.has(task.id)) { // Dependency added mid-execution — discard worktree and move to triage this.depAborted.delete(task.id); await this.handleDepAbortCleanup(task.id, worktreePath); } else if (isInvalidAssistantContinuationErrorMessage(errorMessage)) { /* FNXC:PostDoneContinuation 2026-07-16-11:57: FN-8111 requires a completed task to win over stale-transcript retry handling. An assistant-last error after the task already reached in-review must signal completion and clear the watchdog rather than create a deferred retry that never dispatches. */ if (await this.handleNonContinuableSessionError(task, taskDone, errorMessage)) { return; } /* FNXC:ExecutorSessionRecovery 2026-07-14-06:03: A stale assistant-last transcript gets a bounded fresh-session retry with the shared recovery backoff. The retry counter must survive the deferred move so repeated fresh-session failures eventually become a visible execution failure instead of cycling through Todo forever. FNXC:ExecutorSessionRecovery 2026-07-14-06:19: Deferred self-requeues must mark the workflow graph recovery and release the active worktree slot after the executor lock drops; otherwise graph failure cleanup can overwrite the recovery and the parked task can keep consuming maxWorktrees capacity. */ const liveTask = await this.store.getTask(task.id); const decision = computeRecoveryDecision({ recoveryRetryCount: liveTask.recoveryRetryCount, nextRecoveryAt: liveTask.nextRecoveryAt, }); if (!decision.shouldRetry) { executorLog.error(`✗ ${task.id} stale assistant-continuation retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`); await this.store.logEntry( task.id, `Stale assistant-continuation fresh-session retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, errorStack ?? errorDetail, this.getRunContextFor(task.id), ); await this.store.updateTask(task.id, { status: "failed", error: errorMessage, recoveryRetryCount: null, nextRecoveryAt: null, }); await this.persistTokenUsage(task.id); this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage)); return; } staleAssistantContinuationRequeue = true; const attempt = decision.nextState.recoveryRetryCount; const delay = formatDelay(decision.delayMs); executorLog.warn(`${task.id} stale assistant-continuation session detected — fresh-session retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay} after executor lock release`); await this.store.logEntry( task.id, `Detected stale assistant-continuation session — fresh-session retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay} with progress preserved: ${errorMessage}`, undefined, this.getRunContextFor(task.id), ); await this.store.updateTask(task.id, { sessionFile: null, recoveryRetryCount: decision.nextState.recoveryRetryCount, nextRecoveryAt: decision.nextState.nextRecoveryAt, }); return; } else if (errorMessage.includes("Invalid transition")) { // Task was moved by user/process while executor was running — already in desired state // This check must come before pausedAborted since it's more specific const transitionMatch = errorMessage.match(/Invalid transition: '([^']+)' → '([^']+)'/); const fromColumn = transitionMatch?.[1] ?? "unknown"; const toColumn = transitionMatch?.[2] ?? "unknown"; const logMessage = `Task already moved from '${fromColumn}' — skipping transition to '${toColumn}'`; executorLog.log(`${task.id} ${logMessage}`); await this.store.logEntry(task.id, logMessage, errorMessage, this.getRunContextFor(task.id)); /* FNXC:WorkflowResolvedColumns 2026-07-31-09:25 (fleet: executor lifecycle roles): `fromColumn`/`toColumn` are parsed out of the store's rejection message, so they carry whatever ids that workflow declares. Comparing them to the literal `in-review` meant a renamed review lane never matched and the duplicate-handoff finalize never ran, leaving the card mid-transition with nothing to complete it. Resolve the task's own review role; an unresolvable workflow keeps the legacy literal, so behaviour is unchanged wherever the vocabulary cannot be read. */ const reviewLane = (await resolveTaskLifecycleColumns(this.store, task.id).catch(() => undefined))?.review ?? "in-review"; if (fromColumn === reviewLane && toColumn === reviewLane) { try { const finalizeResult = await this.finalizeAlreadyReviewedTask(task.id); executorLog.debug(`${task.id} duplicate in-review finalization result: ${finalizeResult}`); } catch (finalizeErr: unknown) { const finalizeErrMessage = finalizeErr instanceof Error ? finalizeErr.message : String(finalizeErr); executorLog.warn(`${task.id} failed to finalize duplicate in-review transition: ${finalizeErrMessage}`); } } // Task finished successfully (just already moved), so call onComplete this.signalTaskComplete(task); } else if (this.pausedAborted.has(task.id)) { // Task was paused mid-execution — clean up worktree and move to todo if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); return; } if (await this.parkApprovalSuspension(task.id, "executor session")) return; this.clearPausedAborted(task.id); const latestTask = await this.store.getTask(task.id); if ( /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): the HOLD lane — this recognises a card the abort already parked with its progress preserved, and skipping the cleanup is what keeps that progress. On a renamed board the cleanup ran anyway and discarded it. */ latestTask?.column === (await this.resolveResumeLanes(task.id)).hold && latestTask.paused === true && ((latestTask.currentStep ?? 0) > 0 || latestTask.steps?.some((step) => step.status === "done" || step.status === "in-progress")) ) { executorLog.debug(`${task.id} paused-abort cleanup skipped — incomplete task is already parked with progress preserved`); await this.store.logEntry( task.id, "Execution abort cleanup skipped — incomplete stuck-loop task is already parked with progress preserved", undefined, this.getRunContextFor(task.id), ); return; } const finalizationDecision = await this.getCompletedTaskFinalizationDecision(task.id, taskDone); if (finalizationDecision === "finalize") { if (await this.shouldDeferCompletionForGlobalPause(task.id, "paused after completion")) { return; } executorLog.log(`${task.id} paused after completion — finalizing to in-review`); await this.store.logEntry(task.id, "Execution paused after completion — finalizing to in-review", undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); /* FNXC:WorkflowLifecycle 2026-06-17-23:33: FN-6625: the completed/no-commit handoff may dispose graph execution after the task is already in-review. Mark that abort as completion-finalize so a trailing FN-6614-style graph failure resolves benignly instead of looking like a user/global pause; FN-6568 uses the same provenance seam for merge aborts. FNXC:WorkflowLifecycle 2026-06-18-10:59: FN-6644/FN-6641: the finally-block handoff must record durable completed-finalize state because a later teardown can overwrite provenance to `hard-cancel`. The classifier must still resolve that completed no-commit tail failure benignly without weakening genuine pause or active hard-cancel behavior. */ this.markCompletionFinalized(task.id); reportImplementationExit?.("review-handoff-paused-after-completion"); await this.handoffTaskToReview(task, "paused-after-completion"); this.signalTaskComplete(task); } else if (finalizationDecision === "blocked") { await this.persistTokenUsage(task.id); return; } else { executorLog.log(`${task.id} paused — moving to todo`); if (worktreePath && existsSync(worktreePath)) { try { const settings = await this.store.getSettings(); await removeWorktree({ worktreePath, rootDir: this.rootDir, settings, taskId: task.id, audit, reason: RemovalReason.ExecutorDispose, expectedOwnerTaskId: task.id, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); executorLog.log(`Removed old worktree for paused task: ${worktreePath}`); } catch (cleanupErr: unknown) { const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr); executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`); } } // FNXC:WorkflowLifecycle 2026-06-21-00:00: FN-6722 — a mid-run abort on // a task that already has real step progress must not discard that // progress on the bounce to todo. The sibling pause-park path moves // with preserveResumeState; // this teardown branch historically did not — it cleared `branch` AND // moved without preservation, which reset every step to pending // (store.moveTaskInternal ~7322 resetAllStepsToPending) and dropped the // pointer to the commits already on the task branch. The next dispatch // then re-planned from Step 0 even though the work was committed on the // branch — observably a "lost all progress / stuck" failure. Preserve the // branch + resume state when there is resumable progress so execute() // resumes onto the existing branch (the `acquisition.isResume && // task.branch` reconciliation ~7679) from the first incomplete step. The // worktree is still removed above and its binding cleared below to free // the concurrency slot (FN-6782) — only the durable pointers (branch + // step state) are kept. The 9227 guard above covers the same intent but // is race-contingent on the move having already landed; this makes the // fall-through path safe regardless. // // Read progress from `latestTask` (the store snapshot fetched at ~9226), // NOT the `task` parameter: `task` is frozen at dispatch time and never // mutated mid-run, so a fresh task (currentStep 0, all steps pending at // dispatch) whose agent committed step progress to the store during this // session would otherwise look progress-less here and hit the destructive // reset — the exact FN-6722 failure mode. Fall back to `task` when the // store read came back empty. const progressSource = latestTask ?? task; const hasResumableProgress = (progressSource.currentStep ?? 0) > 0 || (progressSource.steps?.some((step) => step.status === "done" || step.status === "in-progress") ?? false); /* FNXC:WorkflowLifecycle 2026-07-12-09:05: Pause-bounce loop (observed on FN-7851): this teardown runs BECAUSE the user paused the task, but the plain move-to-todo below wiped the pause flags (store reopen block), leaving an unpaused dispatchable todo row. The graph-failure classifier then read `paused=false, userPaused=false`, misclassified the abort as engine-internal, and auto-continued the session; once the shared graphResumeRetryCount budget was exhausted the scheduler simply re-dispatched the row seconds later — so pausing an in-progress task could never stick. When the pause that caused this abort is still in force at teardown time, move with `preservePause` so the row lands in todo still parked (`paused` kept; scheduler skips paused/userPaused todo rows) and the classifier sees the pause and routes benignly. An unpause during the teardown window leaves `paused` unset and restores the old requeue-for-normal-scheduling behavior. */ const pauseStillInForce = latestTask?.paused === true; await this.store.updateTask( task.id, hasResumableProgress ? { worktree: undefined } : { worktree: undefined, branch: undefined }, ); await this.store.logEntry( task.id, pauseStillInForce ? "Execution paused — agent terminated, parked in todo (pause preserved, awaiting explicit unpause)" : "Execution paused — agent terminated, moved to todo", undefined, this.getRunContextFor(task.id), ); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { ...(hasResumableProgress ? { preserveResumeState: true } : {}), ...(pauseStillInForce ? { preservePause: true } : {}), }); } } else if (this.stuckAborted.has(task.id)) { // Task was killed by stuck task detector — defer requeue to finally block // (after this.executing is cleared) to prevent re-dispatch race. if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); return; } stuckRequeue = this.stuckAborted.get(task.id) ?? true; this.stuckAborted.delete(task.id); executorLog.log(`${task.id} terminated by stuck task detector — will ${stuckRequeue ? "retry" : "not retry (budget exhausted)"}`); } else { // Context-limit error reached the executor after promptWithFallback's auto-compaction // already attempted to recover. Recovery strategy (in order): // 1. Reduced-prompt retry in the same session (up to MAX_REDUCED_PROMPT_ATTEMPTS) // 2. Fresh-session requeue — terminate the saturated session and move the task // back to "todo" so the next dispatch gets a clean session (bounded by // recoveryRetryCount / MAX_RECOVERY_RETRIES). // FN-2182 class: Step 7 overflow after earlier compaction used to hit the // loopAttempts<1 guard and fail permanently; the requeue path below recovers // by restarting with a fresh session against the already-written step output. const MAX_REDUCED_PROMPT_ATTEMPTS = 3; const loopState = this.loopRecoveryState.get(task.id); const loopAttempts = loopState?.attempts ?? 0; const isContextError = isContextLimitError(errorMessage); if (isContextError && loopAttempts < MAX_REDUCED_PROMPT_ATTEMPTS) { const activeEntry = this.activeSessions.get(task.id); if (activeEntry) { executorLog.log(`${task.id} context limit error after auto-compaction — attempting reduced-prompt retry (${loopAttempts + 1}/${MAX_REDUCED_PROMPT_ATTEMPTS})`); await this.store.logEntry(task.id, `Context limit error after auto-compaction — attempting reduced-prompt retry (${loopAttempts + 1}/${MAX_REDUCED_PROMPT_ATTEMPTS}): ${errorMessage}`, undefined, this.getRunContextFor(task.id)); this.loopRecoveryState.set(task.id, { attempts: loopAttempts + 1, pending: false }); try { this.options.stuckTaskDetector?.recordProgress(task.id); // Build a reduced prompt that's simpler and shorter to avoid context overflow const reducedPrompt = [ "Your previous attempt hit the context window limit.", "Focus on completing the task efficiently with minimal context:", "1. Review git status and git log to see what's been done", "2. Identify the most critical remaining work", "3. Complete it with a simpler, more focused approach", "", "Do not repeat what's already been done. Just complete the task and call fn_task_done.", ].join("\n"); await promptWithFallback(activeEntry.session, reducedPrompt); checkSessionError(activeEntry.session); await this.persistTokenUsage(task.id, activeEntry.session); // Reduced-prompt retry succeeded — return to let the finally block clean up // without marking the task as failed. executorLog.log(`${task.id} reduced-prompt recovery succeeded — continuing`); await this.store.logEntry(task.id, "Reduced-prompt recovery succeeded — continuing execution", undefined, this.getRunContextFor(task.id)); return; } catch (reducedErr: unknown) { const reducedErrorMessage = reducedErr instanceof Error ? reducedErr.message : String(reducedErr); if (!isContextLimitError(reducedErrorMessage)) { executorLog.error(`${task.id} reduced-prompt recovery also failed: ${reducedErrorMessage}`); await this.store.logEntry(task.id, `Reduced-prompt recovery failed: ${reducedErrorMessage}`, undefined, this.getRunContextFor(task.id)); // Non-context failure — fall through to mark task as failed } else { // Still a context error — the session is saturated beyond recovery. // Fall through to the fresh-session requeue path below. executorLog.warn(`${task.id} session still saturated after reduced-prompt retry — will attempt fresh-session requeue`); await this.store.logEntry(task.id, `Reduced-prompt retry still over context — will attempt fresh-session requeue`, undefined, this.getRunContextFor(task.id)); } } } } // Fresh-session requeue for context-limit errors: the saturated session // cannot be salvaged, but the task's git state is intact. Move the task // back to todo so the next scheduling pass creates a new session. if (isContextError) { const decision = computeRecoveryDecision({ recoveryRetryCount: task.recoveryRetryCount, nextRecoveryAt: task.nextRecoveryAt, }); if (decision.shouldRetry) { const attempt = decision.nextState.recoveryRetryCount; const delay = formatDelay(decision.delayMs); executorLog.warn(`⚡ ${task.id} context-overflow fresh-session requeue ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}`); await this.store.logEntry(task.id, `Context-overflow fresh-session requeue (${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id)); // Retain the worktree and accumulated step progress so the fresh // session resumes where the saturated one left off, but clear // sessionFile synchronously here so the next dispatch is forced // to spawn a brand-new session instead of reopening the // over-context one. The session-end finally block also clears // sessionFile, but it runs as fire-and-forget — if moveTask // wins the task lock first, the next executor pass would // observe a stale sessionFile and resume into the saturated // session, looping on the same context-limit failure. await this.store.updateTask(task.id, { recoveryRetryCount: decision.nextState.recoveryRetryCount, nextRecoveryAt: decision.nextState.nextRecoveryAt, sessionFile: null, }); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: true }); return; } executorLog.error(`✗ ${task.id} context-overflow requeue budget exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorMessage}`); await this.store.logEntry(task.id, `Context-overflow requeues exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, undefined, this.getRunContextFor(task.id)); // Reset so downstream failure path can persist cleanly await this.store.updateTask(task.id, { recoveryRetryCount: null, nextRecoveryAt: null, }); // Fall through to terminal failure marking // Contamination recovery lives in executor because branch cross-contamination // is surfaced here from task execution preflight; merger empty-cherry-pick // handling does not throw BranchCrossContaminationError in its own path. } else if (err instanceof BranchCrossContaminationError) { const details = err.foreignCommits .map((commit) => `${commit.sha.slice(0, 12)}:${commit.foreignTaskId}`) .join(", "); await this.store.logEntry(task.id, `[recovery] branch cross-contamination detected on ${err.branchName} since ${err.baseSha}: ${details}`, undefined, this.getRunContextFor(task.id)); try { const recoveredBootstrapMisbinding = await this.tryBootstrapMisbindingRecovery(task, err, audit); if (recoveredBootstrapMisbinding) { return; } const classified = await classifyForeignCommits({ repoDir: this.rootDir, branchName: err.branchName, baseSha: err.baseSha, foreignCommits: err.foreignCommits, }); const misrouted: Array<{ commit: (typeof classified.unique)[number]; foreignTaskId: string; paths: string[] }> = []; const preOrphanUnique: typeof classified.unique = []; for (const commit of classified.unique) { const misroutedResult = await classifyMisroutedForeignCommit({ repoDir: this.rootDir, sha: commit.sha, commitSubject: commit.subject, commitBody: await execAsync(`git log -1 --format=%b ${commit.sha}`, { cwd: this.rootDir, encoding: "utf-8" }).then((r) => r.stdout).catch(() => ""), currentTaskId: task.id, }); if (misroutedResult.misrouted && misroutedResult.foreignTaskId) { misrouted.push({ commit, foreignTaskId: misroutedResult.foreignTaskId, paths: misroutedResult.paths ?? [] }); } else { preOrphanUnique.push(commit); } } // Orphan-our-advance: a "unique" foreign commit attributed to a // task that's already `done` is a stranded merge from the pre-FF // ref-advance bug. FF-rehomeable orphans are advanced onto the // integration branch and then dropped from this task's branch // alongside already-upstream commits. Non-FF orphans (diverged // from current integration tip) are logged with a cherry-pick // hint and left as `genuinelyUnique` for human adjudication. const rehomedOrphans: typeof classified.unique = []; const genuinelyUnique: typeof classified.unique = []; const integrationBranchForOrphan = task.mergeDetails?.mergeTargetBranch ?? task.baseBranch ?? "main"; for (const commit of preOrphanUnique) { const orphanBody = await execAsync(`git log -1 --format=%b ${commit.sha}`, { cwd: this.rootDir, encoding: "utf-8" }) .then((r) => r.stdout) .catch(() => ""); const orphanClass = await classifyOrphanOurAdvance({ repoDir: this.rootDir, taskStore: this.store, integrationBranch: integrationBranchForOrphan, currentTaskId: task.id, commitSha: commit.sha, commitSubject: commit.subject, commitBody: orphanBody, }); if (!orphanClass.orphan) { genuinelyUnique.push(commit); continue; } const rehome = await rehomeOrphanOntoIntegration({ rootDir: this.rootDir, projectRootDir: this.rootDir, integrationBranch: integrationBranchForOrphan, orphanSha: commit.sha, taskId: task.id, audit, }).catch((rehomeError: unknown): { rehomed: false; reason: string } => ({ rehomed: false, reason: rehomeError instanceof Error ? rehomeError.message : String(rehomeError), })); if (rehome.rehomed) { rehomedOrphans.push(commit); await this.store.logEntry( task.id, `[recovery] rehomed orphan-our-advance commit ${commit.sha.slice(0, 12)} (source ${orphanClass.sourceTaskId}) onto ${integrationBranchForOrphan} via fast-forward; dropping from branch`, undefined, this.getRunContextFor(task.id), ); } else { const hint = "cherryPickHint" in rehome && rehome.cherryPickHint ? ` — manual rehome: \`${rehome.cherryPickHint}\`` : ""; await this.store.logEntry( task.id, `[recovery] orphan-our-advance commit ${commit.sha.slice(0, 12)} (source ${orphanClass.sourceTaskId}) refused auto-rehome: ${rehome.reason}${hint}`, undefined, this.getRunContextFor(task.id), ); genuinelyUnique.push(commit); } } const alreadyShas = classified.alreadyUpstream.map((commit) => commit.sha.slice(0, 12)).join(", ") || "none"; const misroutedShas = misrouted.map(({ commit }) => commit.sha.slice(0, 12)).join(", ") || "none"; const rehomedShas = rehomedOrphans.map((commit) => commit.sha.slice(0, 12)).join(", ") || "none"; const uniqueShas = genuinelyUnique.map((commit) => commit.sha.slice(0, 12)).join(", ") || "none"; await this.store.logEntry( task.id, `[recovery] contamination classification: already-upstream=[${alreadyShas}] misrouted=[${misroutedShas}] rehomed-orphan=[${rehomedShas}] unique=[${uniqueShas}]`, undefined, this.getRunContextFor(task.id), ); const alreadyAttemptedRecovery = (task.recoveryRetryCount ?? 0) > 0; if (genuinelyUnique.length === 0 && !alreadyAttemptedRecovery) { // Run the recovery inside the worktree (when one exists) so the final // `git checkout ` step doesn't collide with the worktree's own // checkout. If we operate from this.rootDir while the branch is checked // out in a worktree, git refuses the recheckout with // "branch already used by worktree" and the in-line happy path silently // fails — every contaminated task would then fall through to the // dispatcher pause path even when it could have auto-recovered. const recoveryRepoDir = task.worktree ?? this.rootDir; const recovery = await autoRecoverCrossContamination({ repoDir: recoveryRepoDir, branchName: err.branchName, baseSha: err.baseSha, taskId: task.id, shasToDrop: [ ...classified.alreadyUpstream.map((commit) => commit.sha), ...misrouted.map(({ commit }) => commit.sha), ...rehomedOrphans.map((commit) => commit.sha), ], }); await this.store.logEntry( task.id, `[recovery] auto-recovered branch-cross-contamination: dropped ${recovery.droppedShas.length} commits (already-upstream + misrouted, SHAs: ${recovery.droppedShas.map((sha) => sha.slice(0, 12)).join(", ")}); new tip ${recovery.newTipSha.slice(0, 12)}`, undefined, this.getRunContextFor(task.id), ); for (const dropped of misrouted) { await audit.database({ type: "task:auto-recover-misrouted-foreign-commit", target: task.id, metadata: { droppedSha: dropped.commit.sha, foreignTaskId: dropped.foreignTaskId, paths: dropped.paths, }, }); } await this.store.updateTask(task.id, { recoveryRetryCount: 1, nextRecoveryAt: null, paused: false, pausedReason: null, error: null, }); // FN-4939: preserve the worktree across requeue. The recovery operated // inside the worktree (re-anchored the branch and re-checked it out), so // the worktree directory remains internally consistent and usable. Nulling // task.worktree here was the root cause of transient // `no-worktree-no-merge-confirmed` stall signals — a live mapped worktree // would still exist on disk while task.worktree was null, and downstream // classifiers (in-review-stall.ts, TaskChangesTab) cannot distinguish // "worktree gone" from "pointer not yet repopulated". Matches sibling // recovery paths in auto-recovery-handlers/contamination.ts, // tryBootstrapMisbindingRecovery, and self-healing reclaim. this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: true, preserveWorktree: true }); return; } if (alreadyAttemptedRecovery) { await this.store.logEntry( task.id, "[recovery] auto-recovery already attempted; escalating to human adjudication", undefined, this.getRunContextFor(task.id), ); } else if (genuinelyUnique.length > 0) { await this.store.logEntry( task.id, `[recovery] unique foreign commits require human adjudication: ${genuinelyUnique.map((commit) => commit.sha.slice(0, 12)).join(", ")}`, undefined, this.getRunContextFor(task.id), ); } } catch (recoveryError: unknown) { const recoveryMessage = recoveryError instanceof Error ? recoveryError.message : String(recoveryError); await this.store.logEntry(task.id, `[recovery] contamination auto-recovery failed: ${recoveryMessage}`, undefined, this.getRunContextFor(task.id)); } const autoRecoveryDispatcher = this.getAutoRecoveryDispatcher(audit); const ownCommits = err.foreignCommits.filter((commit) => commit.foreignTaskId === task.id).length; const foreignAttributedCommits = err.foreignCommits.filter((commit) => commit.foreignTaskId !== task.id).length; const foreignOnlyClassification = (task.branch && task.baseCommitSha) ? await classifyForeignOnlyContamination({ repoDir: this.rootDir, branchName: task.branch, baseSha: task.baseCommitSha, taskId: task.id, }).catch(() => null) : null; const decision = await autoRecoveryDispatcher.dispatch({ class: "branch-cross-contamination", taskId: task.id, runId: this.getRunContextFor(task.id)?.runId, pausedReason: "branch-cross-contamination", evidence: { ownCommits, foreignAttributedCommits, foreignOnlyKind: foreignOnlyClassification?.kind, }, underlyingError: err, }, { task, retryCount: task.recoveryRetryCount ?? 0, settings: (await this.store.getSettings()).autoRecovery ?? { mode: "deterministic-only", maxRetries: 3 }, }); if (decision.action === "pause") { await this.store.updateTask(task.id, { status: "failed", error: err.message, paused: true, pausedReason: "branch-cross-contamination", }); } return; } else if (isBranchConflictError(err)) { const conflictCount = (this.branchConflictErrorCount.get(task.id) ?? 0) + 1; this.branchConflictErrorCount.set(task.id, conflictCount); if (conflictCount > this.BRANCH_CONFLICT_TRIPWIRE_THRESHOLD) { const details = [ `branch=${err.branchName}`, `worktree=${err.conflictingWorktreePath}`, `existingTipSha=${err.existingTipSha}`, `startPoint=${err.startPoint}`, ].join(" "); const tripwireMessage = `Branch conflict tripwire fired after ${conflictCount} events (threshold ${this.BRANCH_CONFLICT_TRIPWIRE_THRESHOLD}). ${details}`; await this.store.logEntry(task.id, `[recovery] ${tripwireMessage}`, undefined, this.getRunContextFor(task.id)); const autoRecoveryDispatcher = this.getAutoRecoveryDispatcher(audit); const decision = await autoRecoveryDispatcher.dispatch({ class: "branch-conflict-tripwire", taskId: task.id, runId: this.getRunContextFor(task.id)?.runId, pausedReason: "branch-conflict-tripwire", evidence: { branchName: err.branchName, conflictingWorktreePath: err.conflictingWorktreePath, }, underlyingError: err, }, { task, retryCount: task.recoveryRetryCount ?? 0, settings: (await this.store.getSettings()).autoRecovery ?? { mode: "deterministic-only", maxRetries: 3 }, }); if (decision.action === "pause") { await this.store.updateTask(task.id, { status: "failed", error: tripwireMessage, paused: true, pausedReason: "branch-conflict-tripwire", }); } return; } let outcome: "retry" | "reclaimed" | "sticky" = "sticky"; for (let attempt = 1; attempt <= this.MAX_AUTO_RECOVERY_ATTEMPTS; attempt += 1) { outcome = await this.handleBranchConflict(task, err); if (outcome !== "retry") break; await this.store.logEntry(task.id, `[recovery] ${task.id} branch-conflict auto-retry requested (${attempt}/${this.MAX_AUTO_RECOVERY_ATTEMPTS})`, undefined, this.getRunContextFor(task.id)); const taskForRetry = await this.store.getTask(task.id); await recordRetry({ store: this.store, settings: await this.store.getSettings(), task: taskForRetry, category: "branchConflict", role: "executor", agentId: task.assignedAgentId ?? undefined, attempt, }); } if (outcome === "retry") { const autoRecoveryDispatcher = this.getAutoRecoveryDispatcher(audit); const decision = await autoRecoveryDispatcher.dispatch({ class: "branch-conflict-recovery-exhausted", taskId: task.id, runId: this.getRunContextFor(task.id)?.runId, pausedReason: "branch-conflict-recovery-exhausted", evidence: { branchName: err.branchName, conflictingWorktreePath: err.conflictingWorktreePath, }, underlyingError: err, }, { task, retryCount: task.recoveryRetryCount ?? 0, settings: (await this.store.getSettings()).autoRecovery ?? { mode: "deterministic-only", maxRetries: 3 }, }); if (decision.action === "pause") { await this.store.updateTask(task.id, { status: "failed", error: err.message, paused: true, pausedReason: "branch-conflict-recovery-exhausted", }); } return; } return; } else if (await this.handleNonContinuableSessionError(task, taskDone, errorMessage)) { return; } else if (await this.handleNonContinuableSessionRetry(task, errorMessage)) { return; } else if (this.options.usageLimitPauser && isUsageLimitError(errorMessage)) { await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, errorMessage); } else if (isTransientError(errorMessage)) { // Transient network/infrastructure error — use bounded recovery policy const decision = computeRecoveryDecision({ recoveryRetryCount: task.recoveryRetryCount, nextRecoveryAt: task.nextRecoveryAt, }); if (decision.shouldRetry) { const attempt = decision.nextState.recoveryRetryCount; const delay = formatDelay(decision.delayMs); // Silent transient errors (e.g., "request was aborted") are noisy — skip logging if (!isSilentTransientError(errorMessage)) { executorLog.warn(`⚡ ${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${errorMessage}`); await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${errorMessage}`, undefined, this.getRunContextFor(task.id)); } // Clean up the old worktree so the retry gets a fresh one if (worktreePath && existsSync(worktreePath)) { try { const settings = await this.store.getSettings(); await removeWorktree({ worktreePath, rootDir: this.rootDir, settings, taskId: task.id, audit, reason: RemovalReason.ExecutorTransientRetry, expectedOwnerTaskId: task.id, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); executorLog.log(`Removed old worktree for transient retry: ${worktreePath}`); } catch (cleanupErr: unknown) { const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr); executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`); } } await this.store.updateTask(task.id, { recoveryRetryCount: decision.nextState.recoveryRetryCount, nextRecoveryAt: decision.nextState.nextRecoveryAt, worktree: null, branch: null, }); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true }); return; } // Recovery budget exhausted — escalate to real failure executorLog.error(`✗ ${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${errorDetail}`); await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${errorMessage}`, errorStack ?? errorDetail, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { status: "failed", error: errorMessage, recoveryRetryCount: null, nextRecoveryAt: null, }); await this.persistTokenUsage(task.id); executorLog.log(`✗ ${task.id} transient retries exhausted — failed in execution`); this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage)); return; } const terminalError = err instanceof RetryStormError ? JSON.stringify(serializeRetryStormError(err)) : errorMessage; executorLog.error(`✗ ${task.id} execution failed:`, errorDetail); await this.store.logEntry(task.id, `Execution failed: ${terminalError}`, errorStack ?? errorDetail, this.getRunContextFor(task.id)); await this.store.updateTask(task.id, { status: "failed", error: terminalError }); await this.persistTokenUsage(task.id); executorLog.log(`✗ ${task.id} execution failed`); this.options.onError?.(task, err instanceof Error ? err : new Error(errorMessage)); } } finally { if (reviewAddressingActivated) { const latestTask = await this.store.getTask(task.id); if (taskDone) { await this.transitionReviewAddressing(task.id, ["in-progress", "queued"], "addressed"); } else if (latestTask.status === "failed") { await this.transitionReviewAddressing(task.id, ["in-progress", "queued"], "failed"); } } /* FNXC:GlobalConcurrencyControls 2026-07-15-02:55: Belt-and-suspenders for graph→legacy pre-held handoff inside the lock-claimed try: release any still-registered slot before lock/executing cleanup. execute()'s outer finally also drops (no-op once take/drop already cleared the registration). */ if (dropPreHeldExecutorSlot(task.id)) this.options.semaphore?.release(); this.executing.delete(task.id); executingTaskLock.release(task.id); // Clear run context at end of execute() lifecycle this.currentRunContexts.delete(task.id); // U5 (R6) leak guard: effectiveColumnAgentByTask is set() in the outer execute() // scope (execute-seam ~6191, step-session ~5674) BEFORE the session-entry try // whose finally (deleteActiveSession / deleteActiveStepExecutor) normally clears // it. A throw between the set() and that try would otherwise leak the entry and // permanently block the column agent's heartbeat ticks. Deleting here in the // outer finally covers BOTH paths since both run inside execute(). this.effectiveColumnAgentByTask.delete(task.id); // Terminate all spawned child agents on ALL exit paths. // This must run here (in the outer finally) rather than only in agentWork's // finally block, because failures during worktree creation or before // agentWork is entered leave children orphaned with no other cleanup path. try { await this.terminateAllChildren(task.id); } catch (err) { executorLog.warn(`terminateAllChildren failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`); } // Reset loop recovery state at end of execute() lifecycle. // State is in-memory and per-run — should not persist across attempts. this.loopRecoveryState.delete(task.id); this.tokenUsageBaselines.delete(task.id); if (taskDone) { this.branchConflictErrorCount.delete(task.id); } else { const latestTask = await this.store.getTask(task.id); if ((await resolveTerminalColumnsFor(this.store, task.id)).includes(latestTask.column)) { this.branchConflictErrorCount.delete(task.id); } } // Requeue stale assistant-continuation sessions AFTER this.executing is cleared. // Moving the task while the execution guard is still held can cause the scheduler's // task:moved dispatch to no-op, stranding the task in todo with no fresh run. if (staleAssistantContinuationRequeue) { /* FNXC:ExecutorSessionRecovery 2026-07-14-06:26: Claim the process-wide executor lock for deferred cleanup, release it immediately before moveTask emits task:moved, and always drop the claim on errors. This closes the guard-release race without recreating the original no-op dispatch: a fresh retry cannot start while stale state is being cleared, but can claim the task when the committed move event fires. FNXC:ExecutorSessionRecovery 2026-07-14-06:34: Release the stale run's activeWorktrees slot before releasing the executor lock. Once the lock is open, the fresh retry may install its own slot while moveTask dispatches; deleting afterward would erase the new run's capacity and liveness tracking. */ const cleanupClaimed = executingTaskLock.tryClaim(task.id); if (!cleanupClaimed) { executorLog.debug(`${task.id} stale assistant-continuation requeue skipped — a fresh executor already claimed the task`); } else { let cleanupLockHeld = true; try { const latestTask = await this.store.getTask(task.id); const continuationLanes = await this.resolveResumeLanes(task.id); if (latestTask.column === continuationLanes.wip || latestTask.column === continuationLanes.hold) { await this.store.updateTask(task.id, { sessionFile: null, status: null, error: null, }); const continuationReboundColumn = await resolveReboundColumnFor(this.store, task.id); if (latestTask.column !== continuationReboundColumn) { this.markGraphExecuteSelfRequeued(task.id); this.activeWorktrees.delete(task.id); executingTaskLock.release(task.id); cleanupLockHeld = false; await this.store.moveTask(task.id, continuationReboundColumn, { preserveResumeState: true }); } else { this.activeWorktrees.delete(task.id); } executorLog.log(`${task.id} stale assistant-continuation session cleared — requeued to ${continuationReboundColumn} with progress preserved`); } else { executorLog.debug(`${task.id} stale assistant-continuation requeue skipped — task is now in '${latestTask.column}'`); } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to requeue stale assistant-continuation task ${task.id}: ${errorMessage}`); } finally { if (cleanupLockHeld) { executingTaskLock.release(task.id); } } } } // Requeue stuck-killed task AFTER this.executing is cleared. // This prevents the race where the scheduler re-dispatches the task // (via task:moved → execute()) while the old execution guard is still set, // which caused the new execute() call to silently no-op, stranding the // task in "in-progress" with no active session or worktree. if (stuckRequeue === true) { if (this.userCanceledTaskIds.has(task.id)) { this.clearPausedAborted(task.id); this.stuckAborted.delete(task.id); this.userCanceledTaskIds.delete(task.id); await this.store.logEntry(task.id, "Execution canceled by user — leaving task in todo"); } else { try { // Re-read latest task state. While this execute() invocation was // unwinding, self-healing (e.g. recoverCompletedTasks) may have // already transitioned the task to in-review or done. Continuing // the stuck-requeue cleanup in that case would destroy the worktree // the recovery now relies on and clobber the task back to todo with // all step progress reset, undoing valid completion. Skip the // entire cleanup if the column has moved on past in-progress/todo. const latestTask = await this.store.getTask(task.id); const outerRequeueLanes = await this.resolveResumeLanes(task.id); if (latestTask.column !== outerRequeueLanes.wip && latestTask.column !== outerRequeueLanes.hold) { executorLog.log( `${task.id} stuck-requeue skipped — task is now in '${latestTask.column}' (recovered concurrently)`, ); } else { const settings = await this.store.getSettings(); const preserveProgress = settings.preserveProgressOnStuckRequeue !== false; /* FNXC:StuckRequeue 2026-06-27-23:15: Preserve-progress stuck requeues still remove the old checkout. Reconcile steps first so uncommitted-only output is reset to pending while committed progress can remain complete. */ await this.resetStepsIfWorkLost(latestTask); // Clean up the old worktree so the retry gets a fresh one if (worktreePath && existsSync(worktreePath)) { try { await removeWorktree({ worktreePath, rootDir: this.rootDir, settings, taskId: task.id, audit, reason: RemovalReason.ExecutorStuckKilled, expectedOwnerTaskId: task.id, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); executorLog.log(`Removed old worktree for stuck-killed retry: ${worktreePath}`); } catch (cleanupErr: unknown) { const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr); executorLog.warn(`Failed to remove old worktree ${worktreePath}: ${cleanupErrMessage}`); } } await this.store.updateTask(task.id, { status: "queued", error: null, worktree: null, branch: null, }); // Only move to todo if not already there. Use the freshly-read // latestTask.column rather than the stale captured task.column — // the captured snapshot can be hours old and would race against // any concurrent recovery (see comment above). const stuckReboundColumn = await resolveReboundColumnFor(this.store, task.id); if (latestTask.column !== stuckReboundColumn) { this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, stuckReboundColumn, preserveProgress ? { preserveProgress: true } : undefined); /* Audit trail: record task move (FN-1404). FNXC:WorkflowLifecycleColumns 2026-07-30-15:15: `to` records the column the card was ACTUALLY moved to. It was hardcoded `"todo"` while the move target was already resolved from the workflow, so on a renamed board the audit row named a column the move never touched — a run-audit trail that disagrees with the move it describes is worse than none, because it is the record an operator reaches for afterwards. */ await audit.database({ type: "task:move", target: task.id, metadata: { to: stuckReboundColumn } }); executorLog.log(`${task.id} moved to ${stuckReboundColumn} for retry after stuck kill${preserveProgress ? " (progress preserved)" : ""}`); } else { executorLog.debug(`${task.id} already in ${stuckReboundColumn} — skipping redundant move`); } } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to requeue stuck task ${task.id}: ${errorMessage}`); } } } /* * FNXC:AgentGating 2026-07-12-17:12: * MAIN-008 closes the approval-decision/unwind race. The dashboard can * unpause while the original executor still owns its process-wide lock; * consume that single deferred edge only after every old-session cleanup * path above has run, then bootstrap one new executor session. A Set plus * resumingUnpaused makes duplicate task updates idempotent. */ await this.resumeApprovalAfterUnwindIfNeeded(task.id); } } // ── Custom tools for the worker agent ────────────────────────────── private createTaskUpdateTool( taskId: string, codeReviewVerdicts: Map, sessionRef: { current: AgentSession | null }, stuckDetector?: StuckTaskDetector, ): ToolDefinition { const store = this.store; return { name: "fn_task_update", label: "Update Step", description: "Update a step's status. Call before starting a step (in-progress), " + "after completing it (done), or to skip it (skipped). " + "Optionally update task dependencies by passing a dependencies array. " + "Optionally set workflow-defined custom field values by passing a custom_fields patch " + "(keyed by field id; validated against the workflow's field schema; pass null to clear a field). " + "step/status may be omitted to update only custom_fields or dependencies. " + "The board updates in real-time.", parameters: taskUpdateParams, execute: async (_id: string, params: Static) => { const { step, status, dependencies, custom_fields } = params; // Bare-call guard (P1 api-contract): a call with none of // step/status/dependencies/custom_fields silently no-op'd, which the // agent cannot observe. Reject it up front so the failure is visible and // self-describing. The legacy no-op text is preserved as the detail. if (step === undefined && status === undefined && dependencies === undefined && custom_fields === undefined) { return { content: [{ type: "text" as const, text: "ERROR: fn_task_update requires at least one of: step+status (report step progress), " + "dependencies (array of task ids), or custom_fields (workflow-defined field patch). " + "No-op: provide a step+status, dependencies, or custom_fields to update.", }], details: {}, isError: true, }; } // Custom-field patch (KTD-13): routed through the store's single write // authority, which validates each value against the task's workflow field // schema. A typed rejection surfaces the offending field id + reason as a // tool error so the agent can correct it. Applied first so a field-only // call (step omitted) returns here. if (custom_fields !== undefined) { const res = await store.updateTaskCustomFields(taskId, custom_fields); if (!res.ok) { const r = res.rejection; // Self-correcting rejection text: append the valid field ids (and, // for an enum violation, the valid values for the offending field) // resolved from the task's workflow field schema so a failed write // carries everything the agent needs to retry. Best-effort: a // resolution failure just omits the hint (the base reason still ships). let hint = ""; try { const defs = await this.resolveTaskCustomFieldDefs(taskId); if (defs && defs.length > 0) { if (r.code === "unknown-field" || r.code === "no-fields-defined") { hint = ` Valid field ids: ${defs.map((f) => f.id).join(", ")}.`; } else if (r.code === "enum-violation") { const field = defs.find((f) => f.id === r.fieldId); const opts = field?.options?.map((o) => o.value) ?? []; if (opts.length > 0) hint = ` Valid values for '${r.fieldId}': ${opts.join(", ")}.`; } } } catch { /* hint is best-effort */ } return { content: [{ type: "text" as const, text: `ERROR: custom field '${r.fieldId}' rejected (${r.code}): ${r.detail}${hint}`, }], details: { fieldId: r.fieldId, code: r.code, detail: r.detail }, isError: true, }; } // A custom-fields-only update (no step) succeeds here. if (step === undefined && status === undefined && dependencies === undefined) { const updatedKeys = Object.keys(custom_fields); return { content: [{ type: "text" as const, text: `Updated custom field(s): ${updatedKeys.join(", ")}.`, }], details: { updatedFields: updatedKeys }, }; } } // Record step progress for stuck task detection. // Step transitions (in-progress, done, skipped) indicate real progress // and reset the loop detection counter. Generic activity (text deltas, // tool calls) is tracked separately via recordActivity in AgentLogger. if (status === "in-progress" || status === "done" || status === "skipped") { stuckDetector?.recordProgress(taskId); } // Dependencies-only update (no step) is permitted; handle deps then return. if (step === undefined) { if (dependencies !== undefined) { if (dependencies.includes(taskId)) { return { content: [{ type: "text" as const, text: `Cannot add self-dependency: ${taskId} cannot depend on itself.` }], details: {}, }; } const invalidIds: string[] = []; for (const depId of dependencies) { try { await store.getTask(depId); } catch { invalidIds.push(depId); } } if (invalidIds.length > 0) { return { content: [{ type: "text" as const, text: `Cannot set dependencies — the following task(s) do not exist: ${invalidIds.join(", ")}` }], details: {}, }; } await store.updateTask(taskId, { dependencies }); return { content: [{ type: "text" as const, text: `Dependencies updated.` }], details: {}, }; } return { content: [{ type: "text" as const, text: `No-op: provide a step+status, dependencies, or custom_fields to update.` }], details: {}, }; } if (status === undefined) { return { content: [{ type: "text" as const, text: `Step ${step} provided without a status. Pass status (pending/in-progress/done/skipped).` }], details: {}, }; } if (!Number.isInteger(step) || step < 0) { return { content: [{ type: "text" as const, text: `Invalid step number: ${step}. Steps are 0-indexed; Step 0 is Preflight.`, }], details: {}, }; } /* * FNXC:StepNumbering 2026-06-17-00:00: * FN-6607 makes fn_task_update.step the same 0-based number agents see in PROMPT.md (`### Step N:`) and TaskStore.updateStep uses internally. The prior `step - 1` conversion made Step 0 impossible to mark done and shifted every review/progress update one array slot early. */ const stepIndex = step; if (status === "in-progress") { try { const latestTask = await store.getTask(taskId); const otherInProgressStepIndex = latestTask.steps.findIndex( (taskStep, index) => index !== stepIndex && taskStep.status === "in-progress", ); if (otherInProgressStepIndex !== -1) { executorLog.warn( `${taskId}: fn_task_update marking step ${step} in-progress while step ${otherInProgressStepIndex} is already in-progress`, ); } } catch (err) { executorLog.warn(`${taskId}: failed to inspect step lease state before fn_task_update: ${err}`); } } /* FNXC:WorkflowReviewGates 2026-07-19-02:30: U10 (R9): the in-session code-review REVISE gate on `fn_task_update(status="done")` is deleted. Its verdict source was the legacy `fn_review_step` tool, which no longer exists, so the map it read is permanently empty. A REVISE from a graph-owned Code Review node routes back to the implementation node as a graph edge instead of blocking a step-status tool call. */ // Handle dependencies parameter if provided if (dependencies !== undefined) { // Validate: prevent self-dependency if (dependencies.includes(taskId)) { return { content: [{ type: "text" as const, text: `Cannot add self-dependency: ${taskId} cannot depend on itself.`, }], details: {}, }; } // Validate: all dependency task IDs must exist const invalidIds: string[] = []; for (const depId of dependencies) { try { await store.getTask(depId); } catch { invalidIds.push(depId); } } if (invalidIds.length > 0) { return { content: [{ type: "text" as const, text: `Cannot set dependencies — the following task(s) do not exist: ${invalidIds.join(", ")}`, }], details: {}, }; } // Update dependencies await store.updateTask(taskId, { dependencies }); } const task = await store.updateStep(taskId, stepIndex, status as StepStatus); const stepInfo = task.steps[stepIndex]; if (!stepInfo) { return { content: [{ type: "text" as const, text: `Invalid step number: ${step}. This task has ${task.steps.length} step(s) (0-indexed; valid range 0-${Math.max(0, task.steps.length - 1)}).`, }], details: {}, }; } const persistedStatus = stepInfo.status; const progress = task.steps.filter((s) => s.status === "done").length; /* FNXC:WorkflowReviewGates 2026-07-19-02:30: U10 (R9): the pre-step conversation-checkpoint capture is deleted with `fn_review_step`. Its only consumer was that tool's RETHINK rewind (`session.navigateTree`); a graph-owned RETHINK re-enters the implementation node instead of rewinding the live conversation. */ // FNXC:StepLifecycle 2026-07-22-09:50: A persisted-status mismatch means // the store rejected the transition (for example, a completed-step // regression or an out-of-order start/completion). FN-5168 treats // repeated rebuffs after loop recovery as a deterministic churn signal. if (persistedStatus !== status) { stuckDetector?.recordIgnoredStepUpdate(taskId); const ignoredStepUpdates = stuckDetector?.getIgnoredStepUpdateCount(taskId) ?? 0; const loopAttempts = this.loopRecoveryState.get(taskId)?.attempts ?? 0; if (loopAttempts >= 1 && ignoredStepUpdates === 25) { executorLog.warn( `${taskId}: no-progress churn detected ` + `(ignoredStepUpdates=${ignoredStepUpdates}, stuckKillStreak=${task.stuckKillCount ?? 0}) — ` + `escalating to STUCK_NO_PROGRESS_CHURN`, ); } return { content: [{ type: "text" as const, text: `Step ${step} (${stepInfo.name}) remains ${persistedStatus} — ${status} request ignored to preserve step lifecycle invariants. Progress: ${progress}/${task.steps.length} done.`, }], details: {}, }; } return { content: [{ type: "text" as const, text: `Step ${step} (${stepInfo.name}) → ${persistedStatus}. Progress: ${progress}/${task.steps.length} done.`, }], details: {}, }; }, }; } private createTaskLogTool(taskId: string): ToolDefinition { return sharedCreateTaskLogTool(this.store, taskId); } private createTaskLogsReadTool(taskId: string): ToolDefinition { return sharedCreateTaskLogsReadTool(this.store, taskId); } /* FNXC:EphemeralAgentTaskCreation 2026-07-01-00:00: A task-execution session is an ephemeral worker when no permanent identity agent governs it (default executor-FN-XXXX worker) or the governing agent is itself ephemeral. Pass that through so fn_task_create honors the project `ephemeralAgentsCanCreateTasks` toggle; permanent-agent sessions are never gated. */ private createTaskCreateTool(callerIsEphemeral: boolean, sourceTaskId?: string, sourceAgentId?: string): ToolDefinition { return sharedCreateTaskCreateTool(this.store, { sourceType: "api", sourceAgentId, sourceParentTaskId: sourceTaskId }, { rootDir: this.rootDir, callerIsEphemeral, sourceTaskId, sourceAgentId, messageStore: this.options.messageStore }); } private createTaskDocumentWriteTool(taskId: string): ToolDefinition { return sharedCreateTaskDocumentWriteTool(this.store, taskId); } private createTaskDocumentReadTool(taskId: string): ToolDefinition { return sharedCreateTaskDocumentReadTool(this.store, taskId); } private createTaskPromptWriteTool(taskId: string): ToolDefinition { return sharedCreateTaskPromptWriteTool(this.store, taskId, this.getRunContextFor(taskId)); } private createTaskFileScopeAddTool(taskId: string): ToolDefinition { return sharedCreateTaskFileScopeAddTool(this.store, taskId, this.getRunContextFor(taskId)); } /* FNXC:ArtifactRegistry 2026-07-10-14:30: Executor-lane registration anchors relative `path` payloads at the task worktree (where the agent saves screenshots/wireframes/mocks) and defaults taskId to the executing task so agent-produced media surfaces in the per-task Artifacts tab without the agent having to repeat its own task id. */ private createArtifactRegisterTool(authorId: string, taskId: string, worktreePath: string): ToolDefinition { return sharedCreateArtifactRegisterTool(this.store, authorId, this.options.messageStore, { baseDir: worktreePath, defaultTaskId: taskId, }); } private createArtifactListTool(): ToolDefinition { return sharedCreateArtifactListTool(this.store); } private createArtifactViewTool(): ToolDefinition { return sharedCreateArtifactViewTool(this.store); } private createWorkflowListTool(): ToolDefinition { return sharedCreateWorkflowListTool(this.store); } private createWorkflowGetTool(): ToolDefinition { return sharedCreateWorkflowGetTool(this.store); } private createWorkflowValidateTool(): ToolDefinition { return sharedCreateWorkflowValidateTool(this.store); } private createWorkflowSelectTool(taskId: string): ToolDefinition { return sharedCreateWorkflowSelectTool(this.store, taskId); } private createTaskPromoteTool(taskId: string): ToolDefinition { return sharedCreateTaskPromoteTool(this.store, taskId); } private createWorkflowCreateTool(): ToolDefinition { return sharedCreateWorkflowCreateTool(this.store); } private createWorkflowUpdateTool(): ToolDefinition { return sharedCreateWorkflowUpdateTool(this.store); } private createWorkflowDeleteTool(): ToolDefinition { return sharedCreateWorkflowDeleteTool(this.store); } private createWorkflowSettingsTool(): ToolDefinition { return sharedCreateWorkflowSettingsTool(this.store); } private createTraitListTool(): ToolDefinition { return sharedCreateTraitListTool(); } private createTaskAddDepTool(taskId: string): ToolDefinition { const store = this.store; return { name: "fn_task_add_dep", label: "Add Dependency", description: "Declare a dependency on an existing task. Use when you discover " + "mid-execution that another task must be completed first. " + "Adding a dependency to an in-progress task will stop execution " + "and discard current work, so confirm=true is required. " + "Without confirm=true, a warning is returned first.", parameters: taskAddDepParams, execute: async (_id: string, params: Static) => { const targetId = params.task_id; // Prevent self-dependency if (targetId === taskId) { return { content: [{ type: "text" as const, text: `Cannot add self-dependency: ${taskId} cannot depend on itself.`, }], details: {}, }; } // Validate target task exists try { await store.getTask(targetId); } catch { return { content: [{ type: "text" as const, text: `Task ${targetId} not found. Cannot add dependency on a non-existent task.`, }], details: {}, }; } // Read current task to get existing dependencies const currentTask = await store.getTask(taskId); const existing = currentTask.dependencies; // Dedup check if (existing.includes(targetId)) { return { content: [{ type: "text" as const, text: `${targetId} is already a dependency of ${taskId}. No changes made.`, }], details: {}, }; } // Confirmation gate — destructive action for in-progress tasks if (!params.confirm) { return { content: [{ type: "text" as const, text: `Warning: adding a dependency to an in-progress task will stop execution and discard current work. Call with confirm=true to proceed.`, }], details: {}, }; } // Add the dependency await store.updateTask(taskId, { dependencies: [...existing, targetId] }); await store.logEntry(taskId, `Added dependency on ${targetId} — stopping execution for re-planning`); // Trigger abort flow (same pattern as pausedAborted) this.depAborted.add(taskId); const activeSession = this.activeSessions.get(taskId); activeSession?.session.dispose(); // Also terminate step sessions if active const stepExecutor = this.activeStepExecutors.get(taskId); if (stepExecutor) { stepExecutor.terminateAllSessions().catch(err => executorLog.warn(`Failed to terminate step sessions for dep-abort ${taskId}: ${err}`) ); } return { content: [{ type: "text" as const, text: `Added dependency on ${targetId}. Stopping execution — task will move to triage for re-planning.`, }], details: {}, }; }, }; } private async transitionReviewAddressing(taskId: string, from: Array<"queued" | "in-progress" | "addressed" | "failed">, to: "queued" | "in-progress" | "addressed" | "failed"): Promise { const task = await this.store.getTask(taskId); const reviewState = task.reviewState; if (!reviewState || reviewState.addressing.length === 0) { return; } const now = new Date().toISOString(); let changed = false; const addressing = reviewState.addressing.map((record) => { if (!from.includes(record.status)) { return record; } changed = true; /* FNXC:ReviewAddressing 2026-07-30-16:40 DELIBERATE-LITERAL: `to` here is a review-addressing RECORD STATUS (`"queued" | "in-progress" | "addressed" | "failed"`, see this method's signature), NOT a board column — the very next lines test it against `"addressed"` and `"failed"`, which are not columns at all. The lifecycle-column census matches the bare string and counted it; resolving it to a workflow role would be nonsense. */ return { ...record, status: to, startedAt: to === "in-progress" ? now : record.startedAt, completedAt: to === "addressed" || to === "failed" ? now : record.completedAt, error: to === "addressed" ? undefined : record.error, }; }); if (!changed) { return; } await this.store.updateTask(taskId, { reviewState: { ...reviewState, addressing, }, }); } private async verifyWorktreeInvariants( task: Task, worktreePathOverride?: string, allowReanchor = true, options?: { noOpCompletion?: boolean; noOpCompletionReason?: string }, ): Promise<{ ok: true } | { ok: false; reason: "wrong_toplevel" | "wrong_branch" | "no_commits"; observed: string; expected: string; repo?: string }> { const settings = await this.store.getSettings(); // FNXC:Workspace 2026-06-21-23:30: KTD2 — un-stubbed per-repo worktree-invariant verification. // Phase A returned a flat {ok:true} stub here (no root worktree to verify against the non-git root). Phase B iterates every `task.workspaceWorktrees` entry, asserting (a) the sub-repo worktree's git toplevel matches the recorded repo.worktreePath and (b) its HEAD is on the recorded `fusion/` branch (repo.branch). The result union is PRESERVED EXACTLY — `{ok:true} | {ok:false; reason:'wrong_toplevel'|'wrong_branch'|'no_commits'; observed; expected}` — because the :10889 consumer switches on `reason` to drive requeue/handoff (:10894-10936). We ADD an optional `repo` field to the failure shape (purely additive; the consumer only reads reason/observed/expected) and return the FIRST failing repo. A zero-acquire workspace task (empty map) verifies vacuously → {ok:true}, matching Phase A so fn_task_done does not requeue it. if (this.workspaceConfig) { const workspaceWorktrees = task.workspaceWorktrees ?? {}; // FNXC:Workspace 2026-06-22-00:00: KTD2 — resolve the SAME task-wide no-commit eligibility the singular path // uses (getNoCommitEligibilityReason / no-op-completion sentinel / prompt-derived), once, before the per-repo // loop. When eligible (Plan-Only, verified no-op, etc.) the per-repo no_commits guard below is skipped so an // intentionally commit-free workspace task is not blocked from completion. const workspacePromptContent = (task as Task & { prompt?: unknown }).prompt; const workspacePromptEligibility = evaluatePromptDerivedNoCommitEligibility( task, typeof workspacePromptContent === "string" ? workspacePromptContent : "", ); const workspaceNoCommitEligibilityReason = getNoCommitEligibilityReason(task) ?? (options?.noOpCompletion ? options.noOpCompletionReason ?? "verified no-op/duplicate completion sentinel" : null) ?? (workspacePromptEligibility.eligible ? workspacePromptEligibility.reason ?? "prompt-derived no-commit eligibility" : null); if (workspaceNoCommitEligibilityReason) { executorLog.debug(`${task.id}: workspace fn_task_done no_commits guard skipped (${workspaceNoCommitEligibilityReason})`); } // FNXC:Workspace 2026-06-21-15:00: F6 — iterate sorted repo keys so the FIRST failing repo // returned here is deterministic across runs/rehydrate (the value is surfaced to the operator). for (const repoRel of Object.keys(workspaceWorktrees).sort()) { const repo = workspaceWorktrees[repoRel]; const expectedBranch = repo.branch || canonicalFusionBranchName(task.id); // Skip git checks if the worktree dir is gone (mirrors the singular FN-009 carve-out below): completion does not require a live worktree on disk. if (!existsSync(repo.worktreePath)) { executorLog.log(`${task.id}: workspace worktree for ${repoRel} not found at ${repo.worktreePath} — skipping git validation`); continue; } let expectedWorktreeRealpath: string; try { expectedWorktreeRealpath = canonicalizePath(repo.worktreePath); } catch (error) { return { ok: false, reason: "wrong_toplevel", repo: repoRel, observed: `unresolvable repo worktree (${repo.worktreePath}): ${error instanceof Error ? error.message : String(error)}`, expected: `resolvable worktree for ${repoRel}`, }; } try { const { stdout } = await execAsync("git rev-parse --show-toplevel", { cwd: repo.worktreePath, encoding: "utf-8", timeout: 10_000, maxBuffer: 1024 * 1024, }); const observedTopLevelRaw = stdout.trim(); if (observedTopLevelRaw) { const observedTopLevel = canonicalizePath(observedTopLevelRaw); if (observedTopLevel !== expectedWorktreeRealpath) { return { ok: false, reason: "wrong_toplevel", repo: repoRel, observed: observedTopLevel, expected: expectedWorktreeRealpath, }; } } } catch (error) { return { ok: false, reason: "wrong_toplevel", repo: repoRel, observed: error instanceof Error ? error.message : String(error), expected: expectedWorktreeRealpath, }; } try { const { stdout } = await execAsync("git rev-parse --abbrev-ref HEAD", { cwd: repo.worktreePath, encoding: "utf-8", timeout: 10_000, maxBuffer: 1024 * 1024, }); const observedBranch = stdout.trim(); if (observedBranch && observedBranch !== expectedBranch) { return { ok: false, reason: "wrong_branch", repo: repoRel, observed: observedBranch, expected: expectedBranch, }; } } catch (error) { return { ok: false, reason: "wrong_branch", repo: repoRel, observed: error instanceof Error ? error.message : String(error), expected: expectedBranch, }; } // FNXC:Workspace 2026-06-22-00:00: KTD2 — per-repo no_commits guard (parity with the singular path at :10821). // Phase B originally returned {ok:true} after the toplevel/branch checks, so a workspace task could call // fn_task_done having committed NOTHING in any sub-repo (scope-leak sees zero touched files, branch names match) // and still advance to in-review. Enforce the same `git rev-list --count ..HEAD > 0` invariant per repo, // gated by the SAME task-wide no-commit eligibility below so Plan-Only / no-op-sentinel tasks stay exempt. // The first sub-repo with zero commits fails with reason:'no_commits' (consumer-stable union). if (!workspaceNoCommitEligibilityReason) { const repoBaseRef = await this.resolveDiffBaseRef(repo.worktreePath, repo.baseCommitSha); if (repoBaseRef) { try { const { stdout } = await execAsync(`git rev-list --count ${repoBaseRef}..HEAD`, { cwd: repo.worktreePath, encoding: "utf-8", timeout: 10_000, maxBuffer: 1024 * 1024, }); const trimmedCount = stdout.trim(); if (trimmedCount) { const count = Number.parseInt(trimmedCount, 10); if (!Number.isFinite(count) || count <= 0) { return { ok: false, reason: "no_commits", repo: repoRel, observed: Number.isFinite(count) ? String(count) : trimmedCount, expected: "> 0", }; } } } catch (error) { return { ok: false, reason: "no_commits", repo: repoRel, observed: error instanceof Error ? error.message : String(error), expected: `git rev-list --count ${repoBaseRef}..HEAD > 0`, }; } } else { executorLog.warn(`${task.id}: unable to resolve diff base for ${repoRel} no_commits guard; skipping for this sub-repo`); } } } return { ok: true }; } const branchName = resolveTaskWorkingBranch(task); // Non-workspace tasks hold a one-element set; fall back to its sole member to preserve the original singular resolution. const worktreePath = worktreePathOverride ?? task.worktree ?? this.getActiveWorktreePaths(task.id)[0] ?? null; if (!worktreePath) { return { ok: false, reason: "wrong_toplevel", observed: "missing task.worktree", expected: `registered task worktree under ${resolveWorktreesDir(this.rootDir, settings)}/*`, }; } const expectedRoot = canonicalizePath(this.rootDir); let expectedWorktreeRealpath: string; try { expectedWorktreeRealpath = canonicalizePath(worktreePath); } catch (error) { return { ok: false, reason: "wrong_toplevel", observed: `unresolvable task.worktree (${worktreePath}): ${error instanceof Error ? error.message : String(error)}`, expected: `resolvable task worktree under ${resolveWorktreesDir(this.rootDir, settings)}/*`, }; } // FN-009: If worktree directory doesn't exist, skip git validation for task completion. // This is safe because: // 1. Task completion doesn't modify the worktree // FNXC:PostgresRuntimeStorage 2026-07-14-18:47: Deliverables (task documents and follow-up tasks) are stored in the project-scoped PostgreSQL store. // 3. If code changes were made, the worktree would exist // 4. This prevents ENOENT errors when agents complete documentation/coordination tasks if (!existsSync(worktreePath)) { executorLog.log( `${task.id}: worktree directory not found at ${worktreePath} — skipping git validation for task completion`, ); return { ok: true }; } try { const { stdout } = await execAsync("git rev-parse --show-toplevel", { cwd: worktreePath, encoding: "utf-8", timeout: 10_000, maxBuffer: 1024 * 1024, }); const observedTopLevelRaw = stdout.trim(); if (observedTopLevelRaw) { const observedTopLevel = canonicalizePath(observedTopLevelRaw); if ( observedTopLevel === expectedRoot || !isInsideWorktreesDir(this.rootDir, observedTopLevel, settings) || observedTopLevel !== expectedWorktreeRealpath ) { if (allowReanchor && observedTopLevel !== expectedRoot && isInsideWorktreesDir(this.rootDir, observedTopLevel, settings)) { const reanchor = await detectNestedWorktreeRoot(this.rootDir, worktreePath, settings); if (reanchor.reanchored) { await this.store.updateTask(task.id, { worktree: reanchor.root }); executorLog.log(`${task.id}: re-anchored nested task.worktree ${worktreePath} -> ${reanchor.root}`); await this.store.logEntry(task.id, `Re-anchored nested task.worktree from ${worktreePath} to ${reanchor.root}`, undefined, this.getRunContextFor(task.id)); await this.emitWorktreeReanchoredAudit(task.id, worktreePath, reanchor.root, "verify-worktree-invariants"); return this.verifyWorktreeInvariants(task, reanchor.root, false, options); } } return { ok: false, reason: "wrong_toplevel", observed: observedTopLevel, expected: expectedWorktreeRealpath, }; } } } catch (error) { return { ok: false, reason: "wrong_toplevel", observed: error instanceof Error ? error.message : String(error), expected: expectedWorktreeRealpath, }; } try { const { stdout } = await execAsync("git rev-parse --abbrev-ref HEAD", { cwd: worktreePath, encoding: "utf-8", timeout: 10_000, maxBuffer: 1024 * 1024, }); const observedBranch = stdout.trim(); if (observedBranch && observedBranch !== branchName) { if (observedBranch.toLowerCase() === branchName.toLowerCase()) { executorLog.log(`${task.id}: branch case-mismatch detected; canonicalizing observed=${observedBranch} expected=${branchName}`); const autocorrectResult = await attemptBranchAutocorrect({ worktreePath, observedBranch, expectedBranch: branchName, rootDir: this.rootDir, }); if (autocorrectResult.status !== "failed") { const auditor = createRunAuditor(this.store, this.getRunContextFor(task.id)); await auditor.git({ type: "branch:auto-canonicalize-case", target: worktreePath, metadata: { taskId: task.id, observed: observedBranch, expected: branchName, worktreePath, mode: autocorrectResult.status, }, }); return { ok: true }; } executorLog.warn(`${task.id}: failed to canonicalize branch case mismatch: ${autocorrectResult.reason ?? "unknown"}`); } return { ok: false, reason: "wrong_branch", observed: observedBranch, expected: branchName, }; } } catch (error) { return { ok: false, reason: "wrong_branch", observed: error instanceof Error ? error.message : String(error), expected: branchName, }; } const promptContent = (task as Task & { prompt?: unknown }).prompt; const promptDerivedEligibility = evaluatePromptDerivedNoCommitEligibility( task, typeof promptContent === "string" ? promptContent : "", ); const noCommitEligibilityReason = getNoCommitEligibilityReason(task) ?? (options?.noOpCompletion ? options.noOpCompletionReason ?? "verified no-op/duplicate completion sentinel" : null) ?? (promptDerivedEligibility.eligible ? promptDerivedEligibility.reason ?? "prompt-derived no-commit eligibility" : null); if (noCommitEligibilityReason) { executorLog.debug(`${task.id}: fn_task_done no_commits guard skipped (${noCommitEligibilityReason})`); try { await this.store.logEntry( task.id, `fn_task_done no_commits guard skipped (${noCommitEligibilityReason})`, undefined, this.getRunContextFor(task.id), ); } catch (error) { executorLog.warn( `${task.id}: failed to write no_commits guard skip audit log: ${error instanceof Error ? error.message : String(error)}`, ); } return { ok: true }; } const baseRef = await this.resolveDiffBaseRef(worktreePath, task.baseCommitSha); if (!baseRef) { executorLog.warn(`${task.id}: unable to resolve diff base for invariant commit-count check; skipping no_commits guard`); return { ok: true }; } try { const { stdout } = await execAsync(`git rev-list --count ${baseRef}..HEAD`, { cwd: worktreePath, encoding: "utf-8", timeout: 10_000, maxBuffer: 1024 * 1024, }); const trimmedCount = stdout.trim(); if (!trimmedCount) { return { ok: true }; } const count = Number.parseInt(trimmedCount, 10); if (!Number.isFinite(count) || count <= 0) { return { ok: false, reason: "no_commits", observed: Number.isFinite(count) ? String(count) : stdout.trim(), expected: "> 0", }; } } catch (error) { return { ok: false, reason: "no_commits", observed: error instanceof Error ? error.message : String(error), expected: `git rev-list --count ${baseRef}..HEAD > 0`, }; } return { ok: true }; } private async evaluateTaskDoneScopeLeak( task: Task, worktreePath: string, promptContent: string, settings: Settings, audit?: RunAuditor, ): Promise<{ blocked: false } | { blocked: true; message: string }> { if (task.scopeOverride === true) { executorLog.debug(`${task.id}: scope-leak guard bypassed (scopeOverride=true)`); await this.store.logEntry(task.id, "[scope-leak] scope guard bypassed via task.scopeOverride", undefined, this.getRunContextFor(task.id)); return { blocked: false }; } const declaredScope = await this.store.parseFileScopeFromPrompt(task.id).catch(() => [] as string[]); if (declaredScope.length === 0) { return { blocked: false }; } const reviewLevel = parseReviewLevelFromPrompt(promptContent); const configuredMode = settings.planOnlyScopeLeakEnforcement ?? "warn"; const enforcementMode: "off" | "warn" | "block" = reviewLevel === 1 ? configuredMode : "warn"; if (enforcementMode === "off") { return { blocked: false }; } // FNXC:Workspace 2026-06-22-00:30: KTD4 — per-repo scope-leak guard. // The singular capture below runs `captureUncommittedModifiedFiles` + `captureModifiedFiles` // against `worktreePath`. In workspace mode `worktreePath` is the browse-only non-git workspace // root, so both silently return [] (git failures swallowed) and the uncommitted-in-scope block // never fires — a workspace task could complete with off-scope changes in any sub-repo. So we // ITERATE every acquired sub-repo (cwd = repo.worktreePath, base = repo.baseCommitSha) and block // on the FIRST repo carrying off-scope changes — naming the repo. The task-level preamble above // (scopeOverride / declaredScope / enforcementMode) is shared and runs once. Return shape is // preserved: `{blocked:false} | {blocked:true; message}`. // // FNXC:Workspace 2026-06-21-15:00: F1/F2/F5/F6 hardening of the per-repo scope-leak guard. // F5 (false-block fix + dead-code wiring + single filter surface): we previously repo-prefixed each // touched file (`${repoRel}/${file}`) BEFORE filtering, so `isAlwaysAllowedScopeLeakPath`'s // `startsWith(".changeset/")` carve-out never matched a sub-repo changeset (`repo-a/.changeset/x.md`) // and a legit per-repo changeset was wrongly flagged off-scope → fn_task_done wrongly REFUSED. Now we // derive each repo's repo-LOCAL declared-scope subset (`deriveRepoScopeSubset`) and run the SAME // `workflowPathMatchesDeclaredScope` + `isAlwaysAllowedScopeLeakPath` filter the non-workspace path // uses against the repo-LOCAL touched file — one filter surface, not two. This wires in the formerly // dead `deriveRepoScopeSubset`/`splitRepoScopedPath` helpers. // F1 (fail CLOSED on throw): each repo iteration is wrapped in its own try/catch (like the // attribution-audit loop). A thrown capture/diff error in workspace mode surfaces as a BLOCK naming // the repo instead of bubbling to the outer `.catch()` that fails OPEN — an incomplete scope check // must never let fn_task_done proceed. // F2 (scoped-but-zero-acquire): a scoped task that acquired NO sub-repo worktrees aggregates zero // off-scope files and would silently pass; we block it (scope is declared but unverifiable). // F6 (deterministic ordering): iterate sorted repo keys so the reported offending repo is stable // across runs/rehydrate. let touchedFiles: string[]; let offendingRepo: string | undefined; if (this.workspaceConfig) { const workspaceWorktrees = task.workspaceWorktrees ?? {}; const repoKeys = Object.keys(workspaceWorktrees).sort(); // F2: declaredScope is non-empty here (the `declaredScope.length === 0` early-return above // handled the unscoped case). A scoped task that acquired no sub-repo worktrees cannot have its // scope verified at all — refuse rather than silently passing scope enforcement. if (repoKeys.length === 0) { const message = "workspace task declares File Scope but acquired no sub-repo worktrees — cannot verify scope"; executorLog.warn(`${task.id}: [scope-leak] ${message}`); await this.store.logEntry(task.id, `[scope-leak] ${message}`, undefined, this.getRunContextFor(task.id)); return { blocked: true, message }; } const aggregatedOffScope: string[] = []; for (const repoRel of repoKeys) { const repo = workspaceWorktrees[repoRel]; try { const [repoUncommitted, repoCommitted] = await Promise.all([ this.captureUncommittedModifiedFiles(repo.worktreePath), this.captureModifiedFiles(repo.worktreePath, repo.baseCommitSha, task.id, audit, "scope-leak-guard"), ]); // Repo-LOCAL touched files (no `${repoRel}/` prefix) so the always-allowed `.changeset/` // carve-out and the scope match operate as the reviewer/cwd=repo sees them (F5). const repoTouched = [...new Set([...repoUncommitted, ...repoCommitted])]; // Repo-LOCAL declared-scope subset for THIS repo (prefix stripped). Same filter as the // non-workspace branch below — one surface. const repoScopeSubset = deriveRepoScopeSubset(declaredScope, repoRel); const repoOffScope = repoTouched .filter((filePath) => !workflowPathMatchesDeclaredScope(filePath, repoScopeSubset)) .filter((filePath) => !isAlwaysAllowedScopeLeakPath(filePath)) // Re-prefix the surviving off-scope files for the operator-facing message/attribution. .map((filePath) => `${repoRel}/${filePath}`); if (repoOffScope.length > 0) { // First offending repo wins (mirrors verifyWorktreeInvariants' first-failing-repo return). if (!offendingRepo) offendingRepo = repoRel; aggregatedOffScope.push(...repoOffScope); } } catch (repoErr: unknown) { // F1: fail CLOSED. A capture/diff throw means scope is UNVERIFIED for this repo; refuse // fn_task_done as a precaution rather than letting the outer `.catch()` fail open. const errMessage = repoErr instanceof Error ? repoErr.message : String(repoErr); const message = `workspace scope-leak guard failed to evaluate (${repoRel}/${errMessage}) — refusing fn_task_done as a precaution`; executorLog.warn(`${task.id}: [scope-leak] ${message}`); await this.store.logEntry(task.id, `[scope-leak] ${message}`, undefined, this.getRunContextFor(task.id)); return { blocked: true, message }; } } touchedFiles = aggregatedOffScope; if (touchedFiles.length === 0) { return { blocked: false }; } } else { const [uncommittedTouchedFiles, branchCommittedFiles] = await Promise.all([ this.captureUncommittedModifiedFiles(worktreePath), this.captureModifiedFiles(worktreePath, task.baseCommitSha, task.id, audit, "scope-leak-guard"), ]); touchedFiles = [...new Set([...uncommittedTouchedFiles, ...branchCommittedFiles])]; if (touchedFiles.length === 0) { return { blocked: false }; } } const offScopeFiles = (this.workspaceConfig // In workspace mode `touchedFiles` is already the off-scope set (filtered per repo above). ? touchedFiles : touchedFiles .filter((filePath) => !workflowPathMatchesDeclaredScope(filePath, declaredScope)) // FN-4811 follow-up: by convention every task may add its own changeset entry // under `.changeset/`, so changeset files are always considered in-scope and // never flagged by the scope-leak guard. The file-scope invariant at squash and // the broader contamination guards still catch cross-task changeset leakage at // a higher signal-to-noise ratio than the per-execution scope-leak warning. .filter((filePath) => !isAlwaysAllowedScopeLeakPath(filePath))); if (offScopeFiles.length === 0) { return { blocked: false }; } const renderListPreview = (items: string[], cap = 10): string => { if (items.length <= cap) { return items.join(", "); } const remaining = items.length - cap; return `${items.slice(0, cap).join(", ")}, … (+${remaining} more)`; }; const offScopePreview = renderListPreview(offScopeFiles); const declaredScopePreview = renderListPreview(declaredScope); // Name the offending sub-repo in workspace mode so the operator/agent knows where to revert. const repoTag = offendingRepo ? ` repo=${offendingRepo}` : ""; const message = `[scope-leak] reviewLevel=${reviewLevel} enforcement=${enforcementMode}${repoTag} off-scope touched files [${offScopePreview}]; declared scope [${declaredScopePreview}]; total off-scope=${offScopeFiles.length} total scope=${declaredScope.length}`; executorLog.warn(`${task.id}: ${message}`); await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); if (enforcementMode === "block") { return { blocked: true, message: `Plan-Only scope-leak guard refused fn_task_done${offendingRepo ? ` (sub-repo ${offendingRepo})` : ""}. Off-scope paths: [${offScopePreview}]. Revert them before retrying (for example: git checkout -- ).`, }; } return { blocked: false }; } /* FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — an IMPLICIT completion (agent exits with every step done/skipped and no explicit fn_task_done) is an AUTO-promotion, so it must honor the skip-bypass taint. A synthesized taint refusal here re-parks the run through the existing refusal budget rather than laundering skipped-after-refusal steps into review. The explicit fn_task_done tool path is NOT routed here — that call remains the honest exit. */ private evaluateImplicitCompletionRefusal( task: Task, codeReviewVerdicts: Map, ): ReturnType { const refusal = evaluateTaskDoneRefusal(task, {}, codeReviewVerdicts); if (!refusal.ok) return refusal; const taint = evaluateSkipBypassTaint(task); if (taint.blocked) return buildSkipBypassTaintRefusal(taint); return { ok: true }; } /* FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — a `bulk-step-completion-without-review` refusal stamps the durable taint marker so that later skips (in this or a requeued lifecycle) cannot auto-promote. The marker is cleared only on an honest exit (accepted fn_task_done / operator retry). */ private skipBypassTaintUpdateForRefusal( refusal: Extract, { ok: false }>, ): { bulkCompletionRefusalAt: string } | Record { if (refusal.refusalClass !== "bulk-step-completion-without-review") return {}; return { bulkCompletionRefusalAt: new Date().toISOString() }; } private async handleImplicitTaskDoneRefusal( task: Task, refusal: Extract, { ok: false }>, ): Promise { await this.store.logEntry(task.id, refusal.message, undefined, this.getRunContextFor(task.id)); executorLog.error(`${task.id}: fn_task_done refused (${refusal.refusalClass}) — ${refusal.reason} (implicit completion)`); const taintUpdate = this.skipBypassTaintUpdateForRefusal(refusal); const priorRequeues = task.taskDoneRetryCount ?? 0; const nextRequeueCount = priorRequeues + 1; if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) { await this.store.updateTask(task.id, { status: "queued", error: null, taskDoneRetryCount: nextRequeueCount, ...taintUpdate, paused: false, pausedByAgentId: null, worktree: null, branch: null, sessionFile: null, }); await this.store.logEntry( task.id, `${refusal.message} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`, undefined, this.getRunContextFor(task.id), ); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveProgress: true }); } else { await this.store.updateTask(task.id, { status: "failed", error: refusal.message, ...taintUpdate, paused: false, pausedByAgentId: null, worktree: null, branch: null, sessionFile: null, }); await this.store.logEntry(task.id, `${refusal.message} — execution failed because implicit fn_task_done was refused`, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(task.id); } this.deleteActiveSession(task.id); this.tokenUsageBaselines.delete(task.id); } private createTaskDoneTool( taskId: string, worktreePath: string, promptContent: string, codeReviewVerdicts: Map, onDone: () => void, audit?: RunAuditor, ): ToolDefinition { const store = this.store; return { name: "fn_task_done", label: "Mark Task Done", description: "End the task. With outcome=\"completed\" (default): signal that all steps are complete, tests pass, and " + "documentation is updated — call as the final action after finishing all work; automatically marks all " + "remaining steps as done; optionally provide a summary of what was changed/fixed. " + "With outcome=\"blocked\": honestly park the task when the work genuinely cannot proceed (upstream API break, " + "missing dependency task, unresolvable external blocker). Blocked is NOT a completion claim — it does not " + "trip the review/completion gates, does not auto-complete or auto-skip steps, and preserves your worktree/" + "branch/step progress so the task can be requeued once the blocker clears. Prefer blocked over marking steps " + "skipped when the task cannot be finished.", parameters: Type.Object({ summary: Type.Optional(Type.String({ description: "Optional summary of what was changed/fixed and what was verified (2-4 sentences). Used when outcome=\"completed\".", })), /* FNXC:Lifecycle 2026-07-16-10:20: FN-8141 laundered a genuinely-impossible task into `done`: fn_task_done only expressed success, the bulk-completion gate refused it, the requeue budget re-ran the doomed task 5 times, and the only remaining affordance (skip every step) made `isTaskComplete()` return true so self-healing + the AI merger finalized an empty diff as done. The `blocked` outcome is the sanctioned honest exit: it parks the task `failed` (error `BLOCKED: `) without any completion claim, so laundering is never the cheapest path. */ outcome: Type.Optional(Type.Union( [Type.Literal("completed"), Type.Literal("blocked")], { description: "\"completed\" (default) finishes the task; \"blocked\" honestly parks it as failed because the work cannot proceed. Use \"blocked\" instead of skipping steps + completing when you are stuck." }, )), blockedBy: Type.Optional(Type.Array(Type.String(), { description: "When outcome=\"blocked\": task IDs (e.g. [\"FN-8145\"]) that must complete before this task can proceed. Recorded as real dependency edges so the task requeues behind the blocker.", })), reason: Type.Optional(Type.String({ description: "Required when outcome=\"blocked\": concrete explanation of what is blocking the work and what is needed to unblock it.", })), }), execute: async (_id: string, params: { summary?: string; outcome?: "completed" | "blocked"; blockedBy?: string[]; reason?: string }) => { /* FNXC:Lifecycle 2026-07-16-10:20: FN-8141 — the blocked exit runs BEFORE every completion gate (completion blocker, verdict providers, worktree invariants, bulk-completion refusal). Blocked is not a completion claim, so none of those gates apply; parking `failed` with a `BLOCKED:` error + real dependency edges is the whole action. Steps keep their true statuses (no auto-done, no auto-skip) so a laundered "all steps skipped ⇒ complete" state can never form. */ if (params.outcome === "blocked") { const reason = params.reason?.trim(); if (!reason) { const message = "fn_task_done(outcome=\"blocked\") requires a non-empty `reason` describing what is blocking the work. Provide `reason` (and optional `blockedBy` task IDs) and call again."; return { content: [{ type: "text" as const, text: message }], details: { error: message }, }; } const blockedTask = await store.getTask(taskId); const blockedByIds = Array.from( new Set((params.blockedBy ?? []).map((id) => id.trim()).filter((id) => id.length > 0)), ); const parkError = `BLOCKED: ${reason}`; // Record blockedBy as real dependency edges (union with existing) so the task requeues // behind the blocker rather than re-running the doomed work. Preserve worktree/branch/ // step progress (FN-7863 EXECUTION_DISPATCH_LOOP_EXHAUSTED park convention) — do NOT // call onDone() so the outer loop's `liveTask.status === "failed"` honor-park branch keeps // the row parked for the blocker/operator instead of handing off to review. const mergedDependencies = blockedByIds.length > 0 ? Array.from(new Set([...(blockedTask.dependencies ?? []), ...blockedByIds])) : undefined; /* FNXC:HonestBlockedExit 2026-08-01-01:40 (operator: FN-8634 "shouldn't show a failed badge"): When `blockedBy` is EMPTY there is nothing external to wait for — the block is a defect in the PLAN (contradictory requirements, impossible step), and the designed recovery is a replan, which the planner overseer was already performing after the alarm: park failed → overseer "stage=executor signal=failed" → steer to replan. Skip the alarm: park directly as `needs-replan` (the graph's durable replan signal) with the BLOCKED reason in the log, and move to the replan column so triage re-admits it. A blocked exit WITH dependencies keeps the failed park — waiting on other tasks is a real park an operator may act on. The run-audit event fires for both shapes (`parkedAs` discriminates, ids/outcomes-only). */ const autoReplanPark = blockedByIds.length === 0; if (autoReplanPark) { const replanColumn = await resolveReplanTargetColumn(this.store, taskId); await store.logEntry( taskId, `${parkError} — no blocking dependencies recorded; parking for automatic replan in ${replanColumn} (steps preserved)`, undefined, this.getRunContextFor(taskId), ); this.workflowLifecycleMovesInFlight.add(taskId); try { await moveTaskToReplanColumn(this.store, { id: taskId, column: blockedTask.column }, replanColumn); } finally { this.workflowLifecycleMovesInFlight.delete(taskId); } await store.updateTask(taskId, { status: "needs-replan", error: null, paused: false, pausedByAgentId: null, }, this.getRunContextFor(taskId)); } else { await store.updateTask(taskId, { status: "failed", error: parkError, paused: false, pausedByAgentId: null, ...(mergedDependencies ? { dependencies: mergedDependencies } : {}), }, this.getRunContextFor(taskId)); await store.logEntry( taskId, `${parkError} — recorded dependencies: ${blockedByIds.join(", ")} — parked failed (honest blocked exit; steps preserved)`, undefined, this.getRunContextFor(taskId), ); } await this.store.recordRunAuditEvent?.({ taskId, agentId: "executor", runId: generateSyntheticRunId("execution-blocked", taskId), domain: "database", mutationType: "task:execution-blocked-parked", target: taskId, metadata: { taskId, blockedBy: blockedByIds, hasReason: true, parkedAs: autoReplanPark ? "auto-replan" : "failed", }, }); await this.persistTokenUsage(taskId); executorLog.log(`⛔ ${taskId} ${autoReplanPark ? "parked for automatic replan via blocked exit (no dependencies)" : `parked failed via blocked exit (blockedBy: ${blockedByIds.join(", ")})`}`); return { content: [{ type: "text" as const, text: autoReplanPark ? "Task parked as blocked with no blocking dependencies — queued for automatic replan so the plan can resolve the conflict. Steps left in their true statuses; no completion recorded." : `Task parked as blocked (failed). Recorded ${blockedByIds.length} blocking dependency(ies); it will requeue once they complete. Steps left in their true statuses; no completion recorded.`, }], details: {}, }; } const task = await store.getTask(taskId); const completionBlocker = await this.getTaskCompletionBlocker(task); if (completionBlocker) { return { content: [{ type: "text" as const, text: `Cannot mark task done yet — ${completionBlocker}. Resolve the blocker before calling fn_task_done().`, }], details: {}, }; } const providerVerdict = await this.evaluateTaskVerdictProviders(task, { summary: params.summary, source: "fn_task_done", }); if (!providerVerdict.ok) { await store.logEntry(taskId, providerVerdict.message, undefined, this.getRunContextFor(task.id)); executorLog.error(`${taskId}: ${providerVerdict.message}`); return { content: [{ type: "text" as const, text: providerVerdict.message }], details: { error: providerVerdict.message, }, }; } const noOpMarker = parseNoOpCompletionMarker(params.summary); const invariantCheck = await this.verifyWorktreeInvariants(task, worktreePath, true, { noOpCompletion: Boolean(noOpMarker), noOpCompletionReason: noOpMarker ? `verified ${noOpMarker.kind} completion sentinel${noOpMarker.canonicalId ? ` (${noOpMarker.canonicalId})` : ""}` : undefined, }); if (!invariantCheck.ok) { const refusalMessage = `fn_task_done refused: ${invariantCheck.reason} — observed=${invariantCheck.observed}, expected=${invariantCheck.expected}`; await store.logEntry(taskId, refusalMessage, undefined, this.getRunContextFor(task.id)); executorLog.error(`${taskId}: fn_task_done refused (${invariantCheck.reason}) — observed=${invariantCheck.observed}, expected=${invariantCheck.expected}`); const priorRequeues = task.taskDoneRetryCount ?? 0; const nextRequeueCount = priorRequeues + 1; if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) { await store.updateTask(taskId, { status: "queued", error: null, taskDoneRetryCount: nextRequeueCount, paused: false, pausedByAgentId: null, worktree: null, branch: null, sessionFile: null, }); await store.logEntry( taskId, `${refusalMessage} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`, undefined, this.getRunContextFor(task.id), ); await store.moveTask(taskId, await resolveReboundColumnFor(store, taskId), { preserveProgress: true }); executorLog.log(`✗ ${taskId} failed invariant check — requeued to todo (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`); } else { await store.updateTask(taskId, { status: "failed", error: refusalMessage, paused: false, pausedByAgentId: null, worktree: null, branch: null, sessionFile: null, }); await store.logEntry(taskId, `${refusalMessage} — invariant-check retry budget exhausted`, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(taskId); executorLog.log(`✗ ${taskId} failed invariant check`); } return { content: [{ type: "text" as const, text: refusalMessage }], details: { error: refusalMessage, }, }; } const taskDoneRefusal = evaluateTaskDoneRefusal(task, params, codeReviewVerdicts); if (!taskDoneRefusal.ok) { const refusalMessage = taskDoneRefusal.message; await store.logEntry(taskId, refusalMessage, undefined, this.getRunContextFor(task.id)); executorLog.error(`${taskId}: fn_task_done refused (${taskDoneRefusal.refusalClass}) — ${taskDoneRefusal.reason}`); // FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — stamp the skip-bypass taint marker so a // later skip-then-exit (in this or a requeued lifecycle) cannot auto-promote. const taintUpdate = this.skipBypassTaintUpdateForRefusal(taskDoneRefusal); const priorRequeues = task.taskDoneRetryCount ?? 0; const nextRequeueCount = priorRequeues + 1; if (priorRequeues < MAX_TASK_DONE_REQUEUE_RETRIES) { await store.updateTask(taskId, { status: "queued", error: null, taskDoneRetryCount: nextRequeueCount, ...taintUpdate, paused: false, pausedByAgentId: null, worktree: null, branch: null, sessionFile: null, }); await store.logEntry( taskId, `${refusalMessage} — requeued to todo immediately (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`, undefined, this.getRunContextFor(task.id), ); await store.moveTask(taskId, await resolveReboundColumnFor(store, taskId), { preserveProgress: true }); executorLog.log(`✗ ${taskId} fn_task_done refusal (${taskDoneRefusal.refusalClass}) — requeued to todo (${nextRequeueCount}/${MAX_TASK_DONE_REQUEUE_RETRIES})`); } else { await store.updateTask(taskId, { status: "failed", error: refusalMessage, ...taintUpdate, paused: false, pausedByAgentId: null, worktree: null, branch: null, sessionFile: null, }); await store.logEntry(taskId, `${refusalMessage} — fn_task_done refusal retry budget exhausted`, undefined, this.getRunContextFor(task.id)); await this.persistTokenUsage(taskId); executorLog.log(`✗ ${taskId} fn_task_done refusal (${taskDoneRefusal.refusalClass})`); } return { content: [{ type: "text" as const, text: refusalMessage }], details: { error: refusalMessage, refusalClass: taskDoneRefusal.refusalClass, }, }; } // Merge per-task effective workflow settings (U3, KTD-3) so the // planOnlyScopeLeakEnforcement read in evaluateTaskDoneScopeLeak picks up // workflow values. Behavior-inert by default. const settings = await mergeEffectiveSettings(store, task, await store.getSettings()); const scopeLeakCheck = await this.evaluateTaskDoneScopeLeak(task, worktreePath, promptContent, settings, audit) .catch((error: unknown) => { const errorMessage = error instanceof Error ? error.message : String(error); executorLog.warn(`${taskId}: scope-leak guard failed open: ${errorMessage}`); return { blocked: false } as const; }); if (scopeLeakCheck.blocked) { await store.logEntry(taskId, `[scope-leak] blocked fn_task_done: ${scopeLeakCheck.message}`, undefined, this.getRunContextFor(task.id)); return { content: [{ type: "text" as const, text: scopeLeakCheck.message }], details: { error: scopeLeakCheck.message, }, }; } if (noOpMarker) { const runContext = this.getRunContextFor(taskId); await store.updateTask(taskId, { noCommitsExpected: true }); await store.logEntry( taskId, `Verified ${noOpMarker.kind} completion sentinel accepted; no commits expected for terminal handoff`, JSON.stringify({ kind: noOpMarker.kind, reason: noOpMarker.reason, canonicalId: noOpMarker.canonicalId, summary: params.summary, runId: runContext?.runId, agentId: runContext?.agentId, }), runContext, ); const recordActivity = (store as typeof store & { recordActivity?: (entry: { type: "task:updated"; taskId: string; taskTitle?: string; details: string; metadata?: Record; }) => Promise; }).recordActivity; if (recordActivity) { await recordActivity.call(store, { type: "task:updated", taskId, taskTitle: task.title, details: `Task marked as verified ${noOpMarker.kind}; no commits expected`, metadata: { taskId, kind: noOpMarker.kind, reason: noOpMarker.reason, canonicalId: noOpMarker.canonicalId, summary: params.summary, runId: runContext?.runId, agentId: runContext?.agentId, }, }).catch((error: unknown) => { executorLog.warn(`${taskId}: failed to record no-op completion activity: ${error instanceof Error ? error.message : String(error)}`); }); } } onDone(); // Mark all pending/in-progress steps as done for (let i = 0; i < task.steps.length; i++) { if (task.steps[i].status !== "done" && task.steps[i].status !== "skipped") { await store.updateStep(taskId, i, "done"); } } // FN-4106: preserve the original completion summary on workflow-step reruns. const newSummary = params.summary?.trim(); if (newSummary) { const currentTask = await store.getTask(taskId); const existingSummary = currentTask.summary?.trim(); const hasRunWorkflowSteps = (currentTask.workflowStepResults?.length ?? 0) > 0; const rerunSuffix = `---\nRerun after workflow step revision:\n${newSummary}`; if (existingSummary && hasRunWorkflowSteps && !existingSummary.endsWith(rerunSuffix)) { await store.updateTask(taskId, { summary: `${currentTask.summary}\n\n${rerunSuffix}`, }); await store.logEntry(taskId, "fn_task_done summary appended to existing summary (workflow-step rerun)", undefined, this.getRunContextFor(taskId)); } else if (!existingSummary || !hasRunWorkflowSteps) { await store.updateTask(taskId, { summary: params.summary }); } } const hardPauseActive = Boolean(settings.globalPause); // Task-level pause prevents new work from starting, not completion of // in-flight work. Always clear it on explicit agent completion so the // board cannot strand a completed task in a paused state. await store.updateTask(taskId, { paused: false, pausedByAgentId: null, status: null, // FNXC:Lifecycle 2026-07-16-21:40: FN-8141 — an ACCEPTED explicit fn_task_done is the // honest completion signal (covers the PREMISE STALE skip-then-done flow); clear any // skip-bypass taint so a subsequent auto-promotion path is not blocked. bulkCompletionRefusalAt: null, }); await store.logEntry(taskId, "Task marked done by agent", undefined, this.getRunContextFor(taskId)); const latestTask = await store.getTask(taskId); let latestColumn = latestTask.column; if (latestColumn === await resolveReboundColumnFor(store, taskId)) { await store.logEntry( taskId, hardPauseActive ? "fn_task_done called while task was in todo during pause — promoting to in-progress for deferred completion handoff" : "fn_task_done called while task was in todo — promoting to in-progress before completion handoff", undefined, this.getRunContextFor(taskId), ); /* FNXC:WorkflowResolvedColumns 2026-07-30-21:40: census-invisible moveTask DESTINATION, and `latestColumn` must be set from the SAME resolved value or the check below it compares against a lane the card is not in. */ const wipTarget = await resolveWipTargetForTask(store, taskId); await store.moveTask(taskId, wipTarget); latestColumn = wipTarget; } /* FNXC:WorkflowResolvedColumns 2026-07-31-09:20 (fleet: executor lifecycle roles): The completed-task watchdog arms when the card is in its IMPLEMENTATION lane. Naming `in-progress` literally meant a renamed wip column never armed it — a watchdog that silently never fires, on exactly the boards this program converted. The branch directly above already resolves that lane through `resolveWipTargetForTask`; this asks the same question of the same resolver rather than of an id. */ if (latestColumn === await resolveWipTargetForTask(store, taskId) && !hardPauseActive) { this.scheduleCompletedTaskWatchdog(taskId, "fn_task_done"); } const successMessage = hardPauseActive ? "Task marked complete. Completion handoff deferred until pause is cleared." : params.summary ? "Task marked complete with summary. All steps done. Moving to in-review." : "Task marked complete. All steps done. Moving to in-review."; return { content: [{ type: "text" as const, text: successMessage }], details: {}, }; }, }; } /** * Clean up after a dep-abort: remove worktree, delete branch, move task to triage. * Shared between the try-block (graceful return) and catch-block (error) paths. */ private async handleDepAbortCleanup(taskId: string, worktreePath: string): Promise { executorLog.log(`${taskId} dependency added — work discarded, moved to triage for re-planning`); // Remove worktree try { const settings = await this.store.getSettings(); await this.removeOwnWorktreeWithReconcile({ worktreePath, settings, taskId, reason: RemovalReason.ExecutorDispose, }); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: failed to remove worktree during dep-abort cleanup (${worktreePath}): ${msg}`); } // Delete the branch — use stored branch name if available, fall back to convention const task = await this.store.getTask(taskId); const branch = resolveTaskWorkingBranch(task); let branchDeleted = false; try { await execAsync(`git branch -D "${branch}"`, { cwd: this.rootDir }); branchDeleted = true; } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: failed to delete branch during dep-abort cleanup (${branch}): ${msg}`); } if (branchDeleted) { // FN-2165 regression guard: null baseBranch on any task that stored this branch try { await this.store.clearStaleExecutionStartBranchReferences([branch], taskId); } catch { /* best-effort */ } } // Clear worktree tracking this.activeWorktrees.delete(taskId); // Update task: clear worktree and status, move to triage await this.store.updateTask(taskId, { worktree: null, status: null }); /* FNXC:WorkflowLifecycleColumns 2026-07-29-15:10 (P0 audit after the Planning-column merge): This wrote the LITERAL `triage`. The default coding lineage no longer declares that column — it has one pre-implementation column, id `todo` — so a card that gained a dependency mid-execution had its work discarded and was then parked in a column its own workflow does not define. Nothing in the graph routes a card out of an undeclared column, and the only rescue is `reconcileUndeclaredTaskColumns` on the NEXT ENGINE START, so between the abort and a restart the card is stalled with no automatic recovery. It does not throw, which is why it would have surfaced as a user report rather than a red test. Resolve the rebound target from the task's own workflow (hold -> intake -> first declared column), the same helper the other ~16 executor rebounds already use. */ await this.store.moveTask(taskId, await resolveReboundColumnFor(this.store, taskId)); await this.store.logEntry(taskId, "Execution stopped — work discarded, requeued for re-planning"); } /** * Re-open the implementation-bearing slice of work for a revision/failure * handler. Returns the earliest reopened step and all reopened indexes, or * null when there was nothing to re-open. */ private async reopenLastStepForRevision( taskId: string, task: Task, ): Promise<{ index: number; name: string; indexes: number[] } | null> { const steps = task.steps; if (steps.length === 0) return null; let lastNonPendingIndex = -1; for (let i = steps.length - 1; i >= 0; i--) { if (steps[i].status !== "pending") { lastNonPendingIndex = i; break; } } if (lastNonPendingIndex === -1) { await this.store.updateTask(taskId, { currentStep: 0 }); return null; } // Match step-title words rather than arbitrary substrings so an implementation step // like "DataVerificationLayer" is not treated as a trailing delivery/check step. const isTerminalVerificationOrDeliveryStep = (name: string): boolean => /(^|[^a-z])(testing|verification|documentation|delivery)([^a-z]|$)/i.test(name); const resetIndexes = new Set([lastNonPendingIndex]); if (isTerminalVerificationOrDeliveryStep(steps[lastNonPendingIndex].name)) { let cursor = lastNonPendingIndex; while (cursor >= 0 && isTerminalVerificationOrDeliveryStep(steps[cursor].name)) { resetIndexes.add(cursor); cursor--; } while (cursor >= 0 && steps[cursor].status === "pending") { cursor--; } if (cursor >= 0) { resetIndexes.add(cursor); } } const indexes = [...resetIndexes].sort((a, b) => a - b); /* FNXC:WorkflowOptionalStepFix 2026-06-27-18:03: Code Review / Browser Verification REVISE bounces must reopen the step that can actually make the requested code change, not only a trailing Documentation & Delivery or Testing & Verification step. Otherwise the graph rerun can complete a trivial terminal step, re-evaluate the optional group against unchanged code, and loop or strand pending work. Reopen the trailing verification/delivery suffix plus the nearest preceding implementation step so both in-progress and in-review bounce sources re-launch execution on actionable work before optional-step re-evaluation. */ for (const index of indexes) { if (steps[index].status !== "pending") { await this.store.updateStep(taskId, index, "pending"); } } const currentStep = indexes[0] ?? lastNonPendingIndex; await this.store.updateTask(taskId, { currentStep }); return { index: currentStep, name: steps[currentStep].name, indexes }; } /** * Run deterministic verification (test + build commands) in the task's worktree. * Returns a structured result indicating whether all commands passed. */ private async runExecutorDeterministicVerification( task: Task, worktreePath: string, settings: Settings, extraEnv?: NodeJS.ProcessEnv, ): Promise { const testCommand = settings.testCommand?.trim(); const buildCommand = settings.buildCommand?.trim(); if (!testCommand && !buildCommand) { executorLog.debug(`${task.id}: no test/build commands configured — skipping verification`); return { allPassed: true }; } const parts: string[] = []; if (testCommand) parts.push(`test: ${testCommand}`); if (buildCommand) parts.push(`build: ${buildCommand}`); // FNXC:EngineDiagnostics 2026-07-26-09:33: green path verification start/pass is expected work — debug so failures stay prominent. executorLog.debug(`${task.id}: [verification] running deterministic verification (${parts.join(", ")})`); await this.store.logEntry( task.id, `[verification] Running deterministic verification (${parts.join(", ")})`, undefined, this.getRunContextFor(task.id), ); const result: VerificationResult = { allPassed: true }; // Run test command first if configured if (testCommand) { const testResult = await runVerificationCommand( this.store, worktreePath, task.id, testCommand, "test", undefined, executorLog, "executor", extraEnv, settings.verificationCommandTimeoutMs, ); result.testResult = testResult; if (!testResult.success) { result.allPassed = false; result.failedCommand = "testCommand"; executorLog.log(`${task.id}: [verification] test failed (exit ${testResult.exitCode})`); return result; } } // Run build command second if configured if (buildCommand) { const buildResult = await runVerificationCommand( this.store, worktreePath, task.id, buildCommand, "build", undefined, executorLog, "executor", extraEnv, settings.verificationCommandTimeoutMs, ); result.buildResult = buildResult; if (!buildResult.success) { result.allPassed = false; result.failedCommand = "buildCommand"; executorLog.log(`${task.id}: [verification] build failed (exit ${buildResult.exitCode})`); return result; } } executorLog.debug(`${task.id}: [verification] passed`); await this.store.logEntry( task.id, `[verification] Deterministic verification passed`, undefined, this.getRunContextFor(task.id), ); return result; } /** * Attempt to fix verification failures by spawning a dedicated AI fix agent. * Follows the pattern established by the merger's attemptInMergeVerificationFix. * Returns true if verification passes after the fix attempt, false otherwise. */ private async attemptExecutorVerificationFix( task: Task, worktreePath: string, failureContext: { command: string; exitCode: number | null; output: string; type: "test" | "build"; }, settings: Settings, retryNumber: number, maxRetries: number, extraEnv?: NodeJS.ProcessEnv, ): Promise { try { executorLog.log(`${task.id}: spawning executor verification fix agent (attempt ${retryNumber}/${maxRetries})`); const logger = new AgentLogger({ store: this.store, taskId: task.id, agent: "executor", persistAgentToolOutput: settings.persistAgentToolOutput, // Executor sessions are task-scoped ephemeral workers. persistAgentThinkingLog: resolvePersistAgentThinkingLog(settings, { ephemeral: true }), onAgentText: this.options.onAgentText, onAgentTool: this.options.onAgentTool, }); // Build skill selection context let skillContext: Awaited> | undefined; if (this.options.agentStore) { try { skillContext = await buildSessionSkillContext({ agentStore: this.options.agentStore, task, sessionPurpose: "executor", projectRootDir: worktreePath, pluginRunner: this.options.pluginRunner, }); } catch { // Graceful fallback - no skill selection } } // Resolve model using the executor's model hierarchy const assignedRuntimeConfig = await this.getAssignedAgentRuntimeConfig(task.assignedAgentId); const executorSessionModel = resolveExecutorSessionModel( task.modelProvider, task.modelId, settings, assignedRuntimeConfig, task.credentialInstanceId, ); const { provider: executorProvider, modelId: executorModelId } = executorSessionModel; const executorFallback = resolveExecutorFallbackModel(settings); // Create the fix agent session const { session } = await createResolvedAgentSession({ sessionPurpose: "executor", pluginRunner: this.options.pluginRunner, cwd: worktreePath, // Run in the task's worktree systemPrompt: `You are a verification fix agent running during task execution in a worktree. All step-session steps completed successfully but the deterministic verification command failed. Your job is to fix the failing code directly in the working directory. ## Scope Only fix what is required to make the failing verification pass. Do not refactor, rename broadly, or make opportunistic improvements. ## Rules 1. Read the error output carefully to understand what is failing before editing anything 2. Before assuming a code fix is needed, check whether the failure is caused by stale/missing build artifacts in a sibling workspace package — typical signatures: \`Failed to resolve import "./X.js"\` pointing into another package's \`dist/\`, \`Cannot find module\`, or \`ERR_MODULE_NOT_FOUND\` referencing a workspace-internal path. In that case, rebuild the affected package(s) (e.g. \`pnpm --filter build\`, or \`pnpm --filter "/*" build\` for a group) and re-run verification before editing source files. 3. Make targeted fixes to the failing code path 4. After fixing, run the verification command to confirm the fix works 5. Do NOT make any git commits — just fix the code 6. You MAY modify any files needed to make the verification pass, including files unrelated to this task's original change. Pre-existing build/test breakage is in scope: fix it. Prefer the smallest change that makes verification green. 7. If you cannot fix the issue within scope, explain why and what evidence indicates a deeper/root problem`, tools: "coding", onText: logger.onText, onThinking: logger.onThinking, onToolStart: logger.onToolStart, onToolEnd: logger.onToolEnd, defaultProvider: executorProvider, defaultModelId: executorModelId, ...(executorSessionModel.credentialInstanceId ? { credentialInstanceId: executorSessionModel.credentialInstanceId } : {}), fallbackProvider: executorFallback.provider, fallbackModelId: executorFallback.modelId, fallbackThinkingLevel: resolveExecutorFallbackThinkingLevel(task.thinkingLevel, settings), defaultThinkingLevel: resolveExecutorThinkingLevel(task.thinkingLevel, settings), runAuditor: createRunAuditor(this.store, this.getRunContextFor(task.id)), settings, taskEnv: extraEnv, mcpServers: await this.resolveMcpServers(undefined), // FNXC:SessionRouting 2026-06-24-11:20: // #1675: propagate task id so verification-fix requests carry the same // X-Session-Id/X-Session-Affinity as the primary session. taskId: task.id, // FNXC:PluginSkills 2026-07-12-00:00: Verification-fix sessions share task skill selection; include plugin skill body dirs so fixes can use plugin-authored guidance. ...(skillContext?.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}), ...(skillContext && skillContext.additionalSkillPaths.length > 0 ? { additionalSkillPaths: skillContext.additionalSkillPaths } : {}), }); await this.store.logEntry( task.id, `Executor verification fix agent started (model: ${describeModel(session)}, attempt ${retryNumber}/${maxRetries})`, undefined, this.getRunContextFor(task.id), ); await this.store.appendAgentLog( task.id, `Fix agent started (model: ${describeModel(session)}, attempt ${retryNumber}/${maxRetries})`, "status", undefined, "executor", ); try { // Build the fix prompt const fixPrompt = `Fix the failing ${failureContext.type} verification for task ${task.id}. ## Failed command Command: \`${failureContext.command}\` Exit code: ${failureContext.exitCode} ## Error output ${failureContext.output.slice(0, VERIFICATION_LOG_MAX_CHARS)} ## Instructions 1. Read the error output and identify the root cause 2. Make targeted fixes to resolve the failure 3. Run the verification command \`${failureContext.command}\` to confirm your fix works 4. If the fix doesn't work, try a different approach 5. Do NOT make any git commits`; // Run the agent with rate limit retry await withRateLimitRetry(async () => { await promptWithFallback(session, fixPrompt); }, { onRetry: (attempt, delayMs, error) => { const delaySec = Math.round(delayMs / 1000); executorLog.warn(`⏳ ${task.id} executor fix agent rate limited — retry ${attempt} in ${delaySec}s: ${error.message}`); }, }); await accumulateSessionTokenUsage(this.store, task.id, session, { agentId: task.assignedAgentId ?? undefined, role: "executor", }); // Re-run full deterministic verification (test AND build) after the fix attempt executorLog.log(`${task.id}: re-running deterministic verification after fix attempt ${retryNumber}/${maxRetries}`); await this.store.logEntry( task.id, `Re-running deterministic verification (attempt ${retryNumber}/${maxRetries})`, undefined, this.getRunContextFor(task.id), ); await this.store.appendAgentLog( task.id, `Re-running verification (attempt ${retryNumber}/${maxRetries})`, "status", undefined, "executor", ); const reRunResult = await this.runExecutorDeterministicVerification(task, worktreePath, settings, extraEnv); return reRunResult.allPassed; } finally { await logger.flush(); session.dispose(); } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`${task.id}: executor verification fix agent error: ${errorMessage}`); await this.store.logEntry( task.id, `Executor verification fix agent encountered an error`, errorMessage, this.getRunContextFor(task.id), ); await this.store.appendAgentLog( task.id, "Fix agent encountered an error", "tool_error", errorMessage, "executor", ); return false; } } /** * Send a task back to in-progress after verification failure. * Injects failure feedback into PROMPT.md, resets steps, clears session, * and schedules a move to todo → in-progress after the executing guard clears. */ private async sendTaskBackForFix( task: Task, worktreePath: string, failureFeedback: string, stepName: string, reason: string, preserveResumeState: boolean = true, mergeVerificationFailure: boolean = false, retryPresentation?: { attempt: number; max?: number }, ): Promise { const taskId = task.id; this.clearCompletedTaskWatchdog(taskId); // 1. Add a task comment explaining the failure await this.store.addTaskComment( taskId, `${reason}. The failing workflow step was "${stepName}". ` + `Feedback:\n${failureFeedback}\n\n` + `Please fix the issues so the verification can pass on the next attempt.`, "agent", ); // 2. Log an entry explaining the task was sent back await this.store.logEntry( taskId, `${reason} — moved back to in-progress for remediation`, ); /* * FNXC:CodeReviewRetryBudget 2026-07-22-00:00: * A graph-owned Code Review REVISE is not a workflow-step hard-failure retry. * Preserve its resolved per-step budget in PROMPT.md: unset Code Review policy * is unlimited, while an explicit finite value (including zero at the gate) * remains operator-visible. The execute requeue progress-signature guard, not * this display, remains the safety boundary for unchanged remediation loops. */ await this.injectWorkflowStepFailureInstructions( task, failureFeedback, stepName, retryPresentation ?? { attempt: MAX_WORKFLOW_STEP_RETRIES, max: MAX_WORKFLOW_STEP_RETRIES }, ); // 4. Re-open only the last step for a single in-place fix pass. Earlier // done steps stay done so the executor doesn't redo finished work. const updatedTask = await this.store.getTask(taskId); await this.reopenLastStepForRevision(taskId, updatedTask); // 5. Clear error/status/session fields and reset workflow step retries. // FNXC:ReviewLeniency 2026-07-02-02:10: prior terminal failure results // (incl. optional gate nodes like code-review) are cleared by the rerun // bounce AFTER the task leaves the mergeable in-review column (see // clearTerminalStepFailuresForRetry), NOT here — clearing them while the // task is still in-review would drop the merge blocker during the async // bounce window and let a concurrent auto-merge sweep merge an // empty-`steps` graph-native task with the gate failure unaddressed. await this.store.updateTask(taskId, { status: mergeVerificationFailure ? "merging-fix" : null, error: null, sessionFile: null, workflowStepRetries: 0, }); // 6. Schedule the move after the guard unwinds (per guard-unwind requirement) this.scheduleWorkflowRerun( taskId, worktreePath, `${taskId}: sent back to in-progress for remediation`, preserveResumeState, ); } /** * Inject or update the "Workflow Step Failure" section in PROMPT.md. * This section contains failure feedback from workflow steps that hard-failed. * The section is replaced entirely to avoid accumulation of old feedback. */ private async injectWorkflowStepFailureInstructions( task: Task, failureFeedback: string, stepName: string, retry: { attempt: number; max?: number }, ): Promise { const promptPath = join(this.store.getFusionDir(), "tasks", task.id, "PROMPT.md"); // Read existing PROMPT.md let content: string; try { content = await readFile(promptPath, "utf-8"); } catch { executorLog.warn(`${task.id}: PROMPT.md not found at ${promptPath}, skipping workflow failure injection`); return; } const retryLabel = retry.max === undefined ? "unbounded" : String(retry.max); const remainingRetries = retry.max === undefined ? "unlimited" : String(Math.max(0, retry.max - retry.attempt)); const failureSectionHeader = "## Workflow Step Failure"; const scopeGuard = this.buildWorkflowFailureScopeGuard(task, content); const failureSectionContent = `${failureSectionHeader} The following workflow step failed and requires implementation fixes: **Step:** ${stepName} **Failure Feedback:** ${failureFeedback} ${scopeGuard} **Retry:** ${retry.attempt}/${retryLabel} (${remainingRetries} remaining) **Important:** This is a workflow step failure — fix the issues above by making the necessary code changes. The task has been sent back to in-progress for remediation. The executor will attempt to fix the issues on the next pass. `; let newContent: string; if (content.includes(failureSectionHeader)) { // Replace existing section const sectionRegex = new RegExp( `${failureSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`, "i" ); if (sectionRegex.test(content)) { newContent = content.replace(sectionRegex, failureSectionContent); } else { // Fallback: append at end newContent = content + "\n" + failureSectionContent; } } else { // Remove any existing Workflow Revision Instructions section first (conflicting state) const revisionSectionHeader = "## Workflow Revision Instructions"; if (content.includes(revisionSectionHeader)) { const revisionRegex = new RegExp( `${revisionSectionHeader}[\\s\\S]*?(?=\\n## |\\n# |$)`, "i" ); content = content.replace(revisionRegex, ""); } // Append new section before any closing markers or at end const acceptanceCriteriaMatch = content.match(/\n##\s+Acceptance Criteria\n/); if (acceptanceCriteriaMatch) { const insertIdx = acceptanceCriteriaMatch.index!; newContent = content.slice(0, insertIdx) + "\n" + failureSectionContent + content.slice(insertIdx); } else { newContent = content + "\n" + failureSectionContent; } } // Write updated content try { await writeFile(promptPath, newContent); executorLog.log(`${task.id}: injected workflow step failure instructions into PROMPT.md (retry ${retry.attempt}/${retryLabel})`); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`${task.id}: failed to inject workflow step failure instructions: ${errorMessage}`); } } private buildWorkflowFailureScopeGuard(task: Task, promptContent: string): string { const promptScopeEntries = extractPromptListEntries(extractPromptSection(promptContent, "File Scope")); const metadataScope = Array.isArray(task.sourceMetadata?.fileScope) ? task.sourceMetadata.fileScope.filter((entry): entry is string => typeof entry === "string") : []; const declaredScope = Array.from(new Set([...promptScopeEntries, ...metadataScope].map((entry) => entry.trim()).filter(Boolean))); /* * FNXC:WorkflowRemediationScope 2026-06-29-13:56: * Review remediation must not let one task silently implement unrelated behavior. If reviewer feedback points outside the declared File Scope, the executor should remove/split the unrelated work instead of expanding the task, while still allowing already-scoped fixes to proceed automatically. */ if (declaredScope.length === 0) { return "**Scope Guard:** Keep remediation limited to this task's stated mission and existing implementation surface. If the feedback requires unrelated behavior, remove or split that work instead of implementing it here."; } return [ "**Scope Guard:** Treat the declared File Scope as the remediation boundary. Fix only the scoped files unless PROMPT.md already authorizes a scope expansion. If the feedback requires unrelated behavior outside this scope, remove those unrelated changes or split them into a separate task instead of implementing them here.", "", "**Declared File Scope:**", ...declaredScope.map((entry) => `- ${entry}`), ].join("\n"); } private async captureBaseCommitSha( task: Task, worktreePath: string, audit: { git: (event: { type: "commit:create"; target: string; metadata: Record }) => Promise }, options: { isResume: boolean } = { isResume: false }, ): Promise { try { // Preserve an existing baseCommitSha only on RESUME of the same // worktree, where diff-base stability across sessions of the same task // matters. On fresh/pooled acquisitions the branch was just // force-reset to current main, so any stored baseCommitSha is by // definition behind the new merge-base — preserving it would yield // stale diff math and (when reused as a contamination reference) the // FN-4417 false-positive cascade. Always recapture on non-resume. if (options.isResume && task.baseCommitSha) { try { execSync(`git merge-base --is-ancestor ${task.baseCommitSha} HEAD`, { cwd: worktreePath, stdio: "pipe", }); executorLog.log(`${task.id}: preserved baseCommitSha ${task.baseCommitSha.slice(0, 7)} (resume)`); await audit.git({ type: "commit:create", target: task.baseCommitSha, metadata: { purpose: "base", preserved: true }, }); return; } catch { // Existing baseCommitSha is stale or invalid. Recapture below. } } const baseCommitSha = await resolveCapturedBaseCommitSha(worktreePath, { warn: (msg) => executorLog.warn(`${task.id}: ${msg}`), }); if (!baseCommitSha) { throw new Error("could not resolve base commit SHA"); } await this.store.updateTask(task.id, { baseCommitSha }); executorLog.log(`${task.id}: captured baseCommitSha ${baseCommitSha.slice(0, 7)}`); await audit.git({ type: "commit:create", target: baseCommitSha, metadata: { purpose: "base", preserved: false } }); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.debug(`Failed to capture baseCommitSha for ${task.id}: ${errorMessage}`); // Non-fatal: task can continue without baseCommitSha } } /** * Resolve a fresh merge-base against the integration branch for use as a * contamination check reference. Unlike {@link resolveDiffBaseRef}, this * NEVER falls back to `task.baseCommitSha`, because a stale stored base * would make the contamination check flag every legitimately-merged commit * since that snapshot as "foreign" (FN-4417). It also never falls back to * `HEAD~1`, because for a newly force-reset pooled branch HEAD~1 is a * commit on main itself, which would yield the same false positive on a * smaller scale. * * Returns `undefined` when neither `origin/main` nor `main` is resolvable; * the caller is expected to treat that as "contamination check skipped". */ private async resolveContaminationBaseRef(worktreePath: string): Promise { // Prefer LOCAL main over origin/main. origin/main is a tracking ref that // is only as fresh as the last `git fetch` — on dev machines that haven't // pushed in a while it can lag local main by hundreds of commits, which // re-introduces the FN-4417 false positive at a smaller scale (the // merge-base falls back to the last common ancestor between HEAD and the // stale origin/main, and every commit on local main since then looks // "foreign"). Local main is the canonical integration target for Fusion. try { const { stdout } = await execAsync( "git merge-base HEAD main 2>/dev/null || git merge-base HEAD origin/main", { cwd: worktreePath, encoding: "utf-8" }, ); const ref = stdout.trim(); return ref || undefined; } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed merge-base lookup for contamination check in ${worktreePath}: ${errorMessage}`); return undefined; } } /** * Capture the list of files modified during agent execution. * Uses git diff against the stored baseCommitSha to determine what changed. * Returns an empty array if no changes or if git commands fail. */ private async resolveDiffBaseRef(worktreePath: string, baseCommitSha?: string): Promise { if (baseCommitSha) return baseCommitSha; try { const { stdout } = await execAsync( "git merge-base HEAD origin/main 2>/dev/null || git merge-base HEAD main", { cwd: worktreePath, encoding: "utf-8" }, ); const ref = stdout.trim(); if (ref) return ref; } catch (mergeBaseErr: unknown) { const mergeBaseMsg = mergeBaseErr instanceof Error ? mergeBaseErr.message : String(mergeBaseErr); executorLog.warn(`Failed merge-base lookup for diff base in ${worktreePath}, trying HEAD~1 fallback: ${mergeBaseMsg}`); } try { const { stdout } = await execAsync("git rev-parse HEAD~1", { cwd: worktreePath, encoding: "utf-8", }); return stdout.trim() || undefined; } catch { executorLog.debug(`Could not determine base commit for diff in ${worktreePath}`); return undefined; } } private async captureModifiedFiles( worktreePath: string, baseCommitSha: string | undefined, taskId: string, audit?: RunAuditor, source = "unspecified", ): Promise { try { const baseRef = await this.resolveDiffBaseRef(worktreePath, baseCommitSha); if (!baseRef) { return []; } try { const attributed = await filterFilesToOwnTaskCommits({ worktreePath, baseRef, taskId, }); const divergence = attributed.rawDiffFileCount - attributed.files.length; if (divergence > 0) { await audit?.database({ type: "task:worktree-contamination-detected", target: taskId, metadata: { rawDiffFileCount: attributed.rawDiffFileCount, attributedFileCount: attributed.files.length, foreignCommitCount: attributed.foreignCommits.length, foreignCommitShas: attributed.foreignCommits.slice(0, 5).map((commit) => commit.sha), source, }, }); executorLog.warn( `${taskId}: contamination detected — raw diff ${attributed.rawDiffFileCount} files, attributed ${attributed.files.length} (foreign commits: ${attributed.foreignCommits.length})`, ); } return attributed.files; } catch (error) { if (error instanceof BranchAttributionError) { executorLog.warn(`${taskId}: branch-attribution failed (${error.message}); falling back to raw diff`); const { stdout } = await execAsync(`git diff --name-only ${baseRef}..HEAD`, { cwd: worktreePath, encoding: "utf-8", }); const output = stdout.trim(); return output ? output.split("\n").filter(Boolean) : []; } throw error; } } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.debug(`Failed to capture modified files: ${errorMessage}`); return []; } } /** * FNXC:Workspace 2026-06-21-23:30: KTD1 — per-repo modified-file capture for workspace tasks. * Loops `task.workspaceWorktrees` and REUSES `captureModifiedFiles` per sub-repo (NOT a hand-built `git diff`), so each repo gets: (a) resolveDiffBaseRef's merge-base fallback when repo.baseCommitSha is undefined, and (b) the filterFilesToOwnTaskCommits raw-vs-attributed divergence/contamination audit for free. Returned files are repo-prefixed (`/`) and aggregated, so a downstream File-Scope check / merge can attribute each change to its sub-repo. Returns [] for a zero-acquire workspace task. */ private async captureWorkspaceModifiedFiles( task: Task, audit?: RunAuditor, source = "post-session", ): Promise { const workspaceWorktrees = task.workspaceWorktrees ?? {}; // FNXC:Workspace 2026-06-21-15:00: F4/F6 — per-repo error isolation + deterministic ordering. // F4: an unexpected throw from one repo's `captureModifiedFiles` must NOT escape and skip the // downstream `updateTask({modifiedFiles})` write — that would leave `task.modifiedFiles` empty and // blind the merge file audit. Wrap each per-repo call (log + continue), mirroring the post-session // branch-attribution loop. F6: iterate sorted repo keys so aggregation order is stable across runs. const aggregated: string[] = []; for (const repoRel of Object.keys(workspaceWorktrees).sort()) { const repo = workspaceWorktrees[repoRel]; try { const repoFiles = await this.captureModifiedFiles(repo.worktreePath, repo.baseCommitSha, task.id, audit, source); for (const file of repoFiles) { aggregated.push(`${repoRel}/${file}`); } } catch (repoErr: unknown) { executorLog.warn(`${task.id}: per-repo modified-file capture failed for ${repoRel}: ${repoErr instanceof Error ? repoErr.message : String(repoErr)}`); } } return aggregated; } /** * FNXC:Workspace 2026-06-22-00:30: KTD3 — per-repo review by looping the EXISTING single-cwd reviewStep. * The reviewer is an AGENT spawned with `cwd = worktree`, told (in prompt text, reviewer.ts) to run `git diff` * itself — it does NOT read a diff passed in code. So per-repo review = ONE reviewer agent per sub-repo. We keep * `reviewStep` single-cwd; the CALLERS loop. This helper is the shared loop+aggregate so both review entry points * (historically the deleted in-session review tool, now only the step-inversion `stepReview` seam) iterate * identically: it invokes the caller's * own `invokeForCwd(cwd)` once per acquired worktree (cwd = repo.worktreePath) and aggregates the repo-tagged * verdicts as a CONJUNCTION — the task is "reviewed" only if EVERY repo passes; the FIRST non-APPROVE repo's * verdict becomes the aggregate verdict (mirroring verifyWorktreeInvariants' first-failing-repo return), and its * findings are repo-tagged. A zero-acquire workspace task (empty map) returns UNAVAILABLE so the caller routes it * rather than fabricating an APPROVE. * * Verdict severity for the conjunction: any RETHINK/REVISE/UNAVAILABLE fails the whole review; only all-APPROVE * (or all-skipped UNAVAILABLE-advisory, handled by the caller) approves. We surface the first failing repo's exact * verdict so the caller's existing verdict→edge mapping (APPROVE done-marking, REVISE block, RETHINK reset, * UNAVAILABLE retry) is unchanged. */ private async reviewWorkspacePerRepo( // FNXC:Workspace 2026-06-21-15:00: F7 — drop the dead `repoRel` callback param. // Both call sites bind `(cwd) => runForCwd(cwd)` and discard the second arg, so the type wrongly // implied repo identity is observable inside `runForCwd`. Removed until a real consumer needs it // (Phase C). The loop below still tags findings with `repoRel` from its own iteration key. task: Task, invokeForCwd: (cwd: string) => Promise, ): Promise { const workspaceWorktrees = task.workspaceWorktrees ?? {}; // FNXC:Workspace 2026-06-21-15:00: F6 — sort repo keys so the reported FIRST failing repo is // deterministic across runs/rehydrate. const repoKeys = Object.keys(workspaceWorktrees).sort(); if (repoKeys.length === 0) { // No acquired worktree — surface UNAVAILABLE so the caller routes it rather than // fabricating an authoritative APPROVE for an un-reviewable workspace task. return { verdict: "UNAVAILABLE", review: "No acquired sub-repo worktree to review (workspace task with zero worktrees).", summary: "Skipped: no sub-repo worktree", }; } const reviewSections: string[] = []; const summarySections: string[] = []; let firstFailing: { repo: string; result: ReviewResult } | undefined; for (const repoRel of repoKeys) { const repo = workspaceWorktrees[repoRel]; const result = await invokeForCwd(repo.worktreePath); // Tag every per-repo finding with its sub-repo so downstream readers attribute it correctly. reviewSections.push(`### [${repoRel}] ${result.verdict}\n${result.review}`); summarySections.push(`[${repoRel}] ${result.verdict}: ${result.summary}`); if (result.verdict !== "APPROVE") { // FNXC:Workspace 2026-06-21-15:00: F3 — BREAK on the first non-APPROVE repo. // The contract is "the FIRST non-APPROVE repo's verdict becomes the aggregate". Without the // break, a LATER repo's reviewer throwing would discard this already-determined REVISE/RETHINK // and the caller would see UNAVAILABLE — masking the real verdict. Stop at the first failure. firstFailing = { repo: repoRel, result }; break; } } if (firstFailing) { // Conjunction failed: the aggregate carries the FIRST failing repo's verdict (so the caller's // verdict→edge mapping is identical to single-cwd), with the full repo-tagged review body. return { verdict: firstFailing.result.verdict, // FNXC:Workspace 2026-06-22-00:00: the conjunction BREAKS on the first non-APPROVE repo, // so reviewSections holds only the repos evaluated up to (and including) the failure — not // every sub-repo. Label it honestly so operators don't read a partial list as exhaustive. review: `Workspace review failed in sub-repo \`${firstFailing.repo}\` (verdict ${firstFailing.result.verdict}). Per-repo verdicts (evaluation stopped at first failure; later repos not reviewed):\n\n${reviewSections.join("\n\n")}`, summary: `${firstFailing.repo}: ${firstFailing.result.verdict} — ${summarySections.join(" | ")}`, }; } // Every sub-repo approved → the task is reviewed (conjunction satisfied). return { verdict: "APPROVE", review: `All ${repoKeys.length} sub-repo(s) approved. Per-repo verdicts:\n\n${reviewSections.join("\n\n")}`, summary: `APPROVE across ${repoKeys.length} sub-repo(s): ${summarySections.join(" | ")}`, }; } private async captureUncommittedModifiedFiles(worktreePath: string): Promise { try { const [unstaged, staged] = await Promise.all([ execAsync("git diff --name-only", { cwd: worktreePath, encoding: "utf-8" }), execAsync("git diff --name-only --cached", { cwd: worktreePath, encoding: "utf-8" }), ]); const files = [...unstaged.stdout.split("\n"), ...staged.stdout.split("\n")] .map((entry) => entry.trim()) .filter(Boolean); return [...new Set(files)]; } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to capture uncommitted modified files: ${errorMessage}`); return []; } } // ── Worktree management ──────────────────────────────────────────── /** * Execute a script-mode workflow step by resolving the scriptName to a command * from project settings and running it in the task worktree. */ private async executeScriptWorkflowStep( task: Task, workflowStep: WorkflowStep, worktreePath: string, settings: Settings, extraEnv?: NodeJS.ProcessEnv, ): Promise<{ success: boolean; output?: string; error?: string }> { const scriptName = workflowStep.scriptName!.trim(); const scriptCommand = settings.scripts?.[scriptName]; if (!scriptCommand) { const available = settings.scripts ? Object.keys(settings.scripts).join(", ") : "none"; const msg = `Script '${scriptName}' not found in project settings. Available scripts: ${available}`; await this.store.logEntry(task.id, msg); return { success: false, error: msg }; } executorLog.log(`${task.id}: workflow step '${workflowStep.name}' executing script '${scriptName}': ${scriptCommand}`); await this.store.logEntry(task.id, `Workflow step '${workflowStep.name}' executing script '${scriptName}': ${scriptCommand}`); const scriptAbortController = new AbortController(); this.registerConfiguredCommandController(task.id, scriptAbortController); try { const scriptResult = await runConfiguredCommand( scriptCommand, worktreePath, 120_000, extraEnv, createRunAuditor(this.store, { runId: this.getRunContextFor(task.id)?.runId ?? generateSyntheticRunId("exec-script", task.id), agentId: this.getRunContextFor(task.id)?.agentId ?? (task.assignedAgentId ?? "executor"), taskId: task.id, phase: "execute", }), scriptAbortController.signal, ); if (scriptAbortController.signal.aborted) { throw this.createConfiguredCommandAbortError(task.id, scriptCommand); } if (scriptResult.spawnError || scriptResult.timedOut || scriptResult.exitCode !== 0) { return { success: false, error: configuredCommandErrorMessage(scriptResult) }; } return { success: true, output: `Script '${scriptName}' completed successfully` }; } catch (err: unknown) { if (err instanceof Error && err.name === "AbortError") { throw err; } const execError = err instanceof Error ? err : new Error(String(err)); const stderr = "stderr" in execError && typeof execError.stderr === "string" ? execError.stderr.trim() : ""; const stdout = "stdout" in execError && typeof execError.stdout === "string" ? execError.stdout.trim() : ""; const exitCode = "code" in execError ? execError.code : ("status" in execError ? execError.status : undefined); const parts: string[] = []; if (exitCode !== undefined) parts.push(`Exit code: ${exitCode}`); if (stdout) parts.push(`stdout: ${truncateWorkflowScriptOutput(stdout)}`); if (stderr) parts.push(`stderr: ${truncateWorkflowScriptOutput(stderr)}`); if (!parts.length) parts.push(execError.message || "Unknown error"); const errorOutput = parts.join("\n"); return { success: false, error: errorOutput }; } finally { this.unregisterConfiguredCommandController(task.id, scriptAbortController); } } /** Parse structured JSON verdict from workflow step output. */ private parseWorkflowStepOutput(rawOutput: string): { output: string; verdict?: "APPROVE" | "APPROVE_WITH_NOTES" | "REVISE"; notes?: string; malformed?: boolean; } { return parseWorkflowStepOutput(rawOutput); } private workflowInputRepliesAfterWatermark(task: TaskDetail, marker: string): Array<{ createdAt?: string }> { const pausedReason = task.pausedReason ?? ""; const watermark = (() => { const match = pausedReason.slice(marker.length).match(/^@(\d+)/); const parsed = match ? Number(match[1]) : NaN; return Number.isFinite(parsed) ? parsed : undefined; })(); const steering = Array.isArray(task.steeringComments) ? task.steeringComments : []; return watermark === undefined ? steering : steering.filter((comment) => { const created = Date.parse((comment as { createdAt?: string }).createdAt ?? ""); return Number.isFinite(created) ? created >= watermark : false; }); } private async resolveWorkflowInputMarkerForGraphNode(live: TaskDetail, nodeId: string): Promise<"clear" | "waiting" | "none"> { const pausedReason = live.pausedReason ?? ""; if (!pausedReason.startsWith("workflow-input:")) return "none"; const markerMatch = /^workflow-input:([^:@\s]+)(?:@\d+)?[:]/.exec(pausedReason); if (!markerMatch) return "none"; const marker = `workflow-input:${markerMatch[1]}`; const replies = this.workflowInputRepliesAfterWatermark(live, marker); if (live.paused || replies.length === 0) { await this.store.updateTask(live.id, { status: "awaiting-user-input", paused: true }, this.getRunContextFor(live.id)); return "waiting"; } /* * FNXC:WorkflowInput 2026-06-29-10:00: * A workflow graph can restart at an earlier node after pause/resume recovery while the durable pausedReason still points at the later skill node that asked the question. If the user already supplied a post-watermark reply, clear that stale marker before any node executes so Compound Engineering cannot loop at Plan while Commit & open PR's answered question remains attached. */ await this.store.updateTask(live.id, { status: null, pausedReason: null }, this.getRunContextFor(live.id)); await this.store.logEntry( live.id, marker === `workflow-input:${nodeId}` ? `Workflow input received for step '${nodeId}' — resuming` : `Workflow input marker '${markerMatch[1]}' already has a reply — clearing stale marker before step '${nodeId}'`, undefined, this.getRunContextFor(live.id), ); return "clear"; } /** * Execute a single workflow step by spawning an agent with the step's prompt. * Returns structured outcome with support for revision requests. */ private async executeWorkflowStep( task: Task, workflowStep: WorkflowStep, worktreePath: string, settings: Settings, taskEnv?: NodeJS.ProcessEnv, stepOptions?: { unattended?: boolean }, ): Promise { let toolMode: "coding" | "readonly" = workflowStep.toolMode || "readonly"; // (U3) Genuinely-unattended run — set FUSION_HEADLESS=1 below so skills record // assumptions and proceed instead of parking on a question. Explicit opt-in // only (default false = board run); see runGraphCustomNode / KTD-3. const unattended = stepOptions?.unattended === true; const isPlanReviewStep = workflowStep.id === "graph:plan-review-step" || workflowStep.name === "Plan Review"; const workflowStepMetadata = workflowStep as WorkflowStep & { optionalGroupId?: string; reviewCanFixInline?: boolean; requireExternalIntegrationEvidence?: boolean; }; const optionalGroupId = workflowStepMetadata.optionalGroupId; const isReviewTypeWorkflowStep = isPlanReviewStep || workflowStepMetadata.reviewCanFixInline === true || /(?:^|\b)(?:review|verification)(?:\b|$)/i.test(workflowStep.name) || optionalGroupId === "plan-review" || optionalGroupId === "code-review" || optionalGroupId === "browser-verification"; const reviewerInlineFixesEnabled = (settings as Settings & { reviewerInlineFixes?: boolean }).reviewerInlineFixes !== false; const allowReviewerInlineFixes = reviewerInlineFixesEnabled && isReviewTypeWorkflowStep && workflowStep.mode === "prompt"; const allowPlanReviewPromptWrite = allowReviewerInlineFixes && isPlanReviewStep; if (allowReviewerInlineFixes && !isPlanReviewStep) { /* * FNXC:WorkflowReviewers 2026-07-01-12:36: * Review-type workflow nodes can now repair their own findings when the workflow setting `reviewerInlineFixes` is on. Use coding tools for implementation review sessions so Code Review, Browser Verification, and custom review/verification gates do not have to bounce through executor remediation for issues they can safely fix inline. Plan Review stays on a narrow PROMPT.md writer because it runs before implementation. */ toolMode = "coding"; } const requireExternalIntegrationEvidence = workflowStepMetadata.requireExternalIntegrationEvidence === true; /* * FNXC:WorkflowReviewSpecInjection 2026-07-18-18:15: * FN-7561 established that review agents cannot reliably locate the project-root PROMPT.md from a task worktree. Load it once through the store and embed it for every review-type node. FN-8288 extends that invariant beyond Plan Review: approved planning revisions are authoritative, the original task description is historical, and a failed artifact read must stay visible instead of silently restoring superseded scope. */ let workflowReviewSpecArtifact: string | undefined; if (isReviewTypeWorkflowStep) { try { workflowReviewSpecArtifact = await this.readTaskArtifact(task.id, "PROMPT.md"); } catch (error) { const diagnostic = `PROMPT.md could not be read because task storage failed; ${workflowStep.name} must retry without replanning. ${error instanceof Error ? error.message : String(error)}`; await this.store.logEntry(task.id, `[pre-merge] ${workflowStep.name} artifact read failed: ${diagnostic}`); return { success: false, error: diagnostic, output: diagnostic, failureValue: requiredArtifactReadFailedValue("PROMPT.md"), }; } } const workflowReviewSpecText = typeof workflowReviewSpecArtifact === "string" ? workflowReviewSpecArtifact : ""; const planReviewSpecText = isPlanReviewStep ? workflowReviewSpecText : ""; /* FNXC:PlanReview 2026-07-21-16:30: Review steps must never approve or execute against an unavailable contract. Confirmed missing or whitespace-only PROMPT.md fails closed before reviewer creation; typed recovery routes ownership back to planning without spending the review-revision budget. */ if (isReviewTypeWorkflowStep && !workflowReviewSpecText.trim()) { const diagnostic = `PROMPT.md could not be loaded; ${workflowStep.name} cannot approve without the authoritative task contract.`; await this.store.logEntry( task.id, `[pre-merge] ${workflowStep.name} refused to run without PROMPT.md: ${diagnostic}`, ); return { success: false, revisionRequested: true, output: `REVISE: ${diagnostic}`, verdict: "REVISE", notes: diagnostic, failureValue: requiredArtifactMissingValue(["PROMPT.md"]), }; } if (isPlanReviewStep && requireExternalIntegrationEvidence) { /* * FNXC:PlanValidation 2026-06-30-09:03: * Coding (per-step review) intentionally keeps external-integration evidence as a Plan Review gate. Enforce it here, not in triage, so only workflows that set `requireExternalIntegrationEvidence` block and failures route through the graph's normal plan-replan loop. */ const evidenceGaps = detectExternalIntegrationEvidenceGaps({ promptContent: planReviewSpecText, }); if (evidenceGaps.length > 0) { const diagnostic = formatExternalIntegrationEvidenceDiagnostic(evidenceGaps); const output = `REVISE: ${diagnostic}`; await this.store.logEntry( task.id, `[pre-merge] Plan Review deterministic external-integration evidence check requested revision: ${diagnostic}`, ); return { success: false, revisionRequested: true, output, verdict: "REVISE", notes: diagnostic, }; } } // Compute the diff scope so the workflow step agent reviews only what THIS // task changed — not unrelated files it might wander into. Without this, // open-ended review prompts (e.g. "verify visual polish") have been // observed to spend the entire timeout budget reading pre-existing files // that match the task description's keywords. See FN-3327 post-mortem. const scopedFiles = await this.captureModifiedFiles(worktreePath, task.baseCommitSha, task.id, undefined, "workflow-step-handler"); let diffShortstat: string | undefined; try { const baseRef = await this.resolveDiffBaseRef(worktreePath, task.baseCommitSha); if (baseRef) { const { stdout } = await execAsync(`git diff --shortstat ${baseRef}..HEAD`, { cwd: worktreePath, encoding: "utf-8", }); diffShortstat = stdout.trim() || undefined; } } catch { // best-effort — fall through with no shortstat } const MAX_SCOPE_FILES = 100; const scopeFileBlock = scopedFiles.length === 0 ? "(no modified files detected for this task — review the worktree directly, but do NOT browse unrelated files)" : scopedFiles.length > MAX_SCOPE_FILES ? `${scopedFiles.slice(0, MAX_SCOPE_FILES).map((f) => `- ${f}`).join("\n")}\n- ... (${scopedFiles.length - MAX_SCOPE_FILES} more files truncated)` : scopedFiles.map((f) => `- ${f}`).join("\n"); /* * FNXC:PlanReviewScope 2026-06-29-00:57: * Plan Review validates the planned PROMPT.md before execution. It must not * inherit the generic workflow-step diff scope, because dirty worktrees or * unrelated local commits can make a plan-only gate reject implementation * state and loop back to triage after the planner already approved the spec. */ const approvedContractBlock = isReviewTypeWorkflowStep && !isPlanReviewStep ? ` Approved Task Contract: - PROMPT.md is the authoritative current contract for this review. It includes any approved planning revisions and scope decisions. - The Task Description is historical input only. Do not enforce superseded requirements from the original Task Description when they conflict with PROMPT.md. - Do not request behavior that PROMPT.md explicitly defers, excludes, or forbids. Review the implementation against the approved contract reproduced below. - Scope exclusions do not waive security, correctness, or data-integrity defects in the approved implementation. --- BEGIN APPROVED PROMPT.md --- ${workflowReviewSpecText} --- END APPROVED PROMPT.md ---` : ""; const scopeBlock = isPlanReviewStep ? `Plan Review Scope: - Review the task plan artifact (PROMPT.md), reproduced verbatim below, and task metadata only. - The plan is embedded in this prompt — do NOT go looking for a PROMPT.md file in the worktree; it lives at the project root (\`.fusion/tasks/${task.id}/PROMPT.md\`), outside this worktree, so review the embedded copy. - Do NOT judge current implementation diffs, uncommitted worktree changes, or unrelated repository changes. - If the plan is internally consistent, complete, scoped, and verifiable, approve even when the worktree contains unrelated changes from another task. --- BEGIN PROMPT.md --- ${planReviewSpecText} --- END PROMPT.md ---` : `Diff Scope (files changed by THIS task vs base): ${scopeFileBlock}${diffShortstat ? `\nDiff stat: ${diffShortstat}` : ""} CRITICAL SCOPING RULES — read before doing anything else: - Review ONLY the files listed above. Do NOT analyze unmodified files or unrelated parts of the codebase. - If NONE of the files in the diff scope are relevant to your review category (e.g. a UX/design reviewer with no UI/CSS/component files in scope, a security reviewer with no auth/network code in scope, an a11y reviewer with no markup changes), respond IMMEDIATELY with a single short approval line such as "No relevant changes in scope — approved." and STOP. Do not start exploring the codebase. - Your wall-clock budget is short. Spending it browsing unmodified files will cause this step to time out and block merge.${approvedContractBlock}`; const latestTaskForUserComments = await this.store.getTask(task.id).catch(() => task); const workflowStepUserComments = selectUserCommentsForAgentContext(latestTaskForUserComments, { limit: null }); const workflowStepUserCommentSection = buildUserCommentsPromptSection(workflowStepUserComments); /* * FNXC:AgentSteering 2026-06-30-14:08: * Prompt/custom workflow-step reviewers, including Browser Verification agents, do not call reviewStep. They still gate quality, so their system prompt must carry the same canonical uncapped user comments plus legacy steering selected from a fresh task snapshot. */ // (KTD-6) Verdict-contract reconciliation. The trailing-verdict JSON is the // gate-parsing contract — it only matters for steps that gate merge. A skill // step that isn't a gate (e.g. ce-plan / ce-work / ce-compound) produces // skill-native output (and may emit a ===FUSION_AWAIT_INPUT=== sentinel and // stop), so forcing a verdict would contradict the U2 preamble. Require the // verdict only for gate steps (and skill-less prompt steps, which keep the // legacy reviewer contract); relax it for non-gate skill steps. The executor // runs parseAwaitInputSentinel on output regardless, so the await-input // sentinel always takes priority when present. const isSkillStep = typeof workflowStep.skillName === "string" && workflowStep.skillName.trim().length > 0; const isSummaryProjectionStep = (workflowStep as WorkflowStep & { summaryTarget?: string }).summaryTarget === "task"; const requireVerdict = !isSummaryProjectionStep && (workflowStep.gateMode === "gate" || !isSkillStep); const verdictBlock = requireVerdict ? ` ## Feedback Format When your review is complete, your final line MUST be a single JSON object (no markdown fences): {"verdict":"APPROVE|APPROVE_WITH_NOTES|REVISE","notes":"..."} Rules: - Output exactly one trailing JSON object and stop. - verdict must be exactly APPROVE, APPROVE_WITH_NOTES, or REVISE. - notes should be concise and actionable. Use an empty string when there are no notes. - For out-of-scope fast-bail responses, use: {"verdict":"APPROVE","notes":"out of scope: no UI files changed"} Backward compat fallback: if JSON is unavailable, you may still begin output with REQUEST REVISION to request changes.` : ` ## Output Format Follow the skill's own output conventions. You are NOT required to end with a verdict JSON object — this step does not gate merge. If you need to ask the user a question, emit a single ===FUSION_AWAIT_INPUT=== block and stop (see the workflow-step conventions in your instructions).`; const inlineFixBlock = allowReviewerInlineFixes ? ` ## Same-Session Fix Policy This review-type node may fix issues it finds before returning a final verdict. - If you find an in-scope issue you can fix safely, edit the relevant files in this same session, run the smallest relevant verification, and then return APPROVE or APPROVE_WITH_NOTES. - Return REVISE only when the issue is still present, cannot be safely fixed in this reviewer session, needs broader executor remediation, or needs user input. - Plan Review may use fn_task_prompt_write to replace the task's PROMPT.md with the complete revised plan. Do not implement product code from Plan Review. - Code Review and Browser Verification may fix implementation issues inside the assigned task worktree and should mention the fix in notes.` : ""; const systemPrompt = `You are a workflow step agent executing: ${workflowStep.name} Task Context: - Task ID: ${task.id} - Task Description: ${task.description} - Worktree: ${worktreePath} ${scopeBlock}${workflowStepUserCommentSection ? `\n\n${workflowStepUserCommentSection}` : ""} Your role: - Execute this workflow step exactly as scoped. - Prioritize high-impact correctness/risk findings over stylistic nits. - Keep feedback actionable and directly tied to evidence in files/outputs. Your Instructions: ${workflowStep.prompt} You have access to the file system to review changes.${inlineFixBlock}${verdictBlock}`; const agentLogger = new AgentLogger({ store: this.store, taskId: task.id, agent: "reviewer", persistAgentToolOutput: settings.persistAgentToolOutput, // Review-in-executor sessions are task-scoped ephemeral workers. persistAgentThinkingLog: resolvePersistAgentThinkingLog(settings, { ephemeral: true }), onAgentText: (taskId, delta) => { this.options.onAgentText?.(taskId, delta); }, onAgentTool: (taskId, toolName, detail) => { this.options.onAgentTool?.(taskId, toolName, detail); }, }); // Determine primary model and an explicit fallback. Review-type workflow // steps use the validator lane; ordinary workflow prompts use the executor // lane. A complete per-step override remains authoritative for either lane. // FNXC:ModelResolution 2026-06-25-12:00: FN-7039 requires ordinary workflow // steps to inherit project execution-lane model settings before defaults. // Review gates are independent validation surfaces and must not silently use // the same implementation model merely because they execute in this method. const assignedRuntimeConfig = await this.getAssignedAgentRuntimeConfig(task.assignedAgentId); const laneModel = isReviewTypeWorkflowStep ? resolveValidatorSessionModel( task.validatorModelProvider, task.validatorModelId, settings, assignedRuntimeConfig, task.validatorCredentialInstanceId, ) : resolveExecutorSessionModel( task.modelProvider, task.modelId, settings, assignedRuntimeConfig, task.credentialInstanceId, ); const useOverride = !!(workflowStep.modelProvider && workflowStep.modelId); const primaryProvider = useOverride ? workflowStep.modelProvider : laneModel.provider; const primaryModelId = useOverride ? workflowStep.modelId : laneModel.modelId; // FNXC:ProviderAuth 2026-08-01-08:39: A workflow-step model override has no paired instance selection, so only the resolved primary task lane may carry its requested credential instance. Fallback attempts must retain their provider-default behavior rather than inheriting a primary-provider identity. const primaryCredentialInstanceId = useOverride ? undefined : laneModel.credentialInstanceId; const workflowFallback = isReviewTypeWorkflowStep ? resolveValidatorFallbackModel(settings) : resolveExecutorFallbackModel(settings); const fallback = workflowFallback.provider && workflowFallback.modelId && (workflowFallback.provider !== primaryProvider || workflowFallback.modelId !== primaryModelId) ? workflowFallback : undefined; const fallbackSettingsHint = isReviewTypeWorkflowStep ? "settings.validatorFallbackProvider/validatorFallbackModelId or fallbackProvider/fallbackModelId" : "settings.executionFallbackProvider/executionFallbackModelId or fallbackProvider/fallbackModelId"; const fallbackLaneLabel = isReviewTypeWorkflowStep ? "validator" : "executor"; const timeoutMs = Math.max(60_000, settings.workflowStepTimeoutMs ?? 900_000); const runOnce = async ( provider: string | undefined, modelId: string | undefined, attemptLabel: string, ): Promise => { const stepInstructions = await this.resolveInstructionsForRole("executor", settings); const stepSystemPrompt = buildSystemPromptWithInstructions(systemPrompt, stepInstructions); // Build skill selection context for workflow step session const skillContext = await buildSessionSkillContext({ agentStore: this.options.agentStore!, task, sessionPurpose: "executor", projectRootDir: this.rootDir, pluginRunner: this.options.pluginRunner, }); const workflowAgent = await this.getAuthoritativeAssignedAgent(task.assignedAgentId); const workflowRuntimeHint = extractRuntimeHint(workflowAgent?.runtimeConfig); // Signal to skills running in this step (e.g. compound-engineering ce-plan / // ce-work) that they are inside a Fusion autonomous workflow step, NOT an // interactive Claude Code session. There is no synchronous blocking-question // tool here, so a skill must surface user questions via the await-input // convention (which the dashboard / task card renders) instead of calling // AskUserQuestion into the void. Scoped to the step session — the main // executor session deliberately does not carry it. // (U3) FUSION_HEADLESS=1 marks a genuinely-unattended run (LFG/pipeline) so // skills record assumptions and proceed instead of parking. Set ONLY when // the explicit `unattended` flag is true; absent on a board run. const stepEnv: NodeJS.ProcessEnv = { ...(taskEnv ?? process.env), FUSION_WORKFLOW_STEP: "1", }; // FNXC:WorkflowSteps 2026-06-21-06:30: // Default-safe invariant (KTD-3): a board run must NEVER be headless. Since // stepEnv spreads taskEnv/process.env, an inherited FUSION_HEADLESS (e.g. an // outer pipeline exported it) would otherwise leak in and silently skip user // questions. Set it ONLY on an explicit opt-in; strip any inherited value // otherwise so absence of the flag always yields a board run. if (unattended) { stepEnv.FUSION_HEADLESS = "1"; } else { delete stepEnv.FUSION_HEADLESS; } // (U1) Load the step's named skill into THIS session. The interactive fix // proved the resolver works when fed BOTH a requested name AND a discovery // path (compound-engineering-skill-resolution.test.ts). Here we mirror it: // merge the step's skillName (both namespaced `compound-engineering:ce-work` // and bare `ce-work` — the resolver matches bare names case-insensitively) // into the resolved requestedSkillNames, and pass the CE install root (from // the injected FUSION_CE_SKILLS_DIR env) as additionalSkillPaths so the // loader can actually discover the bundled SKILL.md. Without both halves the // named skill was only prompt text pointing at a skill the session never had. let effectiveSkillSelection = skillContext.skillSelectionContext; const ceSkillsDir = typeof stepEnv.FUSION_CE_SKILLS_DIR === "string" && stepEnv.FUSION_CE_SKILLS_DIR.trim() ? stepEnv.FUSION_CE_SKILLS_DIR.trim() : undefined; if (workflowStep.skillName && workflowStep.skillName.trim()) { const namespaced = workflowStep.skillName.trim(); const bare = namespaced.includes(":") ? namespaced.slice(namespaced.lastIndexOf(":") + 1) : namespaced; const existing = effectiveSkillSelection?.requestedSkillNames ?? []; const mergedNames = [...new Set([...existing, namespaced, bare])]; effectiveSkillSelection = { projectRootDir: effectiveSkillSelection?.projectRootDir ?? this.rootDir, ...(effectiveSkillSelection?.sessionPurpose ? { sessionPurpose: effectiveSkillSelection.sessionPurpose } : { sessionPurpose: "executor" }), requestedSkillNames: mergedNames, }; } const additionalSkillPaths = mergeAdditionalSkillPaths(skillContext.additionalSkillPaths, ceSkillsDir ? [ceSkillsDir] : undefined); // FNXC:WorkflowSteps 2026-07-30-21:40: // FN-8461 / GitHub #2388: workflow steps resolve skills from enabled-plugin // body directories and the optional CE install root. Warn only after merging // those sources when THIS named skill remains undiscoverable: a non-empty path // array for another skill is not viable, while an actual plugin body makes CE // env absence expected rather than misleading operator-facing noise. if ( workflowStep.skillName?.trim() && !isWorkflowStepSkillDiscoverable(workflowStep.skillName.trim(), additionalSkillPaths, ceSkillsDir) ) { await this.store.logEntry( task.id, `[skill-load] Workflow step '${workflowStep.name}' requests skill '${workflowStep.skillName}' but it cannot be discovered from configured plugin body directories or FUSION_CE_SKILLS_DIR; the step runs with role-fallback skills only.`, ); } const logBrowserVerificationActivity = async (message: string) => { await this.store.logEntry(task.id, message); await this.store.appendAgentLog(task.id, message, "status", undefined, "reviewer"); }; if (workflowStep.requiresBrowser === true) { effectiveSkillSelection = augmentSessionSkillsForBrowserStep(effectiveSkillSelection, this.rootDir); await logBrowserVerificationActivity(`[browser-verification] starting browser verification for task ${task.id} using step '${workflowStep.name}'`); const browserProbe = await probeAgentBrowserAvailability(execAsync as AgentBrowserExec, { cwd: worktreePath, env: stepEnv, timeoutMs: 5_000, }); await logBrowserVerificationActivity(formatAgentBrowserAvailabilityLog(browserProbe)); } // (U8b) Coding-mode skill steps fan out to ce- subagents via // fn_spawn_agent (read the persona def, pass its body as systemPromptOverride). // That tool is registered only in the main executor session — never here — // so coding mode granted write/edit but NOT spawn. Register it for // coding-mode steps now; readonly steps keep no spawn (filterCustomToolsForReadonly // strips it). The spawn tool inherits the injected env so children also see // FUSION_CE_AGENTS_DIR. // // (U9 / KTD-4, Risk-1) ACCEPTED WRITE-CAPABILITY POSTURE: coding mode also // exposes write/edit. The CE plan/code-review steps run coding ONLY to gain // spawn (they are not supposed to mutate the tree), but the tool policy is // binary today — coding is the only mode that carries fn_spawn_agent. There // is NO engine guard preventing those steps from writing; the only protection // is skill discipline plus the U6 no-diff detection assertion. The proper fix // (a dedicated readonly-plus-spawn tool mode) is deferred; this is a // knowingly-accepted gap, not a closed one — re-evaluate before enabling the // CE workflow for genuinely-unattended (FUSION_HEADLESS) LFG/pipeline runs. const planReviewPromptTools: ToolDefinition[] = allowPlanReviewPromptWrite ? [this.createTaskPromptWriteTool(task.id)] : []; const codingCustomTools: ToolDefinition[] = toolMode === "coding" ? [this.createSpawnAgentTool(task.id, worktreePath, settings, stepEnv)] : []; const workflowCustomTools = [...planReviewPromptTools, ...codingCustomTools]; const readonlyCustomTools = toolMode === "readonly" ? filterCustomToolsForReadonly(workflowCustomTools, { allowTool: (tool) => allowPlanReviewPromptWrite && tool.name === "fn_task_prompt_write", }) : { allowed: workflowCustomTools, denied: [] as string[] }; if (toolMode === "readonly" && readonlyCustomTools.denied.length > 0) { await this.store.logEntry( task.id, `[readonly-violation] Workflow step '${workflowStep.name}' dropped denied custom tools: ${readonlyCustomTools.denied.join(", ")}`, ); } /* * FNXC:Settings-ThinkingLevel 2026-07-10-00:00: * WorkflowStep sessions resolve reasoning effort as node/step `thinkingLevel` first, then the task override for their selected model lane, then settings defaults/lane fallbacks. * * FNXC:Settings-ThinkingLevel 2026-07-10-14:20: * The step's own `fallback` attempt already swaps to a distinct model (validator fallback OR global fallback pair) — it must honor THAT model's fallback thinking level, not silently reuse the primary lane's thinking level. Route by which candidate `fallback.label` actually matched instead of only special-casing `validatorFallback`. */ const workflowStepThinkingSource = workflowStep.thinkingLevel ?? (isReviewTypeWorkflowStep ? task.validatorThinkingLevel ?? task.thinkingLevel : task.thinkingLevel); const workflowStepThinkingLevel = attemptLabel === "fallback" ? isReviewTypeWorkflowStep ? resolveValidatorFallbackThinkingLevel(workflowStepThinkingSource, settings) : resolveExecutorFallbackThinkingLevel(workflowStepThinkingSource, settings) : isReviewTypeWorkflowStep ? resolveValidatorThinkingLevel(workflowStepThinkingSource, settings) : resolveExecutorThinkingLevel(workflowStepThinkingSource, settings); const workflowStepFallbackThinkingLevel = isReviewTypeWorkflowStep ? resolveValidatorFallbackThinkingLevel(workflowStepThinkingSource, settings) : resolveExecutorFallbackThinkingLevel(workflowStepThinkingSource, settings); const { session } = await createResolvedAgentSession({ sessionPurpose: "executor", runtimeHint: workflowRuntimeHint, pluginRunner: this.options.pluginRunner, cwd: worktreePath, systemPrompt: stepSystemPrompt, tools: toolMode, defaultProvider: provider, defaultModelId: modelId, ...(attemptLabel !== "fallback" && primaryCredentialInstanceId ? { credentialInstanceId: primaryCredentialInstanceId } : {}), fallbackProvider: workflowFallback.provider, fallbackModelId: workflowFallback.modelId, fallbackThinkingLevel: workflowStepFallbackThinkingLevel, defaultThinkingLevel: workflowStepThinkingLevel, runAuditor: createRunAuditor(this.store, this.getRunContextFor(task.id)), settings, taskEnv: stepEnv, mcpServers: await this.resolveMcpServers(undefined), // FNXC:SessionRouting 2026-06-24-11:20: // #1675: propagate task id so workflow-step requests carry the same // X-Session-Id/X-Session-Affinity as the primary session. taskId: task.id, // FNXC:PluginSkills 2026-07-12-00:00: Workflow-step sessions union plugin skill body dirs with CE's FUSION_CE_SKILLS_DIR so neither plugin-package nor compound-engineering skills are overwritten. // Skill selection: assigned-agent / role-fallback skills, plus the step's own named skill (U1) made discoverable via additionalSkillPaths. ...(effectiveSkillSelection ? { skillSelection: effectiveSkillSelection } : {}), ...(additionalSkillPaths ? { additionalSkillPaths } : {}), ...(readonlyCustomTools.allowed.length > 0 ? { customTools: readonlyCustomTools.allowed } : {}), }); const workflowModelDetails = formatModelMarkerDetails( describeModel(session), workflowStepThinkingLevel, [ useOverride && attemptLabel === "primary" ? "workflow step override" : "", attemptLabel === "fallback" ? "fallback after timeout" : "", ], ); executorLog.debug(`${task.id}: workflow step '${workflowStep.name}' using model ${workflowModelDetails}`); await this.store.logEntry( task.id, `Workflow step '${workflowStep.name}' using model: ${workflowModelDetails}`, ); this.setActiveWorkflowStepSession(task.id, session, worktreePath, this.createSeenSteeringIds(task)); // FNXC:TaskTiming 2026-07-30-21:40: graph-owned Plan Review is the only // post-spec planning lane. Start before prompting and finalize in finally before any replan handoff. const ownsPlanningSegment = workflowStep.id === "graph:plan-review-step" || workflowStep.name === "Plan Review"; if (ownsPlanningSegment) { this.activePlanningWorkflowSessions.add(task.id); const planningStart = startPlanningSegment(task); try { if (planningStart.planningStartedAt) await this.store.updateTask(task.id, planningStart); } catch (error) { this.activePlanningWorkflowSessions.delete(task.id); throw error; } } let output = ""; const deltaNormalizer = createStreamingDeltaNormalizer(); let detectedQuestion: string | null = null; let resolveQuestion: ((value: "await-input") => void) | undefined; const questionPromise = new Promise<"await-input">((resolve) => { resolveQuestion = resolve; }); session.subscribe((event) => { if (event.type === "message_update") { const msgEvent = event.assistantMessageEvent; if (msgEvent.type === "text_delta") { // Repair dropped sentence-boundary spaces at the shared engine delta chokepoint, // including tool-call cross-message boundaries (see streaming-delta.ts). const delta = deltaNormalizer.normalize(msgEvent.partial, msgEvent.contentIndex, msgEvent.delta, "text"); output += delta; agentLogger.onText(delta); } else if (msgEvent.type === "thinking_delta") { // Repair dropped sentence-boundary spaces at the shared engine delta chokepoint, // including tool-call cross-message boundaries (see streaming-delta.ts). const delta = deltaNormalizer.normalize(msgEvent.partial, msgEvent.contentIndex, msgEvent.delta, "thinking"); agentLogger.onThinking(delta); } } if (event.type === "tool_execution_start") { agentLogger.onToolStart(event.toolName, event.args as Record | undefined); if (!unattended && detectedQuestion === null) { const question = parseAwaitInputQuestionToolCall( event.toolName, event.args as Record | undefined, ); if (question) { detectedQuestion = question; resolveQuestion?.("await-input"); } } } if (event.type === "tool_execution_end") { agentLogger.onToolEnd(event.toolName, event.isError, event.result); } }); let timedOut = false; let timeoutHandle: ReturnType | undefined; const timeoutPromise = new Promise<"timeout">((resolveTimeout) => { timeoutHandle = setTimeout(() => { timedOut = true; resolveTimeout("timeout"); }, timeoutMs); }); try { const promptPromise = promptWithFallback( session, `Execute the workflow step "${workflowStep.name}" for task ${task.id}.\n\n` + `Review the work done in this worktree and evaluate it against the criteria in your instructions.`, ); const outcome = await Promise.race([ promptPromise.then(() => "completed" as const), timeoutPromise, questionPromise, ]); if (outcome === "await-input" && detectedQuestion) { try { session.dispose(); } catch { /* best-effort */ } await agentLogger.flush(); return { success: true, output: `===FUSION_AWAIT_INPUT===\n${detectedQuestion}\n===END_FUSION_AWAIT_INPUT===`, }; } if (outcome === "timeout") { executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' (${attemptLabel}) timed out after ${timeoutMs}ms — disposing session`); await this.store.logEntry( task.id, `Workflow step '${workflowStep.name}' ${attemptLabel === "primary" ? "primary" : "fallback"} model timed out after ${Math.round(timeoutMs / 1000)}s — aborting session`, ); if (workflowStep.requiresBrowser === true) { await logBrowserVerificationActivity(`[browser-verification] finished browser verification for task ${task.id}: timed out`); } // FNXC:TaskCost 2026-07-30-21:40: Plan Review tokens are task cost; // snapshot before timeout disposal just like normal completion. await accumulateSessionTokenUsage(this.store, task.id, session, { agentId: task.assignedAgentId ?? undefined, role: "executor" }); try { session.dispose(); } catch { /* best-effort */ } await agentLogger.flush(); return { success: false, error: `workflow step timed out after ${timeoutMs}ms`, timedOut: true }; } // Completed within the timeout — let any post-completion errors surface. checkSessionError(session); await accumulateSessionTokenUsage(this.store, task.id, session, { agentId: task.assignedAgentId ?? undefined, role: "executor", }); session.dispose(); await agentLogger.flush(); const parsed = requireVerdict ? parseWorkflowStepOutput(output) : parseWorkflowStepOutput(output, { requireVerdict: false }); if (parsed.verdict) { const revisionRequested = parsed.verdict === "REVISE"; if (workflowStep.requiresBrowser === true) { await logBrowserVerificationActivity(`[browser-verification] finished browser verification for task ${task.id}: verdict ${parsed.verdict}`); } return { success: !revisionRequested, revisionRequested, output: parsed.output, verdict: parsed.verdict, notes: parsed.notes, }; } if (parsed.malformed) { // FNXC:ReviewLeniency 2026-07-02-00:30: malformed output (after the // fallback-model retry) is recorded as a NON-BLOCKING advisory, not a // hard gate block — see runGraphCustomNode's outcome mapping. await this.store.logEntry( task.id, `[pre-merge] Workflow step '${workflowStep.name}' produced malformed output (no parseable verdict) — recorded as non-blocking advisory`, ); if (workflowStep.requiresBrowser === true) { await logBrowserVerificationActivity(`[browser-verification] finished browser verification for task ${task.id}: malformed output`); } return { success: false, output: parsed.output, error: "malformed output — no verdict extracted", notes: undefined, malformed: true, }; } if (workflowStep.requiresBrowser === true) { await logBrowserVerificationActivity(`[browser-verification] finished browser verification for task ${task.id}: completed`); } return { success: true, output: parsed.output }; } catch (err: unknown) { await agentLogger.flush(); // Persist the delta before error disposal so graph-owned planning reviews // cannot disappear from operator cost totals. await accumulateSessionTokenUsage(this.store, task.id, session, { agentId: task.assignedAgentId ?? undefined, role: "executor" }); try { session.dispose(); } catch { /* best-effort */ } if ((err instanceof ReadonlyViolationError) || ((err as { code?: string } | null)?.code === "READONLY_VIOLATION")) { const violation = err as ReadonlyViolationError; const deniedTool = violation.toolName || "unknown"; await this.store.logEntry( task.id, `[readonly-violation] Workflow step '${workflowStep.name}' attempted denied tool '${deniedTool}'`, ); if (workflowStep.requiresBrowser === true) { await logBrowserVerificationActivity(`[browser-verification] finished browser verification for task ${task.id}: readonly violation`); } return { success: false, error: `[readonly-violation] ${violation.message}` }; } const errorMessage = err instanceof Error ? err.message : String(err); if (workflowStep.requiresBrowser === true) { await logBrowserVerificationActivity(`[browser-verification] finished browser verification for task ${task.id}: failed — ${errorMessage}`); } return { success: false, error: errorMessage }; } finally { if (timeoutHandle) clearTimeout(timeoutHandle); if (ownsPlanningSegment) { try { const livePlanningTask = await this.store.getTask(task.id); if (livePlanningTask) { const planningEnd = finalizePlanningSegment(livePlanningTask); if (planningEnd.planningStartedAt === null) await this.store.updateTask(task.id, planningEnd); } } finally { // Finalize before releasing Plan Review ownership so triage can only // begin a subsequent, non-overlapping planning segment. this.activePlanningWorkflowSessions.delete(task.id); } } const activeWorkflowStepSession = this.activeWorkflowStepSessions.get(task.id); if (activeWorkflowStepSession === session) { this.deleteActiveWorkflowStepSession(task.id, worktreePath); } // Suppress unused-variable warning; `timedOut` documents intent. void timedOut; } }; const primaryOutcome = await runOnce(primaryProvider, primaryModelId, "primary"); /* FNXC:ReviewLeniency 2026-07-02-00:30: Retry the fallback model on a MALFORMED (unparseable-verdict) primary response, not only on a timeout. A single fumbled response — reasoning with no trailing verdict — should get one more attempt on the fallback model before the gate result is recorded, mirroring the reviewer path's UNAVAILABLE retry. If no fallback is configured the malformed primary is returned as-is (and is treated as a non-blocking advisory downstream, see runGraphCustomNode). */ const primaryMalformed = (primaryOutcome as { malformed?: boolean }).malformed === true; if (!primaryOutcome.timedOut && !primaryMalformed) return primaryOutcome; if (!fallback) { /* * FNXC:ReviewLeniency 2026-07-05-17:24: * FN-7561: when NO fallback model is configured, a MALFORMED primary (unparseable verdict — a single fumbled response) still deserves one retry so a transient formatting fumble does not feed the plan-review replan loop. Self-retry once on the SAME primary model. Timeouts are NOT self-retried — they would likely just time out again and burn another full budget. If the self-retry is still malformed it is returned as a non-blocking advisory downstream. */ if (primaryMalformed && !primaryOutcome.timedOut) { executorLog.log(`${task.id}: workflow step '${workflowStep.name}' produced malformed output and no fallback is configured — retrying once on the primary model`); const retryOutcome = await runOnce(primaryProvider, primaryModelId, "primary-retry"); const retryMalformed = (retryOutcome as { malformed?: boolean }).malformed === true; if (!retryMalformed) return retryOutcome; await this.store.logEntry( task.id, `Workflow step '${workflowStep.name}' produced malformed output on both the primary attempt and one self-retry — no fallback model configured (set ${fallbackSettingsHint})`, ); return retryOutcome; } const reason = primaryOutcome.timedOut ? "timed out" : "produced malformed output"; executorLog.warn(`${task.id}: workflow step '${workflowStep.name}' ${reason} and no fallback model is configured`); await this.store.logEntry( task.id, `Workflow step '${workflowStep.name}' ${reason} — no fallback model configured (set ${fallbackSettingsHint})`, ); return primaryOutcome; } executorLog.log(`${task.id}: retrying workflow step '${workflowStep.name}' with ${fallbackLaneLabel} fallback ${fallback.provider}/${fallback.modelId} after primary ${primaryOutcome.timedOut ? "timeout" : "malformed output"}`); return runOnce(fallback.provider, fallback.modelId, "fallback"); } private MAX_WORKTREE_RETRIES = 3; private WORKTREE_RETRY_DELAYS = [100, 500, 1000]; // ms /** * Create a git worktree with automatic recovery from conflicts. * Implements retry logic with exponential backoff for transient failures. * * @param branch - The branch name to create (e.g., "fusion/fn-123") * @param path - The desired worktree path * @param taskId - The task ID for logging * @param startPoint - Optional base branch/commit for new branch * @returns The actual worktree path (may differ if recovery generated new name) */ private formatBranchConflictLifecycleLog(taskId: string, error: BranchConflictError): string { const strandedSummary = error.strandedCommits.length > 0 ? error.strandedCommits.map((commit) => `${commit.sha.slice(0, 12)} ${commit.subject}`).join("; ") : "none"; const recommendation = "Resolve the local branch/worktree conflict with git tooling (inspect/reclaim or discard) before retrying."; return [ `Branch conflict: ${error.branchName} is already checked out at ${error.conflictingWorktreePath}`, `Existing tip: ${error.existingTipSha}`, `Stranded commits since ${error.startPoint}: ${strandedSummary}`, recommendation, ].join("\n"); } private formatBranchConflictAgentLog(taskId: string, error: BranchConflictError): string { const lines = [ `branch=${error.branchName}`, `worktree=${error.conflictingWorktreePath}`, `existingTipSha=${error.existingTipSha}`, `startPoint=${error.startPoint}`, ]; if (error.strandedCommits.length > 0) { lines.push( ...error.strandedCommits.map((commit) => `stranded=${commit.sha.slice(0, 12)} ${commit.subject}`), ); } else { lines.push("stranded=none"); } lines.push( `recommendation=Resolve the local branch/worktree conflict with git tooling (inspect/reclaim or discard) before retrying.`, ); return lines.join("\n"); } private readonly MAX_AUTO_RECOVERY_ATTEMPTS = 3; private readonly BRANCH_CONFLICT_TRIPWIRE_THRESHOLD = 5; private async tryBootstrapMisbindingRecovery( task: Task, contamination: BranchCrossContaminationError, audit: ReturnType, ): Promise { const bootstrap = await classifyBootstrapMisbinding({ repoDir: this.rootDir, branchName: contamination.branchName, baseSha: contamination.baseSha, taskId: task.id, foreignCommits: contamination.foreignCommits, }); if (!bootstrap.isBootstrapMisbinding) { return false; } const worktreePath = task.worktree; const worktreeClassification = worktreePath ? await classifyTaskWorktree(this.rootDir, worktreePath) : { ok: false as const }; if (!worktreePath || !worktreeClassification.ok) { await this.store.logEntry(task.id, `[recovery] bootstrap misbinding detected but worktree unavailable for re-anchor: ${worktreePath ?? "none"}`, undefined, this.getRunContextFor(task.id)); return false; } await this.store.logEntry(task.id, `[recovery] bootstrap-time branch misbinding detected on ${contamination.branchName}: 0 own commits, re-anchoring to ${contamination.baseSha}`, undefined, this.getRunContextFor(task.id)); try { const reanchor = await reanchorBranchToBase({ repoDir: this.rootDir, worktreePath, branchName: contamination.branchName, baseSha: contamination.baseSha, taskId: task.id, }); await audit.git({ type: "branch:reanchor", target: contamination.branchName, metadata: { taskId: task.id, baseSha: contamination.baseSha, previousTipSha: reanchor.previousTipSha, newTipSha: reanchor.newTipSha, trigger: "bootstrap-misbinding", }, }); await this.store.updateTask(task.id, { recoveryRetryCount: null, nextRecoveryAt: null, error: null, paused: false, pausedReason: null, }); this.markGraphExecuteSelfRequeued(task.id); await this.store.moveTask(task.id, await resolveReboundColumnFor(this.store, task.id), { preserveResumeState: false, preserveWorktree: true }); return true; } catch (error) { await this.store.logEntry(task.id, `[recovery] bootstrap re-anchor failed; falling back to contamination safety path: ${formatError(error)}`, undefined, this.getRunContextFor(task.id)); return false; } } private async reclaimExistingWorktree( task: Task, livePath: string, branch: string, tipSha: string, count: number, settings: Partial, ): Promise { const targetPath = preservedWorktreeTargetPathForTask(task.id, livePath, settings, this.rootDir); const normalizedPath = await this.normalizeReclaimableWorktreePath(livePath, targetPath, task.id, settings); await this.store.updateTask(task.id, { worktree: normalizedPath, branch }); const latestTask = await this.store.getTask(task.id); const baseRef = await this.resolveDiffBaseRef(normalizedPath, latestTask.baseCommitSha); if (baseRef) { await assertCleanBranchAtBase(this.rootDir, branch, baseRef, task.id); } const message = `[recovery] reclaimed existing worktree for ${task.id} at ${normalizedPath} (${count} commits preserved, tip ${tipSha.slice(0, 12)})`; await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.store.appendAgentLog(task.id, "Branch conflict auto-recovery", "status", message, "executor"); } private async handleBranchConflict(task: Task, error: BranchConflictError): Promise<"retry" | "reclaimed" | "sticky"> { // FN-4811: Before invoking inspection-based recovery (which may force-remove the // conflicting worktree), verify the conflict isn't currently bound to a live session. // If it is, refuse the whole recovery dance — a force-remove here would yank an active // task's filesystem out from under it, producing FN-4781/FN-4804-style cascade failures. const activeOwner = await this.findActiveWorktreeOwner(error.conflictingWorktreePath, task.id); if (activeOwner !== null) { const refusalMessage = `[FN-4811] Branch conflict on ${error.branchName} deferred: conflicting worktree ${error.conflictingWorktreePath} is actively owned by ${activeOwner}`; executorLog.warn(refusalMessage); await this.store.logEntry(task.id, refusalMessage, undefined, this.getRunContextFor(task.id)); return "sticky"; } const settings = await mergeEffectiveSettings(this.store, task, await this.store.getSettings()); const integrationRef = task.mergeDetails?.mergeTargetBranch ?? task.baseBranch ?? task.executionStartBranch ?? await resolveIntegrationBranch(this.rootDir, undefined); const inspection = await inspectBranchConflict({ repoDir: this.rootDir, branchName: error.branchName, conflictingWorktreePath: error.conflictingWorktreePath, requestingTaskId: task.id, ownerTaskId: task.id, startPoint: error.startPoint, integrationRef, }); if (inspection.kind === "stale-resolved") { await this.store.updateTask(task.id, { worktree: null, branch: null, baseCommitSha: null }); const message = `[recovery] ${task.id} stage-A: pruned stale admin entry for ${error.branchName}`; await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.store.appendAgentLog(task.id, "Branch conflict auto-recovery", "status", message, "executor"); return "retry"; } if (inspection.kind === "tip-already-merged") { if (inspection.livePath) { await this.cleanupConflictingWorktree(inspection.livePath, error.branchName, task.id); } try { await execAsync("git worktree prune", { cwd: this.rootDir, timeout: 120_000, maxBuffer: 10 * 1024 * 1024, }); } catch { // best-effort } try { await execAsync(`git branch -D ${JSON.stringify(error.branchName)}`, { cwd: this.rootDir, timeout: 120_000, maxBuffer: 10 * 1024 * 1024, }); } catch { // best-effort } await this.store.updateTask(task.id, { worktree: null, branch: null, baseCommitSha: null }); const message = `[recovery] ${task.id} stage-A: tip-already-merged cleanup for ${error.branchName} (${inspection.tipSha.slice(0, 12)} on ${inspection.integrationRef})`; await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); await this.store.appendAgentLog(task.id, "Branch conflict auto-recovery", "status", message, "executor"); return "retry"; } if (inspection.kind === "reclaimable") { await this.reclaimExistingWorktree(task, inspection.livePath, error.branchName, inspection.tipSha, inspection.taskAttributedCommitCount, settings); return "reclaimed"; } if (inspection.kind === "fully-subsumed") { await this.reclaimExistingWorktree(task, inspection.livePath, error.branchName, inspection.tipSha, 0, settings); return "reclaimed"; } if (inspection.kind === "live-foreign") { const cleanupSuccess = await this.cleanupConflictingWorktree(inspection.livePath, error.branchName, task.id); if (cleanupSuccess) { try { await execAsync("git worktree prune", { cwd: this.rootDir }); } catch { // best-effort } try { const worktreeMap = await this.getWorktreeBranchMap(); if (!worktreeMap.has(error.branchName)) { await execAsync(`git branch -D "${error.branchName}"`, { cwd: this.rootDir }); } } catch { // best-effort } return "retry"; } } const conflictMessage = `Task branch conflict: ${error.branchName} is already checked out at ${error.conflictingWorktreePath}. ` + `Resolve the local branch/worktree conflict with git tooling (inspect/reclaim or discard) before retrying.`; await this.store.logEntry(task.id, this.formatBranchConflictLifecycleLog(task.id, error), undefined, this.getRunContextFor(task.id)); await this.store.appendAgentLog(task.id, "Branch conflict recovery required", "tool_error", this.formatBranchConflictAgentLog(task.id, error), "executor"); const autoRecoveryDispatcher = this.getAutoRecoveryDispatcher(createRunAuditor(this.store, this.getRunContextFor(task.id))); const decision = await autoRecoveryDispatcher.dispatch({ class: "branch-conflict-unrecoverable", taskId: task.id, runId: this.getRunContextFor(task.id)?.runId, pausedReason: "branch-conflict-unrecoverable", evidence: { branchName: error.branchName, conflictingWorktreePath: error.conflictingWorktreePath, }, underlyingError: error, }, { task, retryCount: task.recoveryRetryCount ?? 0, settings: (await this.store.getSettings()).autoRecovery ?? { mode: "deterministic-only", maxRetries: 3 }, }); if (decision.action === "pause") { await this.store.updateTask(task.id, { status: "failed", error: conflictMessage, branch: error.branchName, worktree: error.conflictingWorktreePath, paused: true, pausedReason: "branch-conflict-unrecoverable", }); await this.persistTokenUsage(task.id); executorLog.warn(`✗ ${task.id} branch conflict sticky failure: ${error.branchName} @ ${error.conflictingWorktreePath}`); this.options.onError?.(task, error); return "sticky"; } return "retry"; } /* FNXC:Worktrees 2026-07-19-15:47: Branch-needing task work must be created with `git worktree add` in an isolated checkout. Per the AGENTS.md “Prefer main For Direct Work; Use Worktrees For Branches” standing rule, this.rootDir is never switched with `git checkout` or `git switch` to select a task branch; see the primary-checkout invariant regression test for the executable guard. */ private async createWorktree( branch: string, path: string, taskId: string, startPoint?: string, allowSiblingBranchRename = false, ): Promise<{ path: string; branch: string }> { // Track the worktree path we're attempting to use (may change during recovery) const currentPath = path; let resolvedStartPoint: string | undefined; if (startPoint) { const resolved = await this.resolveWorktreeStartPoint(startPoint, taskId); if (resolved === null) { // Stored baseBranch no longer exists (e.g., upstream dep merged and branch // deleted while this task sat queued/stuck). Clear it on the task so any // subsequent retry branches from the default base, and proceed from HEAD. await this.store.updateTask(taskId, { executionStartBranch: null }); } else { resolvedStartPoint = resolved; } } // When the task declares a non-main base (a sibling task's branch), the // legacy behavior was to fork the worktree from that branch's tip, // inheriting all of its commits. That caused content leakage when the // dep was later squash-merged to main: the dep's raw commits became // orphans whose content already existed in main, blocking the // dependent's own merge with phantom conflicts. // // Prevention: instead of forking from the dep's tip, fork from `main` // (or the configured remote/main if rebase-from-remote is enabled) and // then `git merge --squash` the dep's content into a single import // commit. The dependent branch then carries main's history + 1 commit // for the dep's content; if the dep is later squash-merged to main, the // patch-id on that import commit will match main's squash and Layer 2 // recovery (or a clean rebase) handles it. // // Fall-soft: any failure in this path falls back to the legacy behavior // so we don't break worktree creation for setups where the squash flow // can't run (no main branch resolvable, network down, etc.). const squashImport = resolvedStartPoint ? await this.planSquashImportFromDep(taskId, resolvedStartPoint, startPoint) : null; const initialStartPoint = squashImport ? squashImport.mainBase : resolvedStartPoint; const settings = await this.store.getSettings(); for (let attempt = 0; attempt < this.MAX_WORKTREE_RETRIES; attempt++) { try { const result = await this.tryCreateWorktree( branch, currentPath, taskId, initialStartPoint, attempt, 0, allowSiblingBranchRename, settings, ); // Squash-import dep content into the freshly created worktree so the // branch contains main's history + 1 import commit instead of the // dep's raw commits. if (squashImport) { await this.squashImportDepIntoWorktree( result.path, taskId, squashImport.depTip, squashImport.label, ).catch((importErr: unknown) => { executorLog.warn( `Squash-import of ${squashImport.label} into ${result.branch} failed for ${taskId} (continuing without): ${importErr instanceof Error ? importErr.message : String(importErr)}`, ); }); } // Mirror the merge-time rebase behavior: when worktreeRebaseBeforeMerge // is enabled, fetch the remote and rebase the just-created task branch // onto the latest /. This makes the worktree // start from origin/main + local main both, so divergence only matters // if the user actively skips this setting. Best-effort: failures here // don't abort task setup. await this.rebaseNewWorktreeOntoRemote(result.path, result.branch, taskId).catch((err: unknown) => { executorLog.warn( `Post-create worktree rebase failed for ${taskId} (continuing): ${err instanceof Error ? err.message : String(err)}`, ); }); return result; } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); const isLastAttempt = attempt === this.MAX_WORKTREE_RETRIES - 1; const isBranchConflict = isBranchConflictError(error); const isTerminalWorktreeError = error instanceof NonRetryableWorktreeError || error instanceof StaleWorktreeIndexLockError || isBranchConflict; if (isLastAttempt || isTerminalWorktreeError) { await this.store.logEntry( taskId, `Worktree creation failed after ${this.MAX_WORKTREE_RETRIES} attempts`, errorMessage, ); if (isBranchConflict) { throw error; } throw new Error( `Failed to create worktree after ${this.MAX_WORKTREE_RETRIES} attempts: ${errorMessage}`, ); } // Wait before retry (exponential backoff) const delay = this.WORKTREE_RETRY_DELAYS[attempt] || 1000; await new Promise((resolve) => setTimeout(resolve, delay)); } } // Should never reach here, but TypeScript needs a return throw new Error("Unexpected exit from worktree creation retry loop"); } private quoteShellArg(value: string): string { return `'${value.replace(/'/g, "'\\''")}'`; } /** * Decide whether a task's declared dep base should be squash-imported * (instead of forked from). Returns the planned operation's data when the * dep tip differs from the resolvable main base; returns null when no * import is needed (dep is already at main) or when no main base is * resolvable (caller falls back to legacy fork-from-dep). * * `originalStartPoint` is the user-facing label (typically the branch name * like `fusion/fn-2729`) used purely for log messages. `depTip` is the * resolved SHA of the dep's tip — that's what gets squash-merged. */ private async planSquashImportFromDep( _taskId: string, depTip: string, originalStartPoint: string | undefined, ): Promise<{ depTip: string; mainBase: string; label: string } | null> { let settings; try { settings = await this.store.getSettings(); } catch { return null; } // Resolve the main base. Preference order: // 1. / when worktreeRebaseBeforeMerge is enabled // and a remote is resolvable (settings.worktreeRebaseRemote wins; // otherwise fall back to "origin" or the lone remote). // 2. rootDir's HEAD (i.e., whatever local main is currently checked out // to). Used when remote rebase is disabled or no remote exists. let mainBase = ""; if (settings.worktreeRebaseBeforeMerge !== false) { let remote = settings.worktreeRebaseRemote?.trim() || ""; if (!remote) { try { const { stdout } = await execAsync("git remote", { cwd: this.rootDir }); const remotes = stdout.split("\n").map((s) => s.trim()).filter(Boolean); if (remotes.includes("origin")) remote = "origin"; else if (remotes.length === 1) remote = remotes[0]; } catch { // No remote resolvable. } } if (remote) { let defaultBranch = ""; try { const { stdout } = await execAsync( `git rev-parse --abbrev-ref ${this.quoteShellArg(remote)}/HEAD`, { cwd: this.rootDir }, ); defaultBranch = stdout.trim().replace(new RegExp(`^${remote}/`), ""); } catch { // origin/HEAD not set; will fall through to local HEAD below. } if (defaultBranch && defaultBranch !== "HEAD") { // Fetch best-effort so the remote ref reflects upstream tip. await execAsync( `git fetch ${this.quoteShellArg(remote)} ${this.quoteShellArg(defaultBranch)}`, { cwd: this.rootDir }, ).catch(() => undefined); try { const { stdout } = await execAsync( `git rev-parse --verify "${remote}/${defaultBranch}^{commit}"`, { cwd: this.rootDir, encoding: "utf-8" }, ); mainBase = stdout.trim(); } catch { // Couldn't resolve remote ref — fall through. } } } } if (!mainBase) { try { const { stdout } = await execAsync("git rev-parse HEAD", { cwd: this.rootDir, encoding: "utf-8", }); mainBase = stdout.trim(); } catch { return null; } } if (!mainBase) return null; // If the dep tip is already an ancestor of main, no squash import is // needed — the dep's content is already represented in main. try { await execAsync( `git merge-base --is-ancestor ${this.quoteShellArg(depTip)} ${this.quoteShellArg(mainBase)}`, { cwd: this.rootDir }, ); // Exit code 0 → ancestor → no import needed; legacy fork-from-main is fine. // Returning the plan with mainBase but signalling "no work" via dep===main. if (depTip === mainBase) return null; // Dep is ancestor of main but its tip SHA differs from main's tip; the // worktree should still branch off main, no squash needed. return { depTip: mainBase, mainBase, label: originalStartPoint || depTip.slice(0, 8) }; } catch { // Not an ancestor — squash-import is the safer path. } return { depTip, mainBase, label: originalStartPoint || depTip.slice(0, 8) }; } /** * Squash-merge the dep's content into a worktree that's already branched * off main. Produces one commit on the worktree branch carrying the dep's * content, instead of inheriting the dep's individual commits. Best-effort: * any failure (conflict, hooks, IO) leaves the worktree at main and the * caller proceeds — the dependent task will then need to import the dep's * content itself, but the worktree itself is still usable. */ private async squashImportDepIntoWorktree( worktreePath: string, taskId: string, depTip: string, label: string, ): Promise { // No-op when dep is already represented in the worktree's history. try { await execAsync( `git merge-base --is-ancestor ${this.quoteShellArg(depTip)} HEAD`, { cwd: worktreePath }, ); return; } catch { // Not an ancestor — proceed. } // Try a squash-merge. `--no-commit` is implied by `--squash`; the merge // either stages the dep's diff or fails (conflicts / unrelated histories). try { await execAsync( `git merge --squash --allow-unrelated-histories ${this.quoteShellArg(depTip)}`, { cwd: worktreePath }, ); } catch (err) { // Reset any partial state so the worktree stays usable, then rethrow // so the caller can decide whether to log/fall-through. await execAsync("git reset --hard HEAD", { cwd: worktreePath }).catch( () => undefined, ); throw err; } // If no diff was staged the dep is content-equivalent to main; nothing // to commit. try { await execAsync("git diff --cached --quiet", { cwd: worktreePath }); return; // exit 0 → no staged changes, nothing to commit } catch { // exit non-zero → staged changes exist, proceed to commit. } // Always non-empty (subject + body via two -m args). Drop // --allow-empty-message: we never want git to silently accept an empty // message — a missing message here would make the commit hard to // attribute / explain in `git log` and break downstream consumers that // parse merge metadata from commit messages. const subject = `chore(${taskId}): import dependency content from ${label}`; const body = `Squash-imported the working tree of ${label} as a single commit so this ` + `branch carries the dep's content without inheriting its individual commits. ` + `If the dep is later squash-merged to main, this commit's patch-id should ` + `match the merge and rebase cleanly.`; try { await execAsync( `git commit -m ${this.quoteShellArg(subject)} -m ${this.quoteShellArg(body)}`, { cwd: worktreePath }, ); } catch (commitErr) { await execAsync("git reset --hard HEAD", { cwd: worktreePath }).catch( () => undefined, ); throw commitErr; } await this.store.logEntry( taskId, `Squash-imported dependency content from ${label} into worktree (single import commit instead of inheriting raw commits)`, ); } /** * After creating a fresh task worktree, fetch the configured remote and * rebase the task branch onto `/`. The result is a * branch that contains origin's tip plus any local main commits, so the * eventual merge has fewer surprises and the executor sees the freshest * code its peers/CI may have published. * * No-op when `worktreeRebaseBeforeMerge` is disabled, no remote is * configured/resolvable, or the rebase produces conflicts (we abort and * leave the worktree as-is so the executor can still run). */ private async rebaseNewWorktreeOntoRemote( worktreePath: string, branch: string, taskId: string, ): Promise { let settings; try { settings = await this.store.getSettings(); } catch { return; } if (settings.worktreeRebaseBeforeMerge === false) return; let remote = settings.worktreeRebaseRemote?.trim() || ""; if (!remote) { try { const { stdout } = await execAsync("git remote", { cwd: this.rootDir }); const remotes = stdout.split("\n").map((s) => s.trim()).filter(Boolean); if (remotes.includes("origin")) remote = "origin"; else if (remotes.length === 1) remote = remotes[0]; } catch { // No remote resolvable — nothing to rebase against. } } if (!remote) return; let defaultBranch = ""; try { const { stdout } = await execAsync(`git rev-parse --abbrev-ref ${remote}/HEAD`, { cwd: this.rootDir }); defaultBranch = stdout.trim().replace(new RegExp(`^${remote}/`), ""); } catch { // origin/HEAD not set — fall back to current branch in rootDir. } if (!defaultBranch) { try { const { stdout } = await execAsync("git rev-parse --abbrev-ref HEAD", { cwd: this.rootDir }); defaultBranch = stdout.trim(); } catch { return; } } if (!defaultBranch || defaultBranch === "HEAD") return; const remoteRef = `${remote}/${defaultBranch}`; try { await execAsync(`git fetch ${this.quoteShellArg(remote)} ${this.quoteShellArg(defaultBranch)}`, { cwd: this.rootDir }); } catch (err) { executorLog.warn( `Worktree rebase: fetch ${remote} ${defaultBranch} failed for ${taskId}: ${err instanceof Error ? err.message : String(err)}`, ); return; } try { await execAsync(`git rebase ${this.quoteShellArg(remoteRef)}`, { cwd: worktreePath }); await this.store.logEntry( taskId, `Rebased new worktree branch ${branch} onto ${remoteRef}`, ); } catch (rebaseErr) { const msg = rebaseErr instanceof Error ? rebaseErr.message : String(rebaseErr); executorLog.warn( `Worktree rebase: rebase onto ${remoteRef} failed for ${taskId} — aborting and leaving local base intact: ${msg}`, ); try { await execAsync("git rebase --abort", { cwd: worktreePath }); } catch { // best-effort } await this.store.logEntry( taskId, `Could not rebase new worktree onto ${remoteRef} — kept local base. The merge-time rebase will retry with conflict resolution.`, ); } } /** * Resolve a stored baseBranch to a concrete commit SHA. * * Returns `null` (not throw) when the ref cannot be resolved — typically * because the upstream dep's branch was merged and deleted while this task * sat queued/stuck. Callers should treat null as "fall back to default base" * rather than fail the task permanently. */ private async resolveWorktreeStartPoint(startPoint: string, taskId: string): Promise { const command = isAbsolute(startPoint) && existsSync(startPoint) ? `git -C "${startPoint}" rev-parse --verify HEAD^{commit}` : `git rev-parse --verify "${startPoint}^{commit}"`; try { const { stdout } = await execAsync(command, { cwd: this.rootDir }); return stdout.trim() || startPoint; } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); await this.store.logEntry( taskId, `Worktree base ref "${startPoint}" is missing — falling back to default base`, errorMessage, ); return null; } } /* FNXC:MissingWorktreeRecovery 2026-07-16-18:35: Returns the recovery outcome (not a bare boolean) so the FN-7996 graph-failure router can distinguish "requeued for clean retry" (handled — stop failure processing) from "escalate-exhausted" (fall through to the visible terminal park for human inspection). Existing session-start callers treat any truthy outcome as handled, unchanged. */ private async recoverMissingWorktreeSessionStartFailure( task: Task, worktreePath: string, error: unknown, audit: RunAuditor, ): Promise { const errorText = error instanceof Error ? error.message : String(error); const missingWorktreeFailure = isMissingWorktreeSessionStartFailure(errorText); const missingTaskJsonFailure = isTransientMissingTaskJsonError(error, task); if (!missingWorktreeFailure && !missingTaskJsonFailure) return false; const classification = classifyMissingWorktreeSessionStartFailure(errorText); const missingTaskJsonPath = errorText.match(TRANSIENT_WORKTREE_TASK_JSON_ENOENT_PATTERN)?.[1] ?? null; const staleWorktreePath = extractMissingWorktreePathFromSessionStartFailure(errorText) ?? (missingTaskJsonPath ? resolvePath(missingTaskJsonPath, "..", "..", "..") : null) ?? worktreePath; if (missingTaskJsonFailure) { executorLog.log(`[transient-task-json-suppressed] taskId=${task.id} elapsedMs=0 reason=missing-task-json-under-worktree path=${missingTaskJsonPath ?? "unknown"}`); } await audit.git({ type: "worktree:incomplete-detected", target: staleWorktreePath, metadata: { classification, reason: errorText, source: "session-start", taskId: task.id }, }); if (isInsideWorktreesDir(this.rootDir, staleWorktreePath)) { try { await removeWorktree({ rootDir: this.rootDir, worktreePath: staleWorktreePath, settings: await this.store.getSettings(), reason: RemovalReason.PoolPrune, taskId: task.id, audit, expectedOwnerTaskId: task.id, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); } catch (removeErr) { executorLog.warn(`${task.id}: failed to remove unusable session-start worktree ${staleWorktreePath}: ${formatError(removeErr)}`); } } const recovery = await autoRecoverWorktreeSessionStartFailure(this.store, task, { failure: error, source: "executor-session-start", auditor: audit, rootDir: this.rootDir, }); if (recovery.outcome !== "escalate-exhausted") { this.markGraphExecuteSelfRequeued(task.id); } await audit.git({ type: "worktree:auto-recovered", target: staleWorktreePath, metadata: { classification: recovery.classification, action: recovery.outcome === "escalate-exhausted" ? "escalate-exhausted" : "requeue-todo", retries: recovery.retries, maxRetries: MAX_WORKTREE_SESSION_RETRIES, staleWorktree: staleWorktreePath, taskId: task.id, }, }); if (recovery.outcome === "escalate-exhausted") { await this.store.logEntry( task.id, `Worktree session-start auto-recovery exhausted (${recovery.retries}/${MAX_WORKTREE_SESSION_RETRIES}); task left for human inspection`, undefined, this.getRunContextFor(task.id), ); } else { await this.store.logEntry( task.id, `Worktree was ${classification} at session start; requeued to todo for clean retry (attempt ${recovery.retries}/${MAX_WORKTREE_SESSION_RETRIES})`, undefined, this.getRunContextFor(task.id), ); } return recovery.outcome === "escalate-exhausted" ? "escalate-exhausted" : "requeue-todo"; } private async emitWorktreeReanchoredAudit( taskId: string, fromPath: string, toPath: string, source: "verify-worktree-invariants" | "executor-liveness-gate", ): Promise { const runContext = this.getRunContextFor(taskId); if (!runContext?.runId || !runContext.agentId) return; const auditor = createRunAuditor(this.store, { runId: runContext.runId, agentId: runContext.agentId, taskId, phase: "execute", }); await auditor.git({ type: "worktree:reanchored", target: toPath, metadata: { taskId, fromPath, toPath, source, }, }); } private async emitStaleLockAudit( taskId: string, event: | "worktree:stale-lock-detected" | "worktree:stale-lock-recovered" | "worktree:stale-lock-recovery-failed" | "worktree:stale-lock-refused" | "worktree:stale-registration-detected" | "worktree:stale-registration-recovered" | "worktree:stale-registration-recovery-failed", targetPath: string, metadata: Record, ): Promise { const runContext = this.getRunContextFor(taskId); if (!runContext?.runId || !runContext.agentId) return; const auditor = createRunAuditor(this.store, { runId: runContext.runId, agentId: runContext.agentId, taskId, phase: "execute", }); await auditor.git({ type: event, target: targetPath, metadata }); } private async recoverIndexLockIfStale(taskId: string, path: string, conflictInfo: { lockPath?: string; message?: string }): Promise { const lockPath = conflictInfo.lockPath; if (!lockPath) return false; const classification = await classifyStaleLock({ rootDir: this.rootDir, lockPath, activeSessionRegistry, }); await this.emitStaleLockAudit(taskId, "worktree:stale-lock-detected", path, { lockPath, classification: classification.kind, reason: classification.reason, ageMs: classification.ageMs ?? null, owningWorktreePath: classification.owningWorktreePath ?? null, }); if (classification.kind !== "stale") { await this.emitStaleLockAudit(taskId, "worktree:stale-lock-refused", path, { lockPath, classification: classification.kind, reason: classification.reason, ageMs: classification.ageMs ?? null, owningWorktreePath: classification.owningWorktreePath ?? null, }); throw new StaleWorktreeIndexLockError({ message: `Worktree creation blocked: index.lock at ${resolvePath(this.rootDir, lockPath)} is held by another git process (reason: ${classification.reason}, owning worktree ${classification.owningWorktreePath ?? "unknown"}). Resolve manually before retrying.`, lockPath: resolvePath(this.rootDir, lockPath), classification: classification.kind, reason: classification.reason, }); } try { const removed = await tryRemoveStaleLock({ lockPath: resolvePath(this.rootDir, lockPath) }); if (removed.removed) { await this.emitStaleLockAudit(taskId, "worktree:stale-lock-recovered", path, { lockPath }); await this.store.logEntry(taskId, `Recovered stale worktree index.lock and retrying`, resolvePath(this.rootDir, lockPath), this.getRunContextFor(taskId)); return true; } await this.emitStaleLockAudit(taskId, "worktree:stale-lock-recovery-failed", path, { lockPath, reason: removed.reason ?? "not-removed", }); return false; } catch (error) { await this.emitStaleLockAudit(taskId, "worktree:stale-lock-recovery-failed", path, { lockPath, reason: error instanceof Error ? error.message : String(error), }); return false; } } /** * Single attempt to create a worktree with conflict detection and recovery. * Returns the actual worktree path used (may differ from input if recovery generated new name). */ private async recoverStaleRegistration(taskId: string, path: string, conflictInfo: { path?: string; message?: string }): Promise { const staleRegistrationPath = conflictInfo.path ?? path; await this.emitStaleLockAudit(taskId, "worktree:stale-registration-detected", path, { staleRegistrationPath, worktreePath: path, }); const recovery = await recoverStaleRegistration({ rootDir: this.rootDir, worktreePath: path, logger: executorLog, }); if (recovery.recovered) { await this.emitStaleLockAudit(taskId, "worktree:stale-registration-recovered", path, { actions: recovery.actions, }); await this.store.logEntry(taskId, "Recovered stale worktree registration and retrying", staleRegistrationPath, this.getRunContextFor(taskId)); return true; } await this.emitStaleLockAudit(taskId, "worktree:stale-registration-recovery-failed", path, { actions: recovery.actions, reason: recovery.reason ?? "unknown", }); return false; } private async tryCreateWorktree( branch: string, path: string, taskId: string, startPoint?: string, attemptNumber = 0, recoveryDepth = 0, allowSiblingBranchRename = false, settings: Partial = {}, ): Promise<{ path: string; branch: string }> { // Guard: refuse to create a worktree nested inside another worktree. // Nested worktrees happen when the executor is launched with rootDir pointed // at a worktree directory instead of the main repo — produces paths like // `.worktrees/green-finch/.worktrees/amber-panda` that bloat the filesystem // and confuse every tool that walks git state. await this.assertWorktreePathNotNested(path, taskId); const installGuardOrCleanup = async () => { try { await installTaskWorktreeIdentityGuard({ worktreePath: path, taskId, commitMsgHookEnabled: settings.commitMsgHookEnabled, taskPrefix: settings.taskPrefix, taskAttributionTrailerName: settings.taskAttributionTrailerNames?.[0], commitAuthorEnabled: settings.commitAuthorEnabled, commitAuthorName: settings.commitAuthorName, commitAuthorEmail: settings.commitAuthorEmail, }); } catch (error) { try { await rm(path, { recursive: true, force: true }); } catch { executorLog.log(`Warning: failed to remove worktree after identity-guard install failure: ${path}`); } throw error; } }; // If directory exists but is not a registered worktree, remove it first if (existsSync(path)) { const isRegistered = await this.isRegisteredWorktree(path); if (!isRegistered) { await this.store.logEntry( taskId, `Removing existing directory (not a registered worktree): ${path}`, ); try { await rm(path, { recursive: true, force: true }); } catch (e: unknown) { const eMessage = e instanceof Error ? e.message : String(e); throw new Error(`Failed to remove existing directory ${path}: ${eMessage}`); } } else { executorLog.debug(`Worktree already exists: ${path}`); await installGuardOrCleanup(); return { path, branch }; } } const createWithBranch = async (branchToCreate: string) => { const cmd = startPoint ? `git worktree add -b "${branchToCreate}" "${path}" "${startPoint}"` : `git worktree add -b "${branchToCreate}" "${path}"`; try { await execAsync(cmd, { cwd: this.rootDir }); } catch (err) { // Remove any partial directory left behind so the invariant holds: // "if .worktrees/ exists on disk, it is a fully registered git worktree." try { await rm(path, { recursive: true, force: true }); } catch { // best-effort cleanup; log but don't mask the original error executorLog.log(`Warning: failed to remove partial worktree directory after creation failure: ${path}`); } throw err; } }; const createFromExistingBranch = async () => { try { await execAsync(`git worktree add "${path}" "${branch}"`, { cwd: this.rootDir }); } catch (err) { // Remove any partial directory left behind so the invariant holds: // "if .worktrees/ exists on disk, it is a fully registered git worktree." try { await rm(path, { recursive: true, force: true }); } catch { // best-effort cleanup; log but don't mask the original error executorLog.log(`Warning: failed to remove partial worktree directory after creation failure: ${path}`); } throw err; } }; let staleLockRecoveryAttempted = false; let staleRegistrationRecoveryAttempted = false; try { await createWithBranch(branch); executorLog.log(`Worktree created: ${path}${startPoint ? ` (from ${startPoint})` : ""}`); if (attemptNumber > 0) { await this.store.logEntry(taskId, `Worktree created on attempt ${attemptNumber + 1}`, path); } await installGuardOrCleanup(); return { path, branch }; } catch (initialError: unknown) { const conflictInfo = this.extractWorktreeConflictInfo(initialError); if (conflictInfo.type === "index-lock-contention" && !staleLockRecoveryAttempted) { staleLockRecoveryAttempted = true; const recovered = await this.recoverIndexLockIfStale(taskId, path, conflictInfo); if (recovered) { await createWithBranch(branch); executorLog.log(`Worktree created after stale lock recovery: ${path}`); await installGuardOrCleanup(); return { path, branch }; } } if (conflictInfo.type === "stale-registration" && !staleRegistrationRecoveryAttempted) { staleRegistrationRecoveryAttempted = true; const recovered = await this.recoverStaleRegistration(taskId, path, conflictInfo); if (recovered) { await createWithBranch(branch); executorLog.log(`Worktree created after stale registration recovery: ${path}`); await installGuardOrCleanup(); return { path, branch }; } } if (conflictInfo.type === "not-git-repo") { throw new NonRetryableWorktreeError( "Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.", ); } // Handle "already used by worktree" conflict if (conflictInfo.type === "already-used" && conflictInfo.path) { const result = await this.handleWorktreeConflict( conflictInfo.path, branch, path, taskId, startPoint, attemptNumber, allowSiblingBranchRename, settings, ); if (result) { return result; } throw new Error( `Worktree conflict at ${conflictInfo.path}: automatic cleanup failed`, ); } // Handle "invalid reference" - stale branch that doesn't exist if (conflictInfo.type === "invalid-reference") { if (recoveryDepth >= this.MAX_WORKTREE_RETRIES - 1) { throw new NonRetryableWorktreeError( `Stale branch reference for ${branch} remained invalid after ${this.MAX_WORKTREE_RETRIES} cleanup attempts`, ); } const branchCleaned = await this.cleanupStaleBranch(branch, taskId); if (branchCleaned) { await this.store.logEntry(taskId, `Removed stale branch reference, retrying`); return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, recoveryDepth + 1, allowSiblingBranchRename, settings); } throw new Error( `Invalid reference for branch ${branch}: unable to clean up stale reference`, ); } // Handle "could not create leading directories" - permission/path issues if (conflictInfo.type === "leading-directories") { throw new Error( `Cannot create worktree at ${path}: permission or path issue. ` + `Check that parent directories are writable.`, ); } // Try creating from existing branch (branch might already exist) try { await createFromExistingBranch(); executorLog.log(`Worktree created from existing branch: ${path}`); await installGuardOrCleanup(); return { path, branch }; } catch (fallbackError: unknown) { const fallbackErrorMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError); // Check if the fallback also hit an "already used" conflict const fallbackConflictInfo = this.extractWorktreeConflictInfo(fallbackError); if (fallbackConflictInfo.type === "index-lock-contention" && !staleLockRecoveryAttempted) { staleLockRecoveryAttempted = true; const recovered = await this.recoverIndexLockIfStale(taskId, path, fallbackConflictInfo); if (recovered) { await createFromExistingBranch(); executorLog.log(`Worktree created from existing branch after stale lock recovery: ${path}`); await installGuardOrCleanup(); return { path, branch }; } } if (fallbackConflictInfo.type === "stale-registration" && !staleRegistrationRecoveryAttempted) { staleRegistrationRecoveryAttempted = true; const recovered = await this.recoverStaleRegistration(taskId, path, fallbackConflictInfo); if (recovered) { await createFromExistingBranch(); executorLog.log(`Worktree created from existing branch after stale registration recovery: ${path}`); await installGuardOrCleanup(); return { path, branch }; } } if (fallbackConflictInfo.type === "not-git-repo") { throw new NonRetryableWorktreeError( "Project directory is not a Git repository. Fusion requires a Git repository for worktree creation. Initialize with 'git init' or run from a Git project directory.", ); } if (fallbackConflictInfo.type === "already-used" && fallbackConflictInfo.path) { const result = await this.handleWorktreeConflict( fallbackConflictInfo.path, branch, path, taskId, startPoint, attemptNumber, allowSiblingBranchRename, settings, ); if (result) { return result; } throw new Error( `Worktree conflict at ${fallbackConflictInfo.path}: automatic cleanup failed`, ); } // Handle stale reference in fallback path too if (fallbackConflictInfo.type === "invalid-reference") { if (recoveryDepth >= this.MAX_WORKTREE_RETRIES - 1) { throw new NonRetryableWorktreeError( `Stale branch reference for ${branch} remained invalid after ${this.MAX_WORKTREE_RETRIES} cleanup attempts`, ); } const branchCleaned = await this.cleanupStaleBranch(branch, taskId); if (branchCleaned) { await this.store.logEntry(taskId, `Cleaned up stale reference in fallback, retrying`); return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, recoveryDepth + 1, allowSiblingBranchRename, settings); } } throw new Error(`Failed to create worktree: ${fallbackErrorMessage}`); } } } /** * Handle "already used by worktree" conflict. * Either generates a new worktree name (if conflicting worktree is in use by active task) * or cleans up the conflicting worktree and retries. * * @returns The worktree path if recovery succeeded, null if recovery failed */ private async handleWorktreeConflict( conflictPath: string, branch: string, path: string, taskId: string, startPoint?: string, attemptNumber?: number, allowSiblingBranchRename = false, settings: Partial = {}, ): Promise<{ path: string; branch: string } | null> { const tryFreshFallback = () => this.tryFreshWorktreeAfterLiveConflict({ conflictPath, branch, taskId, startPoint, attemptNumber, allowSiblingBranchRename, settings, }); const shouldGenerateNewName = await this.shouldGenerateNewWorktreeName( conflictPath, taskId, ); /* * FNXC:ExecutorWorktree 2026-07-18-17:20: * Inspect every branch/worktree collision before cleanup, including inactive * same-task bindings. The old inactive path skipped inspection and called * cleanupConflictingWorktree directly, which force-deleted a branch carrying * completed task commits during workflow-node recovery. Liveness determines * whether a sibling checkout is needed; it must never determine whether task * history is disposable. */ const inspection = await inspectBranchConflict({ repoDir: this.rootDir, branchName: branch, conflictingWorktreePath: conflictPath, requestingTaskId: taskId, ownerTaskId: taskId, startPoint, integrationRef: await resolveIntegrationBranch(this.rootDir, settings), }); if (inspection.kind === "reclaimable") { const livePath = isInsideWorktreesDir(this.rootDir, inspection.livePath, settings) ? inspection.livePath : await this.normalizeReclaimableWorktreePath(inspection.livePath, path, taskId, settings); await this.store.logEntry( taskId, `[recovery] reclaimed existing worktree for ${taskId} at ${livePath} (${inspection.taskAttributedCommitCount} commits preserved)`, inspection.tipSha, ); return { path: livePath, branch }; } if (inspection.kind === "fully-subsumed") { const livePath = isInsideWorktreesDir(this.rootDir, inspection.livePath, settings) ? inspection.livePath : await this.normalizeReclaimableWorktreePath(inspection.livePath, path, taskId, settings); await this.store.logEntry( taskId, `[recovery] reclaimed existing worktree for ${taskId} at ${livePath} (0 commits preserved)`, inspection.tipSha, ); return { path: livePath, branch }; } if (shouldGenerateNewName) { if (inspection.kind === "stale" || inspection.kind === "stale-resolved" || inspection.kind === "tip-already-merged") { const cleanupSuccess = await this.cleanupConflictingWorktree(conflictPath, branch, taskId); if (cleanupSuccess) { await this.store.logEntry(taskId, `Cleaned up conflicting worktree, retrying`, path); return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, 0, allowSiblingBranchRename, settings); } // FN-4811: When git classifies a worktree as stale but the DB liveness gate refuses // removal (an active task still has this worktree bound), fall through to the // sibling-rename path rather than failing the whole conflict-recovery attempt. This // preserves the live task while letting the requesting task proceed with a fresh // worktree name. } if (inspection.kind === "live-foreign") { const cleanupSuccess = await this.cleanupConflictingWorktree(inspection.livePath, branch, taskId); if (cleanupSuccess) { await this.store.logEntry(taskId, `Removed foreign conflicting worktree and retrying`, inspection.livePath); return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, 0, allowSiblingBranchRename, settings); } // FN-4811: Cleanup was refused because the foreign worktree is actively bound to a // live session. Force-removing would yank an active task's filesystem. Fall through // to the sibling-rename path (suffix-2 through suffix-6) so the requesting task can // proceed without disturbing the live owner. If sibling-rename is disabled, the // generic conflict error below will trigger the caller's auto-recovery dispatcher. } if (!allowSiblingBranchRename) { throw new Error(`Branch ${branch} conflict could not be auto-resolved`); } return tryFreshFallback(); } const cleanupSuccess = await this.cleanupConflictingWorktree(conflictPath, branch, taskId); if (cleanupSuccess) { await this.store.logEntry(taskId, `Cleaned up conflicting worktree, retrying`, path); return this.tryCreateWorktree(branch, path, taskId, startPoint, attemptNumber, 0, allowSiblingBranchRename, settings); } if (await this.isLiveCleanupRefusal(conflictPath, taskId)) { return tryFreshFallback(); } return null; } private async normalizeReclaimableWorktreePath( sourcePath: string, targetPath: string, taskId: string, settings: Partial, ): Promise { const isRelocationActive = async (path: string) => this.hasActiveWorktreeBinding(taskId, path) || await this.isLiveCleanupRefusal(path, taskId); try { const placement = await relocateReclaimableWorktreeIntoRoot({ rootDir: this.rootDir, sourcePath, targetPath, taskId, settings, isPathActive: isRelocationActive, }); if (placement.kind === "deferred-live") { await this.store.logEntry( taskId, `[recovery] deferred relocation of active preserved worktree ${sourcePath}`, sourcePath, ); return placement.path; } if (placement.relocated) { await this.store.logEntry( taskId, `[recovery] relocated preserved worktree from ${sourcePath} to ${placement.path}`, placement.path, ); } return placement.path; } catch (error) { const detail = error instanceof Error ? error.message : String(error); await this.store.logEntry( taskId, `[recovery] failed to relocate preserved worktree from ${sourcePath} to ${targetPath}: ${detail}`, sourcePath, ); throw new NonRetryableWorktreeError( `Could not relocate preserved ${taskId} worktree into the configured worktrees directory: ${detail}`, ); } } private async tryFreshWorktreeAfterLiveConflict(input: { conflictPath: string; branch: string; taskId: string; startPoint?: string; attemptNumber?: number; allowSiblingBranchRename: boolean; settings: Partial; }): Promise<{ path: string; branch: string }> { const { conflictPath, branch, taskId, attemptNumber, allowSiblingBranchRename, settings } = input; if (!allowSiblingBranchRename) { throw new Error(`Branch ${branch} conflict could not be auto-resolved`); } const conflictStartPoint = branch; for (let suffix = 2; suffix <= 6; suffix++) { const suffixedBranch = `${branch}-${suffix}`; const newPath = resolveTaskWorktreePath(this.rootDir, settings, generateWorktreeName(this.rootDir, settings)); try { await this.store.logEntry( taskId, `Preserved active conflicting worktree and retrying with fresh worktree branch ${suffixedBranch}`, `${conflictPath} -> ${newPath}`, ); /* * FNXC:ExecutorWorktree 2026-07-01-00:00: * Active-session cleanup refusal must allocate a fresh worktree/branch instead of bubbling automatic cleanup failure. Removing the live conflicting path violates the FN-4811 invariant, so bounded sibling branches preserve the owner while letting the requesting task continue. */ return await this.tryCreateWorktree(suffixedBranch, newPath, taskId, conflictStartPoint, attemptNumber, 0, true, settings); } catch (suffixErr: unknown) { const info = this.extractWorktreeConflictInfo(suffixErr); if (info.type === "already-used") { continue; } throw suffixErr; } } throw new Error( `Cannot create branch for task: "${branch}"; live conflicting worktree ${conflictPath} was preserved and suffixes -2 through -6 are all in use by other worktrees`, ); } private async isLiveCleanupRefusal(worktreePath: string, taskId: string): Promise { const activeOwner = await this.findActiveWorktreeOwner(worktreePath, taskId); if (activeOwner !== null) return true; const activeRecord = activeSessionRegistry.lookupByPath(worktreePath); if (!activeRecord) return false; if (activeRecord.taskId !== taskId) return true; return executingTaskLock.has(taskId) || this.hasActiveWorktreeBinding(taskId, worktreePath); } /** * Check if a path is registered as a git worktree. */ private async isRegisteredWorktree(path: string): Promise { return isRegisteredGitWorktree(this.rootDir, path); } /** * Throw if `path` lies inside an existing registered worktree other than the * repo root. The repo root itself is a worktree (main branch) and must be * allowed — we only reject paths strictly *inside* a non-root worktree. */ private async assertWorktreePathNotNested(path: string, taskId: string): Promise { const target = resolvePath(path); const rootResolved = resolvePath(this.rootDir); const registered = await getRegisteredWorktreePaths(this.rootDir); for (const wt of registered) { if (wt === rootResolved) continue; // root is allowed as ancestor if (wt === target) continue; // exact match handled later as "already registered" const rel = relative(wt, target); if (rel && !rel.startsWith("..") && !isAbsolute(rel)) { await this.store.logEntry( taskId, `Refusing to create nested worktree`, `target ${target} is inside registered worktree ${wt}`, ); throw new NonRetryableWorktreeError( `Refusing to create worktree at ${target}: path is nested inside existing worktree ${wt}. ` + `This usually means the executor was launched with rootDir pointing at a worktree instead of the main repo.`, ); } } } /** * Determine if we should generate a new worktree name instead of cleaning up. * Returns true if the conflicting worktree is used by an active task. */ private async getWorktreeBranchMap(): Promise> { const { stdout } = await execAsync("git worktree list --porcelain", { cwd: this.rootDir, encoding: "utf-8" }); const map = new Map(); let currentWorktree: string | null = null; for (const line of stdout.split("\n")) { if (line.startsWith("worktree ")) { currentWorktree = line.slice("worktree ".length).trim(); } else if (line.startsWith("branch refs/heads/") && currentWorktree) { map.set(line.slice("branch refs/heads/".length).trim(), currentWorktree); } else if (!line.trim()) { currentWorktree = null; } } return map; } private async shouldGenerateNewWorktreeName( conflictPath: string, currentTaskId: string, ): Promise { // FNXC:Workspace 2026-06-21-12:00: KTD2 — a task may hold N worktree paths; the conflict check is membership across the set, not equality on a single path. for (const [taskId, worktreePaths] of this.activeWorktrees) { if (taskId !== currentTaskId && worktreePaths.has(conflictPath)) { return true; } } // Check if another non-done task uses this worktree const otherUser = await findWorktreeUser(this.store, conflictPath, currentTaskId); return otherUser !== null; } /** * FN-4811: Determine whether `worktreePath` is currently bound to an active executor or * merger session. If so, removing it would pull the rug out from under a live agent, * producing the FN-4781/FN-4804 symptoms (worktree disappears mid-task, two parallel runs, * cross-task contamination). Returns the task ID currently using the worktree, or null if * the worktree is safe to remove. * * Liveness sources, in order: * 1. In-memory `activeWorktrees` map (per-executor session tracking). * 2. DB-level: any non-done, non-paused, in-progress task with `task.worktree === path`. * * The requesting task is excluded from the check because `cleanupConflictingWorktree` is * only called for worktrees the requesting task is trying to displace. */ /** * FN-6782 leaked-slot reaper support: expose a read-only snapshot of the * in-memory `activeWorktrees` holders so SelfHealingManager can cross-check * each holder's task column and reclaim a slot whose holder is no longer * legitimately in-progress (the "in todo yet still maxWorktrees holder" * leak). Returns a copied array — never the live Map — so callers cannot * mutate executor state. The actual release still goes through * `clearPhantomExecutorBinding`, which refuses to detach live session * surfaces, so this introspection cannot by itself pull a worktree out from * under a running agent. */ listWorktreeHolders(): Array<{ taskId: string; worktreePath: string }> { const holders: Array<{ taskId: string; worktreePath: string }> = []; // FNXC:Workspace 2026-06-21-12:00: KTD2 — flat-map each task's Set into one holder row per worktree path. A workspace task emits N rows; the FN-6782 reaper (self-healing.ts) and in-process-runtime adapter key purely off taskId (verified) and are idempotent across duplicate-task rows, so multi-row holders do not mis-count maxWorktrees slots. for (const [taskId, worktreePaths] of this.activeWorktrees) { for (const worktreePath of worktreePaths) { holders.push({ taskId, worktreePath }); } } return holders; } private async findActiveWorktreeOwner( worktreePath: string, requestingTaskId: string, ): Promise { // FNXC:Workspace 2026-06-21-12:00: KTD2 — membership across the task's worktree set (a workspace task holds N). for (const [taskId, paths] of this.activeWorktrees) { if (taskId !== requestingTaskId && paths.has(worktreePath)) { return taskId; } } try { const tasks = await this.store.listTasks({ slim: true, includeArchived: false }); /* FNXC:WorkflowResolvedColumns 2026-07-30-16:40 (executor): "Who else is actively working in this worktree?" is the WIP role, not the id. NOT the query-filter class — this listTasks call passes no `column`. On a renamed board the check matched nobody, so the worktree read as unowned and a second task could be handed a checkout already in use. Resolved per CANDIDATE task, one IR cache for the scan, and only for rows that could still match. */ const ownerIrCache = new Map>>(); for (const t of tasks) { if (t.id === requestingTaskId) continue; const wipColumns = new Set(["in-progress"]); try { const ir = await resolveWorkflowIrForTask(this.store, t.id, ownerIrCache); if (ir) { const resolved = columnsWithFlag(ir, "countsTowardWip"); if (resolved.length > 0) { wipColumns.clear(); for (const id of resolved) wipColumns.add(id); } } } catch { /* degraded: legacy id only */ } if (!wipColumns.has(t.column)) continue; if (t.paused === true) continue; if (t.worktree === worktreePath) return t.id; // FNXC:Workspace 2026-06-22-09:00: workspace tasks hold their worktrees in // task.workspaceWorktrees, not the singular task.worktree column. The DB liveness // fallback must check those per-sub-repo paths too — otherwise a conflict against a // sub-repo worktree owned by an in-progress workspace task is missed, especially // before its in-memory activeWorktrees entry is (re)registered after restart. const wsEntries = t.workspaceWorktrees; if (wsEntries && Object.values(wsEntries).some((entry) => entry.worktreePath === worktreePath)) { return t.id; } } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`findActiveWorktreeOwner: DB liveness check failed for ${worktreePath}: ${msg}`); } return null; } /** * Clean up a conflicting worktree and its branch. * Handles locked worktrees by unlocking first. * Returns true if cleanup succeeded. */ private hasActiveWorktreeBinding(taskId: string, worktreePath: string): boolean { // FNXC:Workspace 2026-06-21-12:00: KTD2 — membership across the task's worktree set. const paths = this.activeWorktrees.get(taskId); return paths ? paths.has(worktreePath) : false; } private async reconcileSelfOwnedBeforeRemove(worktreePath: string, taskId: string): Promise { const outcome = reconcileSelfOwnedActiveSessionForRemoval( activeSessionRegistry, worktreePath, taskId, (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), { processActiveProbe: (probeTaskId) => executingTaskLock.has(probeTaskId), }, ); if (outcome.action === "reconciled") { executorLog.warn( `[FN-5346] ${taskId}: dropped stale self-owned activeSessionRegistry entry before removeWorktree at ${worktreePath}`, ); await this.store.logEntry(taskId, "Cleared stale self-owned active-session entry before remove", worktreePath); } else if (outcome.action === "process-active-refuses") { executorLog.warn( `[FN-5256] refused stale-self-owned reconcile for ${taskId}: process-active=true at ${worktreePath}`, ); await this.store.logEntry( taskId, "Refused stale self-owned reconcile — task still actively executing", worktreePath, ).catch(() => undefined); } else if (outcome.action === "too-recent-refuses") { executorLog.warn( `[FN-5256] refused stale-self-owned reconcile for ${taskId}: age=${outcome.ageMs}ms (<${outcome.minIdleMs}ms) at ${worktreePath}`, ); await this.store.logEntry( taskId, `Refused stale self-owned reconcile — registration too recent (${outcome.ageMs}ms < ${outcome.minIdleMs}ms)`, worktreePath, ).catch(() => undefined); } } /** Remove only this executor's store-scoped lifecycle disposer registrations. */ disposeStoreLifecycleDisposers(): void { this.unregisterTaskMoveDisposer?.(); this.unregisterTaskMoveDisposer = undefined; this.unregisterArchiveWorktreeDisposer?.(); this.unregisterArchiveWorktreeDisposer = undefined; this.unregisterArchiveWorkspaceWorktreeDisposer?.(); this.unregisterArchiveWorkspaceWorktreeDisposer = undefined; } private async removeOwnWorktreeWithReconcile(input: { worktreePath: string; settings: Settings; taskId: string; reason: RemovalReason; audit?: Parameters[0]["audit"]; }): Promise { await this.reconcileSelfOwnedBeforeRemove(input.worktreePath, input.taskId); const removeArgs = { worktreePath: input.worktreePath, rootDir: this.rootDir, settings: input.settings, taskId: input.taskId, reason: input.reason, audit: input.audit, expectedOwnerTaskId: input.taskId, liveOwnerProbe: (path: string, ownerTaskId: string) => this.hasActiveWorktreeBinding(ownerTaskId, path), // FN-5256: route the worktree-backend defensive reconcile through the // hardened gates (process-active + min-idle window). processActiveProbe: (probeTaskId: string) => executingTaskLock.has(probeTaskId), } as const; try { await removeWorktree(removeArgs); } catch (error: unknown) { if ( error instanceof ActiveSessionWorktreeRemovalError && error.details.taskId === input.taskId && !this.hasActiveWorktreeBinding(input.taskId, input.worktreePath) ) { // FN-5256: route the post-throw reconcile through the hardened path so // process-active and too-recent signals also gate this leg. const outcome = reconcileSelfOwnedActiveSessionForRemoval( activeSessionRegistry, input.worktreePath, input.taskId, (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), { processActiveProbe: (probeTaskId) => executingTaskLock.has(probeTaskId), }, ); if (outcome.action === "reconciled") { await this.store.logEntry( input.taskId, "Reconciled stale self-owned active-session registration (post-throw)", input.worktreePath, ); await removeWorktree(removeArgs); return; } if (outcome.action === "process-active-refuses" || outcome.action === "too-recent-refuses") { executorLog.warn( `[FN-5256] post-throw reconcile refused for ${input.taskId} at ${input.worktreePath}: action=${outcome.action}`, ); // Refused — surface the original error so the caller can decide. } } throw error; } } private async cleanupConflictingWorktree( worktreePath: string, branch: string, taskId: string, ): Promise { await this.reconcileSelfOwnedBeforeRemove(worktreePath, taskId); // FN-4811: Hard liveness gate — refuse to remove a worktree that is currently bound to // an active executor/merger session, regardless of git-level conflict classification. // This is the canonical guard against the FN-4781/FN-4804 race where a startup cleanup // pass or branch-conflict recovery yanked the worktree of a still-running session, causing // "assigned worktree path disappeared mid-task" + parallel-runs + cross-task contamination. const activeOwner = await this.findActiveWorktreeOwner(worktreePath, taskId); if (activeOwner !== null) { const refusalMessage = `[FN-4811] Refused to remove worktree ${worktreePath}: actively owned by ${activeOwner} (requested by ${taskId})`; executorLog.warn(refusalMessage); await this.store.logEntry(taskId, `Refused to remove conflicting worktree — actively owned by another task`, `${worktreePath} (owner: ${activeOwner})`); return false; } try { // Check if worktree is locked and unlock if needed try { await execAsync(`git worktree unlock "${worktreePath}"`, { cwd: this.rootDir, }); await this.store.logEntry(taskId, `Unlocked worktree`, worktreePath); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: failed to unlock conflicting worktree ${worktreePath} before cleanup: ${msg}`); } // Remove the worktree const settings = await this.store.getSettings(); await this.removeOwnWorktreeWithReconcile({ worktreePath, settings, taskId, reason: RemovalReason.ExecutorDispose, }); await this.store.logEntry(taskId, `Removed conflicting worktree`, worktreePath); // Delete the branch if it exists try { await execAsync(`git branch -D "${branch}"`, { cwd: this.rootDir, }); await this.store.logEntry(taskId, `Deleted branch`, branch); // FN-2165 regression guard: null baseBranch on any task that stored this branch await this.store.clearStaleExecutionStartBranchReferences([branch], taskId); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: failed to delete conflicting branch ${branch}: ${msg}`); } return true; } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); // FN-4811 follow-up (FN-4813): when `git worktree remove --force` fails because the // conflicting path isn't a recoverable git worktree, treat it as already-cleaned: // prune any stale admin entry, force-remove the leftover directory, best-effort delete // the branch, and return success so the caller can proceed with fresh worktree creation. // Without this recovery, every `tryCreateWorktree` retry on such a path fails with // "automatic cleanup failed". // // Three variants land here, all meaning "no live worktree to preserve at this path": // 1. `validation failed, cannot remove working tree` — stale admin entry, dir missing. // 2. `is not a working tree` — an orphan directory exists on disk but git never // registered it (e.g. a leaked worktree dir that outlived its admin entry). This // is the FN-6782 leak residue that collides with freshly generated worktree names. // 3. `No such file or directory` / ENOENT — the path is already gone. // // Exclude spawn failures (e.g. `spawn git ENOENT` when the git binary is missing or not // on PATH): those are environment errors, not "path is not a worktree" signals, and must // not be misread as a successful stale-path cleanup. const err = error as NodeJS.ErrnoException; const isSpawnFailure = typeof err?.syscall === "string" && err.syscall.startsWith("spawn"); const staleConflictPath = !isSpawnFailure && ( /validation failed, cannot remove working tree/i.test(errorMessage) || /is not a working tree/i.test(errorMessage) || /no such file or directory|ENOENT/i.test(errorMessage) ); if (staleConflictPath) { // The error string alone is NOT authoritative — it can name an unrelated path, or fire // on a live worktree under a racing/transient failure. Re-verify on disk before any // destructive action and refuse to force-remove anything that is still a real worktree, // out of bounds, reached through a symlink, or actively owned by a live session. Only a // genuine orphan directory inside the configured worktrees tree is safe to delete. const settings = await this.store.getSettings(); const stillRegistered = await isRegisteredGitWorktree(this.rootDir, worktreePath).catch(() => true); const activeOwner = await this.findActiveWorktreeOwner(worktreePath, taskId).catch(() => "unknown"); let safeToRemove = isInsideWorktreesDir(this.rootDir, worktreePath, settings) && !stillRegistered && activeOwner === null; if (safeToRemove && existsSync(worktreePath)) { try { if (lstatSync(worktreePath).isSymbolicLink()) { safeToRemove = false; } else if (!isInsideWorktreesDir(this.rootDir, realpathSync(worktreePath), settings)) { safeToRemove = false; } } catch { // Stat failed (path vanished mid-check) — nothing to remove; the prune/branch // cleanup below is still safe to run. } } if (!safeToRemove) { // A real/registered/out-of-bounds/owned/symlinked path we must not touch. Surface as a // cleanup failure so the operator-recovery path handles it instead of silently // claiming success (and never `rm -rf`-ing something we shouldn't). await this.store.logEntry( taskId, `Refused stale-path cleanup — path is not a safe orphan (registered=${stillRegistered}, owner=${activeOwner ?? "none"})`, worktreePath, ); return false; } try { await execAsync("git worktree prune", { cwd: this.rootDir, timeout: 30_000, maxBuffer: 10 * 1024 * 1024, }); } catch (pruneErr: unknown) { const pruneMsg = pruneErr instanceof Error ? pruneErr.message : String(pruneErr); executorLog.warn(`${taskId}: git worktree prune failed during stale-path cleanup of ${worktreePath}: ${pruneMsg}`); } // An orphan directory ("is not a working tree") won't be removed by prune — git // doesn't track it. Force-remove the leftover dir so the colliding name is free. if (existsSync(worktreePath)) { try { await rm(worktreePath, { recursive: true, force: true }); } catch (rmErr: unknown) { const rmMsg = rmErr instanceof Error ? rmErr.message : String(rmErr); executorLog.warn(`${taskId}: failed to remove orphan worktree directory ${worktreePath}: ${rmMsg}`); } } try { await execAsync(`git branch -D "${branch}"`, { cwd: this.rootDir }); await this.store.clearStaleExecutionStartBranchReferences([branch], taskId); } catch { // best-effort — branch may not exist, which is fine for a stale-path cleanup } await this.store.logEntry( taskId, `Cleaned up stale conflicting worktree (no live worktree at path — pruned admin entry and removed orphan directory)`, worktreePath, ); return true; } await this.store.logEntry( taskId, `Failed to clean up conflicting worktree`, `${worktreePath}: ${errorMessage}`, ); return false; } } /** * Clean up a stale branch that no longer has a valid reference. * * Recovery strategy (in order): * 1. `git worktree prune` — remove stale worktree metadata that may * hold a lock on the branch reference * 2. `git branch -D` — delete the branch normally * 3. `git update-ref -d refs/heads/` — force-remove a corrupted * or dangling reference when `git branch -D` fails * * Each step is logged so operators can trace the recovery path. * Returns true if the branch reference was successfully removed. */ private async cleanupStaleBranch(branch: string, taskId: string): Promise { // Step 1: Prune stale worktree metadata that may hold a lock on the branch try { await execAsync("git worktree prune", { cwd: this.rootDir }); await this.store.logEntry(taskId, `Pruned stale worktree metadata`, branch); } catch { // Prune is best-effort — continue even if it fails } // Step 2: Try normal branch deletion try { await execAsync(`git branch -D "${branch}"`, { cwd: this.rootDir, }); await this.store.logEntry(taskId, `Removed stale branch`, branch); // FN-2165 regression guard: null baseBranch on any task that stored this branch try { await this.store.clearStaleExecutionStartBranchReferences([branch], taskId); } catch { /* best-effort */ } return true; } catch (branchDeleteError: unknown) { const branchDeleteErrorMessage = branchDeleteError instanceof Error ? branchDeleteError.message : String(branchDeleteError); await this.store.logEntry( taskId, `git branch -D failed for stale branch, trying update-ref`, `${branch}: ${branchDeleteErrorMessage}`, ); } // Step 3: Force-remove the reference directly try { const refPath = `refs/heads/${branch}`; await execAsync(`git update-ref -d "${refPath}"`, { cwd: this.rootDir, }); await this.store.logEntry(taskId, `Force-removed stale branch reference via update-ref`, refPath); // FN-2165 regression guard: null baseBranch on any task that stored this branch try { await this.store.clearStaleExecutionStartBranchReferences([branch], taskId); } catch { /* best-effort */ } return true; } catch (updateRefError: unknown) { const updateRefErrorMessage = updateRefError instanceof Error ? updateRefError.message : String(updateRefError); await this.store.logEntry( taskId, `Failed to remove stale branch reference`, `${branch}: ${updateRefErrorMessage}`, ); return false; } } /** * Extract conflict information from git worktree error output. * Handles multiple error patterns: * - "already used by worktree at '...'" * - "invalid reference" / "unable to resolve reference" / "stale file handle" * - "could not create leading directories" * - "working tree already exists" */ private extractWorktreeConflictInfo(error: unknown): { type: "already-used" | "invalid-reference" | "leading-directories" | "already-exists" | "not-git-repo" | "index-lock-contention" | "stale-registration" | "unknown"; path?: string; lockPath?: string; message?: string; } { const execError = error instanceof Error ? error : new Error(String(error)); const output = [ execError.message, "stderr" in execError && typeof execError.stderr === "string" ? execError.stderr.toString() : undefined, "stdout" in execError && typeof execError.stdout === "string" ? execError.stdout.toString() : undefined, ] .filter(Boolean) .join("\n"); // Pattern: already used by worktree at '/path/to/worktree' const alreadyUsedMatch = output.match(/already used by worktree at '([^']+)'/); if (alreadyUsedMatch) { return { type: "already-used", path: alreadyUsedMatch[1], message: output }; } // Pattern: already checked out at '/path/to/worktree' const alreadyCheckedOutMatch = output.match(/is already checked out at '([^']+)'/); if (alreadyCheckedOutMatch) { return { type: "already-used", path: alreadyCheckedOutMatch[1], message: output }; } const lockPath = parseIndexLockPath(output); if (lockPath) { return { type: "index-lock-contention", lockPath, message: output }; } const staleRegistrationPath = parseStaleRegistrationPath(output); if (staleRegistrationPath) { return { type: "stale-registration", path: staleRegistrationPath, message: output }; } // Pattern: invalid reference: 'branch-name' // Also covers: unable to resolve reference, stale file handle, not a valid ref if ( output.match(/invalid reference/i) || output.match(/unable to resolve reference/i) || output.match(/stale file handle/i) || output.match(/not a valid ref/i) || output.match(/unable to delete.*ref/i) ) { return { type: "invalid-reference", message: output }; } // Pattern: could not create leading directories if (output.match(/could not create leading directories/i)) { return { type: "leading-directories", message: output }; } // Pattern: working tree already exists if (output.match(/working tree already exists/i)) { return { type: "already-exists", message: output }; } // Pattern: not a git repository / not a git repo if (output.match(/not a git repo(sitory)?/i)) { return { type: "not-git-repo", message: output }; } return { type: "unknown", message: output }; } /** * Remove a task's worktree, but only if no other in-progress or todo task * shares the same worktree path (dependency-chain reuse). The branch is * always cleaned up by the merger on a per-task basis. */ async cleanup(taskId: string): Promise { const worktreePaths = this.getActiveWorktreePaths(taskId); if (worktreePaths.length === 0) return; this.activeWorktrees.delete(taskId); // FNXC:Workspace 2026-06-21-12:00: KTD1 — in workspace mode the tracked path is the non-git workspace root (browse-only), never a removable worktree. Drop the in-memory tracking above but never remove the root. Per-repo worktree teardown returns in Phase B. if (this.workspaceConfig) { return; } // Non-workspace tasks hold a one-element set — preserve the original single-path removal semantics. const worktreePath = worktreePaths[0]; // Check if another task still needs this worktree const otherUser = await findWorktreeUser(this.store, worktreePath, taskId); if (otherUser) { executorLog.log(`Worktree retained for ${taskId} — still needed by ${otherUser}`); return; } try { const settings = await this.store.getSettings(); await this.removeOwnWorktreeWithReconcile({ worktreePath, settings, taskId, reason: RemovalReason.ExecutorDispose, }); executorLog.log(`Cleaned up worktree for ${taskId}`); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to clean up worktree for ${taskId}:`, errorMessage); } } /** * When the engine restarts mid-step, an `in-progress` step may have already * passed its code review (log: `code review Step N: APPROVE`) but not yet * been flipped to `done` by the agent's next `fn_task_update` call. Without * intervention, the next executor pass re-enters the step and replays plan * + code review, which we've measured at 5–20 min of pure waste per restart. * * This reconciler scans the task log for any in-progress step whose most * recent approved code review is newer than its most recent `→ pending` * transition, and marks those steps `done`. Subsequent resume logic then * advances to the next actually-pending step. */ private async recoverApprovedStepsOnResume(taskId: string): Promise { let detail: TaskDetail; try { detail = await this.store.getTask(taskId); } catch (err) { executorLog.warn(`${taskId}: recoverApprovedStepsOnResume getTask failed: ${err instanceof Error ? err.message : String(err)}`); return; } const log = detail.log ?? []; if (log.length === 0) return; let recovered = 0; for (let i = 0; i < detail.steps.length; i++) { if (detail.steps[i].status !== "in-progress") continue; let lastPendingAt = -1; let lastApproveAt = -1; const stepName = detail.steps[i].name; // Matches "Step 3 (My Step) → pending"; name is user-controlled, so match // on prefix rather than a regex built from the name. const transitionPrefix = `Step ${i} (${stepName}) → `; const approvePrefix = `code review Step ${i}:`; for (let j = 0; j < log.length; j++) { const action = log[j].action || ""; if (action.startsWith(transitionPrefix)) { const status = action.slice(transitionPrefix.length).trim(); if (status === "pending") lastPendingAt = j; } else if (action.startsWith(approvePrefix) && action.includes("APPROVE")) { lastApproveAt = j; } } if (lastApproveAt > lastPendingAt) { executorLog.log( `${taskId}: step ${i} ("${stepName}") already has an approved code review — marking done on resume (skipping review replay)`, ); try { await this.store.logEntry( taskId, `Step ${i} (${stepName}) recovered as done on resume — code review had already approved before the engine stopped`, ); await this.store.updateStep(taskId, i, "done"); recovered++; } catch (err) { executorLog.warn( `${taskId}: failed to recover step ${i} on resume: ${err instanceof Error ? err.message : String(err)}`, ); } } } if (recovered > 0) { executorLog.log(`${taskId}: recovered ${recovered} approved step(s) on resume`); } } /** * On resume (task already has a branch from a prior run), walk git history * and mark steps as done when a commit matching the step-completion convention * is found. This prevents the agent from redoing already-committed work after * an auto-requeue. * * Commit message convention (case-insensitive): * feat|chore|fix(FN-XXXX): complete Step N * * Called after the worktree is acquired and before the agent session starts. */ private async reconcileStepsFromGitHistory(taskId: string, detail: TaskDetail, worktreePath: string): Promise { const baseCommitSha = detail.baseCommitSha; if (!baseCommitSha) return; // Step-inversion read-through (KTD-12, U12): for graph-owned tasks, resolve // which artifact/parser governs the step list from the workflow's parse-steps // declaration so reconcile knows the step source. The `complete step N` // commit convention is parser-agnostic (every parser yields the same step // ordering the agent commits against), so the git-history reconcile below is // unchanged — this read-through records the governing source for diagnostics // and is the seam a future parser-specific reconcile would consult. Legacy // tasks (no parse-steps node) resolve to undefined and are untouched. try { const ir = await resolveWorkflowIrForTask(this.store, taskId); const stepSource = this.resolveTaskStepSource(ir); if (stepSource) { executorLog.log( `${taskId}: reconcile step source governed by parse-steps(artifact=${stepSource.artifact}, parser=${stepSource.parser})`, ); } } catch { // Read-through is diagnostic only; never block reconcile on it. } const pendingOrInProgressSteps = detail.steps.filter( (s, i) => (s.status === "pending" || s.status === "in-progress") && i > 0, ); if (pendingOrInProgressSteps.length === 0) return; let logOutput: string; try { const { stdout } = await execAsync( `git log "${baseCommitSha}..HEAD" --format=%ct%x09%s`, { cwd: worktreePath }, ); logOutput = stdout; } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`${taskId}: reconcileStepsFromGitHistory — git log failed: ${msg}`); return; } if (!logOutput.trim()) return; const latestPendingByStep = new Map(); for (const entry of detail.log ?? []) { const action = entry.action ?? ""; const match = action.match(/^Step (\d+) \(.+\) → pending$/); if (!match) continue; const stepIndex = Number.parseInt(match[1], 10); const pendingAt = Date.parse(entry.timestamp); if (!Number.isInteger(stepIndex) || !Number.isFinite(pendingAt)) continue; latestPendingByStep.set(stepIndex, Math.max(latestPendingByStep.get(stepIndex) ?? -1, pendingAt)); } /* FNXC:WorkflowResume 2026-06-30-08:02: Browser Verification and Code Review REVISE intentionally reopen the trailing implementation/verification suffix. FN-7273 showed git-history resume then found older `complete Step 5` commits from the previous attempt, tried to mark Step 5 done while Step 3 was active, and logged a false reconciliation after TaskStore rejected the out-of-order write. A reopened step may only be reconciled from a commit whose author time is newer than the latest `→ pending` transition for that step, and success is logged only after the store confirms the step is terminal. */ // Match: feat(FN-2978): complete Step 3 / chore(fn-2978)!: Complete step 3 const stepCommitRegex = /^(?:feat|chore|fix)\([Ff][Nn]-\d+\)(?:!)?:\s*complete\s+step\s+(\d+)/i; const reconciledStepIndices = new Set(); for (const line of logOutput.split("\n")) { const [commitSecondsRaw, ...messageParts] = line.split("\t"); const commitMs = Number.parseInt(commitSecondsRaw ?? "", 10) * 1000; const message = messageParts.join("\t").trim(); const match = message.match(stepCommitRegex); if (!match) continue; const stepIndex = parseInt(match[1], 10); if (Number.isNaN(stepIndex) || stepIndex < 0 || stepIndex >= detail.steps.length) continue; const latestPendingAt = latestPendingByStep.get(stepIndex); if (latestPendingAt !== undefined && (!Number.isFinite(commitMs) || commitMs <= latestPendingAt)) continue; const step = detail.steps[stepIndex]; if (step.status === "pending" || step.status === "in-progress") { reconciledStepIndices.add(stepIndex); } } for (const stepIndex of reconciledStepIndices) { const updated = await this.store.updateStep(taskId, stepIndex, "done"); const updatedStepStatus = updated.steps?.[stepIndex]?.status; if (updatedStepStatus !== "done" && updatedStepStatus !== "skipped") { executorLog.warn( `${taskId}: skipped git-history reconciliation log for Step ${stepIndex}; store kept status ${updatedStepStatus ?? "missing"}`, ); continue; } await this.store.logEntry( taskId, `Reconciled Step ${stepIndex} as done from git history (resume)`, undefined, this.getRunContextFor(taskId), ); executorLog.log(`${taskId}: reconciled Step ${stepIndex} as done from git history`); } if (reconciledStepIndices.size > 0) { // Refresh task and update currentStep to the lowest pending index const updated = await this.store.getTask(taskId); const lowestPending = updated.steps.findIndex((s) => s.status === "pending" || s.status === "in-progress"); if (lowestPending >= 0 && lowestPending !== updated.currentStep) { await this.store.updateTask(taskId, { currentStep: lowestPending }); executorLog.log(`${taskId}: set currentStep to ${lowestPending} after step reconciliation`); } } } /** * Check whether the task's branch has any unique commits compared to main. * If the branch has no unique commits and the task has steps marked done, * those steps represent lost uncommitted work — reset them to "pending" * so the next execution doesn't skip them. * * Called during stuck-kill cleanup when the worktree is about to be destroyed. */ private async resetStepsIfWorkLost(task: Task): Promise { const completedSteps = task.steps.filter( (s) => s.status === "done" || s.status === "in-progress", ); if (completedSteps.length === 0) return; const branchName = resolveTaskWorkingBranch(task); try { // Check if the branch has any unique commits vs main const { stdout: mergeBaseStdout } = await execAsync( `git merge-base "${branchName}" HEAD 2>/dev/null`, { cwd: this.rootDir, encoding: "utf-8" }, ); const { stdout: branchHeadStdout } = await execAsync( `git rev-parse "${branchName}" 2>/dev/null`, { cwd: this.rootDir, encoding: "utf-8" }, ); const mergeBase = mergeBaseStdout.trim(); const branchHead = branchHeadStdout.trim(); if (mergeBase === branchHead) { await this.resetLostWorkStepProgress(task, completedSteps.length, "branch had no commits"); } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn( `${task.id}: unable to prove surviving branch commits before worktree removal — resetting ${completedSteps.length} step(s) to pending: ${msg}`, ); /* FNXC:StuckRequeue 2026-06-27-23:55: Stuck-requeue cleanup is about to delete the checkout. If git cannot prove the branch has durable commits, treat completed/in-progress steps as lost work rather than preserving progress that may point at deleted uncommitted output. */ await this.resetLostWorkStepProgress(task, completedSteps.length, `git proof failed: ${msg}`); } } private async resetLostWorkStepProgress(task: Task, completedStepCount: number, reason: string): Promise { executorLog.warn( `${task.id} ${reason} — resetting ${completedStepCount} step(s) to pending`, ); for (let i = 0; i < task.steps.length; i++) { if (task.steps[i].status === "done" || task.steps[i].status === "in-progress") { await this.store.updateStep(task.id, i, "pending"); } } const refreshedTask = await this.store.getTask(task.id); const prevCurrentStep = refreshedTask.currentStep; if (refreshedTask.steps.length > 0) { const firstPendingStep = refreshedTask.steps.findIndex((s) => s.status === "pending"); const newCurrentStep = firstPendingStep >= 0 ? firstPendingStep : 0; if (newCurrentStep !== prevCurrentStep) { await this.store.updateTask(task.id, { currentStep: newCurrentStep }); executorLog.log( `${task.id}: reset currentStep to ${newCurrentStep} after lost-work reset (was ${prevCurrentStep})`, ); await this.store.logEntry( task.id, `Reset currentStep to ${newCurrentStep} after lost-work step reset (was ${prevCurrentStep})`, ); } } await this.store.logEntry( task.id, `Reset ${completedStepCount} step(s) to pending — ${reason} (uncommitted work lost with worktree)`, ); } /** * Mark a task as stuck-aborted so the executor's error handling * knows not to treat the disposed session as a genuine failure. * Called by the stuck task detector's onStuck callback. * * @param shouldRequeue — true to move the task back to "todo" for retry, * false if the stuck kill budget is exhausted (task already marked failed). */ markStuckAborted(taskId: string, shouldRequeue: boolean = true): void { // Terminate step-session executor if active const stepExecutor = this.activeStepExecutors.get(taskId); if (stepExecutor) { stepExecutor.terminateAllSessions().catch(err => executorLog.warn(`Failed to terminate step sessions for stuck task ${taskId}: ${err}`) ); } this.stuckAborted.set(taskId, shouldRequeue); // Safety net: if the executor's Promise never resolves (e.g. a bash subprocess // is blocking the agent session even after dispose()), force-requeue the task // directly after a short grace period. Without this, a task with a hung tool // call stays stranded in "in-progress" until the engine restarts. if (shouldRequeue && this.executing.has(taskId)) { const FORCE_REQUEUE_GRACE_MS = 60_000; // 60 s — generous, but bounded setTimeout(async () => { if (!this.executing.has(taskId)) return; // executor unwound normally — nothing to do // Re-check the latest column: self-healing may have already moved the // task out of in-progress (e.g. recoverCompletedTasks → in-review). // Force-requeueing in that case would clobber a valid recovery, undo // the worktree/branch state that recovery now relies on, and reset // step progress. let latestColumn: string | undefined; try { const latestTask = await this.store.getTask(taskId); latestColumn = latestTask.column; } catch (err: unknown) { executorLog.warn( `${taskId} force-requeue could not read latest task state: ${err instanceof Error ? err.message : String(err)}`, ); } /* FNXC:WorkflowLifecycleColumns 2026-07-30-21:40 (fleet): the board's wip lane; with the literal a renamed board skipped every force-requeue as "recovered concurrently". */ if (latestColumn && latestColumn !== (await this.resolveResumeLanes(taskId)).wip) { executorLog.log( `${taskId} force-requeue skipped — task is now in '${latestColumn}' (recovered concurrently)`, ); this.executing.delete(taskId); executingTaskLock.release(taskId); this.stuckAborted.delete(taskId); return; } executorLog.warn( `${taskId} still executing ${FORCE_REQUEUE_GRACE_MS / 1000}s after stuck-kill signal ` + `(likely a hung subprocess) — force-requeueing`, ); try { const settings = await this.store.getSettings(); const preserveProgress = settings.preserveProgressOnStuckRequeue !== false; const latestTask = await this.store.getTask(taskId); const worktreePath = this.getWorktreePath(taskId) ?? latestTask.worktree; /* FNXC:Workspace 2026-06-21-22:30: F8 — observability for the workspace case. A workspace task has no singular worktree (getWorktreePath returns undefined for a multi-worktree task, and latestTask.worktree is null on the browse-only root), so the removeWorktree block below silently no-ops. Per-repo teardown is Phase B; until then make the skip visible rather than silent. Behavior is unchanged. */ if (this.workspaceConfig && !worktreePath) { await this.store.logEntry( taskId, `workspace task ${taskId}: no singular worktree to force-requeue (per-repo teardown is Phase B)`, ); } await this.store.logEntry( taskId, `Force-kill cleanup starting after stuck-kill unwind timeout — reaping in-flight surfaces and worktree`, ); // Spawned children must be terminated before the canonical reaper clears // spawnedAgents bookkeeping; otherwise child agent sessions would be orphaned. await this.terminateAllChildren(taskId).catch((err: unknown) => { executorLog.warn(`${taskId}: spawned child cleanup failed during force-requeue: ${err instanceof Error ? err.message : String(err)}`); }); await this.awaitAbortInFlightTaskWork(taskId, "force-requeue after stuck-kill unwind timeout"); // awaitAbortInFlightTaskWork marks pausedAborted as a generic abort // signal (KB-PROV 2026-07-26: `engine-abort`, since the force-requeue is // engine-initiated and passes no `userCanceled`). // The force-requeue path has already handled the task move, so // clear it to prevent a later subprocess unwind from logging/moving as a pause. this.clearPausedAborted(taskId); /* FNXC:StuckRequeue 2026-06-27-23:15: The force path mirrors normal stuck-requeue cleanup: before reaping a hung executor's worktree, reconcile step progress against committed branch state so preserved progress never points at deleted uncommitted work. */ await this.resetStepsIfWorkLost(latestTask); let cleanupFailed = false; if (worktreePath && existsSync(worktreePath)) { try { await removeWorktree({ worktreePath, rootDir: this.rootDir, settings, taskId, reason: RemovalReason.ExecutorStuckKilled, expectedOwnerTaskId: taskId, liveOwnerProbe: (path, ownerTaskId) => this.hasActiveWorktreeBinding(ownerTaskId, path), }); executorLog.log(`${taskId}: removed worktree during force-requeue cleanup: ${worktreePath}`); } catch (cleanupErr: unknown) { cleanupFailed = true; const cleanupErrMessage = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr); executorLog.warn(`${taskId}: worktree removal failed during force-requeue cleanup (${worktreePath}): ${cleanupErrMessage}`); await this.store.logEntry(taskId, `Force-kill cleanup failed to remove worktree ${worktreePath}: ${cleanupErrMessage}`); } } this.activeWorktrees.delete(taskId); await this.store.logEntry( taskId, `Force-requeued after stuck-kill: executor did not unwind within ${FORCE_REQUEUE_GRACE_MS / 1000}s (hung subprocess)${preserveProgress ? " — progress preserved" : ""}`, ); await this.store.updateTask(taskId, { status: "queued", error: null, worktree: null, branch: null, }); await this.store.moveTask(taskId, await resolveReboundColumnFor(this.store, taskId), preserveProgress ? { preserveProgress: true } : undefined); // Remove from executing only after the hung surfaces and worktree have // been reaped, preventing a scheduler re-dispatch onto stale resources. this.executing.delete(taskId); executingTaskLock.release(taskId); this.stuckAborted.delete(taskId); this.loopRecoveryState.delete(taskId); await this.store.logEntry( taskId, cleanupFailed ? "Force-kill cleanup completed with non-fatal worktree removal failure — task requeued" : "Force-kill cleanup completed — in-flight surfaces reaped and task requeued", ); executorLog.log(`${taskId} force-requeued to todo after stuck-kill cleanup`); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`Failed to force-requeue stuck task ${taskId}: ${errorMessage}`); await this.store.logEntry(taskId, `Force-kill cleanup failed during stuck-kill force-requeue: ${errorMessage}`).catch(() => undefined); } }, FORCE_REQUEUE_GRACE_MS); } } /** * Handle a loop-detected event from the stuck task detector. * Attempts an in-process compact-and-resume before falling back to kill/requeue. * * This method is the `onLoopDetected` callback wired through the dashboard. * It: * 1. Checks if the task has an active session * 2. Rejects if the one-attempt ceiling has been reached * 3. Calls `compactSessionContext()` to compact the conversation * 4. Sets recovery-pending state so the execution flow can resume * * @returns true if the executor accepted recovery ownership (detector skips kill), * false if recovery should not be attempted (detector proceeds with kill/requeue) */ async handleLoopDetected(event: StuckTaskEvent): Promise { const { taskId } = event; const activeEntry = this.activeSessions.get(taskId); // No active session — can't compact, let detector kill/requeue if (!activeEntry) { executorLog.log(`${taskId} loop detected but no active session — falling back to kill/requeue`); return false; } // Check attempt ceiling (max 1 compact-and-resume per execute() lifecycle). // After this fallback, StuckTaskDetector -> SelfHealingManager.checkStuckBudget // enforces STUCK_LOOP_EXHAUSTED terminalization when retry budget is spent. const state = this.loopRecoveryState.get(taskId); if (state && state.attempts >= 1) { executorLog.log(`${taskId} loop detected but compact ceiling reached — falling back to kill/requeue`); return false; } // Attempt compaction const attempt = (state?.attempts ?? 0) + 1; executorLog.log(`${taskId} loop detected (attempt ${attempt}) — attempting compact-and-resume`); await this.store.logEntry(taskId, `Loop detected (${event.activitySinceProgress} events since last progress) — attempting compact-and-resume (attempt ${attempt})`); let compactionTimedOut = false; let compactionTimer: ReturnType | undefined; const abortActiveSession = () => { const sessionWithAbort = activeEntry.session as unknown as { abort?: () => Promise }; if (typeof sessionWithAbort.abort === "function") { void sessionWithAbort.abort().catch((err: unknown) => { executorLog.warn(`${taskId} loop compaction abort after timeout failed: ${err instanceof Error ? err.message : String(err)}`); }); } }; let compactResult: Awaited> | null; try { compactResult = await Promise.race([ compactSessionContext(activeEntry.session), new Promise((resolve) => { compactionTimer = setTimeout(() => { compactionTimedOut = true; abortActiveSession(); resolve(null); }, LOOP_COMPACTION_TIMEOUT_MS); }), ]); } finally { if (compactionTimer) clearTimeout(compactionTimer); } if (!compactResult) { const reason = compactionTimedOut ? `Context compaction timed out after ${LOOP_COMPACTION_TIMEOUT_MS / 1000}s` : "Context compaction failed or unavailable"; executorLog.log(`${taskId} ${reason.toLowerCase()} — falling back to kill/requeue`); await this.store.logEntry(taskId, `${reason} — falling back to kill/requeue`); return false; } if (this.activeSessions.get(taskId)?.session !== activeEntry.session) { executorLog.log(`${taskId} compaction completed after session changed — falling back to kill/requeue`); await this.store.logEntry(taskId, "Context compaction completed after session changed — falling back to kill/requeue"); return false; } executorLog.log(`${taskId} compaction succeeded (freed ${compactResult.tokensBefore} tokens) — setting recovery-pending`); await this.store.logEntry(taskId, `Context compacted successfully — will resume with fresh context`); // FN-5168: once loop recovery has fired in this execute() lifecycle, // ignored fn_task_update rebuffs can be promoted to no-progress churn. this.options.stuckTaskDetector?.markLoopObserved(taskId); // Mark recovery-pending so the execution flow can consume it this.loopRecoveryState.set(taskId, { attempts: attempt, pending: true }); // Steer the session with a resume prompt to break the loop try { await activeEntry.session.steer( "⚠️ Loop detected: you were repeating actions without making progress. " + "The conversation has been compacted. Review the current state carefully, " + "check what's already been done (git log, file contents), and take a different " + "approach. Do NOT repeat the same actions. Advance to the next step if the " + "current work is complete.", ); } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.error(`${taskId} failed to steer after compaction: ${errorMessage}`); // Recovery-pending is still set — the execution flow will handle it } return true; } /** * FNXC:Workspace 2026-06-21-12:00: KTD2 single-path-getter contract. Returns the task's sole worktree path for single-repo tasks (one-element set). For a multi-worktree workspace task there is no single answer — callers must read the per-repo `task.workspaceWorktrees` entry instead — so this returns undefined. A workspace task tracked only at the browse-only root also returns undefined, matching the "no removable single worktree" semantics. */ getWorktreePath(taskId: string): string | undefined { if (this.workspaceConfig) { return undefined; } return this.getActiveWorktreePaths(taskId)[0]; } // ── Agent Spawning ───────────────────────────────────────────────────── /** * Terminate all child agents spawned by a parent task. * Called from the finally block of agentWork when the parent session ends. */ private async terminateAllChildren(parentTaskId: string): Promise { const childIds = this.spawnedAgents.get(parentTaskId); if (!childIds || childIds.size === 0) return; executorLog.log(`Terminating ${childIds.size} child agents for parent ${parentTaskId}`); // Detach the parent generation before any agent-store await. A replacement // execution may register a new set for the same task ID while cleanup is // still settling; the old generation must never delete that new set. this.spawnedAgents.delete(parentTaskId); await Promise.all([...childIds].map((childId) => this.terminateChildAgent(childId))); } /** * Terminate a single child agent by ID. * Disposes the session, updates AgentStore state, and cleans up tracking Maps. */ private async terminateChildAgent(childId: string): Promise { const childSession = this.childSessions.get(childId); if (childSession) { childSession.dispose(); this.childSessions.delete(childId); } try { await this.options.agentStore?.updateAgentState(childId, "paused"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to update spawned child ${childId} state to 'terminated' during cleanup: ${msg}`); } this.pendingEphemeralDeletions.add(childId); try { await this.options.agentStore?.deleteAgent(childId); } catch (err: unknown) { if (!this.isBenignEphemeralDeleteRaceError(childId, err)) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to delete spawned agent ${childId}: ${msg}`); } } finally { this.pendingEphemeralDeletions.delete(childId); } this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1); } /** * Run a spawned child agent's task to completion. * Handles state transitions and cleanup. */ private async runSpawnedChild( agentId: string, childSession: AgentSession, taskPrompt: string, ): Promise { try { await this.options.agentStore?.updateAgentState(agentId, "running"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to update spawned child ${agentId} state to 'running': ${msg}`); } try { await promptWithFallback(childSession, taskPrompt); // Normal completion — mark as active (available) try { await this.options.agentStore?.updateAgentState(agentId, "active"); } catch (markActiveErr) { executorLog.warn(`Child agent ${agentId} updateAgentState(active) failed: ${markActiveErr instanceof Error ? markActiveErr.message : String(markActiveErr)}`); } } catch (err: unknown) { // Error during execution — mark as error try { await this.options.agentStore?.updateAgentState(agentId, "error"); } catch (markErrorErr) { executorLog.warn(`Child agent ${agentId} updateAgentState(error) failed: ${markErrorErr instanceof Error ? markErrorErr.message : String(markErrorErr)}`); } const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`Child agent ${agentId} failed: ${errorMessage}`); } finally { /* FNXC:AgentSpawning 2026-06-23-12:25: Server memory must return to baseline after spawned child execution. A normally completed child session owns provider/runtime state until disposed; deleting it from childSessions first makes later parent cleanup unable to reach it. */ if (this.childSessions.get(agentId) === childSession) { try { await childSession.dispose(); } catch (disposeErr) { executorLog.warn(`Child agent ${agentId} session dispose failed: ${disposeErr instanceof Error ? disposeErr.message : String(disposeErr)}`); } this.childSessions.delete(agentId); } this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1); } } /** * Create the fn_spawn_agent tool definition. * Allows the parent agent to spawn child agents with delegated tasks. */ private createSpawnAgentTool( taskId: string, worktreePath: string, settings: Settings, taskEnv?: NodeJS.ProcessEnv, ): ToolDefinition { return { name: "fn_spawn_agent", label: "Spawn Agent", description: "Spawn a child agent to handle parallel work or specialized sub-tasks. " + "Each child runs in its own git worktree (branched from your worktree) and executes autonomously. " + "When you end (fn_task_done), all spawned children are terminated.", parameters: spawnAgentParams, execute: async (_id: string, params: Static) => { const { name, role, task: taskPrompt, systemPromptOverride } = params; // Check if AgentStore is available if (!this.options.agentStore) { return { content: [{ type: "text" as const, text: "Agent spawning is not available (no AgentStore configured)" }], details: { agentId: "", state: "error" }, }; } /* FNXC:CapacityModel 2026-07-29-14:10 (two numbers — spawned agents count): `maxSpawnedAgentsPerParent` (5) and `maxSpawnedAgentsGlobal` (20) are DELETED. They were a THIRD and FOURTH limiter with their own private budgets, invisible to the two the operator configures — and they measured the wrong thing: a child that finished still counted against `totalSpawnedCount` until its parent task ended, so the cap throttled cumulative spawns rather than concurrent ones. A spawned child IS an agent and runs in its own git worktree (branched from the parent's), so it consumes both configured dimensions. It now checks the SAME project agent count every other lane checks, via the shared live-claim helper — one number, one answer, no private budget that can disagree with the board. This closes a real hole rather than only deleting knobs: children were counted by NEITHER capacity gate, so a fan-out could put up to 20 extra worktrees on disk while the scheduler believed the project was at its limit. */ const spawnClaimed = await computeTopLevelConcurrencyClaimedFromStore({ store: this.store, tasks: await this.store.listTasks({ slim: true, includeArchived: false }), }); const spawnCap = settings.maxConcurrent ?? 2; const liveChildren = this.totalSpawnedCount; if (spawnClaimed + liveChildren >= spawnCap) { return { content: [{ type: "text" as const, text: `Agent capacity reached (${spawnClaimed + liveChildren}/${spawnCap} running, including ${liveChildren} spawned child agent(s)). Wait for work to finish, or raise Max Concurrent Tasks.`, }], details: { agentId: "", state: "error" }, }; } /* FNXC:CapacityModel 2026-07-29-19:20 (PR #2579 review — greptile P1, TOCTOU): RESERVE THE SLOT SYNCHRONOUSLY, before the first await. The check above reads capacity, then several awaits follow (createAgent, createWorktree, updateAgentState) before `totalSpawnedCount` was incremented. Two parents calling fn_spawn_agent with one slot left both passed the check and both spawned — more agents and more worktrees than Max Concurrent Tasks permits, which is the very hole this change set out to close. JS is single-threaded, so incrementing here — with NO await between the read and the increment — makes check-and-reserve atomic against every other spawn call. The reservation is rolled back on any failure below, and the success path no longer double-counts. */ this.totalSpawnedCount++; let spawnReservationHeld = true; const releaseSpawnReservation = () => { if (!spawnReservationHeld) return; spawnReservationHeld = false; this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1); }; /* FNXC:CapacityModel 2026-08-01-02:40 (same class as the planning-admission gap, 374956ef23): The FNXC above says a child "consumes both configured dimensions" — and then gated only ONE. A child's worktree is not a task row, so the task-ledger gates never see it; count live children against the worktree budget here at the acquisition source, like planning admission now does. Runs AFTER the synchronous agent-slot reservation (its own TOCTOU rule: the awaits in this check must not reopen the two-racing-spawns hole — the reservation is already held, and a worktree refusal unwinds it). Absent/null maxWorktrees (worktrees off) falls through to the agent gate alone, matching every other lane. */ { const spawnMaxWorktrees = (settings as { maxWorktrees?: number | null }).maxWorktrees ?? 4; if (typeof spawnMaxWorktrees === "number" && Number.isFinite(spawnMaxWorktrees)) { const spawnTasks = await this.store.listTasks({ slim: true, includeArchived: false }); /* FNXC:WorkflowResolvedColumns 2026-08-01-03:05: TERMINAL IS A ROLE, NOT A NAME — same conversion as the planning-admission ledger this gate was copied from. Against the literals a RENAMED board matches neither `done` nor `archived`, so finished cards keep counting as live worktree holders, `heldWorktrees` only ever grows, and every spawn is refused on a board with free slots. A permanent refusal is worse than the over-spawn this gate exists to prevent, because it is silent. PROJECT-level (`resolveProjectColumnsForRoles`) because the ledger spans the whole board with no single task to resolve against; it is legacy-seeded, so a default board still excludes exactly `done` and `archived` and this is byte-identical there. */ const spawnTerminalColumns = await resolveProjectColumnsForRoles(this.store, ["complete", "archived"]); const heldWorktrees = spawnTasks.filter((t) => !spawnTerminalColumns.has(t.column) && typeof t.worktree === "string" && t.worktree.length > 0).length; // totalSpawnedCount already includes THIS reservation; heldWorktrees covers task lanes. if (heldWorktrees + this.totalSpawnedCount > spawnMaxWorktrees) { releaseSpawnReservation(); return { content: [{ type: "text" as const, text: `Worktree capacity reached (${heldWorktrees + this.totalSpawnedCount - 1}/${spawnMaxWorktrees} held, including spawned child agent(s)). Wait for work to finish, or raise Max Worktrees.`, }], details: { agentId: "", state: "error" }, }; } } } try { // Create agent in AgentStore with reportsTo = parent task ID const agent = await this.options.agentStore.createAgent({ name: name.trim(), role: role as AgentCapability, reportsTo: taskId, metadata: { type: "spawned", parentTaskId: taskId }, }); // Create git worktree for child (branched from parent's worktree) const childWorktreeName = generateWorktreeName(this.rootDir, settings); const childWorktreePath = resolveTaskWorktreePath(this.rootDir, settings, childWorktreeName); const childBranch = `fusion/spawn-${agent.id}`; await this.createWorktree(childBranch, childWorktreePath, taskId, worktreePath); // Transition agent to active state await this.options.agentStore.updateAgentState(agent.id, "active"); // Child agents inherit executor instructions const childInstructions = await this.resolveInstructionsForRole("executor", settings); // A non-empty systemPromptOverride lets the caller run the child as a // specific persona (e.g. a compound-engineering reviewer) instead of the // generic child executor. Executor instructions are still appended below. // // (U9 / KTD-7) The engine does NOT itself resolve the persona def file — // the calling skill reads `$FUSION_CE_AGENTS_DIR/.md` (the // FUSION_WORKFLOW_STEP_CONVENTIONS_PREAMBLE instructs a path-confined // read: confined to the install dir, `../` rejected, body-size sanity // checked) and passes the stripped body here. The override body is // therefore trusted only to the extent that read was confined; the // agents dir is plugin-installer-owned and lives OUTSIDE the task // worktree (so coding-mode plan/code-review steps can't write into it — // see assertPluginLocalAgentsTarget in the CE plugin installer). const personaOverride = systemPromptOverride?.trim(); const childBasePrompt = personaOverride ? `${personaOverride} Parent task: ${taskId} Child agent: ${agent.id} (${name})` : `You are a child agent spawned by a parent task executor. Your role: - Complete the delegated task in your own worktree. - Work autonomously, but stay tightly scoped to the delegated request. - Prefer existing project patterns over inventing new ones. - Run relevant tests and report what you verified. - Do not widen scope or refactor unrelated areas. Output expectations: - Provide a concise summary of what you changed. - Call out files touched and validations run. - Explicitly mention unresolved blockers if you could not finish. Parent task: ${taskId} Child agent: ${agent.id} (${name})`; const childSystemPrompt = buildSystemPromptWithInstructions(childBasePrompt, childInstructions); // Build skill selection context for child agent session const childTask = await this.store.getTask(taskId); const skillContext = await buildSessionSkillContext({ agentStore: this.options.agentStore!, task: childTask, sessionPurpose: "executor", projectRootDir: this.rootDir, pluginRunner: this.options.pluginRunner, }); const parentAgent = childTask.assignedAgentId ? await this.options.agentStore.getAgent(childTask.assignedAgentId).catch(() => null) : null; const childRuntimeHint = extractRuntimeHint(agent.runtimeConfig) ?? extractRuntimeHint(parentAgent?.runtimeConfig); // Resolve executor model via canonical lane hierarchy so child agents // honor project executionProvider/executionModelId overrides (parity // with main executor at the top of agentWork()). const childExecutorSessionModel = resolveExecutorSessionModel( undefined, undefined, settings, agent.runtimeConfig as Record | undefined, ); const { provider: childExecutorProvider, modelId: childExecutorModelId } = childExecutorSessionModel; const childExecutorFallback = resolveExecutorFallbackModel(settings); // Create child agent session const { session: childSession } = await createResolvedAgentSession({ sessionPurpose: "executor", runtimeHint: childRuntimeHint, pluginRunner: this.options.pluginRunner, cwd: childWorktreePath, systemPrompt: childSystemPrompt, tools: "coding", defaultProvider: childExecutorProvider, defaultModelId: childExecutorModelId, ...(childExecutorSessionModel.credentialInstanceId ? { credentialInstanceId: childExecutorSessionModel.credentialInstanceId } : {}), fallbackProvider: childExecutorFallback.provider, fallbackModelId: childExecutorFallback.modelId, fallbackThinkingLevel: resolveExecutorFallbackThinkingLevel(undefined, settings), runAuditor: createRunAuditor(this.store, this.getRunContextFor(taskId)), settings, taskEnv, mcpServers: await this.resolveMcpServers(agent.id), // FNXC:SessionRouting 2026-06-24-11:20: // #1675: propagate task id so child-agent requests carry the same // X-Session-Id/X-Session-Affinity as the parent task session. taskId, // FNXC:PluginSkills 2026-07-12-00:00: Child-agent sessions inherit plugin skill body directories from the task skill context so delegated work can load plugin skill guidance. ...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}), ...(skillContext.additionalSkillPaths.length > 0 ? { additionalSkillPaths: skillContext.additionalSkillPaths } : {}), }); // Store tracking state this.childSessions.set(agent.id, childSession); if (!this.spawnedAgents.has(taskId)) { this.spawnedAgents.set(taskId, new Set()); } this.spawnedAgents.get(taskId)!.add(agent.id); // The slot was already reserved before the awaits above; converting the // reservation into the live count is a no-op rather than a second increment. spawnReservationHeld = false; // Run child asynchronously (don't await — parent continues working) this.runSpawnedChild(agent.id, childSession, taskPrompt).catch((err: unknown) => { const errorMessage = err instanceof Error ? err.message : String(err); executorLog.warn(`Child agent ${agent.id} async error: ${errorMessage}`); }); const result: SpawnAgentResult = { agentId: agent.id, name: agent.name, state: "running", role: agent.role, message: `Agent "${name}" spawned and executing task: ${taskPrompt.slice(0, 100)}${taskPrompt.length > 100 ? "..." : ""}`, }; return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }], details: result, }; } catch (err: unknown) { // FNXC:CapacityModel 2026-07-29-19:20: a failed spawn must return the slot // it reserved, or a project permanently loses capacity to a spawn that // never happened. releaseSpawnReservation(); const errorMessage = err instanceof Error ? err.message : String(err); return { content: [{ type: "text" as const, text: `Failed to spawn agent: ${errorMessage}` }], details: { agentId: "", state: "error", message: errorMessage }, }; } }, }; } } /** * Format a timestamp for display in steering comments. * Returns relative time for recent comments, absolute date for older ones. */ function formatTimestamp(iso: string): string { const date = new Date(iso); const now = new Date(); const diffMs = now.getTime() - date.getTime(); const diffMin = Math.floor(diffMs / 60000); const diffHr = Math.floor(diffMin / 60); const diffDay = Math.floor(diffHr / 24); if (diffMin < 1) return "just now"; if (diffMin < 60) return `${diffMin}m ago`; if (diffHr < 24) return `${diffHr}h ago`; if (diffDay < 7) return `${diffDay}d ago`; return date.toLocaleDateString(); } // Project commands are injected here (for reliability) and also in the PROMPT.md (by triage). // This ensures the executor agent always sees the authoritative commands from settings, // even if the PROMPT.md was written manually or before commands were configured. function scopePromptToWorktree(prompt: string | undefined, rootDir?: string, worktreePath?: string, workspaceConfig?: WorkspaceConfig | null): string { /* * FNXC:ExecutorPrompts 2026-06-29-13:55: * Some legacy direct-dispatch tests and recovered task rows can lack a persisted prompt. Treat a missing prompt as empty before worktree path scoping so prompt construction cannot fail before pause-abort and graph-path recovery code handles the task state. */ const promptText = prompt ?? ""; // FNXC:Workspace 2026-06-21-12:00: KTD1 — in workspace mode the session is rooted at the workspace root itself (worktreePath === rootDir) and path rewriting to a per-task root worktree is meaningless: edits happen in per-sub-repo worktrees the agent acquires, not at the root. No-op the rewrite. (The rootDir === worktreePath guard below already covers this, but gate explicitly so intent survives future refactors.) if (workspaceConfig) { return promptText; } if (!rootDir || !worktreePath || rootDir === worktreePath || !promptText.includes(rootDir)) { return promptText; } return promptText .replaceAll(`${rootDir}/`, `${worktreePath}/`) .replaceAll(`${worktreePath}/.fusion/`, `${rootDir}/.fusion/`); } function buildSourceIssueRef(sourceIssue: TaskDetail["sourceIssue"]): string { if (!sourceIssue || sourceIssue.provider !== "github" || !sourceIssue.repository) { return ""; } const issueNumber = sourceIssue.issueNumber ?? Number.parseInt(sourceIssue.externalIssueId ?? "", 10); if (!Number.isInteger(issueNumber) || issueNumber < 1) { return ""; } return `${sourceIssue.repository}#${issueNumber}`; } export function buildExecutionPrompt( task: TaskDetail, rootDir?: string, settings?: Settings, worktreePath?: string, pluginRunner?: PluginRunner, customFieldDefs?: WorkflowFieldDefinition[], workspaceConfig?: WorkspaceConfig | null, options?: { pluginTaskContributions?: string }, ): string { const prompt = scopePromptToWorktree(task.prompt, rootDir, worktreePath, workspaceConfig); const reviewLevel = parseReviewLevelFromPrompt(prompt); /* * FNXC:WorkflowReviewGates 2026-06-29-20:41: * Default Coding and other workflow-graph tasks run review gates as graph nodes, so the executor prompt must not ask implementation agents to call legacy per-step review tools. This keeps Plan Review once-before-execution and Code Review once-before-merge unless a workflow explicitly adds a step-review node. */ // Build co-author trailer arg for git commits based on settings. The user's // configured git identity remains the primary author; Fusion is appended as // a `Co-authored-by` trailer for shared credit (recognized by GitHub). // FNXC:CommitAttribution 2026-06-26-12:48: this prompt hint is best-effort for humans/agents reading commit examples; the worktree commit-msg hook is the authoritative deterministic source for the co-author trailer. const authorArg = settings?.commitAuthorEnabled !== false ? ` -m "Co-authored-by: ${settings?.commitAuthorName || "Fusion"} <${settings?.commitAuthorEmail || "noreply@runfusion.ai"}>"` : ""; const sourceIssueRef = buildSourceIssueRef(task.sourceIssue); // Build step progress for resume const hasProgress = task.steps.length > 0 && task.steps.some((s) => s.status !== "pending"); let progressSection = ""; if (hasProgress) { const doneSteps = task.steps .map((s, i) => ({ ...s, index: i })) .filter((s) => s.status === "done"); const currentStep = task.currentStep; const currentStepInfo = task.steps[currentStep]; progressSection = ` ## ⚠️ RESUMING — Previous progress exists This task was already partially executed. DO NOT redo completed steps. ### Step status: ${task.steps.map((s, i) => `- Step ${i} (${s.name}): **${s.status}**`).join("\n")} ### Resume from: Step ${currentStep}${currentStepInfo ? ` (${currentStepInfo.name})` : ""} ${doneSteps.length > 0 ? `Steps ${doneSteps.map((s) => s.index).join(", ")} are already complete — skip them entirely.` : ""} Check the git log to understand what was already implemented: \`\`\`bash git log --oneline \`\`\` `; } // Build attachments section let attachmentsSection = ""; if (task.attachments && task.attachments.length > 0 && rootDir) { const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]); const lines = ["## Attachments", ""]; for (const att of task.attachments) { const absPath = `${rootDir}/.fusion/tasks/${task.id}/attachments/${att.filename}`; if (IMAGE_MIMES.has(att.mimeType)) { lines.push(`- **${att.originalName}** (screenshot): \`${absPath}\``); } else { lines.push(`- **${att.originalName}** (${att.mimeType}): \`${absPath}\` — read for context`); } } attachmentsSection = "\n" + lines.join("\n") + "\n"; } // Build project commands section from settings let commandsSection = ""; if (settings?.testCommand || settings?.buildCommand) { const lines = ["## Project Commands"]; if (settings.testCommand) lines.push(`- **Test:** \`${settings.testCommand}\``); if (settings.buildCommand) lines.push(`- **Build:** \`${settings.buildCommand}\``); commandsSection = "\n" + lines.join("\n") + "\n"; } // Build project memory section from settings // When enabled, agents consult and update project memory for durable project learnings. // Backend-aware: instructions branch based on memoryBackendType (file, readonly, qmd) const memoryEnabled = settings?.memoryEnabled !== false; const memoryMode: AgentMemoryInclusionMode = settings?.agentMemoryInclusionMode ?? "full"; let memorySection = ""; if (memoryEnabled && rootDir && memoryMode !== "off") { memorySection = memoryMode === "index" ? "\n## Project Memory (Index Only)\n\nUse fn_memory_search first to find relevant memory, then fn_memory_get for specific excerpts.\n" : "\n" + buildExecutionMemoryInstructions(rootDir, settings); } // Build steering comments section (last 10 comments only to avoid context bloat) let steeringSection = ""; if (task.steeringComments && task.steeringComments.length > 0) { const recentComments = [...task.steeringComments].slice(-10); const lines = [ "", "## Steering Comments", "", "The following comments were added by the user during execution. Consider adjusting your approach or replanning remaining steps based on this feedback.", "", ]; for (const comment of recentComments) { const timestamp = formatTimestamp(comment.createdAt); lines.push(`**${comment.author}** — ${timestamp}`); lines.push(`> ${comment.text}`); lines.push(""); } steeringSection = lines.join("\n"); } // Build custom fields section (KTD-13): when the task's workflow declares // custom fields, the executor agent can write them via fn_task_update // (custom_fields) — but without the schema it is writing blind. List each // field's id/name/type, enum options, required flag, and current value so // the write is informed and self-correcting. Compact: one line per field. let customFieldsSection = ""; if (customFieldDefs && customFieldDefs.length > 0) { const current = task.customFields ?? {}; const lines = [ "", "## Custom fields", "", "This task's workflow declares custom fields. Set them with `fn_task_update(custom_fields={...})` keyed by field id (pass null to clear).", "", ]; for (const f of customFieldDefs) { const parts = [`- \`${f.id}\` (${f.name}) — type: ${f.type}`]; if ((f.type === "enum" || f.type === "multi-enum") && f.options && f.options.length > 0) { const opts = f.options.map((o) => (o.label && o.label !== o.value ? `${o.value} (${o.label})` : o.value)).join(", "); parts.push(`options: [${opts}]`); } if (f.required) parts.push("required"); const hasValue = Object.prototype.hasOwnProperty.call(current, f.id) && current[f.id] !== null && current[f.id] !== undefined; parts.push(`current: ${hasValue ? JSON.stringify(current[f.id]) : "unset"}`); lines.push(parts.join("; ")); } customFieldsSection = lines.join("\n") + "\n"; } const pluginTaskContributions = options?.pluginTaskContributions ?? ""; if (pluginTaskContributions) { executorLog.debug(`${task.id}: applied plugin prompt contributions for executor-task surface`); } const executionPrompt = `Execute this task. ## Task: ${task.id} ${task.title ? `**${task.title}**` : ""} ${task.dependencies.length > 0 ? `Dependencies: ${task.dependencies.join(", ")}` : ""} ## PROMPT.md ${prompt} ${attachmentsSection}${commandsSection}${memorySection}${progressSection}${steeringSection}${customFieldsSection} ## Review level: ${reviewLevel} Workflow review gates are handled by the workflow graph outside this implementation session. Do not request per-step plan review or per-step code review from inside execution; complete the implementation steps and let the graph run enabled Plan Review, Browser Verification, and Code Review nodes at their configured positions. ${pluginTaskContributions ? ` ${pluginTaskContributions} ` : ""} ## Worktree Boundaries You are running in an **isolated git worktree**. This means: - **All code changes must be made inside the current worktree directory.** Do not modify files outside the worktree. - **Exception — Project memory:** You MAY read and write to files under \`.fusion/memory/\` at the project root to save durable project learnings. - **Exception — Task attachments:** You MAY read files under \`.fusion/tasks/{taskId}/attachments/\` at the project root for context. - **Exception — Sibling task specs:** You MAY read \`.fusion/tasks/{taskId}/PROMPT.md\` and \`.fusion/tasks/{taskId}/task.json\` at the project root (read-only) to consult dependency tasks' specifications. If those files do not exist, the dependency has been archived — call \`fn_task_show\` with its ID to load the spec from the archive. - **Shell commands** run inside the worktree by default. Avoid using \`cd\` to navigate outside the worktree. ## Begin ${hasProgress ? `Resume from Step ${task.currentStep}. Do NOT redo completed steps.` : "Start with Step 0 (Preflight). Work through each step in order."} Use \`fn_task_update\` to report progress on every step transition; its \`step\` value is 0-based and equals the \`### Step N:\` number in PROMPT.md. Use \`fn_task_log\` for important actions and decisions. Use \`fn_task_create\` for truly separate follow-up work, including unrelated/pre-existing broad-suite failures. Commit at step boundaries: \`git commit -m "feat(${task.id}): complete Step N — "${sourceIssueRef ? ` -m "Ref: ${sourceIssueRef}"` : ""}${authorArg}\` The \`\` is required — replace it with a concrete 5–10 word description of what the step changed. When all steps are complete: call \`fn_task_done()\` If a build command is configured, run that exact command in this worktree before calling \`fn_task_done()\`. Treat a non-zero exit code as a blocking failure. Do not claim success without a real passing run. Run impacted/package-scoped tests before completion. Run the configured workspace test command only when the task/workflow explicitly requires it or after impacted checks pass for final integration. If any broad command fails, classify the failure before editing: caused-by-this-task failures are blocking; unrelated or pre-existing failures should be logged and split into a follow-up instead of expanding this task. If the repo has a lint command (e.g. \`pnpm lint\`, \`npm run lint\`), run it before \`fn_task_done()\` and fix any failures it reports. If the repo has a typecheck command, run it before \`fn_task_done()\` and fix any failures it reports. Use \`fn_task_create\` for truly separate follow-up work, including unrelated/pre-existing broad-suite failures. If lint is configured and failing, fix that too before completion. Do not repeatedly rerun a broad failing or hanging workspace command without a new hypothesis and a narrower confirming command.`; if (workspaceConfig && workspaceConfig.repos.length > 0) { return executionPrompt + `\n\n## Workspace mode\n` + `This project is a workspace containing multiple git repositories.\n` + `Available repos:\n` + workspaceConfig.repos.map((r: string) => `- \`${r}\``).join("\n") + `\n\nBefore editing files in any sub-repo, call \`fn_acquire_repo_worktree\` ` + `with the repo name to get an isolated worktree path. ` + `Work exclusively inside that returned path — never edit the repo's main checkout directly.\n`; } return executionPrompt; } /** * Format a comment for injection into a running agent session. * Used for real-time steering during task execution. */ function formatCommentForInjection(comment: import("@fusion/core").SteeringComment): string { const timestamp = formatTimestamp(comment.createdAt); return `📣 **New feedback** — ${timestamp} (${comment.author}):\n\n${comment.text}\n\nPlease adjust your approach based on this feedback.`; } function hasNonTerminalWorkflowSteps(task: Pick): boolean { return task.steps.length > 0 && task.steps.some((step) => step.status !== "done" && step.status !== "skipped"); } export { clearTerminalWorkflowStepFailures } from "./executor/workflow-step-failures.js"; import { clearTerminalWorkflowStepFailures } from "./executor/workflow-step-failures.js"; function workflowStepResultPassed(task: Pick | undefined, workflowStepId: string): boolean { const results = task?.workflowStepResults ?? []; return results.some((result) => result.workflowStepId === workflowStepId && result.phase === "pre-merge" && result.status === "passed", ); } function areExplicitEnabledWorkflowStepsSatisfied( task: Pick | undefined, ): boolean { const enabled = task?.enabledWorkflowSteps; if (!Array.isArray(enabled) || enabled.length === 0) return false; return enabled.every((id) => workflowStepResultPassed(task, id)); } function hasUnsatisfiedExplicitEnabledWorkflowSteps( task: Pick | undefined, ): boolean { const enabled = task?.enabledWorkflowSteps; return Array.isArray(enabled) && enabled.length > 0 && !areExplicitEnabledWorkflowStepsSatisfied(task); } function areEnabledPreMergeWorkflowStepsSatisfied( task: Pick | undefined, ): boolean { const preMergeGateIds = new Set(["plan-review", "browser-verification", "code-review"]); const enabled = task?.enabledWorkflowSteps; /* * FNXC:WorkflowLifecycle 2026-06-29-04:46: * Older/default coding tasks may not persist an explicit enabledWorkflowSteps * list even though default-on Plan Review and Code Review have already run. * Treat those two passed rows as satisfied defaults; keep explicit arrays * strict so custom/unknown enabled gates still re-enter the graph. */ const enabledPreMerge = Array.isArray(enabled) && enabled.length > 0 ? enabled.filter((id) => preMergeGateIds.has(id)) : ["plan-review", "code-review"]; if (enabledPreMerge.length === 0) return false; if (Array.isArray(enabled) && enabledPreMerge.length !== enabled.length) return false; return enabledPreMerge.every((id) => workflowStepResultPassed(task, id)); } function preservePreExecutionWorkflowStepResults(task: Pick): CoreWorkflowStepResult[] { /* * FNXC:WorkflowLifecycle 2026-06-29-03:50: * Reverification cleanup must clear post-implementation verification residue * without erasing pre-execution Plan Review evidence. FN-7228 passed Plan * Review, then stale merge-state cleanup reset `workflowStepResults` to `[]`; * the dashboard showed Plan Review with no status while execution continued and * the graph no longer had durable proof to skip duplicate plan review. * * FNXC:WorkflowLifecycle 2026-06-29-04:19: * The durable Plan Review row may already be missing when stale merge cleanup * runs, while the task log still has the authoritative terminal Plan Review * entry. Reconstruct the passed row from that log so execution can continue * with a visible pre-execution review status instead of showing an active task * card with Plan Review blank. */ const preserved = (task.workflowStepResults ?? []).filter((result) => result.workflowStepId === "plan-review"); if (preserved.length > 0) return preserved; let latest: { timestamp?: string; outcome?: string; status: "passed" | "failed" } | undefined; for (const entry of task.log ?? []) { if (entry.action === "[pre-merge] Workflow step completed: Plan Review") { latest = { timestamp: entry.timestamp, outcome: entry.outcome, status: "passed" }; } else if (entry.action === "[pre-merge] Workflow step failed: Plan Review") { latest = { timestamp: entry.timestamp, outcome: entry.outcome, status: "failed" }; } } if (latest?.status !== "passed") return []; return [ { workflowStepId: "plan-review", workflowStepName: "Plan Review", phase: "pre-merge", status: "passed", verdict: "APPROVE", ...(latest.outcome ? { output: latest.outcome, notes: latest.outcome } : {}), ...(latest.timestamp ? { startedAt: latest.timestamp, completedAt: latest.timestamp } : {}), }, ]; } export { detectPseudoPause, detectReviewHandoffIntent, } from "./executor/pseudo-pause.js"; export type { PseudoPauseResult } from "./executor/pseudo-pause.js"; import { detectPseudoPause, detectReviewHandoffIntent, } from "./executor/pseudo-pause.js";