diff --git a/.changeset/fn-workflow-executor-agent-store-wiring.md b/.changeset/fn-workflow-executor-agent-store-wiring.md new file mode 100644 index 0000000000..a7d233d15e --- /dev/null +++ b/.changeset/fn-workflow-executor-agent-store-wiring.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix tasks stalling forever in progress with no session after the workflow role-agent rollout. +category: fix +dev: Two deadlocks in FN-8764's role routing, both silent. (1) The in-process runtime never passed its AgentStore into `TaskExecutorOptions`, so routing failed closed at every role-classified node. (2) A resumed run keeps its continuation work item active until the interpreter returns, so the next node's principal-fence upsert hit `idx_workflow_work_items_one_active_task_continuation` (not its ON CONFLICT target) and raised; the run then re-suspended every dispatch until an operator bounced the card. The executor now supersedes an active work item for a node the run has already left and retries once. Adds the `task:workflow-run-suspended` and `task:workflow-continuation-superseded` run-audit events, and logs principal holds and fence-write errors instead of swallowing them. diff --git a/packages/engine/src/__tests__/executor-continuation-slot-handover.test.ts b/packages/engine/src/__tests__/executor-continuation-slot-handover.test.ts new file mode 100644 index 0000000000..aae2f9f255 --- /dev/null +++ b/packages/engine/src/__tests__/executor-continuation-slot-handover.test.ts @@ -0,0 +1,166 @@ +/* +FNXC:WorkflowAgentRouting 2026-08-07-23:35: +A task may hold only ONE active workflow work item +(`idx_workflow_work_items_one_active_task_continuation`: kind='task' AND state IN +runnable/running/held/retrying). A RESUMED graph run keeps the continuation it woke on active +until the interpreter returns — which is strictly AFTER a later role-classified node needs to +write its own durable principal fence. That fence upsert's ON CONFLICT target is +(run_id, task_id, node_id, kind), a DIFFERENT index, so the stale row does not upsert: the +write raises, routing returns `workflow-principal-fence-unavailable:`, and the run +suspends. Nothing about that state changes between dispatches, so the card deadlocked in its +wip column forever with no task error, and the only recovery was an operator manually bouncing +it back to the hold column (which clears the continuation). + +These tests pin the handover invariant and its two limits, because the repair is only safe if +it stays narrow: it must release a row for a node the run has LEFT, must never steal a live +claim on the node currently executing, and must report "nothing released" so the caller still +fails closed rather than looping on a conflict it cannot resolve. +*/ + +import "./executor-test-helpers.js"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { TaskExecutor } from "../executor.js"; +import { resetExecutorMocks } from "./executor-test-helpers.js"; + +interface FakeWorkItem { + id: string; + taskId: string; + nodeId: string; + nodeInstanceId?: string; + kind: string; + state: string; +} + +function createStore(items: FakeWorkItem[]) { + const rows = [...items]; + const store = { + on: vi.fn(), + off: vi.fn(), + getRootDir: () => "/tmp/test", + listWorkflowWorkItemsForTask: vi.fn(async (_taskId: string, _opts?: unknown) => rows), + transitionWorkflowWorkItem: vi.fn(async (id: string, state: string) => { + const row = rows.find((candidate) => candidate.id === id); + if (!row) throw new Error(`unknown work item ${id}`); + row.state = state; + return row; + }), + recordRunAuditEvent: vi.fn().mockResolvedValue(undefined), + logEntry: vi.fn().mockResolvedValue(undefined), + } as any; + return { store, rows }; +} + +/** The private repair is reached through routing in production; call it directly here. */ +function supersede( + executor: TaskExecutor, + taskId: string, + nodeInstanceId: string, + nodeId: string, +): Promise { + return (executor as unknown as { + supersedeStaleActiveWorkItems: (t: string, i: string, n: string) => Promise; + }).supersedeStaleActiveWorkItems(taskId, nodeInstanceId, nodeId); +} + +describe("workflow continuation slot handover", () => { + beforeEach(() => { + resetExecutorMocks(); + }); + + it("releases the active continuation of a node the run has already left", async () => { + const { store, rows } = createStore([ + { id: "wi-parse", taskId: "FN-1", nodeId: "parse", kind: "task", state: "running" }, + ]); + const executor = new TaskExecutor(store, "/tmp/test"); + + const released = await supersede(executor, "FN-1", "steps#0:step-execute", "step-execute"); + + expect(released).toBe(1); + expect(rows.find((row) => row.id === "wi-parse")?.state).toBe("succeeded"); + // Truthful terminal state, and the lease must be dropped so nothing reads it as live. + expect(store.transitionWorkflowWorkItem).toHaveBeenCalledWith( + "wi-parse", + "succeeded", + expect.objectContaining({ leaseOwner: null, leaseExpiresAt: null }), + ); + }); + + /* + Every active state the partial unique index covers occupies the slot, so every one of them + must be releasable — a repair that only handled `running` would still deadlock on a card + parked `held` by an earlier availability hold. + */ + it.each(["running", "held", "runnable", "retrying"])( + "releases a stale continuation in the '%s' state", + async (state) => { + const { store, rows } = createStore([ + { id: "wi-stale", taskId: "FN-1", nodeId: "parse", kind: "task", state }, + ]); + const executor = new TaskExecutor(store, "/tmp/test"); + + expect(await supersede(executor, "FN-1", "steps#0:step-execute", "step-execute")).toBe(1); + expect(rows.find((row) => row.id === "wi-stale")?.state).toBe("succeeded"); + }, + ); + + it("never steals a claim on the node currently executing", async () => { + const { store, rows } = createStore([ + { + id: "wi-same-node", + taskId: "FN-1", + nodeId: "step-execute", + nodeInstanceId: "steps#0:step-execute", + kind: "task", + state: "running", + }, + ]); + const executor = new TaskExecutor(store, "/tmp/test"); + + // 0 released → the caller must fail closed instead of retrying the fence write. + expect(await supersede(executor, "FN-1", "steps#0:step-execute", "step-execute")).toBe(0); + expect(rows.find((row) => row.id === "wi-same-node")?.state).toBe("running"); + expect(store.transitionWorkflowWorkItem).not.toHaveBeenCalled(); + }); + + it("leaves already-terminal rows untouched", async () => { + const { store, rows } = createStore([ + { id: "wi-done", taskId: "FN-1", nodeId: "plan", kind: "task", state: "succeeded" }, + { id: "wi-failed", taskId: "FN-1", nodeId: "plan-review", kind: "task", state: "failed" }, + ]); + const executor = new TaskExecutor(store, "/tmp/test"); + + expect(await supersede(executor, "FN-1", "steps#0:step-execute", "step-execute")).toBe(0); + expect(store.transitionWorkflowWorkItem).not.toHaveBeenCalled(); + expect(rows.map((row) => row.state)).toEqual(["succeeded", "failed"]); + }); + + /* + Bookkeeping must never decide the run: an unreadable or unwritable work-item table reports + "nothing released" so routing fails closed, rather than throwing out of `beforeNodeExecution` + and terminalizing the task on a storage hiccup. + */ + it("reports nothing released when the work-item table cannot be read", async () => { + const { store } = createStore([]); + store.listWorkflowWorkItemsForTask = vi.fn().mockRejectedValue(new Error("db down")); + const executor = new TaskExecutor(store, "/tmp/test"); + + await expect(supersede(executor, "FN-1", "steps#0:step-execute", "step-execute")).resolves.toBe(0); + }); + + it("reports nothing released when the transition itself fails", async () => { + const { store, rows } = createStore([ + { id: "wi-parse", taskId: "FN-1", nodeId: "parse", kind: "task", state: "running" }, + ]); + store.transitionWorkflowWorkItem = vi.fn().mockRejectedValue(new Error("cas lost")); + const executor = new TaskExecutor(store, "/tmp/test"); + + await expect(supersede(executor, "FN-1", "steps#0:step-execute", "step-execute")).resolves.toBe(0); + expect(rows.find((row) => row.id === "wi-parse")?.state).toBe("running"); + }); + + it("degrades to no-op on a store without the work-item API", async () => { + const executor = new TaskExecutor({ on: vi.fn(), off: vi.fn(), getRootDir: () => "/tmp/test" } as any, "/tmp/test"); + + await expect(supersede(executor, "FN-1", "steps#0:step-execute", "step-execute")).resolves.toBe(0); + }); +}); diff --git a/packages/engine/src/__tests__/in-process-runtime.pg.test.ts b/packages/engine/src/__tests__/in-process-runtime.pg.test.ts index 1bd71cef4c..27e9e241ef 100644 --- a/packages/engine/src/__tests__/in-process-runtime.pg.test.ts +++ b/packages/engine/src/__tests__/in-process-runtime.pg.test.ts @@ -112,13 +112,35 @@ pgDescribe("InProcessRuntime PostgreSQL composition", () => { */ const secretsStore = await lifecycle.secretsStoreGetter?.mock.results[0]?.value; const runtimeConsumers = runtime as unknown as { - executor?: { options?: { secretsStore?: unknown } }; - heartbeatMonitor?: { secretsStore?: unknown }; + executor?: { options?: { secretsStore?: unknown; agentStore?: unknown } }; + heartbeatMonitor?: { secretsStore?: unknown; configStore?: unknown }; + scheduler?: { options?: { agentStore?: unknown } }; + triageProcessor?: { options?: { agentStore?: unknown } }; + selfHealingManager?: { options?: { agentStore?: unknown } }; }; expect(lifecycle.secretsStoreGetter).toHaveBeenCalled(); expect(runtimeConsumers.executor?.options?.secretsStore).toBe(secretsStore); expect(runtimeConsumers.heartbeatMonitor?.secretsStore).toBe(secretsStore); + /* + FNXC:WorkflowAgentRouting 2026-08-07-22:39: + Every runtime consumer that resolves a permanent workflow principal must receive the ONE + long-lived engine AgentStore — asserted across all of them, not only the consumer that + regressed. FN-8764 gave the executor a fail-closed role-routing gate keyed on + `options.agentStore` but never wired that option, so a store the runtime had already built + was simply absent at the seam: every role-classified node (execute / step-execute / review / + merge) failed closed, the executor's `workflow-principal-*` branch swallowed it as a + recoverable hold, and the board deadlocked with no log, audit, or task error. The invariant + is the shared instance at every seam; an undefined here is the deadlock. + */ + const runtimeAgentStore = runtime.getAgentStore(); + expect(runtimeAgentStore).toBeDefined(); + expect(runtimeConsumers.executor?.options?.agentStore).toBe(runtimeAgentStore); + expect(runtimeConsumers.scheduler?.options?.agentStore).toBe(runtimeAgentStore); + expect(runtimeConsumers.triageProcessor?.options?.agentStore).toBe(runtimeAgentStore); + expect(runtimeConsumers.selfHealingManager?.options?.agentStore).toBe(runtimeAgentStore); + expect(runtimeConsumers.heartbeatMonitor?.configStore).toBe(runtimeAgentStore); + /* FNXC:SecretsEnvRuntimeWiring 2026-08-05-21:58: Production coverage must invoke the runtime-created executor and heartbeat monitor, not diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index e01568823a..56a0b89792 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -6803,10 +6803,26 @@ export class TaskExecutor { beforeNodeExecution: async (node, nodeTask, context) => { const classifiedRole = classifyWorkflowAgentNode(node); if (!classifiedRole) return undefined; - // A classified session without the authoritative IR/agent store must - // fail closed; running it as an ambient executor defeats role routing. + /* + * A classified session without the authoritative IR/agent store must fail closed; + * running it as an ambient executor defeats role routing. + * + * FNXC:WorkflowAgentRouting 2026-08-07-23:05: + * Name WHICH dependency is missing and log it. Unlike every other refusal below, + * this one persists no held work item (the durable-hold helper needs the very IR + * that is missing), so it is the one routing outcome with no durable trace at all: + * the run suspends with a bare `capacity` marker and the card re-suspends at the + * same node every poll, indistinguishable from a dead engine. A missing agent-store + * wire deadlocked the whole board this way. Neither condition is transient — both + * are boot-time composition faults — so log at error, not warn. + */ if (!this.options.agentStore || !columnAgentIr) { - return { outcome: "failure" as const, value: `workflow-principal-routing-unavailable:${classifiedRole}` }; + const missing = !this.options.agentStore ? "no-agent-store" : "no-workflow-ir"; + executorLog.error( + `[workflow-graph] ${nodeTask.id}: cannot route node '${node.id}' to a '${classifiedRole}' principal — ${missing}. ` + + "This is a runtime composition fault, not a transient wait: the node will re-suspend every dispatch until it is repaired.", + ); + return { outcome: "failure" as const, value: `workflow-principal-routing-unavailable:${missing}:${classifiedRole}` }; } const agents = await this.options.agentStore.listAgents({ includeEphemeral: true }); const activeSessions = new Map(agents.map((agent) => [agent.id, this.workflowAgentCapacity.activeSessions(agent.id, this.store.getRootDir())])); @@ -6956,25 +6972,84 @@ export class TaskExecutor { * just-acquired capacity and fails closed rather than running ambient. */ if (!durableWorkItemId && typeof this.store.upsertWorkflowWorkItem === "function") { + const writeFence = (): Promise => this.store.upsertWorkflowWorkItem!({ + runId: `${resolvedRunId ?? `${nodeTask.id}:workflow`}:${nodeInstanceId}`, + taskId: nodeTask.id, + nodeId: node.id, + kind: "task", + state: "running", + leaseOwner: `executor:${nodeTask.id}`, + leaseExpiresAt: null, + principalAgentId: routed.route.agent.id, + workflowRole: routed.route.role, + authorityKind: routed.route.authority, + nodeInstanceId, + }); + /* + * FNXC:WorkflowAgentRouting 2026-08-07-23:12: + * Never swallow the fence-write error. Failing closed is correct — a session must + * not run without its durable principal record — but the original bare `catch {}` + * discarded the ONLY evidence of why, and this hold is not transient: the same + * write fails on every dispatch. The store wraps driver errors, so the actionable + * text (constraint name, NOT NULL column) is on `cause`, not `message`. + */ + const handleFenceFailure = async (fenceErr: unknown): Promise => { + const detail = fenceErr instanceof Error ? fenceErr.message : String(fenceErr); + const cause = fenceErr instanceof Error && fenceErr.cause instanceof Error + ? ` [cause: ${fenceErr.cause.message}]` + : ""; + executorLog.error( + `[workflow-graph] ${nodeTask.id}: durable principal fence write failed for node '${node.id}' ` + + `(role=${routed.route.role}, authority=${routed.route.authority}, agent=${routed.route.agent.id}): ${detail}${cause}`, + ); + await this.store.logEntry( + nodeTask.id, + `Workflow principal fence write failed at node '${node.id}' — ${detail.slice(0, 300)}${cause}`, + ).catch(() => undefined); + void this.workflowAgentCapacity.release(attemptId, this.options.agentStore!.workflowProjectId ?? this.store.getRootDir()); + return { outcome: "failure" as const, value: `workflow-principal-fence-unavailable:${routed.route.role}` }; + }; try { - const item = await this.store.upsertWorkflowWorkItem({ - runId: `${resolvedRunId ?? `${nodeTask.id}:workflow`}:${nodeInstanceId}`, - taskId: nodeTask.id, - nodeId: node.id, - kind: "task", - state: "running", - leaseOwner: `executor:${nodeTask.id}`, - leaseExpiresAt: null, - principalAgentId: routed.route.agent.id, - workflowRole: routed.route.role, - authorityKind: routed.route.authority, - nodeInstanceId, - }); + const item = await writeFence(); durableWorkItemId = item.id; directWorkflowPrincipalWorkItemIds.add(item.id); - } catch { - void this.workflowAgentCapacity.release(attemptId, this.options.agentStore.workflowProjectId ?? this.store.getRootDir()); - return { outcome: "failure" as const, value: `workflow-principal-fence-unavailable:${routed.route.role}` }; + } catch (firstFenceErr) { + /* + * FNXC:WorkflowAgentRouting 2026-08-07-23:20: + * HAND OVER the task's single active-continuation slot before failing closed. + * + * `idx_workflow_work_items_one_active_task_continuation` allows at most ONE + * active (`runnable`/`running`/`held`/`retrying`) task work item per task, and + * the fence upsert's ON CONFLICT target is (run_id, task_id, node_id, kind) — + * a different index — so a still-active row for a node this run has ALREADY + * LEFT does not upsert, it raises. That is exactly the steady state of a resumed + * run: the continuation the run woke up on stays active until the interpreter + * returns, which is after this node needs its fence. + * + * The result was a permanent deadlock with no trace: every dispatch re-resumed + * at the continuation node, walked forward to the first role-classified node, + * failed this write, suspended, and left the same rows behind — so the card sat + * in its wip column forever and the only recovery was an operator bouncing it + * back to the hold column (which clears the continuation). Superseding the + * stale row here is what makes that bounce unnecessary. + * + * Only rows for a DIFFERENT node are superseded, and only after the write has + * actually failed, so the invariant still holds and a genuine concurrent claim + * on this same node is never stolen. `succeeded` is truthful: traversal reached + * a later node, which is only reachable through that one. + */ + const superseded = await this.supersedeStaleActiveWorkItems(nodeTask.id, nodeInstanceId, node.id); + if (superseded > 0) { + try { + const item = await writeFence(); + durableWorkItemId = item.id; + directWorkflowPrincipalWorkItemIds.add(item.id); + } catch (retryErr) { + return await handleFenceFailure(retryErr); + } + } else { + return await handleFenceFailure(firstFenceErr); + } } } workflowCapacityAttemptIds.add(attemptId); @@ -7203,6 +7278,24 @@ export class TaskExecutor { * handling; the next direct resume must receive the same fenced identity. */ if (principalHoldReason) { + /* + * FNXC:WorkflowAgentRouting 2026-08-07-22:39: + * A principal hold is a WAIT, so it writes no task error — but it must never be + * INVISIBLE. `workflow-principal-routing-unavailable:*` is a misconfiguration + * (no agent store / no resolvable IR), not a transient wait: nothing will ever + * clear it, so every resume re-parks and the card deadlocks in its wip column. + * A missing `agentStore` wire did exactly that to every task after FN-8764, with + * no log line, no audit row, and no task-log entry to find it by. Log the hold — + * loudly for the never-clears variant — so the next occurrence is greppable. + */ + const neverClears = principalHoldReason.startsWith("workflow-principal-routing-unavailable:"); + const holdMessage = `[workflow-graph] ${task.id} held at graph node — ${principalHoldReason}`; + if (neverClears) { + executorLog.error(`${holdMessage} (workflow principal routing is unavailable; this hold cannot self-clear)`); + } else { + executorLog.warn(holdMessage); + } + await this.store.logEntry(task.id, `Workflow stage held — ${principalHoldReason}`).catch(() => undefined); if (continuation && typeof this.store.transitionWorkflowWorkItem === "function") { await this.store.transitionWorkflowWorkItem(continuation.id, "held", { leaseOwner: null, @@ -7238,6 +7331,40 @@ export class TaskExecutor { return; } if (result.disposition === "suspended") { + /* + * FNXC:WorkflowExecution 2026-08-07-22:52: + * A suspend is a WAIT, so it writes no task error — but a bare `return` made it + * INVISIBLE, and an invisible wait that never clears is indistinguishable from a + * dead board. `onSuspend` deliberately writes no fresh continuation when an ACTIVE + * work item already exists, so a card re-suspending at the SAME node leaves zero + * new state anywhere: no log line, no audit row, no work-item update. Operators saw + * only "Resuming execution after unpause" every poll forever, and the only recovery + * was manually bouncing the card to the hold column. Record the suspension point so + * the wait is answerable after the fact ("why is this card parked?") without a debug + * build. Metadata is ids/outcomes-only — node/run identifiers, reason, and columns. + */ + const suspension = result.suspension; + await this.store.recordRunAuditEvent?.({ + taskId: task.id, + agentId: "executor", + runId: resolvedRunId ?? generateSyntheticRunId("workflow-run-suspended", task.id), + domain: "database", + mutationType: "task:workflow-run-suspended", + target: task.id, + metadata: { + taskId: task.id, + nodeId: suspension?.nodeId ?? "unknown", + reason: suspension?.reason ?? "unknown", + fromColumn: suspension?.fromColumn ?? null, + toColumn: suspension?.toColumn ?? null, + continuationId: continuation?.id ?? null, + continuationNodeId: continuation?.nodeId ?? null, + continuationState: continuation?.state ?? null, + }, + }).catch(() => undefined); + executorLog.log( + `[workflow-graph] ${task.id} suspended at node '${suspension?.nodeId ?? "unknown"}' (${suspension?.reason ?? "unknown"})`, + ); return; } if (result.disposition === "failed") { @@ -7573,6 +7700,86 @@ export class TaskExecutor { return undefined; } + /** + * FNXC:WorkflowAgentRouting 2026-08-07-23:20: + * Release the task's single active-continuation slot from rows this run has already + * advanced past, so the node currently executing can write its own durable principal + * fence. + * + * `idx_workflow_work_items_one_active_task_continuation` permits ONE active task work + * item per task. A resumed run keeps the continuation it woke on active until the + * interpreter returns, which is strictly after a later node needs its fence — so the + * fence write hits that partial unique index (NOT the upsert's ON CONFLICT target) and + * raises. Before this repair the run suspended there on every dispatch and the card + * deadlocked in its wip column until an operator bounced it back to the hold column. + * + * Deliberately narrow: called ONLY after a fence write has actually failed, and only rows + * for a DIFFERENT node/instance are superseded — a live claim on this same node is never + * stolen, so a genuine concurrent-owner conflict still fails closed. `succeeded` is the + * truthful terminal state: traversal reached a later node, which is reachable only + * through the one being superseded. + * + * @returns how many rows were released; 0 means the conflict was not a stale continuation + * and the caller must fail closed. + */ + private async supersedeStaleActiveWorkItems( + taskId: string, + currentNodeInstanceId: string, + currentNodeId: string, + ): Promise { + if (typeof this.store.listWorkflowWorkItemsForTask !== "function" + || typeof this.store.transitionWorkflowWorkItem !== "function") { + return 0; + } + let items: WorkflowWorkItem[]; + try { + items = await this.store.listWorkflowWorkItemsForTask(taskId, { kinds: ["task"] }); + } catch (err) { + executorLog.warn( + `[workflow-graph] ${taskId}: could not read work items to supersede a stale continuation: ${err instanceof Error ? err.message : String(err)}`, + ); + return 0; + } + const stale = items.filter((item) => + ACTIVE_WORKFLOW_WORK_ITEM_STATES.includes(item.state) + && item.nodeId !== currentNodeId + && (item.nodeInstanceId ?? item.nodeId) !== currentNodeInstanceId); + let released = 0; + for (const item of stale) { + try { + await this.store.transitionWorkflowWorkItem(item.id, "succeeded", { + leaseOwner: null, + leaseExpiresAt: null, + lastError: null, + }); + released += 1; + executorLog.log( + `[workflow-graph] ${taskId}: superseded stale active work item at node '${item.nodeId}' (state=${item.state}) so node '${currentNodeId}' can claim the continuation slot`, + ); + await this.store.recordRunAuditEvent?.({ + taskId, + agentId: "executor", + runId: generateSyntheticRunId("workflow-supersede-continuation", taskId), + domain: "database", + mutationType: "task:workflow-continuation-superseded", + target: taskId, + metadata: { + taskId, + supersededNodeId: item.nodeId, + supersededState: item.state, + currentNodeId, + currentNodeInstanceId, + }, + }).catch(() => undefined); + } catch (err) { + executorLog.warn( + `[workflow-graph] ${taskId}: failed to supersede stale work item at node '${item.nodeId}': ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + return released; + } + private buildParseStepsDeps(runId?: string): ParseStepsHandlerDeps { return { readArtifact: (task, key): Promise => this.readTaskArtifact(task.id, key), diff --git a/packages/engine/src/runtimes/in-process-runtime.ts b/packages/engine/src/runtimes/in-process-runtime.ts index 10b9d3ae5a..7a7342b936 100644 --- a/packages/engine/src/runtimes/in-process-runtime.ts +++ b/packages/engine/src/runtimes/in-process-runtime.ts @@ -1329,6 +1329,19 @@ export class InProcessRuntime attribution. Reading it lazily at runner-construction time picks up the resolved id. */ getLocalNodeId: () => this.localNodeId, + /* + FNXC:WorkflowAgentRouting 2026-08-07-22:39: + FN-8764 made every role-classified graph node (execute / step-execute / review / merge) + resolve a permanent principal through `options.agentStore`, and fail closed with + `workflow-principal-routing-unavailable:` when that store is absent. The runtime built + the one long-lived engine AgentStore above but never handed it to the executor, so EVERY + task deadlocked on entry to `steps#0:step-execute`: routing failed closed, the executor's + `workflow-principal-*` hold branch swallowed the result as a recoverable wait, and the card + sat in-progress with its foreach instance pinned `in-progress` and no session, no log, and + no audit row. Wire the same instance the scheduler, triage, merger, and heartbeat already + receive — an executor without it can no longer run any classified node at all. + */ + agentStore: this.agentStore, pool: this.worktreePool, usageLimitPauser: this.usageLimitPauser, credentialRotator: this.credentialRotator, diff --git a/packages/engine/src/workflows/workflow-graph-executor.ts b/packages/engine/src/workflows/workflow-graph-executor.ts index fd76df3cd3..e6d1825a8b 100644 --- a/packages/engine/src/workflows/workflow-graph-executor.ts +++ b/packages/engine/src/workflows/workflow-graph-executor.ts @@ -1664,6 +1664,17 @@ export class WorkflowGraphExecutor { * the task or convert an operator-visible hold into graph failure. */ if (preflight.outcome === "failure" && typeof preflight.value === "string" && preflight.value.startsWith("workflow-principal-")) { + /* + * FNXC:WorkflowAgentRouting 2026-08-07-23:05: + * Carry the refusal REASON out on the shared context. The suspension marker + * itself has no field for it, so throwing alone reduced every distinct routing + * refusal — unavailable owner, exhausted pool, missing agent store — to an + * indistinguishable `capacity` suspend at the caller. Context survives the + * unwind (see the catch below), which is what lets the executor recognise this + * as a principal hold, park the continuation `held` instead of leaving it + * `running` forever, and name the reason in the task log. + */ + context[`node:${node.id}:principal-hold`] = preflight.value; throw new WorkflowGraphSuspended({ reason: "capacity", nodeId: node.id,