From 340da924008fffefddff6b618af7cd00789d8604 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 19 Jun 2026 10:51:55 -0700 Subject: [PATCH] FN-6735: treat benign merge-seam pause aborts as retryable Treat benign pause/resume aborts at workflow merge seams as retryable transient merge failures. - Classify legacy, merge-region, manual-hold, and retry node ids as merge-seam graph failures. - Route clean in-review pause/resume aborts back through bounded auto-merge retry when eligible. - Keep conflict, contamination, foreign-work, exhausted-retry, global/user pause, and confirmed-merge cases terminal. - Cover the lifecycle with a reliability regression test and architecture notes. Files changed: docs/architecture.md | 4 +- .../merge-node-paused-abort-retryable.test.ts | 227 +++++++++++++++++++++ packages/engine/src/executor.ts | 77 ++++++- packages/engine/src/workflow-graph-executor.ts | 2 +- 4 files changed, 301 insertions(+), 9 deletions(-) Fusion-Task-Id: FN-6735 Fusion-Task-Lineage: 3560c1e3-91e9-405a-bdfc-a243f7052803 --- docs/architecture.md | 4 +- .../merge-node-paused-abort-retryable.test.ts | 227 ++++++++++++++++++ packages/engine/src/executor.ts | 77 +++++- .../engine/src/workflow-graph-executor.ts | 2 +- 4 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 packages/engine/src/__tests__/reliability-interactions/merge-node-paused-abort-retryable.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index cf6892c863..66865f5fce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1280,7 +1280,7 @@ The columns/traits track moved *board* policy (transitions, capacity, hold, merg - A `parse-steps` node reads a workflow-declared **artifact** (PROMPT.md is just the default workflow's declared `step-source` artifact) and runs a registry **parser** (`step-headings`, `json-steps`, or a plugin-contributed parser) to write `Task.steps[]`. It is the only graph-side step-list writer and must dominate any `foreach`. Parsers fail closed to a routable `outcome:parse-error`. - A `foreach(source:"task-steps")` node instantiates an inline template subgraph once per planned step, with `mode` (sequential/parallel) and `isolation` (shared/worktree) as explicit axes and per-instance run-state pinned + persisted for crash-safe resume. - Resume-limbo graph failures are retried only through a narrow persisted counter (`Task.graphResumeRetryCount`, max 2). The executor classifies a failure as transient only when it happens immediately after the engine restart/unpause resume log marker, reports no graph `reason`, has no completed step progress, and the task has no durable `lastError`/`failureReason`; it clears transient `status`/`error`, logs the auto-retry, and schedules one more graph execution. Any explicit graph reason, completed step progress, durable task error, missing resume marker, or exhausted counter remains a genuine `status:"failed"` disposition and goes to review handoff, preserving the FN-5704 anti-loop contract. -- Paused graph exits are benign only while the task is still in `in-progress`; that is the user-pause/engine-pause state where preserving the pause without requeueing is intentional. If the graph reports a pause/abort exit after the task has already advanced to another live column (for example `in-review` after an unpause/resume race), `TaskExecutor.handleGraphFailure()` surfaces the boundary as operator-actionable failure evidence (`status:"failed"`/`error` when no failure is already present, plus a task-log entry) and does **not** move, rewind, or auto-merge the task. The exception is completed/no-commit finalize-to-review teardown (FN-6625/FN-6644/FN-6647): once the persisted task row proves a completed finalize handoff (non-`in-progress`, all steps done/skipped, no live pause/status/error, and the finalize-to-review log entry), a trailing graph abort resolves as an already-advanced benign graph exit even if volatile completion markers were cleared by teardown/restart and later abort provenance was re-marked from `completion-finalize` to `hard-cancel`. Genuine `userPaused`/global-pause exits and active-execution hard-cancels still use the operator-action path. `done` and `archived` remain terminal and keep their column/status, while existing failure details are preserved. +- Paused graph exits are benign only while the task is still in `in-progress`; that is the user-pause/engine-pause state where preserving the pause without requeueing is intentional. If the graph reports a pause/abort exit after the task has already advanced to another live column (for example `in-review` after an unpause/resume race), `TaskExecutor.handleGraphFailure()` surfaces the boundary as operator-actionable failure evidence (`status:"failed"`/`error` when no failure is already present, plus a task-log entry) and does **not** move, rewind, or auto-merge the task. The exceptions are completed/no-commit finalize-to-review teardown (FN-6625/FN-6644/FN-6647) and benign merge-seam pause/resume aborts (FN-6735). For completed finalize handoff, once the persisted task row proves a completed finalize handoff (non-`in-progress`, all steps done/skipped, no live pause/status/error, and the finalize-to-review log entry), a trailing graph abort resolves as an already-advanced benign graph exit even if volatile completion markers were cleared by teardown/restart and later abort provenance was re-marked from `completion-finalize` to `hard-cancel`. For merge-seam aborts, `in-review` tasks with no persisted status/error and no confirmed merge may re-enter bounded auto-merge retry only when the failed graph node is a merge/request-merge seam, the graph value is not conflict/contamination/foreign/retry-exhaustion evidence, project settings allow auto-merge processing (or the task is a shared-branch local integration member), and the merge retry budget is not exhausted. Genuine `userPaused`/global-pause exits, active-execution hard-cancels, terminal merge failure values, `autoMerge:false` human-gated tasks, retry-exhausted tasks, and already-confirmed merges still use the operator-action path. `done` and `archived` remain terminal and keep their column/status, while existing failure details are preserved. - A `step-review` node surfaces reviewer verdicts (APPROVE/REVISE/RETHINK/UNAVAILABLE) as outcome edges; `rework` edges (the only legal graph cycles, bounded per instance) route REVISE/RETHINK back to `step-execute`, with RETHINK traversal triggering the reset seam. - A `code` node runs sandboxed TypeScript (esbuild + child process, clamped timeout, no store handle) for arbitrary computed routing/field logic — the same trust tier as project-local script steps. @@ -1794,7 +1794,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f - **Worktrunk-managed lifecycles**: when `worktrunk.enabled`, self-healing defers prune/idle/worktree-cap sweeps to the worktrunk backend; branch-level stale/ conflict reclaim stays native. Orphan `fusion/*` branches are operator-managed via standard git tooling (no auto-rescue task filing). - **Post-finalize verification no-op (FN-4944)**: when auto-merge receives a delayed `VerificationError` after a task is already `done` with `mergeDetails.mergeConfirmed === true` (already-on-main fast-path), it must log one `[verification] ... no action` diagnostic and must not bounce the task back to `in-progress` / `merging-fix`. Defense-in-depth now re-checks the done+mergeConfirmed condition immediately before each verification-failure status write site, and emits `task:post-finalize-verification-no-op` database audit events with failure metadata for forensics. - **Transient auto-merge retry classification (FN-5697)**: non-conflict auto-merge errors now run through `isTransientError(...)` before terminal parking. Transient provider/network failures (for example `This operation was aborted`, `socket hang up`, and `server_error` payloads) are retried with bounded exponential backoff (`5s/10s/20s`) and `status=null` for both direct and pull-request merge strategies; once `MAX_AUTO_MERGE_TRANSIENT_RETRIES` is exhausted, tasks are parked `in-review/failed` with explicit transient-exhaustion logs. -- **Merge-seam abort provenance (FN-6568)**: workflow graph merge-node failures must not be classified as pause/resume aborts merely because the merge seam hard-canceled an in-flight session. `TaskExecutor` tracks paused-abort provenance separately (`global-pause`, `merge-seam`, `hard-cancel`); genuine user/global pauses still preserve FN-6478/FN-5147 parking, while non-paused `merge`/`requestMerge` graph failures route back into the bounded auto-merge retry path instead of being parked `status:"failed"` with `mergeRetries=NULL`. +- **Merge-seam abort provenance (FN-6568/FN-6735)**: workflow graph merge-node failures must not be classified as pause/resume aborts merely because the merge seam hard-canceled an in-flight session. `TaskExecutor` tracks paused-abort provenance separately (`global-pause`, `merge-seam`, `hard-cancel`); genuine user/global pauses still preserve FN-6478/FN-5147 parking, while non-paused merge-seam graph failures (`merge`, `requestMerge`, built-in merge-region node ids, `merge-manual-hold`, and `merge-retry`) route back into the bounded auto-merge retry path instead of being parked `status:"failed"` with `mergeRetries=NULL`. Benign pause/resume aborts at these seams are also retryable when the task is already `in-review`, has no durable failure/status, has not confirmed a merge, remains auto-merge eligible (or is a shared-branch local integration), and has merge retries remaining. Conflict/contamination/foreign-work/retry-exhaustion values, `autoMerge:false` human-gated review tasks, pre-existing failures, global/user pauses, and post-confirmation partial landings remain terminal operator-action evidence. - **Worktree pool exclusivity (FN-4954)**: `WorktreePool.acquire(taskId)` / `release(path, taskId?)` track a `leased` map so every pooled path is either idle or leased, never both. Cross-task double-lease detection throws `PoolDoubleLeaseError` and emits `worktree:pool-double-lease-detected`; merger Step 8 now detaches HEAD and clears `task.worktree` / `task.branch` before releasing paths back to the pool. - **Stale registration recovery (FN-5056)**: `NativeWorktreeBackend.create` and `executor.tryCreateWorktree` detect `missing but already registered worktree` failures, run `git worktree prune` (plus `remove --force` / `add -f` fallbacks) before retrying, and emit `worktree:stale-registration-{detected,recovered,recovery-failed}` audit events. - **Raw worktree deletion must be paired with prune (FN-5058)**: any direct filesystem deletion of a worktree directory (`rm -rf` / `rmSync`) must be followed by best-effort `git worktree prune` via `pruneWorktreeAdminEntries` so `.git/worktrees/*` admin entries are not stranded in a missing-but-registered state (FN-5056 class). diff --git a/packages/engine/src/__tests__/reliability-interactions/merge-node-paused-abort-retryable.test.ts b/packages/engine/src/__tests__/reliability-interactions/merge-node-paused-abort-retryable.test.ts new file mode 100644 index 0000000000..9095f1a901 --- /dev/null +++ b/packages/engine/src/__tests__/reliability-interactions/merge-node-paused-abort-retryable.test.ts @@ -0,0 +1,227 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import "../executor-test-helpers.js"; +import { TaskExecutor } from "../../executor.js"; +import { createMockStore, resetExecutorMocks } from "../executor-test-helpers.js"; +import type { TaskDetail } from "@fusion/core"; + +const now = "2026-06-19T00:00:00.000Z"; + +function makeInReviewTask(overrides: Partial = {}): TaskDetail { + return { + id: "FN-6735-T", + title: "Merge paused abort repro", + description: "Reproduces benign merge pause abort classification", + column: "in-review", + dependencies: [], + steps: [ + { name: "Preflight", status: "done" }, + { name: "Implement", status: "done" }, + ], + currentStep: 1, + log: [], + branch: "fusion/fn-6735-t", + baseBranch: "main", + worktree: "/tmp/fusion-fn-6735-t", + status: null, + error: null, + paused: true, + userPaused: false, + autoMerge: true, + mergeRetries: 0, + createdAt: now, + updatedAt: now, + ...overrides, + } as TaskDetail; +} + +function makeHarness(taskOverrides: Partial = {}, settingsOverrides: Record = {}) { + const store = createMockStore(); + const task = makeInReviewTask(taskOverrides); + store.getTask.mockResolvedValue(task); + store.getSettings.mockResolvedValue({ + maxConcurrent: 2, + maxWorktrees: 4, + pollIntervalMs: 15000, + groupOverlappingFiles: false, + autoMerge: true, + maxAutoMergeRetries: 3, + worktreeInitCommand: undefined, + ...settingsOverrides, + }); + const executor = new TaskExecutor(store, "/tmp/test", {}); + const mergeRequester = vi.fn(async () => ({ + task, + branch: task.branch ?? "fusion/fn-6735-t", + merged: true, + noOp: false, + worktreeRemoved: true, + branchDeleted: true, + })); + executor.setMergeRequester(mergeRequester as any); + (executor as any).markPausedAborted(task.id, "hard-cancel"); + return { store, task, executor, mergeRequester }; +} + +async function invokeGraphFailure(executor: TaskExecutor, task: TaskDetail, nodeId: string, value?: string) { + await (executor as any).handleGraphFailure(task, { + disposition: "failed", + outcome: "failure", + visitedNodeIds: ["review", nodeId], + context: value === undefined ? {} : { [`node:${nodeId}:value`]: value }, + }); +} + +function logText(store: ReturnType): string { + return store.logEntry.mock.calls.map((call: unknown[]) => call[1]).join("\n"); +} + +describe("merge-node paused-abort retry classification (FN-6735)", () => { + beforeEach(() => { + resetExecutorMocks(); + }); + + /* + Surface Enumeration coverage: + - Merge seam node ids: legacy `merge`, `requestMerge`, primitive merge-region ids, and historical aliases all route through the same classifier. + - Auto-merge paths: autopilot autoMerge:true and shared-branch local integration are both exercised. + - Pause sources: benign hard-cancel/undefined-like generic pause is retried; global/user pause controls remain terminal. + - Retry/data states: retry budget, mergeConfirmed partial landing, conflict, foreign/contamination, and pre-existing failure all avoid retry. + - FN-5147: autoMerge:false human-gated in-review tasks are not re-enqueued. + */ + it.each([ + "merge", + "requestMerge", + "merge-gate", + "merge-attempt", + "manual-merge-hold", + "merge-manual-hold", + "retry-backoff", + "merge-retry", + ] as const)("re-enqueues benign paused merge graph failure at node %s without operator-action failure", async (nodeId) => { + const { store, task, executor, mergeRequester } = makeHarness(); + + await invokeGraphFailure(executor, task, nodeId); + + expect(mergeRequester).toHaveBeenCalledWith(task.id); + const messages = logText(store); + expect(messages).toContain(`Workflow graph merge failure at node '${nodeId}' routed to bounded auto-merge retry after benign pause/resume abort`); + expect(messages).not.toContain("Workflow graph failure surfaced after paused engine abort during pause/resume"); + expect(messages).not.toContain("operator action required"); + expect(store.updateTask).not.toHaveBeenCalledWith( + task.id, + expect.objectContaining({ status: "failed" }), + undefined, + ); + }); + + it("allows shared-branch-group local integration to retry even when global autoMerge is off", async () => { + const { store, task, executor, mergeRequester } = makeHarness({ + autoMerge: undefined, + branchContext: { groupId: "BG-6735", source: "mission", assignmentMode: "shared" }, + }, { autoMerge: false }); + + await invokeGraphFailure(executor, task, "merge-gate"); + + expect(mergeRequester).toHaveBeenCalledWith(task.id); + expect(logText(store)).toContain("routed to bounded auto-merge retry after benign pause/resume abort"); + expect(store.updateTask).not.toHaveBeenCalledWith(task.id, expect.objectContaining({ status: "failed" }), undefined); + }); + + it("parks genuine merge conflicts as terminal instead of retrying forever", async () => { + const { store, task, executor, mergeRequester } = makeHarness(); + + await invokeGraphFailure(executor, task, "merge", "merge-conflict"); + + expect(mergeRequester).not.toHaveBeenCalled(); + expect(store.updateTask).toHaveBeenCalledWith( + task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("operator action required") }), + undefined, + ); + }); + + it("parks contaminated or foreign-only merge graph failures as terminal", async () => { + const { store, task, executor, mergeRequester } = makeHarness(); + + await invokeGraphFailure(executor, task, "merge-attempt", "foreign-only-contamination"); + + expect(mergeRequester).not.toHaveBeenCalled(); + expect(store.updateTask).toHaveBeenCalledWith( + task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("operator action required") }), + undefined, + ); + }); + + it("respects exhausted mergeRetries budget by terminal parking", async () => { + const { store, task, executor, mergeRequester } = makeHarness({ mergeRetries: 3 }); + + await invokeGraphFailure(executor, task, "retry-backoff"); + + expect(mergeRequester).not.toHaveBeenCalled(); + expect(store.updateTask).toHaveBeenCalledWith( + task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("operator action required") }), + undefined, + ); + }); + + it("leaves pre-existing real failures unchanged and does not re-enqueue", async () => { + const { store, task, executor, mergeRequester } = makeHarness({ status: "failed", error: "real failure before graph unwind" }); + + await invokeGraphFailure(executor, task, "merge"); + + expect(mergeRequester).not.toHaveBeenCalled(); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("does not auto-mutate human-gated autoMerge:false in-review tasks", async () => { + const { store, task, executor, mergeRequester } = makeHarness({ autoMerge: undefined }, { autoMerge: false }); + + await invokeGraphFailure(executor, task, "merge"); + + expect(mergeRequester).not.toHaveBeenCalled(); + expect(store.updateTask).toHaveBeenCalledWith( + task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("operator action required") }), + undefined, + ); + }); + + it("preserves global and explicit user pause terminal behavior", async () => { + const globalHarness = makeHarness(); + (globalHarness.executor as any).markPausedAborted(globalHarness.task.id, "global-pause"); + + await invokeGraphFailure(globalHarness.executor, globalHarness.task, "merge"); + + expect(globalHarness.mergeRequester).not.toHaveBeenCalled(); + expect(globalHarness.store.updateTask).toHaveBeenCalledWith( + globalHarness.task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("global pause") }), + undefined, + ); + + const userHarness = makeHarness({ userPaused: true }); + await invokeGraphFailure(userHarness.executor, userHarness.task, "merge"); + + expect(userHarness.mergeRequester).not.toHaveBeenCalled(); + expect(userHarness.store.updateTask).toHaveBeenCalledWith( + userHarness.task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("explicit user pause") }), + undefined, + ); + }); + + it("does not retry merge-confirmed partial landing evidence", async () => { + const { store, task, executor, mergeRequester } = makeHarness({ mergeDetails: { mergeConfirmed: true } as any }); + + await invokeGraphFailure(executor, task, "merge"); + + expect(mergeRequester).not.toHaveBeenCalled(); + expect(store.updateTask).toHaveBeenCalledWith( + task.id, + expect.objectContaining({ status: "failed", error: expect.stringContaining("operator action required") }), + undefined, + ); + }); +}); diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index 47edf2e53d..c9e15bbbfe 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -8,8 +8,8 @@ const execAsync = promisify(exec); import { delimiter, isAbsolute, join, relative, resolve as resolvePath } from "node:path"; import { existsSync, realpathSync } from "node:fs"; import { readFile, rm, writeFile } from "node:fs/promises"; -import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings, MergeResult, WorkflowIrNode } from "@fusion/core"; -import { RetryStormError, TaskDeletedError, serializeRetryStormError, isExperimentalFeatureEnabled, isWorkflowColumnsEnabled, resolveWorkflowIrForTask, resolveColumnAgentBinding, resolveEffectiveAgent, instanceNodeId, getWorkflowExtensionRegistry, getBuiltinWorkflow, parseNoOpCompletionMarker } from "@fusion/core"; +import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings, MergeResult, WorkflowIrNode, WorkflowIrNodeKind } from "@fusion/core"; +import { RetryStormError, TaskDeletedError, serializeRetryStormError, isExperimentalFeatureEnabled, isWorkflowColumnsEnabled, resolveWorkflowIrForTask, resolveColumnAgentBinding, resolveEffectiveAgent, instanceNodeId, getWorkflowExtensionRegistry, getBuiltinWorkflow, parseNoOpCompletionMarker, allowsAutoMergeProcessing, isSharedBranchGroupMemberIntegration, resolveMaxAutoMergeRetries } from "@fusion/core"; import { mergeEffectiveSettings } from "./effective-settings.js"; import type { TaskStep, WorkflowIr, WorkflowFieldDefinition, WorkflowColumnAgent, EffectiveAgentInput, WorkflowWorkEngineDispatchResult } from "@fusion/core"; import { @@ -35,6 +35,7 @@ import { type ForeachActiveContext, type WorkflowLegacySeams, } from "./workflow-node-handlers.js"; +import { MERGE_REGION_KINDS } from "./workflow-graph-executor.js"; import type { WorkflowNodeResult } from "./workflow-graph-executor.js"; import type { AuditPrimitiveInput, @@ -6433,7 +6434,55 @@ export class TaskExecutor { } private isMergeGraphFailure(failedNode: string | undefined): boolean { - return failedNode === "merge" || failedNode === "requestMerge"; + /* + FNXC:WorkflowLifecycle 2026-06-19-00:00: + FN-6735 requires every workflow merge-region node id to classify as a merge-seam graph failure. A benign pause/resume abort can surface as the synthetic legacy `merge`, `requestMerge`, or a primitive merge-region id, and all must route through bounded merge retry rather than terminal operator-action parking. + */ + if (!failedNode) return false; + if (failedNode === "merge" || failedNode === "requestMerge") return true; + if (MERGE_REGION_KINDS.has(failedNode as WorkflowIrNodeKind)) return true; + return failedNode === "merge-manual-hold" || failedNode === "merge-retry"; + } + + private isTerminalMergeGraphFailureValue(value: string | undefined): boolean { + if (!value) return false; + const normalized = value.toLowerCase(); + return normalized.includes("conflict") + || normalized.includes("contamination") + || normalized.includes("foreign") + || normalized.includes("retry-exhausted") + || normalized.includes("retries exhausted") + || normalized.includes("max retries"); + } + + private async isRetryableBenignMergePauseAbort( + live: TaskDetail, + result: WorkflowGraphTaskRunResult, + abortProvenance: "global-pause" | "merge-seam" | "hard-cancel" | "completion-finalize" | undefined, + pausedAborted: boolean, + ): Promise { + /* + FNXC:WorkflowLifecycle 2026-06-19-00:05: + FN-6735 treats a generic engine pause/resume abort at the merge seam as transient only when the row is still a clean in-review auto-merge candidate: no user/global pause, no pre-existing failure, no merge-confirmed partial landing, no terminal conflict/contamination value, within mergeRetries budget, and still eligible for auto-merge or shared-branch local integration. Anything outside those guards keeps the existing terminal operator-action park. + */ + if (!pausedAborted) return false; + if (abortProvenance === "global-pause" || live.userPaused === true) return false; + if (abortProvenance === "completion-finalize") return false; + if (live.column !== "in-review" || live.status != null || live.error != null) return false; + if (live.mergeDetails?.mergeConfirmed === true) return false; + if (this.isTerminalMergeGraphFailureValue(this.graphFailureValue(result))) return false; + const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; + if (!this.isMergeGraphFailure(failedNode)) return false; + let settings: Settings | undefined; + try { + settings = await this.store.getSettings(); + } catch { + return false; + } + const sharedBranchMember = isSharedBranchGroupMemberIntegration(live); + if (!sharedBranchMember && !allowsAutoMergeProcessing(live, settings)) return false; + if ((live.mergeRetries ?? 0) >= resolveMaxAutoMergeRetries(settings)) return false; + return true; } private async routeGraphMergeFailureToRetry( @@ -6443,7 +6492,7 @@ export class TaskExecutor { ): Promise { if (!this.mergeRequester) return false; const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1] ?? "unknown"; - const message = `Workflow graph merge failure at node '${failedNode}' routed to bounded auto-merge retry${abortProvenance === "merge-seam" ? " after merge-seam abort" : ""}`; + const message = `Workflow graph merge failure at node '${failedNode}' routed to bounded auto-merge retry${abortProvenance === "merge-seam" ? " after merge-seam abort" : abortProvenance === "hard-cancel" || abortProvenance === undefined ? " after benign pause/resume abort" : ""}`; executorLog.warn(`${live.id}: ${message}`); await this.store.logEntry(live.id, message, undefined, this.getRunContextFor(live.id)); try { @@ -6524,6 +6573,11 @@ export class TaskExecutor { || (live.paused && !mergeSeamAborted && !suppressFinalizedCompletionAbort) || (pausedAborted && !mergeSeamAborted && !completionFinalizeAborted && !suppressFinalizedCompletionAbort), ); + if (genuinePauseAbort && await this.isRetryableBenignMergePauseAbort(live, result, abortProvenance, pausedAborted)) { + if (await this.routeGraphMergeFailureToRetry(live, result, abortProvenance)) { + return; + } + } if (genuinePauseAbort) { /* FNXC:WorkflowLifecycle 2026-06-15-01:45: @@ -6559,7 +6613,19 @@ export class TaskExecutor { return; } const failedNode = result.visitedNodeIds[result.visitedNodeIds.length - 1]; - if (this.isMergeGraphFailure(failedNode) && await this.routeGraphMergeFailureToRetry(live, result, abortProvenance)) { + const mergeGraphFailure = this.isMergeGraphFailure(failedNode); + const failureValue = this.graphFailureValue(result); + if (mergeGraphFailure && !this.isTerminalMergeGraphFailureValue(failureValue) && await this.routeGraphMergeFailureToRetry(live, result, abortProvenance)) { + return; + } + if (mergeGraphFailure && this.isTerminalMergeGraphFailureValue(failureValue) && live.column !== "done" && live.column !== "archived") { + const message = `Workflow graph terminal merge failure at node '${failedNode ?? "unknown"}' (${failureValue}) — operator action required`; + executorLog.warn(`${task.id}: ${message}`); + await this.store.logEntry(task.id, message, undefined, this.getRunContextFor(task.id)); + if (live.status == null && live.error == null) { + await this.store.updateTask(task.id, { error: message, status: "failed" }, this.getRunContextFor(task.id)); + } + await this.persistTokenUsage(task.id); return; } if (live.column !== "in-progress") { @@ -6568,7 +6634,6 @@ export class TaskExecutor { await this.store.logEntry(task.id, benignMessage, undefined, this.getRunContextFor(task.id)); return; } - const failureValue = this.graphFailureValue(result); if (this.isAwaitingGraphFailureValue(failureValue)) { /* FNXC:WorkflowLifecycle 2026-06-15-12:00: diff --git a/packages/engine/src/workflow-graph-executor.ts b/packages/engine/src/workflow-graph-executor.ts index d001ea53a4..cbc7b8fc8c 100644 --- a/packages/engine/src/workflow-graph-executor.ts +++ b/packages/engine/src/workflow-graph-executor.ts @@ -180,7 +180,7 @@ const TERMINAL_FAILURE: WorkflowGraphExecutorResult = { * seam so observable lifecycle behavior stays byte-identical with the legacy * executor. Consolidate with a core export when one exists. */ -const MERGE_REGION_KINDS = new Set([ +export const MERGE_REGION_KINDS = new Set([ "merge-gate", "merge-attempt", "manual-merge-hold",