FN-6113: fail fast on terminal provider errors

Fail fast on non-retryable provider failures in the in-review stall recovery flow. - classify failed-task provider errors as non-retryable, retryable, or unknown in core stall detection - surface terminal provider errors with dedicated dashboard copy and exported stall signal metadata - pause in-review tasks immediately on non-retryable provider failures and record a dedicated run-audit mutation - add regression coverage for provider error classification, self-healing disposition, and dashboard badge behavior Files changed: .../core/src/__tests__/in-review-stall.test.ts | 70 ++++++++++ packages/core/src/in-review-stall.ts | 53 ++++++- packages/core/src/index.ts | 4 +- .../app/utils/__tests__/inReviewStallCopy.test.ts | 5 +- packages/dashboard/app/utils/inReviewStallCopy.ts | 8 ++ .../in-review-stall-deadlock-disposition.test.ts | 153 +++++++++++++++++++++ packages/engine/src/run-audit.ts | 1 + packages/engine/src/self-healing.ts | 30 +++- 8 files changed, 320 insertions(+), 4 deletions(-) Fusion-Task-Id: FN-6113 Fusion-Task-Lineage: 5e89adb4-6b59-4deb-8811-5c75d6fa19cc
2026-06-09 12:13:05 -07:00
parent 6a77217f94
commit 320b153c45
8 changed files with 320 additions and 4 deletions
--- a/packages/engine/src/tests/reliability-interactions/in-review-stall-deadlock-disposition.test.ts
+++ b/packages/engine/src/tests/reliability-interactions/in-review-stall-deadlock-disposition.test.ts
@@ -115,6 +115,159 @@ describe("reliability interactions: in-review stall deadlock disposition", () =>
    manager.stop();
  });

+  it("FN-6113: terminal provider errors dispose in a single stall cycle", async () => {
+    const task = {
+      id: "FN-6113-TERMINAL",
+      column: "in-review",
+      paused: false,
+      userPaused: false,
+      status: "failed",
+      error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
+      branch: "fusion/fn-6113-terminal",
+      worktree: "/tmp/fn-6113-terminal",
+      mergeDetails: {},
+      mergeRetries: 0,
+      steps: [{ name: "merge", status: "done" }],
+      workflowStepResults: [],
+      updatedAt: "2026-01-01T00:00:00.000Z",
+      log: [],
+    } as any satisfies Task;
+
+    const store = createStore(task);
+    const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
+
+    vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
+    expect(await manager.surfaceInReviewStalls()).toBe(1);
+
+    expect(task.paused).toBe(true);
+    expect(task.pausedReason).toBe("non-retryable-provider-error");
+    expect(task.status).toBe("failed");
+    expect(task.error).toBe(
+      "Terminal provider error (non-retryable): Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
+    );
+    expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed [non-retryable-provider-error]:"))).toHaveLength(1);
+    expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall auto-disposed ["))).toBe(false);
+    expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall surfaced ["))).toBe(false);
+
+    const auditEvents = (store as any).__auditEvents as any[];
+    const terminalAuditEvents = auditEvents.filter((event) => event.mutationType === "task:in-review-stall-terminal-provider-error");
+    expect(terminalAuditEvents).toHaveLength(1);
+    expect(terminalAuditEvents[0]).toMatchObject({
+      domain: "database",
+      taskId: "FN-6113-TERMINAL",
+      target: "FN-6113-TERMINAL",
+      metadata: expect.objectContaining({
+        code: "non-retryable-provider-error",
+        reason: "Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
+        branch: "fusion/fn-6113-terminal",
+        worktree: "/tmp/fn-6113-terminal",
+      }),
+    });
+
+    manager.stop();
+  });
+
+  it("FN-6113: terminal provider errors are ignored when autoMerge is disabled", async () => {
+    const task = {
+      id: "FN-6113-AUTOMERGE-OFF",
+      column: "in-review",
+      paused: false,
+      userPaused: false,
+      status: "failed",
+      error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
+      branch: "fusion/fn-6113-automerge-off",
+      worktree: "/tmp/fn-6113-automerge-off",
+      mergeDetails: {},
+      mergeRetries: 0,
+      steps: [{ name: "merge", status: "done" }],
+      workflowStepResults: [],
+      updatedAt: "2026-01-01T00:00:00.000Z",
+      log: [],
+    } as any satisfies Task;
+
+    const store = createStore(task, { autoMerge: false });
+    const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
+
+    vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
+    expect(await manager.surfaceInReviewStalls()).toBe(0);
+    expect(task.paused).toBe(false);
+    expect((store.updateTask as any).mock.calls.length).toBe(0);
+    expect(task.log).toHaveLength(0);
+
+    manager.stop();
+  });
+
+  it("FN-6113: terminal provider errors do not auto-dispose userPaused tasks", async () => {
+    const task = {
+      id: "FN-6113-USER-PAUSED",
+      column: "in-review",
+      paused: false,
+      userPaused: true,
+      status: "failed",
+      error: "HTTP 403 forbidden for model access",
+      branch: "fusion/fn-6113-user-paused",
+      worktree: "/tmp/fn-6113-user-paused",
+      mergeDetails: {},
+      mergeRetries: 0,
+      steps: [{ name: "merge", status: "done" }],
+      workflowStepResults: [],
+      updatedAt: "2026-01-01T00:00:00.000Z",
+      log: [],
+    } as any satisfies Task;
+
+    const store = createStore(task);
+    const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
+
+    vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
+    expect(await manager.surfaceInReviewStalls()).toBe(1);
+    expect(task.paused).toBe(false);
+    expect(task.pausedReason).toBeUndefined();
+    expect((store.updateTask as any).mock.calls.length).toBe(0);
+    expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed ["))).toBe(false);
+    expect(((store as any).__auditEvents as any[]).some((event) => event.mutationType === "task:in-review-stall-terminal-provider-error")).toBe(false);
+
+    manager.stop();
+  });
+
+  it("FN-6113: retryable provider errors still use repeated-stall deadlock disposition", async () => {
+    const task = {
+      id: "FN-6113-RETRYABLE",
+      column: "in-review",
+      paused: false,
+      userPaused: false,
+      status: "failed",
+      error: "HTTP 503 service unavailable",
+      branch: "fusion/fn-6113-retryable",
+      worktree: "/tmp/fn-6113-retryable",
+      mergeDetails: {},
+      mergeRetries: 0,
+      steps: [{ name: "merge", status: "done" }],
+      workflowStepResults: [],
+      updatedAt: "2026-01-01T00:00:00.000Z",
+      log: [],
+    } as any satisfies Task;
+
+    const store = createStore(task);
+    const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
+
+    vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
+    expect(await manager.surfaceInReviewStalls()).toBe(1);
+    expect(task.paused).toBe(false);
+    vi.setSystemTime(new Date("2026-01-01T00:12:00.000Z"));
+    expect(await manager.surfaceInReviewStalls()).toBe(1);
+    expect(task.paused).toBe(false);
+    vi.setSystemTime(new Date("2026-01-01T00:14:00.000Z"));
+    expect(await manager.surfaceInReviewStalls()).toBe(1);
+
+    expect(task.paused).toBe(true);
+    expect(task.pausedReason).toBe("in-review-stall-deadlock");
+    expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall surfaced [merge-blocker]:"))).toHaveLength(2);
+    expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall auto-disposed [merge-blocker]:"))).toHaveLength(1);
+    expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed ["))).toBe(false);
+
+    manager.stop();
+  });
+
  it("FN-6070: rejected limbo requeues do not increment into deadlock disposition", async () => {
    const task = {
      id: "FN-6070-REJECTED",
--- a/packages/engine/src/run-audit.ts
+++ b/packages/engine/src/run-audit.ts
@@ -523,6 +523,7 @@ export type DatabaseMutationType =
   */
  | "session:runtime-resolved"
  | "task:in-review-stall-deadlock-disposed"
+  | "task:in-review-stall-terminal-provider-error"
  | "task:finalize-unproven-blocked"
  /**
   * FN-5490/FN-5517/FN-5526/FN-5540 lost-work guard: the merger or self-heal
--- a/packages/engine/src/self-healing.ts
+++ b/packages/engine/src/self-healing.ts
@@ -28,7 +28,7 @@ import { promisify } from "node:util";
 import { setImmediate as setImmediateCb } from "node:timers";
 import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { isAbsolute, join, relative, resolve } from "node:path";
-import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
+import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
 import type { MeshLeaseManager } from "./mesh-lease-manager.js";
 import { createLogger, schedulerLog } from "./logger.js";
 import { mergeEffectiveSettings } from "./effective-settings.js";
@@ -5418,6 +5418,34 @@ export class SelfHealingManager {
          continue;
        }

+        if (signal.code === "non-retryable-provider-error" && task.userPaused !== true) {
+          await this.store.logEntry(task.id, `${IN_REVIEW_STALL_TERMINAL_LOG_PREFIX}${signal.code}]: ${signal.reason}`);
+          await this.store.updateTask(task.id, {
+            paused: true,
+            pausedReason: "non-retryable-provider-error",
+            status: "failed",
+            error: `Terminal provider error (non-retryable): ${signal.reason}`,
+          });
+          const auditor = createRunAuditor(this.store, {
+            runId: generateSyntheticRunId("self-healing-stall-terminal-provider-error", task.id),
+            agentId: "self-healing",
+            taskId: task.id,
+            phase: "self-healing",
+          });
+          await auditor.database({
+            type: "task:in-review-stall-terminal-provider-error",
+            target: task.id,
+            metadata: {
+              code: signal.code,
+              reason: signal.reason,
+              branch: task.branch ?? null,
+              worktree: task.worktree ?? null,
+            },
+          });
+          surfaced += 1;
+          continue;
+        }
+
        const previous = [...(task.log ?? [])]
          .reverse()
          .find((entry) => entry.action.startsWith(IN_REVIEW_STALL_LOG_PREFIX));