root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FN-6113: fail fast on terminal provider errors

Browse Source 
Fail fast on non-retryable provider failures in the in-review stall recovery flow.

- classify failed-task provider errors as non-retryable, retryable, or unknown in core stall detection
- surface terminal provider errors with dedicated dashboard copy and exported stall signal metadata
- pause in-review tasks immediately on non-retryable provider failures and record a dedicated run-audit mutation
- add regression coverage for provider error classification, self-healing disposition, and dashboard badge behavior

Files changed:
 .../core/src/__tests__/in-review-stall.test.ts     |  70 ++++++++++
 packages/core/src/in-review-stall.ts               |  53 ++++++-
 packages/core/src/index.ts                         |   4 +-
 .../app/utils/__tests__/inReviewStallCopy.test.ts  |   5 +-
 packages/dashboard/app/utils/inReviewStallCopy.ts  |   8 ++
 .../in-review-stall-deadlock-disposition.test.ts   | 153 +++++++++++++++++++++
 packages/engine/src/run-audit.ts                   |   1 +
 packages/engine/src/self-healing.ts                |  30 +++-
 8 files changed, 320 insertions(+), 4 deletions(-)

Fusion-Task-Id: FN-6113

Fusion-Task-Lineage: 5e89adb4-6b59-4deb-8811-5c75d6fa19cc
This commit is contained in:
gsxdsm
2026-06-09 12:13:05 -07:00
parent 6a77217f94
commit 320b153c45
8 changed files with 320 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
packages/core/src/__tests__/in-review-stall.test.ts
View File

@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import {
classifyProviderError,
countRecentIdenticalStallEntries,
DEFAULT_MAX_AUTO_MERGE_RETRIES,
DEFAULT_STALE_MERGING_MIN_AGE_MS,
@@ -22,6 +23,45 @@ const baseTask = {
updatedAt: new Date(NOW).toISOString(),
};
describe("classifyProviderError", () => {
it.each([
"HTTP 400 invalid_request_error: unsupported parameter",
"model gpt-5.3-codex is not supported by this provider",
"model claude-next not found",
"Configured model provider/model was not found in the pi model registry",
"invalid model: gpt-unknown",
"The requested model does not exist",
"permission denied for model access",
"HTTP 403 forbidden for model",
"401 unauthorized: invalid api key",
])("classifies non-retryable provider errors: %s", (message) => {
expect(classifyProviderError(message)).toBe("non_retryable");
});
it.each([
"HTTP 503 service unavailable",
"429 rate limit exceeded",
"Request timed out after 60s",
"ECONNRESET while reading provider response",
"provider overloaded; try again later",
])("classifies retryable provider errors: %s", (message) => {
expect(classifyProviderError(message)).toBe("retryable");
});
it.each([
"",
"task has incomplete steps",
"merge conflict in package.json",
"unexpected provider response shape",
])("classifies unknown errors: %s", (message) => {
expect(classifyProviderError(message)).toBe("unknown");
});
it("gives non-retryable patterns precedence over retryable patterns", () => {
expect(classifyProviderError("HTTP 400 invalid_request_error after rate limit warning")).toBe("non_retryable");
});
});
describe("countRecentIdenticalStallEntries", () => {
const reason = "Failed to create worktree after 3 attempts";
const task = (log: Array<{ timestamp: string; action: string }>) => ({ log });
@@ -123,6 +163,36 @@ describe("getInReviewStallReason", () => {
expect(signal?.reason).toContain("failed pre-merge workflow steps");
});
it("returns non-retryable-provider-error for failed task provider blockers", () => {
const signal = getInReviewStallReason({
...baseTask,
status: "failed",
error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
}, { now: NOW });
expect(signal?.code).toBe("non-retryable-provider-error");
expect(signal?.reason).toBe("Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported");
});
it("keeps retryable provider errors as merge-blockers", () => {
const signal = getInReviewStallReason({
...baseTask,
status: "failed",
error: "HTTP 503 service unavailable",
}, { now: NOW });
expect(signal?.code).toBe("merge-blocker");
expect(signal?.reason).toBe("task is marked 'failed': HTTP 503 service unavailable");
});
it("keeps unknown failed task errors as merge-blockers", () => {
const signal = getInReviewStallReason({
...baseTask,
status: "failed",
error: "merge conflict requires manual resolution",
}, { now: NOW });
expect(signal?.code).toBe("merge-blocker");
expect(signal?.reason).toBe("task is marked 'failed': merge conflict requires manual resolution");
});
it("suppresses merge-blocker when autoMerge is disabled", () => {
const signal = getInReviewStallReason({
...baseTask,

53
packages/core/src/in-review-stall.ts
View File

@@ -15,7 +15,10 @@ export type InReviewStallCode =
| "merge-blocker"
| "transient-merge-status-no-owner"
| "merge-retries-exhausted"
| "no-worktree-no-merge-confirmed";
| "no-worktree-no-merge-confirmed"
| "non-retryable-provider-error";
export type ProviderErrorClassification = "non_retryable" | "retryable" | "unknown";
export interface InReviewStallSignal {
reason: string;
@@ -40,8 +43,45 @@ export const DEFAULT_STALE_MERGING_MIN_AGE_MS = 5 * 60_000;
export const DEFAULT_MAX_AUTO_MERGE_RETRIES = 3;
export const IN_REVIEW_STALL_LOG_PREFIX = "In-review stall surfaced [";
export const IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX = "In-review stall auto-disposed [";
export const IN_REVIEW_STALL_TERMINAL_LOG_PREFIX = "In-review stall terminal disposed [";
const TRANSIENT_MERGE_STATUSES = new Set(["merging", "merging-pr", "merging-fix"]);
const FAILED_TASK_MERGE_BLOCKER_PREFIX = "task is marked 'failed':";
export function classifyProviderError(error: string): ProviderErrorClassification {
const normalized = error.trim().toLowerCase();
if (!normalized) return "unknown";
if (
(/\b400\b/.test(normalized) && normalized.includes("invalid_request_error"))
|| /model\b.*\bis not supported/.test(normalized)
|| /model\b.*\bnot found/.test(normalized)
|| normalized.includes("was not found in the pi model registry")
|| normalized.includes("invalid model")
|| normalized.includes("model does not exist")
|| (/\b401\b/.test(normalized) && normalized.includes("unauthorized"))
|| (/\b403\b/.test(normalized) && normalized.includes("forbidden"))
|| (/permission denied/.test(normalized) && /model|access/.test(normalized))
) {
return "non_retryable";
}
if (
/\b429\b/.test(normalized)
|| normalized.includes("too many requests")
|| normalized.includes("rate limit")
|| /\b5\d\d\b/.test(normalized)
|| normalized.includes("overloaded")
|| normalized.includes("econnreset")
|| normalized.includes("etimedout")
|| normalized.includes("timed out")
|| normalized.includes("timeout")
) {
return "retryable";
}
return "unknown";
}
export function countRecentIdenticalStallEntries(
task: Pick<Task, "log">,
@@ -135,6 +175,17 @@ export function getInReviewStallReason(
const mergeBlocker = getTaskMergeBlocker(task);
if (mergeBlocker) {
if (mergeBlocker.startsWith(FAILED_TASK_MERGE_BLOCKER_PREFIX)) {
const error = mergeBlocker.slice(FAILED_TASK_MERGE_BLOCKER_PREFIX.length).trim();
if (classifyProviderError(error) === "non_retryable") {
return {
code: "non-retryable-provider-error",
reason: `Terminal provider error: ${error}`,
observedAt,
};
}
}
return {
code: "merge-blocker",
reason: mergeBlocker,

4
packages/core/src/index.ts
View File

@@ -630,14 +630,16 @@ export {
type FindVitestProcessIdsOptions,
} from "./vitest-processes.js";
export {
classifyProviderError,
countRecentIdenticalStallEntries,
getInReviewStallReason,
IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX,
IN_REVIEW_STALL_LOG_PREFIX,
IN_REVIEW_STALL_TERMINAL_LOG_PREFIX,
DEFAULT_STALE_MERGING_MIN_AGE_MS,
DEFAULT_MAX_AUTO_MERGE_RETRIES,
} from "./in-review-stall.js";
export type { InReviewStallSignal, InReviewStallCode } from "./in-review-stall.js";
export type { InReviewStallSignal, InReviewStallCode, ProviderErrorClassification } from "./in-review-stall.js";
export {
getStalePausedReviewSignal,
DEFAULT_STALE_PAUSED_REVIEW_THRESHOLD_MS,

5
packages/dashboard/app/utils/__tests__/inReviewStallCopy.test.ts
View File

@@ -8,6 +8,7 @@ describe("inReviewStallCopy", () => {
["transient-merge-status-no-owner", "Merge stalled"],
["merge-retries-exhausted", "Retries exhausted"],
["no-worktree-no-merge-confirmed", "No worktree"],
["non-retryable-provider-error", "Provider error"],
] as const)("returns populated copy for %s", (code, badgeLabel) => {
const copy = getInReviewStallCopy({
code,
@@ -57,7 +58,7 @@ describe("inReviewStallCopy", () => {
expect(copy.counter).toBeUndefined();
});
it.each(["merge-blocker", "transient-merge-status-no-owner", "no-worktree-no-merge-confirmed"] as const)(
it.each(["merge-blocker", "transient-merge-status-no-owner", "no-worktree-no-merge-confirmed", "non-retryable-provider-error"] as const)(
"does not render counter for non-retry stall code %s",
(code) => {
const copy = getInReviewStallCopy(
@@ -98,6 +99,8 @@ describe("inReviewStallCopy", () => {
["merge-blocker", undefined, true],
["merge-retries-exhausted", "merging", true],
["transient-merge-status-no-owner", "merging", true],
["non-retryable-provider-error", undefined, true],
["non-retryable-provider-error", "merging", true],
["no-worktree-no-merge-confirmed", undefined, false],
["no-worktree-no-merge-confirmed", "merging", false],
] as const)("badge visibility for %s with status %s is %s", (code, status, expected) => {

8
packages/dashboard/app/utils/inReviewStallCopy.ts
View File

@@ -22,6 +22,7 @@ const BADGE_LABEL_BY_CODE: Record<InReviewStallCode, string> = {
"transient-merge-status-no-owner": "Merge stalled",
"merge-retries-exhausted": "Retries exhausted",
"no-worktree-no-merge-confirmed": "No worktree",
"non-retryable-provider-error": "Provider error",
};
const COPY_BY_CODE: Record<InReviewStallCode, Omit<InReviewStallCopy, "badgeLabel" | "counter" | "code">> = {
@@ -50,6 +51,13 @@ const COPY_BY_CODE: Record<InReviewStallCode, Omit<InReviewStallCopy, "badgeLabe
suggestedAction:
"Check the Changes tab and Git history; if the work landed, mark the merge confirmed, otherwise re-create the worktree.",
},
"non-retryable-provider-error": {
headline: "Terminal provider error",
description:
"The provider rejected the task with a non-retryable error such as an invalid model, unsupported request, or permission denial. Self-healing will pause the task instead of retrying the same failure.",
suggestedAction:
"Fix the model/provider configuration or permissions, then unpause and retry the task once the provider can accept the request.",
},
};
function defaultCopy(signal: InReviewStallSignal): InReviewStallCopy {

153
packages/engine/src/__tests__/reliability-interactions/in-review-stall-deadlock-disposition.test.ts
View File

@@ -115,6 +115,159 @@ describe("reliability interactions: in-review stall deadlock disposition", () =>
manager.stop();
});
it("FN-6113: terminal provider errors dispose in a single stall cycle", async () => {
const task = {
id: "FN-6113-TERMINAL",
column: "in-review",
paused: false,
userPaused: false,
status: "failed",
error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
branch: "fusion/fn-6113-terminal",
worktree: "/tmp/fn-6113-terminal",
mergeDetails: {},
mergeRetries: 0,
steps: [{ name: "merge", status: "done" }],
workflowStepResults: [],
updatedAt: "2026-01-01T00:00:00.000Z",
log: [],
} as any satisfies Task;
const store = createStore(task);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
expect(await manager.surfaceInReviewStalls()).toBe(1);
expect(task.paused).toBe(true);
expect(task.pausedReason).toBe("non-retryable-provider-error");
expect(task.status).toBe("failed");
expect(task.error).toBe(
"Terminal provider error (non-retryable): Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
);
expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed [non-retryable-provider-error]:"))).toHaveLength(1);
expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall auto-disposed ["))).toBe(false);
expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall surfaced ["))).toBe(false);
const auditEvents = (store as any).__auditEvents as any[];
const terminalAuditEvents = auditEvents.filter((event) => event.mutationType === "task:in-review-stall-terminal-provider-error");
expect(terminalAuditEvents).toHaveLength(1);
expect(terminalAuditEvents[0]).toMatchObject({
domain: "database",
taskId: "FN-6113-TERMINAL",
target: "FN-6113-TERMINAL",
metadata: expect.objectContaining({
code: "non-retryable-provider-error",
reason: "Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
branch: "fusion/fn-6113-terminal",
worktree: "/tmp/fn-6113-terminal",
}),
});
manager.stop();
});
it("FN-6113: terminal provider errors are ignored when autoMerge is disabled", async () => {
const task = {
id: "FN-6113-AUTOMERGE-OFF",
column: "in-review",
paused: false,
userPaused: false,
status: "failed",
error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported",
branch: "fusion/fn-6113-automerge-off",
worktree: "/tmp/fn-6113-automerge-off",
mergeDetails: {},
mergeRetries: 0,
steps: [{ name: "merge", status: "done" }],
workflowStepResults: [],
updatedAt: "2026-01-01T00:00:00.000Z",
log: [],
} as any satisfies Task;
const store = createStore(task, { autoMerge: false });
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
expect(await manager.surfaceInReviewStalls()).toBe(0);
expect(task.paused).toBe(false);
expect((store.updateTask as any).mock.calls.length).toBe(0);
expect(task.log).toHaveLength(0);
manager.stop();
});
it("FN-6113: terminal provider errors do not auto-dispose userPaused tasks", async () => {
const task = {
id: "FN-6113-USER-PAUSED",
column: "in-review",
paused: false,
userPaused: true,
status: "failed",
error: "HTTP 403 forbidden for model access",
branch: "fusion/fn-6113-user-paused",
worktree: "/tmp/fn-6113-user-paused",
mergeDetails: {},
mergeRetries: 0,
steps: [{ name: "merge", status: "done" }],
workflowStepResults: [],
updatedAt: "2026-01-01T00:00:00.000Z",
log: [],
} as any satisfies Task;
const store = createStore(task);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
expect(await manager.surfaceInReviewStalls()).toBe(1);
expect(task.paused).toBe(false);
expect(task.pausedReason).toBeUndefined();
expect((store.updateTask as any).mock.calls.length).toBe(0);
expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed ["))).toBe(false);
expect(((store as any).__auditEvents as any[]).some((event) => event.mutationType === "task:in-review-stall-terminal-provider-error")).toBe(false);
manager.stop();
});
it("FN-6113: retryable provider errors still use repeated-stall deadlock disposition", async () => {
const task = {
id: "FN-6113-RETRYABLE",
column: "in-review",
paused: false,
userPaused: false,
status: "failed",
error: "HTTP 503 service unavailable",
branch: "fusion/fn-6113-retryable",
worktree: "/tmp/fn-6113-retryable",
mergeDetails: {},
mergeRetries: 0,
steps: [{ name: "merge", status: "done" }],
workflowStepResults: [],
updatedAt: "2026-01-01T00:00:00.000Z",
log: [],
} as any satisfies Task;
const store = createStore(task);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z"));
expect(await manager.surfaceInReviewStalls()).toBe(1);
expect(task.paused).toBe(false);
vi.setSystemTime(new Date("2026-01-01T00:12:00.000Z"));
expect(await manager.surfaceInReviewStalls()).toBe(1);
expect(task.paused).toBe(false);
vi.setSystemTime(new Date("2026-01-01T00:14:00.000Z"));
expect(await manager.surfaceInReviewStalls()).toBe(1);
expect(task.paused).toBe(true);
expect(task.pausedReason).toBe("in-review-stall-deadlock");
expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall surfaced [merge-blocker]:"))).toHaveLength(2);
expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall auto-disposed [merge-blocker]:"))).toHaveLength(1);
expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed ["))).toBe(false);
manager.stop();
});
it("FN-6070: rejected limbo requeues do not increment into deadlock disposition", async () => {
const task = {
id: "FN-6070-REJECTED",

1
packages/engine/src/run-audit.ts
View File

@@ -523,6 +523,7 @@ export type DatabaseMutationType =
*/
| "session:runtime-resolved"
| "task:in-review-stall-deadlock-disposed"
| "task:in-review-stall-terminal-provider-error"
| "task:finalize-unproven-blocked"
/**
* FN-5490/FN-5517/FN-5526/FN-5540 lost-work guard: the merger or self-heal

30
packages/engine/src/self-healing.ts
View File

@@ -28,7 +28,7 @@ import { promisify } from "node:util";
import { setImmediate as setImmediateCb } from "node:timers";
import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
import { isAbsolute, join, relative, resolve } from "node:path";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger, schedulerLog } from "./logger.js";
import { mergeEffectiveSettings } from "./effective-settings.js";
@@ -5418,6 +5418,34 @@ export class SelfHealingManager {
continue;
}
if (signal.code === "non-retryable-provider-error" && task.userPaused !== true) {
await this.store.logEntry(task.id, `${IN_REVIEW_STALL_TERMINAL_LOG_PREFIX}${signal.code}]: ${signal.reason}`);
await this.store.updateTask(task.id, {
paused: true,
pausedReason: "non-retryable-provider-error",
status: "failed",
error: `Terminal provider error (non-retryable): ${signal.reason}`,
});
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-healing-stall-terminal-provider-error", task.id),
agentId: "self-healing",
taskId: task.id,
phase: "self-healing",
});
await auditor.database({
type: "task:in-review-stall-terminal-provider-error",
target: task.id,
metadata: {
code: signal.code,
reason: signal.reason,
branch: task.branch ?? null,
worktree: task.worktree ?? null,
},
});
surfaced += 1;
continue;
}
const previous = [...(task.log ?? [])]
.reverse()
.find((entry) => entry.action.startsWith(IN_REVIEW_STALL_LOG_PREFIX));