diff --git a/packages/core/src/__tests__/in-review-stall.test.ts b/packages/core/src/__tests__/in-review-stall.test.ts index a02aab92fd..3c678be2ed 100644 --- a/packages/core/src/__tests__/in-review-stall.test.ts +++ b/packages/core/src/__tests__/in-review-stall.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + classifyProviderError, countRecentIdenticalStallEntries, DEFAULT_MAX_AUTO_MERGE_RETRIES, DEFAULT_STALE_MERGING_MIN_AGE_MS, @@ -22,6 +23,45 @@ const baseTask = { updatedAt: new Date(NOW).toISOString(), }; +describe("classifyProviderError", () => { + it.each([ + "HTTP 400 invalid_request_error: unsupported parameter", + "model gpt-5.3-codex is not supported by this provider", + "model claude-next not found", + "Configured model provider/model was not found in the pi model registry", + "invalid model: gpt-unknown", + "The requested model does not exist", + "permission denied for model access", + "HTTP 403 forbidden for model", + "401 unauthorized: invalid api key", + ])("classifies non-retryable provider errors: %s", (message) => { + expect(classifyProviderError(message)).toBe("non_retryable"); + }); + + it.each([ + "HTTP 503 service unavailable", + "429 rate limit exceeded", + "Request timed out after 60s", + "ECONNRESET while reading provider response", + "provider overloaded; try again later", + ])("classifies retryable provider errors: %s", (message) => { + expect(classifyProviderError(message)).toBe("retryable"); + }); + + it.each([ + "", + "task has incomplete steps", + "merge conflict in package.json", + "unexpected provider response shape", + ])("classifies unknown errors: %s", (message) => { + expect(classifyProviderError(message)).toBe("unknown"); + }); + + it("gives non-retryable patterns precedence over retryable patterns", () => { + expect(classifyProviderError("HTTP 400 invalid_request_error after rate limit warning")).toBe("non_retryable"); + }); +}); + describe("countRecentIdenticalStallEntries", () => { const reason = "Failed to create worktree after 3 attempts"; const task = (log: Array<{ timestamp: string; action: string }>) => ({ log }); @@ -123,6 +163,36 @@ describe("getInReviewStallReason", () => { expect(signal?.reason).toContain("failed pre-merge workflow steps"); }); + it("returns non-retryable-provider-error for failed task provider blockers", () => { + const signal = getInReviewStallReason({ + ...baseTask, + status: "failed", + error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported", + }, { now: NOW }); + expect(signal?.code).toBe("non-retryable-provider-error"); + expect(signal?.reason).toBe("Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported"); + }); + + it("keeps retryable provider errors as merge-blockers", () => { + const signal = getInReviewStallReason({ + ...baseTask, + status: "failed", + error: "HTTP 503 service unavailable", + }, { now: NOW }); + expect(signal?.code).toBe("merge-blocker"); + expect(signal?.reason).toBe("task is marked 'failed': HTTP 503 service unavailable"); + }); + + it("keeps unknown failed task errors as merge-blockers", () => { + const signal = getInReviewStallReason({ + ...baseTask, + status: "failed", + error: "merge conflict requires manual resolution", + }, { now: NOW }); + expect(signal?.code).toBe("merge-blocker"); + expect(signal?.reason).toBe("task is marked 'failed': merge conflict requires manual resolution"); + }); + it("suppresses merge-blocker when autoMerge is disabled", () => { const signal = getInReviewStallReason({ ...baseTask, diff --git a/packages/core/src/in-review-stall.ts b/packages/core/src/in-review-stall.ts index 7b8b2fd9f2..23bfe0300c 100644 --- a/packages/core/src/in-review-stall.ts +++ b/packages/core/src/in-review-stall.ts @@ -15,7 +15,10 @@ export type InReviewStallCode = | "merge-blocker" | "transient-merge-status-no-owner" | "merge-retries-exhausted" - | "no-worktree-no-merge-confirmed"; + | "no-worktree-no-merge-confirmed" + | "non-retryable-provider-error"; + +export type ProviderErrorClassification = "non_retryable" | "retryable" | "unknown"; export interface InReviewStallSignal { reason: string; @@ -40,8 +43,45 @@ export const DEFAULT_STALE_MERGING_MIN_AGE_MS = 5 * 60_000; export const DEFAULT_MAX_AUTO_MERGE_RETRIES = 3; export const IN_REVIEW_STALL_LOG_PREFIX = "In-review stall surfaced ["; export const IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX = "In-review stall auto-disposed ["; +export const IN_REVIEW_STALL_TERMINAL_LOG_PREFIX = "In-review stall terminal disposed ["; const TRANSIENT_MERGE_STATUSES = new Set(["merging", "merging-pr", "merging-fix"]); +const FAILED_TASK_MERGE_BLOCKER_PREFIX = "task is marked 'failed':"; + +export function classifyProviderError(error: string): ProviderErrorClassification { + const normalized = error.trim().toLowerCase(); + if (!normalized) return "unknown"; + + if ( + (/\b400\b/.test(normalized) && normalized.includes("invalid_request_error")) + || /model\b.*\bis not supported/.test(normalized) + || /model\b.*\bnot found/.test(normalized) + || normalized.includes("was not found in the pi model registry") + || normalized.includes("invalid model") + || normalized.includes("model does not exist") + || (/\b401\b/.test(normalized) && normalized.includes("unauthorized")) + || (/\b403\b/.test(normalized) && normalized.includes("forbidden")) + || (/permission denied/.test(normalized) && /model|access/.test(normalized)) + ) { + return "non_retryable"; + } + + if ( + /\b429\b/.test(normalized) + || normalized.includes("too many requests") + || normalized.includes("rate limit") + || /\b5\d\d\b/.test(normalized) + || normalized.includes("overloaded") + || normalized.includes("econnreset") + || normalized.includes("etimedout") + || normalized.includes("timed out") + || normalized.includes("timeout") + ) { + return "retryable"; + } + + return "unknown"; +} export function countRecentIdenticalStallEntries( task: Pick, @@ -135,6 +175,17 @@ export function getInReviewStallReason( const mergeBlocker = getTaskMergeBlocker(task); if (mergeBlocker) { + if (mergeBlocker.startsWith(FAILED_TASK_MERGE_BLOCKER_PREFIX)) { + const error = mergeBlocker.slice(FAILED_TASK_MERGE_BLOCKER_PREFIX.length).trim(); + if (classifyProviderError(error) === "non_retryable") { + return { + code: "non-retryable-provider-error", + reason: `Terminal provider error: ${error}`, + observedAt, + }; + } + } + return { code: "merge-blocker", reason: mergeBlocker, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 326972e0d8..8857eabb76 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -630,14 +630,16 @@ export { type FindVitestProcessIdsOptions, } from "./vitest-processes.js"; export { + classifyProviderError, countRecentIdenticalStallEntries, getInReviewStallReason, IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, + IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, DEFAULT_STALE_MERGING_MIN_AGE_MS, DEFAULT_MAX_AUTO_MERGE_RETRIES, } from "./in-review-stall.js"; -export type { InReviewStallSignal, InReviewStallCode } from "./in-review-stall.js"; +export type { InReviewStallSignal, InReviewStallCode, ProviderErrorClassification } from "./in-review-stall.js"; export { getStalePausedReviewSignal, DEFAULT_STALE_PAUSED_REVIEW_THRESHOLD_MS, diff --git a/packages/dashboard/app/utils/__tests__/inReviewStallCopy.test.ts b/packages/dashboard/app/utils/__tests__/inReviewStallCopy.test.ts index bbfaace400..02c76d161a 100644 --- a/packages/dashboard/app/utils/__tests__/inReviewStallCopy.test.ts +++ b/packages/dashboard/app/utils/__tests__/inReviewStallCopy.test.ts @@ -8,6 +8,7 @@ describe("inReviewStallCopy", () => { ["transient-merge-status-no-owner", "Merge stalled"], ["merge-retries-exhausted", "Retries exhausted"], ["no-worktree-no-merge-confirmed", "No worktree"], + ["non-retryable-provider-error", "Provider error"], ] as const)("returns populated copy for %s", (code, badgeLabel) => { const copy = getInReviewStallCopy({ code, @@ -57,7 +58,7 @@ describe("inReviewStallCopy", () => { expect(copy.counter).toBeUndefined(); }); - it.each(["merge-blocker", "transient-merge-status-no-owner", "no-worktree-no-merge-confirmed"] as const)( + it.each(["merge-blocker", "transient-merge-status-no-owner", "no-worktree-no-merge-confirmed", "non-retryable-provider-error"] as const)( "does not render counter for non-retry stall code %s", (code) => { const copy = getInReviewStallCopy( @@ -98,6 +99,8 @@ describe("inReviewStallCopy", () => { ["merge-blocker", undefined, true], ["merge-retries-exhausted", "merging", true], ["transient-merge-status-no-owner", "merging", true], + ["non-retryable-provider-error", undefined, true], + ["non-retryable-provider-error", "merging", true], ["no-worktree-no-merge-confirmed", undefined, false], ["no-worktree-no-merge-confirmed", "merging", false], ] as const)("badge visibility for %s with status %s is %s", (code, status, expected) => { diff --git a/packages/dashboard/app/utils/inReviewStallCopy.ts b/packages/dashboard/app/utils/inReviewStallCopy.ts index bf9b3194f3..cee550a734 100644 --- a/packages/dashboard/app/utils/inReviewStallCopy.ts +++ b/packages/dashboard/app/utils/inReviewStallCopy.ts @@ -22,6 +22,7 @@ const BADGE_LABEL_BY_CODE: Record = { "transient-merge-status-no-owner": "Merge stalled", "merge-retries-exhausted": "Retries exhausted", "no-worktree-no-merge-confirmed": "No worktree", + "non-retryable-provider-error": "Provider error", }; const COPY_BY_CODE: Record> = { @@ -50,6 +51,13 @@ const COPY_BY_CODE: Record manager.stop(); }); + it("FN-6113: terminal provider errors dispose in a single stall cycle", async () => { + const task = { + id: "FN-6113-TERMINAL", + column: "in-review", + paused: false, + userPaused: false, + status: "failed", + error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported", + branch: "fusion/fn-6113-terminal", + worktree: "/tmp/fn-6113-terminal", + mergeDetails: {}, + mergeRetries: 0, + steps: [{ name: "merge", status: "done" }], + workflowStepResults: [], + updatedAt: "2026-01-01T00:00:00.000Z", + log: [], + } as any satisfies Task; + + const store = createStore(task); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" }); + + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + expect(await manager.surfaceInReviewStalls()).toBe(1); + + expect(task.paused).toBe(true); + expect(task.pausedReason).toBe("non-retryable-provider-error"); + expect(task.status).toBe("failed"); + expect(task.error).toBe( + "Terminal provider error (non-retryable): Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported", + ); + expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed [non-retryable-provider-error]:"))).toHaveLength(1); + expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall auto-disposed ["))).toBe(false); + expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall surfaced ["))).toBe(false); + + const auditEvents = (store as any).__auditEvents as any[]; + const terminalAuditEvents = auditEvents.filter((event) => event.mutationType === "task:in-review-stall-terminal-provider-error"); + expect(terminalAuditEvents).toHaveLength(1); + expect(terminalAuditEvents[0]).toMatchObject({ + domain: "database", + taskId: "FN-6113-TERMINAL", + target: "FN-6113-TERMINAL", + metadata: expect.objectContaining({ + code: "non-retryable-provider-error", + reason: "Terminal provider error: HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported", + branch: "fusion/fn-6113-terminal", + worktree: "/tmp/fn-6113-terminal", + }), + }); + + manager.stop(); + }); + + it("FN-6113: terminal provider errors are ignored when autoMerge is disabled", async () => { + const task = { + id: "FN-6113-AUTOMERGE-OFF", + column: "in-review", + paused: false, + userPaused: false, + status: "failed", + error: "HTTP 400 invalid_request_error: model gpt-5.3-codex is not supported", + branch: "fusion/fn-6113-automerge-off", + worktree: "/tmp/fn-6113-automerge-off", + mergeDetails: {}, + mergeRetries: 0, + steps: [{ name: "merge", status: "done" }], + workflowStepResults: [], + updatedAt: "2026-01-01T00:00:00.000Z", + log: [], + } as any satisfies Task; + + const store = createStore(task, { autoMerge: false }); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" }); + + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + expect(await manager.surfaceInReviewStalls()).toBe(0); + expect(task.paused).toBe(false); + expect((store.updateTask as any).mock.calls.length).toBe(0); + expect(task.log).toHaveLength(0); + + manager.stop(); + }); + + it("FN-6113: terminal provider errors do not auto-dispose userPaused tasks", async () => { + const task = { + id: "FN-6113-USER-PAUSED", + column: "in-review", + paused: false, + userPaused: true, + status: "failed", + error: "HTTP 403 forbidden for model access", + branch: "fusion/fn-6113-user-paused", + worktree: "/tmp/fn-6113-user-paused", + mergeDetails: {}, + mergeRetries: 0, + steps: [{ name: "merge", status: "done" }], + workflowStepResults: [], + updatedAt: "2026-01-01T00:00:00.000Z", + log: [], + } as any satisfies Task; + + const store = createStore(task); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" }); + + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + expect(await manager.surfaceInReviewStalls()).toBe(1); + expect(task.paused).toBe(false); + expect(task.pausedReason).toBeUndefined(); + expect((store.updateTask as any).mock.calls.length).toBe(0); + expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed ["))).toBe(false); + expect(((store as any).__auditEvents as any[]).some((event) => event.mutationType === "task:in-review-stall-terminal-provider-error")).toBe(false); + + manager.stop(); + }); + + it("FN-6113: retryable provider errors still use repeated-stall deadlock disposition", async () => { + const task = { + id: "FN-6113-RETRYABLE", + column: "in-review", + paused: false, + userPaused: false, + status: "failed", + error: "HTTP 503 service unavailable", + branch: "fusion/fn-6113-retryable", + worktree: "/tmp/fn-6113-retryable", + mergeDetails: {}, + mergeRetries: 0, + steps: [{ name: "merge", status: "done" }], + workflowStepResults: [], + updatedAt: "2026-01-01T00:00:00.000Z", + log: [], + } as any satisfies Task; + + const store = createStore(task); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" }); + + vi.setSystemTime(new Date("2026-01-01T00:10:00.000Z")); + expect(await manager.surfaceInReviewStalls()).toBe(1); + expect(task.paused).toBe(false); + vi.setSystemTime(new Date("2026-01-01T00:12:00.000Z")); + expect(await manager.surfaceInReviewStalls()).toBe(1); + expect(task.paused).toBe(false); + vi.setSystemTime(new Date("2026-01-01T00:14:00.000Z")); + expect(await manager.surfaceInReviewStalls()).toBe(1); + + expect(task.paused).toBe(true); + expect(task.pausedReason).toBe("in-review-stall-deadlock"); + expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall surfaced [merge-blocker]:"))).toHaveLength(2); + expect(task.log.filter((entry: { action: string }) => entry.action.startsWith("In-review stall auto-disposed [merge-blocker]:"))).toHaveLength(1); + expect(task.log.some((entry: { action: string }) => entry.action.startsWith("In-review stall terminal disposed ["))).toBe(false); + + manager.stop(); + }); + it("FN-6070: rejected limbo requeues do not increment into deadlock disposition", async () => { const task = { id: "FN-6070-REJECTED", diff --git a/packages/engine/src/run-audit.ts b/packages/engine/src/run-audit.ts index 6f52b46221..a8cec71a20 100644 --- a/packages/engine/src/run-audit.ts +++ b/packages/engine/src/run-audit.ts @@ -523,6 +523,7 @@ export type DatabaseMutationType = */ | "session:runtime-resolved" | "task:in-review-stall-deadlock-disposed" + | "task:in-review-stall-terminal-provider-error" | "task:finalize-unproven-blocked" /** * FN-5490/FN-5517/FN-5526/FN-5540 lost-work guard: the merger or self-heal diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 7db7f203ef..53878b3044 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -28,7 +28,7 @@ import { promisify } from "node:util"; import { setImmediate as setImmediateCb } from "node:timers"; import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"; import { isAbsolute, join, relative, resolve } from "node:path"; -import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core"; +import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core"; import type { MeshLeaseManager } from "./mesh-lease-manager.js"; import { createLogger, schedulerLog } from "./logger.js"; import { mergeEffectiveSettings } from "./effective-settings.js"; @@ -5418,6 +5418,34 @@ export class SelfHealingManager { continue; } + if (signal.code === "non-retryable-provider-error" && task.userPaused !== true) { + await this.store.logEntry(task.id, `${IN_REVIEW_STALL_TERMINAL_LOG_PREFIX}${signal.code}]: ${signal.reason}`); + await this.store.updateTask(task.id, { + paused: true, + pausedReason: "non-retryable-provider-error", + status: "failed", + error: `Terminal provider error (non-retryable): ${signal.reason}`, + }); + const auditor = createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-healing-stall-terminal-provider-error", task.id), + agentId: "self-healing", + taskId: task.id, + phase: "self-healing", + }); + await auditor.database({ + type: "task:in-review-stall-terminal-provider-error", + target: task.id, + metadata: { + code: signal.code, + reason: signal.reason, + branch: task.branch ?? null, + worktree: task.worktree ?? null, + }, + }); + surfaced += 1; + continue; + } + const previous = [...(task.log ?? [])] .reverse() .find((entry) => entry.action.startsWith(IN_REVIEW_STALL_LOG_PREFIX));