diff --git a/.changeset/transient-request-timeout.md b/.changeset/transient-request-timeout.md new file mode 100644 index 0000000000..4184fd480d --- /dev/null +++ b/.changeset/transient-request-timeout.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Provider request timeouts now retry with backoff instead of looping planning forever. +category: fix +dev: Adds `/\brequest timed out\b/i` and `/\bAPIConnectionTimeoutError\b/i` to `TRANSIENT_ERROR_PATTERNS` (`packages/engine/src/errors/transient-error-patterns.ts`). The Anthropic/OpenAI SDK `APIConnectionTimeoutError` default message `"Request timed out."` matched no pattern, so it fell through to `specifyTask`'s generic failure branch which restores `status: null` with no counter and no `nextRecoveryAt` — triage rediscovery then re-admitted the card every poll indefinitely. Now routes into the bounded `MAX_RECOVERY_RETRIES` (3) + 60s/120s/300s backoff policy. Pattern is anchored to "request timed out" rather than a bare timeout match so agent prose and verification output ("BuildKit timed out", "Test suite timed out after 30000ms") stay permanent. Does not affect model fallback, which pi decides internally. diff --git a/packages/engine/src/__tests__/transient-error-detector.test.ts b/packages/engine/src/__tests__/transient-error-detector.test.ts index cd63c30c71..3c848448fb 100644 --- a/packages/engine/src/__tests__/transient-error-detector.test.ts +++ b/packages/engine/src/__tests__/transient-error-detector.test.ts @@ -179,6 +179,36 @@ describe("Transient Error Detector", () => { expect(isTransientError("Aborted")).toBe(false); expect(isTransientError("The operation was aborted by user")).toBe(false); }); + + /* + FNXC:Reliability-ErrorClassification 2026-08-10-18:32: + `"Request timed out."` is the Anthropic/OpenAI SDK `APIConnectionTimeoutError` default message. + It previously matched no pattern, so it fell through to the generic planning-failure branch that + writes no counter and no backoff — triage re-admitted the card every poll, forever. Measured: + 48 such events across 10 tasks in 30 hours with 0-minute gaps between attempts. Classifying it + transient routes it into the bounded MAX_RECOVERY_RETRIES=3 + backoff policy. + */ + it("classifies a provider request timeout as transient", () => { + expect(isTransientError("Request timed out.")).toBe(true); + expect(isTransientError("Specification failed: Request timed out.")).toBe(true); + expect(isTransientError("request timed out")).toBe(true); + expect(isTransientError("APIConnectionTimeoutError: Request timed out.")).toBe(true); + }); + + /* + The pattern is anchored to "request timed out" rather than a bare /timed? out/ because agent log + prose and verification output legitimately contain "timed out". A broad match would reclassify + real, permanent task failures as retryable — the mistake the connection-only rule was written to + avoid. These strings are all observed in production task logs. + */ + it("does NOT treat non-request timeout prose as transient", () => { + expect(isTransientError("BuildKit timed out while building the image")).toBe(false); + expect(isTransientError("force-requeue after stuck-kill unwind timeout")).toBe(false); + expect(isTransientError("Test suite timed out after 30000ms")).toBe(false); + expect(isTransientError("Verification command timed out")).toBe(false); + // Pre-existing negative cases that must stay negative: "timeout" is not "timed out". + expect(isTransientError("Request timeout")).toBe(false); + }); }); describe("isNonPlanDefectPlanReviewFailure", () => { diff --git a/packages/engine/src/errors/transient-error-patterns.ts b/packages/engine/src/errors/transient-error-patterns.ts index abacbfc35d..7c2bfa4c61 100644 --- a/packages/engine/src/errors/transient-error-patterns.ts +++ b/packages/engine/src/errors/transient-error-patterns.ts @@ -47,10 +47,41 @@ export const TRANSIENT_ERROR_PATTERNS: RegExp[] = [ /ETIMEDOUT/i, /socket hang up/i, - // Timeout patterns (only when related to connections, not general timeouts) + // Timeout patterns (connection-scoped — see the provider-request-timeout block below for the + // narrow exception covering SDK request timeouts) /timeout.*connection/i, /connection.*timeout/i, + /* + FNXC:Reliability-ErrorClassification 2026-08-10-18:32: + Provider REQUEST timeouts are transient. `"Request timed out."` is the literal default message of + the Anthropic and OpenAI SDKs' `APIConnectionTimeoutError`, surfaced to Fusion by + `checkSessionError` after pi-coding-agent exhausts its own in-session retries. + + Previously the connection-scoped patterns above deliberately excluded "general timeouts", so this + string matched NOTHING and fell through to the generic failure branch in `specifyTask` + (triage.ts) — which restores `status: null` and writes no counter, no `nextRecoveryAt`, and no + park. Triage rediscovery then re-admitted the card on the very next poll, forever. + + Measured impact before this change: 48 `Specification failed: Request timed out.` events across + 10 tasks in 30 hours, with zero backoff between attempts (FN-8950 alone burned 8 consecutive + attempts over ~8 hours and never reached implementation). Failed planning attempts averaged 33 + minutes each — 91 of them in 2 days, ~50 hours of wall-clock producing nothing, which was 24% of + all planning time. Classifying these as transient routes them into the BOUNDED recovery policy + (`MAX_RECOVERY_RETRIES` = 3 with 60s/120s/300s jittered backoff via `nextRecoveryAt`) that the + connection-level patterns already use, so a provider blip costs three spaced retries instead of an + unbounded loop. + + ANCHORING IS LOAD-BEARING: match `request timed out`, never a bare /timed? out/. Agent log prose + and verification output legitimately contain "timed out" (observed: "BuildKit timed out", + "stuck-kill unwind timeout"), and a broad pattern would reclassify real, permanent task failures + as retryable — the exact mistake the original connection-only rule was written to avoid. This + does NOT affect model fallback, which pi decides internally and Fusion only observes + (`auth/fallback-model-observer.ts`). + */ + /\brequest timed out\b/i, + /\bAPIConnectionTimeoutError\b/i, + // AI provider abort errors — temporary request cancellations (e.g., Anthropic streaming aborts) // These occur when the provider's infrastructure drops an in-flight request. /request was aborted/i,