From df88cb7289ef3f3608fb8e84e73eabdc1c2ef4eb Mon Sep 17 00:00:00 2001 From: fusion-merge-train Date: Sun, 5 Jul 2026 05:30:12 +0000 Subject: [PATCH] fix(engine): retry transient auth errors in withRateLimitRetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A long-running agent session holds its OAuth access token in memory. When the token rotates mid-run (Claude Max tokens have an ~8h lifetime), the next API call fails with 401 authentication_error and withRateLimitRetry re-throws it immediately — the task is marked failed and the operator is paged, even though the refreshed credentials make the very next call succeed. Observed recurring at every ~8h token boundary, hitting whatever task or heartbeat is in flight. Add isTransientAuthError (authentication_error / invalid authentication credentials / token_expired / oauth scope) with its own small retry budget: 2 retries at a flat ~5s delay (credential refresh completes within seconds, so the 30s -> 2min rate-limit backoff curve would just prolong the outage). Auth retries decrement the loop counter so they never consume rate-limit attempts, and the existing rate-limit path is byte-for-byte unchanged. Genuinely bad credentials still propagate after ~10s. Co-Authored-By: Claude Fable 5 --- .../src/__tests__/rate-limit-retry.test.ts | 87 +++++++++++++++++++ packages/engine/src/rate-limit-retry.ts | 60 +++++++++++-- 2 files changed, 141 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/__tests__/rate-limit-retry.test.ts b/packages/engine/src/__tests__/rate-limit-retry.test.ts index 8dadec9fb5..01f86ec81c 100644 --- a/packages/engine/src/__tests__/rate-limit-retry.test.ts +++ b/packages/engine/src/__tests__/rate-limit-retry.test.ts @@ -218,4 +218,91 @@ describe("withRateLimitRetry", () => { const result = await promise; expect(result).toBe("ok"); }); + + it("retries a transient auth error and succeeds after credential rotation", async () => { + const authErr = new Error( + '401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + ); + const fn = vi + .fn() + .mockRejectedValueOnce(authErr) + .mockResolvedValueOnce("recovered"); + const onRetry = vi.fn(); + + const promise = withRateLimitRetry(fn, { onRetry }); + + // Auth retry uses a flat ~5s delay (5000ms ±10 %), not the 30s backoff + await vi.advanceTimersByTimeAsync(6000); + + const result = await promise; + expect(result).toBe("recovered"); + expect(fn).toHaveBeenCalledTimes(2); + expect(onRetry).toHaveBeenCalledTimes(1); + expect(onRetry).toHaveBeenCalledWith(1, expect.any(Number), authErr); + }); + + it("throws after the transient-auth retry budget is exhausted", async () => { + const authErr = new Error("invalid authentication credentials"); + const fn = vi.fn().mockRejectedValue(authErr); + const onRetry = vi.fn(); + + const promise = withRateLimitRetry(fn, { onRetry }); + const assertion = expect(promise).rejects.toThrow( + "invalid authentication credentials", + ); + + for (let i = 0; i < 4; i++) { + await vi.advanceTimersByTimeAsync(6000); + } + + await assertion; + expect(fn).toHaveBeenCalledTimes(3); // initial + 2 auth retries + expect(onRetry).toHaveBeenCalledTimes(2); + }); + + it("does not let auth retries consume rate-limit attempts", async () => { + const fn = vi + .fn() + .mockRejectedValueOnce(new Error("token_expired")) + .mockRejectedValueOnce(new Error("429 too many requests")) + .mockResolvedValueOnce("ok"); + + // maxRetries: 1 — if the auth retry consumed the single rate-limit + // attempt, the 429 on the next call would exhaust the budget and throw. + const promise = withRateLimitRetry(fn, { + maxRetries: 1, + baseDelayMs: 100, + maxDelayMs: 1000, + }); + + await vi.advanceTimersByTimeAsync(6000); // auth retry delay + await vi.advanceTimersByTimeAsync(500); // rate-limit backoff + + const result = await promise; + expect(result).toBe("ok"); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it("classifies various transient auth error patterns correctly", async () => { + const patterns = [ + '{"type":"error","error":{"type":"authentication_error"}}', + "Invalid authentication credentials", + "token_expired", + "token expired", + "OAuth token does not meet scope requirements", + ]; + + for (const msg of patterns) { + const fn = vi + .fn() + .mockRejectedValueOnce(new Error(msg)) + .mockResolvedValueOnce("ok"); + + const promise = withRateLimitRetry(fn); + await vi.advanceTimersByTimeAsync(6000); + const result = await promise; + expect(result).toBe("ok"); + expect(fn).toHaveBeenCalledTimes(2); + } + }); }); diff --git a/packages/engine/src/rate-limit-retry.ts b/packages/engine/src/rate-limit-retry.ts index 5bf4629a22..905112f3bd 100644 --- a/packages/engine/src/rate-limit-retry.ts +++ b/packages/engine/src/rate-limit-retry.ts @@ -14,13 +14,39 @@ * pending retries when a task is paused, cancelled, or the engine is shutting * down — so agents don't sit in a 2-minute sleep unnecessarily. * - * **Scope:** Only rate-limit errors (as classified by `isUsageLimitError`) are - * retried. All other error types are re-thrown immediately so existing error - * handling (transient-error retry, failure marking, etc.) is unaffected. + * **Scope:** Rate-limit errors (as classified by `isUsageLimitError`) are + * retried with the backoff curve above. Transient authentication errors (as + * classified by `isTransientAuthError` — e.g. an OAuth access token rotating + * mid-run) get their own small retry budget with a short flat delay. All other + * error types are re-thrown immediately so existing error handling + * (transient-error retry, failure marking, etc.) is unaffected. */ import { isUsageLimitError } from "./usage-limit-detector.js"; +/** + * Matches transient authentication failures caused by credential rotation — + * e.g. a Claude Max OAuth access token expiring mid-run (~8 h lifetime). A + * long-running agent session holds the old token in memory; the very next + * call after the provider refreshes credentials succeeds, so these are worth + * a couple of quick retries before propagating as a task failure. + */ +const TRANSIENT_AUTH_ERROR_RE = + /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired|oauth token does not meet scope/i; + +function isTransientAuthError(message: string | undefined): boolean { + return TRANSIENT_AUTH_ERROR_RE.test(message ?? ""); +} + +/** Transient-auth retry budget — separate from the rate-limit `maxRetries`. */ +const AUTH_MAX_RETRIES = 2; +/** + * Flat delay before a transient-auth retry. A credential refresh completes + * within seconds, so the rate-limit backoff curve (30 s → 2 min) would just + * prolong the outage. + */ +const AUTH_RETRY_DELAY_MS = 5_000; + export interface RateLimitRetryOptions { /** Maximum number of retry attempts before re-throwing (default: 3). */ maxRetries?: number; @@ -45,7 +71,10 @@ export interface RateLimitRetryOptions { * * The wrapper calls `fn()`. If it throws a rate-limit error (detected via * `isUsageLimitError`), it sleeps with exponential backoff and retries up to - * `maxRetries` times. Non-rate-limit errors are re-thrown immediately. + * `maxRetries` times. If it throws a transient authentication error (detected + * via `isTransientAuthError`), it retries up to `AUTH_MAX_RETRIES` times after + * a short flat delay — this budget is separate and does not consume rate-limit + * attempts. All other errors are re-thrown immediately. * * After all retries are exhausted, the **original** error is thrown so the * caller's existing catch block can trigger the global pause via @@ -73,20 +102,39 @@ export async function withRateLimitRetry( } = options; let lastError: Error | undefined; + let authRetries = 0; for (let attempt = 0; attempt <= maxRetries; attempt++) { try { return await fn(); } catch (err: unknown) { const error = err instanceof Error ? err : new Error(String(err)); + const authError = isTransientAuthError(error.message); - // Non-rate-limit errors: re-throw immediately — no retry - if (!isUsageLimitError(error.message)) { + // Non-retryable errors: re-throw immediately — no retry + if (!isUsageLimitError(error.message) && !authError) { throw error; } lastError = error; + if (authError) { + if (authRetries >= AUTH_MAX_RETRIES || signal?.aborted) { + throw lastError; + } + authRetries++; + // Don't consume a rate-limit attempt for an auth retry + attempt--; + + const jitter = AUTH_RETRY_DELAY_MS * 0.1 * (2 * Math.random() - 1); // ±10 % + const delay = Math.max(0, Math.round(AUTH_RETRY_DELAY_MS + jitter)); + + onRetry?.(authRetries, delay, error); + + await sleep(delay, signal); + continue; + } + // All retries exhausted — throw so caller can trigger global pause if (attempt >= maxRetries) { throw lastError;