diff --git a/.changeset/fix-transient-auth-scope-classification.md b/.changeset/fix-transient-auth-scope-classification.md new file mode 100644 index 0000000000..f25da92d1e --- /dev/null +++ b/.changeset/fix-transient-auth-scope-classification.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Retry transient OAuth token-rotation errors so in-flight agent calls survive rotation without failing the task. +category: fix +dev: withRateLimitRetry now retries transient auth errors (authentication_error, invalid credentials, token_expired) on a separate ~5s flat-delay budget that does not consume rate-limit attempts. OAuth scope/permission failures are explicitly excluded (operator must re-authorize) so they surface immediately instead of retrying pointlessly. diff --git a/packages/engine/src/__tests__/rate-limit-retry.test.ts b/packages/engine/src/__tests__/rate-limit-retry.test.ts index 01f86ec81c..1de497f3e6 100644 --- a/packages/engine/src/__tests__/rate-limit-retry.test.ts +++ b/packages/engine/src/__tests__/rate-limit-retry.test.ts @@ -289,7 +289,6 @@ describe("withRateLimitRetry", () => { "Invalid authentication credentials", "token_expired", "token expired", - "OAuth token does not meet scope requirements", ]; for (const msg of patterns) { @@ -305,4 +304,46 @@ describe("withRateLimitRetry", () => { expect(fn).toHaveBeenCalledTimes(2); } }); + + it("does not retry OAuth scope errors even when wrapped in an authentication_error envelope", async () => { + const scopeErrors = [ + "OAuth token does not meet scope requirements", + "insufficient_scope", + '{"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}', + ]; + + for (const msg of scopeErrors) { + const fn = vi.fn().mockRejectedValue(new Error(msg)); + const onRetry = vi.fn(); + + await expect(withRateLimitRetry(fn, { onRetry })).rejects.toThrow(msg); + + // Permanent scope failures must surface immediately — no retries. + expect(fn).toHaveBeenCalledTimes(1); + expect(onRetry).not.toHaveBeenCalled(); + } + }); + + it("cancels transient-auth retry sleep when abort signal fires", async () => { + const authErr = new Error( + '401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + ); + const fn = vi.fn().mockRejectedValue(authErr); + const ac = new AbortController(); + + const promise = withRateLimitRetry(fn, { + maxRetries: 5, + signal: ac.signal, + }); + + // Let the first call fail and enter the auth-retry sleep (~5s). + await vi.advanceTimersByTimeAsync(10); + + // Abort before the 5s auth delay elapses. + ac.abort(new Error("Task paused")); + + await expect(promise).rejects.toThrow("Task paused"); + // Only the initial call — the auth retry never fires. + expect(fn).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/engine/src/rate-limit-retry.ts b/packages/engine/src/rate-limit-retry.ts index 905112f3bd..56e9ac2268 100644 --- a/packages/engine/src/rate-limit-retry.ts +++ b/packages/engine/src/rate-limit-retry.ts @@ -24,6 +24,11 @@ import { isUsageLimitError } from "./usage-limit-detector.js"; +/* +FNXC:EngineAuthRetry 2026-07-05-06:07: +A long-running agent session holds its OAuth access token in memory. Claude Max access tokens rotate mid-run (~8 h lifetime); the in-flight call fails with a 401 authentication_error even though the credentials file has already been refreshed, and the very next call succeeds. Retry these a few times so a token-boundary rotation does not surface as a spurious task-failure alert. This budget is separate from the rate-limit retry budget and must not consume rate-limit attempts. +*/ + /** * Matches transient authentication failures caused by credential rotation — * e.g. a Claude Max OAuth access token expiring mid-run (~8 h lifetime). A @@ -32,10 +37,20 @@ import { isUsageLimitError } from "./usage-limit-detector.js"; * a couple of quick retries before propagating as a task failure. */ const TRANSIENT_AUTH_ERROR_RE = - /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired|oauth token does not meet scope/i; + /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired/i; + +/* +FNXC:EngineAuthRetry 2026-07-05-06:07: +OAuth scope/permission-grant failures are NOT transient — the token is valid but lacks required grants, so the operator must re-authorize the connection. Retrying would repeat the failing call for ~10 s before surfacing the real (operator-actionable) error. This exclusion runs BEFORE the transient match because providers wrap scope errors inside a generic {"type":"authentication_error"} envelope that would otherwise match TRANSIENT_AUTH_ERROR_RE and retry pointlessly. +*/ +const SCOPE_ERROR_RE = + /oauth token does not meet scope|insufficient[_\s-]?scope|invalid[_\s-]?scope/i; function isTransientAuthError(message: string | undefined): boolean { - return TRANSIENT_AUTH_ERROR_RE.test(message ?? ""); + const msg = message ?? ""; + // Permanent scope failures must surface immediately instead of retrying. + if (SCOPE_ERROR_RE.test(msg)) return false; + return TRANSIENT_AUTH_ERROR_RE.test(msg); } /** Transient-auth retry budget — separate from the rate-limit `maxRetries`. */ @@ -118,6 +133,10 @@ export async function withRateLimitRetry( lastError = error; + /* + FNXC:EngineAuthRetry 2026-07-05-06:07: + Transient-auth retries use a separate, smaller budget (AUTH_MAX_RETRIES) at a flat ~5s delay, and decrement `attempt` so they never burn a rate-limit attempt. Credential rotation completes in seconds, so the rate-limit backoff curve (30s -> 2min) would only prolong the outage. An already-aborted signal short-circuits to throw the original auth error without sleeping, matching the rate-limit path. + */ if (authError) { if (authRetries >= AUTH_MAX_RETRIES || signal?.aborted) { throw lastError;