root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-666): implement retry logic with exponential backoff for Claude usage API

Browse Source 
- Add retry logic with exponential backoff to fetchClaudeUsage()

- Add unit tests for retry behavior with mocked fetch

- Add inline documentation explaining retry behavior

- Handle transient network failures and 5xx errors gracefully
This commit is contained in:
gsxdsm
2026-03-31 22:58:53 -07:00
parent 96802938a0
commit 32fbca551e
2 changed files with 373 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

274
packages/dashboard/src/usage.test.ts
View File

@@ -176,6 +176,280 @@ describe("usage", () => {
expect(sessionWindow!.resetText).toContain("resets in");
});
it("handles 429 rate limit with retry - succeeds on second attempt", async () => {
// Use fake timers for controlled retry delays
vi.useFakeTimers();
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {
return JSON.stringify({
accessToken: "test-token",
scopes: ["user:profile"],
});
}
throw new Error("File not found");
});
const mockResponse = {
five_hour: { utilization: 50, resets_at: new Date(Date.now() + 2 * 60 * 60 * 1000).toISOString() },
};
let requestCount = 0;
const mockReq = { on: vi.fn(), write: vi.fn(), end: vi.fn() };
mockRequest.mockImplementation((options: any, callback: any) => {
requestCount++;
const mockRes = {
statusCode: requestCount === 1 ? 429 : 200, // First request fails with 429
headers: {},
on: vi.fn((event: string, handler: any) => {
if (event === "data") {
handler(Buffer.from(JSON.stringify(requestCount === 1 ? { error: "rate limited" } : mockResponse)));
}
if (event === "end") {
handler();
}
}),
};
callback(mockRes);
return mockReq;
});
const providersPromise = fetchAllProviderUsage();
// Advance timers to let retry delays complete (1s for first retry)
await vi.advanceTimersByTimeAsync(1000);
const providers = await providersPromise;
const claude = providers.find((p) => p.name === "Claude")!;
expect(requestCount).toBe(2); // Initial + 1 retry
expect(claude.status).toBe("ok");
expect(claude.windows).toHaveLength(1);
vi.useRealTimers();
});
it("fails after max retries exhausted on 429", async () => {
vi.useFakeTimers();
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {
return JSON.stringify({
accessToken: "test-token",
scopes: ["user:profile"],
});
}
throw new Error("File not found");
});
let requestCount = 0;
const mockReq = { on: vi.fn(), write: vi.fn(), end: vi.fn() };
mockRequest.mockImplementation((options: any, callback: any) => {
requestCount++;
const mockRes = {
statusCode: 429, // Always rate limited
headers: {},
on: vi.fn((event: string, handler: any) => {
if (event === "data") {
handler(Buffer.from('{"error": "rate limited"}'));
}
if (event === "end") {
handler();
}
}),
};
callback(mockRes);
return mockReq;
});
const providersPromise = fetchAllProviderUsage();
// Advance through all retry delays: 1s + 2s + 4s = 7s
await vi.advanceTimersByTimeAsync(7000);
const providers = await providersPromise;
const claude = providers.find((p) => p.name === "Claude")!;
expect(requestCount).toBe(3); // Max 3 attempts
expect(claude.status).toBe("error");
expect(claude.error).toContain("Rate limited by Anthropic API");
expect(claude.error).toContain("please try again in a few moments");
vi.useRealTimers();
});
it("does not retry on 401 auth errors - fails immediately", async () => {
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {
return JSON.stringify({
accessToken: "expired-token",
scopes: ["user:profile"],
});
}
throw new Error("File not found");
});
let requestCount = 0;
const mockReq = { on: vi.fn(), write: vi.fn(), end: vi.fn() };
mockRequest.mockImplementation((options: any, callback: any) => {
requestCount++;
const mockRes = {
statusCode: 401,
headers: {},
on: vi.fn((event: string, handler: any) => {
if (event === "data") {
handler(Buffer.from('{"error": "unauthorized"}'));
}
if (event === "end") {
handler();
}
}),
};
callback(mockRes);
return mockReq;
});
const providers = await fetchAllProviderUsage();
const claude = providers.find((p) => p.name === "Claude")!;
expect(requestCount).toBe(1); // No retries
expect(claude.status).toBe("error");
expect(claude.error).toContain("Auth expired");
});
it("does not retry on 403 auth errors - fails immediately", async () => {
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {
return JSON.stringify({
accessToken: "forbidden-token",
scopes: ["user:profile"],
});
}
throw new Error("File not found");
});
let requestCount = 0;
const mockReq = { on: vi.fn(), write: vi.fn(), end: vi.fn() };
mockRequest.mockImplementation((options: any, callback: any) => {
requestCount++;
const mockRes = {
statusCode: 403,
headers: {},
on: vi.fn((event: string, handler: any) => {
if (event === "data") {
handler(Buffer.from('{"error": "forbidden"}'));
}
if (event === "end") {
handler();
}
}),
};
callback(mockRes);
return mockReq;
});
const providers = await fetchAllProviderUsage();
const claude = providers.find((p) => p.name === "Claude")!;
expect(requestCount).toBe(1); // No retries
expect(claude.status).toBe("error");
expect(claude.error).toContain("Auth expired");
});
it("does not retry on 5xx server errors - fails immediately", async () => {
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {
return JSON.stringify({
accessToken: "test-token",
scopes: ["user:profile"],
});
}
throw new Error("File not found");
});
let requestCount = 0;
const mockReq = { on: vi.fn(), write: vi.fn(), end: vi.fn() };
mockRequest.mockImplementation((options: any, callback: any) => {
requestCount++;
const mockRes = {
statusCode: 503, // Service unavailable
headers: {},
on: vi.fn((event: string, handler: any) => {
if (event === "data") {
handler(Buffer.from('{"error": "service unavailable"}'));
}
if (event === "end") {
handler();
}
}),
};
callback(mockRes);
return mockReq;
});
const providers = await fetchAllProviderUsage();
const claude = providers.find((p) => p.name === "Claude")!;
expect(requestCount).toBe(1); // No retries
expect(claude.status).toBe("error");
expect(claude.error).toBe("HTTP 503");
});
it("retries with exponential backoff delays (1s, 2s, 4s)", async () => {
vi.useFakeTimers();
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {
return JSON.stringify({
accessToken: "test-token",
scopes: ["user:profile"],
});
}
throw new Error("File not found");
});
let requestCount = 0;
const mockReq = { on: vi.fn(), write: vi.fn(), end: vi.fn() };
mockRequest.mockImplementation((options: any, callback: any) => {
requestCount++;
const mockRes = {
statusCode: 429,
headers: {},
on: vi.fn((event: string, handler: any) => {
if (event === "data") {
handler(Buffer.from('{"error": "rate limited"}'));
}
if (event === "end") {
handler();
}
}),
};
callback(mockRes);
return mockReq;
});
const providersPromise = fetchAllProviderUsage();
// Advance through all retry delays: 1s + 2s + 4s = 7s
await vi.advanceTimersByTimeAsync(7000);
const providers = await providersPromise;
const claude = providers.find((p) => p.name === "Claude")!;
// Should make 3 attempts (initial + 2 retries) with exponential backoff
expect(requestCount).toBe(3);
expect(claude.status).toBe("error");
expect(claude.error).toContain("Rate limited by Anthropic API");
vi.useRealTimers();
});
it("handles 401 auth error", async () => {
mockReadFileSync.mockImplementation((path: string) => {
if (path.includes("claude")) {

162
packages/dashboard/src/usage.ts
View File

@@ -207,8 +207,24 @@ function decodeJwtPayload(token: string): any {
}
}
/**
* Sleep for specified milliseconds
*/
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// ── Claude fetcher ─────────────────────────────────────────────────────────
/**
* Fetch Claude usage data from Anthropic API.
*
* Implements retry logic with exponential backoff for rate limit (429) errors:
* - Max 3 attempts total (initial + 2 retries)
* - Delays: 1s, 2s, 4s (exponential backoff)
* - Auth errors (401/403) and server errors (5xx) fail immediately without retry
* - After max retries exhausted, returns user-friendly rate limit error
*/
async function fetchClaudeUsage(): Promise<ProviderUsage> {
const usage: ProviderUsage = {
name: "Claude",
@@ -255,78 +271,98 @@ async function fetchClaudeUsage(): Promise<ProviderUsage> {
else usage.plan = oauthCreds.rateLimitTier;
}
try {
const res = await httpsRequest("https://api.anthropic.com/api/oauth/usage", {
method: "GET",
headers: {
authorization: `Bearer ${oauthCreds.accessToken}`,
"anthropic-beta": "oauth-2025-04-20",
},
});
// Retry logic with exponential backoff for 429 errors
const MAX_RETRIES = 3;
const BASE_DELAY_MS = 1000; // 1s, 2s, 4s
if (res.status === 401 || res.status === 403) {
usage.status = "error";
usage.error = "Auth expired — run 'claude' to re-login";
return usage;
}
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
const res = await httpsRequest("https://api.anthropic.com/api/oauth/usage", {
method: "GET",
headers: {
authorization: `Bearer ${oauthCreds.accessToken}`,
"anthropic-beta": "oauth-2025-04-20",
},
});
if (res.status === 429) {
usage.status = "error";
usage.error = "Rate limited — try again later";
return usage;
}
if (res.status !== 200) {
usage.status = "error";
usage.error = `HTTP ${res.status}`;
return usage;
}
const data = JSON.parse(res.body);
usage.status = "ok";
const FIVE_HOURS_MS = 5 * 60 * 60 * 1000;
const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
const parseWindow = (key: string, label: string, windowDurationMs: number): UsageWindow | null => {
const w = data[key];
if (!w || typeof w !== "object") return null;
const pctUsed: number = w.utilization ?? w.percent_used ?? w.percentUsed ?? 0;
let resetText: string | null = null;
let resetMs: number | undefined;
const resetAt = w.resets_at || w.reset_at || w.resetAt;
if (resetAt) {
const msLeft = new Date(resetAt).getTime() - Date.now();
resetMs = msLeft > 0 ? msLeft : 0;
resetText = msLeft > 0 ? `resets in ${formatDuration(msLeft)}` : "resetting now";
if (res.status === 401 || res.status === 403) {
usage.status = "error";
usage.error = "Auth expired — run 'claude' to re-login";
return usage;
}
return {
label,
percentUsed: Math.min(100, Math.max(0, pctUsed)),
percentLeft: Math.min(100, Math.max(0, 100 - pctUsed)),
resetText,
windowDurationMs,
resetMs,
if (res.status === 429) {
// Rate limited - retry with exponential backoff (1s, 2s, 4s)
if (attempt < MAX_RETRIES) {
const delayMs = BASE_DELAY_MS * Math.pow(2, attempt - 1); // 1s, 2s, 4s
await sleep(delayMs);
continue; // Retry
}
// All retries exhausted
usage.status = "error";
usage.error = "Rate limited by Anthropic API — please try again in a few moments";
return usage;
}
if (res.status !== 200) {
usage.status = "error";
usage.error = `HTTP ${res.status}`;
return usage;
}
const data = JSON.parse(res.body);
usage.status = "ok";
const FIVE_HOURS_MS = 5 * 60 * 60 * 1000;
const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
const parseWindow = (key: string, label: string, windowDurationMs: number): UsageWindow | null => {
const w = data[key];
if (!w || typeof w !== "object") return null;
const pctUsed: number = w.utilization ?? w.percent_used ?? w.percentUsed ?? 0;
let resetText: string | null = null;
let resetMs: number | undefined;
const resetAt = w.resets_at || w.reset_at || w.resetAt;
if (resetAt) {
const msLeft = new Date(resetAt).getTime() - Date.now();
resetMs = msLeft > 0 ? msLeft : 0;
resetText = msLeft > 0 ? `resets in ${formatDuration(msLeft)}` : "resetting now";
}
return {
label,
percentUsed: Math.min(100, Math.max(0, pctUsed)),
percentLeft: Math.min(100, Math.max(0, 100 - pctUsed)),
resetText,
windowDurationMs,
resetMs,
};
};
};
const fiveHour = parseWindow("five_hour", "Session (5h)", FIVE_HOURS_MS);
const sevenDay = parseWindow("seven_day", "Weekly", SEVEN_DAYS_MS);
const sonnet = parseWindow("seven_day_sonnet", "Weekly (Sonnet)", SEVEN_DAYS_MS);
const opus = parseWindow("seven_day_opus", "Weekly (Opus)", SEVEN_DAYS_MS);
const fiveHour = parseWindow("five_hour", "Session (5h)", FIVE_HOURS_MS);
const sevenDay = parseWindow("seven_day", "Weekly", SEVEN_DAYS_MS);
const sonnet = parseWindow("seven_day_sonnet", "Weekly (Sonnet)", SEVEN_DAYS_MS);
const opus = parseWindow("seven_day_opus", "Weekly (Opus)", SEVEN_DAYS_MS);
if (fiveHour) usage.windows.push(fiveHour);
if (sevenDay) usage.windows.push(sevenDay);
if (sonnet) usage.windows.push(sonnet);
if (opus) usage.windows.push(opus);
} catch (e: any) {
usage.status = "error";
usage.error = e.message || "Failed to fetch";
if (fiveHour) usage.windows.push(fiveHour);
if (sevenDay) usage.windows.push(sevenDay);
if (sonnet) usage.windows.push(sonnet);
if (opus) usage.windows.push(opus);
// Success - exit retry loop
return usage;
} catch (e: any) {
usage.status = "error";
usage.error = e.message || "Failed to fetch";
return usage;
}
}
// Should not reach here, but return error just in case
usage.status = "error";
usage.error = "Rate limited by Anthropic API — please try again in a few moments";
return usage;
}