Files
fusion/packages/engine/src/recovery-policy.test.ts
gsxdsm 8c246b036f feat(FN-775): add recoverable-retry with bounded exponential backoff
- Add  state to tasks: persisted in DB, gates scheduler pickup to prevent immediate retry of transient failures
- Introduce shared recovery-policy module with bounded exponential backoff (1s → 60s, max 5 attempts)
- Wire recovery policy into executor, scheduler, and triage so all agents respect the same retry cadence
- Persist retry state (attempt count, next eligible time) in task metadata via store and DB schema
- Add DB migration for new retry columns and update schema tests
- Update README with recovery policy documentation
- Refactor dashboard Header component and styles, consolidate header tests
- Fix session-files route tests to align with updated route signatures
2026-04-03 07:58:45 -07:00

161 lines
5.4 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, it, expect, vi, afterEach } from "vitest";
import {
computeRecoveryDecision,
formatDelay,
MAX_RECOVERY_RETRIES,
BASE_DELAY_MS,
MAX_DELAY_MS,
BACKOFF_MULTIPLIER,
} from "./recovery-policy.js";
describe("computeRecoveryDecision", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("returns shouldRetry=true on first failure (count=0)", () => {
const decision = computeRecoveryDecision({});
expect(decision.shouldRetry).toBe(true);
expect(decision.exhausted).toBe(false);
expect(decision.nextState.recoveryRetryCount).toBe(1);
expect(decision.nextState.nextRecoveryAt).toBeDefined();
expect(decision.delayMs).toBeGreaterThan(0);
});
it("increments recovery count on each attempt", () => {
const d1 = computeRecoveryDecision({ recoveryRetryCount: 0 });
expect(d1.nextState.recoveryRetryCount).toBe(1);
const d2 = computeRecoveryDecision({ recoveryRetryCount: 1 });
expect(d2.nextState.recoveryRetryCount).toBe(2);
const d3 = computeRecoveryDecision({ recoveryRetryCount: 2 });
expect(d3.nextState.recoveryRetryCount).toBe(3);
});
it("exhausts after MAX_RECOVERY_RETRIES attempts", () => {
const decision = computeRecoveryDecision({
recoveryRetryCount: MAX_RECOVERY_RETRIES,
});
expect(decision.shouldRetry).toBe(false);
expect(decision.exhausted).toBe(true);
expect(decision.nextState.recoveryRetryCount).toBeUndefined();
expect(decision.nextState.nextRecoveryAt).toBeUndefined();
expect(decision.delayMs).toBe(0);
});
it("also exhausts when count exceeds max (overflow safety)", () => {
const decision = computeRecoveryDecision({
recoveryRetryCount: 999,
});
expect(decision.shouldRetry).toBe(false);
expect(decision.exhausted).toBe(true);
});
it("uses exponential backoff with increasing delays", () => {
// Use fixed random for deterministic test
vi.spyOn(Math, "random").mockReturnValue(0.5); // No jitter when random=0.5
const d1 = computeRecoveryDecision({});
const d2 = computeRecoveryDecision({ recoveryRetryCount: 1 });
const d3 = computeRecoveryDecision({ recoveryRetryCount: 2 });
// Base: 60s, then 120s, then 240s (capped at 300s)
expect(d1.delayMs).toBe(BASE_DELAY_MS); // 60s × 2^0 = 60s
expect(d2.delayMs).toBe(BASE_DELAY_MS * BACKOFF_MULTIPLIER); // 60s × 2^1 = 120s
expect(d3.delayMs).toBe(BASE_DELAY_MS * BACKOFF_MULTIPLIER ** 2); // 60s × 2^2 = 240s
});
it("caps delay at MAX_DELAY_MS", () => {
vi.spyOn(Math, "random").mockReturnValue(0.5);
// With high retry count, delay should be capped
const decision = computeRecoveryDecision({ recoveryRetryCount: 2 });
expect(decision.delayMs).toBeLessThanOrEqual(MAX_DELAY_MS * 1.1); // Allow for jitter
});
it("applies jitter (±10%) to delays", () => {
// Zero jitter
vi.spyOn(Math, "random").mockReturnValue(0.5);
const noJitter = computeRecoveryDecision({});
// Max positive jitter
vi.spyOn(Math, "random").mockReturnValue(1.0);
const maxJitter = computeRecoveryDecision({});
// Max negative jitter
vi.spyOn(Math, "random").mockReturnValue(0.0);
const minJitter = computeRecoveryDecision({});
// All should be within ±10% of base delay
const base = BASE_DELAY_MS;
expect(noJitter.delayMs).toBe(base);
expect(maxJitter.delayMs).toBeGreaterThan(base);
expect(maxJitter.delayMs).toBeLessThanOrEqual(base * 1.1);
expect(minJitter.delayMs).toBeLessThan(base);
expect(minJitter.delayMs).toBeGreaterThanOrEqual(base * 0.9);
});
it("sets nextRecoveryAt to a future ISO timestamp", () => {
const before = Date.now();
const decision = computeRecoveryDecision({});
const after = Date.now();
const recoveryTime = new Date(decision.nextState.nextRecoveryAt!).getTime();
expect(recoveryTime).toBeGreaterThanOrEqual(before + decision.delayMs - 1);
expect(recoveryTime).toBeLessThanOrEqual(after + decision.delayMs + 1);
});
it("treats undefined recoveryRetryCount as 0", () => {
const decision = computeRecoveryDecision({ recoveryRetryCount: undefined });
expect(decision.shouldRetry).toBe(true);
expect(decision.nextState.recoveryRetryCount).toBe(1);
});
it("clears recovery metadata when exhausted", () => {
const decision = computeRecoveryDecision({
recoveryRetryCount: MAX_RECOVERY_RETRIES,
nextRecoveryAt: new Date().toISOString(),
});
expect(decision.nextState.recoveryRetryCount).toBeUndefined();
expect(decision.nextState.nextRecoveryAt).toBeUndefined();
});
});
describe("formatDelay", () => {
it("formats seconds under 60 as Ns", () => {
expect(formatDelay(5000)).toBe("5s");
expect(formatDelay(30000)).toBe("30s");
expect(formatDelay(59000)).toBe("59s");
});
it("formats exact minutes as Nm", () => {
expect(formatDelay(60000)).toBe("1m");
expect(formatDelay(120000)).toBe("2m");
expect(formatDelay(300000)).toBe("5m");
});
it("formats non-exact minutes as seconds", () => {
expect(formatDelay(90000)).toBe("90s");
expect(formatDelay(150000)).toBe("150s");
});
it("handles zero", () => {
expect(formatDelay(0)).toBe("0s");
});
});
describe("constants", () => {
it("MAX_RECOVERY_RETRIES is 3", () => {
expect(MAX_RECOVERY_RETRIES).toBe(3);
});
it("BASE_DELAY_MS is 60 seconds", () => {
expect(BASE_DELAY_MS).toBe(60_000);
});
it("MAX_DELAY_MS is 300 seconds (5 minutes)", () => {
expect(MAX_DELAY_MS).toBe(300_000);
});
});