feat(FN-775): add recoverable-retry with bounded exponential backoff

- Add  state to tasks: persisted in DB, gates scheduler pickup to prevent immediate retry of transient failures
- Introduce shared recovery-policy module with bounded exponential backoff (1s → 60s, max 5 attempts)
- Wire recovery policy into executor, scheduler, and triage so all agents respect the same retry cadence
- Persist retry state (attempt count, next eligible time) in task metadata via store and DB schema
- Add DB migration for new retry columns and update schema tests
- Update README with recovery policy documentation
- Refactor dashboard Header component and styles, consolidate header tests
- Fix session-files route tests to align with updated route signatures
This commit is contained in:
gsxdsm
2026-04-03 07:58:45 -07:00
parent ee8f19fc93
commit 51855f43d9
15 changed files with 961 additions and 30 deletions

View File

@@ -166,13 +166,14 @@ async function migrateTasks(kbDir: string, db: Database): Promise<void> {
id, title, description, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, baseCommitSha, modelPresetId,
modelProvider, modelId, validatorModelProvider, validatorModelId,
mergeRetries, error, summary, thinkingLevel, createdAt, updatedAt,
mergeRetries, recoveryRetryCount, nextRecoveryAt,
error, summary, thinkingLevel, createdAt, updatedAt,
columnMovedAt, dependencies, steps, log, attachments, steeringComments,
comments, workflowStepResults, prInfo, issueInfo, mergeDetails,
breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, sliceId
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`);
@@ -211,6 +212,8 @@ async function migrateTasks(kbDir: string, db: Database): Promise<void> {
task.validatorModelProvider ?? null,
task.validatorModelId ?? null,
task.mergeRetries ?? null,
task.recoveryRetryCount ?? null,
task.nextRecoveryAt ?? null,
task.error ?? null,
task.summary ?? null,
task.thinkingLevel ?? null,

View File

@@ -86,7 +86,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
});
it("seeds lastModified", () => {
@@ -109,7 +109,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
});
it("does not overwrite existing config on re-init", () => {
@@ -704,7 +704,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 5 (includes v1→v2, v2→v3, v3→v4, and v4→v5 migrations)
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -729,11 +729,11 @@ describe("schema migrations", () => {
const db = new Database(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
db.close();
});
@@ -828,7 +828,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 5
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1038,7 +1038,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();

View File

@@ -59,7 +59,7 @@ export function fromJson<T>(json: string | null | undefined): T | undefined {
// ── Schema Definition ────────────────────────────────────────────────
const SCHEMA_VERSION = 6;
const SCHEMA_VERSION = 7;
function normalizeTaskComments(
steeringComments: SteeringComment[] | undefined,
@@ -142,6 +142,8 @@ CREATE TABLE IF NOT EXISTS tasks (
validatorModelProvider TEXT,
validatorModelId TEXT,
mergeRetries INTEGER,
recoveryRetryCount INTEGER,
nextRecoveryAt TEXT,
error TEXT,
summary TEXT,
thinkingLevel TEXT,
@@ -409,8 +411,15 @@ export class Database {
});
}
if (version < 7) {
this.applyMigration(7, () => {
this.addColumnIfMissing("tasks", "recoveryRetryCount", "INTEGER");
this.addColumnIfMissing("tasks", "nextRecoveryAt", "TEXT");
});
}
// Future migrations go here:
// if (version < 7) { this.applyMigration(7, () => { ... }); }
// if (version < 8) { this.applyMigration(8, () => { ... }); }
}
/**

View File

@@ -4630,4 +4630,112 @@ Task with acceptance criteria
expect(movedEvents[0].to).toBe("todo");
});
});
describe("recovery metadata (recoveryRetryCount / nextRecoveryAt)", () => {
async function createTestTask(overrides: Partial<import("./types.js").TaskCreateInput> = {}) {
return store.createTask({ description: "recovery test task", ...overrides });
}
it("new tasks have no recovery metadata (defaults to undefined)", async () => {
const task = await createTestTask();
expect(task.recoveryRetryCount).toBeUndefined();
expect(task.nextRecoveryAt).toBeUndefined();
});
it("updateTask can set and clear recoveryRetryCount and nextRecoveryAt", async () => {
const task = await createTestTask();
const futureTime = new Date(Date.now() + 60_000).toISOString();
// Set
const updated = await store.updateTask(task.id, {
recoveryRetryCount: 2,
nextRecoveryAt: futureTime,
});
expect(updated.recoveryRetryCount).toBe(2);
expect(updated.nextRecoveryAt).toBe(futureTime);
// Re-read to verify persistence
const reread = await store.getTask(task.id);
expect(reread.recoveryRetryCount).toBe(2);
expect(reread.nextRecoveryAt).toBe(futureTime);
// Clear
const cleared = await store.updateTask(task.id, {
recoveryRetryCount: null,
nextRecoveryAt: null,
});
expect(cleared.recoveryRetryCount).toBeUndefined();
expect(cleared.nextRecoveryAt).toBeUndefined();
});
it("moveTask to in-review clears recovery metadata", async () => {
const task = await createTestTask();
await store.moveTask(task.id, "todo");
await store.updateTask(task.id, {
recoveryRetryCount: 3,
nextRecoveryAt: new Date().toISOString(),
});
await store.moveTask(task.id, "in-progress");
const moved = await store.moveTask(task.id, "in-review");
expect(moved.recoveryRetryCount).toBeUndefined();
expect(moved.nextRecoveryAt).toBeUndefined();
});
it("moveTask to done clears recovery metadata", async () => {
const task = await createTestTask();
await store.moveTask(task.id, "todo");
await store.updateTask(task.id, {
recoveryRetryCount: 1,
nextRecoveryAt: new Date().toISOString(),
});
await store.moveTask(task.id, "in-progress");
await store.moveTask(task.id, "in-review");
const done = await store.moveTask(task.id, "done");
expect(done.recoveryRetryCount).toBeUndefined();
expect(done.nextRecoveryAt).toBeUndefined();
});
it("moveTask from in-progress to todo preserves recovery metadata", async () => {
const task = await createTestTask();
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
const futureTime = new Date(Date.now() + 60_000).toISOString();
await store.updateTask(task.id, {
recoveryRetryCount: 2,
nextRecoveryAt: futureTime,
});
const moved = await store.moveTask(task.id, "todo");
expect(moved.recoveryRetryCount).toBe(2);
expect(moved.nextRecoveryAt).toBe(futureTime);
});
it("recovery metadata persists across store re-initialization", async () => {
const task = await createTestTask();
const futureTime = new Date(Date.now() + 60_000).toISOString();
await store.updateTask(task.id, {
recoveryRetryCount: 5,
nextRecoveryAt: futureTime,
});
// Re-create store to simulate restart
store.stopWatching();
const store2 = new TaskStore(rootDir, globalDir);
await store2.init();
const reloaded = await store2.getTask(task.id);
expect(reloaded.recoveryRetryCount).toBe(5);
expect(reloaded.nextRecoveryAt).toBe(futureTime);
store2.stopWatching();
});
it("schema migration: existing rows default to NULL (undefined) for recovery fields", async () => {
// Tasks created before the migration should have undefined recovery fields
const task = await createTestTask();
const detail = await store.getTask(task.id);
expect(detail.recoveryRetryCount).toBeUndefined();
expect(detail.nextRecoveryAt).toBeUndefined();
});
});
});

View File

@@ -178,6 +178,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
validatorModelProvider: row.validatorModelProvider || undefined,
validatorModelId: row.validatorModelId || undefined,
mergeRetries: row.mergeRetries ?? undefined,
recoveryRetryCount: row.recoveryRetryCount ?? undefined,
nextRecoveryAt: row.nextRecoveryAt || undefined,
error: row.error || undefined,
summary: row.summary || undefined,
thinkingLevel: row.thinkingLevel || undefined,
@@ -225,14 +227,15 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
INSERT OR REPLACE INTO tasks (
id, title, description, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, branch, baseCommitSha, modelPresetId, modelProvider,
modelId, validatorModelProvider, validatorModelId, mergeRetries, error,
modelId, validatorModelProvider, validatorModelId, mergeRetries,
recoveryRetryCount, nextRecoveryAt, error,
summary, thinkingLevel, createdAt, updatedAt, columnMovedAt,
dependencies, steps, log, attachments, steeringComments,
comments, workflowStepResults, prInfo, issueInfo, mergeDetails,
breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, missionId, sliceId
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`).run(
task.id,
@@ -255,6 +258,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.validatorModelProvider ?? null,
task.validatorModelId ?? null,
task.mergeRetries ?? null,
task.recoveryRetryCount ?? null,
task.nextRecoveryAt ?? null,
task.error ?? null,
task.summary ?? null,
task.thinkingLevel ?? null,
@@ -930,10 +935,15 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.error = undefined;
task.worktree = undefined;
task.blockedBy = undefined;
task.recoveryRetryCount = undefined;
task.nextRecoveryAt = undefined;
}
// Clear transient fields when moving from in-progress to reset columns (todo/triage)
// This ensures failed tasks don't show failed status after being moved for retry
// This ensures failed tasks don't show failed status after being moved for retry.
// Note: recovery metadata (recoveryRetryCount, nextRecoveryAt) is intentionally
// preserved here — the recovery-policy module manages those fields. They are
// only cleared on terminal transitions (in-review, done, archived).
if (fromColumn === "in-progress" && (toColumn === "todo" || toColumn === "triage")) {
task.status = undefined;
task.error = undefined;
@@ -941,6 +951,12 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.blockedBy = undefined;
}
// Clear recovery metadata when task reaches in-review (successful completion)
if (toColumn === "in-review") {
task.recoveryRetryCount = undefined;
task.nextRecoveryAt = undefined;
}
// Clear workflow step results when moving from in-review back to todo or in-progress
// This ensures fresh workflow step runs on retry
if (fromColumn === "in-review" && (toColumn === "todo" || toColumn === "in-progress")) {
@@ -959,7 +975,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
async updateTask(
id: string,
updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; recoveryRetryCount?: number | null; nextRecoveryAt?: string | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
): Promise<Task> {
return this.withTaskLock(id, async () => {
// Validate that task doesn't depend on itself
@@ -1014,6 +1030,16 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
if (updates.size !== undefined) task.size = updates.size;
if (updates.reviewLevel !== undefined) task.reviewLevel = updates.reviewLevel;
if (updates.mergeRetries !== undefined) task.mergeRetries = updates.mergeRetries;
if (updates.recoveryRetryCount === null) {
task.recoveryRetryCount = undefined;
} else if (updates.recoveryRetryCount !== undefined) {
task.recoveryRetryCount = updates.recoveryRetryCount;
}
if (updates.nextRecoveryAt === null) {
task.nextRecoveryAt = undefined;
} else if (updates.nextRecoveryAt !== undefined) {
task.nextRecoveryAt = updates.nextRecoveryAt;
}
if (updates.modelProvider === null) {
task.modelProvider = undefined;
} else if (updates.modelProvider !== undefined) {

View File

@@ -475,6 +475,15 @@ export interface Task {
workflowStepResults?: WorkflowStepResult[];
/** Number of merge retry attempts made for this task (auto-merge conflict recovery) */
mergeRetries?: number;
/** Number of bounded recovery retry attempts for transient executor/triage failures.
* Distinct from `mergeRetries` (merge-conflict-specific). Incremented by the
* recovery-policy module on each recoverable failure; cleared when work restarts
* cleanly or reaches a terminal column (in-review, done, archived). */
recoveryRetryCount?: number;
/** ISO-8601 timestamp indicating when the task becomes eligible for the next
* recovery retry. Scheduler and triage processor skip tasks whose
* `nextRecoveryAt` is still in the future. Cleared alongside `recoveryRetryCount`. */
nextRecoveryAt?: string;
/** Thinking level for AI agent sessions — controls reasoning effort (off/minimal/low/medium/high) */
thinkingLevel?: ThinkingLevel;
/** Error message from the last failure, if the task failed during execution */
@@ -1008,6 +1017,8 @@ export interface ArchivedTaskEntry {
/** Slice ID this task is linked to */
sliceId?: string;
mergeRetries?: number;
recoveryRetryCount?: number;
nextRecoveryAt?: string;
error?: string;
}

View File

@@ -3551,7 +3551,12 @@ describe("TaskExecutor usage limit detection", () => {
});
expect(onUsageLimitHitSpy).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith("FN-001", "Transient error (will retry): connection refused");
// Recovery policy: first transient error retry 1/3 with backoff
expect(store.logEntry).toHaveBeenCalledWith("FN-001", expect.stringContaining("Transient error (retry 1/3"));
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
recoveryRetryCount: 1,
nextRecoveryAt: expect.any(String),
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
expect(store.updateTask).not.toHaveBeenCalledWith(
"FN-001",
@@ -3663,6 +3668,190 @@ describe("TaskExecutor usage limit detection", () => {
});
});
describe("TaskExecutor bounded recovery retries", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("increments recoveryRetryCount on successive transient failures", async () => {
const store = createMockStore();
const onError = vi.fn();
mockedCreateHaiAgent.mockRejectedValue(new Error("upstream connect error"));
const executor = new TaskExecutor(store, "/tmp/test", { onError });
// First failure: count goes from undefined to 1
await executor.execute({
id: "FN-001",
title: "Test",
description: "Test",
column: "in-progress",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
recoveryRetryCount: 1,
nextRecoveryAt: expect.any(String),
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
expect(onError).not.toHaveBeenCalled();
// Second failure: count goes from 1 to 2
vi.clearAllMocks();
mockedCreateHaiAgent.mockRejectedValue(new Error("upstream connect error"));
await executor.execute({
id: "FN-001",
title: "Test",
description: "Test",
column: "in-progress",
recoveryRetryCount: 1,
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
recoveryRetryCount: 2,
}));
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
expect(onError).not.toHaveBeenCalled();
});
it("escalates to failure when recovery retries are exhausted", async () => {
const store = createMockStore();
const onError = vi.fn();
mockedCreateHaiAgent.mockRejectedValue(new Error("socket hang up"));
const executor = new TaskExecutor(store, "/tmp/test", { onError });
// Task already has 3 retries (max) — next failure should escalate
await executor.execute({
id: "FN-001",
title: "Test",
description: "Test",
column: "in-progress",
recoveryRetryCount: 3,
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
expect(store.updateTask).toHaveBeenCalledWith("FN-001", expect.objectContaining({
status: "failed",
error: "socket hang up",
recoveryRetryCount: null,
nextRecoveryAt: null,
}));
expect(store.moveTask).not.toHaveBeenCalledWith("FN-001", "todo");
expect(onError).toHaveBeenCalled();
});
it("does NOT consume retry budget for paused tasks", async () => {
const store = createMockStore();
const executor = new TaskExecutor(store, "/tmp/test", {});
// Simulate a paused abort — the executor checks pausedAborted set
const task = {
id: "FN-001",
title: "Test",
description: "Test",
column: "in-progress" as const,
recoveryRetryCount: 1,
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
};
// Simulate: task gets paused mid-execution → abort error
mockedCreateHaiAgent.mockRejectedValue(new Error("Aborted"));
(executor as any).pausedAborted.add("FN-001");
await executor.execute(task);
// Should NOT update recoveryRetryCount
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", expect.objectContaining({
recoveryRetryCount: expect.any(Number),
}));
});
it("does NOT consume retry budget for stuck-task-detector kills", async () => {
const store = createMockStore();
const executor = new TaskExecutor(store, "/tmp/test", {});
mockedCreateHaiAgent.mockRejectedValue(new Error("Aborted"));
(executor as any).stuckAborted.add("FN-001");
await executor.execute({
id: "FN-001",
title: "Test",
description: "Test",
column: "in-progress",
recoveryRetryCount: 2,
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
// Should NOT update recoveryRetryCount
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", expect.objectContaining({
recoveryRetryCount: expect.any(Number),
}));
});
it("clears recovery metadata after successful run completes", async () => {
const store = createMockStore();
// Mock successful agent session
const mockSession = {
prompt: vi.fn().mockResolvedValue(undefined),
dispose: vi.fn(),
state: { error: undefined },
};
mockedCreateHaiAgent.mockResolvedValue({ session: mockSession } as any);
const executor = new TaskExecutor(store, "/tmp/test", {});
await executor.execute({
id: "FN-001",
title: "Test",
description: "Test",
column: "in-progress",
recoveryRetryCount: 2,
nextRecoveryAt: new Date().toISOString(),
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
// moveTask to in-review clears recovery metadata (via store's column transition logic)
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-review");
});
});
describe("Per-task model overrides", () => {
beforeEach(() => {
vi.clearAllMocks();

View File

@@ -15,6 +15,7 @@ import { executorLog, reviewerLog } from "./logger.js";
import { isUsageLimitError, checkSessionError, type UsageLimitPauser } from "./usage-limit-detector.js";
import { isTransientError } from "./transient-error-detector.js";
import { withRateLimitRetry } from "./rate-limit-retry.js";
import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js";
import type { StuckTaskDetector } from "./stuck-task-detector.js";
// Re-export for backward compatibility (tests import from executor.ts)
@@ -723,10 +724,35 @@ export class TaskExecutor {
if (this.options.usageLimitPauser && isUsageLimitError(err.message)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, err.message);
} else if (isTransientError(err.message)) {
// Transient network/infrastructure error — retry instead of failing
executorLog.warn(`${task.id} transient error — moving to todo for retry: ${err.message}`);
await this.store.logEntry(task.id, `Transient error (will retry): ${err.message}`);
await this.store.moveTask(task.id, "todo");
// Transient network/infrastructure error — use bounded recovery policy
const decision = computeRecoveryDecision({
recoveryRetryCount: task.recoveryRetryCount,
nextRecoveryAt: task.nextRecoveryAt,
});
if (decision.shouldRetry) {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
executorLog.warn(`${task.id} transient error — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${err.message}`);
await this.store.logEntry(task.id, `Transient error (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${err.message}`);
await this.store.updateTask(task.id, {
recoveryRetryCount: decision.nextState.recoveryRetryCount,
nextRecoveryAt: decision.nextState.nextRecoveryAt,
});
await this.store.moveTask(task.id, "todo");
return;
}
// Recovery budget exhausted — escalate to real failure
executorLog.error(`${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${err.message}`);
await this.store.logEntry(task.id, `Transient error retries exhausted after ${MAX_RECOVERY_RETRIES} attempts: ${err.message}`);
await this.store.updateTask(task.id, {
status: "failed",
error: err.message,
recoveryRetryCount: null,
nextRecoveryAt: null,
});
this.options.onError?.(task, err);
return;
}
executorLog.error(`${task.id} execution failed:`, err.message);

View File

@@ -0,0 +1,160 @@
import { describe, it, expect, vi, afterEach } from "vitest";
import {
computeRecoveryDecision,
formatDelay,
MAX_RECOVERY_RETRIES,
BASE_DELAY_MS,
MAX_DELAY_MS,
BACKOFF_MULTIPLIER,
} from "./recovery-policy.js";
describe("computeRecoveryDecision", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("returns shouldRetry=true on first failure (count=0)", () => {
const decision = computeRecoveryDecision({});
expect(decision.shouldRetry).toBe(true);
expect(decision.exhausted).toBe(false);
expect(decision.nextState.recoveryRetryCount).toBe(1);
expect(decision.nextState.nextRecoveryAt).toBeDefined();
expect(decision.delayMs).toBeGreaterThan(0);
});
it("increments recovery count on each attempt", () => {
const d1 = computeRecoveryDecision({ recoveryRetryCount: 0 });
expect(d1.nextState.recoveryRetryCount).toBe(1);
const d2 = computeRecoveryDecision({ recoveryRetryCount: 1 });
expect(d2.nextState.recoveryRetryCount).toBe(2);
const d3 = computeRecoveryDecision({ recoveryRetryCount: 2 });
expect(d3.nextState.recoveryRetryCount).toBe(3);
});
it("exhausts after MAX_RECOVERY_RETRIES attempts", () => {
const decision = computeRecoveryDecision({
recoveryRetryCount: MAX_RECOVERY_RETRIES,
});
expect(decision.shouldRetry).toBe(false);
expect(decision.exhausted).toBe(true);
expect(decision.nextState.recoveryRetryCount).toBeUndefined();
expect(decision.nextState.nextRecoveryAt).toBeUndefined();
expect(decision.delayMs).toBe(0);
});
it("also exhausts when count exceeds max (overflow safety)", () => {
const decision = computeRecoveryDecision({
recoveryRetryCount: 999,
});
expect(decision.shouldRetry).toBe(false);
expect(decision.exhausted).toBe(true);
});
it("uses exponential backoff with increasing delays", () => {
// Use fixed random for deterministic test
vi.spyOn(Math, "random").mockReturnValue(0.5); // No jitter when random=0.5
const d1 = computeRecoveryDecision({});
const d2 = computeRecoveryDecision({ recoveryRetryCount: 1 });
const d3 = computeRecoveryDecision({ recoveryRetryCount: 2 });
// Base: 60s, then 120s, then 240s (capped at 300s)
expect(d1.delayMs).toBe(BASE_DELAY_MS); // 60s × 2^0 = 60s
expect(d2.delayMs).toBe(BASE_DELAY_MS * BACKOFF_MULTIPLIER); // 60s × 2^1 = 120s
expect(d3.delayMs).toBe(BASE_DELAY_MS * BACKOFF_MULTIPLIER ** 2); // 60s × 2^2 = 240s
});
it("caps delay at MAX_DELAY_MS", () => {
vi.spyOn(Math, "random").mockReturnValue(0.5);
// With high retry count, delay should be capped
const decision = computeRecoveryDecision({ recoveryRetryCount: 2 });
expect(decision.delayMs).toBeLessThanOrEqual(MAX_DELAY_MS * 1.1); // Allow for jitter
});
it("applies jitter (±10%) to delays", () => {
// Zero jitter
vi.spyOn(Math, "random").mockReturnValue(0.5);
const noJitter = computeRecoveryDecision({});
// Max positive jitter
vi.spyOn(Math, "random").mockReturnValue(1.0);
const maxJitter = computeRecoveryDecision({});
// Max negative jitter
vi.spyOn(Math, "random").mockReturnValue(0.0);
const minJitter = computeRecoveryDecision({});
// All should be within ±10% of base delay
const base = BASE_DELAY_MS;
expect(noJitter.delayMs).toBe(base);
expect(maxJitter.delayMs).toBeGreaterThan(base);
expect(maxJitter.delayMs).toBeLessThanOrEqual(base * 1.1);
expect(minJitter.delayMs).toBeLessThan(base);
expect(minJitter.delayMs).toBeGreaterThanOrEqual(base * 0.9);
});
it("sets nextRecoveryAt to a future ISO timestamp", () => {
const before = Date.now();
const decision = computeRecoveryDecision({});
const after = Date.now();
const recoveryTime = new Date(decision.nextState.nextRecoveryAt!).getTime();
expect(recoveryTime).toBeGreaterThanOrEqual(before + decision.delayMs - 1);
expect(recoveryTime).toBeLessThanOrEqual(after + decision.delayMs + 1);
});
it("treats undefined recoveryRetryCount as 0", () => {
const decision = computeRecoveryDecision({ recoveryRetryCount: undefined });
expect(decision.shouldRetry).toBe(true);
expect(decision.nextState.recoveryRetryCount).toBe(1);
});
it("clears recovery metadata when exhausted", () => {
const decision = computeRecoveryDecision({
recoveryRetryCount: MAX_RECOVERY_RETRIES,
nextRecoveryAt: new Date().toISOString(),
});
expect(decision.nextState.recoveryRetryCount).toBeUndefined();
expect(decision.nextState.nextRecoveryAt).toBeUndefined();
});
});
describe("formatDelay", () => {
it("formats seconds under 60 as Ns", () => {
expect(formatDelay(5000)).toBe("5s");
expect(formatDelay(30000)).toBe("30s");
expect(formatDelay(59000)).toBe("59s");
});
it("formats exact minutes as Nm", () => {
expect(formatDelay(60000)).toBe("1m");
expect(formatDelay(120000)).toBe("2m");
expect(formatDelay(300000)).toBe("5m");
});
it("formats non-exact minutes as seconds", () => {
expect(formatDelay(90000)).toBe("90s");
expect(formatDelay(150000)).toBe("150s");
});
it("handles zero", () => {
expect(formatDelay(0)).toBe("0s");
});
});
describe("constants", () => {
it("MAX_RECOVERY_RETRIES is 3", () => {
expect(MAX_RECOVERY_RETRIES).toBe(3);
});
it("BASE_DELAY_MS is 60 seconds", () => {
expect(BASE_DELAY_MS).toBe(60_000);
});
it("MAX_DELAY_MS is 300 seconds (5 minutes)", () => {
expect(MAX_DELAY_MS).toBe(300_000);
});
});

View File

@@ -0,0 +1,125 @@
/**
* Recovery Policy — bounded exponential-backoff retry for recoverable executor/triage failures.
*
* This module provides a **pure decision function** that computes whether a transient
* failure should be retried, and if so, what the updated recovery state should be.
*
* **Design boundary:**
* - `recovery-policy.ts` handles **inter-poll** recoverable retries — tasks moved back
* to todo/triage with backoff, gated by `nextRecoveryAt` in the scheduler/triage poller.
* - `withRateLimitRetry()` in `rate-limit-retry.ts` handles **intra-session** rate-limit
* retries — immediate retry within the same agent session with exponential backoff.
* - `transient-error-detector.ts` provides the low-level error classifier (`isTransientError`,
* `classifyError`). This module consumes those classifiers but does not replace them.
*
* **Retry semantics:**
* - Up to `MAX_RECOVERY_RETRIES` attempts with exponential backoff.
* - Base delay: 60 seconds, multiplied by 2^attempt, capped at 300 seconds.
* - ±10% jitter to avoid thundering-herd effects.
* - Recovery metadata (`recoveryRetryCount`, `nextRecoveryAt`) is persisted on the task
* so retries survive engine restarts.
* - Exhausted retry budgets escalate to a real failure (task marked failed or error set).
*
* **Not retried via this policy:**
* - Usage-limit errors (handled by `UsageLimitPauser` with global pause)
* - User pauses (handled by pause flow)
* - Stuck-task-detector kills (handled by stuck flow)
* - Dependency-abort cleanups (handled by dep-abort flow)
* - Merge-conflict retries (handled by `mergeRetries` separately)
*/
// ── Constants ────────────────────────────────────────────────────────
/** Maximum number of recovery retry attempts before escalating to failure. */
export const MAX_RECOVERY_RETRIES = 3;
/** Base delay in milliseconds for the first retry (60 seconds). */
export const BASE_DELAY_MS = 60_000;
/** Maximum delay cap in milliseconds (300 seconds = 5 minutes). */
export const MAX_DELAY_MS = 300_000;
/** Backoff multiplier (2x exponential). */
export const BACKOFF_MULTIPLIER = 2;
// ── Types ────────────────────────────────────────────────────────────
export interface RecoveryState {
recoveryRetryCount?: number;
nextRecoveryAt?: string;
}
export interface RecoveryDecision {
/** Whether the task should be retried (moved back to todo/triage). */
shouldRetry: boolean;
/** Whether the retry budget is exhausted (terminal failure). */
exhausted: boolean;
/** Updated recovery state to persist on the task. */
nextState: RecoveryState;
/** Computed delay in milliseconds (for logging). Zero when exhausted. */
delayMs: number;
}
// ── Decision function ────────────────────────────────────────────────
/**
* Compute whether a recoverable failure should be retried and what the
* updated recovery state should be.
*
* This is a **pure function** — it does not call TaskStore or perform I/O.
* The caller is responsible for persisting `nextState` via `store.updateTask()`.
*
* @param currentState - Current recovery metadata from the task
* @returns A decision describing whether to retry or escalate
*/
export function computeRecoveryDecision(
currentState: RecoveryState,
): RecoveryDecision {
const currentCount = currentState.recoveryRetryCount ?? 0;
const nextCount = currentCount + 1;
if (nextCount > MAX_RECOVERY_RETRIES) {
// Budget exhausted — escalate to real failure
return {
shouldRetry: false,
exhausted: true,
nextState: { recoveryRetryCount: undefined, nextRecoveryAt: undefined },
delayMs: 0,
};
}
// Exponential backoff: base × 2^(attempt-1), capped at max
const rawDelay = Math.min(
BASE_DELAY_MS * BACKOFF_MULTIPLIER ** (nextCount - 1),
MAX_DELAY_MS,
);
// ±10% jitter to avoid thundering herd
const jitter = rawDelay * 0.1 * (2 * Math.random() - 1);
const delayMs = Math.max(0, Math.round(rawDelay + jitter));
const nextRecoveryAt = new Date(Date.now() + delayMs).toISOString();
return {
shouldRetry: true,
exhausted: false,
nextState: {
recoveryRetryCount: nextCount,
nextRecoveryAt,
},
delayMs,
};
}
/**
* Format a retry delay for human-readable logging.
*
* @param delayMs - Delay in milliseconds
* @returns Human-readable string like "60s" or "120s"
*/
export function formatDelay(delayMs: number): string {
const seconds = Math.round(delayMs / 1000);
if (seconds < 60) return `${seconds}s`;
const minutes = Math.round(seconds / 60);
return seconds % 60 === 0 ? `${minutes}m` : `${seconds}s`;
}

View File

@@ -924,4 +924,86 @@ describe("Scheduler", () => {
expect(mockMissionStore.activateSlice).not.toHaveBeenCalled();
});
});
describe("recovery due-time gating (nextRecoveryAt)", () => {
it("skips todo tasks whose nextRecoveryAt is in the future", async () => {
const future = new Date(Date.now() + 60_000).toISOString();
const task = createMockTask({
id: "FN-010",
column: "todo",
nextRecoveryAt: future,
recoveryRetryCount: 1,
});
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([task]),
getSettings: vi.fn().mockResolvedValue({ maxConcurrent: 2, maxWorktrees: 4 }),
});
const onSchedule = vi.fn();
const scheduler = new Scheduler(store, { onSchedule });
scheduler.start();
await scheduler.schedule();
scheduler.stop();
// Should NOT have been started
expect(onSchedule).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
});
it("picks up todo tasks whose nextRecoveryAt has elapsed", async () => {
const past = new Date(Date.now() - 1000).toISOString();
const task = createMockTask({
id: "FN-011",
column: "todo",
nextRecoveryAt: past,
recoveryRetryCount: 1,
});
// Mock filesystem validation: task dir exists, PROMPT.md exists and non-empty
(existsSync as any).mockReturnValue(true);
(readFile as any).mockResolvedValue("# Task\n\nSome content\n## File Scope\n- foo.ts\n");
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([task]),
getSettings: vi.fn().mockResolvedValue({ maxConcurrent: 2, maxWorktrees: 4 }),
parseFileScopeFromPrompt: vi.fn().mockResolvedValue([]),
getTasksDir: vi.fn().mockReturnValue("/test/project/.fusion/tasks"),
});
const onSchedule = vi.fn();
const scheduler = new Scheduler(store, { onSchedule });
// Call schedule() directly without start() to avoid scheduling guard race
(scheduler as any).running = true;
await scheduler.schedule();
expect(store.moveTask).toHaveBeenCalledWith("FN-011", "in-progress");
});
it("picks up todo tasks without nextRecoveryAt normally", async () => {
const task = createMockTask({
id: "FN-012",
column: "todo",
// No nextRecoveryAt — should be picked up normally
});
(existsSync as any).mockReturnValue(true);
(readFile as any).mockResolvedValue("# Task\n\nSome content\n## File Scope\n- foo.ts\n");
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([task]),
getSettings: vi.fn().mockResolvedValue({ maxConcurrent: 2, maxWorktrees: 4 }),
parseFileScopeFromPrompt: vi.fn().mockResolvedValue([]),
getTasksDir: vi.fn().mockReturnValue("/test/project/.fusion/tasks"),
});
const onSchedule = vi.fn();
const scheduler = new Scheduler(store, { onSchedule });
// Call schedule() directly without start() to avoid scheduling guard race
(scheduler as any).running = true;
await scheduler.schedule();
expect(store.moveTask).toHaveBeenCalledWith("FN-012", "in-progress");
});
});
});

View File

@@ -410,7 +410,13 @@ export class Scheduler {
);
if (available <= 0) return;
const todo = tasks.filter((t) => t.column === "todo" && !t.paused);
const now = Date.now();
const todo = tasks.filter((t) => {
if (t.column !== "todo" || t.paused) return false;
// Skip tasks with a recovery backoff that hasn't elapsed yet
if (t.nextRecoveryAt && new Date(t.nextRecoveryAt).getTime() > now) return false;
return true;
});
if (todo.length === 0) return;
/**

View File

@@ -10,14 +10,21 @@ import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import { mkdir, writeFile, rm } from "node:fs/promises";
const { mockReviewStep } = vi.hoisted(() => ({
const { mockReviewStep, mockCreateKbAgent } = vi.hoisted(() => ({
mockReviewStep: vi.fn(),
mockCreateKbAgent: vi.fn(),
}));
vi.mock("./reviewer.js", () => ({
reviewStep: mockReviewStep,
}));
vi.mock("./pi.js", () => ({
createKbAgent: mockCreateKbAgent,
describeModel: vi.fn().mockReturnValue("mock-model"),
promptWithFallback: vi.fn().mockReturnValue("mock-prompt"),
}));
const __dirname = dirname(fileURLToPath(import.meta.url));
function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
@@ -26,7 +33,7 @@ function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
listTasks: vi.fn().mockResolvedValue([]),
createTask: vi.fn(),
moveTask: vi.fn(),
updateTask: vi.fn(),
updateTask: vi.fn().mockResolvedValue(undefined),
deleteTask: vi.fn(),
mergeTask: vi.fn(),
getSettings: vi.fn().mockResolvedValue({
@@ -718,4 +725,147 @@ describe("taskCreate tool model inheritance", () => {
validatorModelId: undefined,
}));
});
describe("bounded recovery retries for triage", () => {
it("sets recoveryRetryCount and nextRecoveryAt on first transient error via specifyTask", async () => {
const task = {
id: "FN-200",
description: "Test triage task",
column: "triage",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
} as unknown as Task;
const store = createMockStore({
getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
});
const processor = new TriageProcessor(store, "/test/root", {
pollIntervalMs: 100_000,
});
// Mock createKbAgent to throw a transient error
mockCreateKbAgent.mockRejectedValue(new Error("upstream connect error"));
await processor.specifyTask(task);
expect(store.updateTask).toHaveBeenCalledWith("FN-200", expect.objectContaining({
recoveryRetryCount: 1,
nextRecoveryAt: expect.any(String),
}));
});
it("escalates to error state when triage retries are exhausted via specifyTask", async () => {
const task = {
id: "FN-201",
description: "Test triage task",
column: "triage",
recoveryRetryCount: 3, // Already at max
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
} as unknown as Task;
const onSpecifyError = vi.fn();
const store = createMockStore({
getTask: vi.fn().mockResolvedValue({ ...task, attachments: [] }),
});
const processor = new TriageProcessor(store, "/test/root", {
pollIntervalMs: 100_000,
onSpecifyError,
});
mockCreateKbAgent.mockRejectedValue(new Error("connection reset"));
await processor.specifyTask(task);
// Should set error and clear recovery metadata
expect(store.updateTask).toHaveBeenCalledWith("FN-201", expect.objectContaining({
error: expect.stringContaining("Specification failed after 3 transient errors"),
recoveryRetryCount: null,
nextRecoveryAt: null,
}));
expect(onSpecifyError).toHaveBeenCalled();
});
});
describe("recovery due-time gating (nextRecoveryAt)", () => {
it("skips triage tasks whose nextRecoveryAt is in the future", async () => {
const future = new Date(Date.now() + 60_000).toISOString();
const task = {
id: "FN-100",
description: "Test triage task",
column: "triage",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
nextRecoveryAt: future,
recoveryRetryCount: 1,
} as unknown as Task;
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([task]),
});
const processor = new TriageProcessor(store, "/test/root", {
pollIntervalMs: 100_000, // long interval so only manual poll runs
});
// Spy on specifyTask to ensure it's NOT called for gated tasks
const specifySpy = vi.spyOn(processor, "specifyTask");
processor.start();
// Wait a tick for the initial poll
await new Promise((r) => setTimeout(r, 50));
processor.stop();
expect(specifySpy).not.toHaveBeenCalled();
specifySpy.mockRestore();
});
it("processes triage tasks whose nextRecoveryAt has elapsed", async () => {
const past = new Date(Date.now() - 1000).toISOString();
const task = {
id: "FN-101",
description: "Test triage task past",
column: "triage",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
nextRecoveryAt: past,
recoveryRetryCount: 1,
} as unknown as Task;
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([task]),
});
const processor = new TriageProcessor(store, "/test/root", {
pollIntervalMs: 100_000,
});
const specifySpy = vi.spyOn(processor, "specifyTask").mockResolvedValue(undefined);
processor.start();
await new Promise((r) => setTimeout(r, 50));
processor.stop();
expect(specifySpy).toHaveBeenCalledWith(expect.objectContaining({ id: "FN-101" }));
specifySpy.mockRestore();
});
});
});

View File

@@ -23,6 +23,7 @@ import {
} from "./usage-limit-detector.js";
import { isTransientError } from "./transient-error-detector.js";
import { withRateLimitRetry } from "./rate-limit-retry.js";
import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js";
export const TRIAGE_SYSTEM_PROMPT = `You are a task specification agent for "kb", an AI-orchestrated task board.
@@ -373,8 +374,11 @@ export class TriageProcessor {
this.wasEnginePaused = false;
const tasks = await this.store.listTasks();
const now = Date.now();
const triageTasks = tasks.filter(
(t) => t.column === "triage" && !this.processing.has(t.id) && !t.paused,
(t) => t.column === "triage" && !this.processing.has(t.id) && !t.paused
// Skip tasks with a recovery backoff that hasn't elapsed yet
&& !(t.nextRecoveryAt && new Date(t.nextRecoveryAt).getTime() > now),
);
for (const task of triageTasks) {
@@ -685,12 +689,35 @@ export class TriageProcessor {
err.message,
);
} else if (isTransientError(err.message)) {
// Transient network/infrastructure error — don't mark as failed, allow retry
triageLog.warn(`${task.id} transient error during triage — will retry: ${err.message}`);
await this.store.logEntry(task.id, `Transient error during specification (will retry): ${err.message}`).catch(() => {});
// Restore status so triage picks it up again on next pass
const restoreStatus = task.status === "needs-respecify" ? "needs-respecify" : undefined;
await this.store.updateTask(task.id, { status: restoreStatus }).catch(() => {});
// Transient network/infrastructure error — use bounded recovery policy
const decision = computeRecoveryDecision({
recoveryRetryCount: task.recoveryRetryCount,
nextRecoveryAt: task.nextRecoveryAt,
});
if (decision.shouldRetry) {
const attempt = decision.nextState.recoveryRetryCount;
const delay = formatDelay(decision.delayMs);
triageLog.warn(`${task.id} transient error during triage — retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}: ${err.message}`);
await this.store.logEntry(task.id, `Transient error during specification (retry ${attempt}/${MAX_RECOVERY_RETRIES} in ${delay}): ${err.message}`).catch(() => {});
const restoreStatus = task.status === "needs-respecify" ? "needs-respecify" : undefined;
await this.store.updateTask(task.id, {
status: restoreStatus,
recoveryRetryCount: decision.nextState.recoveryRetryCount,
nextRecoveryAt: decision.nextState.nextRecoveryAt,
}).catch(() => {});
return;
}
// Recovery budget exhausted — freeze in triage with error for manual intervention
triageLog.error(`${task.id} transient error retries exhausted (${MAX_RECOVERY_RETRIES} attempts): ${err.message}`);
await this.store.logEntry(task.id, `Specification failed after ${MAX_RECOVERY_RETRIES} transient errors: ${err.message}`).catch(() => {});
await this.store.updateTask(task.id, {
error: `Specification failed after ${MAX_RECOVERY_RETRIES} transient errors: ${err.message}`,
recoveryRetryCount: null,
nextRecoveryAt: null,
}).catch(() => {});
this.options.onSpecifyError?.(task, err);
return;
}
// For re-specification, restore needs-respecify status so it can be retried