root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-775): add recoverable-retry with bounded exponential backoff

Browse Source 
- Add  state to tasks: persisted in DB, gates scheduler pickup to prevent immediate retry of transient failures
- Introduce shared recovery-policy module with bounded exponential backoff (1s → 60s, max 5 attempts)
- Wire recovery policy into executor, scheduler, and triage so all agents respect the same retry cadence
- Persist retry state (attempt count, next eligible time) in task metadata via store and DB schema
- Add DB migration for new retry columns and update schema tests
- Update README with recovery policy documentation
- Refactor dashboard Header component and styles, consolidate header tests
- Fix session-files route tests to align with updated route signatures
This commit is contained in:
gsxdsm
2026-04-03 07:58:45 -07:00
parent ee8f19fc93
commit 51855f43d9
15 changed files with 961 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/core/src/db-migrate.ts
View File

@@ -166,13 +166,14 @@ async function migrateTasks(kbDir: string, db: Database): Promise<void> {
id, title, description, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, baseCommitSha, modelPresetId,
modelProvider, modelId, validatorModelProvider, validatorModelId,
mergeRetries, error, summary, thinkingLevel, createdAt, updatedAt,
mergeRetries, recoveryRetryCount, nextRecoveryAt,
error, summary, thinkingLevel, createdAt, updatedAt,
columnMovedAt, dependencies, steps, log, attachments, steeringComments,
comments, workflowStepResults, prInfo, issueInfo, mergeDetails,
breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, sliceId
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`);
@@ -211,6 +212,8 @@ async function migrateTasks(kbDir: string, db: Database): Promise<void> {
task.validatorModelProvider ?? null,
task.validatorModelId ?? null,
task.mergeRetries ?? null,
task.recoveryRetryCount ?? null,
task.nextRecoveryAt ?? null,
task.error ?? null,
task.summary ?? null,
task.thinkingLevel ?? null,

14
packages/core/src/db.test.ts
View File

@@ -86,7 +86,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
});
it("seeds lastModified", () => {
@@ -109,7 +109,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
});
it("does not overwrite existing config on re-init", () => {
@@ -704,7 +704,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 5 (includes v1→v2, v2→v3, v3→v4, and v4→v5 migrations)
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -729,11 +729,11 @@ describe("schema migrations", () => {
const db = new Database(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
db.close();
});
@@ -828,7 +828,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 5
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1038,7 +1038,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(6);
expect(db.getSchemaVersion()).toBe(7);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();

13
packages/core/src/db.ts
View File

@@ -59,7 +59,7 @@ export function fromJson<T>(json: string | null | undefined): T | undefined {
// ── Schema Definition ────────────────────────────────────────────────
const SCHEMA_VERSION = 6;
const SCHEMA_VERSION = 7;
function normalizeTaskComments(
steeringComments: SteeringComment[] | undefined,
@@ -142,6 +142,8 @@ CREATE TABLE IF NOT EXISTS tasks (
validatorModelProvider TEXT,
validatorModelId TEXT,
mergeRetries INTEGER,
recoveryRetryCount INTEGER,
nextRecoveryAt TEXT,
error TEXT,
summary TEXT,
thinkingLevel TEXT,
@@ -409,8 +411,15 @@ export class Database {
});
}
if (version < 7) {
this.applyMigration(7, () => {
this.addColumnIfMissing("tasks", "recoveryRetryCount", "INTEGER");
this.addColumnIfMissing("tasks", "nextRecoveryAt", "TEXT");
});
}
// Future migrations go here:
// if (version < 7) { this.applyMigration(7, () => { ... }); }
// if (version < 8) { this.applyMigration(8, () => { ... }); }
}
/**

108
packages/core/src/store.test.ts
View File

@@ -4630,4 +4630,112 @@ Task with acceptance criteria
expect(movedEvents[0].to).toBe("todo");
});
});
describe("recovery metadata (recoveryRetryCount / nextRecoveryAt)", () => {
async function createTestTask(overrides: Partial<import("./types.js").TaskCreateInput> = {}) {
return store.createTask({ description: "recovery test task", ...overrides });
}
it("new tasks have no recovery metadata (defaults to undefined)", async () => {
const task = await createTestTask();
expect(task.recoveryRetryCount).toBeUndefined();
expect(task.nextRecoveryAt).toBeUndefined();
});
it("updateTask can set and clear recoveryRetryCount and nextRecoveryAt", async () => {
const task = await createTestTask();
const futureTime = new Date(Date.now() + 60_000).toISOString();
// Set
const updated = await store.updateTask(task.id, {
recoveryRetryCount: 2,
nextRecoveryAt: futureTime,
});
expect(updated.recoveryRetryCount).toBe(2);
expect(updated.nextRecoveryAt).toBe(futureTime);
// Re-read to verify persistence
const reread = await store.getTask(task.id);
expect(reread.recoveryRetryCount).toBe(2);
expect(reread.nextRecoveryAt).toBe(futureTime);
// Clear
const cleared = await store.updateTask(task.id, {
recoveryRetryCount: null,
nextRecoveryAt: null,
});
expect(cleared.recoveryRetryCount).toBeUndefined();
expect(cleared.nextRecoveryAt).toBeUndefined();
});
it("moveTask to in-review clears recovery metadata", async () => {
const task = await createTestTask();
await store.moveTask(task.id, "todo");
await store.updateTask(task.id, {
recoveryRetryCount: 3,
nextRecoveryAt: new Date().toISOString(),
});
await store.moveTask(task.id, "in-progress");
const moved = await store.moveTask(task.id, "in-review");
expect(moved.recoveryRetryCount).toBeUndefined();
expect(moved.nextRecoveryAt).toBeUndefined();
});
it("moveTask to done clears recovery metadata", async () => {
const task = await createTestTask();
await store.moveTask(task.id, "todo");
await store.updateTask(task.id, {
recoveryRetryCount: 1,
nextRecoveryAt: new Date().toISOString(),
});
await store.moveTask(task.id, "in-progress");
await store.moveTask(task.id, "in-review");
const done = await store.moveTask(task.id, "done");
expect(done.recoveryRetryCount).toBeUndefined();
expect(done.nextRecoveryAt).toBeUndefined();
});
it("moveTask from in-progress to todo preserves recovery metadata", async () => {
const task = await createTestTask();
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
const futureTime = new Date(Date.now() + 60_000).toISOString();
await store.updateTask(task.id, {
recoveryRetryCount: 2,
nextRecoveryAt: futureTime,
});
const moved = await store.moveTask(task.id, "todo");
expect(moved.recoveryRetryCount).toBe(2);
expect(moved.nextRecoveryAt).toBe(futureTime);
});
it("recovery metadata persists across store re-initialization", async () => {
const task = await createTestTask();
const futureTime = new Date(Date.now() + 60_000).toISOString();
await store.updateTask(task.id, {
recoveryRetryCount: 5,
nextRecoveryAt: futureTime,
});
// Re-create store to simulate restart
store.stopWatching();
const store2 = new TaskStore(rootDir, globalDir);
await store2.init();
const reloaded = await store2.getTask(task.id);
expect(reloaded.recoveryRetryCount).toBe(5);
expect(reloaded.nextRecoveryAt).toBe(futureTime);
store2.stopWatching();
});
it("schema migration: existing rows default to NULL (undefined) for recovery fields", async () => {
// Tasks created before the migration should have undefined recovery fields
const task = await createTestTask();
const detail = await store.getTask(task.id);
expect(detail.recoveryRetryCount).toBeUndefined();
expect(detail.nextRecoveryAt).toBeUndefined();
});
});
});

34
packages/core/src/store.ts
View File

@@ -178,6 +178,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
validatorModelProvider: row.validatorModelProvider || undefined,
validatorModelId: row.validatorModelId || undefined,
mergeRetries: row.mergeRetries ?? undefined,
recoveryRetryCount: row.recoveryRetryCount ?? undefined,
nextRecoveryAt: row.nextRecoveryAt || undefined,
error: row.error || undefined,
summary: row.summary || undefined,
thinkingLevel: row.thinkingLevel || undefined,
@@ -225,14 +227,15 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
INSERT OR REPLACE INTO tasks (
id, title, description, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, branch, baseCommitSha, modelPresetId, modelProvider,
modelId, validatorModelProvider, validatorModelId, mergeRetries, error,
modelId, validatorModelProvider, validatorModelId, mergeRetries,
recoveryRetryCount, nextRecoveryAt, error,
summary, thinkingLevel, createdAt, updatedAt, columnMovedAt,
dependencies, steps, log, attachments, steeringComments,
comments, workflowStepResults, prInfo, issueInfo, mergeDetails,
breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, missionId, sliceId
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`).run(
task.id,
@@ -255,6 +258,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.validatorModelProvider ?? null,
task.validatorModelId ?? null,
task.mergeRetries ?? null,
task.recoveryRetryCount ?? null,
task.nextRecoveryAt ?? null,
task.error ?? null,
task.summary ?? null,
task.thinkingLevel ?? null,
@@ -930,10 +935,15 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.error = undefined;
task.worktree = undefined;
task.blockedBy = undefined;
task.recoveryRetryCount = undefined;
task.nextRecoveryAt = undefined;
}
// Clear transient fields when moving from in-progress to reset columns (todo/triage)
// This ensures failed tasks don't show failed status after being moved for retry
// This ensures failed tasks don't show failed status after being moved for retry.
// Note: recovery metadata (recoveryRetryCount, nextRecoveryAt) is intentionally
// preserved here — the recovery-policy module manages those fields. They are
// only cleared on terminal transitions (in-review, done, archived).
if (fromColumn === "in-progress" && (toColumn === "todo" || toColumn === "triage")) {
task.status = undefined;
task.error = undefined;
@@ -941,6 +951,12 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.blockedBy = undefined;
}
// Clear recovery metadata when task reaches in-review (successful completion)
if (toColumn === "in-review") {
task.recoveryRetryCount = undefined;
task.nextRecoveryAt = undefined;
}
// Clear workflow step results when moving from in-review back to todo or in-progress
// This ensures fresh workflow step runs on retry
if (fromColumn === "in-review" && (toColumn === "todo" || toColumn === "in-progress")) {
@@ -959,7 +975,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
async updateTask(
id: string,
updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; recoveryRetryCount?: number | null; nextRecoveryAt?: string | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
): Promise<Task> {
return this.withTaskLock(id, async () => {
// Validate that task doesn't depend on itself
@@ -1014,6 +1030,16 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
if (updates.size !== undefined) task.size = updates.size;
if (updates.reviewLevel !== undefined) task.reviewLevel = updates.reviewLevel;
if (updates.mergeRetries !== undefined) task.mergeRetries = updates.mergeRetries;
if (updates.recoveryRetryCount === null) {
task.recoveryRetryCount = undefined;
} else if (updates.recoveryRetryCount !== undefined) {
task.recoveryRetryCount = updates.recoveryRetryCount;
}
if (updates.nextRecoveryAt === null) {
task.nextRecoveryAt = undefined;
} else if (updates.nextRecoveryAt !== undefined) {
task.nextRecoveryAt = updates.nextRecoveryAt;
}
if (updates.modelProvider === null) {
task.modelProvider = undefined;
} else if (updates.modelProvider !== undefined) {

11
packages/core/src/types.ts
View File

@@ -475,6 +475,15 @@ export interface Task {
workflowStepResults?: WorkflowStepResult[];
/** Number of merge retry attempts made for this task (auto-merge conflict recovery) */
mergeRetries?: number;
/** Number of bounded recovery retry attempts for transient executor/triage failures.
* Distinct from `mergeRetries` (merge-conflict-specific). Incremented by the
* recovery-policy module on each recoverable failure; cleared when work restarts
* cleanly or reaches a terminal column (in-review, done, archived). */
recoveryRetryCount?: number;
/** ISO-8601 timestamp indicating when the task becomes eligible for the next
* recovery retry. Scheduler and triage processor skip tasks whose
* `nextRecoveryAt` is still in the future. Cleared alongside `recoveryRetryCount`. */
nextRecoveryAt?: string;
/** Thinking level for AI agent sessions — controls reasoning effort (off/minimal/low/medium/high) */
thinkingLevel?: ThinkingLevel;
/** Error message from the last failure, if the task failed during execution */
@@ -1008,6 +1017,8 @@ export interface ArchivedTaskEntry {
/** Slice ID this task is linked to */
sliceId?: string;
mergeRetries?: number;
recoveryRetryCount?: number;
nextRecoveryAt?: string;
error?: string;
}