root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add SelfHealingManager for unattended multi-day operation

Browse Source 
Adds four self-healing subsystems to enable the engine to recover from
common failure modes without human intervention:

- Auto-unpause: clears rate-limit-triggered globalPause with escalating
  backoff (5 min → 60 min cap), resets on sustained recovery
- Stuck kill budget: caps task stuck-kill retries (default 3) to prevent
  infinite stuck→todo→stuck loops
- Periodic maintenance (every 15 min): git worktree prune, orphan cleanup,
  SQLite WAL checkpoint
- Worktree cap enforcement: removes oldest idle worktrees when count
  exceeds 2× maxWorktrees

New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs,
autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs.
New task field: stuckKillCount (schema v8 migration).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-03 15:35:12 -07:00
parent 6a38b001ba
commit fdf6d0b4a1
11 changed files with 842 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
packages/core/src/db.test.ts
View File

@@ -86,7 +86,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
});
it("seeds lastModified", () => {
@@ -109,7 +109,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
});
it("does not overwrite existing config on re-init", () => {
@@ -188,6 +188,18 @@ describe("Database", () => {
});
});
describe("walCheckpoint", () => {
it("runs WAL checkpoint and returns stats", () => {
const result = db.walCheckpoint();
expect(result).toHaveProperty("busy");
expect(result).toHaveProperty("log");
expect(result).toHaveProperty("checkpointed");
expect(typeof result.busy).toBe("number");
expect(typeof result.log).toBe("number");
expect(typeof result.checkpointed).toBe("number");
});
});
describe("transactions", () => {
it("commits on success", () => {
db.transaction(() => {
@@ -704,7 +716,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 5 (includes v1→v2, v2→v3, v3→v4, and v4→v5 migrations)
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -729,11 +741,11 @@ describe("schema migrations", () => {
const db = new Database(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
db.close();
});
@@ -828,7 +840,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 5
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1038,7 +1050,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(7);
expect(db.getSchemaVersion()).toBe(8);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();

19
packages/core/src/db.ts
View File

@@ -59,7 +59,7 @@ export function fromJson<T>(json: string | null | undefined): T | undefined {
// ── Schema Definition ────────────────────────────────────────────────
const SCHEMA_VERSION = 7;
const SCHEMA_VERSION = 8;
function normalizeTaskComments(
steeringComments: SteeringComment[] | undefined,
@@ -418,8 +418,14 @@ export class Database {
});
}
if (version < 8) {
this.applyMigration(8, () => {
this.addColumnIfMissing("tasks", "stuckKillCount", "INTEGER DEFAULT 0");
});
}
// Future migrations go here:
// if (version < 8) { this.applyMigration(8, () => { ... }); }
// if (version < 9) { this.applyMigration(9, () => { ... }); }
}
/**
@@ -486,6 +492,15 @@ export class Database {
}
}
/**
* Run a WAL checkpoint to truncate the WAL file and reclaim disk space.
* Safe to call periodically. Returns checkpoint stats.
*/
walCheckpoint(): { busy: number; log: number; checkpointed: number } {
const row = this.db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get() as any;
return { busy: row?.busy ?? 0, log: row?.log ?? 0, checkpointed: row?.checkpointed ?? 0 };
}
/**
* Close the database connection.
*/

21
packages/core/src/store.ts
View File

@@ -178,6 +178,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
validatorModelProvider: row.validatorModelProvider || undefined,
validatorModelId: row.validatorModelId || undefined,
mergeRetries: row.mergeRetries ?? undefined,
stuckKillCount: row.stuckKillCount ?? undefined,
recoveryRetryCount: row.recoveryRetryCount ?? undefined,
nextRecoveryAt: row.nextRecoveryAt || undefined,
error: row.error || undefined,
@@ -228,14 +229,14 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
id, title, description, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, branch, baseCommitSha, modelPresetId, modelProvider,
modelId, validatorModelProvider, validatorModelId, mergeRetries,
recoveryRetryCount, nextRecoveryAt, error,
stuckKillCount, recoveryRetryCount, nextRecoveryAt, error,
summary, thinkingLevel, createdAt, updatedAt, columnMovedAt,
dependencies, steps, log, attachments, steeringComments,
comments, workflowStepResults, prInfo, issueInfo, mergeDetails,
breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, missionId, sliceId
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`).run(
task.id,
@@ -258,6 +259,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.validatorModelProvider ?? null,
task.validatorModelId ?? null,
task.mergeRetries ?? null,
task.stuckKillCount ?? 0,
task.recoveryRetryCount ?? null,
task.nextRecoveryAt ?? null,
task.error ?? null,
@@ -975,7 +977,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
async updateTask(
id: string,
updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; recoveryRetryCount?: number | null; nextRecoveryAt?: string | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; stuckKillCount?: number | null; recoveryRetryCount?: number | null; nextRecoveryAt?: string | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
): Promise<Task> {
return this.withTaskLock(id, async () => {
// Validate that task doesn't depend on itself
@@ -1030,6 +1032,11 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
if (updates.size !== undefined) task.size = updates.size;
if (updates.reviewLevel !== undefined) task.reviewLevel = updates.reviewLevel;
if (updates.mergeRetries !== undefined) task.mergeRetries = updates.mergeRetries;
if (updates.stuckKillCount === null) {
task.stuckKillCount = undefined;
} else if (updates.stuckKillCount !== undefined) {
task.stuckKillCount = updates.stuckKillCount;
}
if (updates.recoveryRetryCount === null) {
task.recoveryRetryCount = undefined;
} else if (updates.recoveryRetryCount !== undefined) {
@@ -2715,6 +2722,14 @@ ${stepsSection}`;
}
}
/**
* Run a WAL checkpoint to truncate the WAL file and reclaim disk space.
* Safe to call periodically from the self-healing maintenance timer.
*/
walCheckpoint(): { busy: number; log: number; checkpointed: number } {
return this.db.walCheckpoint();
}
getRootDir(): string {
return this.rootDir;
}

24
packages/core/src/types.ts
View File

@@ -475,6 +475,10 @@ export interface Task {
workflowStepResults?: WorkflowStepResult[];
/** Number of merge retry attempts made for this task (auto-merge conflict recovery) */
mergeRetries?: number;
/** Number of times the stuck-task detector has killed this task's agent session.
* Incremented by the self-healing manager on each stuck kill. When this reaches
* `maxStuckKills`, the task is marked as permanently failed instead of re-queued. */
stuckKillCount?: number;
/** Number of bounded recovery retry attempts for transient executor/triage failures.
* Distinct from `mergeRetries` (merge-conflict-specific). Incremented by the
* recovery-policy module on each recoverable failure; cleared when work restarts
@@ -743,6 +747,21 @@ export interface ProjectSettings {
* than this duration, the task is considered stuck and will be terminated and retried.
* Default: undefined (disabled). Suggested value: 600000 (10 minutes). */
taskStuckTimeoutMs?: number;
/** When true, automatically unpause after rate-limit-triggered globalPause using
* escalating backoff. Allows unattended recovery from transient API rate limits.
* Default: true. */
autoUnpauseEnabled?: boolean;
/** Base delay in milliseconds before first auto-unpause attempt after rate-limit pause.
* Subsequent attempts use exponential backoff (2x). Default: 300000 (5 min). */
autoUnpauseBaseDelayMs?: number;
/** Maximum delay cap in milliseconds for auto-unpause backoff. Default: 3600000 (60 min). */
autoUnpauseMaxDelayMs?: number;
/** Maximum number of times the stuck-task detector can kill and re-queue a task
* before it is marked as permanently failed. Default: 3. */
maxStuckKills?: number;
/** Interval in milliseconds for periodic maintenance (worktree pruning, WAL checkpoint,
* orphan cleanup). 0 disables. Default: 900000 (15 min). */
maintenanceIntervalMs?: number;
/** When true, automatically poll and update PR status badges for tasks linked to GitHub PRs.
* Default: false. */
autoUpdatePrStatus?: boolean;
@@ -845,6 +864,11 @@ export const DEFAULT_PROJECT_SETTINGS: ProjectSettings = {
smartConflictResolution: true,
requirePlanApproval: false,
taskStuckTimeoutMs: undefined,
autoUnpauseEnabled: true,
autoUnpauseBaseDelayMs: 300_000,
autoUnpauseMaxDelayMs: 3_600_000,
maxStuckKills: 3,
maintenanceIntervalMs: 900_000,
autoUpdatePrStatus: false,
autoCreatePr: false,
autoBackupEnabled: false,