feat: add SelfHealingManager for unattended multi-day operation

Adds four self-healing subsystems to enable the engine to recover from
common failure modes without human intervention:

- Auto-unpause: clears rate-limit-triggered globalPause with escalating
  backoff (5 min → 60 min cap), resets on sustained recovery
- Stuck kill budget: caps task stuck-kill retries (default 3) to prevent
  infinite stuck→todo→stuck loops
- Periodic maintenance (every 15 min): git worktree prune, orphan cleanup,
  SQLite WAL checkpoint
- Worktree cap enforcement: removes oldest idle worktrees when count
  exceeds 2× maxWorktrees

New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs,
autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs.
New task field: stuckKillCount (schema v8 migration).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-03 15:35:12 -07:00
parent 10201f6304
commit bcb9915968
11 changed files with 842 additions and 18 deletions

View File

@@ -33,6 +33,10 @@ export interface StuckTaskDetectorOptions {
/** Callback invoked when a stuck task is detected and killed.
* The task will be moved to "todo" for retry by the detector. */
onStuck?: (taskId: string) => void;
/** Called before re-queuing a killed task. Return false to prevent re-queue
* (caller is responsible for marking the task as terminally failed).
* Used by SelfHealingManager to enforce stuck kill budgets. */
beforeRequeue?: (taskId: string) => Promise<boolean>;
}
export class StuckTaskDetector {
@@ -40,6 +44,7 @@ export class StuckTaskDetector {
private interval: ReturnType<typeof setInterval> | null = null;
private pollIntervalMs: number;
private onStuck?: (taskId: string) => void;
private beforeRequeue?: (taskId: string) => Promise<boolean>;
constructor(
private store: TaskStore,
@@ -47,6 +52,7 @@ export class StuckTaskDetector {
) {
this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
this.onStuck = options.onStuck;
this.beforeRequeue = options.beforeRequeue;
}
/**
@@ -158,6 +164,22 @@ export class StuckTaskDetector {
stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
}
// Check stuck kill budget before re-queuing (SelfHealingManager integration).
// If beforeRequeue returns false, the task has been marked failed — skip re-queue.
if (this.beforeRequeue) {
try {
const shouldRequeue = await this.beforeRequeue(taskId);
if (!shouldRequeue) {
stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
this.onStuck?.(taskId);
return;
}
} catch (err) {
stuckLog.error(`beforeRequeue check failed for ${taskId}:`, err);
// Fall through to re-queue on error — safer than dropping the task
}
}
// Set transient "stuck-killed" status, then move to "todo" for retry.
// moveTask from "in-progress" to "todo" automatically clears status,
// so no explicit status clear is needed after the move.