feat: add SelfHealingManager for unattended multi-day operation
Adds four self-healing subsystems to enable the engine to recover from common failure modes without human intervention: - Auto-unpause: clears rate-limit-triggered globalPause with escalating backoff (5 min → 60 min cap), resets on sustained recovery - Stuck kill budget: caps task stuck-kill retries (default 3) to prevent infinite stuck→todo→stuck loops - Periodic maintenance (every 15 min): git worktree prune, orphan cleanup, SQLite WAL checkpoint - Worktree cap enforcement: removes oldest idle worktrees when count exceeds 2× maxWorktrees New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs, autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs. New task field: stuckKillCount (schema v8 migration). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,10 @@ export interface StuckTaskDetectorOptions {
|
||||
/** Callback invoked when a stuck task is detected and killed.
|
||||
* The task will be moved to "todo" for retry by the detector. */
|
||||
onStuck?: (taskId: string) => void;
|
||||
/** Called before re-queuing a killed task. Return false to prevent re-queue
|
||||
* (caller is responsible for marking the task as terminally failed).
|
||||
* Used by SelfHealingManager to enforce stuck kill budgets. */
|
||||
beforeRequeue?: (taskId: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export class StuckTaskDetector {
|
||||
@@ -40,6 +44,7 @@ export class StuckTaskDetector {
|
||||
private interval: ReturnType<typeof setInterval> | null = null;
|
||||
private pollIntervalMs: number;
|
||||
private onStuck?: (taskId: string) => void;
|
||||
private beforeRequeue?: (taskId: string) => Promise<boolean>;
|
||||
|
||||
constructor(
|
||||
private store: TaskStore,
|
||||
@@ -47,6 +52,7 @@ export class StuckTaskDetector {
|
||||
) {
|
||||
this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
|
||||
this.onStuck = options.onStuck;
|
||||
this.beforeRequeue = options.beforeRequeue;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -158,6 +164,22 @@ export class StuckTaskDetector {
|
||||
stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
|
||||
}
|
||||
|
||||
// Check stuck kill budget before re-queuing (SelfHealingManager integration).
|
||||
// If beforeRequeue returns false, the task has been marked failed — skip re-queue.
|
||||
if (this.beforeRequeue) {
|
||||
try {
|
||||
const shouldRequeue = await this.beforeRequeue(taskId);
|
||||
if (!shouldRequeue) {
|
||||
stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
|
||||
this.onStuck?.(taskId);
|
||||
return;
|
||||
}
|
||||
} catch (err) {
|
||||
stuckLog.error(`beforeRequeue check failed for ${taskId}:`, err);
|
||||
// Fall through to re-queue on error — safer than dropping the task
|
||||
}
|
||||
}
|
||||
|
||||
// Set transient "stuck-killed" status, then move to "todo" for retry.
|
||||
// moveTask from "in-progress" to "todo" automatically clears status,
|
||||
// so no explicit status clear is needed after the move.
|
||||
|
||||
Reference in New Issue
Block a user