feat: add SelfHealingManager for unattended multi-day operation

Adds four self-healing subsystems to enable the engine to recover from
common failure modes without human intervention:

- Auto-unpause: clears rate-limit-triggered globalPause with escalating
  backoff (5 min → 60 min cap), resets on sustained recovery
- Stuck kill budget: caps task stuck-kill retries (default 3) to prevent
  infinite stuck→todo→stuck loops
- Periodic maintenance (every 15 min): git worktree prune, orphan cleanup,
  SQLite WAL checkpoint
- Worktree cap enforcement: removes oldest idle worktrees when count
  exceeds 2× maxWorktrees

New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs,
autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs.
New task field: stuckKillCount (schema v8 migration).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-03 15:35:12 -07:00
parent 10201f6304
commit bcb9915968
11 changed files with 842 additions and 18 deletions

View File

@@ -20,6 +20,7 @@ import type {
import { runtimeLog } from "../logger.js";
import type { StuckTaskDetector } from "../stuck-task-detector.js";
import type { UsageLimitPauser } from "../usage-limit-detector.js";
import { SelfHealingManager } from "../self-healing.js";
/**
* InProcessRuntime runs a project within the main process.
@@ -65,6 +66,7 @@ export class InProcessRuntime
private globalSemaphore?: AgentSemaphore;
private stuckTaskDetector?: StuckTaskDetector;
private usageLimitPauser?: UsageLimitPauser;
private selfHealingManager?: SelfHealingManager;
private agentStore?: AgentStore;
private heartbeatMonitor?: HeartbeatMonitor;
/** Maps task IDs to agent IDs for lifecycle tracking */
@@ -225,13 +227,19 @@ export class InProcessRuntime
runtimeLog.warn(`AgentStore initialization failed (continuing without agent monitoring):`, agentErr);
}
// 7. Set up event forwarding from TaskStore
// 7. Initialize SelfHealingManager
this.selfHealingManager = new SelfHealingManager(this.taskStore, {
rootDir: this.config.workingDirectory,
});
this.selfHealingManager.start();
// 8. Set up event forwarding from TaskStore
this.setupEventForwarding();
// 8. Resume orphaned in-progress tasks
// 9. Resume orphaned in-progress tasks
await this.executor.resumeOrphaned();
// 9. Start scheduler
// 10. Start scheduler
this.scheduler.start();
this.setStatus("active");
@@ -266,13 +274,19 @@ export class InProcessRuntime
runtimeLog.log(`Stopping InProcessRuntime for project ${this.config.projectId}`);
try {
// 1. Stop heartbeat monitor
// 1. Stop self-healing manager
if (this.selfHealingManager) {
this.selfHealingManager.stop();
runtimeLog.log("SelfHealingManager stopped");
}
// 2. Stop heartbeat monitor
if (this.heartbeatMonitor) {
this.heartbeatMonitor.stop();
runtimeLog.log("HeartbeatMonitor stopped");
}
// 2. Stop scheduler (prevents new task scheduling)
// 3. Stop scheduler (prevents new task scheduling)
if (this.scheduler) {
this.scheduler.stop();
runtimeLog.log("Scheduler stopped");