feat: add SelfHealingManager for unattended multi-day operation
Adds four self-healing subsystems to enable the engine to recover from common failure modes without human intervention: - Auto-unpause: clears rate-limit-triggered globalPause with escalating backoff (5 min → 60 min cap), resets on sustained recovery - Stuck kill budget: caps task stuck-kill retries (default 3) to prevent infinite stuck→todo→stuck loops - Periodic maintenance (every 15 min): git worktree prune, orphan cleanup, SQLite WAL checkpoint - Worktree cap enforcement: removes oldest idle worktrees when count exceeds 2× maxWorktrees New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs, autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs. New task field: stuckKillCount (schema v8 migration). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ import type {
|
||||
import { runtimeLog } from "../logger.js";
|
||||
import type { StuckTaskDetector } from "../stuck-task-detector.js";
|
||||
import type { UsageLimitPauser } from "../usage-limit-detector.js";
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
|
||||
/**
|
||||
* InProcessRuntime runs a project within the main process.
|
||||
@@ -65,6 +66,7 @@ export class InProcessRuntime
|
||||
private globalSemaphore?: AgentSemaphore;
|
||||
private stuckTaskDetector?: StuckTaskDetector;
|
||||
private usageLimitPauser?: UsageLimitPauser;
|
||||
private selfHealingManager?: SelfHealingManager;
|
||||
private agentStore?: AgentStore;
|
||||
private heartbeatMonitor?: HeartbeatMonitor;
|
||||
/** Maps task IDs to agent IDs for lifecycle tracking */
|
||||
@@ -225,13 +227,19 @@ export class InProcessRuntime
|
||||
runtimeLog.warn(`AgentStore initialization failed (continuing without agent monitoring):`, agentErr);
|
||||
}
|
||||
|
||||
// 7. Set up event forwarding from TaskStore
|
||||
// 7. Initialize SelfHealingManager
|
||||
this.selfHealingManager = new SelfHealingManager(this.taskStore, {
|
||||
rootDir: this.config.workingDirectory,
|
||||
});
|
||||
this.selfHealingManager.start();
|
||||
|
||||
// 8. Set up event forwarding from TaskStore
|
||||
this.setupEventForwarding();
|
||||
|
||||
// 8. Resume orphaned in-progress tasks
|
||||
// 9. Resume orphaned in-progress tasks
|
||||
await this.executor.resumeOrphaned();
|
||||
|
||||
// 9. Start scheduler
|
||||
// 10. Start scheduler
|
||||
this.scheduler.start();
|
||||
|
||||
this.setStatus("active");
|
||||
@@ -266,13 +274,19 @@ export class InProcessRuntime
|
||||
runtimeLog.log(`Stopping InProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
try {
|
||||
// 1. Stop heartbeat monitor
|
||||
// 1. Stop self-healing manager
|
||||
if (this.selfHealingManager) {
|
||||
this.selfHealingManager.stop();
|
||||
runtimeLog.log("SelfHealingManager stopped");
|
||||
}
|
||||
|
||||
// 2. Stop heartbeat monitor
|
||||
if (this.heartbeatMonitor) {
|
||||
this.heartbeatMonitor.stop();
|
||||
runtimeLog.log("HeartbeatMonitor stopped");
|
||||
}
|
||||
|
||||
// 2. Stop scheduler (prevents new task scheduling)
|
||||
// 3. Stop scheduler (prevents new task scheduling)
|
||||
if (this.scheduler) {
|
||||
this.scheduler.stop();
|
||||
runtimeLog.log("Scheduler stopped");
|
||||
|
||||
Reference in New Issue
Block a user