/** * SelfHealingManager — enables unattended multi-day/week operation by * providing automatic recovery from common failure modes. * * Four subsystems: * 1. **Auto-unpause**: Clears rate-limit-triggered `globalPause` with * escalating backoff (5 min → 60 min cap). Resets on sustained unpause. * 2. **Stuck kill budget**: Caps how many times a task can be killed by the * stuck-task detector before marking it as permanently failed. * 3. **Periodic maintenance**: Worktree pruning, orphan cleanup, SQLite * WAL checkpoint — all on a configurable interval (default 15 min). * 4. **Worktree cap enforcement**: Prevents unbounded worktree accumulation * by cleaning oldest idle worktrees when count exceeds 2× maxWorktrees. */ import { execSync } from "node:child_process"; import { existsSync, readdirSync, statSync } from "node:fs"; import { join } from "node:path"; import type { TaskStore, Settings } from "@fusion/core"; import { createLogger } from "./logger.js"; import { scanIdleWorktrees } from "./worktree-pool.js"; const log = createLogger("self-healing"); export interface SelfHealingOptions { /** Project root directory (parent of .worktrees/) */ rootDir: string; } export class SelfHealingManager { // ── Auto-unpause state ────────────────────────────────────────────── private unpauseTimer: ReturnType | null = null; private unpauseAttempt = 0; private lastPauseTriggeredAt = 0; private lastUnpauseAt = 0; // ── Maintenance timer ─────────────────────────────────────────────── private maintenanceInterval: ReturnType | null = null; // ── Event listener cleanup ────────────────────────────────────────── private settingsListener: ((data: { settings: Settings; previous: Settings }) => void) | null = null; constructor( private store: TaskStore, private options: SelfHealingOptions, ) {} // ── Lifecycle ─────────────────────────────────────────────────────── start(): void { // Wire up settings:updated listener for auto-unpause this.settingsListener = ({ settings, previous }) => { this.onSettingsUpdated(settings, previous); }; this.store.on("settings:updated", this.settingsListener); // Start periodic maintenance this.startMaintenance(); log.log("Started"); } stop(): void { // Remove settings listener if (this.settingsListener) { try { this.store.removeListener("settings:updated", this.settingsListener); } catch { // Store may not support removeListener (e.g., test mocks) } this.settingsListener = null; } // Clear timers this.cancelUnpauseTimer(); if (this.maintenanceInterval) { clearInterval(this.maintenanceInterval); this.maintenanceInterval = null; } log.log("Stopped"); } // ── Auto-unpause ─────────────────────────────────────────────────── private onSettingsUpdated(settings: Settings, previous: Settings): void { // globalPause false → true: schedule auto-unpause if (!previous.globalPause && settings.globalPause) { if (!settings.autoUnpauseEnabled) { log.log("Global pause activated — auto-unpause disabled, requires manual intervention"); return; } // If pause re-triggered within 60s of our last unpause, escalate backoff if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) { this.unpauseAttempt++; log.warn(`Global pause re-triggered within 60s — escalating to attempt ${this.unpauseAttempt}`); } this.lastPauseTriggeredAt = Date.now(); const baseDelay = settings.autoUnpauseBaseDelayMs ?? 300_000; const maxDelay = settings.autoUnpauseMaxDelayMs ?? 3_600_000; const delay = Math.min(baseDelay * Math.pow(2, this.unpauseAttempt), maxDelay); this.scheduleUnpause(delay); } // globalPause true → false: check if we should reset backoff if (previous.globalPause && !settings.globalPause) { this.cancelUnpauseTimer(); // If sustained unpause (not a quick re-trigger), reset attempt counter if (this.lastPauseTriggeredAt && (Date.now() - this.lastPauseTriggeredAt) > 60_000) { this.unpauseAttempt = 0; } } } private scheduleUnpause(delayMs: number): void { this.cancelUnpauseTimer(); const delaySec = Math.round(delayMs / 1000); const delayMin = Math.round(delaySec / 60); const display = delayMin >= 1 ? `${delayMin}m` : `${delaySec}s`; log.warn(`Auto-unpause scheduled in ${display} (attempt ${this.unpauseAttempt + 1})`); this.unpauseTimer = setTimeout(() => { this.unpauseTimer = null; void this.attemptUnpause(); }, delayMs); } private async attemptUnpause(): Promise { try { const settings = await this.store.getSettings(); // Already unpaused (manually or by another mechanism) if (!settings.globalPause) { log.log("Auto-unpause: already unpaused — no action needed"); this.unpauseAttempt = 0; return; } log.warn("Auto-unpause: clearing globalPause"); this.lastUnpauseAt = Date.now(); await this.store.updateSettings({ globalPause: false }); // Note: if the rate limit is still active, the next agent session will // hit it again → UsageLimitPauser triggers globalPause → our listener // catches the transition and schedules the next attempt with escalated backoff. } catch (err: any) { log.error(`Auto-unpause failed: ${err.message}`); } } private cancelUnpauseTimer(): void { if (this.unpauseTimer) { clearTimeout(this.unpauseTimer); this.unpauseTimer = null; } } // ── Stuck kill budget ───────────────────────────────────────────── /** * Check whether a stuck-killed task should be re-queued or marked as failed. * Called by StuckTaskDetector's `beforeRequeue` callback. * * @returns `true` if the task should be re-queued, `false` if budget exhausted * (task has been marked as permanently failed). */ async checkStuckBudget(taskId: string): Promise { try { const settings = await this.store.getSettings(); const maxKills = settings.maxStuckKills ?? 3; const task = await this.store.getTask(taskId); const newCount = (task.stuckKillCount ?? 0) + 1; if (newCount > maxKills) { // Budget exhausted — mark as permanently failed log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}) — marking failed`); await this.store.updateTask(taskId, { stuckKillCount: newCount, status: "failed", error: `Task stuck ${newCount} times — exceeded maximum of ${maxKills} stuck kills`, }); await this.store.logEntry( taskId, `Permanently failed: agent stuck ${newCount} times (max: ${maxKills})`, ); return false; } // Budget remaining — allow re-queue log.log(`${taskId} stuck kill ${newCount}/${maxKills} — will re-queue`); await this.store.updateTask(taskId, { stuckKillCount: newCount }); await this.store.logEntry( taskId, `Stuck kill ${newCount}/${maxKills} — re-queuing for retry`, ); return true; } catch (err: any) { log.error(`checkStuckBudget failed for ${taskId}: ${err.message}`); // On error, allow re-queue — safer than permanently failing return true; } } // ── Periodic maintenance ────────────────────────────────────────── private async startMaintenance(): Promise { const settings = await this.store.getSettings(); const intervalMs = settings.maintenanceIntervalMs ?? 900_000; if (intervalMs <= 0) { log.log("Periodic maintenance disabled (maintenanceIntervalMs <= 0)"); return; } log.log(`Periodic maintenance every ${Math.round(intervalMs / 60_000)}m`); this.maintenanceInterval = setInterval(() => { void this.runMaintenance(); }, intervalMs); } private async runMaintenance(): Promise { const startMs = Date.now(); log.log("Maintenance cycle starting"); try { await this.pruneWorktrees(); await this.cleanupOrphans(); this.checkpointWal(); await this.enforceWorktreeCap(); const elapsedMs = Date.now() - startMs; log.log(`Maintenance cycle completed in ${elapsedMs}ms`); } catch (err: any) { log.error(`Maintenance cycle failed: ${err.message}`); } } /** Run `git worktree prune` to clean stale metadata. */ private async pruneWorktrees(): Promise { try { execSync("git worktree prune", { cwd: this.options.rootDir, stdio: "pipe", timeout: 30_000, }); log.log("Worktree prune completed"); } catch (err: any) { log.error(`Worktree prune failed: ${err.message}`); } } /** Remove orphaned worktrees not assigned to any active task. */ private async cleanupOrphans(): Promise { try { const orphaned = await scanIdleWorktrees(this.options.rootDir, this.store); if (orphaned.length === 0) return 0; // Only clean up if recycling is disabled — otherwise they belong in the pool const settings = await this.store.getSettings(); if (settings.recycleWorktrees) { return 0; } let cleaned = 0; for (const worktreePath of orphaned) { try { execSync(`git worktree remove "${worktreePath}" --force`, { cwd: this.options.rootDir, stdio: "pipe", timeout: 30_000, }); cleaned++; } catch { // Individual failure is non-fatal } } if (cleaned > 0) { log.log(`Cleaned ${cleaned} orphaned worktree(s)`); } return cleaned; } catch (err: any) { log.error(`Orphan cleanup failed: ${err.message}`); return 0; } } /** Run SQLite WAL checkpoint to reclaim disk space. */ private checkpointWal(): void { try { const result = this.store.walCheckpoint(); if (result.log > 0) { log.log(`WAL checkpoint: ${result.checkpointed}/${result.log} pages checkpointed` + (result.busy > 0 ? ` (${result.busy} busy)` : "")); } } catch (err: any) { log.error(`WAL checkpoint failed: ${err.message}`); } } /** Remove oldest idle worktrees if total count exceeds 2× maxWorktrees. */ private async enforceWorktreeCap(): Promise { const worktreesDir = join(this.options.rootDir, ".worktrees"); if (!existsSync(worktreesDir)) return; try { const settings = await this.store.getSettings(); const cap = (settings.maxWorktrees ?? 4) * 2; const entries = readdirSync(worktreesDir, { withFileTypes: true }); const dirs = entries.filter((e) => e.isDirectory()); if (dirs.length <= cap) return; // Find idle worktrees that can be safely removed const idle = await scanIdleWorktrees(this.options.rootDir, this.store); if (idle.length === 0) return; // Sort by mtime ascending (oldest first) const withMtime = idle.map((p) => { try { return { path: p, mtime: statSync(p).mtimeMs }; } catch { return { path: p, mtime: 0 }; } }); withMtime.sort((a, b) => a.mtime - b.mtime); let removed = 0; const excess = dirs.length - cap; for (const { path: worktreePath } of withMtime) { if (removed >= excess) break; try { execSync(`git worktree remove "${worktreePath}" --force`, { cwd: this.options.rootDir, stdio: "pipe", timeout: 30_000, }); removed++; } catch { // Individual failure is non-fatal } } if (removed > 0) { log.warn(`Worktree cap: removed ${removed} idle worktree(s) (was ${dirs.length}, cap ${cap})`); } } catch (err: any) { log.error(`Worktree cap enforcement failed: ${err.message}`); } } }