feat: add SelfHealingManager for unattended multi-day operation

Adds four self-healing subsystems to enable the engine to recover from
common failure modes without human intervention:

- Auto-unpause: clears rate-limit-triggered globalPause with escalating
  backoff (5 min → 60 min cap), resets on sustained recovery
- Stuck kill budget: caps task stuck-kill retries (default 3) to prevent
  infinite stuck→todo→stuck loops
- Periodic maintenance (every 15 min): git worktree prune, orphan cleanup,
  SQLite WAL checkpoint
- Worktree cap enforcement: removes oldest idle worktrees when count
  exceeds 2× maxWorktrees

New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs,
autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs.
New task field: stuckKillCount (schema v8 migration).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-03 15:35:12 -07:00
parent 6a38b001ba
commit fdf6d0b4a1
11 changed files with 842 additions and 18 deletions

View File

@@ -4,7 +4,7 @@ import { createInterface } from "node:readline";
import { TaskStore, AutomationStore } from "@fusion/core";
import type { Settings, TaskDetail, PrInfo } from "@fusion/core";
import { createServer, GitHubClient } from "@fusion/dashboard";
import { TriageProcessor, TaskExecutor, Scheduler, AgentSemaphore, WorktreePool, aiMergeTask, UsageLimitPauser, PRIORITY_MERGE, scanIdleWorktrees, cleanupOrphanedWorktrees, NtfyNotifier, PrMonitor, PrCommentHandler, CronRunner, StuckTaskDetector } from "@fusion/engine";
import { TriageProcessor, TaskExecutor, Scheduler, AgentSemaphore, WorktreePool, aiMergeTask, UsageLimitPauser, PRIORITY_MERGE, scanIdleWorktrees, cleanupOrphanedWorktrees, NtfyNotifier, PrMonitor, PrCommentHandler, CronRunner, StuckTaskDetector, SelfHealingManager } from "@fusion/engine";
import { AuthStorage, DefaultPackageManager, ModelRegistry, SettingsManager, discoverAndLoadExtensions, getAgentDir, createExtensionRuntime } from "@mariozechner/pi-coding-agent";
/**
@@ -506,12 +506,16 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
onSpecifyError: (t, e) => console.log(`[engine] ✗ ${t.id}: ${e.message}`),
});
// ── Self-healing: auto-unpause, stuck kill budgets, maintenance ─────
const selfHealing = new SelfHealingManager(store, { rootDir: cwd });
// ── Stuck task detector: monitors agent sessions for stagnation ────
// Created before the executor so it can be passed in options.
// The onStuck callback is wired to executor.markStuckAborted after
// executor creation (late-binding via closure on executorRef).
const executorRef: { current: TaskExecutor | null } = { current: null };
const stuckTaskDetector = new StuckTaskDetector(store, {
beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId),
onStuck: (taskId) => {
executorRef.current?.markStuckAborted(taskId);
console.log(`[engine] ⚠ ${taskId} stuck — terminated, will retry`);
@@ -550,6 +554,7 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
triage.start();
scheduler.start();
stuckTaskDetector.start();
selfHealing.start();
// ── Startup sweep: resume orphaned in-progress tasks ──────────────
executor.resumeOrphaned().catch((err) =>
@@ -660,6 +665,7 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
scheduleMergeRetry();
const shutdown = () => {
selfHealing.stop();
stuckTaskDetector.stop();
triage.stop();
scheduler.stop();