feat: add SelfHealingManager for unattended multi-day operation

Adds four self-healing subsystems to enable the engine to recover from common failure modes without human intervention: - Auto-unpause: clears rate-limit-triggered globalPause with escalating backoff (5 min → 60 min cap), resets on sustained recovery - Stuck kill budget: caps task stuck-kill retries (default 3) to prevent infinite stuck→todo→stuck loops - Periodic maintenance (every 15 min): git worktree prune, orphan cleanup, SQLite WAL checkpoint - Worktree cap enforcement: removes oldest idle worktrees when count exceeds 2× maxWorktrees New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs, autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs. New task field: stuckKillCount (schema v8 migration). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-03 15:35:12 -07:00
parent 6a38b001ba
commit fdf6d0b4a1
11 changed files with 842 additions and 18 deletions
--- a/packages/cli/src/commands/dashboard.ts
+++ b/packages/cli/src/commands/dashboard.ts
@@ -4,7 +4,7 @@ import { createInterface } from "node:readline";
 import { TaskStore, AutomationStore } from "@fusion/core";
 import type { Settings, TaskDetail, PrInfo } from "@fusion/core";
 import { createServer, GitHubClient } from "@fusion/dashboard";
-import { TriageProcessor, TaskExecutor, Scheduler, AgentSemaphore, WorktreePool, aiMergeTask, UsageLimitPauser, PRIORITY_MERGE, scanIdleWorktrees, cleanupOrphanedWorktrees, NtfyNotifier, PrMonitor, PrCommentHandler, CronRunner, StuckTaskDetector } from "@fusion/engine";
+import { TriageProcessor, TaskExecutor, Scheduler, AgentSemaphore, WorktreePool, aiMergeTask, UsageLimitPauser, PRIORITY_MERGE, scanIdleWorktrees, cleanupOrphanedWorktrees, NtfyNotifier, PrMonitor, PrCommentHandler, CronRunner, StuckTaskDetector, SelfHealingManager } from "@fusion/engine";
 import { AuthStorage, DefaultPackageManager, ModelRegistry, SettingsManager, discoverAndLoadExtensions, getAgentDir, createExtensionRuntime } from "@mariozechner/pi-coding-agent";

 /**
@@ -506,12 +506,16 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
      onSpecifyError: (t, e) => console.log(`[engine] ✗ ${t.id}: ${e.message}`),
    });

+    // ── Self-healing: auto-unpause, stuck kill budgets, maintenance ─────
+    const selfHealing = new SelfHealingManager(store, { rootDir: cwd });
+
    // ── Stuck task detector: monitors agent sessions for stagnation ────
    // Created before the executor so it can be passed in options.
    // The onStuck callback is wired to executor.markStuckAborted after
    // executor creation (late-binding via closure on executorRef).
    const executorRef: { current: TaskExecutor | null } = { current: null };
    const stuckTaskDetector = new StuckTaskDetector(store, {
+      beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId),
      onStuck: (taskId) => {
        executorRef.current?.markStuckAborted(taskId);
        console.log(`[engine] ⚠ ${taskId} stuck — terminated, will retry`);
@@ -550,6 +554,7 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
    triage.start();
    scheduler.start();
    stuckTaskDetector.start();
+    selfHealing.start();

    // ── Startup sweep: resume orphaned in-progress tasks ──────────────
    executor.resumeOrphaned().catch((err) =>
@@ -660,6 +665,7 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
    scheduleMergeRetry();

    const shutdown = () => {
+      selfHealing.stop();
      stuckTaskDetector.stop();
      triage.stop();
      scheduler.stop();
--- a/packages/core/src/db.test.ts
+++ b/packages/core/src/db.test.ts
@@ -86,7 +86,7 @@ describe("Database", () => {
    });

    it("seeds schema version", () => {
-      expect(db.getSchemaVersion()).toBe(7);
+      expect(db.getSchemaVersion()).toBe(8);
    });

    it("seeds lastModified", () => {
@@ -109,7 +109,7 @@ describe("Database", () => {

    it("is idempotent - calling init() twice does not fail", () => {
      expect(() => db.init()).not.toThrow();
-      expect(db.getSchemaVersion()).toBe(7);
+      expect(db.getSchemaVersion()).toBe(8);
    });

    it("does not overwrite existing config on re-init", () => {
@@ -188,6 +188,18 @@ describe("Database", () => {
    });
  });

+  describe("walCheckpoint", () => {
+    it("runs WAL checkpoint and returns stats", () => {
+      const result = db.walCheckpoint();
+      expect(result).toHaveProperty("busy");
+      expect(result).toHaveProperty("log");
+      expect(result).toHaveProperty("checkpointed");
+      expect(typeof result.busy).toBe("number");
+      expect(typeof result.log).toBe("number");
+      expect(typeof result.checkpointed).toBe("number");
+    });
+  });
+
  describe("transactions", () => {
    it("commits on success", () => {
      db.transaction(() => {
@@ -704,7 +716,7 @@ describe("schema migrations", () => {
    db.init();

    // Verify version bumped to 5 (includes v1→v2, v2→v3, v3→v4, and v4→v5 migrations)
-    expect(db.getSchemaVersion()).toBe(7);
+    expect(db.getSchemaVersion()).toBe(8);

    // Verify new columns exist and existing data is intact
    const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -729,11 +741,11 @@ describe("schema migrations", () => {
    const db = new Database(kbDir);
    db.init();

-    expect(db.getSchemaVersion()).toBe(7);
+    expect(db.getSchemaVersion()).toBe(8);

    // Re-init should not fail
    db.init();
-    expect(db.getSchemaVersion()).toBe(7);
+    expect(db.getSchemaVersion()).toBe(8);

    db.close();
  });
@@ -828,7 +840,7 @@ describe("schema migrations", () => {
    db.init();

    // Verify version bumped to 5
-    expect(db.getSchemaVersion()).toBe(7);
+    expect(db.getSchemaVersion()).toBe(8);

    // Verify new columns exist and existing data is intact
    const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1038,7 +1050,7 @@ describe("createDatabase factory", () => {
    const db = createDatabase(kbDir);
    db.init();

-    expect(db.getSchemaVersion()).toBe(7);
+    expect(db.getSchemaVersion()).toBe(8);
    expect(db.getLastModified()).toBeGreaterThan(0);

    db.close();
--- a/packages/core/src/db.ts
+++ b/packages/core/src/db.ts
@@ -59,7 +59,7 @@ export function fromJson<T>(json: string | null | undefined): T | undefined {

 // ── Schema Definition ────────────────────────────────────────────────

-const SCHEMA_VERSION = 7;
+const SCHEMA_VERSION = 8;

 function normalizeTaskComments(
  steeringComments: SteeringComment[] | undefined,
@@ -418,8 +418,14 @@ export class Database {
      });
    }

+    if (version < 8) {
+      this.applyMigration(8, () => {
+        this.addColumnIfMissing("tasks", "stuckKillCount", "INTEGER DEFAULT 0");
+      });
+    }
+
    // Future migrations go here:
-    // if (version < 8) { this.applyMigration(8, () => { ... }); }
+    // if (version < 9) { this.applyMigration(9, () => { ... }); }
  }

  /**
@@ -486,6 +492,15 @@ export class Database {
    }
  }

+  /**
+   * Run a WAL checkpoint to truncate the WAL file and reclaim disk space.
+   * Safe to call periodically. Returns checkpoint stats.
+   */
+  walCheckpoint(): { busy: number; log: number; checkpointed: number } {
+    const row = this.db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get() as any;
+    return { busy: row?.busy ?? 0, log: row?.log ?? 0, checkpointed: row?.checkpointed ?? 0 };
+  }
+
  /**
   * Close the database connection.
   */
--- a/packages/core/src/store.ts
+++ b/packages/core/src/store.ts
@@ -178,6 +178,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
      validatorModelProvider: row.validatorModelProvider || undefined,
      validatorModelId: row.validatorModelId || undefined,
      mergeRetries: row.mergeRetries ?? undefined,
+      stuckKillCount: row.stuckKillCount ?? undefined,
      recoveryRetryCount: row.recoveryRetryCount ?? undefined,
      nextRecoveryAt: row.nextRecoveryAt || undefined,
      error: row.error || undefined,
@@ -228,14 +229,14 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
        id, title, description, "column", status, size, reviewLevel, currentStep,
        worktree, blockedBy, paused, baseBranch, branch, baseCommitSha, modelPresetId, modelProvider,
        modelId, validatorModelProvider, validatorModelId, mergeRetries,
-        recoveryRetryCount, nextRecoveryAt, error,
+        stuckKillCount, recoveryRetryCount, nextRecoveryAt, error,
        summary, thinkingLevel, createdAt, updatedAt, columnMovedAt,
        dependencies, steps, log, attachments, steeringComments,
        comments, workflowStepResults, prInfo, issueInfo, mergeDetails,
        breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, missionId, sliceId
      ) VALUES (
        ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
-        ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+        ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
      )
    `).run(
      task.id,
@@ -258,6 +259,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
      task.validatorModelProvider ?? null,
      task.validatorModelId ?? null,
      task.mergeRetries ?? null,
+      task.stuckKillCount ?? 0,
      task.recoveryRetryCount ?? null,
      task.nextRecoveryAt ?? null,
      task.error ?? null,
@@ -975,7 +977,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {

  async updateTask(
    id: string,
-    updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; recoveryRetryCount?: number | null; nextRecoveryAt?: string | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
+    updates: { title?: string; description?: string; prompt?: string; worktree?: string; status?: string | null; dependencies?: string[]; blockedBy?: string | null; paused?: boolean; baseBranch?: string; branch?: string; baseCommitSha?: string; size?: "S" | "M" | "L"; reviewLevel?: number; mergeRetries?: number; stuckKillCount?: number | null; recoveryRetryCount?: number | null; nextRecoveryAt?: string | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; error?: string | null; summary?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
  ): Promise<Task> {
    return this.withTaskLock(id, async () => {
      // Validate that task doesn't depend on itself
@@ -1030,6 +1032,11 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
      if (updates.size !== undefined) task.size = updates.size;
      if (updates.reviewLevel !== undefined) task.reviewLevel = updates.reviewLevel;
      if (updates.mergeRetries !== undefined) task.mergeRetries = updates.mergeRetries;
+      if (updates.stuckKillCount === null) {
+        task.stuckKillCount = undefined;
+      } else if (updates.stuckKillCount !== undefined) {
+        task.stuckKillCount = updates.stuckKillCount;
+      }
      if (updates.recoveryRetryCount === null) {
        task.recoveryRetryCount = undefined;
      } else if (updates.recoveryRetryCount !== undefined) {
@@ -2715,6 +2722,14 @@ ${stepsSection}`;
    }
  }

+  /**
+   * Run a WAL checkpoint to truncate the WAL file and reclaim disk space.
+   * Safe to call periodically from the self-healing maintenance timer.
+   */
+  walCheckpoint(): { busy: number; log: number; checkpointed: number } {
+    return this.db.walCheckpoint();
+  }
+
  getRootDir(): string {
    return this.rootDir;
  }
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -475,6 +475,10 @@ export interface Task {
  workflowStepResults?: WorkflowStepResult[];
  /** Number of merge retry attempts made for this task (auto-merge conflict recovery) */
  mergeRetries?: number;
+  /** Number of times the stuck-task detector has killed this task's agent session.
+   *  Incremented by the self-healing manager on each stuck kill. When this reaches
+   *  `maxStuckKills`, the task is marked as permanently failed instead of re-queued. */
+  stuckKillCount?: number;
  /** Number of bounded recovery retry attempts for transient executor/triage failures.
   *  Distinct from `mergeRetries` (merge-conflict-specific). Incremented by the
   *  recovery-policy module on each recoverable failure; cleared when work restarts
@@ -743,6 +747,21 @@ export interface ProjectSettings {
   *  than this duration, the task is considered stuck and will be terminated and retried.
   *  Default: undefined (disabled). Suggested value: 600000 (10 minutes). */
  taskStuckTimeoutMs?: number;
+  /** When true, automatically unpause after rate-limit-triggered globalPause using
+   *  escalating backoff. Allows unattended recovery from transient API rate limits.
+   *  Default: true. */
+  autoUnpauseEnabled?: boolean;
+  /** Base delay in milliseconds before first auto-unpause attempt after rate-limit pause.
+   *  Subsequent attempts use exponential backoff (2x). Default: 300000 (5 min). */
+  autoUnpauseBaseDelayMs?: number;
+  /** Maximum delay cap in milliseconds for auto-unpause backoff. Default: 3600000 (60 min). */
+  autoUnpauseMaxDelayMs?: number;
+  /** Maximum number of times the stuck-task detector can kill and re-queue a task
+   *  before it is marked as permanently failed. Default: 3. */
+  maxStuckKills?: number;
+  /** Interval in milliseconds for periodic maintenance (worktree pruning, WAL checkpoint,
+   *  orphan cleanup). 0 disables. Default: 900000 (15 min). */
+  maintenanceIntervalMs?: number;
  /** When true, automatically poll and update PR status badges for tasks linked to GitHub PRs.
   *  Default: false. */
  autoUpdatePrStatus?: boolean;
@@ -845,6 +864,11 @@ export const DEFAULT_PROJECT_SETTINGS: ProjectSettings = {
  smartConflictResolution: true,
  requirePlanApproval: false,
  taskStuckTimeoutMs: undefined,
+  autoUnpauseEnabled: true,
+  autoUnpauseBaseDelayMs: 300_000,
+  autoUnpauseMaxDelayMs: 3_600_000,
+  maxStuckKills: 3,
+  maintenanceIntervalMs: 900_000,
  autoUpdatePrStatus: false,
  autoCreatePr: false,
  autoBackupEnabled: false,
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -15,6 +15,7 @@ export { PrCommentHandler } from "./pr-comment-handler.js";
 export { NtfyNotifier, type NtfyNotifierOptions } from "./notifier.js";
 export { CronRunner, type CronRunnerOptions } from "./cron-runner.js";
 export { StuckTaskDetector, type StuckTaskDetectorOptions, type DisposableSession } from "./stuck-task-detector.js";
+export { SelfHealingManager, type SelfHealingOptions } from "./self-healing.js";
 export { ProjectManager } from "./project-manager.js";
 // Multi-project runtime types
 export {
--- a/packages/engine/src/runtimes/in-process-runtime.ts
+++ b/packages/engine/src/runtimes/in-process-runtime.ts
@@ -20,6 +20,7 @@ import type {
 import { runtimeLog } from "../logger.js";
 import type { StuckTaskDetector } from "../stuck-task-detector.js";
 import type { UsageLimitPauser } from "../usage-limit-detector.js";
+import { SelfHealingManager } from "../self-healing.js";

 /**
 * InProcessRuntime runs a project within the main process.
@@ -65,6 +66,7 @@ export class InProcessRuntime
  private globalSemaphore?: AgentSemaphore;
  private stuckTaskDetector?: StuckTaskDetector;
  private usageLimitPauser?: UsageLimitPauser;
+  private selfHealingManager?: SelfHealingManager;
  private agentStore?: AgentStore;
  private heartbeatMonitor?: HeartbeatMonitor;
  /** Maps task IDs to agent IDs for lifecycle tracking */
@@ -225,13 +227,19 @@ export class InProcessRuntime
        runtimeLog.warn(`AgentStore initialization failed (continuing without agent monitoring):`, agentErr);
      }

-      // 7. Set up event forwarding from TaskStore
+      // 7. Initialize SelfHealingManager
+      this.selfHealingManager = new SelfHealingManager(this.taskStore, {
+        rootDir: this.config.workingDirectory,
+      });
+      this.selfHealingManager.start();
+
+      // 8. Set up event forwarding from TaskStore
      this.setupEventForwarding();

-      // 8. Resume orphaned in-progress tasks
+      // 9. Resume orphaned in-progress tasks
      await this.executor.resumeOrphaned();

-      // 9. Start scheduler
+      // 10. Start scheduler
      this.scheduler.start();

      this.setStatus("active");
@@ -266,13 +274,19 @@ export class InProcessRuntime
    runtimeLog.log(`Stopping InProcessRuntime for project ${this.config.projectId}`);

    try {
-      // 1. Stop heartbeat monitor
+      // 1. Stop self-healing manager
+      if (this.selfHealingManager) {
+        this.selfHealingManager.stop();
+        runtimeLog.log("SelfHealingManager stopped");
+      }
+
+      // 2. Stop heartbeat monitor
      if (this.heartbeatMonitor) {
        this.heartbeatMonitor.stop();
        runtimeLog.log("HeartbeatMonitor stopped");
      }

-      // 2. Stop scheduler (prevents new task scheduling)
+      // 3. Stop scheduler (prevents new task scheduling)
      if (this.scheduler) {
        this.scheduler.stop();
        runtimeLog.log("Scheduler stopped");
--- a/packages/engine/src/self-healing.test.ts
+++ b/packages/engine/src/self-healing.test.ts
@@ -0,0 +1,294 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { SelfHealingManager } from "./self-healing.js";
+import type { TaskStore, Settings, Task } from "@fusion/core";
+import { EventEmitter } from "node:events";
+
+// ── Mock helpers ────────────────────────────────────────────────────
+
+/** TaskStore mock backed by a real EventEmitter so settings:updated works. */
+function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
+  const emitter = new EventEmitter();
+  const store = Object.assign(emitter, {
+    getSettings: vi.fn().mockResolvedValue({
+      autoUnpauseEnabled: true,
+      autoUnpauseBaseDelayMs: 100,
+      autoUnpauseMaxDelayMs: 800,
+      maxStuckKills: 3,
+      maintenanceIntervalMs: 0,
+      maxWorktrees: 4,
+      globalPause: true, // default: paused (for auto-unpause tests)
+    } as unknown as Settings),
+    updateSettings: vi.fn().mockResolvedValue({} as Settings),
+    getTask: vi.fn().mockResolvedValue({
+      id: "FN-001",
+      stuckKillCount: 0,
+    } as unknown as Task),
+    updateTask: vi.fn().mockResolvedValue({} as Task),
+    logEntry: vi.fn().mockResolvedValue(undefined),
+    moveTask: vi.fn().mockResolvedValue(undefined),
+    walCheckpoint: vi.fn().mockReturnValue({ busy: 0, log: 5, checkpointed: 5 }),
+    listTasks: vi.fn().mockResolvedValue([]),
+    getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
+    ...overrides,
+  }) as unknown as TaskStore & EventEmitter;
+  return store;
+}
+
+describe("SelfHealingManager", () => {
+  let store: TaskStore & EventEmitter;
+  let manager: SelfHealingManager;
+
+  beforeEach(() => {
+    vi.useFakeTimers({ shouldAdvanceTime: true });
+    store = createMockStore();
+    manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
+  });
+
+  afterEach(() => {
+    manager.stop();
+    vi.useRealTimers();
+  });
+
+  // ── Auto-unpause ─────────────────────────────────────────────────
+
+  describe("auto-unpause", () => {
+    it("schedules unpause when globalPause transitions false→true", async () => {
+      manager.start();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      await vi.advanceTimersByTimeAsync(150);
+
+      expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: false });
+    });
+
+    it("does not schedule unpause when autoUnpauseEnabled is false", async () => {
+      manager.start();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: false },
+        previous: { globalPause: false },
+      });
+
+      await vi.advanceTimersByTimeAsync(500);
+
+      expect(store.updateSettings).not.toHaveBeenCalled();
+    });
+
+    it("does not fire when already unpaused before timer", async () => {
+      // When the timer fires, getSettings returns globalPause: false
+      (store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
+        globalPause: false,
+        maintenanceIntervalMs: 0,
+      } as unknown as Settings);
+
+      manager.start();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      await vi.advanceTimersByTimeAsync(150);
+
+      expect(store.updateSettings).not.toHaveBeenCalled();
+    });
+
+    it("escalates backoff when pause re-triggers within 60s", async () => {
+      manager.start();
+
+      // First pause
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      await vi.advanceTimersByTimeAsync(150);
+      expect(store.updateSettings).toHaveBeenCalledTimes(1);
+
+      // Simulate successful unpause
+      store.emit("settings:updated", {
+        settings: { globalPause: false },
+        previous: { globalPause: true },
+      });
+
+      // Immediately re-trigger pause (within 60s window)
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      // Escalated delay = 200ms. At 150ms it should NOT have fired yet.
+      await vi.advanceTimersByTimeAsync(150);
+      expect(store.updateSettings).toHaveBeenCalledTimes(1);
+
+      // At 250ms total (100ms more) it should fire
+      await vi.advanceTimersByTimeAsync(100);
+      expect(store.updateSettings).toHaveBeenCalledTimes(2);
+    });
+
+    it("cancels timer on manual unpause (true→false)", async () => {
+      manager.start();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 200, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      // Manual unpause before timer fires
+      store.emit("settings:updated", {
+        settings: { globalPause: false },
+        previous: { globalPause: true },
+      });
+
+      await vi.advanceTimersByTimeAsync(300);
+
+      expect(store.updateSettings).not.toHaveBeenCalled();
+    });
+
+    it("ignores false→false transitions", async () => {
+      manager.start();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: false },
+        previous: { globalPause: false },
+      });
+
+      await vi.advanceTimersByTimeAsync(500);
+
+      expect(store.updateSettings).not.toHaveBeenCalled();
+    });
+  });
+
+  // ── Stuck kill budget ─────────────────────────────────────────────
+
+  describe("checkStuckBudget", () => {
+    it("returns true and increments count when within budget", async () => {
+      manager.start();
+
+      const result = await manager.checkStuckBudget("FN-001");
+
+      expect(result).toBe(true);
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 1 });
+      expect(store.logEntry).toHaveBeenCalledWith(
+        "FN-001",
+        expect.stringContaining("Stuck kill 1/3"),
+      );
+    });
+
+    it("returns true for subsequent kills within budget", async () => {
+      (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
+        id: "FN-001",
+        stuckKillCount: 2,
+      } as unknown as Task);
+
+      manager.start();
+
+      const result = await manager.checkStuckBudget("FN-001");
+
+      expect(result).toBe(true);
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 3 });
+    });
+
+    it("returns false and marks failed when budget exceeded", async () => {
+      (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
+        id: "FN-001",
+        stuckKillCount: 3,
+      } as unknown as Task);
+
+      manager.start();
+
+      const result = await manager.checkStuckBudget("FN-001");
+
+      expect(result).toBe(false);
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", {
+        stuckKillCount: 4,
+        status: "failed",
+        error: expect.stringContaining("exceeded maximum of 3"),
+      });
+      expect(store.logEntry).toHaveBeenCalledWith(
+        "FN-001",
+        expect.stringContaining("Permanently failed"),
+      );
+    });
+
+    it("respects custom maxStuckKills setting", async () => {
+      (store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
+        maxStuckKills: 1,
+        maintenanceIntervalMs: 0,
+      } as unknown as Settings);
+      (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
+        id: "FN-001",
+        stuckKillCount: 1,
+      } as unknown as Task);
+
+      manager.start();
+
+      const result = await manager.checkStuckBudget("FN-001");
+
+      expect(result).toBe(false);
+    });
+
+    it("returns true on error (safe fallback)", async () => {
+      (store.getTask as ReturnType<typeof vi.fn>).mockRejectedValue(new Error("DB error"));
+
+      manager.start();
+
+      const result = await manager.checkStuckBudget("FN-001");
+
+      expect(result).toBe(true);
+    });
+
+    it("handles undefined stuckKillCount as 0", async () => {
+      (store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
+        id: "FN-001",
+      } as unknown as Task);
+
+      manager.start();
+
+      const result = await manager.checkStuckBudget("FN-001");
+
+      expect(result).toBe(true);
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 1 });
+    });
+  });
+
+  // ── Lifecycle ─────────────────────────────────────────────────────
+
+  describe("lifecycle", () => {
+    it("starts and stops without error", () => {
+      manager.start();
+      manager.stop();
+    });
+
+    it("cleans up timers on stop", async () => {
+      manager.start();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 500, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      manager.stop();
+
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(store.updateSettings).not.toHaveBeenCalled();
+    });
+
+    it("does not respond to events after stop", async () => {
+      manager.start();
+      manager.stop();
+
+      store.emit("settings:updated", {
+        settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
+        previous: { globalPause: false },
+      });
+
+      await vi.advanceTimersByTimeAsync(200);
+      expect(store.updateSettings).not.toHaveBeenCalled();
+    });
+  });
+});
--- a/packages/engine/src/self-healing.ts
+++ b/packages/engine/src/self-healing.ts
@@ -0,0 +1,360 @@
+/**
+ * SelfHealingManager — enables unattended multi-day/week operation by
+ * providing automatic recovery from common failure modes.
+ *
+ * Four subsystems:
+ * 1. **Auto-unpause**: Clears rate-limit-triggered `globalPause` with
+ *    escalating backoff (5 min → 60 min cap). Resets on sustained unpause.
+ * 2. **Stuck kill budget**: Caps how many times a task can be killed by the
+ *    stuck-task detector before marking it as permanently failed.
+ * 3. **Periodic maintenance**: Worktree pruning, orphan cleanup, SQLite
+ *    WAL checkpoint — all on a configurable interval (default 15 min).
+ * 4. **Worktree cap enforcement**: Prevents unbounded worktree accumulation
+ *    by cleaning oldest idle worktrees when count exceeds 2× maxWorktrees.
+ */
+
+import { execSync } from "node:child_process";
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import type { TaskStore, Settings } from "@fusion/core";
+import { createLogger } from "./logger.js";
+import { scanIdleWorktrees } from "./worktree-pool.js";
+
+const log = createLogger("self-healing");
+
+export interface SelfHealingOptions {
+  /** Project root directory (parent of .worktrees/) */
+  rootDir: string;
+}
+
+export class SelfHealingManager {
+  // ── Auto-unpause state ──────────────────────────────────────────────
+  private unpauseTimer: ReturnType<typeof setTimeout> | null = null;
+  private unpauseAttempt = 0;
+  private lastPauseTriggeredAt = 0;
+  private lastUnpauseAt = 0;
+
+  // ── Maintenance timer ───────────────────────────────────────────────
+  private maintenanceInterval: ReturnType<typeof setInterval> | null = null;
+
+  // ── Event listener cleanup ──────────────────────────────────────────
+  private settingsListener: ((data: { settings: Settings; previous: Settings }) => void) | null = null;
+
+  constructor(
+    private store: TaskStore,
+    private options: SelfHealingOptions,
+  ) {}
+
+  // ── Lifecycle ───────────────────────────────────────────────────────
+
+  start(): void {
+    // Wire up settings:updated listener for auto-unpause
+    this.settingsListener = ({ settings, previous }) => {
+      this.onSettingsUpdated(settings, previous);
+    };
+    this.store.on("settings:updated", this.settingsListener);
+
+    // Start periodic maintenance
+    this.startMaintenance();
+
+    log.log("Started");
+  }
+
+  stop(): void {
+    // Remove settings listener
+    if (this.settingsListener) {
+      try {
+        this.store.removeListener("settings:updated", this.settingsListener);
+      } catch {
+        // Store may not support removeListener (e.g., test mocks)
+      }
+      this.settingsListener = null;
+    }
+
+    // Clear timers
+    this.cancelUnpauseTimer();
+    if (this.maintenanceInterval) {
+      clearInterval(this.maintenanceInterval);
+      this.maintenanceInterval = null;
+    }
+
+    log.log("Stopped");
+  }
+
+  // ── Auto-unpause ───────────────────────────────────────────────────
+
+  private onSettingsUpdated(settings: Settings, previous: Settings): void {
+    // globalPause false → true: schedule auto-unpause
+    if (!previous.globalPause && settings.globalPause) {
+      if (!settings.autoUnpauseEnabled) {
+        log.log("Global pause activated — auto-unpause disabled, requires manual intervention");
+        return;
+      }
+
+      // If pause re-triggered within 60s of our last unpause, escalate backoff
+      if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
+        this.unpauseAttempt++;
+        log.warn(`Global pause re-triggered within 60s — escalating to attempt ${this.unpauseAttempt}`);
+      }
+
+      this.lastPauseTriggeredAt = Date.now();
+
+      const baseDelay = settings.autoUnpauseBaseDelayMs ?? 300_000;
+      const maxDelay = settings.autoUnpauseMaxDelayMs ?? 3_600_000;
+      const delay = Math.min(baseDelay * Math.pow(2, this.unpauseAttempt), maxDelay);
+
+      this.scheduleUnpause(delay);
+    }
+
+    // globalPause true → false: check if we should reset backoff
+    if (previous.globalPause && !settings.globalPause) {
+      this.cancelUnpauseTimer();
+
+      // If sustained unpause (not a quick re-trigger), reset attempt counter
+      if (this.lastPauseTriggeredAt && (Date.now() - this.lastPauseTriggeredAt) > 60_000) {
+        this.unpauseAttempt = 0;
+      }
+    }
+  }
+
+  private scheduleUnpause(delayMs: number): void {
+    this.cancelUnpauseTimer();
+
+    const delaySec = Math.round(delayMs / 1000);
+    const delayMin = Math.round(delaySec / 60);
+    const display = delayMin >= 1 ? `${delayMin}m` : `${delaySec}s`;
+    log.warn(`Auto-unpause scheduled in ${display} (attempt ${this.unpauseAttempt + 1})`);
+
+    this.unpauseTimer = setTimeout(() => {
+      this.unpauseTimer = null;
+      void this.attemptUnpause();
+    }, delayMs);
+  }
+
+  private async attemptUnpause(): Promise<void> {
+    try {
+      const settings = await this.store.getSettings();
+
+      // Already unpaused (manually or by another mechanism)
+      if (!settings.globalPause) {
+        log.log("Auto-unpause: already unpaused — no action needed");
+        this.unpauseAttempt = 0;
+        return;
+      }
+
+      log.warn("Auto-unpause: clearing globalPause");
+      this.lastUnpauseAt = Date.now();
+      await this.store.updateSettings({ globalPause: false });
+
+      // Note: if the rate limit is still active, the next agent session will
+      // hit it again → UsageLimitPauser triggers globalPause → our listener
+      // catches the transition and schedules the next attempt with escalated backoff.
+    } catch (err: any) {
+      log.error(`Auto-unpause failed: ${err.message}`);
+    }
+  }
+
+  private cancelUnpauseTimer(): void {
+    if (this.unpauseTimer) {
+      clearTimeout(this.unpauseTimer);
+      this.unpauseTimer = null;
+    }
+  }
+
+  // ── Stuck kill budget ─────────────────────────────────────────────
+
+  /**
+   * Check whether a stuck-killed task should be re-queued or marked as failed.
+   * Called by StuckTaskDetector's `beforeRequeue` callback.
+   *
+   * @returns `true` if the task should be re-queued, `false` if budget exhausted
+   *          (task has been marked as permanently failed).
+   */
+  async checkStuckBudget(taskId: string): Promise<boolean> {
+    try {
+      const settings = await this.store.getSettings();
+      const maxKills = settings.maxStuckKills ?? 3;
+
+      const task = await this.store.getTask(taskId);
+      const newCount = (task.stuckKillCount ?? 0) + 1;
+
+      if (newCount > maxKills) {
+        // Budget exhausted — mark as permanently failed
+        log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}) — marking failed`);
+        await this.store.updateTask(taskId, {
+          stuckKillCount: newCount,
+          status: "failed",
+          error: `Task stuck ${newCount} times — exceeded maximum of ${maxKills} stuck kills`,
+        });
+        await this.store.logEntry(
+          taskId,
+          `Permanently failed: agent stuck ${newCount} times (max: ${maxKills})`,
+        );
+        return false;
+      }
+
+      // Budget remaining — allow re-queue
+      log.log(`${taskId} stuck kill ${newCount}/${maxKills} — will re-queue`);
+      await this.store.updateTask(taskId, { stuckKillCount: newCount });
+      await this.store.logEntry(
+        taskId,
+        `Stuck kill ${newCount}/${maxKills} — re-queuing for retry`,
+      );
+      return true;
+    } catch (err: any) {
+      log.error(`checkStuckBudget failed for ${taskId}: ${err.message}`);
+      // On error, allow re-queue — safer than permanently failing
+      return true;
+    }
+  }
+
+  // ── Periodic maintenance ──────────────────────────────────────────
+
+  private async startMaintenance(): Promise<void> {
+    const settings = await this.store.getSettings();
+    const intervalMs = settings.maintenanceIntervalMs ?? 900_000;
+
+    if (intervalMs <= 0) {
+      log.log("Periodic maintenance disabled (maintenanceIntervalMs <= 0)");
+      return;
+    }
+
+    log.log(`Periodic maintenance every ${Math.round(intervalMs / 60_000)}m`);
+    this.maintenanceInterval = setInterval(() => {
+      void this.runMaintenance();
+    }, intervalMs);
+  }
+
+  private async runMaintenance(): Promise<void> {
+    const startMs = Date.now();
+    log.log("Maintenance cycle starting");
+
+    try {
+      await this.pruneWorktrees();
+      await this.cleanupOrphans();
+      this.checkpointWal();
+      await this.enforceWorktreeCap();
+
+      const elapsedMs = Date.now() - startMs;
+      log.log(`Maintenance cycle completed in ${elapsedMs}ms`);
+    } catch (err: any) {
+      log.error(`Maintenance cycle failed: ${err.message}`);
+    }
+  }
+
+  /** Run `git worktree prune` to clean stale metadata. */
+  private async pruneWorktrees(): Promise<void> {
+    try {
+      execSync("git worktree prune", {
+        cwd: this.options.rootDir,
+        stdio: "pipe",
+        timeout: 30_000,
+      });
+      log.log("Worktree prune completed");
+    } catch (err: any) {
+      log.error(`Worktree prune failed: ${err.message}`);
+    }
+  }
+
+  /** Remove orphaned worktrees not assigned to any active task. */
+  private async cleanupOrphans(): Promise<number> {
+    try {
+      const orphaned = await scanIdleWorktrees(this.options.rootDir, this.store);
+      if (orphaned.length === 0) return 0;
+
+      // Only clean up if recycling is disabled — otherwise they belong in the pool
+      const settings = await this.store.getSettings();
+      if (settings.recycleWorktrees) {
+        return 0;
+      }
+
+      let cleaned = 0;
+      for (const worktreePath of orphaned) {
+        try {
+          execSync(`git worktree remove "${worktreePath}" --force`, {
+            cwd: this.options.rootDir,
+            stdio: "pipe",
+            timeout: 30_000,
+          });
+          cleaned++;
+        } catch {
+          // Individual failure is non-fatal
+        }
+      }
+
+      if (cleaned > 0) {
+        log.log(`Cleaned ${cleaned} orphaned worktree(s)`);
+      }
+      return cleaned;
+    } catch (err: any) {
+      log.error(`Orphan cleanup failed: ${err.message}`);
+      return 0;
+    }
+  }
+
+  /** Run SQLite WAL checkpoint to reclaim disk space. */
+  private checkpointWal(): void {
+    try {
+      const result = this.store.walCheckpoint();
+      if (result.log > 0) {
+        log.log(`WAL checkpoint: ${result.checkpointed}/${result.log} pages checkpointed` +
+          (result.busy > 0 ? ` (${result.busy} busy)` : ""));
+      }
+    } catch (err: any) {
+      log.error(`WAL checkpoint failed: ${err.message}`);
+    }
+  }
+
+  /** Remove oldest idle worktrees if total count exceeds 2× maxWorktrees. */
+  private async enforceWorktreeCap(): Promise<void> {
+    const worktreesDir = join(this.options.rootDir, ".worktrees");
+    if (!existsSync(worktreesDir)) return;
+
+    try {
+      const settings = await this.store.getSettings();
+      const cap = (settings.maxWorktrees ?? 4) * 2;
+
+      const entries = readdirSync(worktreesDir, { withFileTypes: true });
+      const dirs = entries.filter((e) => e.isDirectory());
+
+      if (dirs.length <= cap) return;
+
+      // Find idle worktrees that can be safely removed
+      const idle = await scanIdleWorktrees(this.options.rootDir, this.store);
+      if (idle.length === 0) return;
+
+      // Sort by mtime ascending (oldest first)
+      const withMtime = idle.map((p) => {
+        try {
+          return { path: p, mtime: statSync(p).mtimeMs };
+        } catch {
+          return { path: p, mtime: 0 };
+        }
+      });
+      withMtime.sort((a, b) => a.mtime - b.mtime);
+
+      let removed = 0;
+      const excess = dirs.length - cap;
+
+      for (const { path: worktreePath } of withMtime) {
+        if (removed >= excess) break;
+        try {
+          execSync(`git worktree remove "${worktreePath}" --force`, {
+            cwd: this.options.rootDir,
+            stdio: "pipe",
+            timeout: 30_000,
+          });
+          removed++;
+        } catch {
+          // Individual failure is non-fatal
+        }
+      }
+
+      if (removed > 0) {
+        log.warn(`Worktree cap: removed ${removed} idle worktree(s) (was ${dirs.length}, cap ${cap})`);
+      }
+    } catch (err: any) {
+      log.error(`Worktree cap enforcement failed: ${err.message}`);
+    }
+  }
+}
--- a/packages/engine/src/stuck-task-detector.test.ts
+++ b/packages/engine/src/stuck-task-detector.test.ts
@@ -244,6 +244,67 @@ describe("StuckTaskDetector", () => {
      // Should not throw
      expect(store.moveTask).not.toHaveBeenCalled();
    });
+
+    it("calls beforeRequeue and skips re-queue when it returns false", async () => {
+      const beforeRequeue = vi.fn().mockResolvedValue(false);
+      const onStuck = vi.fn();
+      const customDetector = new StuckTaskDetector(store, { beforeRequeue, onStuck });
+      const session = createMockSession();
+
+      customDetector.trackTask("FN-001", session);
+
+      vi.useFakeTimers({ shouldAdvanceTime: true });
+      vi.advanceTimersByTime(61000);
+
+      await customDetector.killAndRetry("FN-001", 60000);
+
+      expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
+      expect(session.dispose).toHaveBeenCalled();
+      // onStuck should still be called (so executor can mark stuck-aborted)
+      expect(onStuck).toHaveBeenCalledWith("FN-001");
+      // But task should NOT be moved to todo
+      expect(store.moveTask).not.toHaveBeenCalled();
+      expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
+
+      vi.useRealTimers();
+    });
+
+    it("calls beforeRequeue and proceeds with re-queue when it returns true", async () => {
+      const beforeRequeue = vi.fn().mockResolvedValue(true);
+      const customDetector = new StuckTaskDetector(store, { beforeRequeue });
+      const session = createMockSession();
+
+      customDetector.trackTask("FN-001", session);
+
+      vi.useFakeTimers({ shouldAdvanceTime: true });
+      vi.advanceTimersByTime(61000);
+
+      await customDetector.killAndRetry("FN-001", 60000);
+
+      expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
+      expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
+      expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
+
+      vi.useRealTimers();
+    });
+
+    it("falls through to re-queue when beforeRequeue throws", async () => {
+      const beforeRequeue = vi.fn().mockRejectedValue(new Error("check failed"));
+      const customDetector = new StuckTaskDetector(store, { beforeRequeue });
+      const session = createMockSession();
+
+      customDetector.trackTask("FN-001", session);
+
+      vi.useFakeTimers({ shouldAdvanceTime: true });
+      vi.advanceTimersByTime(61000);
+
+      await customDetector.killAndRetry("FN-001", 60000);
+
+      // Should still re-queue on error (safe fallback)
+      expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
+
+      vi.useRealTimers();
+    });
  });

  describe("checkNow", () => {
--- a/packages/engine/src/stuck-task-detector.ts
+++ b/packages/engine/src/stuck-task-detector.ts
@@ -33,6 +33,10 @@ export interface StuckTaskDetectorOptions {
  /** Callback invoked when a stuck task is detected and killed.
   *  The task will be moved to "todo" for retry by the detector. */
  onStuck?: (taskId: string) => void;
+  /** Called before re-queuing a killed task. Return false to prevent re-queue
+   *  (caller is responsible for marking the task as terminally failed).
+   *  Used by SelfHealingManager to enforce stuck kill budgets. */
+  beforeRequeue?: (taskId: string) => Promise<boolean>;
 }

 export class StuckTaskDetector {
@@ -40,6 +44,7 @@ export class StuckTaskDetector {
  private interval: ReturnType<typeof setInterval> | null = null;
  private pollIntervalMs: number;
  private onStuck?: (taskId: string) => void;
+  private beforeRequeue?: (taskId: string) => Promise<boolean>;

  constructor(
    private store: TaskStore,
@@ -47,6 +52,7 @@ export class StuckTaskDetector {
  ) {
    this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
    this.onStuck = options.onStuck;
+    this.beforeRequeue = options.beforeRequeue;
  }

  /**
@@ -158,6 +164,22 @@ export class StuckTaskDetector {
      stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
    }

+    // Check stuck kill budget before re-queuing (SelfHealingManager integration).
+    // If beforeRequeue returns false, the task has been marked failed — skip re-queue.
+    if (this.beforeRequeue) {
+      try {
+        const shouldRequeue = await this.beforeRequeue(taskId);
+        if (!shouldRequeue) {
+          stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
+          this.onStuck?.(taskId);
+          return;
+        }
+      } catch (err) {
+        stuckLog.error(`beforeRequeue check failed for ${taskId}:`, err);
+        // Fall through to re-queue on error — safer than dropping the task
+      }
+    }
+
    // Set transient "stuck-killed" status, then move to "todo" for retry.
    // moveTask from "in-progress" to "todo" automatically clears status,
    // so no explicit status clear is needed after the move.