feat: add SelfHealingManager for unattended multi-day operation

Adds four self-healing subsystems to enable the engine to recover from
common failure modes without human intervention:

- Auto-unpause: clears rate-limit-triggered globalPause with escalating
  backoff (5 min → 60 min cap), resets on sustained recovery
- Stuck kill budget: caps task stuck-kill retries (default 3) to prevent
  infinite stuck→todo→stuck loops
- Periodic maintenance (every 15 min): git worktree prune, orphan cleanup,
  SQLite WAL checkpoint
- Worktree cap enforcement: removes oldest idle worktrees when count
  exceeds 2× maxWorktrees

New settings: autoUnpauseEnabled, autoUnpauseBaseDelayMs,
autoUnpauseMaxDelayMs, maxStuckKills, maintenanceIntervalMs.
New task field: stuckKillCount (schema v8 migration).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-03 15:35:12 -07:00
parent 6a38b001ba
commit fdf6d0b4a1
11 changed files with 842 additions and 18 deletions

View File

@@ -15,6 +15,7 @@ export { PrCommentHandler } from "./pr-comment-handler.js";
export { NtfyNotifier, type NtfyNotifierOptions } from "./notifier.js";
export { CronRunner, type CronRunnerOptions } from "./cron-runner.js";
export { StuckTaskDetector, type StuckTaskDetectorOptions, type DisposableSession } from "./stuck-task-detector.js";
export { SelfHealingManager, type SelfHealingOptions } from "./self-healing.js";
export { ProjectManager } from "./project-manager.js";
// Multi-project runtime types
export {

View File

@@ -20,6 +20,7 @@ import type {
import { runtimeLog } from "../logger.js";
import type { StuckTaskDetector } from "../stuck-task-detector.js";
import type { UsageLimitPauser } from "../usage-limit-detector.js";
import { SelfHealingManager } from "../self-healing.js";
/**
* InProcessRuntime runs a project within the main process.
@@ -65,6 +66,7 @@ export class InProcessRuntime
private globalSemaphore?: AgentSemaphore;
private stuckTaskDetector?: StuckTaskDetector;
private usageLimitPauser?: UsageLimitPauser;
private selfHealingManager?: SelfHealingManager;
private agentStore?: AgentStore;
private heartbeatMonitor?: HeartbeatMonitor;
/** Maps task IDs to agent IDs for lifecycle tracking */
@@ -225,13 +227,19 @@ export class InProcessRuntime
runtimeLog.warn(`AgentStore initialization failed (continuing without agent monitoring):`, agentErr);
}
// 7. Set up event forwarding from TaskStore
// 7. Initialize SelfHealingManager
this.selfHealingManager = new SelfHealingManager(this.taskStore, {
rootDir: this.config.workingDirectory,
});
this.selfHealingManager.start();
// 8. Set up event forwarding from TaskStore
this.setupEventForwarding();
// 8. Resume orphaned in-progress tasks
// 9. Resume orphaned in-progress tasks
await this.executor.resumeOrphaned();
// 9. Start scheduler
// 10. Start scheduler
this.scheduler.start();
this.setStatus("active");
@@ -266,13 +274,19 @@ export class InProcessRuntime
runtimeLog.log(`Stopping InProcessRuntime for project ${this.config.projectId}`);
try {
// 1. Stop heartbeat monitor
// 1. Stop self-healing manager
if (this.selfHealingManager) {
this.selfHealingManager.stop();
runtimeLog.log("SelfHealingManager stopped");
}
// 2. Stop heartbeat monitor
if (this.heartbeatMonitor) {
this.heartbeatMonitor.stop();
runtimeLog.log("HeartbeatMonitor stopped");
}
// 2. Stop scheduler (prevents new task scheduling)
// 3. Stop scheduler (prevents new task scheduling)
if (this.scheduler) {
this.scheduler.stop();
runtimeLog.log("Scheduler stopped");

View File

@@ -0,0 +1,294 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { SelfHealingManager } from "./self-healing.js";
import type { TaskStore, Settings, Task } from "@fusion/core";
import { EventEmitter } from "node:events";
// ── Mock helpers ────────────────────────────────────────────────────
/** TaskStore mock backed by a real EventEmitter so settings:updated works. */
function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
const emitter = new EventEmitter();
const store = Object.assign(emitter, {
getSettings: vi.fn().mockResolvedValue({
autoUnpauseEnabled: true,
autoUnpauseBaseDelayMs: 100,
autoUnpauseMaxDelayMs: 800,
maxStuckKills: 3,
maintenanceIntervalMs: 0,
maxWorktrees: 4,
globalPause: true, // default: paused (for auto-unpause tests)
} as unknown as Settings),
updateSettings: vi.fn().mockResolvedValue({} as Settings),
getTask: vi.fn().mockResolvedValue({
id: "FN-001",
stuckKillCount: 0,
} as unknown as Task),
updateTask: vi.fn().mockResolvedValue({} as Task),
logEntry: vi.fn().mockResolvedValue(undefined),
moveTask: vi.fn().mockResolvedValue(undefined),
walCheckpoint: vi.fn().mockReturnValue({ busy: 0, log: 5, checkpointed: 5 }),
listTasks: vi.fn().mockResolvedValue([]),
getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
...overrides,
}) as unknown as TaskStore & EventEmitter;
return store;
}
describe("SelfHealingManager", () => {
let store: TaskStore & EventEmitter;
let manager: SelfHealingManager;
beforeEach(() => {
vi.useFakeTimers({ shouldAdvanceTime: true });
store = createMockStore();
manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
});
afterEach(() => {
manager.stop();
vi.useRealTimers();
});
// ── Auto-unpause ─────────────────────────────────────────────────
describe("auto-unpause", () => {
it("schedules unpause when globalPause transitions false→true", async () => {
manager.start();
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(150);
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: false });
});
it("does not schedule unpause when autoUnpauseEnabled is false", async () => {
manager.start();
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: false },
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(500);
expect(store.updateSettings).not.toHaveBeenCalled();
});
it("does not fire when already unpaused before timer", async () => {
// When the timer fires, getSettings returns globalPause: false
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
globalPause: false,
maintenanceIntervalMs: 0,
} as unknown as Settings);
manager.start();
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(150);
expect(store.updateSettings).not.toHaveBeenCalled();
});
it("escalates backoff when pause re-triggers within 60s", async () => {
manager.start();
// First pause
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(150);
expect(store.updateSettings).toHaveBeenCalledTimes(1);
// Simulate successful unpause
store.emit("settings:updated", {
settings: { globalPause: false },
previous: { globalPause: true },
});
// Immediately re-trigger pause (within 60s window)
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
// Escalated delay = 200ms. At 150ms it should NOT have fired yet.
await vi.advanceTimersByTimeAsync(150);
expect(store.updateSettings).toHaveBeenCalledTimes(1);
// At 250ms total (100ms more) it should fire
await vi.advanceTimersByTimeAsync(100);
expect(store.updateSettings).toHaveBeenCalledTimes(2);
});
it("cancels timer on manual unpause (true→false)", async () => {
manager.start();
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 200, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
// Manual unpause before timer fires
store.emit("settings:updated", {
settings: { globalPause: false },
previous: { globalPause: true },
});
await vi.advanceTimersByTimeAsync(300);
expect(store.updateSettings).not.toHaveBeenCalled();
});
it("ignores false→false transitions", async () => {
manager.start();
store.emit("settings:updated", {
settings: { globalPause: false },
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(500);
expect(store.updateSettings).not.toHaveBeenCalled();
});
});
// ── Stuck kill budget ─────────────────────────────────────────────
describe("checkStuckBudget", () => {
it("returns true and increments count when within budget", async () => {
manager.start();
const result = await manager.checkStuckBudget("FN-001");
expect(result).toBe(true);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 1 });
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
expect.stringContaining("Stuck kill 1/3"),
);
});
it("returns true for subsequent kills within budget", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
stuckKillCount: 2,
} as unknown as Task);
manager.start();
const result = await manager.checkStuckBudget("FN-001");
expect(result).toBe(true);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 3 });
});
it("returns false and marks failed when budget exceeded", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
stuckKillCount: 3,
} as unknown as Task);
manager.start();
const result = await manager.checkStuckBudget("FN-001");
expect(result).toBe(false);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", {
stuckKillCount: 4,
status: "failed",
error: expect.stringContaining("exceeded maximum of 3"),
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
expect.stringContaining("Permanently failed"),
);
});
it("respects custom maxStuckKills setting", async () => {
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
maxStuckKills: 1,
maintenanceIntervalMs: 0,
} as unknown as Settings);
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
stuckKillCount: 1,
} as unknown as Task);
manager.start();
const result = await manager.checkStuckBudget("FN-001");
expect(result).toBe(false);
});
it("returns true on error (safe fallback)", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockRejectedValue(new Error("DB error"));
manager.start();
const result = await manager.checkStuckBudget("FN-001");
expect(result).toBe(true);
});
it("handles undefined stuckKillCount as 0", async () => {
(store.getTask as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "FN-001",
} as unknown as Task);
manager.start();
const result = await manager.checkStuckBudget("FN-001");
expect(result).toBe(true);
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { stuckKillCount: 1 });
});
});
// ── Lifecycle ─────────────────────────────────────────────────────
describe("lifecycle", () => {
it("starts and stops without error", () => {
manager.start();
manager.stop();
});
it("cleans up timers on stop", async () => {
manager.start();
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 500, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
manager.stop();
await vi.advanceTimersByTimeAsync(1000);
expect(store.updateSettings).not.toHaveBeenCalled();
});
it("does not respond to events after stop", async () => {
manager.start();
manager.stop();
store.emit("settings:updated", {
settings: { globalPause: true, autoUnpauseEnabled: true, autoUnpauseBaseDelayMs: 100, autoUnpauseMaxDelayMs: 800 },
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(200);
expect(store.updateSettings).not.toHaveBeenCalled();
});
});
});

View File

@@ -0,0 +1,360 @@
/**
* SelfHealingManager — enables unattended multi-day/week operation by
* providing automatic recovery from common failure modes.
*
* Four subsystems:
* 1. **Auto-unpause**: Clears rate-limit-triggered `globalPause` with
* escalating backoff (5 min → 60 min cap). Resets on sustained unpause.
* 2. **Stuck kill budget**: Caps how many times a task can be killed by the
* stuck-task detector before marking it as permanently failed.
* 3. **Periodic maintenance**: Worktree pruning, orphan cleanup, SQLite
* WAL checkpoint — all on a configurable interval (default 15 min).
* 4. **Worktree cap enforcement**: Prevents unbounded worktree accumulation
* by cleaning oldest idle worktrees when count exceeds 2× maxWorktrees.
*/
import { execSync } from "node:child_process";
import { existsSync, readdirSync, statSync } from "node:fs";
import { join } from "node:path";
import type { TaskStore, Settings } from "@fusion/core";
import { createLogger } from "./logger.js";
import { scanIdleWorktrees } from "./worktree-pool.js";
const log = createLogger("self-healing");
export interface SelfHealingOptions {
/** Project root directory (parent of .worktrees/) */
rootDir: string;
}
export class SelfHealingManager {
// ── Auto-unpause state ──────────────────────────────────────────────
private unpauseTimer: ReturnType<typeof setTimeout> | null = null;
private unpauseAttempt = 0;
private lastPauseTriggeredAt = 0;
private lastUnpauseAt = 0;
// ── Maintenance timer ───────────────────────────────────────────────
private maintenanceInterval: ReturnType<typeof setInterval> | null = null;
// ── Event listener cleanup ──────────────────────────────────────────
private settingsListener: ((data: { settings: Settings; previous: Settings }) => void) | null = null;
constructor(
private store: TaskStore,
private options: SelfHealingOptions,
) {}
// ── Lifecycle ───────────────────────────────────────────────────────
start(): void {
// Wire up settings:updated listener for auto-unpause
this.settingsListener = ({ settings, previous }) => {
this.onSettingsUpdated(settings, previous);
};
this.store.on("settings:updated", this.settingsListener);
// Start periodic maintenance
this.startMaintenance();
log.log("Started");
}
stop(): void {
// Remove settings listener
if (this.settingsListener) {
try {
this.store.removeListener("settings:updated", this.settingsListener);
} catch {
// Store may not support removeListener (e.g., test mocks)
}
this.settingsListener = null;
}
// Clear timers
this.cancelUnpauseTimer();
if (this.maintenanceInterval) {
clearInterval(this.maintenanceInterval);
this.maintenanceInterval = null;
}
log.log("Stopped");
}
// ── Auto-unpause ───────────────────────────────────────────────────
private onSettingsUpdated(settings: Settings, previous: Settings): void {
// globalPause false → true: schedule auto-unpause
if (!previous.globalPause && settings.globalPause) {
if (!settings.autoUnpauseEnabled) {
log.log("Global pause activated — auto-unpause disabled, requires manual intervention");
return;
}
// If pause re-triggered within 60s of our last unpause, escalate backoff
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
this.unpauseAttempt++;
log.warn(`Global pause re-triggered within 60s — escalating to attempt ${this.unpauseAttempt}`);
}
this.lastPauseTriggeredAt = Date.now();
const baseDelay = settings.autoUnpauseBaseDelayMs ?? 300_000;
const maxDelay = settings.autoUnpauseMaxDelayMs ?? 3_600_000;
const delay = Math.min(baseDelay * Math.pow(2, this.unpauseAttempt), maxDelay);
this.scheduleUnpause(delay);
}
// globalPause true → false: check if we should reset backoff
if (previous.globalPause && !settings.globalPause) {
this.cancelUnpauseTimer();
// If sustained unpause (not a quick re-trigger), reset attempt counter
if (this.lastPauseTriggeredAt && (Date.now() - this.lastPauseTriggeredAt) > 60_000) {
this.unpauseAttempt = 0;
}
}
}
private scheduleUnpause(delayMs: number): void {
this.cancelUnpauseTimer();
const delaySec = Math.round(delayMs / 1000);
const delayMin = Math.round(delaySec / 60);
const display = delayMin >= 1 ? `${delayMin}m` : `${delaySec}s`;
log.warn(`Auto-unpause scheduled in ${display} (attempt ${this.unpauseAttempt + 1})`);
this.unpauseTimer = setTimeout(() => {
this.unpauseTimer = null;
void this.attemptUnpause();
}, delayMs);
}
private async attemptUnpause(): Promise<void> {
try {
const settings = await this.store.getSettings();
// Already unpaused (manually or by another mechanism)
if (!settings.globalPause) {
log.log("Auto-unpause: already unpaused — no action needed");
this.unpauseAttempt = 0;
return;
}
log.warn("Auto-unpause: clearing globalPause");
this.lastUnpauseAt = Date.now();
await this.store.updateSettings({ globalPause: false });
// Note: if the rate limit is still active, the next agent session will
// hit it again → UsageLimitPauser triggers globalPause → our listener
// catches the transition and schedules the next attempt with escalated backoff.
} catch (err: any) {
log.error(`Auto-unpause failed: ${err.message}`);
}
}
private cancelUnpauseTimer(): void {
if (this.unpauseTimer) {
clearTimeout(this.unpauseTimer);
this.unpauseTimer = null;
}
}
// ── Stuck kill budget ─────────────────────────────────────────────
/**
* Check whether a stuck-killed task should be re-queued or marked as failed.
* Called by StuckTaskDetector's `beforeRequeue` callback.
*
* @returns `true` if the task should be re-queued, `false` if budget exhausted
* (task has been marked as permanently failed).
*/
async checkStuckBudget(taskId: string): Promise<boolean> {
try {
const settings = await this.store.getSettings();
const maxKills = settings.maxStuckKills ?? 3;
const task = await this.store.getTask(taskId);
const newCount = (task.stuckKillCount ?? 0) + 1;
if (newCount > maxKills) {
// Budget exhausted — mark as permanently failed
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}) — marking failed`);
await this.store.updateTask(taskId, {
stuckKillCount: newCount,
status: "failed",
error: `Task stuck ${newCount} times — exceeded maximum of ${maxKills} stuck kills`,
});
await this.store.logEntry(
taskId,
`Permanently failed: agent stuck ${newCount} times (max: ${maxKills})`,
);
return false;
}
// Budget remaining — allow re-queue
log.log(`${taskId} stuck kill ${newCount}/${maxKills} — will re-queue`);
await this.store.updateTask(taskId, { stuckKillCount: newCount });
await this.store.logEntry(
taskId,
`Stuck kill ${newCount}/${maxKills} — re-queuing for retry`,
);
return true;
} catch (err: any) {
log.error(`checkStuckBudget failed for ${taskId}: ${err.message}`);
// On error, allow re-queue — safer than permanently failing
return true;
}
}
// ── Periodic maintenance ──────────────────────────────────────────
private async startMaintenance(): Promise<void> {
const settings = await this.store.getSettings();
const intervalMs = settings.maintenanceIntervalMs ?? 900_000;
if (intervalMs <= 0) {
log.log("Periodic maintenance disabled (maintenanceIntervalMs <= 0)");
return;
}
log.log(`Periodic maintenance every ${Math.round(intervalMs / 60_000)}m`);
this.maintenanceInterval = setInterval(() => {
void this.runMaintenance();
}, intervalMs);
}
private async runMaintenance(): Promise<void> {
const startMs = Date.now();
log.log("Maintenance cycle starting");
try {
await this.pruneWorktrees();
await this.cleanupOrphans();
this.checkpointWal();
await this.enforceWorktreeCap();
const elapsedMs = Date.now() - startMs;
log.log(`Maintenance cycle completed in ${elapsedMs}ms`);
} catch (err: any) {
log.error(`Maintenance cycle failed: ${err.message}`);
}
}
/** Run `git worktree prune` to clean stale metadata. */
private async pruneWorktrees(): Promise<void> {
try {
execSync("git worktree prune", {
cwd: this.options.rootDir,
stdio: "pipe",
timeout: 30_000,
});
log.log("Worktree prune completed");
} catch (err: any) {
log.error(`Worktree prune failed: ${err.message}`);
}
}
/** Remove orphaned worktrees not assigned to any active task. */
private async cleanupOrphans(): Promise<number> {
try {
const orphaned = await scanIdleWorktrees(this.options.rootDir, this.store);
if (orphaned.length === 0) return 0;
// Only clean up if recycling is disabled — otherwise they belong in the pool
const settings = await this.store.getSettings();
if (settings.recycleWorktrees) {
return 0;
}
let cleaned = 0;
for (const worktreePath of orphaned) {
try {
execSync(`git worktree remove "${worktreePath}" --force`, {
cwd: this.options.rootDir,
stdio: "pipe",
timeout: 30_000,
});
cleaned++;
} catch {
// Individual failure is non-fatal
}
}
if (cleaned > 0) {
log.log(`Cleaned ${cleaned} orphaned worktree(s)`);
}
return cleaned;
} catch (err: any) {
log.error(`Orphan cleanup failed: ${err.message}`);
return 0;
}
}
/** Run SQLite WAL checkpoint to reclaim disk space. */
private checkpointWal(): void {
try {
const result = this.store.walCheckpoint();
if (result.log > 0) {
log.log(`WAL checkpoint: ${result.checkpointed}/${result.log} pages checkpointed` +
(result.busy > 0 ? ` (${result.busy} busy)` : ""));
}
} catch (err: any) {
log.error(`WAL checkpoint failed: ${err.message}`);
}
}
/** Remove oldest idle worktrees if total count exceeds 2× maxWorktrees. */
private async enforceWorktreeCap(): Promise<void> {
const worktreesDir = join(this.options.rootDir, ".worktrees");
if (!existsSync(worktreesDir)) return;
try {
const settings = await this.store.getSettings();
const cap = (settings.maxWorktrees ?? 4) * 2;
const entries = readdirSync(worktreesDir, { withFileTypes: true });
const dirs = entries.filter((e) => e.isDirectory());
if (dirs.length <= cap) return;
// Find idle worktrees that can be safely removed
const idle = await scanIdleWorktrees(this.options.rootDir, this.store);
if (idle.length === 0) return;
// Sort by mtime ascending (oldest first)
const withMtime = idle.map((p) => {
try {
return { path: p, mtime: statSync(p).mtimeMs };
} catch {
return { path: p, mtime: 0 };
}
});
withMtime.sort((a, b) => a.mtime - b.mtime);
let removed = 0;
const excess = dirs.length - cap;
for (const { path: worktreePath } of withMtime) {
if (removed >= excess) break;
try {
execSync(`git worktree remove "${worktreePath}" --force`, {
cwd: this.options.rootDir,
stdio: "pipe",
timeout: 30_000,
});
removed++;
} catch {
// Individual failure is non-fatal
}
}
if (removed > 0) {
log.warn(`Worktree cap: removed ${removed} idle worktree(s) (was ${dirs.length}, cap ${cap})`);
}
} catch (err: any) {
log.error(`Worktree cap enforcement failed: ${err.message}`);
}
}
}

View File

@@ -244,6 +244,67 @@ describe("StuckTaskDetector", () => {
// Should not throw
expect(store.moveTask).not.toHaveBeenCalled();
});
it("calls beforeRequeue and skips re-queue when it returns false", async () => {
const beforeRequeue = vi.fn().mockResolvedValue(false);
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { beforeRequeue, onStuck });
const session = createMockSession();
customDetector.trackTask("FN-001", session);
vi.useFakeTimers({ shouldAdvanceTime: true });
vi.advanceTimersByTime(61000);
await customDetector.killAndRetry("FN-001", 60000);
expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
expect(session.dispose).toHaveBeenCalled();
// onStuck should still be called (so executor can mark stuck-aborted)
expect(onStuck).toHaveBeenCalledWith("FN-001");
// But task should NOT be moved to todo
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
vi.useRealTimers();
});
it("calls beforeRequeue and proceeds with re-queue when it returns true", async () => {
const beforeRequeue = vi.fn().mockResolvedValue(true);
const customDetector = new StuckTaskDetector(store, { beforeRequeue });
const session = createMockSession();
customDetector.trackTask("FN-001", session);
vi.useFakeTimers({ shouldAdvanceTime: true });
vi.advanceTimersByTime(61000);
await customDetector.killAndRetry("FN-001", 60000);
expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
vi.useRealTimers();
});
it("falls through to re-queue when beforeRequeue throws", async () => {
const beforeRequeue = vi.fn().mockRejectedValue(new Error("check failed"));
const customDetector = new StuckTaskDetector(store, { beforeRequeue });
const session = createMockSession();
customDetector.trackTask("FN-001", session);
vi.useFakeTimers({ shouldAdvanceTime: true });
vi.advanceTimersByTime(61000);
await customDetector.killAndRetry("FN-001", 60000);
// Should still re-queue on error (safe fallback)
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
vi.useRealTimers();
});
});
describe("checkNow", () => {

View File

@@ -33,6 +33,10 @@ export interface StuckTaskDetectorOptions {
/** Callback invoked when a stuck task is detected and killed.
* The task will be moved to "todo" for retry by the detector. */
onStuck?: (taskId: string) => void;
/** Called before re-queuing a killed task. Return false to prevent re-queue
* (caller is responsible for marking the task as terminally failed).
* Used by SelfHealingManager to enforce stuck kill budgets. */
beforeRequeue?: (taskId: string) => Promise<boolean>;
}
export class StuckTaskDetector {
@@ -40,6 +44,7 @@ export class StuckTaskDetector {
private interval: ReturnType<typeof setInterval> | null = null;
private pollIntervalMs: number;
private onStuck?: (taskId: string) => void;
private beforeRequeue?: (taskId: string) => Promise<boolean>;
constructor(
private store: TaskStore,
@@ -47,6 +52,7 @@ export class StuckTaskDetector {
) {
this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
this.onStuck = options.onStuck;
this.beforeRequeue = options.beforeRequeue;
}
/**
@@ -158,6 +164,22 @@ export class StuckTaskDetector {
stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
}
// Check stuck kill budget before re-queuing (SelfHealingManager integration).
// If beforeRequeue returns false, the task has been marked failed — skip re-queue.
if (this.beforeRequeue) {
try {
const shouldRequeue = await this.beforeRequeue(taskId);
if (!shouldRequeue) {
stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
this.onStuck?.(taskId);
return;
}
} catch (err) {
stuckLog.error(`beforeRequeue check failed for ${taskId}:`, err);
// Fall through to re-queue on error — safer than dropping the task
}
}
// Set transient "stuck-killed" status, then move to "todo" for retry.
// moveTask from "in-progress" to "todo" automatically clears status,
// so no explicit status clear is needed after the move.