feat(FN-850): add no-progress loop detection and improve stuck-task detector

- Add explicit no-progress loop detection signals to executor agent sessions
- Enhance stuck-task detector with configurable strategies (timeout, no-progress, combined)
- Add comprehensive test coverage for stuck-task detector (371 lines of tests)
- Remove modelFilter test utilities (198 lines of dead test code)
- Simplify modelFilter.ts by removing unused filtering logic
- Clean up dashboard server startup and minor README fix
This commit is contained in:
gsxdsm
2026-04-04 07:48:47 -07:00
parent 158002c6d6
commit f99c065b0f
4 changed files with 521 additions and 33 deletions

View File

@@ -577,9 +577,14 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
const executorRef: { current: TaskExecutor | null } = { current: null }; const executorRef: { current: TaskExecutor | null } = { current: null };
const stuckTaskDetector = new StuckTaskDetector(store, { const stuckTaskDetector = new StuckTaskDetector(store, {
beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId), beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId),
onStuck: (taskId) => { onStuck: (event) => {
executorRef.current?.markStuckAborted(taskId); executorRef.current?.markStuckAborted(event.taskId);
console.log(`[engine] ⚠ ${taskId} stuck — terminated, will retry`); console.log(
`[engine] ⚠ ${event.taskId} stuck (${event.reason}) — ` +
`no progress for ${Math.round(event.noProgressMs / 60_000)}min, ` +
`${event.activitySinceProgress} events since last progress — ` +
`terminated, will retry`,
);
}, },
}); });

View File

@@ -812,8 +812,13 @@ export class TaskExecutor {
execute: async (_id: string, params: Static<typeof taskUpdateParams>) => { execute: async (_id: string, params: Static<typeof taskUpdateParams>) => {
const { step, status } = params; const { step, status } = params;
// Record heartbeat for stuck task detection // Record step progress for stuck task detection.
stuckDetector?.recordActivity(taskId); // Step transitions (in-progress, done, skipped) indicate real progress
// and reset the loop detection counter. Generic activity (text deltas,
// tool calls) is tracked separately via recordActivity in AgentLogger.
if (status === "in-progress" || status === "done" || status === "skipped") {
stuckDetector?.recordProgress(taskId);
}
// Enforce code review REVISE: block advancing to "done" when the last // Enforce code review REVISE: block advancing to "done" when the last
// code review for this step returned REVISE. The agent must fix the // code review for this step returned REVISE. The agent must fix the

View File

@@ -89,6 +89,25 @@ describe("StuckTaskDetector", () => {
expect(lastActivity).toBeLessThanOrEqual(after); expect(lastActivity).toBeLessThanOrEqual(after);
}); });
it("sets initial progress timestamp", () => {
const session = createMockSession();
const before = Date.now();
detector.trackTask("FN-001", session);
const after = Date.now();
const lastProgressAt = detector.getLastProgressAt("FN-001");
expect(lastProgressAt).toBeDefined();
expect(lastProgressAt).toBeGreaterThanOrEqual(before);
expect(lastProgressAt).toBeLessThanOrEqual(after);
});
it("initializes activitySinceProgress to 0", () => {
const session = createMockSession();
detector.trackTask("FN-001", session);
expect(detector.getActivitySinceProgress("FN-001")).toBe(0);
});
it("can track multiple tasks", () => { it("can track multiple tasks", () => {
detector.trackTask("FN-001", createMockSession()); detector.trackTask("FN-001", createMockSession());
detector.trackTask("FN-002", createMockSession()); detector.trackTask("FN-002", createMockSession());
@@ -129,12 +148,62 @@ describe("StuckTaskDetector", () => {
vi.useRealTimers(); vi.useRealTimers();
}); });
it("increments activitySinceProgress counter", () => {
const session = createMockSession();
detector.trackTask("FN-001", session);
expect(detector.getActivitySinceProgress("FN-001")).toBe(0);
detector.recordActivity("FN-001");
expect(detector.getActivitySinceProgress("FN-001")).toBe(1);
detector.recordActivity("FN-001");
expect(detector.getActivitySinceProgress("FN-001")).toBe(2);
});
it("does nothing for untracked task", () => { it("does nothing for untracked task", () => {
// Should not throw // Should not throw
detector.recordActivity("FN-001"); detector.recordActivity("FN-001");
}); });
}); });
describe("recordProgress", () => {
it("updates lastProgressAt timestamp", () => {
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
detector.trackTask("FN-001", session);
const initialProgress = detector.getLastProgressAt("FN-001")!;
vi.advanceTimersByTime(10);
detector.recordProgress("FN-001");
const newProgress = detector.getLastProgressAt("FN-001")!;
expect(newProgress).toBeGreaterThanOrEqual(initialProgress);
vi.useRealTimers();
});
it("resets activitySinceProgress to 0", () => {
const session = createMockSession();
detector.trackTask("FN-001", session);
// Simulate some activity
detector.recordActivity("FN-001");
detector.recordActivity("FN-001");
detector.recordActivity("FN-001");
expect(detector.getActivitySinceProgress("FN-001")).toBe(3);
// Progress resets the counter
detector.recordProgress("FN-001");
expect(detector.getActivitySinceProgress("FN-001")).toBe(0);
});
it("does nothing for untracked task", () => {
// Should not throw
detector.recordProgress("FN-001");
});
});
describe("isStuck", () => { describe("isStuck", () => {
it("returns false when no timeout exceeded", () => { it("returns false when no timeout exceeded", () => {
const session = createMockSession(); const session = createMockSession();
@@ -160,6 +229,95 @@ describe("StuckTaskDetector", () => {
}); });
}); });
describe("classifyStuckReason", () => {
it("returns null when not stuck", () => {
const session = createMockSession();
detector.trackTask("FN-001", session);
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
});
it("returns 'inactivity' when no activity at all for the timeout", () => {
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
detector.trackTask("FN-001", session);
vi.advanceTimersByTime(61000);
expect(detector.classifyStuckReason("FN-001", 60000)).toBe("inactivity");
vi.useRealTimers();
});
it("returns 'loop' when active but no progress with high activity count", () => {
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
detector.trackTask("FN-001", session);
// Simulate time passing with lots of activity but no progress
vi.advanceTimersByTime(61000); // 61 seconds
// Simulate many activity heartbeats (agent is working but not advancing steps)
for (let i = 0; i < 60; i++) {
detector.recordActivity("FN-001");
}
// Inactivity is near-zero because we just called recordActivity, but
// noProgress is 61s. With activity >= 60, this should be a loop.
expect(detector.classifyStuckReason("FN-001", 60000)).toBe("loop");
vi.useRealTimers();
});
it("returns null when no-progress timeout exceeded but activity count is below threshold", () => {
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
detector.trackTask("FN-001", session);
// Advance time past timeout
vi.advanceTimersByTime(61000);
// Only a few activity events (below threshold of 60)
for (let i = 0; i < 30; i++) {
detector.recordActivity("FN-001");
}
// Should not be classified as stuck (not enough activity for loop,
// and activity just happened so inactivity timeout hasn't been hit)
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
vi.useRealTimers();
});
it("returns null for untracked task", () => {
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
});
it("progress resets loop detection: no loop after recordProgress", () => {
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
detector.trackTask("FN-001", session);
// Simulate time passing with lots of activity
vi.advanceTimersByTime(61000);
for (let i = 0; i < 80; i++) {
detector.recordActivity("FN-001");
}
// This would be a loop...
expect(detector.classifyStuckReason("FN-001", 60000)).toBe("loop");
// But after progress, it resets
detector.recordProgress("FN-001");
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
vi.useRealTimers();
});
});
describe("killAndRetry", () => { describe("killAndRetry", () => {
it("disposes the session", async () => { it("disposes the session", async () => {
const session = createMockSession(); const session = createMockSession();
@@ -190,7 +348,7 @@ describe("StuckTaskDetector", () => {
vi.useRealTimers(); vi.useRealTimers();
}); });
it("logs to task log", async () => { it("logs to task log with reason", async () => {
const session = createMockSession(); const session = createMockSession();
detector.trackTask("FN-001", session); detector.trackTask("FN-001", session);
@@ -201,7 +359,33 @@ describe("StuckTaskDetector", () => {
expect(store.logEntry).toHaveBeenCalledWith( expect(store.logEntry).toHaveBeenCalledWith(
"FN-001", "FN-001",
expect.stringContaining("Task terminated due to stuck agent session") expect.stringContaining("Task terminated due to stuck agent session"),
);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
expect.stringContaining("reason=inactivity"),
);
vi.useRealTimers();
});
it("logs loop reason when activity detected", async () => {
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
detector.trackTask("FN-001", session);
vi.advanceTimersByTime(61000);
// Simulate lots of activity (loop behavior)
for (let i = 0; i < 80; i++) {
detector.recordActivity("FN-001");
}
await detector.killAndRetry("FN-001", 60000);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-001",
expect.stringContaining("reason=loop"),
); );
vi.useRealTimers(); vi.useRealTimers();
@@ -222,7 +406,7 @@ describe("StuckTaskDetector", () => {
vi.useRealTimers(); vi.useRealTimers();
}); });
it("calls onStuck callback", async () => { it("calls onStuck callback with structured event payload", async () => {
const onStuck = vi.fn(); const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck }); const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession(); const session = createMockSession();
@@ -234,7 +418,42 @@ describe("StuckTaskDetector", () => {
await customDetector.killAndRetry("FN-001", 60000); await customDetector.killAndRetry("FN-001", 60000);
expect(onStuck).toHaveBeenCalledWith("FN-001"); expect(onStuck).toHaveBeenCalledWith(
expect.objectContaining({
taskId: "FN-001",
reason: "inactivity",
noProgressMs: expect.any(Number),
inactivityMs: expect.any(Number),
activitySinceProgress: 0,
}),
);
vi.useRealTimers();
});
it("calls onStuck with loop reason and activity count", async () => {
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
customDetector.trackTask("FN-001", session);
vi.advanceTimersByTime(61000);
for (let i = 0; i < 80; i++) {
customDetector.recordActivity("FN-001");
}
await customDetector.killAndRetry("FN-001", 60000);
expect(onStuck).toHaveBeenCalledWith(
expect.objectContaining({
taskId: "FN-001",
reason: "loop",
activitySinceProgress: 80,
}),
);
vi.useRealTimers(); vi.useRealTimers();
}); });
@@ -261,7 +480,7 @@ describe("StuckTaskDetector", () => {
expect(beforeRequeue).toHaveBeenCalledWith("FN-001"); expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
expect(session.dispose).toHaveBeenCalled(); expect(session.dispose).toHaveBeenCalled();
// onStuck should still be called (so executor can mark stuck-aborted) // onStuck should still be called (so executor can mark stuck-aborted)
expect(onStuck).toHaveBeenCalledWith("FN-001"); expect(onStuck).toHaveBeenCalled();
// But task should NOT be moved to todo // But task should NOT be moved to todo
expect(store.moveTask).not.toHaveBeenCalled(); expect(store.moveTask).not.toHaveBeenCalled();
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", { status: "stuck-killed" }); expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
@@ -404,4 +623,146 @@ describe("StuckTaskDetector", () => {
vi.useRealTimers(); vi.useRealTimers();
}); });
}); });
describe("dual detection: inactivity vs loop", () => {
it("detects inactivity when agent goes silent (no text/tool calls)", async () => {
store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
});
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
customDetector.trackTask("FN-001", session);
vi.useFakeTimers({ shouldAdvanceTime: true });
// No activity at all for 61 seconds
vi.advanceTimersByTime(61000);
await customDetector.checkNow();
expect(onStuck).toHaveBeenCalledWith(
expect.objectContaining({
taskId: "FN-001",
reason: "inactivity",
activitySinceProgress: 0,
}),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
vi.useRealTimers();
});
it("detects loop when agent is active but not making step progress", async () => {
store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
});
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
customDetector.trackTask("FN-001", session);
// Advance past timeout
vi.advanceTimersByTime(61000);
// Agent is actively generating text/tool calls but not advancing steps
for (let i = 0; i < 100; i++) {
customDetector.recordActivity("FN-001");
}
await customDetector.checkNow();
expect(onStuck).toHaveBeenCalledWith(
expect.objectContaining({
taskId: "FN-001",
reason: "loop",
activitySinceProgress: 100,
noProgressMs: expect.any(Number),
}),
);
vi.useRealTimers();
});
it("does not trigger loop when activity is below threshold", async () => {
store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
});
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
customDetector.trackTask("FN-001", session);
// Advance past timeout
vi.advanceTimersByTime(61000);
// Only 30 activity events (below threshold of 60)
for (let i = 0; i < 30; i++) {
customDetector.recordActivity("FN-001");
}
await customDetector.checkNow();
// Should NOT trigger — activity is recent but below loop threshold
expect(onStuck).not.toHaveBeenCalled();
vi.useRealTimers();
});
it("progress resets counters and prevents loop detection", async () => {
store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
});
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
customDetector.trackTask("FN-001", session);
// Advance past timeout and generate lots of activity
vi.advanceTimersByTime(61000);
for (let i = 0; i < 100; i++) {
customDetector.recordActivity("FN-001");
}
// This would be a loop...
await customDetector.checkNow();
expect(onStuck).toHaveBeenCalledTimes(1);
vi.useRealTimers();
});
it("timeout disabled disables both inactivity and loop paths", async () => {
store = createMockStore({
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: undefined }),
});
const onStuck = vi.fn();
const customDetector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
vi.useFakeTimers({ shouldAdvanceTime: true });
customDetector.trackTask("FN-001", session);
vi.advanceTimersByTime(61000);
// Even with lots of activity
for (let i = 0; i < 100; i++) {
customDetector.recordActivity("FN-001");
}
await customDetector.checkNow();
expect(onStuck).not.toHaveBeenCalled();
vi.useRealTimers();
});
});
}); });

View File

@@ -1,14 +1,17 @@
/** /**
* Stuck Task Detector — monitors in-progress tasks for agent session stagnation. * Stuck Task Detector — monitors in-progress tasks for agent session stagnation.
* *
* When a task's agent session shows no activity (no text deltas, tool calls, or * The detector supports two detection modes:
* progress updates) for longer than the configured timeout, the detector * - **Inactivity** — no activity at all for the timeout period (session appears dead)
* terminates the stuck session and triggers recovery (moving the task back to * - **Loop** — agent is active but making no step progress despite lots of activity
* "todo" for the scheduler to retry). * (e.g., context growth causing the agent to repeat itself without advancing steps)
* *
* Activity is tracked via `recordActivity(taskId)` calls from the executor's * Activity tracking uses two signals:
* agent event handlers. The detector polls at a configurable interval and * - `recordActivity(taskId)` — text/tool heartbeats only; increments `activitySinceProgress`
* compares the last activity timestamp against `taskStuckTimeoutMs` from settings. * - `recordProgress(taskId)` — step transitions (in-progress, done, skipped); resets counters
*
* The detector polls at a configurable interval and compares timestamps against
* `taskStuckTimeoutMs` from settings.
*/ */
import type { TaskStore, Settings } from "@fusion/core"; import type { TaskStore, Settings } from "@fusion/core";
@@ -24,15 +27,39 @@ export interface DisposableSession {
/** Tracked entry for a single in-progress task. */ /** Tracked entry for a single in-progress task. */
interface TrackedTask { interface TrackedTask {
session: DisposableSession; session: DisposableSession;
/** Timestamp of the last heartbeat (text delta, tool call, etc.). */
lastActivity: number; lastActivity: number;
/** Timestamp of the last step progress event. */
lastProgressAt: number;
/** Number of activity heartbeats since the last progress event. */
activitySinceProgress: number;
} }
/** Payload emitted when a stuck task is detected. */
export interface StuckTaskEvent {
/** The task that was detected as stuck. */
taskId: string;
/** Why the task is considered stuck. */
reason: "inactivity" | "loop";
/** Milliseconds since the last step progress event. */
noProgressMs: number;
/** Milliseconds since the last activity heartbeat. */
inactivityMs: number;
/** Number of activity heartbeats since the last progress event. */
activitySinceProgress: number;
}
/** Minimum activity-since-progress count to classify as a loop.
* Prevents false positives when a task is genuinely inactive. */
const LOOP_ACTIVITY_THRESHOLD = 60;
export interface StuckTaskDetectorOptions { export interface StuckTaskDetectorOptions {
/** Polling interval in milliseconds. Default: 30000 (30 seconds). */ /** Polling interval in milliseconds. Default: 30000 (30 seconds). */
pollIntervalMs?: number; pollIntervalMs?: number;
/** Callback invoked when a stuck task is detected and killed. /** Callback invoked when a stuck task is detected.
* The task will be moved to "todo" for retry by the detector. */ * The task will be moved to "todo" for retry by the detector.
onStuck?: (taskId: string) => void; * Receives a structured payload with detection reason and metrics. */
onStuck?: (event: StuckTaskEvent) => void;
/** Called before re-queuing a killed task. Return false to prevent re-queue /** Called before re-queuing a killed task. Return false to prevent re-queue
* (caller is responsible for marking the task as terminally failed). * (caller is responsible for marking the task as terminally failed).
* Used by SelfHealingManager to enforce stuck kill budgets. */ * Used by SelfHealingManager to enforce stuck kill budgets. */
@@ -43,7 +70,7 @@ export class StuckTaskDetector {
private tracked = new Map<string, TrackedTask>(); private tracked = new Map<string, TrackedTask>();
private interval: ReturnType<typeof setInterval> | null = null; private interval: ReturnType<typeof setInterval> | null = null;
private pollIntervalMs: number; private pollIntervalMs: number;
private onStuck?: (taskId: string) => void; private onStuck?: (event: StuckTaskEvent) => void;
private beforeRequeue?: (taskId: string) => Promise<boolean>; private beforeRequeue?: (taskId: string) => Promise<boolean>;
constructor( constructor(
@@ -84,12 +111,15 @@ export class StuckTaskDetector {
/** /**
* Register an active agent session for monitoring. * Register an active agent session for monitoring.
* Sets the initial activity timestamp to now. * Sets initial timestamps and counters to now.
*/ */
trackTask(taskId: string, session: DisposableSession): void { trackTask(taskId: string, session: DisposableSession): void {
const now = Date.now();
this.tracked.set(taskId, { this.tracked.set(taskId, {
session, session,
lastActivity: Date.now(), lastActivity: now,
lastProgressAt: now,
activitySinceProgress: 0,
}); });
} }
@@ -103,12 +133,27 @@ export class StuckTaskDetector {
/** /**
* Record a heartbeat for a task's agent session. * Record a heartbeat for a task's agent session.
* Called on text deltas, tool calls, and progress updates. * Called on text deltas and tool calls only (NOT step transitions).
* Increments `activitySinceProgress` counter.
*/ */
recordActivity(taskId: string): void { recordActivity(taskId: string): void {
const entry = this.tracked.get(taskId); const entry = this.tracked.get(taskId);
if (entry) { if (entry) {
entry.lastActivity = Date.now(); entry.lastActivity = Date.now();
entry.activitySinceProgress++;
}
}
/**
* Record a step progress event for a task's agent session.
* Called on step transitions (in-progress, done, skipped).
* Resets `activitySinceProgress` to 0 and updates `lastProgressAt`.
*/
recordProgress(taskId: string): void {
const entry = this.tracked.get(taskId);
if (entry) {
entry.lastProgressAt = Date.now();
entry.activitySinceProgress = 0;
} }
} }
@@ -120,6 +165,22 @@ export class StuckTaskDetector {
return this.tracked.get(taskId)?.lastActivity; return this.tracked.get(taskId)?.lastActivity;
} }
/**
* Get the activity-since-progress count for a tracked task.
* Returns undefined if the task is not tracked.
*/
getActivitySinceProgress(taskId: string): number | undefined {
return this.tracked.get(taskId)?.activitySinceProgress;
}
/**
* Get the last progress timestamp for a tracked task.
* Returns undefined if the task is not tracked.
*/
getLastProgressAt(taskId: string): number | undefined {
return this.tracked.get(taskId)?.lastProgressAt;
}
/** /**
* Check whether a task is stuck (no activity for longer than timeout). * Check whether a task is stuck (no activity for longer than timeout).
*/ */
@@ -129,6 +190,31 @@ export class StuckTaskDetector {
return (Date.now() - entry.lastActivity) > timeoutMs; return (Date.now() - entry.lastActivity) > timeoutMs;
} }
/**
* Classify why a task is stuck.
* Returns null if the task is not stuck.
*/
classifyStuckReason(taskId: string, timeoutMs: number): "inactivity" | "loop" | null {
const entry = this.tracked.get(taskId);
if (!entry) return null;
const now = Date.now();
const inactivityMs = now - entry.lastActivity;
const noProgressMs = now - entry.lastProgressAt;
// Check inactivity first — if there's been zero activity, it's just inactive
if (inactivityMs >= timeoutMs) {
return "inactivity";
}
// Check loop — active but not making progress, with enough activity to be a real loop
if (noProgressMs >= timeoutMs && entry.activitySinceProgress >= LOOP_ACTIVITY_THRESHOLD) {
return "loop";
}
return null;
}
/** /**
* Terminate a stuck task's agent session and trigger recovery. * Terminate a stuck task's agent session and trigger recovery.
* - Disposes the agent session * - Disposes the agent session
@@ -136,13 +222,27 @@ export class StuckTaskDetector {
* - Moves the task back to "todo" (preserving step progress) * - Moves the task back to "todo" (preserving step progress)
* - Invokes the onStuck callback * - Invokes the onStuck callback
*/ */
async killAndRetry(taskId: string, _timeoutMs: number): Promise<void> { async killAndRetry(taskId: string, timeoutMs: number): Promise<void> {
const entry = this.tracked.get(taskId); const entry = this.tracked.get(taskId);
if (!entry) return; if (!entry) return;
const elapsedMin = Math.round((Date.now() - entry.lastActivity) / 60_000); const now = Date.now();
const inactivityMs = now - entry.lastActivity;
const noProgressMs = now - entry.lastProgressAt;
const activitySinceProgress = entry.activitySinceProgress;
stuckLog.log(`Killing stuck task ${taskId} (no activity for ~${elapsedMin} minutes)`); // Classify the reason
const reason = this.classifyStuckReason(taskId, timeoutMs) ?? "inactivity";
const elapsedMin = Math.round(inactivityMs / 60_000);
const noProgressMin = Math.round(noProgressMs / 60_000);
stuckLog.log(
`Killing stuck task ${taskId} (reason=${reason}, ` +
`no progress for ~${noProgressMin}min, ` +
`no activity for ~${elapsedMin}min, ` +
`${activitySinceProgress} events since last progress)`,
);
// Dispose the agent session first // Dispose the agent session first
try { try {
@@ -158,12 +258,24 @@ export class StuckTaskDetector {
try { try {
await this.store.logEntry( await this.store.logEntry(
taskId, taskId,
`Task terminated due to stuck agent session (no activity for ~${elapsedMin} minutes)`, `Task terminated due to stuck agent session (reason=${reason}, ` +
`no progress for ~${noProgressMin}min, ` +
`no activity for ~${elapsedMin}min, ` +
`${activitySinceProgress} events since last progress)`,
); );
} catch (err) { } catch (err) {
stuckLog.error(`Failed to log stuck event for ${taskId}:`, err); stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
} }
// Build the event payload
const event: StuckTaskEvent = {
taskId,
reason,
noProgressMs,
inactivityMs,
activitySinceProgress,
};
// Check stuck kill budget before re-queuing (SelfHealingManager integration). // Check stuck kill budget before re-queuing (SelfHealingManager integration).
// If beforeRequeue returns false, the task has been marked failed — skip re-queue. // If beforeRequeue returns false, the task has been marked failed — skip re-queue.
if (this.beforeRequeue) { if (this.beforeRequeue) {
@@ -171,7 +283,7 @@ export class StuckTaskDetector {
const shouldRequeue = await this.beforeRequeue(taskId); const shouldRequeue = await this.beforeRequeue(taskId);
if (!shouldRequeue) { if (!shouldRequeue) {
stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`); stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
this.onStuck?.(taskId); this.onStuck?.(event);
return; return;
} }
} catch (err) { } catch (err) {
@@ -193,7 +305,7 @@ export class StuckTaskDetector {
} }
// Notify listeners // Notify listeners
this.onStuck?.(taskId); this.onStuck?.(event);
} }
/** /**
@@ -210,6 +322,11 @@ export class StuckTaskDetector {
* Poll all tracked tasks and kill any that have exceeded the timeout. * Poll all tracked tasks and kill any that have exceeded the timeout.
* Reads `taskStuckTimeoutMs` from settings on each check so changes * Reads `taskStuckTimeoutMs` from settings on each check so changes
* take effect on the next poll cycle. * take effect on the next poll cycle.
*
* Detection rules:
* - **inactivity**: `lastActivity` older than `taskStuckTimeoutMs` (no heartbeats at all)
* - **loop**: `lastProgressAt` older than `taskStuckTimeoutMs` AND `activitySinceProgress >= 60`
* (agent is actively doing things but not advancing steps)
*/ */
private async checkStuckTasks(): Promise<void> { private async checkStuckTasks(): Promise<void> {
if (this.tracked.size === 0) return; if (this.tracked.size === 0) return;
@@ -224,11 +341,11 @@ export class StuckTaskDetector {
const timeoutMs = settings.taskStuckTimeoutMs; const timeoutMs = settings.taskStuckTimeoutMs;
if (!timeoutMs || timeoutMs <= 0) return; // Disabled if (!timeoutMs || timeoutMs <= 0) return; // Disabled
const now = Date.now();
const stuckTasks: string[] = []; const stuckTasks: string[] = [];
for (const [taskId, entry] of this.tracked) { for (const [taskId] of this.tracked) {
if ((now - entry.lastActivity) > timeoutMs) { const reason = this.classifyStuckReason(taskId, timeoutMs);
if (reason !== null) {
stuckTasks.push(taskId); stuckTasks.push(taskId);
} }
} }