feat(FN-850): add no-progress loop detection and improve stuck-task detector
- Add explicit no-progress loop detection signals to executor agent sessions - Enhance stuck-task detector with configurable strategies (timeout, no-progress, combined) - Add comprehensive test coverage for stuck-task detector (371 lines of tests) - Remove modelFilter test utilities (198 lines of dead test code) - Simplify modelFilter.ts by removing unused filtering logic - Clean up dashboard server startup and minor README fix
This commit is contained in:
@@ -577,9 +577,14 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
|
|||||||
const executorRef: { current: TaskExecutor | null } = { current: null };
|
const executorRef: { current: TaskExecutor | null } = { current: null };
|
||||||
const stuckTaskDetector = new StuckTaskDetector(store, {
|
const stuckTaskDetector = new StuckTaskDetector(store, {
|
||||||
beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId),
|
beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId),
|
||||||
onStuck: (taskId) => {
|
onStuck: (event) => {
|
||||||
executorRef.current?.markStuckAborted(taskId);
|
executorRef.current?.markStuckAborted(event.taskId);
|
||||||
console.log(`[engine] ⚠ ${taskId} stuck — terminated, will retry`);
|
console.log(
|
||||||
|
`[engine] ⚠ ${event.taskId} stuck (${event.reason}) — ` +
|
||||||
|
`no progress for ${Math.round(event.noProgressMs / 60_000)}min, ` +
|
||||||
|
`${event.activitySinceProgress} events since last progress — ` +
|
||||||
|
`terminated, will retry`,
|
||||||
|
);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -812,8 +812,13 @@ export class TaskExecutor {
|
|||||||
execute: async (_id: string, params: Static<typeof taskUpdateParams>) => {
|
execute: async (_id: string, params: Static<typeof taskUpdateParams>) => {
|
||||||
const { step, status } = params;
|
const { step, status } = params;
|
||||||
|
|
||||||
// Record heartbeat for stuck task detection
|
// Record step progress for stuck task detection.
|
||||||
stuckDetector?.recordActivity(taskId);
|
// Step transitions (in-progress, done, skipped) indicate real progress
|
||||||
|
// and reset the loop detection counter. Generic activity (text deltas,
|
||||||
|
// tool calls) is tracked separately via recordActivity in AgentLogger.
|
||||||
|
if (status === "in-progress" || status === "done" || status === "skipped") {
|
||||||
|
stuckDetector?.recordProgress(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
// Enforce code review REVISE: block advancing to "done" when the last
|
// Enforce code review REVISE: block advancing to "done" when the last
|
||||||
// code review for this step returned REVISE. The agent must fix the
|
// code review for this step returned REVISE. The agent must fix the
|
||||||
|
|||||||
@@ -89,6 +89,25 @@ describe("StuckTaskDetector", () => {
|
|||||||
expect(lastActivity).toBeLessThanOrEqual(after);
|
expect(lastActivity).toBeLessThanOrEqual(after);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("sets initial progress timestamp", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
const before = Date.now();
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
const after = Date.now();
|
||||||
|
|
||||||
|
const lastProgressAt = detector.getLastProgressAt("FN-001");
|
||||||
|
expect(lastProgressAt).toBeDefined();
|
||||||
|
expect(lastProgressAt).toBeGreaterThanOrEqual(before);
|
||||||
|
expect(lastProgressAt).toBeLessThanOrEqual(after);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("initializes activitySinceProgress to 0", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
expect(detector.getActivitySinceProgress("FN-001")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
it("can track multiple tasks", () => {
|
it("can track multiple tasks", () => {
|
||||||
detector.trackTask("FN-001", createMockSession());
|
detector.trackTask("FN-001", createMockSession());
|
||||||
detector.trackTask("FN-002", createMockSession());
|
detector.trackTask("FN-002", createMockSession());
|
||||||
@@ -129,12 +148,62 @@ describe("StuckTaskDetector", () => {
|
|||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("increments activitySinceProgress counter", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
expect(detector.getActivitySinceProgress("FN-001")).toBe(0);
|
||||||
|
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
expect(detector.getActivitySinceProgress("FN-001")).toBe(1);
|
||||||
|
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
expect(detector.getActivitySinceProgress("FN-001")).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
it("does nothing for untracked task", () => {
|
it("does nothing for untracked task", () => {
|
||||||
// Should not throw
|
// Should not throw
|
||||||
detector.recordActivity("FN-001");
|
detector.recordActivity("FN-001");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("recordProgress", () => {
|
||||||
|
it("updates lastProgressAt timestamp", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
const initialProgress = detector.getLastProgressAt("FN-001")!;
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(10);
|
||||||
|
detector.recordProgress("FN-001");
|
||||||
|
|
||||||
|
const newProgress = detector.getLastProgressAt("FN-001")!;
|
||||||
|
expect(newProgress).toBeGreaterThanOrEqual(initialProgress);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resets activitySinceProgress to 0", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Simulate some activity
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
expect(detector.getActivitySinceProgress("FN-001")).toBe(3);
|
||||||
|
|
||||||
|
// Progress resets the counter
|
||||||
|
detector.recordProgress("FN-001");
|
||||||
|
expect(detector.getActivitySinceProgress("FN-001")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does nothing for untracked task", () => {
|
||||||
|
// Should not throw
|
||||||
|
detector.recordProgress("FN-001");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("isStuck", () => {
|
describe("isStuck", () => {
|
||||||
it("returns false when no timeout exceeded", () => {
|
it("returns false when no timeout exceeded", () => {
|
||||||
const session = createMockSession();
|
const session = createMockSession();
|
||||||
@@ -160,6 +229,95 @@ describe("StuckTaskDetector", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("classifyStuckReason", () => {
|
||||||
|
it("returns null when not stuck", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns 'inactivity' when no activity at all for the timeout", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBe("inactivity");
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns 'loop' when active but no progress with high activity count", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Simulate time passing with lots of activity but no progress
|
||||||
|
vi.advanceTimersByTime(61000); // 61 seconds
|
||||||
|
|
||||||
|
// Simulate many activity heartbeats (agent is working but not advancing steps)
|
||||||
|
for (let i = 0; i < 60; i++) {
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inactivity is near-zero because we just called recordActivity, but
|
||||||
|
// noProgress is 61s. With activity >= 60, this should be a loop.
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBe("loop");
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null when no-progress timeout exceeded but activity count is below threshold", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Advance time past timeout
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
// Only a few activity events (below threshold of 60)
|
||||||
|
for (let i = 0; i < 30; i++) {
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should not be classified as stuck (not enough activity for loop,
|
||||||
|
// and activity just happened so inactivity timeout hasn't been hit)
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null for untracked task", () => {
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("progress resets loop detection: no loop after recordProgress", () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Simulate time passing with lots of activity
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
for (let i = 0; i < 80; i++) {
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
// This would be a loop...
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBe("loop");
|
||||||
|
|
||||||
|
// But after progress, it resets
|
||||||
|
detector.recordProgress("FN-001");
|
||||||
|
expect(detector.classifyStuckReason("FN-001", 60000)).toBeNull();
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("killAndRetry", () => {
|
describe("killAndRetry", () => {
|
||||||
it("disposes the session", async () => {
|
it("disposes the session", async () => {
|
||||||
const session = createMockSession();
|
const session = createMockSession();
|
||||||
@@ -190,7 +348,7 @@ describe("StuckTaskDetector", () => {
|
|||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("logs to task log", async () => {
|
it("logs to task log with reason", async () => {
|
||||||
const session = createMockSession();
|
const session = createMockSession();
|
||||||
detector.trackTask("FN-001", session);
|
detector.trackTask("FN-001", session);
|
||||||
|
|
||||||
@@ -201,7 +359,33 @@ describe("StuckTaskDetector", () => {
|
|||||||
|
|
||||||
expect(store.logEntry).toHaveBeenCalledWith(
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
"FN-001",
|
"FN-001",
|
||||||
expect.stringContaining("Task terminated due to stuck agent session")
|
expect.stringContaining("Task terminated due to stuck agent session"),
|
||||||
|
);
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-001",
|
||||||
|
expect.stringContaining("reason=inactivity"),
|
||||||
|
);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("logs loop reason when activity detected", async () => {
|
||||||
|
const session = createMockSession();
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-001", session);
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
// Simulate lots of activity (loop behavior)
|
||||||
|
for (let i = 0; i < 80; i++) {
|
||||||
|
detector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
await detector.killAndRetry("FN-001", 60000);
|
||||||
|
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-001",
|
||||||
|
expect.stringContaining("reason=loop"),
|
||||||
);
|
);
|
||||||
|
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
@@ -222,7 +406,7 @@ describe("StuckTaskDetector", () => {
|
|||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("calls onStuck callback", async () => {
|
it("calls onStuck callback with structured event payload", async () => {
|
||||||
const onStuck = vi.fn();
|
const onStuck = vi.fn();
|
||||||
const customDetector = new StuckTaskDetector(store, { onStuck });
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
const session = createMockSession();
|
const session = createMockSession();
|
||||||
@@ -234,7 +418,42 @@ describe("StuckTaskDetector", () => {
|
|||||||
|
|
||||||
await customDetector.killAndRetry("FN-001", 60000);
|
await customDetector.killAndRetry("FN-001", 60000);
|
||||||
|
|
||||||
expect(onStuck).toHaveBeenCalledWith("FN-001");
|
expect(onStuck).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
taskId: "FN-001",
|
||||||
|
reason: "inactivity",
|
||||||
|
noProgressMs: expect.any(Number),
|
||||||
|
inactivityMs: expect.any(Number),
|
||||||
|
activitySinceProgress: 0,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("calls onStuck with loop reason and activity count", async () => {
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
customDetector.trackTask("FN-001", session);
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
for (let i = 0; i < 80; i++) {
|
||||||
|
customDetector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
await customDetector.killAndRetry("FN-001", 60000);
|
||||||
|
|
||||||
|
expect(onStuck).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
taskId: "FN-001",
|
||||||
|
reason: "loop",
|
||||||
|
activitySinceProgress: 80,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
});
|
});
|
||||||
@@ -261,7 +480,7 @@ describe("StuckTaskDetector", () => {
|
|||||||
expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
|
expect(beforeRequeue).toHaveBeenCalledWith("FN-001");
|
||||||
expect(session.dispose).toHaveBeenCalled();
|
expect(session.dispose).toHaveBeenCalled();
|
||||||
// onStuck should still be called (so executor can mark stuck-aborted)
|
// onStuck should still be called (so executor can mark stuck-aborted)
|
||||||
expect(onStuck).toHaveBeenCalledWith("FN-001");
|
expect(onStuck).toHaveBeenCalled();
|
||||||
// But task should NOT be moved to todo
|
// But task should NOT be moved to todo
|
||||||
expect(store.moveTask).not.toHaveBeenCalled();
|
expect(store.moveTask).not.toHaveBeenCalled();
|
||||||
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
|
expect(store.updateTask).not.toHaveBeenCalledWith("FN-001", { status: "stuck-killed" });
|
||||||
@@ -404,4 +623,146 @@ describe("StuckTaskDetector", () => {
|
|||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("dual detection: inactivity vs loop", () => {
|
||||||
|
it("detects inactivity when agent goes silent (no text/tool calls)", async () => {
|
||||||
|
store = createMockStore({
|
||||||
|
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
|
||||||
|
});
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
customDetector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
// No activity at all for 61 seconds
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
await customDetector.checkNow();
|
||||||
|
|
||||||
|
expect(onStuck).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
taskId: "FN-001",
|
||||||
|
reason: "inactivity",
|
||||||
|
activitySinceProgress: 0,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo");
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("detects loop when agent is active but not making step progress", async () => {
|
||||||
|
store = createMockStore({
|
||||||
|
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
|
||||||
|
});
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
customDetector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Advance past timeout
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
// Agent is actively generating text/tool calls but not advancing steps
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
customDetector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
await customDetector.checkNow();
|
||||||
|
|
||||||
|
expect(onStuck).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
taskId: "FN-001",
|
||||||
|
reason: "loop",
|
||||||
|
activitySinceProgress: 100,
|
||||||
|
noProgressMs: expect.any(Number),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not trigger loop when activity is below threshold", async () => {
|
||||||
|
store = createMockStore({
|
||||||
|
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
|
||||||
|
});
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
customDetector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Advance past timeout
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
// Only 30 activity events (below threshold of 60)
|
||||||
|
for (let i = 0; i < 30; i++) {
|
||||||
|
customDetector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
await customDetector.checkNow();
|
||||||
|
|
||||||
|
// Should NOT trigger — activity is recent but below loop threshold
|
||||||
|
expect(onStuck).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("progress resets counters and prevents loop detection", async () => {
|
||||||
|
store = createMockStore({
|
||||||
|
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: 60000 }),
|
||||||
|
});
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
customDetector.trackTask("FN-001", session);
|
||||||
|
|
||||||
|
// Advance past timeout and generate lots of activity
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
customDetector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
// This would be a loop...
|
||||||
|
await customDetector.checkNow();
|
||||||
|
expect(onStuck).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("timeout disabled disables both inactivity and loop paths", async () => {
|
||||||
|
store = createMockStore({
|
||||||
|
getSettings: vi.fn().mockResolvedValue({ taskStuckTimeoutMs: undefined }),
|
||||||
|
});
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const customDetector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||||
|
|
||||||
|
customDetector.trackTask("FN-001", session);
|
||||||
|
vi.advanceTimersByTime(61000);
|
||||||
|
|
||||||
|
// Even with lots of activity
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
customDetector.recordActivity("FN-001");
|
||||||
|
}
|
||||||
|
|
||||||
|
await customDetector.checkNow();
|
||||||
|
|
||||||
|
expect(onStuck).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,14 +1,17 @@
|
|||||||
/**
|
/**
|
||||||
* Stuck Task Detector — monitors in-progress tasks for agent session stagnation.
|
* Stuck Task Detector — monitors in-progress tasks for agent session stagnation.
|
||||||
*
|
*
|
||||||
* When a task's agent session shows no activity (no text deltas, tool calls, or
|
* The detector supports two detection modes:
|
||||||
* progress updates) for longer than the configured timeout, the detector
|
* - **Inactivity** — no activity at all for the timeout period (session appears dead)
|
||||||
* terminates the stuck session and triggers recovery (moving the task back to
|
* - **Loop** — agent is active but making no step progress despite lots of activity
|
||||||
* "todo" for the scheduler to retry).
|
* (e.g., context growth causing the agent to repeat itself without advancing steps)
|
||||||
*
|
*
|
||||||
* Activity is tracked via `recordActivity(taskId)` calls from the executor's
|
* Activity tracking uses two signals:
|
||||||
* agent event handlers. The detector polls at a configurable interval and
|
* - `recordActivity(taskId)` — text/tool heartbeats only; increments `activitySinceProgress`
|
||||||
* compares the last activity timestamp against `taskStuckTimeoutMs` from settings.
|
* - `recordProgress(taskId)` — step transitions (in-progress, done, skipped); resets counters
|
||||||
|
*
|
||||||
|
* The detector polls at a configurable interval and compares timestamps against
|
||||||
|
* `taskStuckTimeoutMs` from settings.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TaskStore, Settings } from "@fusion/core";
|
import type { TaskStore, Settings } from "@fusion/core";
|
||||||
@@ -24,15 +27,39 @@ export interface DisposableSession {
|
|||||||
/** Tracked entry for a single in-progress task. */
|
/** Tracked entry for a single in-progress task. */
|
||||||
interface TrackedTask {
|
interface TrackedTask {
|
||||||
session: DisposableSession;
|
session: DisposableSession;
|
||||||
|
/** Timestamp of the last heartbeat (text delta, tool call, etc.). */
|
||||||
lastActivity: number;
|
lastActivity: number;
|
||||||
|
/** Timestamp of the last step progress event. */
|
||||||
|
lastProgressAt: number;
|
||||||
|
/** Number of activity heartbeats since the last progress event. */
|
||||||
|
activitySinceProgress: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Payload emitted when a stuck task is detected. */
|
||||||
|
export interface StuckTaskEvent {
|
||||||
|
/** The task that was detected as stuck. */
|
||||||
|
taskId: string;
|
||||||
|
/** Why the task is considered stuck. */
|
||||||
|
reason: "inactivity" | "loop";
|
||||||
|
/** Milliseconds since the last step progress event. */
|
||||||
|
noProgressMs: number;
|
||||||
|
/** Milliseconds since the last activity heartbeat. */
|
||||||
|
inactivityMs: number;
|
||||||
|
/** Number of activity heartbeats since the last progress event. */
|
||||||
|
activitySinceProgress: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Minimum activity-since-progress count to classify as a loop.
|
||||||
|
* Prevents false positives when a task is genuinely inactive. */
|
||||||
|
const LOOP_ACTIVITY_THRESHOLD = 60;
|
||||||
|
|
||||||
export interface StuckTaskDetectorOptions {
|
export interface StuckTaskDetectorOptions {
|
||||||
/** Polling interval in milliseconds. Default: 30000 (30 seconds). */
|
/** Polling interval in milliseconds. Default: 30000 (30 seconds). */
|
||||||
pollIntervalMs?: number;
|
pollIntervalMs?: number;
|
||||||
/** Callback invoked when a stuck task is detected and killed.
|
/** Callback invoked when a stuck task is detected.
|
||||||
* The task will be moved to "todo" for retry by the detector. */
|
* The task will be moved to "todo" for retry by the detector.
|
||||||
onStuck?: (taskId: string) => void;
|
* Receives a structured payload with detection reason and metrics. */
|
||||||
|
onStuck?: (event: StuckTaskEvent) => void;
|
||||||
/** Called before re-queuing a killed task. Return false to prevent re-queue
|
/** Called before re-queuing a killed task. Return false to prevent re-queue
|
||||||
* (caller is responsible for marking the task as terminally failed).
|
* (caller is responsible for marking the task as terminally failed).
|
||||||
* Used by SelfHealingManager to enforce stuck kill budgets. */
|
* Used by SelfHealingManager to enforce stuck kill budgets. */
|
||||||
@@ -43,7 +70,7 @@ export class StuckTaskDetector {
|
|||||||
private tracked = new Map<string, TrackedTask>();
|
private tracked = new Map<string, TrackedTask>();
|
||||||
private interval: ReturnType<typeof setInterval> | null = null;
|
private interval: ReturnType<typeof setInterval> | null = null;
|
||||||
private pollIntervalMs: number;
|
private pollIntervalMs: number;
|
||||||
private onStuck?: (taskId: string) => void;
|
private onStuck?: (event: StuckTaskEvent) => void;
|
||||||
private beforeRequeue?: (taskId: string) => Promise<boolean>;
|
private beforeRequeue?: (taskId: string) => Promise<boolean>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@@ -84,12 +111,15 @@ export class StuckTaskDetector {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Register an active agent session for monitoring.
|
* Register an active agent session for monitoring.
|
||||||
* Sets the initial activity timestamp to now.
|
* Sets initial timestamps and counters to now.
|
||||||
*/
|
*/
|
||||||
trackTask(taskId: string, session: DisposableSession): void {
|
trackTask(taskId: string, session: DisposableSession): void {
|
||||||
|
const now = Date.now();
|
||||||
this.tracked.set(taskId, {
|
this.tracked.set(taskId, {
|
||||||
session,
|
session,
|
||||||
lastActivity: Date.now(),
|
lastActivity: now,
|
||||||
|
lastProgressAt: now,
|
||||||
|
activitySinceProgress: 0,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -103,12 +133,27 @@ export class StuckTaskDetector {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Record a heartbeat for a task's agent session.
|
* Record a heartbeat for a task's agent session.
|
||||||
* Called on text deltas, tool calls, and progress updates.
|
* Called on text deltas and tool calls only (NOT step transitions).
|
||||||
|
* Increments `activitySinceProgress` counter.
|
||||||
*/
|
*/
|
||||||
recordActivity(taskId: string): void {
|
recordActivity(taskId: string): void {
|
||||||
const entry = this.tracked.get(taskId);
|
const entry = this.tracked.get(taskId);
|
||||||
if (entry) {
|
if (entry) {
|
||||||
entry.lastActivity = Date.now();
|
entry.lastActivity = Date.now();
|
||||||
|
entry.activitySinceProgress++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a step progress event for a task's agent session.
|
||||||
|
* Called on step transitions (in-progress, done, skipped).
|
||||||
|
* Resets `activitySinceProgress` to 0 and updates `lastProgressAt`.
|
||||||
|
*/
|
||||||
|
recordProgress(taskId: string): void {
|
||||||
|
const entry = this.tracked.get(taskId);
|
||||||
|
if (entry) {
|
||||||
|
entry.lastProgressAt = Date.now();
|
||||||
|
entry.activitySinceProgress = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,6 +165,22 @@ export class StuckTaskDetector {
|
|||||||
return this.tracked.get(taskId)?.lastActivity;
|
return this.tracked.get(taskId)?.lastActivity;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the activity-since-progress count for a tracked task.
|
||||||
|
* Returns undefined if the task is not tracked.
|
||||||
|
*/
|
||||||
|
getActivitySinceProgress(taskId: string): number | undefined {
|
||||||
|
return this.tracked.get(taskId)?.activitySinceProgress;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the last progress timestamp for a tracked task.
|
||||||
|
* Returns undefined if the task is not tracked.
|
||||||
|
*/
|
||||||
|
getLastProgressAt(taskId: string): number | undefined {
|
||||||
|
return this.tracked.get(taskId)?.lastProgressAt;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check whether a task is stuck (no activity for longer than timeout).
|
* Check whether a task is stuck (no activity for longer than timeout).
|
||||||
*/
|
*/
|
||||||
@@ -129,6 +190,31 @@ export class StuckTaskDetector {
|
|||||||
return (Date.now() - entry.lastActivity) > timeoutMs;
|
return (Date.now() - entry.lastActivity) > timeoutMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify why a task is stuck.
|
||||||
|
* Returns null if the task is not stuck.
|
||||||
|
*/
|
||||||
|
classifyStuckReason(taskId: string, timeoutMs: number): "inactivity" | "loop" | null {
|
||||||
|
const entry = this.tracked.get(taskId);
|
||||||
|
if (!entry) return null;
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const inactivityMs = now - entry.lastActivity;
|
||||||
|
const noProgressMs = now - entry.lastProgressAt;
|
||||||
|
|
||||||
|
// Check inactivity first — if there's been zero activity, it's just inactive
|
||||||
|
if (inactivityMs >= timeoutMs) {
|
||||||
|
return "inactivity";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check loop — active but not making progress, with enough activity to be a real loop
|
||||||
|
if (noProgressMs >= timeoutMs && entry.activitySinceProgress >= LOOP_ACTIVITY_THRESHOLD) {
|
||||||
|
return "loop";
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Terminate a stuck task's agent session and trigger recovery.
|
* Terminate a stuck task's agent session and trigger recovery.
|
||||||
* - Disposes the agent session
|
* - Disposes the agent session
|
||||||
@@ -136,13 +222,27 @@ export class StuckTaskDetector {
|
|||||||
* - Moves the task back to "todo" (preserving step progress)
|
* - Moves the task back to "todo" (preserving step progress)
|
||||||
* - Invokes the onStuck callback
|
* - Invokes the onStuck callback
|
||||||
*/
|
*/
|
||||||
async killAndRetry(taskId: string, _timeoutMs: number): Promise<void> {
|
async killAndRetry(taskId: string, timeoutMs: number): Promise<void> {
|
||||||
const entry = this.tracked.get(taskId);
|
const entry = this.tracked.get(taskId);
|
||||||
if (!entry) return;
|
if (!entry) return;
|
||||||
|
|
||||||
const elapsedMin = Math.round((Date.now() - entry.lastActivity) / 60_000);
|
const now = Date.now();
|
||||||
|
const inactivityMs = now - entry.lastActivity;
|
||||||
|
const noProgressMs = now - entry.lastProgressAt;
|
||||||
|
const activitySinceProgress = entry.activitySinceProgress;
|
||||||
|
|
||||||
stuckLog.log(`Killing stuck task ${taskId} (no activity for ~${elapsedMin} minutes)`);
|
// Classify the reason
|
||||||
|
const reason = this.classifyStuckReason(taskId, timeoutMs) ?? "inactivity";
|
||||||
|
|
||||||
|
const elapsedMin = Math.round(inactivityMs / 60_000);
|
||||||
|
const noProgressMin = Math.round(noProgressMs / 60_000);
|
||||||
|
|
||||||
|
stuckLog.log(
|
||||||
|
`Killing stuck task ${taskId} (reason=${reason}, ` +
|
||||||
|
`no progress for ~${noProgressMin}min, ` +
|
||||||
|
`no activity for ~${elapsedMin}min, ` +
|
||||||
|
`${activitySinceProgress} events since last progress)`,
|
||||||
|
);
|
||||||
|
|
||||||
// Dispose the agent session first
|
// Dispose the agent session first
|
||||||
try {
|
try {
|
||||||
@@ -158,12 +258,24 @@ export class StuckTaskDetector {
|
|||||||
try {
|
try {
|
||||||
await this.store.logEntry(
|
await this.store.logEntry(
|
||||||
taskId,
|
taskId,
|
||||||
`Task terminated due to stuck agent session (no activity for ~${elapsedMin} minutes)`,
|
`Task terminated due to stuck agent session (reason=${reason}, ` +
|
||||||
|
`no progress for ~${noProgressMin}min, ` +
|
||||||
|
`no activity for ~${elapsedMin}min, ` +
|
||||||
|
`${activitySinceProgress} events since last progress)`,
|
||||||
);
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
|
stuckLog.error(`Failed to log stuck event for ${taskId}:`, err);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Build the event payload
|
||||||
|
const event: StuckTaskEvent = {
|
||||||
|
taskId,
|
||||||
|
reason,
|
||||||
|
noProgressMs,
|
||||||
|
inactivityMs,
|
||||||
|
activitySinceProgress,
|
||||||
|
};
|
||||||
|
|
||||||
// Check stuck kill budget before re-queuing (SelfHealingManager integration).
|
// Check stuck kill budget before re-queuing (SelfHealingManager integration).
|
||||||
// If beforeRequeue returns false, the task has been marked failed — skip re-queue.
|
// If beforeRequeue returns false, the task has been marked failed — skip re-queue.
|
||||||
if (this.beforeRequeue) {
|
if (this.beforeRequeue) {
|
||||||
@@ -171,7 +283,7 @@ export class StuckTaskDetector {
|
|||||||
const shouldRequeue = await this.beforeRequeue(taskId);
|
const shouldRequeue = await this.beforeRequeue(taskId);
|
||||||
if (!shouldRequeue) {
|
if (!shouldRequeue) {
|
||||||
stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
|
stuckLog.log(`${taskId} exceeded stuck kill budget — not re-queuing`);
|
||||||
this.onStuck?.(taskId);
|
this.onStuck?.(event);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -193,7 +305,7 @@ export class StuckTaskDetector {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Notify listeners
|
// Notify listeners
|
||||||
this.onStuck?.(taskId);
|
this.onStuck?.(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -210,6 +322,11 @@ export class StuckTaskDetector {
|
|||||||
* Poll all tracked tasks and kill any that have exceeded the timeout.
|
* Poll all tracked tasks and kill any that have exceeded the timeout.
|
||||||
* Reads `taskStuckTimeoutMs` from settings on each check so changes
|
* Reads `taskStuckTimeoutMs` from settings on each check so changes
|
||||||
* take effect on the next poll cycle.
|
* take effect on the next poll cycle.
|
||||||
|
*
|
||||||
|
* Detection rules:
|
||||||
|
* - **inactivity**: `lastActivity` older than `taskStuckTimeoutMs` (no heartbeats at all)
|
||||||
|
* - **loop**: `lastProgressAt` older than `taskStuckTimeoutMs` AND `activitySinceProgress >= 60`
|
||||||
|
* (agent is actively doing things but not advancing steps)
|
||||||
*/
|
*/
|
||||||
private async checkStuckTasks(): Promise<void> {
|
private async checkStuckTasks(): Promise<void> {
|
||||||
if (this.tracked.size === 0) return;
|
if (this.tracked.size === 0) return;
|
||||||
@@ -224,11 +341,11 @@ export class StuckTaskDetector {
|
|||||||
const timeoutMs = settings.taskStuckTimeoutMs;
|
const timeoutMs = settings.taskStuckTimeoutMs;
|
||||||
if (!timeoutMs || timeoutMs <= 0) return; // Disabled
|
if (!timeoutMs || timeoutMs <= 0) return; // Disabled
|
||||||
|
|
||||||
const now = Date.now();
|
|
||||||
const stuckTasks: string[] = [];
|
const stuckTasks: string[] = [];
|
||||||
|
|
||||||
for (const [taskId, entry] of this.tracked) {
|
for (const [taskId] of this.tracked) {
|
||||||
if ((now - entry.lastActivity) > timeoutMs) {
|
const reason = this.classifyStuckReason(taskId, timeoutMs);
|
||||||
|
if (reason !== null) {
|
||||||
stuckTasks.push(taskId);
|
stuckTasks.push(taskId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user