feat(KB-249): immediate stuck task check on timeout setting change

- Add checkNow() method to StuckTaskDetector for immediate stuck task detection\n- Add comprehensive tests for checkNow() method\n- Wire up settings change handler in dashboard command\n- Update AGENTS.md documentation with immediate check behavior
This commit is contained in:
gsxdsm
2026-03-30 23:39:04 -07:00
parent ee4a64fc98
commit cea55fa665
4 changed files with 119 additions and 0 deletions

View File

@@ -220,6 +220,7 @@ Timeout in milliseconds for detecting stuck tasks. When a task's agent session s
- When the timeout value is changed (e.g., reduced from 30 to 10 minutes), the system immediately checks for stuck tasks under the new timer rather than waiting for the next 30-second poll cycle - When the timeout value is changed (e.g., reduced from 30 to 10 minutes), the system immediately checks for stuck tasks under the new timer rather than waiting for the next 30-second poll cycle
- Paused tasks are automatically untracked from monitoring - Paused tasks are automatically untracked from monitoring
- The timeout is read from settings on every poll cycle, so changes take effect immediately - The timeout is read from settings on every poll cycle, so changes take effect immediately
- When the timeout value is changed (e.g., reduced from 30 to 10 minutes), the system immediately checks for stuck tasks under the new timer rather than waiting for the next poll cycle
### `worktreeNaming` (default: `"random"`) ### `worktreeNaming` (default: `"random"`)

View File

@@ -582,6 +582,16 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
} }
}); });
// ── Stuck task timeout change: immediate check ────────────────────
// When taskStuckTimeoutMs is changed (e.g., user reduces timeout),
// immediately check for stuck tasks under the new timer value.
store.on("settings:updated", async ({ settings: s, previous: prev }) => {
if (s.taskStuckTimeoutMs !== prev.taskStuckTimeoutMs) {
console.log(`[stuck-detector] Timeout changed to ${s.taskStuckTimeoutMs}ms — running immediate check`);
await stuckTaskDetector.checkNow();
}
});
// ── Periodic retry: catch failed merges on each poll cycle ──────── // ── Periodic retry: catch failed merges on each poll cycle ────────
// Uses a setTimeout chain so the interval dynamically follows // Uses a setTimeout chain so the interval dynamically follows
// settings.pollIntervalMs without requiring an engine restart. // settings.pollIntervalMs without requiring an engine restart.

View File

@@ -340,6 +340,103 @@ describe("StuckTaskDetector", () => {
}); });
}); });
describe("checkNow (immediate check)", () => {
it("detects and kills stuck tasks immediately", async () => {
const store = createMockStore({ taskStuckTimeoutMs: 60_000 });
const onStuck = vi.fn();
detector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
detector.trackTask("KB-001", session);
// Backdate to be stuck (2 minutes > 1 minute timeout)
const entry = (detector as any).tracked.get("KB-001");
entry.lastActivity = Date.now() - 120_000;
// Call checkNow instead of polling
await detector.checkNow();
// Should kill the stuck task
expect(session.dispose).toHaveBeenCalledOnce();
expect(onStuck).toHaveBeenCalledWith("KB-001");
expect(detector.trackedCount).toBe(0);
});
it("is safe to call when no tasks are tracked", async () => {
const store = createMockStore({ taskStuckTimeoutMs: 60_000 });
detector = new StuckTaskDetector(store);
// Should not throw or call settings when no tasks tracked
await detector.checkNow();
expect(store.getSettings).not.toHaveBeenCalled();
});
it("is safe to call when timeout is disabled (undefined)", async () => {
const store = createMockStore({ taskStuckTimeoutMs: undefined });
detector = new StuckTaskDetector(store);
const session = createMockSession();
detector.trackTask("KB-001", session);
// Backdate to be "stuck"
const entry = (detector as any).tracked.get("KB-001");
entry.lastActivity = Date.now() - 9999_000;
// Should not throw or kill anything
await detector.checkNow();
expect(session.dispose).not.toHaveBeenCalled();
expect(detector.trackedCount).toBe(1); // Still tracked
});
it("respects current timeout value from settings", async () => {
const store = createMockStore({ taskStuckTimeoutMs: 300_000 }); // 5 minutes
const onStuck = vi.fn();
detector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
detector.trackTask("KB-001", session);
// Backdate to 2 minutes ago
const entry = (detector as any).tracked.get("KB-001");
entry.lastActivity = Date.now() - 120_000;
// With 5-minute timeout, should not be stuck
await detector.checkNow();
expect(onStuck).not.toHaveBeenCalled();
expect(session.dispose).not.toHaveBeenCalled();
// Change settings to 1-minute timeout
store._settings.taskStuckTimeoutMs = 60_000;
// Now checkNow should detect it as stuck
await detector.checkNow();
expect(onStuck).toHaveBeenCalledWith("KB-001");
expect(session.dispose).toHaveBeenCalledOnce();
});
it("is safe to call when detector is stopped", async () => {
const store = createMockStore({ taskStuckTimeoutMs: 60_000 });
const onStuck = vi.fn();
detector = new StuckTaskDetector(store, { onStuck });
const session = createMockSession();
detector.trackTask("KB-001", session);
// Backdate to be stuck
const entry = (detector as any).tracked.get("KB-001");
entry.lastActivity = Date.now() - 120_000;
// Don't start the detector - just call checkNow
await detector.checkNow();
// Should still work even though detector was never started
expect(session.dispose).toHaveBeenCalledOnce();
expect(onStuck).toHaveBeenCalledWith("KB-001");
});
});
describe("start / stop", () => { describe("start / stop", () => {
it("starts polling and can be stopped", async () => { it("starts polling and can be stopped", async () => {
const store = createMockStore({ taskStuckTimeoutMs: 100 }); const store = createMockStore({ taskStuckTimeoutMs: 100 });

View File

@@ -56,6 +56,7 @@ export class StuckTaskDetector {
start(): void { start(): void {
if (this.interval) return; if (this.interval) return;
this.interval = setInterval(() => { this.interval = setInterval(() => {
stuckLog.log("Running periodic stuck task check (polling)");
this.checkStuckTasks().catch((err) => { this.checkStuckTasks().catch((err) => {
stuckLog.error("Error checking stuck tasks:", err); stuckLog.error("Error checking stuck tasks:", err);
}); });
@@ -173,6 +174,16 @@ export class StuckTaskDetector {
this.onStuck?.(taskId); this.onStuck?.(taskId);
} }
/**
* Check for stuck tasks immediately, outside the normal polling cycle.
* Safe to call at any time — will no-op if no tasks are tracked or timeout is disabled.
* Logs at debug level to distinguish manual checks from polling.
*/
async checkNow(): Promise<void> {
stuckLog.log("Running immediate stuck task check (triggered manually)");
await this.checkStuckTasks();
}
/** /**
* Poll all tracked tasks and kill any that have exceeded the timeout. * Poll all tracked tasks and kill any that have exceeded the timeout.
* Reads `taskStuckTimeoutMs` from settings on each check so changes * Reads `taskStuckTimeoutMs` from settings on each check so changes