feat(KB-249): immediate stuck task check on timeout setting change
- Add checkNow() method to StuckTaskDetector for immediate stuck task detection\n- Add comprehensive tests for checkNow() method\n- Wire up settings change handler in dashboard command\n- Update AGENTS.md documentation with immediate check behavior
This commit is contained in:
@@ -220,6 +220,7 @@ Timeout in milliseconds for detecting stuck tasks. When a task's agent session s
|
|||||||
- When the timeout value is changed (e.g., reduced from 30 to 10 minutes), the system immediately checks for stuck tasks under the new timer rather than waiting for the next 30-second poll cycle
|
- When the timeout value is changed (e.g., reduced from 30 to 10 minutes), the system immediately checks for stuck tasks under the new timer rather than waiting for the next 30-second poll cycle
|
||||||
- Paused tasks are automatically untracked from monitoring
|
- Paused tasks are automatically untracked from monitoring
|
||||||
- The timeout is read from settings on every poll cycle, so changes take effect immediately
|
- The timeout is read from settings on every poll cycle, so changes take effect immediately
|
||||||
|
- When the timeout value is changed (e.g., reduced from 30 to 10 minutes), the system immediately checks for stuck tasks under the new timer rather than waiting for the next poll cycle
|
||||||
|
|
||||||
### `worktreeNaming` (default: `"random"`)
|
### `worktreeNaming` (default: `"random"`)
|
||||||
|
|
||||||
|
|||||||
@@ -582,6 +582,16 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── Stuck task timeout change: immediate check ────────────────────
|
||||||
|
// When taskStuckTimeoutMs is changed (e.g., user reduces timeout),
|
||||||
|
// immediately check for stuck tasks under the new timer value.
|
||||||
|
store.on("settings:updated", async ({ settings: s, previous: prev }) => {
|
||||||
|
if (s.taskStuckTimeoutMs !== prev.taskStuckTimeoutMs) {
|
||||||
|
console.log(`[stuck-detector] Timeout changed to ${s.taskStuckTimeoutMs}ms — running immediate check`);
|
||||||
|
await stuckTaskDetector.checkNow();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// ── Periodic retry: catch failed merges on each poll cycle ────────
|
// ── Periodic retry: catch failed merges on each poll cycle ────────
|
||||||
// Uses a setTimeout chain so the interval dynamically follows
|
// Uses a setTimeout chain so the interval dynamically follows
|
||||||
// settings.pollIntervalMs without requiring an engine restart.
|
// settings.pollIntervalMs without requiring an engine restart.
|
||||||
|
|||||||
@@ -340,6 +340,103 @@ describe("StuckTaskDetector", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("checkNow (immediate check)", () => {
|
||||||
|
it("detects and kills stuck tasks immediately", async () => {
|
||||||
|
const store = createMockStore({ taskStuckTimeoutMs: 60_000 });
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
detector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("KB-001", session);
|
||||||
|
|
||||||
|
// Backdate to be stuck (2 minutes > 1 minute timeout)
|
||||||
|
const entry = (detector as any).tracked.get("KB-001");
|
||||||
|
entry.lastActivity = Date.now() - 120_000;
|
||||||
|
|
||||||
|
// Call checkNow instead of polling
|
||||||
|
await detector.checkNow();
|
||||||
|
|
||||||
|
// Should kill the stuck task
|
||||||
|
expect(session.dispose).toHaveBeenCalledOnce();
|
||||||
|
expect(onStuck).toHaveBeenCalledWith("KB-001");
|
||||||
|
expect(detector.trackedCount).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is safe to call when no tasks are tracked", async () => {
|
||||||
|
const store = createMockStore({ taskStuckTimeoutMs: 60_000 });
|
||||||
|
detector = new StuckTaskDetector(store);
|
||||||
|
|
||||||
|
// Should not throw or call settings when no tasks tracked
|
||||||
|
await detector.checkNow();
|
||||||
|
|
||||||
|
expect(store.getSettings).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is safe to call when timeout is disabled (undefined)", async () => {
|
||||||
|
const store = createMockStore({ taskStuckTimeoutMs: undefined });
|
||||||
|
detector = new StuckTaskDetector(store);
|
||||||
|
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("KB-001", session);
|
||||||
|
|
||||||
|
// Backdate to be "stuck"
|
||||||
|
const entry = (detector as any).tracked.get("KB-001");
|
||||||
|
entry.lastActivity = Date.now() - 9999_000;
|
||||||
|
|
||||||
|
// Should not throw or kill anything
|
||||||
|
await detector.checkNow();
|
||||||
|
|
||||||
|
expect(session.dispose).not.toHaveBeenCalled();
|
||||||
|
expect(detector.trackedCount).toBe(1); // Still tracked
|
||||||
|
});
|
||||||
|
|
||||||
|
it("respects current timeout value from settings", async () => {
|
||||||
|
const store = createMockStore({ taskStuckTimeoutMs: 300_000 }); // 5 minutes
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
detector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("KB-001", session);
|
||||||
|
|
||||||
|
// Backdate to 2 minutes ago
|
||||||
|
const entry = (detector as any).tracked.get("KB-001");
|
||||||
|
entry.lastActivity = Date.now() - 120_000;
|
||||||
|
|
||||||
|
// With 5-minute timeout, should not be stuck
|
||||||
|
await detector.checkNow();
|
||||||
|
expect(onStuck).not.toHaveBeenCalled();
|
||||||
|
expect(session.dispose).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// Change settings to 1-minute timeout
|
||||||
|
store._settings.taskStuckTimeoutMs = 60_000;
|
||||||
|
|
||||||
|
// Now checkNow should detect it as stuck
|
||||||
|
await detector.checkNow();
|
||||||
|
expect(onStuck).toHaveBeenCalledWith("KB-001");
|
||||||
|
expect(session.dispose).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is safe to call when detector is stopped", async () => {
|
||||||
|
const store = createMockStore({ taskStuckTimeoutMs: 60_000 });
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
detector = new StuckTaskDetector(store, { onStuck });
|
||||||
|
|
||||||
|
const session = createMockSession();
|
||||||
|
detector.trackTask("KB-001", session);
|
||||||
|
|
||||||
|
// Backdate to be stuck
|
||||||
|
const entry = (detector as any).tracked.get("KB-001");
|
||||||
|
entry.lastActivity = Date.now() - 120_000;
|
||||||
|
|
||||||
|
// Don't start the detector - just call checkNow
|
||||||
|
await detector.checkNow();
|
||||||
|
|
||||||
|
// Should still work even though detector was never started
|
||||||
|
expect(session.dispose).toHaveBeenCalledOnce();
|
||||||
|
expect(onStuck).toHaveBeenCalledWith("KB-001");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("start / stop", () => {
|
describe("start / stop", () => {
|
||||||
it("starts polling and can be stopped", async () => {
|
it("starts polling and can be stopped", async () => {
|
||||||
const store = createMockStore({ taskStuckTimeoutMs: 100 });
|
const store = createMockStore({ taskStuckTimeoutMs: 100 });
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ export class StuckTaskDetector {
|
|||||||
start(): void {
|
start(): void {
|
||||||
if (this.interval) return;
|
if (this.interval) return;
|
||||||
this.interval = setInterval(() => {
|
this.interval = setInterval(() => {
|
||||||
|
stuckLog.log("Running periodic stuck task check (polling)");
|
||||||
this.checkStuckTasks().catch((err) => {
|
this.checkStuckTasks().catch((err) => {
|
||||||
stuckLog.error("Error checking stuck tasks:", err);
|
stuckLog.error("Error checking stuck tasks:", err);
|
||||||
});
|
});
|
||||||
@@ -173,6 +174,16 @@ export class StuckTaskDetector {
|
|||||||
this.onStuck?.(taskId);
|
this.onStuck?.(taskId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check for stuck tasks immediately, outside the normal polling cycle.
|
||||||
|
* Safe to call at any time — will no-op if no tasks are tracked or timeout is disabled.
|
||||||
|
* Logs at debug level to distinguish manual checks from polling.
|
||||||
|
*/
|
||||||
|
async checkNow(): Promise<void> {
|
||||||
|
stuckLog.log("Running immediate stuck task check (triggered manually)");
|
||||||
|
await this.checkStuckTasks();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Poll all tracked tasks and kill any that have exceeded the timeout.
|
* Poll all tracked tasks and kill any that have exceeded the timeout.
|
||||||
* Reads `taskStuckTimeoutMs` from settings on each check so changes
|
* Reads `taskStuckTimeoutMs` from settings on each check so changes
|
||||||
|
|||||||
Reference in New Issue
Block a user