675 lines
26 KiB
TypeScript
675 lines
26 KiB
TypeScript
/**
|
|
* Stuck Task Detector — monitors in-progress tasks for agent session stagnation.
|
|
*
|
|
* The detector supports two detection modes:
|
|
* - **Inactivity** — no activity at all for the timeout period (session appears dead)
|
|
* - **Loop** — agent is active but making no step progress despite lots of activity
|
|
* (e.g., context growth causing the agent to repeat itself without advancing steps)
|
|
*
|
|
* Activity tracking uses two signals:
|
|
* - `recordActivity(taskId)` — text/tool heartbeats only; increments `activitySinceProgress`
|
|
* - `recordProgress(taskId)` — step transitions (in-progress, done, skipped); resets counters
|
|
*
|
|
* The detector polls at a configurable interval and compares timestamps against
|
|
* `taskStuckTimeoutMs` from settings. Project defaults keep this active by
|
|
* default while keeping workflow-step execution timeouts independent.
|
|
*/
|
|
|
|
import type { TaskStore, Settings } from "@fusion/core";
|
|
import { createLogger } from "./logger.js";
|
|
|
|
const stuckLog = createLogger("stuck-detector");
|
|
|
|
/** Minimal session interface — matches what TaskExecutor stores. */
|
|
export interface DisposableSession {
|
|
dispose: () => void;
|
|
}
|
|
|
|
/** Tracked entry for a single in-progress task. */
|
|
interface TrackedTask {
|
|
session: DisposableSession;
|
|
/** Timestamp of the last heartbeat (text delta, tool call, etc.). */
|
|
lastActivity: number;
|
|
/** Timestamp of the last step progress event. */
|
|
lastProgressAt: number;
|
|
/** Number of activity heartbeats since the last progress event. */
|
|
activitySinceProgress: number;
|
|
/** Number of ignored fn_task_update rebuffs since the last progress event. */
|
|
ignoredStepUpdateCount: number;
|
|
/**
|
|
* Set once the executor has already observed a loop in the current execute()
|
|
* lifecycle. FN-5168 only escalates to no-progress churn after the existing
|
|
* loop recovery path already had one chance to compact-and-resume.
|
|
*/
|
|
loopObservedInLifecycle: boolean;
|
|
/**
|
|
* True while compact-and-resume recovery has been accepted but the executor
|
|
* has not yet resumed normal prompting. Keep tracking alive so FN-5168 can
|
|
* still accumulate ignored-step rebuffs after recovery resumes.
|
|
*/
|
|
recoveryInProgress: boolean;
|
|
/**
|
|
* The canonical task ID used for all external callbacks (beforeRequeue,
|
|
* onStuck, onLoopDetected). In step-session mode the map key is a compound
|
|
* string like "FN-1452-step-1"; this field always holds the bare task ID
|
|
* ("FN-1452") so callbacks can look up the right task record.
|
|
*/
|
|
canonicalTaskId: string;
|
|
}
|
|
|
|
/** Payload emitted when a stuck task is detected. */
|
|
export interface StuckTaskEvent {
|
|
/** The task that was detected as stuck. */
|
|
taskId: string;
|
|
/** Why the task is considered stuck. */
|
|
reason: "inactivity" | "loop" | "no-progress-churn";
|
|
/** Milliseconds since the last step progress event. */
|
|
noProgressMs: number;
|
|
/** Milliseconds since the last activity heartbeat. */
|
|
inactivityMs: number;
|
|
/** Number of activity heartbeats since the last progress event. */
|
|
activitySinceProgress: number;
|
|
/** Number of ignored fn_task_update rebuffs since the last progress event. */
|
|
ignoredStepUpdateCount: number;
|
|
/** Whether the task should be re-queued (budget not exhausted). */
|
|
shouldRequeue: boolean;
|
|
}
|
|
|
|
/** Minimum activity-since-progress count to classify as a loop.
|
|
* Prevents false positives when a task is genuinely inactive. */
|
|
const LOOP_ACTIVITY_THRESHOLD = 60;
|
|
|
|
/**
|
|
* FN-5168 root cause: once compact-and-resume has already fired, repeated
|
|
* ignored step-update rebuffs are a strong deterministic signal that the agent
|
|
* is still churning on completed work instead of advancing.
|
|
*/
|
|
const NO_PROGRESS_CHURN_THRESHOLD = 25;
|
|
|
|
export interface StuckTaskDetectorOptions {
|
|
/** Polling interval in milliseconds. Default: 30000 (30 seconds). */
|
|
pollIntervalMs?: number;
|
|
/** Callback invoked when a stuck task is detected.
|
|
* The task will be moved to "todo" for retry by the detector.
|
|
* Receives a structured payload with detection reason and metrics. */
|
|
onStuck?: (event: StuckTaskEvent) => void;
|
|
/** Called before re-queuing a killed task. Return false to prevent re-queue
|
|
* (caller is responsible for marking the task as terminally failed).
|
|
* Used by SelfHealingManager to enforce stuck kill budgets. */
|
|
beforeRequeue?: (taskId: string, reason: "inactivity" | "loop" | "no-progress-churn", event: StuckTaskEvent) => Promise<boolean>;
|
|
/** Pre-kill callback invoked ONLY when reason is "loop".
|
|
* Called BEFORE session.dispose() / moveTask("todo") so the caller can
|
|
* attempt in-process recovery (e.g. compact-and-resume) without killing
|
|
* the agent session.
|
|
*
|
|
* Return `true` to signal "executor accepted ownership of recovery for this
|
|
* run" — the detector will skip dispose/requeue and keep the task tracked
|
|
* for FN-5168 churn accounting while the caller owns recovery.
|
|
* Return `false` to let the detector proceed with the normal kill/requeue path.
|
|
*
|
|
* Errors in this callback fall through to the normal kill path (treated as `false`). */
|
|
onLoopDetected?: (event: StuckTaskEvent) => Promise<boolean>;
|
|
/**
|
|
* U8 (CLI Agent Executor): returns true when a task's live CLI agent session
|
|
* is `waitingOnInput` — expected idleness (a permission/question prompt). When
|
|
* true, stuck/inactivity flagging is SUPPRESSED for that task this cycle: the
|
|
* agent is intentionally quiet waiting for a human, not stalled. The U3 stall
|
|
* backstop (the CLI session state machine's own watchdog) remains the only
|
|
* escalation path while waiting. Narrow seam: a single lookup; absence
|
|
* (undefined) preserves the prior behavior. Errors are treated as "not
|
|
* waiting" (fail toward the normal stuck path, never silently suppress). */
|
|
isCliSessionWaitingOnInput?: (taskId: string) => boolean;
|
|
}
|
|
|
|
export class StuckTaskDetector {
|
|
private tracked = new Map<string, TrackedTask>();
|
|
private interval: ReturnType<typeof setInterval> | null = null;
|
|
private pollIntervalMs: number;
|
|
private onStuck?: (event: StuckTaskEvent) => void;
|
|
private beforeRequeue?: (taskId: string, reason: "inactivity" | "loop" | "no-progress-churn", event: StuckTaskEvent) => Promise<boolean>;
|
|
private onLoopDetected?: (event: StuckTaskEvent) => Promise<boolean>;
|
|
private isCliSessionWaitingOnInput?: (taskId: string) => boolean;
|
|
private paused = false;
|
|
private exhaustedTasks = new Set<string>();
|
|
|
|
constructor(
|
|
private store: TaskStore,
|
|
options: StuckTaskDetectorOptions = {},
|
|
) {
|
|
this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
|
|
this.onStuck = options.onStuck;
|
|
this.beforeRequeue = options.beforeRequeue;
|
|
this.onLoopDetected = options.onLoopDetected;
|
|
this.isCliSessionWaitingOnInput = options.isCliSessionWaitingOnInput;
|
|
}
|
|
|
|
/**
|
|
* U8: whether stuck flagging should be suppressed for a task because its live
|
|
* CLI agent session is `waitingOnInput` (expected idleness). Defensive: a
|
|
* throw in the injected lookup is treated as "not waiting" so a broken seam
|
|
* never silently disables stuck detection.
|
|
*/
|
|
private isWaitingOnInput(canonicalTaskId: string): boolean {
|
|
if (!this.isCliSessionWaitingOnInput) return false;
|
|
try {
|
|
return this.isCliSessionWaitingOnInput(canonicalTaskId);
|
|
} catch (err) {
|
|
stuckLog.error(`waitingOnInput lookup threw for ${canonicalTaskId}; treating as not waiting:`, err);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start the polling loop that checks for stuck tasks.
|
|
* Safe to call multiple times (no-ops if already running).
|
|
*/
|
|
start(): void {
|
|
if (this.interval) return;
|
|
this.interval = setInterval(() => {
|
|
stuckLog.log("Running periodic stuck task check (polling)");
|
|
this.checkStuckTasks().catch((err) => {
|
|
stuckLog.error("Error checking stuck tasks:", err);
|
|
});
|
|
}, this.pollIntervalMs);
|
|
stuckLog.log(`Started (poll interval: ${this.pollIntervalMs}ms)`);
|
|
}
|
|
|
|
/**
|
|
* Stop the polling loop.
|
|
* Does not untrack any tasks — just stops checking.
|
|
*/
|
|
stop(): void {
|
|
if (this.interval) {
|
|
clearInterval(this.interval);
|
|
this.interval = null;
|
|
stuckLog.log("Stopped");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Register an active agent session for monitoring.
|
|
* Sets initial timestamps and counters to now.
|
|
*
|
|
* @param trackingKey The key used internally to identify this session.
|
|
* In step-session mode this is a compound string like "FN-1452-step-1".
|
|
* @param session The disposable agent session.
|
|
* @param canonicalTaskId The bare task ID ("FN-1452") used for all external
|
|
* callbacks (beforeRequeue, onStuck, onLoopDetected). When omitted the
|
|
* trackingKey is used as-is (single-session mode where they are identical).
|
|
*/
|
|
trackTask(trackingKey: string, session: DisposableSession, canonicalTaskId?: string): void {
|
|
const canonicalId = canonicalTaskId ?? trackingKey;
|
|
if (this.exhaustedTasks.has(canonicalId)) {
|
|
void this.store.getTask(canonicalId)
|
|
.then((task) => {
|
|
const isExhausted = task.status === "failed"
|
|
&& (
|
|
task.error?.startsWith("STUCK_LOOP_EXHAUSTED:")
|
|
|| task.error?.startsWith("STUCK_NO_PROGRESS_CHURN:")
|
|
);
|
|
if (isExhausted) {
|
|
stuckLog.log(`Skipping tracking for ${trackingKey} (canonical=${canonicalId}) — task is in terminal stuck state`);
|
|
return;
|
|
}
|
|
this.exhaustedTasks.delete(canonicalId);
|
|
const now = Date.now();
|
|
this.tracked.set(trackingKey, {
|
|
session,
|
|
lastActivity: now,
|
|
lastProgressAt: now,
|
|
activitySinceProgress: 0,
|
|
ignoredStepUpdateCount: 0,
|
|
loopObservedInLifecycle: false,
|
|
recoveryInProgress: false,
|
|
canonicalTaskId: canonicalId,
|
|
});
|
|
stuckLog.log(`Tracking task ${trackingKey} (canonical=${canonicalId}, total tracked: ${this.tracked.size})`);
|
|
})
|
|
.catch((err) => {
|
|
stuckLog.error(`Failed to validate exhausted status for ${canonicalId}; proceeding to track:`, err);
|
|
this.exhaustedTasks.delete(canonicalId);
|
|
const now = Date.now();
|
|
this.tracked.set(trackingKey, {
|
|
session,
|
|
lastActivity: now,
|
|
lastProgressAt: now,
|
|
activitySinceProgress: 0,
|
|
ignoredStepUpdateCount: 0,
|
|
loopObservedInLifecycle: false,
|
|
recoveryInProgress: false,
|
|
canonicalTaskId: canonicalId,
|
|
});
|
|
stuckLog.log(`Tracking task ${trackingKey} (canonical=${canonicalId}, total tracked: ${this.tracked.size})`);
|
|
});
|
|
return;
|
|
}
|
|
|
|
const now = Date.now();
|
|
this.tracked.set(trackingKey, {
|
|
session,
|
|
lastActivity: now,
|
|
lastProgressAt: now,
|
|
activitySinceProgress: 0,
|
|
ignoredStepUpdateCount: 0,
|
|
loopObservedInLifecycle: false,
|
|
recoveryInProgress: false,
|
|
canonicalTaskId: canonicalId,
|
|
});
|
|
stuckLog.log(`Tracking task ${trackingKey} (canonical=${canonicalId}, total tracked: ${this.tracked.size})`);
|
|
}
|
|
|
|
/**
|
|
* Remove a task from monitoring.
|
|
* Called when a task finishes (success, failure, or pause).
|
|
*
|
|
* Handles both direct keys and step-scoped keys:
|
|
* - Direct key (single-session mode): removes the entry with the given ID
|
|
* - Step-scoped keys (step-session mode): removes ALL entries for the canonical task ID
|
|
*
|
|
* In step-session mode, tasks are tracked with compound keys like "FN-200-step-0".
|
|
* When the executor calls untrackTask with the bare task ID "FN-200", this method
|
|
* cleans up all step-scoped entries for that task.
|
|
*/
|
|
untrackTask(taskId: string): void {
|
|
// First, try to delete the direct key (single-session mode)
|
|
this.tracked.delete(taskId);
|
|
|
|
// Also clean up any step-scoped entries for this task.
|
|
// In step-session mode, entries are keyed by "taskId-step-N" but we need to
|
|
// clean them up when given the bare task ID.
|
|
// Pattern: "{taskId}-step-{N}" where N is a number
|
|
const stepPrefix = `${taskId}-step-`;
|
|
for (const key of this.tracked.keys()) {
|
|
if (key.startsWith(stepPrefix)) {
|
|
this.tracked.delete(key);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Record a heartbeat for a task's agent session.
|
|
* Called on text deltas and tool calls only (NOT step transitions).
|
|
* Increments `activitySinceProgress` counter.
|
|
*
|
|
* In step-session mode, called with step-scoped keys (e.g., "FN-200-step-0").
|
|
*/
|
|
recordActivity(taskId: string): void {
|
|
const entry = this.tracked.get(taskId);
|
|
if (entry) {
|
|
entry.lastActivity = Date.now();
|
|
entry.activitySinceProgress++;
|
|
if (entry.activitySinceProgress <= 3 || entry.activitySinceProgress % 50 === 0) {
|
|
stuckLog.log(`Activity recorded for ${taskId} (sinceProgress=${entry.activitySinceProgress})`);
|
|
}
|
|
}
|
|
}
|
|
|
|
private findTrackedEntry(taskId: string): TrackedTask | undefined {
|
|
const direct = this.tracked.get(taskId);
|
|
if (direct) return direct;
|
|
for (const entry of this.tracked.values()) {
|
|
if (entry.canonicalTaskId === taskId) {
|
|
return entry;
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/**
|
|
* Record a step progress event for a task's agent session.
|
|
* Called on step transitions (in-progress, done, skipped).
|
|
* Resets `activitySinceProgress` to 0 and updates `lastProgressAt`.
|
|
*
|
|
* In step-session mode, called with the bare task ID (e.g., "FN-200").
|
|
* This method finds entries by canonical task ID when the direct key lookup fails.
|
|
*/
|
|
recordProgress(taskId: string): void {
|
|
const entry = this.findTrackedEntry(taskId);
|
|
if (entry) {
|
|
entry.lastProgressAt = Date.now();
|
|
entry.activitySinceProgress = 0;
|
|
entry.ignoredStepUpdateCount = 0;
|
|
entry.recoveryInProgress = false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the last activity timestamp for a tracked task.
|
|
* Returns undefined if the task is not tracked.
|
|
*/
|
|
getLastActivity(taskId: string): number | undefined {
|
|
return this.tracked.get(taskId)?.lastActivity;
|
|
}
|
|
|
|
/**
|
|
* Get the activity-since-progress count for a tracked task.
|
|
* Returns undefined if the task is not tracked.
|
|
*/
|
|
getActivitySinceProgress(taskId: string): number | undefined {
|
|
return this.tracked.get(taskId)?.activitySinceProgress;
|
|
}
|
|
|
|
/**
|
|
* Get the last progress timestamp for a tracked task.
|
|
* Returns undefined if the task is not tracked.
|
|
*/
|
|
getLastProgressAt(taskId: string): number | undefined {
|
|
return this.tracked.get(taskId)?.lastProgressAt;
|
|
}
|
|
|
|
getIgnoredStepUpdateCount(taskId: string): number | undefined {
|
|
return this.findTrackedEntry(taskId)?.ignoredStepUpdateCount;
|
|
}
|
|
|
|
recordIgnoredStepUpdate(taskId: string): void {
|
|
const entry = this.findTrackedEntry(taskId);
|
|
if (entry) {
|
|
entry.lastActivity = Date.now();
|
|
entry.activitySinceProgress++;
|
|
entry.ignoredStepUpdateCount++;
|
|
if (entry.ignoredStepUpdateCount <= 3 || entry.ignoredStepUpdateCount % 10 === 0) {
|
|
stuckLog.log(
|
|
`Ignored step update recorded for ${taskId} ` +
|
|
`(ignored=${entry.ignoredStepUpdateCount}, sinceProgress=${entry.activitySinceProgress})`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
markLoopObserved(taskId: string): void {
|
|
const entry = this.findTrackedEntry(taskId);
|
|
if (entry) {
|
|
entry.loopObservedInLifecycle = true;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check whether a task is stuck (no activity for longer than timeout).
|
|
*/
|
|
isStuck(taskId: string, timeoutMs: number): boolean {
|
|
const entry = this.tracked.get(taskId);
|
|
if (!entry) return false;
|
|
return (Date.now() - entry.lastActivity) > timeoutMs;
|
|
}
|
|
|
|
/**
|
|
* Classify why a task is stuck.
|
|
* Returns null if the task is not stuck.
|
|
*/
|
|
classifyStuckReason(taskId: string, timeoutMs: number): "inactivity" | "loop" | "no-progress-churn" | null {
|
|
const entry = this.tracked.get(taskId);
|
|
if (!entry) return null;
|
|
|
|
const now = Date.now();
|
|
const inactivityMs = now - entry.lastActivity;
|
|
const noProgressMs = now - entry.lastProgressAt;
|
|
|
|
// Check inactivity first — if there's been zero activity, it's just inactive
|
|
if (inactivityMs >= timeoutMs) {
|
|
return "inactivity";
|
|
}
|
|
|
|
// FN-5168: only escalate to churn after loop recovery already had one chance
|
|
// in this execute() lifecycle and the agent kept hammering ignored step updates.
|
|
if (
|
|
noProgressMs >= timeoutMs
|
|
&& entry.loopObservedInLifecycle
|
|
&& entry.ignoredStepUpdateCount >= NO_PROGRESS_CHURN_THRESHOLD
|
|
) {
|
|
return "no-progress-churn";
|
|
}
|
|
|
|
// After onLoopDetected accepts compact-and-resume, the executor owns the
|
|
// recovery prompt. Do not immediately classify the same stale timestamps as
|
|
// another loop before the executor has a chance to emit fresh progress. The
|
|
// deterministic no-progress-churn terminal path above still has to fire if
|
|
// the recovered session keeps hammering rejected step updates.
|
|
if (entry.recoveryInProgress) {
|
|
return null;
|
|
}
|
|
|
|
// Check loop — active but not making progress, with enough activity to be a real loop
|
|
if (noProgressMs >= timeoutMs && entry.activitySinceProgress >= LOOP_ACTIVITY_THRESHOLD) {
|
|
return "loop";
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Terminate a stuck task's agent session and trigger recovery.
|
|
* - Disposes the agent session
|
|
* - Logs the stuck event to the task log
|
|
* - Moves the task back to "todo" (preserving step progress)
|
|
* - Invokes the onStuck callback
|
|
*/
|
|
async killAndRetry(taskId: string, timeoutMs: number): Promise<void> {
|
|
const entry = this.tracked.get(taskId);
|
|
if (!entry) return;
|
|
|
|
// In step-session mode the map key is a compound string like "FN-1452-step-1".
|
|
// All external callbacks (beforeRequeue, onStuck, onLoopDetected) must use
|
|
// the canonical task ID so they can look up the right task record and signal
|
|
// the right executor entry.
|
|
const canonicalId = entry.canonicalTaskId;
|
|
|
|
const now = Date.now();
|
|
const inactivityMs = now - entry.lastActivity;
|
|
const noProgressMs = now - entry.lastProgressAt;
|
|
const activitySinceProgress = entry.activitySinceProgress;
|
|
const ignoredStepUpdateCount = entry.ignoredStepUpdateCount;
|
|
|
|
// Classify the reason
|
|
const reason = this.classifyStuckReason(taskId, timeoutMs) ?? "inactivity";
|
|
|
|
const elapsedMin = Math.round(inactivityMs / 60_000);
|
|
const noProgressMin = Math.round(noProgressMs / 60_000);
|
|
|
|
stuckLog.log(
|
|
`Killing stuck task ${taskId} (canonical=${canonicalId}, reason=${reason}, ` +
|
|
`no progress for ~${noProgressMin}min, ` +
|
|
`no activity for ~${elapsedMin}min, ` +
|
|
`${activitySinceProgress} events since last progress, ` +
|
|
`${ignoredStepUpdateCount} ignored step-update rebuffs)`,
|
|
);
|
|
|
|
// Log the event to the task log using the canonical task ID so it
|
|
// appears in the correct task's log (not a phantom step-key task).
|
|
try {
|
|
await this.store.logEntry(
|
|
canonicalId,
|
|
`Task terminated due to stuck agent session (reason=${reason}, ` +
|
|
`no progress for ~${noProgressMin}min, ` +
|
|
`no activity for ~${elapsedMin}min, ` +
|
|
`${activitySinceProgress} events since last progress, ` +
|
|
`${ignoredStepUpdateCount} ignored step-update rebuffs)`,
|
|
);
|
|
} catch (err) {
|
|
stuckLog.error(`Failed to log stuck event for ${canonicalId}:`, err);
|
|
}
|
|
|
|
// Build the event payload using the canonical task ID so executor callbacks
|
|
// can look up the right entry in stuckAborted / activeStepExecutors.
|
|
const event: StuckTaskEvent = {
|
|
taskId: canonicalId,
|
|
reason,
|
|
noProgressMs,
|
|
inactivityMs,
|
|
activitySinceProgress,
|
|
ignoredStepUpdateCount,
|
|
shouldRequeue: true,
|
|
};
|
|
|
|
// Check stuck kill budget BEFORE disposing the session so the result
|
|
// is available to the executor's cleanup path via the event payload.
|
|
let shouldRequeue = true;
|
|
if (this.beforeRequeue) {
|
|
try {
|
|
shouldRequeue = await this.beforeRequeue(canonicalId, reason, event);
|
|
if (!shouldRequeue) {
|
|
stuckLog.log(
|
|
reason === "no-progress-churn"
|
|
? `${canonicalId} hit no-progress churn terminalization — not re-queuing`
|
|
: `${canonicalId} exceeded stuck kill budget — not re-queuing`,
|
|
);
|
|
}
|
|
} catch (err) {
|
|
stuckLog.error(`beforeRequeue check failed for ${canonicalId}:`, err);
|
|
// Fall through with shouldRequeue=true — safer than dropping the task
|
|
}
|
|
}
|
|
event.shouldRequeue = shouldRequeue;
|
|
|
|
// ── Pre-kill loop interception ──────────────────────────────────
|
|
// When reason is "loop" and an onLoopDetected callback is registered,
|
|
// give the caller a chance to handle recovery in-process (e.g.
|
|
// compact-and-resume) before falling through to the kill/requeue path.
|
|
//
|
|
// If the callback returns true, the caller owns the task — we skip
|
|
// dispose/requeue but keep tracking alive for FN-5168 churn accounting.
|
|
// Errors fall through to normal kill.
|
|
if (reason === "loop" && this.onLoopDetected) {
|
|
try {
|
|
const handled = await this.onLoopDetected(event);
|
|
if (handled) {
|
|
stuckLog.log(
|
|
`${canonicalId} loop recovery accepted by onLoopDetected callback — ` +
|
|
`skipping kill/requeue (caller owns recovery)`,
|
|
);
|
|
// Keep the task tracked so FN-5168 can keep counting ignored
|
|
// step-update churn after recovery resumes, but suppress detector
|
|
// polling until the executor emits fresh progress on resume.
|
|
entry.recoveryInProgress = true;
|
|
return;
|
|
}
|
|
} catch (err) {
|
|
stuckLog.error(`onLoopDetected callback failed for ${canonicalId}:`, err);
|
|
// Fall through to normal kill path
|
|
}
|
|
}
|
|
|
|
// Notify listeners before disposing the session so executor cleanup can
|
|
// mark the abort as intentional before the disposed session unwinds.
|
|
this.onStuck?.(event);
|
|
|
|
if (!shouldRequeue) {
|
|
this.exhaustedTasks.add(canonicalId);
|
|
stuckLog.log(
|
|
reason === "no-progress-churn"
|
|
? `${canonicalId} untracked due to STUCK_NO_PROGRESS_CHURN terminal state (no automatic retries)`
|
|
: `${canonicalId} untracked due to STUCK_LOOP_EXHAUSTED terminal state (no automatic retries)`,
|
|
);
|
|
}
|
|
|
|
// Dispose the agent session after listeners have marked the abort.
|
|
try {
|
|
entry.session.dispose();
|
|
} catch (err) {
|
|
stuckLog.error(`Failed to dispose session for ${taskId}:`, err);
|
|
}
|
|
|
|
// Remove from tracking.
|
|
// The actual moveTask("todo") is handled by the executor's catch/finally
|
|
// block after it cleans up this.executing. This prevents a race where the
|
|
// scheduler re-dispatches the task while the old execution is still active.
|
|
this.tracked.delete(taskId);
|
|
}
|
|
|
|
/**
|
|
* Pause stuck detection checks while the engine is in a paused lifecycle.
|
|
* Active tracked sessions are preserved and refreshed on resume.
|
|
*/
|
|
pause(): void {
|
|
if (this.paused) return;
|
|
this.paused = true;
|
|
}
|
|
|
|
/**
|
|
* Resume stuck detection checks and refresh tracked timestamps so the paused
|
|
* interval does not count as inactivity/no-progress time.
|
|
*/
|
|
resume(): void {
|
|
if (!this.paused) return;
|
|
this.paused = false;
|
|
if (this.tracked.size === 0) return;
|
|
const now = Date.now();
|
|
for (const entry of this.tracked.values()) {
|
|
entry.lastActivity = now;
|
|
entry.lastProgressAt = now;
|
|
entry.activitySinceProgress = 0;
|
|
entry.ignoredStepUpdateCount = 0;
|
|
entry.loopObservedInLifecycle = false;
|
|
entry.recoveryInProgress = false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check for stuck tasks immediately, outside the normal polling cycle.
|
|
* Safe to call at any time — will no-op if no tasks are tracked or timeout is disabled.
|
|
* Logs at debug level to distinguish manual checks from polling.
|
|
*/
|
|
async checkNow(): Promise<void> {
|
|
stuckLog.log("Running immediate stuck task check (triggered manually)");
|
|
await this.checkStuckTasks();
|
|
}
|
|
|
|
/**
|
|
* Poll all tracked tasks and kill any that have exceeded the timeout.
|
|
* Reads `taskStuckTimeoutMs` from settings on each check so changes
|
|
* take effect on the next poll cycle.
|
|
*
|
|
* Detection rules:
|
|
* - **inactivity**: `lastActivity` older than `taskStuckTimeoutMs` (no heartbeats at all)
|
|
* - **no-progress-churn**: after an earlier loop recovery in the same execute() lifecycle,
|
|
* `lastProgressAt` still exceeds `taskStuckTimeoutMs` and ignored step-update rebuffs reach 25+
|
|
* - **loop**: `lastProgressAt` older than `taskStuckTimeoutMs` AND `activitySinceProgress >= 60`
|
|
* (agent is actively doing things but not advancing steps)
|
|
*/
|
|
private async checkStuckTasks(): Promise<void> {
|
|
if (this.tracked.size === 0) return;
|
|
|
|
// Fast pause gate: if lifecycle hooks paused the detector, skip the cycle
|
|
// without reading settings (avoids noisy settings-read errors while paused).
|
|
if (this.paused) return;
|
|
|
|
let settings: Settings;
|
|
try {
|
|
settings = await this.store.getSettings();
|
|
} catch (err) {
|
|
stuckLog.error("Failed to read settings — skipping stuck task detection cycle:", err);
|
|
return; // Can't read settings — skip this cycle
|
|
}
|
|
|
|
// Defensive fallback for pause windows where lifecycle hooks haven't run yet.
|
|
if (settings.globalPause || settings.enginePaused) return;
|
|
|
|
const timeoutMs = settings.taskStuckTimeoutMs;
|
|
if (!timeoutMs || timeoutMs <= 0) return; // Disabled when task stuck timeout is explicitly unset/disabled
|
|
|
|
const stuckTasks: string[] = [];
|
|
|
|
for (const [taskId, entry] of this.tracked) {
|
|
const reason = this.classifyStuckReason(taskId, timeoutMs);
|
|
if (reason !== null) {
|
|
// U8: suppress flagging while the CLI session is waitingOnInput
|
|
// (expected idleness). The U3 stall backstop remains the only escalation
|
|
// path while genuinely waiting.
|
|
if (this.isWaitingOnInput(entry.canonicalTaskId)) {
|
|
stuckLog.log(`Suppressing stuck flag for ${taskId} (canonical=${entry.canonicalTaskId}) — CLI session waitingOnInput`);
|
|
continue;
|
|
}
|
|
stuckTasks.push(taskId);
|
|
}
|
|
}
|
|
|
|
for (const taskId of stuckTasks) {
|
|
await this.killAndRetry(taskId, timeoutMs);
|
|
}
|
|
}
|
|
|
|
/** Number of currently tracked tasks (for testing). */
|
|
get trackedCount(): number {
|
|
return this.tracked.size;
|
|
}
|
|
}
|