fix(FN-703): fix overly broad usage limit pattern and add transient error retry

- Narrow 'insufficient' usage-limit regex to require quota/credit/balance/fund context
- Add transient error detection in executor and triage to retry instead of failing
- Improve scheduler pause/resume logging with actionable messages and resume confirmation
- Add SSE active connection counter for observability
- Add duplicate usage-limit pause suppression logging
This commit is contained in:
gsxdsm
2026-04-02 13:29:48 -07:00
parent 0518fdd977
commit 2acd2cfb17
5 changed files with 37 additions and 3 deletions

View File

@@ -1,6 +1,13 @@
import type { Request, Response } from "express";
import type { TaskStore } from "@fusion/core";
let activeConnections = 0;
/** Returns the current number of active SSE connections. */
export function getActiveSSEConnections(): number {
return activeConnections;
}
export function createSSE(store: TaskStore) {
return (_req: Request, res: Response) => {
res.setHeader("Content-Type", "text/event-stream");
@@ -9,6 +16,8 @@ export function createSSE(store: TaskStore) {
res.setHeader("X-Accel-Buffering", "no");
res.flushHeaders();
activeConnections++;
// Send initial heartbeat
res.write(": connected\n\n");
@@ -40,6 +49,7 @@ export function createSSE(store: TaskStore) {
}, 30_000);
_req.on("close", () => {
activeConnections--;
clearInterval(heartbeat);
store.off("task:created", onCreated);
store.off("task:moved", onMoved);

View File

@@ -13,6 +13,7 @@ import type { WorktreePool } from "./worktree-pool.js";
import { AgentLogger } from "./agent-logger.js";
import { executorLog, reviewerLog } from "./logger.js";
import { isUsageLimitError, checkSessionError, type UsageLimitPauser } from "./usage-limit-detector.js";
import { isTransientError } from "./transient-error-detector.js";
import type { StuckTaskDetector } from "./stuck-task-detector.js";
// Re-export for backward compatibility (tests import from executor.ts)
@@ -688,6 +689,12 @@ export class TaskExecutor {
// Check if the error is a usage-limit error and trigger global pause
if (this.options.usageLimitPauser && isUsageLimitError(err.message)) {
await this.options.usageLimitPauser.onUsageLimitHit("executor", task.id, err.message);
} else if (isTransientError(err.message)) {
// Transient network/infrastructure error — retry instead of failing
executorLog.warn(`${task.id} transient error — moving to todo for retry: ${err.message}`);
await this.store.logEntry(task.id, `Transient error (will retry): ${err.message}`);
await this.store.moveTask(task.id, "todo");
return;
}
executorLog.error(`${task.id} execution failed:`, err.message);
await this.store.logEntry(task.id, `Execution failed: ${err.message}`);

View File

@@ -331,21 +331,27 @@ export class Scheduler {
// Global pause (hard stop): halt all scheduling activity
if (settings.globalPause) {
if (!this.wasGlobalPaused) {
schedulerLog.log("Global pause active — scheduling halted");
schedulerLog.warn("Global pause active — scheduling halted. To resume: set globalPause to false in settings.");
this.wasGlobalPaused = true;
}
return;
}
if (this.wasGlobalPaused) {
schedulerLog.log("Global pause cleared — scheduling resumed");
}
this.wasGlobalPaused = false;
// Engine paused (soft pause): halt new work dispatch, but let agents finish
if (settings.enginePaused) {
if (!this.wasEnginePaused) {
schedulerLog.log("Engine paused — scheduling halted (in-flight agents continue)");
schedulerLog.warn("Engine paused — scheduling halted (in-flight agents continue). To resume: set enginePaused to false.");
this.wasEnginePaused = true;
}
return;
}
if (this.wasEnginePaused) {
schedulerLog.log("Engine pause cleared — scheduling resumed");
}
this.wasEnginePaused = false;
// Count only in-progress tasks toward the worktree limit.

View File

@@ -21,6 +21,7 @@ import {
checkSessionError,
type UsageLimitPauser,
} from "./usage-limit-detector.js";
import { isTransientError } from "./transient-error-detector.js";
export const TRIAGE_SYSTEM_PROMPT = `You are a task specification agent for "kb", an AI-orchestrated task board.
@@ -664,6 +665,14 @@ export class TriageProcessor {
task.id,
err.message,
);
} else if (isTransientError(err.message)) {
// Transient network/infrastructure error — don't mark as failed, allow retry
triageLog.warn(`${task.id} transient error during triage — will retry: ${err.message}`);
await this.store.logEntry(task.id, `Transient error during specification (will retry): ${err.message}`).catch(() => {});
// Restore status so triage picks it up again on next pass
const restoreStatus = task.status === "needs-respecify" ? "needs-respecify" : undefined;
await this.store.updateTask(task.id, { status: restoreStatus }).catch(() => {});
return;
}
// For re-specification, restore needs-respecify status so it can be retried
const restoreStatus = task.status === "needs-respecify" ? "needs-respecify" : undefined;

View File

@@ -27,7 +27,7 @@ const USAGE_LIMIT_PATTERNS: RegExp[] = [
/quota/i,
/billing/i,
/\bcredit/i,
/insufficient/i,
/insufficient.*(quota|credit|balance|fund)/i,
];
/**
@@ -88,6 +88,7 @@ export class UsageLimitPauser {
const settings = await this.store.getSettings();
if (settings.globalPause) {
// Still paused — no need to trigger again
log.log(`Global pause already active — ignoring duplicate from ${agentType}/${taskId}`);
return;
}
// External reset detected — allow re-triggering
@@ -97,6 +98,7 @@ export class UsageLimitPauser {
this.paused = true;
log.warn(`${agentType} hit usage limit on ${taskId}: ${errorMessage}`);
log.warn(`Matched pattern in error: "${errorMessage.slice(0, 200)}"`);
// Log the triggering error on the task
await this.store.logEntry(