From e51ebff381f28b2ebf3a115f4b36254b46d9e38e Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 31 Jul 2026 18:08:48 -0700 Subject: [PATCH] =?UTF-8?q?fix(engine):=20hung-poll=20watchdog=20=E2=80=94?= =?UTF-8?q?=20triage=20admission=20recovers=20loudly=20instead=20of=20dyin?= =?UTF-8?q?g=20silently?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .changeset/triage-poll-watchdog.md | 7 +++++++ packages/engine/src/triage.ts | 26 +++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 .changeset/triage-poll-watchdog.md diff --git a/.changeset/triage-poll-watchdog.md b/.changeset/triage-poll-watchdog.md new file mode 100644 index 0000000000..3042995594 --- /dev/null +++ b/.changeset/triage-poll-watchdog.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: New tasks no longer sit queued for minutes when a triage poll hangs — a watchdog recovers admission loudly. +category: fix +dev: One hung poll left this.polling true forever, silently dropping every 15s tick and task:created wake (observed as 5-10 min "Queued to plan" with open capacity, rescued only by unrelated sweeps). Past 120s the guard force-opens with a WARN naming the stuck duration. diff --git a/packages/engine/src/triage.ts b/packages/engine/src/triage.ts index 487bca3bad..f32a306afd 100644 --- a/packages/engine/src/triage.ts +++ b/packages/engine/src/triage.ts @@ -381,6 +381,8 @@ function isOrphanedLegacyTriageRow(column: string, declaresTriage: boolean): boo export class TriageProcessor { private running = false; private polling = false; + /** FNXC:TriagePollWatchdog 2026-08-01-01:25: wall-clock start of the in-flight poll, for the hung-poll watchdog below. */ + private pollingSince = 0; private pollInterval: ReturnType | null = null; /** The interval (ms) of the currently active `setInterval` timer. */ private activePollMs: number | null = null; @@ -2004,6 +2006,8 @@ export class TriageProcessor { /** Coalescing window for requestImmediatePoll, so a multi-card drag causes one poll, not N. */ private static readonly NUDGE_DEBOUNCE_MS = 150; + /** FNXC:TriagePollWatchdog 2026-08-01-01:25: a poll marked in-flight past this long is treated as hung. */ + private static readonly POLL_WATCHDOG_MS = 120_000; /** * FNXC:DuplicateIntake 2026-07-26-10:40: @@ -2015,8 +2019,28 @@ export class TriageProcessor { private async poll(): Promise { if (!this.running) return; - if (this.polling) return; + if (this.polling) { + /* + FNXC:TriagePollWatchdog 2026-08-01-01:25 (live incident — planning admission silently dead): + The re-entrance guard makes ONE hung poll a PERMANENT silent triage death: `this.polling` + stays true forever, every 15s tick and every task:created wake drops here without a log + line, and new tasks sit "Queued to plan" until an unrelated sweep rescues them (observed + twice on the live board: 5m50s and ~10m admission delays with capacity wide open, both + ending in a batch admission the moment something else poked the store). A dropped poll is + normal for seconds — a poll is genuinely in flight — but minutes means the in-flight poll + hung (store call, provider probe). Recover loudly instead of dying silently: past the + watchdog threshold, log at WARN with the stuck duration and force the guard open so this + tick's poll proceeds. The hung promise, if it ever resolves, resets `polling` in its own + finally — hitting the guard already open is harmless (it just sets it true again). + */ + const stuckMs = this.pollingSince > 0 ? Date.now() - this.pollingSince : 0; + if (stuckMs < TriageProcessor.POLL_WATCHDOG_MS) return; + planLog.warn( + `triage poll watchdog: previous poll still marked in-flight after ${Math.round(stuckMs / 1000)}s — forcing the guard open so planning admission resumes`, + ); + } this.polling = true; + this.pollingSince = Date.now(); this.nudgeDuringPoll = false; try {