From 581b7d0a490b32b8ccbcad7f7486a066a49891df Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 26 Jul 2026 07:23:44 -0700 Subject: [PATCH] fix(triage): clear stale planning statuses periodically, not only at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed on FN-8596: a plan-review REVISE routed to `plan-replan`, triage claimed the card with `status:"planning"` and ran the revision session, and the session wrote the revised PROMPT.md then died without finalizing. The card sat in `triage` with `status:"planning"`, no live planner, and no workflow continuation. That status makes the card invisible to triage rediscovery (it looks claimed), and the only sweep that cleared it ran at STARTUP — so the card was unrecoverable short of an engine restart. The leaked-slot reaper then reclaimed its concurrency slot, which made it look idle without making it runnable. Adds a periodic counterpart in the poll loop. Clearing the status is the whole repair: the card is back in triage with a real spec, so ordinary rediscovery re-picks it. It does not move, pause, or fail the card. Guards against racing a healthy planner: the in-process `processing` set, plus a 20-minute staleness floor that also covers a planner owned by another node this process cannot see. Operator parks are never touched. This fixes the recovery gap, not the trigger — why that session failed to finalize is still under investigation. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/triage-stale-planning-sweep.md | 7 +++ packages/engine/src/__tests__/triage.test.ts | 61 ++++++++++++++++++++ packages/engine/src/triage.ts | 58 +++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 .changeset/triage-stale-planning-sweep.md diff --git a/.changeset/triage-stale-planning-sweep.md b/.changeset/triage-stale-planning-sweep.md new file mode 100644 index 0000000000..f4a39c1861 --- /dev/null +++ b/.changeset/triage-stale-planning-sweep.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Recover cards left stuck with a stale "planning" status instead of stranding them until an engine restart. +category: fix +dev: Adds `TriageProcessor.sweepStalePlanningStatuses`, a periodic counterpart to the startup-only `clearStaleSpecifyingStatuses`. A planner that dies after doing its work but before finalizing left `status:"planning"` on a triage/todo card; rediscovery skips such cards (they look claimed), so the card was unrecoverable short of a restart. The sweep clears the status once past a 20-minute floor with no live planner, letting ordinary rediscovery re-pick it. Guards: the in-process `processing` set, the staleness floor (covers planners owned by another node), and operator parks are never touched. diff --git a/packages/engine/src/__tests__/triage.test.ts b/packages/engine/src/__tests__/triage.test.ts index fff48904f9..fa2d95a19a 100644 --- a/packages/engine/src/__tests__/triage.test.ts +++ b/packages/engine/src/__tests__/triage.test.ts @@ -7053,3 +7053,64 @@ describe("FN-4774 regression: triage duplicate detection over done/archived task expect(text).toContain("(done):"); }); }); + +/* +FNXC:TriageStalePlanning 2026-07-26-17:30: +Regression for the FN-8596 strand: a plan-review REVISE routed to `plan-replan`, triage claimed the +card with `status:"planning"` and ran the revision, and the session died after writing the revised +PROMPT.md but before finalizing. The card sat in `triage` with `status:"planning"` and no workflow +continuation — invisible to rediscovery (it looks claimed) and unrecoverable until an engine +restart, because the only sweep that clears that status ran at startup. + +These cases pin the periodic sweep AND its guards. The guards are the risky half: a sweep that +clears too eagerly would yank the status out from under a healthy planner and let a second planner +claim the same card, so "in-process planner" and "recently touched" are asserted as protected. +*/ +describe("TriageProcessor.sweepStalePlanningStatuses", () => { + const NOW = Date.parse("2026-07-26T14:30:00.000Z"); + const STALE = "2026-07-26T13:53:00.000Z"; // ~37m old, past the 20m floor + const FRESH = "2026-07-26T14:29:00.000Z"; // 1m old + + function sweep(store: ReturnType, tasks: Task[], processing: string[] = []) { + const processor = new TriageProcessor(store as never, "/tmp/root"); + for (const id of processing) (processor as unknown as { processing: Set }).processing.add(id); + return (processor as unknown as { + sweepStalePlanningStatuses(t: Task[], n: number): Promise; + }).sweepStalePlanningStatuses(tasks, NOW); + } + + it("clears a stale planning status so triage can re-pick the card", async () => { + const store = createMockStore(); + await sweep(store, [createTriageTask({ id: "FN-8596", status: "planning", updatedAt: STALE })]); + expect(store.updateTask).toHaveBeenCalledWith("FN-8596", { status: null }); + }); + + it("does not touch a card whose planner is live in this process", async () => { + const store = createMockStore(); + await sweep(store, [createTriageTask({ id: "FN-LIVE", status: "planning", updatedAt: STALE })], ["FN-LIVE"]); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("does not touch a recently-touched card (may be another node's live planner)", async () => { + const store = createMockStore(); + await sweep(store, [createTriageTask({ id: "FN-FRESH", status: "planning", updatedAt: FRESH })]); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("never disturbs an operator park", async () => { + const store = createMockStore(); + await sweep(store, [ + createTriageTask({ id: "FN-PAUSED", status: "planning", updatedAt: STALE, userPaused: true } as Partial), + ]); + expect(store.updateTask).not.toHaveBeenCalled(); + }); + + it("ignores cards outside the planning columns and non-planning statuses", async () => { + const store = createMockStore(); + await sweep(store, [ + createTriageTask({ id: "FN-INPROG", column: "in-progress", status: "planning", updatedAt: STALE }), + createTriageTask({ id: "FN-OTHER", status: "needs-replan", updatedAt: STALE }), + ]); + expect(store.updateTask).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/engine/src/triage.ts b/packages/engine/src/triage.ts index 9d3e5bdafc..28acc9996d 100644 --- a/packages/engine/src/triage.ts +++ b/packages/engine/src/triage.ts @@ -159,6 +159,15 @@ import { isOperatorActionableAgentError, isTransientError, isSilentTransientErro import { withRateLimitRetry } from "./rate-limit-retry.js"; import { computeRecoveryDecision, formatDelay, MAX_RECOVERY_RETRIES } from "./recovery-policy.js"; import type { StuckTaskDetector } from "./stuck-task-detector.js"; + +/* +FNXC:TriageStalePlanning 2026-07-26-17:20: +Staleness floor before the periodic sweep may clear a `status:"planning"` claim. Generous on +purpose: it must never race a slow-but-healthy planner, including one owned by another node whose +in-process `processing` set this engine cannot see. A genuinely stranded card waits at most this +long instead of until the next engine restart. +*/ +const STALE_PLANNING_STATUS_GRACE_MS = 20 * 60_000; import { exec } from "node:child_process"; import { readFile, writeFile, rm } from "node:fs/promises"; import { join } from "node:path"; @@ -651,6 +660,53 @@ export class TriageProcessor { planLog.log("Processor started"); } + /* + FNXC:TriageStalePlanning 2026-07-26-17:20: + PERIODIC counterpart to `clearStaleSpecifyingStatuses`, which runs at STARTUP ONLY. + Observed strand (FN-8596): a plan-review REVISE routed to `plan-replan`, triage claimed the card + with `status:"planning"` and ran the revision session, the session wrote the revised PROMPT.md and + then died WITHOUT finalizing. The card was left in `triage` with `status:"planning"`, a live + worktree, and no workflow continuation. Nothing re-dispatched it: triage rediscovery skips cards + already marked `planning` (they look claimed), and the only sweep that clears that status ran at + startup — so the card sat stranded until an operator restarted the engine. The leaked-slot reaper + then reclaimed its concurrency slot, which made the card look idle without making it runnable. + + Clearing the status is the whole repair: the card is back in triage with a real spec, so ordinary + rediscovery re-picks it on the next poll. This does NOT move, pause, or fail the card. + + Two guards keep it from racing a healthy planner: + - `this.processing` excludes sessions this process owns. + - a staleness floor excludes cards touched recently, which covers planners owned by ANOTHER + node/process that this process's `processing` set cannot see. + User-paused cards are never touched (an operator park is authoritative). + */ + private async sweepStalePlanningStatuses(allTasks: Task[], now: number): Promise { + try { + const stale = allTasks.filter((t) => { + if (t.status !== "planning") return false; + if (t.column !== "triage" && t.column !== "todo") return false; + if (this.processing.has(t.id)) return false; + if (t.userPaused === true || t.paused === true) return false; + const touchedAt = Date.parse(t.updatedAt ?? t.columnMovedAt ?? ""); + if (!Number.isFinite(touchedAt)) return false; + return now - touchedAt >= STALE_PLANNING_STATUS_GRACE_MS; + }); + for (const t of stale) { + planLog.warn( + `Stale 'planning' status on ${t.id} (column=${t.column}, no live planner) — clearing so triage can re-pick it`, + ); + await this.store.updateTask(t.id, { status: null }); + await this.store.logEntry( + t.id, + "Auto-recovered: cleared stale planning status left by a planner that never finished", + ).catch(() => undefined); + } + } catch (err) { + // Never let a housekeeping sweep break the poll. + planLog.warn(`Stale planning-status sweep failed: ${err instanceof Error ? err.message : String(err)}`); + } + } + private async clearStaleSpecifyingStatuses(): Promise { /* FNXC:CodingIdeasWorkflow 2026-07-04-12:00: @@ -1373,6 +1429,8 @@ export class TriageProcessor { const allTasks = await this.store.listTasks({ slim: true, includeArchived: false }); const now = Date.now(); + await this.sweepStalePlanningStatuses(allTasks, now); + if (this.options.semaphore) { const result = recoverIdleSemaphoreLeakCandidate({ semaphore: this.options.semaphore,