root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(engine): hung-pass watchdogs on scheduler, merge queue, and continuation drain

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-07-31 18:19:16 -07:00
committed by gsxdsm
parent 210a89d74f
commit 6834ba35bd
4 changed files with 33 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/pump-watchdogs.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Scheduler, merge queue, and continuation drain recover loudly from a hung pass instead of dying silently.
category: fix
dev: Same shape as the triage-poll death — a stuck re-entrance guard dropped every later tick without a log. Each pump now records its pass start and force-opens the guard past a duration sized to its legitimate work (schedule 10min, merge drain 30min, continuation drain 5min), logging a WARN with the stuck duration.

9
packages/engine/src/project-engine.ts
View File

@@ -458,6 +458,7 @@ export class ProjectEngine {
private unregisterMergeAdmissionProvider?: () => void;
private pausedReviewTaskIds = new Set<string>();
private mergeRunning = false;
private mergeRunningSince = 0;
private activeMergeSession: { dispose: () => void } | null = null;
private activeMergeTaskId: string | null = null;
/** Wall-clock when `activeMergeTaskId` was claimed; self-healing uses this when agent logs are silent. */
@@ -3297,8 +3298,14 @@ export class ProjectEngine {
}
private async drainMergeQueue(): Promise<void> {
if (this.mergeRunning) return;
if (this.mergeRunning) {
/* FNXC:PumpWatchdog 2026-08-01-02:00: one hung pass leaves the guard closed forever and every later tick/wake drops SILENTLY (the triage-poll death, 00769fad7c/e51ebff381). Past the threshold, warn with the stuck duration and force the guard open; the hung pass's own finally re-clearing it later is harmless. A legitimate merge runs many minutes (rebase, verification, land), so the threshold is 30min — past it the merge is wedged (stale-merge sweeps own the TASK, this owns the PUMP flag). */
const stuckMs = this.mergeRunningSince > 0 ? Date.now() - this.mergeRunningSince : 0;
if (stuckMs < 1_800_000) return;
runtimeLog.warn(`merge-queue watchdog: previous drain still marked in-flight after ${Math.round(stuckMs / 1000)}s — forcing the guard open so merging resumes`);
}
this.mergeRunning = true;
this.mergeRunningSince = Date.now();
try {
this.reconcileStaleMergeActive();

12
packages/engine/src/runtimes/in-process-runtime.ts
View File

@@ -620,6 +620,7 @@ export class InProcessRuntime
private triageProcessor?: TriageProcessor;
private workflowContinuationTimer?: ReturnType<typeof setInterval>;
private workflowContinuationDrainActive = false;
private workflowContinuationDrainSince = 0;
private messageStore?: MessageStore;
/** FNXC:TaskDeleteNotice 2026-07-26-16:10: identity-guarded teardown for the delete-notice mailbox seam. */
private unregisterTaskDeleteNoticeMailbox?: () => void;
@@ -2307,7 +2308,15 @@ export class InProcessRuntime
* adapters that bind the pass to this runtime's store and executor.
*/
private async drainWorkflowContinuations(): Promise<void> {
if (this.workflowContinuationDrainActive || this.status !== "active") return;
if (this.status !== "active") return;
if (this.workflowContinuationDrainActive) {
/* FNXC:PumpWatchdog 2026-08-01-02:00: one hung pass leaves the guard closed forever and every later tick/wake drops SILENTLY (the triage-poll death, 00769fad7c/e51ebff381). Past the threshold, warn with the stuck duration and force the guard open; the hung pass's own finally re-clearing it later is harmless. */
const stuckMs = this.workflowContinuationDrainSince > 0 ? Date.now() - this.workflowContinuationDrainSince : 0;
if (stuckMs < 300_000) return;
runtimeLog.warn(`continuation-drain watchdog: previous drain still marked in-flight after ${Math.round(stuckMs / 1000)}s — forcing the guard open`);
}
this.workflowContinuationDrainActive = true;
this.workflowContinuationDrainSince = Date.now();
/*
FNXC:EnginePause 2026-08-01-00:20:
A pause-suspended run persists a runnable continuation (same mechanism as capacity). Without
@@ -2322,7 +2331,6 @@ export class InProcessRuntime
} catch {
/* unreadable settings: proceed as before rather than wedging the pump */
}
this.workflowContinuationDrainActive = true;
try {
await drainDuePlanningContinuations({
listDue: () => this.taskStore.listDueWorkflowWorkItems({

9
packages/engine/src/scheduler.ts
View File

@@ -893,6 +893,7 @@ export interface SchedulerOptions {
export class Scheduler {
private running = false;
private scheduling = false;
private schedulingSince = 0;
private wasWorktreeLimited = false;
private wasGlobalPaused = false;
private wasEnginePaused = false;
@@ -1969,8 +1970,14 @@ export class Scheduler {
*/
async schedule(): Promise<void> {
if (!this.running) return;
if (this.scheduling) return;
if (this.scheduling) {
/* FNXC:PumpWatchdog 2026-08-01-02:00: one hung pass leaves the guard closed forever and every later tick/wake drops SILENTLY (the triage-poll death, 00769fad7c/e51ebff381). Past the threshold, warn with the stuck duration and force the guard open; the hung pass's own finally re-clearing it later is harmless. Dispatch prep runs real git work per candidate, so the threshold is generous. */
const stuckMs = this.schedulingSince > 0 ? Date.now() - this.schedulingSince : 0;
if (stuckMs < 600_000) return;
schedulerLog.warn(`schedule watchdog: previous pass still marked in-flight after ${Math.round(stuckMs / 1000)}s — forcing the guard open so dispatch resumes`);
}
this.scheduling = true;
this.schedulingSince = Date.now();
try {
let tasks = await this.store.listTasks({ slim: true, includeArchived: false, startupMemo: false });