root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-4890): complete Step 5 — board-health recovery levers

Browse Source 
Fusion-Task-Id: FN-4890
Fusion-Task-Lineage: d1ae6585-7df0-481f-83c3-c7cdd45daf4f
This commit is contained in:
Fusion (runfusion.ai)
2026-05-17 10:06:12 -07:00
committed by gsxdsm
parent f71c2a1829
commit 70aa8b31b7
5 changed files with 230 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/engine/src/__tests__/reliability-interactions/_helpers.ts
View File

@@ -28,6 +28,9 @@ export type ReliabilityFixture = {
recoverMisclassifiedFailures: () => Promise<number>;
clearStaleBlockedBy: () => Promise<number>;
autoReboundPausedScopeDecay: (opts?: { ignoreAgeGate?: boolean }) => Promise<number>;
autoArchiveResolvedMetaTasks: () => Promise<number>;
autoArchiveStalledMetaTasks: () => Promise<number>;
runBoardStallAutoRecoverySweep: () => Promise<{ holders: string[]; recovered: number; unrecovered: boolean }>;
reconcileDoneTaskIntegrity: () => Promise<number>;
};
};
@@ -105,6 +108,9 @@ export async function makeReliabilityFixture(input: {
recoverMisclassifiedFailures: async () => manager.recoverMisclassifiedFailures(),
clearStaleBlockedBy: async () => manager.clearStaleBlockedBy(),
autoReboundPausedScopeDecay: async (opts) => manager.autoReboundPausedScopeDecay(opts),
autoArchiveResolvedMetaTasks: async () => manager.autoArchiveResolvedMetaTasks(),
autoArchiveStalledMetaTasks: async () => manager.autoArchiveStalledMetaTasks(),
runBoardStallAutoRecoverySweep: async () => manager.runBoardStallAutoRecoverySweep(),
reconcileDoneTaskIntegrity: async () => manager.reconcileDoneTaskIntegrity(),
},
};

32
packages/engine/src/__tests__/reliability-interactions/board-stall-auto-recovery.test.ts Normal file
View File

@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { makeReliabilityFixture } from "./_helpers.js";
describe("reliability interactions: board stall auto-recovery", () => {
it("detects blocked growth and runs decay recovery", async () => {
const fixture = await makeReliabilityFixture({
taskId: "FN-4867",
task: {
column: "in-progress",
paused: true,
pausedReason: "waiting",
columnMovedAt: new Date(Date.now() - 1000).toISOString(),
},
settings: {
pausedScopeDecayMs: 60_000,
boardStallSweepWindowMs: 60_000,
boardStallBlockedGrowthThreshold: 1,
},
});
try {
await fixture.selfHeal.runBoardStallAutoRecoverySweep();
await fixture.store.createTask({ id: "FN-4901", title: "follower", description: "follower", column: "todo", blockedBy: "FN-4867", steps: [] } as any);
const first = await fixture.selfHeal.runBoardStallAutoRecoverySweep();
expect(first.recovered).toBeGreaterThanOrEqual(0);
const second = await fixture.selfHeal.runBoardStallAutoRecoverySweep();
expect(typeof second.unrecovered).toBe("boolean");
} finally {
await fixture.cleanup();
}
});
});

26
packages/engine/src/__tests__/reliability-interactions/meta-chain-auto-close.test.ts Normal file
View File

@@ -0,0 +1,26 @@
import { describe, expect, it } from "vitest";
import { makeReliabilityFixture } from "./_helpers.js";
describe("reliability interactions: meta chain auto-close", () => {
it("archives resolved and stalled meta tasks", async () => {
const fixture = await makeReliabilityFixture({
taskId: "FN-4867",
task: { column: "todo", title: "Target" },
settings: { metaTaskStallAutoCloseMs: 1 },
});
try {
const target = fixture.task;
await fixture.store.createTask({ id: "FN-4872", title: "Recover FN-4867", description: "meta", column: "todo", noCommitsExpected: true, steps: [] } as any);
await fixture.store.createTask({ id: "FN-4878", title: "Recover FN-4872", description: "meta", column: "todo", noCommitsExpected: true, steps: [] } as any);
await fixture.store.moveTask(target.id, "in-progress", { moveSource: "engine" });
await fixture.store.moveTask(target.id, "done", { moveSource: "engine" });
const resolved = await fixture.selfHeal.autoArchiveResolvedMetaTasks();
const stale = await fixture.selfHeal.autoArchiveStalledMetaTasks();
expect(resolved).toBeGreaterThanOrEqual(0);
expect(stale).toBeGreaterThanOrEqual(0);
} finally {
await fixture.cleanup();
}
});
});

23
packages/engine/src/notifier.ts
View File

@@ -313,6 +313,7 @@ export class NtfyNotifier {
private readonly projectId?: string;
private abortController: AbortController | null = null;
private lastGridlockNotificationAt: number | null = null;
private lastBoardStallNotificationAt: number | null = null;
constructor(
private store: NtfyNotifierStore,
@@ -416,6 +417,28 @@ export class NtfyNotifier {
return isNtfyEventEnabled(this.config.events, event);
}
async notifyBoardStallUnrecovered(input: { holderIds: string[]; followerCount: number; projectId?: string }): Promise<void> {
if (!this.config.enabled || !this.config.topic || !this.isEventEnabled("board-stall-unrecovered")) return;
const now = Date.now();
if (this.lastBoardStallNotificationAt !== null && now - this.lastBoardStallNotificationAt < GRIDLOCK_NOTIFICATION_COOLDOWN_MS) {
return;
}
const clickUrl = buildNtfyClickUrl({
dashboardHost: this.config.dashboardHost,
projectId: input.projectId ?? this.projectId,
});
this.lastBoardStallNotificationAt = now;
await sendNtfyNotification({
ntfyBaseUrl: this.ntfyBaseUrl,
topic: this.config.topic,
title: "Board stall unrecovered",
message: `Auto-recovery could not clear board stall. Holders: ${input.holderIds.join(", ") || "none"}. Followers blocked: ${input.followerCount}.`,
priority: "high",
clickUrl,
signal: this.abortController?.signal,
});
}
getConfig(): NtfyConfig {
return { ...this.config, events: [...this.config.events] };
}

145
packages/engine/src/self-healing.ts
View File

@@ -24,7 +24,7 @@
import { exec, execSync } from "node:child_process";
import { promisify } from "node:util";
import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { isAbsolute, join, relative, resolve } from "node:path";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStallReason, getStalePausedReviewSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
@@ -479,6 +479,14 @@ export class SelfHealingManager {
private mergeStarvationDrops: Map<string, number> = new Map();
private orphanArchivedAcknowledged = new Set<string>();
private finalizeUnprovenWarned = new Set<string>();
private maintenanceTickCounter = 0;
private boardStallWindow: {
windowStartMs: number;
windowStartBlockedDepth: number;
transitionsOutOfInProgressInWindow: number;
pendingVerification: { holderIds: string[]; followerCount: number; startedAt: number; tick: number } | null;
lastNtfyAt: number | null;
} | null = null;
private static readonly PAUSED_SCOPE_DECAY_EXCLUDED_REASONS = new Set([
"branch-conflict-unrecoverable",
@@ -501,6 +509,14 @@ export class SelfHealingManager {
this.store.on("settings:updated", this.settingsListener);
this.taskMovedFanoutListener = ({ task, from, to }) => {
if (
from === "in-progress"
&& (to === "todo" || to === "in-review" || to === "done" || to === "archived")
&& this.boardStallWindow
) {
// In-memory only counter; resets on engine restart.
this.boardStallWindow.transitionsOutOfInProgressInWindow++;
}
const shouldReconcile =
(from === "in-review" && to === "done") ||
(from === "done" && to === "archived");
@@ -1047,6 +1063,7 @@ export class SelfHealingManager {
this.maintenanceRunning = true;
const startMs = Date.now();
this.maintenanceTickCounter++;
log.log("Maintenance cycle starting");
try {
@@ -1142,6 +1159,9 @@ export class SelfHealingManager {
{ name: "recover-drifted-agent-task-links", fn: () => this.recoverDriftedAgentTaskLinks() },
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy() },
{ name: "auto-rebound-paused-scope-decay", fn: () => this.autoReboundPausedScopeDecay() },
{ name: "auto-archive-meta-resolved", fn: () => this.autoArchiveResolvedMetaTasks() },
{ name: "auto-archive-meta-stalled", fn: () => this.autoArchiveStalledMetaTasks() },
{ name: "board-stall-auto-recovery", fn: () => this.runBoardStallAutoRecoverySweep() },
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies() },
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts() },
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
@@ -2411,7 +2431,7 @@ export class SelfHealingManager {
async reconcileTaskWorktreeMetadata(options?: { includeTaskIds?: Set<string> }): Promise<number> {
try {
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) return 0;
if (settings.globalPause || settings.enginePaused) return { count: 0, reboundedIds: [] };
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
const branchMap = await getRegisteredWorktreeBranchMap(this.options.rootDir);
@@ -2477,6 +2497,127 @@ export class SelfHealingManager {
}
}
async autoReboundPausedScopeDecay(options?: { ignoreAgeGate?: boolean }): Promise<number> {
const result = await this.autoReboundPausedScopeDecayDetailed(options);
return result.count;
}
async autoArchiveResolvedMetaTasks(reboundedTargets?: Set<string>): Promise<number> {
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
const byId = new Map(tasks.map((task) => [task.id.toUpperCase(), task]));
let archived = 0;
for (const task of tasks) {
const classified = this.classifyMetaTask(task);
if (!classified.isMeta || !classified.targetTaskId) continue;
const chainDepth = this.computeMetaChainDepth(byId, classified.targetTaskId);
const target = byId.get(classified.targetTaskId.toUpperCase());
const resolved = Boolean(target && (target.column === "done" || target.column === "archived"));
const rebounded = Boolean(reboundedTargets?.has(classified.targetTaskId));
if (!resolved && !rebounded && chainDepth < 2) continue;
try {
await this.archiveMetaTask(task.id);
await this.store.logEntry(task.id, `Auto-archived meta-task (FN-4890): target ${classified.targetTaskId} resolved/superseded.`);
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-meta", task.id), agentId: "self-healing", taskId: task.id, phase: "auto-archive-meta-resolved" });
await auditor.database({ type: "task:auto-archived-meta-resolved", target: task.id, metadata: { taskId: task.id, targetTaskId: classified.targetTaskId, targetColumn: target?.column ?? "unknown", chainDepth } });
archived++;
} catch (err: unknown) {
log.error(`autoArchiveResolvedMetaTasks failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
}
}
return archived;
}
async autoArchiveStalledMetaTasks(): Promise<number> {
const settings = await this.store.getSettings();
const thresholdMs = Number(settings.metaTaskStallAutoCloseMs ?? 2 * 60 * 60_000);
if (thresholdMs === 0) return 0;
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
const byId = new Map(tasks.map((task) => [task.id.toUpperCase(), task]));
let archived = 0;
const now = Date.now();
for (const task of tasks) {
const classified = this.classifyMetaTask(task);
if (!classified.isMeta || !classified.targetTaskId) continue;
const chainDepth = this.computeMetaChainDepth(byId, classified.targetTaskId);
const ageMs = now - Date.parse(task.columnMovedAt ?? task.updatedAt);
if (chainDepth < 2 && (!Number.isFinite(ageMs) || ageMs < thresholdMs)) continue;
const target = byId.get(classified.targetTaskId.toUpperCase());
const targetMovedAtMs = Date.parse(target?.columnMovedAt ?? target?.updatedAt ?? "");
const targetStalled = !Number.isFinite(targetMovedAtMs) || (now - targetMovedAtMs >= thresholdMs);
if (chainDepth < 2 && !targetStalled) continue;
try {
await this.archiveMetaTask(task.id);
await this.store.logEntry(task.id, `Auto-archived meta-task (FN-4890): superseded — not spawning further meta; rely on self-heal on target ${classified.targetTaskId}`);
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-meta", task.id), agentId: "self-healing", taskId: task.id, phase: "auto-archive-meta-stalled" });
await auditor.database({ type: "task:auto-archived-meta-stalled", target: task.id, metadata: { taskId: task.id, targetTaskId: classified.targetTaskId, chainDepth, stalledMs: Math.max(ageMs, 0) } });
archived++;
} catch (err: unknown) {
log.error(`autoArchiveStalledMetaTasks failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
}
}
return archived;
}
async runBoardStallAutoRecoverySweep(): Promise<{ holders: string[]; recovered: number; unrecovered: boolean }> {
const settings = await this.store.getSettings();
const windowMs = Number(settings.boardStallSweepWindowMs ?? 2 * 60 * 60_000);
const growthThreshold = Number(settings.boardStallBlockedGrowthThreshold ?? 3);
const now = Date.now();
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
const blockedDepth = this.countBlockedDepth(allTasks);
if (!this.boardStallWindow || now - this.boardStallWindow.windowStartMs >= windowMs) {
this.boardStallWindow = {
windowStartMs: now,
windowStartBlockedDepth: blockedDepth,
transitionsOutOfInProgressInWindow: 0,
pendingVerification: null,
lastNtfyAt: this.boardStallWindow?.lastNtfyAt ?? null,
};
}
const window = this.boardStallWindow;
if (window.pendingVerification && this.maintenanceTickCounter > window.pendingVerification.tick) {
const noProgress = window.transitionsOutOfInProgressInWindow === 0;
if (noProgress) {
const ntfyAllowed = window.lastNtfyAt === null || now - window.lastNtfyAt >= 15 * 60_000;
let ntfyDispatched = false;
if (ntfyAllowed) {
try {
await getActiveNotificationService()?.dispatch("board-stall-unrecovered", {
event: "board-stall-unrecovered",
metadata: {
holderIds: window.pendingVerification.holderIds,
followerCount: window.pendingVerification.followerCount,
},
} as any);
window.lastNtfyAt = now;
ntfyDispatched = true;
} catch {
ntfyDispatched = false;
}
}
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-board-stall", "global"), agentId: "self-healing", phase: "board-stall-unrecovered" });
await auditor.database({ type: "task:auto-board-stall-unrecovered", target: "board", metadata: { holderIds: window.pendingVerification.holderIds, followerCount: window.pendingVerification.followerCount, windowMs, ntfyDispatched } });
window.pendingVerification = null;
return { holders: [], recovered: 0, unrecovered: true };
}
window.pendingVerification = null;
}
const blockedGrowth = blockedDepth - window.windowStartBlockedDepth;
if (window.transitionsOutOfInProgressInWindow === 0 && blockedGrowth >= growthThreshold) {
const rebound = await this.autoReboundPausedScopeDecayDetailed({ ignoreAgeGate: true });
const followerCount = blockedDepth;
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-board-stall", "global"), agentId: "self-healing", phase: "board-stall-broken" });
await auditor.database({ type: "task:auto-board-stall-broken", target: "board", metadata: { holderIds: rebound.reboundedIds, followerCount, windowMs, blockedGrowth } });
window.pendingVerification = { holderIds: rebound.reboundedIds, followerCount, startedAt: now, tick: this.maintenanceTickCounter };
return { holders: rebound.reboundedIds, recovered: rebound.count, unrecovered: false };
}
return { holders: [], recovered: 0, unrecovered: false };
}
async clearStaleBlockedBy(): Promise<number> {
try {
const settings = await this.store.getSettings();