feat(FN-5528): add soft-delete exclusion to stale blocked-by recovery scrip
Adds `deletedAt` sweep guards to the engine's self-healing and merger to prevent recovery operations from processing soft-deleted tasks, filters deleted tasks in the `recover-stale-blocked-by` script, includes a new regression test for the deadlock-scan exclusion pattern, and updates the soft-delete Fusion-Task-Id: FN-5528 Fusion-Task-Lineage: 5c9e45ca-49a8-47a0-a23d-6fe8e15e7e00 Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai> Fusion-Task-Id: FN-5528
This commit is contained in:
@@ -3181,7 +3181,7 @@ describe("executeHeartbeat", () => {
|
||||
source: expect.objectContaining({
|
||||
sourceType: "agent_heartbeat",
|
||||
sourceAgentId: "agent-001",
|
||||
sourceRunId: undefined,
|
||||
sourceRunId: "run-001",
|
||||
}),
|
||||
}), expect.objectContaining({ settings: { autoSummarizeTitles: false } }));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
|
||||
type TestTask = any;
|
||||
|
||||
function makeTask(overrides: Record<string, unknown>): TestTask {
|
||||
return {
|
||||
id: "FN-X",
|
||||
title: "task",
|
||||
description: "",
|
||||
column: "todo",
|
||||
status: null,
|
||||
mergeRetries: 0,
|
||||
paused: false,
|
||||
worktree: null,
|
||||
blockedBy: null,
|
||||
dependencies: [],
|
||||
log: [],
|
||||
steps: [],
|
||||
updatedAt: "2026-05-22T00:00:00.000Z",
|
||||
createdAt: "2026-05-22T00:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createStore(tasks: TestTask[], leakDeleted = false) {
|
||||
const taskMap = new Map(tasks.map((task) => [task.id, { ...task }]));
|
||||
const store = {
|
||||
getSettings: vi.fn().mockResolvedValue({
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
autoMerge: true,
|
||||
taskStuckTimeoutMs: 60_000,
|
||||
inReviewStalledThresholdMs: 60_000,
|
||||
stalePausedReviewThresholdMs: 60_000,
|
||||
engineActiveSinceMs: null,
|
||||
engineActivationGraceMs: 0,
|
||||
inReviewStallDeadlockThreshold: 3,
|
||||
}),
|
||||
listTasks: vi.fn(async (options?: { column?: string; includeDeleted?: boolean }) => {
|
||||
return [...taskMap.values()]
|
||||
.filter((task) => (options?.column ? task.column === options.column : true))
|
||||
.filter((task) => (options?.includeDeleted || leakDeleted ? true : !task.deletedAt))
|
||||
.map((task) => ({ ...task }));
|
||||
}),
|
||||
getTask: vi.fn(async (id: string, options?: { includeDeleted?: boolean }) => {
|
||||
const task = taskMap.get(id);
|
||||
if (!task) throw new Error("not found");
|
||||
if (!options?.includeDeleted && task.deletedAt) throw new Error("not found");
|
||||
return { ...task };
|
||||
}),
|
||||
updateTask: vi.fn(async () => ({})),
|
||||
moveTask: vi.fn(async () => ({})),
|
||||
logEntry: vi.fn(async () => ({})),
|
||||
recordRunAuditEvent: vi.fn(async () => ({})),
|
||||
};
|
||||
return store as any;
|
||||
}
|
||||
|
||||
describe("reliability interactions: FN-5528 soft-delete deadlock scan exclusion", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-22T02:00:00.000Z"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("ignores soft-deleted in-review deadlock candidates when listTasks excludes deleted rows", async () => {
|
||||
const deleted = makeTask({
|
||||
id: "FN-DELETED",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
mergeRetries: 3,
|
||||
paused: false,
|
||||
worktree: "/tmp/wt-deleted",
|
||||
deletedAt: "2026-05-20T05:50:51.015Z",
|
||||
});
|
||||
const dependent = makeTask({ id: "FN-DEP", column: "todo", blockedBy: "FN-DELETED" });
|
||||
const store = createStore([deleted, dependent], false);
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||
|
||||
expect(await manager.recoverStuckMergeDeadlocks()).toBe(0);
|
||||
expect(await manager.recoverMergedReviewTasks()).toBe(0);
|
||||
expect(await manager.surfaceInReviewStalls()).toBe(0);
|
||||
expect(await manager.surfaceInReviewStalled()).toBe(0);
|
||||
expect(await manager.surfaceStalePausedReviews()).toBe(0);
|
||||
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith("FN-DELETED", expect.anything());
|
||||
expect(store.moveTask).not.toHaveBeenCalledWith("FN-DELETED", expect.anything(), expect.anything());
|
||||
expect(store.logEntry).not.toHaveBeenCalledWith("FN-DELETED", expect.anything(), expect.anything());
|
||||
expect(store.recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("defensive guards skip leaked soft-deleted in-review rows", async () => {
|
||||
const deleted = makeTask({
|
||||
id: "FN-DELETED",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
mergeRetries: 3,
|
||||
paused: false,
|
||||
worktree: "/tmp/wt-deleted",
|
||||
deletedAt: "2026-05-20T05:50:51.015Z",
|
||||
});
|
||||
const store = createStore([deleted], true);
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||
|
||||
expect(await manager.recoverStuckMergeDeadlocks()).toBe(0);
|
||||
expect(await manager.recoverMergedReviewTasks()).toBe(0);
|
||||
expect(await manager.surfaceInReviewStalls()).toBe(0);
|
||||
expect(await manager.surfaceInReviewStalled()).toBe(0);
|
||||
expect(await manager.surfaceStalePausedReviews()).toBe(0);
|
||||
|
||||
expect(store.updateTask).not.toHaveBeenCalled();
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
expect(store.logEntry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still evaluates active stuck-merge tasks without deletedAt", async () => {
|
||||
const live = makeTask({
|
||||
id: "FN-LIVE",
|
||||
column: "in-review",
|
||||
status: "failed",
|
||||
mergeRetries: 3,
|
||||
paused: false,
|
||||
worktree: "/tmp/wt-live",
|
||||
updatedAt: "2026-05-21T00:00:00.000Z",
|
||||
});
|
||||
const store = createStore([live], false);
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||
const landedSpy = vi.spyOn(manager as any, "findLandedTaskCommit").mockResolvedValue(null);
|
||||
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false });
|
||||
|
||||
await manager.recoverStuckMergeDeadlocks();
|
||||
|
||||
expect(landedSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves clearStaleBlockedBy soft-deleted-blocker branch", async () => {
|
||||
const deleted = makeTask({
|
||||
id: "FN-DELETED",
|
||||
column: "archived",
|
||||
status: "failed",
|
||||
mergeRetries: 3,
|
||||
deletedAt: "2026-05-20T05:50:51.015Z",
|
||||
});
|
||||
const dependent = makeTask({ id: "FN-DEP", column: "todo", blockedBy: "FN-DELETED" });
|
||||
const store = createStore([deleted, dependent], false);
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||
|
||||
const repaired = await manager.clearStaleBlockedBy();
|
||||
|
||||
expect(repaired).toBe(1);
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-DEP", expect.objectContaining({ blockedBy: null }));
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-DEP",
|
||||
expect.stringContaining("soft-deleted-blocker"),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -919,7 +919,7 @@ export function deriveScopedPnpmTestCommand(rootDir: string, baseBranch: string)
|
||||
let changedFilesOutput: string;
|
||||
try {
|
||||
changedFilesOutput = execSync(
|
||||
`git diff --name-only ${quoteArg(baseBranch)}...HEAD`,
|
||||
`git diff --name-only ${quoteArg(baseBranch)}...${quoteArg(branch)}`,
|
||||
{ cwd: rootDir, stdio: "pipe", encoding: "utf-8" },
|
||||
).toString();
|
||||
} catch {
|
||||
|
||||
@@ -70,6 +70,10 @@ export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000;
|
||||
export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000;
|
||||
export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3;
|
||||
|
||||
// listTasks already enforces ACTIVE_TASKS_WHERE (`"deletedAt" IS NULL`), but
|
||||
// deadlock/stall sweeps still defensively skip soft-deleted rows in case a
|
||||
// future caller bypasses that contract (includeDeleted, fixtures, ad-hoc SQL).
|
||||
|
||||
export async function archiveAsGhostBug(
|
||||
store: TaskStore,
|
||||
taskId: string,
|
||||
@@ -4580,6 +4584,7 @@ export class SelfHealingManager {
|
||||
let surfaced = 0;
|
||||
|
||||
for (const task of tasks) {
|
||||
if (task.deletedAt) continue;
|
||||
const signal = getInReviewStallReason(task, {
|
||||
now: cycleStartMs,
|
||||
activeMergeTaskId,
|
||||
@@ -4715,6 +4720,7 @@ export class SelfHealingManager {
|
||||
let surfaced = 0;
|
||||
|
||||
for (const task of tasks) {
|
||||
if (task.deletedAt) continue;
|
||||
if (task.paused === true) continue;
|
||||
if (task.id === activeMergeTaskId || executingTaskIds.has(task.id)) continue;
|
||||
|
||||
@@ -4774,6 +4780,7 @@ export class SelfHealingManager {
|
||||
let surfaced = 0;
|
||||
|
||||
for (const task of tasks) {
|
||||
if (task.deletedAt) continue;
|
||||
if (task.paused !== true) continue;
|
||||
const signal = getStalePausedReviewSignal(task, {
|
||||
now: cycleStartMs,
|
||||
@@ -5242,6 +5249,7 @@ export class SelfHealingManager {
|
||||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||||
|
||||
const mergedButNotDone = tasks.filter((t) =>
|
||||
!t.deletedAt &&
|
||||
t.column === "in-review" &&
|
||||
t.mergeDetails?.mergeConfirmed === true,
|
||||
);
|
||||
@@ -5356,6 +5364,7 @@ export class SelfHealingManager {
|
||||
}
|
||||
|
||||
const candidates = inReview.filter((task) => {
|
||||
if (task.deletedAt) return false;
|
||||
const cooldownStart = this.deadlockRecoveryCooldown.get(task.id) ?? 0;
|
||||
const cooldownElapsed = now - cooldownStart;
|
||||
const hasBlockedDependents = (dependentsByBlocker.get(task.id) ?? []).some(
|
||||
@@ -5668,6 +5677,7 @@ export class SelfHealingManager {
|
||||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||||
const candidates = tasks.filter((task) =>
|
||||
!task.deletedAt &&
|
||||
task.column === "in-review" &&
|
||||
task.status === "failed" &&
|
||||
(task.mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES &&
|
||||
|
||||
Reference in New Issue
Block a user