feat(FN-5528): add soft-delete exclusion to stale blocked-by recovery scrip

Adds `deletedAt` sweep guards to the engine's self-healing and merger to prevent recovery operations from processing soft-deleted tasks, filters deleted tasks in the `recover-stale-blocked-by` script, includes a new regression test for the deadlock-scan exclusion pattern, and updates the soft-delete

Fusion-Task-Id: FN-5528

Fusion-Task-Lineage: 5c9e45ca-49a8-47a0-a23d-6fe8e15e7e00

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
Fusion-Task-Id: FN-5528
This commit is contained in:
gsxdsm
2026-05-23 03:33:53 -07:00
parent 2209c57dd6
commit d4ec82bf02
7 changed files with 241 additions and 7 deletions

View File

@@ -3181,7 +3181,7 @@ describe("executeHeartbeat", () => {
source: expect.objectContaining({
sourceType: "agent_heartbeat",
sourceAgentId: "agent-001",
sourceRunId: undefined,
sourceRunId: "run-001",
}),
}), expect.objectContaining({ settings: { autoSummarizeTitles: false } }));
});

View File

@@ -0,0 +1,162 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { SelfHealingManager } from "../../self-healing.js";
type TestTask = any;
function makeTask(overrides: Record<string, unknown>): TestTask {
return {
id: "FN-X",
title: "task",
description: "",
column: "todo",
status: null,
mergeRetries: 0,
paused: false,
worktree: null,
blockedBy: null,
dependencies: [],
log: [],
steps: [],
updatedAt: "2026-05-22T00:00:00.000Z",
createdAt: "2026-05-22T00:00:00.000Z",
...overrides,
};
}
function createStore(tasks: TestTask[], leakDeleted = false) {
const taskMap = new Map(tasks.map((task) => [task.id, { ...task }]));
const store = {
getSettings: vi.fn().mockResolvedValue({
globalPause: false,
enginePaused: false,
autoMerge: true,
taskStuckTimeoutMs: 60_000,
inReviewStalledThresholdMs: 60_000,
stalePausedReviewThresholdMs: 60_000,
engineActiveSinceMs: null,
engineActivationGraceMs: 0,
inReviewStallDeadlockThreshold: 3,
}),
listTasks: vi.fn(async (options?: { column?: string; includeDeleted?: boolean }) => {
return [...taskMap.values()]
.filter((task) => (options?.column ? task.column === options.column : true))
.filter((task) => (options?.includeDeleted || leakDeleted ? true : !task.deletedAt))
.map((task) => ({ ...task }));
}),
getTask: vi.fn(async (id: string, options?: { includeDeleted?: boolean }) => {
const task = taskMap.get(id);
if (!task) throw new Error("not found");
if (!options?.includeDeleted && task.deletedAt) throw new Error("not found");
return { ...task };
}),
updateTask: vi.fn(async () => ({})),
moveTask: vi.fn(async () => ({})),
logEntry: vi.fn(async () => ({})),
recordRunAuditEvent: vi.fn(async () => ({})),
};
return store as any;
}
describe("reliability interactions: FN-5528 soft-delete deadlock scan exclusion", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-22T02:00:00.000Z"));
});
afterEach(() => {
vi.useRealTimers();
});
it("ignores soft-deleted in-review deadlock candidates when listTasks excludes deleted rows", async () => {
const deleted = makeTask({
id: "FN-DELETED",
column: "in-review",
status: "failed",
mergeRetries: 3,
paused: false,
worktree: "/tmp/wt-deleted",
deletedAt: "2026-05-20T05:50:51.015Z",
});
const dependent = makeTask({ id: "FN-DEP", column: "todo", blockedBy: "FN-DELETED" });
const store = createStore([deleted, dependent], false);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
expect(await manager.recoverStuckMergeDeadlocks()).toBe(0);
expect(await manager.recoverMergedReviewTasks()).toBe(0);
expect(await manager.surfaceInReviewStalls()).toBe(0);
expect(await manager.surfaceInReviewStalled()).toBe(0);
expect(await manager.surfaceStalePausedReviews()).toBe(0);
expect(store.updateTask).not.toHaveBeenCalledWith("FN-DELETED", expect.anything());
expect(store.moveTask).not.toHaveBeenCalledWith("FN-DELETED", expect.anything(), expect.anything());
expect(store.logEntry).not.toHaveBeenCalledWith("FN-DELETED", expect.anything(), expect.anything());
expect(store.recordRunAuditEvent).not.toHaveBeenCalled();
});
it("defensive guards skip leaked soft-deleted in-review rows", async () => {
const deleted = makeTask({
id: "FN-DELETED",
column: "in-review",
status: "failed",
mergeRetries: 3,
paused: false,
worktree: "/tmp/wt-deleted",
deletedAt: "2026-05-20T05:50:51.015Z",
});
const store = createStore([deleted], true);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
expect(await manager.recoverStuckMergeDeadlocks()).toBe(0);
expect(await manager.recoverMergedReviewTasks()).toBe(0);
expect(await manager.surfaceInReviewStalls()).toBe(0);
expect(await manager.surfaceInReviewStalled()).toBe(0);
expect(await manager.surfaceStalePausedReviews()).toBe(0);
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.logEntry).not.toHaveBeenCalled();
});
it("still evaluates active stuck-merge tasks without deletedAt", async () => {
const live = makeTask({
id: "FN-LIVE",
column: "in-review",
status: "failed",
mergeRetries: 3,
paused: false,
worktree: "/tmp/wt-live",
updatedAt: "2026-05-21T00:00:00.000Z",
});
const store = createStore([live], false);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
const landedSpy = vi.spyOn(manager as any, "findLandedTaskCommit").mockResolvedValue(null);
vi.spyOn(manager as any, "evaluateBackwardMoveTripleProof").mockResolvedValue({ ok: false });
await manager.recoverStuckMergeDeadlocks();
expect(landedSpy).toHaveBeenCalled();
});
it("preserves clearStaleBlockedBy soft-deleted-blocker branch", async () => {
const deleted = makeTask({
id: "FN-DELETED",
column: "archived",
status: "failed",
mergeRetries: 3,
deletedAt: "2026-05-20T05:50:51.015Z",
});
const dependent = makeTask({ id: "FN-DEP", column: "todo", blockedBy: "FN-DELETED" });
const store = createStore([deleted, dependent], false);
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
const repaired = await manager.clearStaleBlockedBy();
expect(repaired).toBe(1);
expect(store.updateTask).toHaveBeenCalledWith("FN-DEP", expect.objectContaining({ blockedBy: null }));
expect(store.logEntry).toHaveBeenCalledWith(
"FN-DEP",
expect.stringContaining("soft-deleted-blocker"),
);
});
});

View File

@@ -919,7 +919,7 @@ export function deriveScopedPnpmTestCommand(rootDir: string, baseBranch: string)
let changedFilesOutput: string;
try {
changedFilesOutput = execSync(
`git diff --name-only ${quoteArg(baseBranch)}...HEAD`,
`git diff --name-only ${quoteArg(baseBranch)}...${quoteArg(branch)}`,
{ cwd: rootDir, stdio: "pipe", encoding: "utf-8" },
).toString();
} catch {

View File

@@ -70,6 +70,10 @@ export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000;
export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000;
export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3;
// listTasks already enforces ACTIVE_TASKS_WHERE (`"deletedAt" IS NULL`), but
// deadlock/stall sweeps still defensively skip soft-deleted rows in case a
// future caller bypasses that contract (includeDeleted, fixtures, ad-hoc SQL).
export async function archiveAsGhostBug(
store: TaskStore,
taskId: string,
@@ -4580,6 +4584,7 @@ export class SelfHealingManager {
let surfaced = 0;
for (const task of tasks) {
if (task.deletedAt) continue;
const signal = getInReviewStallReason(task, {
now: cycleStartMs,
activeMergeTaskId,
@@ -4715,6 +4720,7 @@ export class SelfHealingManager {
let surfaced = 0;
for (const task of tasks) {
if (task.deletedAt) continue;
if (task.paused === true) continue;
if (task.id === activeMergeTaskId || executingTaskIds.has(task.id)) continue;
@@ -4774,6 +4780,7 @@ export class SelfHealingManager {
let surfaced = 0;
for (const task of tasks) {
if (task.deletedAt) continue;
if (task.paused !== true) continue;
const signal = getStalePausedReviewSignal(task, {
now: cycleStartMs,
@@ -5242,6 +5249,7 @@ export class SelfHealingManager {
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
const mergedButNotDone = tasks.filter((t) =>
!t.deletedAt &&
t.column === "in-review" &&
t.mergeDetails?.mergeConfirmed === true,
);
@@ -5356,6 +5364,7 @@ export class SelfHealingManager {
}
const candidates = inReview.filter((task) => {
if (task.deletedAt) return false;
const cooldownStart = this.deadlockRecoveryCooldown.get(task.id) ?? 0;
const cooldownElapsed = now - cooldownStart;
const hasBlockedDependents = (dependentsByBlocker.get(task.id) ?? []).some(
@@ -5668,6 +5677,7 @@ export class SelfHealingManager {
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
const candidates = tasks.filter((task) =>
!task.deletedAt &&
task.column === "in-review" &&
task.status === "failed" &&
(task.mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES &&