feat(FN-3972): recover in-review tasks with missing worktrees

- Add self-healing recovery to detect in-review tasks that lost their worktree and clear stale blockedBy state
- Extend restart recovery coordinator handling and tests for missing-worktree classification behavior
- Add focused self-healing and restart recovery test coverage for the new recovery path
- Document missing-worktree review recovery behavior in architecture docs

Fusion-Task-Id: FN-3972
This commit is contained in:
Fusion
2026-05-10 23:32:21 -07:00
committed by gsxdsm
parent be404e5967
commit ccd899dab4
5 changed files with 176 additions and 2 deletions

View File

@@ -584,6 +584,7 @@ Runtime action-gate flow (v1):
- `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification
- `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions
- `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`.
- `recoverMissingWorktreeReviewFailures()` is a narrow failed-review recovery: only `status: "failed"` `in-review` tasks with the explicit session-start signature `Refusing to start coding agent in missing worktree:` (from `assertValidWorktreeSession()`) are requeued. Recovery clears stale session metadata (`worktree`, `branch`, `sessionFile`, transient failure state), preserves valid step progress/retry counters, logs the auto-recovery reason, and moves the task back to `todo` for a clean retry.
- `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`. - `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`.
- `recoverAlreadyMergedReviewTasks()` auto-finalizes retry-exhausted `in-review` tasks when self-healing can prove their work already landed on the merge target, preventing deadlocked cards from remaining indefinitely in failed review state. - `recoverAlreadyMergedReviewTasks()` auto-finalizes retry-exhausted `in-review` tasks when self-healing can prove their work already landed on the merge target, preventing deadlocked cards from remaining indefinitely in failed review state.
- `clearStaleBlockedBy()` clears `blockedBy` (and transient `status`) on todo tasks when their blocker is missing, done, archived, paused in-review, or failed in-review with merge retries exhausted. FN-3924 extends this with a dependency-integrity guard: if a task has explicit dependencies and `blockedBy` is not one of the currently unresolved deps, the stale marker is cleared. This repairs rows corrupted by historical overlap re-stamping and lets scheduler re-evaluate from live dependency state. - `clearStaleBlockedBy()` clears `blockedBy` (and transient `status`) on todo tasks when their blocker is missing, done, archived, paused in-review, or failed in-review with merge retries exhausted. FN-3924 extends this with a dependency-integrity guard: if a task has explicit dependencies and `blockedBy` is not one of the currently unresolved deps, the stale marker is cleared. This repairs rows corrupted by historical overlap re-stamping and lets scheduler re-evaluate from live dependency state.

View File

@@ -1,6 +1,10 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import type { TaskStore, Task } from "@fusion/core"; import type { TaskStore, Task } from "@fusion/core";
import { RestartRecoveryCoordinator } from "../restart-recovery-coordinator.js"; import {
RestartRecoveryCoordinator,
isMissingWorktreeSessionStartFailure,
isRecoverableMissingWorktreeReviewFailure,
} from "../restart-recovery-coordinator.js";
function createTask(overrides: Partial<Task>): Task { function createTask(overrides: Partial<Task>): Task {
return { return {
@@ -19,6 +23,26 @@ function createTask(overrides: Partial<Task>): Task {
} }
describe("RestartRecoveryCoordinator", () => { describe("RestartRecoveryCoordinator", () => {
it("classifies missing-worktree session-start failures narrowly", () => {
expect(isMissingWorktreeSessionStartFailure("Refusing to start coding agent in missing worktree: /tmp/wt")).toBe(true);
expect(isMissingWorktreeSessionStartFailure("Refusing to start coding agent in incomplete worktree: /tmp/wt")).toBe(false);
expect(isMissingWorktreeSessionStartFailure("Deterministic test verification failed")).toBe(false);
});
it("identifies recoverable in-review missing-worktree failures with step progress", () => {
const task = createTask({
column: "in-review",
paused: false,
status: "failed",
error: "Refusing to start coding agent in missing worktree: /tmp/wt",
steps: [{ id: "s1", title: "step", status: "done" }] as any,
});
expect(isRecoverableMissingWorktreeReviewFailure(task)).toBe(true);
expect(isRecoverableMissingWorktreeReviewFailure({ ...task, paused: true })).toBe(false);
expect(isRecoverableMissingWorktreeReviewFailure({ ...task, error: "other" })).toBe(false);
expect(isRecoverableMissingWorktreeReviewFailure({ ...task, steps: [{ id: "s2", title: "y", status: "pending" }] as any })).toBe(false);
});
it("requeues interrupted failed tasks with no progress, then resumes remaining orphans", async () => { it("requeues interrupted failed tasks with no progress, then resumes remaining orphans", async () => {
const store = { const store = {
listTasks: vi.fn().mockResolvedValue([ listTasks: vi.fn().mockResolvedValue([

View File

@@ -1324,6 +1324,86 @@ describe("SelfHealingManager", () => {
}); });
}); });
describe("recoverMissingWorktreeReviewFailures", () => {
it("requeues failed in-review tasks with missing-worktree session-start errors", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-3900",
column: "in-review",
paused: false,
status: "failed",
worktree: "/tmp/project/.worktrees/fn-3900-stale",
branch: "fusion/fn-3900",
sessionFile: "/tmp/project/.fusion/sessions/fn-3900.json",
error: "Refusing to start coding agent in missing worktree: /tmp/other/.worktrees/fn-3900",
steps: [{ status: "done" }, { status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();
expect(result).toBe(1);
expect(store.updateTask).toHaveBeenCalledWith("FN-3900", {
status: null,
error: null,
worktree: null,
branch: null,
sessionFile: null,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-3900",
expect.stringContaining("missing worktree"),
);
expect(store.logEntry).toHaveBeenCalledWith(
"FN-3900",
expect.stringContaining("/tmp/project/.worktrees/fn-3900-stale"),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-3900", "todo", { preserveProgress: true });
managerWithRecovery.stop();
});
it("does not requeue non-matching in-review failures", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-3901",
column: "in-review",
paused: false,
status: "failed",
error: "Deterministic test verification failed",
steps: [{ status: "done" }, { status: "pending" }],
log: [],
},
{
id: "FN-3902",
column: "in-review",
paused: true,
status: "failed",
error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-3902",
steps: [{ status: "done" }, { status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();
expect(result).toBe(0);
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
});
describe("recoverMisclassifiedFailures", () => { describe("recoverMisclassifiedFailures", () => {
it("clears failed status when all steps are done and error is no-task_done", async () => { it("clears failed status when all steps are done and error is no-task_done", async () => {
const managerWithRecovery = new SelfHealingManager(store, { const managerWithRecovery = new SelfHealingManager(store, {

View File

@@ -4,7 +4,7 @@ import { createLogger } from "./logger.js";
const log = createLogger("restart-recovery"); const log = createLogger("restart-recovery");
function hasStepProgress(task: Task): boolean { export function hasStepProgress(task: Task): boolean {
const steps = Array.isArray(task.steps) ? task.steps : []; const steps = Array.isArray(task.steps) ? task.steps : [];
return steps.some((step) => step.status === "done" || step.status === "in-progress" || step.status === "skipped"); return steps.some((step) => step.status === "done" || step.status === "in-progress" || step.status === "skipped");
} }
@@ -15,6 +15,21 @@ function isNoTaskDoneFailure(task: Task): boolean {
&& task.error.toLowerCase().includes("without calling fn_task_done"); && task.error.toLowerCase().includes("without calling fn_task_done");
} }
export function isMissingWorktreeSessionStartFailure(error: unknown): boolean {
if (typeof error !== "string") {
return false;
}
return error.includes("Refusing to start coding agent in missing worktree:");
}
export function isRecoverableMissingWorktreeReviewFailure(task: Task): boolean {
return task.column === "in-review"
&& !task.paused
&& task.status === "failed"
&& isMissingWorktreeSessionStartFailure(task.error)
&& hasStepProgress(task);
}
export class RestartRecoveryCoordinator { export class RestartRecoveryCoordinator {
constructor( constructor(
private readonly store: TaskStore, private readonly store: TaskStore,

View File

@@ -21,6 +21,7 @@ import { getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore,
import type { MeshLeaseManager } from "./mesh-lease-manager.js"; import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js"; import { createLogger } from "./logger.js";
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js"; import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
const log = createLogger("self-healing"); const log = createLogger("self-healing");
const execAsync = promisify(exec); const execAsync = promisify(exec);
@@ -239,6 +240,7 @@ export class SelfHealingManager {
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) }, { name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) },
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) }, { name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) },
{ name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) }, { name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
{ name: "missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures().then(() => undefined) },
{ name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) }, { name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) },
{ name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) }, { name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) },
{ name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) }, { name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) },
@@ -847,6 +849,7 @@ export class SelfHealingManager {
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() }, { name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() },
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() }, { name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() },
{ name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() }, { name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
{ name: "recover-missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures() },
{ name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() }, { name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() },
{ name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() }, { name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() },
{ name: "recover-orphaned-executions", fn: () => this.recoverOrphanedExecutions() }, { name: "recover-orphaned-executions", fn: () => this.recoverOrphanedExecutions() },
@@ -2256,6 +2259,57 @@ export class SelfHealingManager {
} }
} }
/**
* Recover failed `in-review` retries that point at a missing worktree path.
*
* This is a narrow guard for session-start failures thrown by
* assertValidWorktreeSession() (`Refusing to start coding agent in missing worktree:`).
* We clear stale worktree metadata and failure state, keep step progress and
* retry counters, then requeue to todo for a clean retry.
*/
async recoverMissingWorktreeReviewFailures(): Promise<number> {
try {
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
const candidates = tasks.filter((task) => isRecoverableMissingWorktreeReviewFailure(task));
if (candidates.length === 0) return 0;
log.warn(`Found ${candidates.length} in-review task(s) failed by missing-worktree session start`);
let recovered = 0;
for (const task of candidates) {
try {
const staleWorktree = task.worktree;
await this.store.updateTask(task.id, {
status: null,
error: null,
worktree: null,
branch: null,
sessionFile: null,
});
await this.store.logEntry(
task.id,
`Auto-recovered: retry/verification session targeted missing worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo`,
);
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
recovered++;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Failed to recover missing-worktree review failure ${task.id}: ${errorMessage}`);
}
}
if (recovered > 0) {
log.log(`Recovered ${recovered} missing-worktree review failure(s) → todo`);
}
return recovered;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Missing-worktree review recovery failed: ${errorMessage}`);
return 0;
}
}
/** /**
* Recover `in-review` tasks marked as `failed` because the agent exited * Recover `in-review` tasks marked as `failed` because the agent exited
* without calling `task_done` *with partial step progress* (some steps done, * without calling `task_done` *with partial step progress* (some steps done,