feat(FN-3972): recover in-review tasks with missing worktrees
- Add self-healing recovery to detect in-review tasks that lost their worktree and clear stale blockedBy state - Extend restart recovery coordinator handling and tests for missing-worktree classification behavior - Add focused self-healing and restart recovery test coverage for the new recovery path - Document missing-worktree review recovery behavior in architecture docs Fusion-Task-Id: FN-3972
This commit is contained in:
@@ -584,6 +584,7 @@ Runtime action-gate flow (v1):
|
|||||||
- `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification
|
- `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification
|
||||||
- `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions
|
- `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions
|
||||||
- `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`.
|
- `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`.
|
||||||
|
- `recoverMissingWorktreeReviewFailures()` is a narrow failed-review recovery: only `status: "failed"` `in-review` tasks with the explicit session-start signature `Refusing to start coding agent in missing worktree:` (from `assertValidWorktreeSession()`) are requeued. Recovery clears stale session metadata (`worktree`, `branch`, `sessionFile`, transient failure state), preserves valid step progress/retry counters, logs the auto-recovery reason, and moves the task back to `todo` for a clean retry.
|
||||||
- `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`.
|
- `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`.
|
||||||
- `recoverAlreadyMergedReviewTasks()` auto-finalizes retry-exhausted `in-review` tasks when self-healing can prove their work already landed on the merge target, preventing deadlocked cards from remaining indefinitely in failed review state.
|
- `recoverAlreadyMergedReviewTasks()` auto-finalizes retry-exhausted `in-review` tasks when self-healing can prove their work already landed on the merge target, preventing deadlocked cards from remaining indefinitely in failed review state.
|
||||||
- `clearStaleBlockedBy()` clears `blockedBy` (and transient `status`) on todo tasks when their blocker is missing, done, archived, paused in-review, or failed in-review with merge retries exhausted. FN-3924 extends this with a dependency-integrity guard: if a task has explicit dependencies and `blockedBy` is not one of the currently unresolved deps, the stale marker is cleared. This repairs rows corrupted by historical overlap re-stamping and lets scheduler re-evaluate from live dependency state.
|
- `clearStaleBlockedBy()` clears `blockedBy` (and transient `status`) on todo tasks when their blocker is missing, done, archived, paused in-review, or failed in-review with merge retries exhausted. FN-3924 extends this with a dependency-integrity guard: if a task has explicit dependencies and `blockedBy` is not one of the currently unresolved deps, the stale marker is cleared. This repairs rows corrupted by historical overlap re-stamping and lets scheduler re-evaluate from live dependency state.
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import type { TaskStore, Task } from "@fusion/core";
|
import type { TaskStore, Task } from "@fusion/core";
|
||||||
import { RestartRecoveryCoordinator } from "../restart-recovery-coordinator.js";
|
import {
|
||||||
|
RestartRecoveryCoordinator,
|
||||||
|
isMissingWorktreeSessionStartFailure,
|
||||||
|
isRecoverableMissingWorktreeReviewFailure,
|
||||||
|
} from "../restart-recovery-coordinator.js";
|
||||||
|
|
||||||
function createTask(overrides: Partial<Task>): Task {
|
function createTask(overrides: Partial<Task>): Task {
|
||||||
return {
|
return {
|
||||||
@@ -19,6 +23,26 @@ function createTask(overrides: Partial<Task>): Task {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("RestartRecoveryCoordinator", () => {
|
describe("RestartRecoveryCoordinator", () => {
|
||||||
|
it("classifies missing-worktree session-start failures narrowly", () => {
|
||||||
|
expect(isMissingWorktreeSessionStartFailure("Refusing to start coding agent in missing worktree: /tmp/wt")).toBe(true);
|
||||||
|
expect(isMissingWorktreeSessionStartFailure("Refusing to start coding agent in incomplete worktree: /tmp/wt")).toBe(false);
|
||||||
|
expect(isMissingWorktreeSessionStartFailure("Deterministic test verification failed")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("identifies recoverable in-review missing-worktree failures with step progress", () => {
|
||||||
|
const task = createTask({
|
||||||
|
column: "in-review",
|
||||||
|
paused: false,
|
||||||
|
status: "failed",
|
||||||
|
error: "Refusing to start coding agent in missing worktree: /tmp/wt",
|
||||||
|
steps: [{ id: "s1", title: "step", status: "done" }] as any,
|
||||||
|
});
|
||||||
|
expect(isRecoverableMissingWorktreeReviewFailure(task)).toBe(true);
|
||||||
|
expect(isRecoverableMissingWorktreeReviewFailure({ ...task, paused: true })).toBe(false);
|
||||||
|
expect(isRecoverableMissingWorktreeReviewFailure({ ...task, error: "other" })).toBe(false);
|
||||||
|
expect(isRecoverableMissingWorktreeReviewFailure({ ...task, steps: [{ id: "s2", title: "y", status: "pending" }] as any })).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("requeues interrupted failed tasks with no progress, then resumes remaining orphans", async () => {
|
it("requeues interrupted failed tasks with no progress, then resumes remaining orphans", async () => {
|
||||||
const store = {
|
const store = {
|
||||||
listTasks: vi.fn().mockResolvedValue([
|
listTasks: vi.fn().mockResolvedValue([
|
||||||
|
|||||||
@@ -1324,6 +1324,86 @@ describe("SelfHealingManager", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("recoverMissingWorktreeReviewFailures", () => {
|
||||||
|
it("requeues failed in-review tasks with missing-worktree session-start errors", async () => {
|
||||||
|
const managerWithRecovery = new SelfHealingManager(store, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
});
|
||||||
|
|
||||||
|
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: "FN-3900",
|
||||||
|
column: "in-review",
|
||||||
|
paused: false,
|
||||||
|
status: "failed",
|
||||||
|
worktree: "/tmp/project/.worktrees/fn-3900-stale",
|
||||||
|
branch: "fusion/fn-3900",
|
||||||
|
sessionFile: "/tmp/project/.fusion/sessions/fn-3900.json",
|
||||||
|
error: "Refusing to start coding agent in missing worktree: /tmp/other/.worktrees/fn-3900",
|
||||||
|
steps: [{ status: "done" }, { status: "pending" }],
|
||||||
|
log: [],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();
|
||||||
|
|
||||||
|
expect(result).toBe(1);
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith("FN-3900", {
|
||||||
|
status: null,
|
||||||
|
error: null,
|
||||||
|
worktree: null,
|
||||||
|
branch: null,
|
||||||
|
sessionFile: null,
|
||||||
|
});
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-3900",
|
||||||
|
expect.stringContaining("missing worktree"),
|
||||||
|
);
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-3900",
|
||||||
|
expect.stringContaining("/tmp/project/.worktrees/fn-3900-stale"),
|
||||||
|
);
|
||||||
|
expect(store.moveTask).toHaveBeenCalledWith("FN-3900", "todo", { preserveProgress: true });
|
||||||
|
|
||||||
|
managerWithRecovery.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not requeue non-matching in-review failures", async () => {
|
||||||
|
const managerWithRecovery = new SelfHealingManager(store, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
});
|
||||||
|
|
||||||
|
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: "FN-3901",
|
||||||
|
column: "in-review",
|
||||||
|
paused: false,
|
||||||
|
status: "failed",
|
||||||
|
error: "Deterministic test verification failed",
|
||||||
|
steps: [{ status: "done" }, { status: "pending" }],
|
||||||
|
log: [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "FN-3902",
|
||||||
|
column: "in-review",
|
||||||
|
paused: true,
|
||||||
|
status: "failed",
|
||||||
|
error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-3902",
|
||||||
|
steps: [{ status: "done" }, { status: "pending" }],
|
||||||
|
log: [],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalled();
|
||||||
|
expect(store.moveTask).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
managerWithRecovery.stop();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("recoverMisclassifiedFailures", () => {
|
describe("recoverMisclassifiedFailures", () => {
|
||||||
it("clears failed status when all steps are done and error is no-task_done", async () => {
|
it("clears failed status when all steps are done and error is no-task_done", async () => {
|
||||||
const managerWithRecovery = new SelfHealingManager(store, {
|
const managerWithRecovery = new SelfHealingManager(store, {
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { createLogger } from "./logger.js";
|
|||||||
|
|
||||||
const log = createLogger("restart-recovery");
|
const log = createLogger("restart-recovery");
|
||||||
|
|
||||||
function hasStepProgress(task: Task): boolean {
|
export function hasStepProgress(task: Task): boolean {
|
||||||
const steps = Array.isArray(task.steps) ? task.steps : [];
|
const steps = Array.isArray(task.steps) ? task.steps : [];
|
||||||
return steps.some((step) => step.status === "done" || step.status === "in-progress" || step.status === "skipped");
|
return steps.some((step) => step.status === "done" || step.status === "in-progress" || step.status === "skipped");
|
||||||
}
|
}
|
||||||
@@ -15,6 +15,21 @@ function isNoTaskDoneFailure(task: Task): boolean {
|
|||||||
&& task.error.toLowerCase().includes("without calling fn_task_done");
|
&& task.error.toLowerCase().includes("without calling fn_task_done");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isMissingWorktreeSessionStartFailure(error: unknown): boolean {
|
||||||
|
if (typeof error !== "string") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return error.includes("Refusing to start coding agent in missing worktree:");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecoverableMissingWorktreeReviewFailure(task: Task): boolean {
|
||||||
|
return task.column === "in-review"
|
||||||
|
&& !task.paused
|
||||||
|
&& task.status === "failed"
|
||||||
|
&& isMissingWorktreeSessionStartFailure(task.error)
|
||||||
|
&& hasStepProgress(task);
|
||||||
|
}
|
||||||
|
|
||||||
export class RestartRecoveryCoordinator {
|
export class RestartRecoveryCoordinator {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly store: TaskStore,
|
private readonly store: TaskStore,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import { getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore,
|
|||||||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||||||
import { createLogger } from "./logger.js";
|
import { createLogger } from "./logger.js";
|
||||||
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||||
|
import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
|
||||||
|
|
||||||
const log = createLogger("self-healing");
|
const log = createLogger("self-healing");
|
||||||
const execAsync = promisify(exec);
|
const execAsync = promisify(exec);
|
||||||
@@ -239,6 +240,7 @@ export class SelfHealingManager {
|
|||||||
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) },
|
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) },
|
||||||
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) },
|
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) },
|
||||||
{ name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
|
{ name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
|
||||||
|
{ name: "missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures().then(() => undefined) },
|
||||||
{ name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) },
|
{ name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) },
|
||||||
{ name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) },
|
{ name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) },
|
||||||
{ name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) },
|
{ name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) },
|
||||||
@@ -847,6 +849,7 @@ export class SelfHealingManager {
|
|||||||
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() },
|
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() },
|
||||||
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() },
|
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() },
|
||||||
{ name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
|
{ name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
|
||||||
|
{ name: "recover-missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures() },
|
||||||
{ name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() },
|
{ name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() },
|
||||||
{ name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() },
|
{ name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() },
|
||||||
{ name: "recover-orphaned-executions", fn: () => this.recoverOrphanedExecutions() },
|
{ name: "recover-orphaned-executions", fn: () => this.recoverOrphanedExecutions() },
|
||||||
@@ -2256,6 +2259,57 @@ export class SelfHealingManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recover failed `in-review` retries that point at a missing worktree path.
|
||||||
|
*
|
||||||
|
* This is a narrow guard for session-start failures thrown by
|
||||||
|
* assertValidWorktreeSession() (`Refusing to start coding agent in missing worktree:`).
|
||||||
|
* We clear stale worktree metadata and failure state, keep step progress and
|
||||||
|
* retry counters, then requeue to todo for a clean retry.
|
||||||
|
*/
|
||||||
|
async recoverMissingWorktreeReviewFailures(): Promise<number> {
|
||||||
|
try {
|
||||||
|
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||||||
|
const candidates = tasks.filter((task) => isRecoverableMissingWorktreeReviewFailure(task));
|
||||||
|
|
||||||
|
if (candidates.length === 0) return 0;
|
||||||
|
|
||||||
|
log.warn(`Found ${candidates.length} in-review task(s) failed by missing-worktree session start`);
|
||||||
|
|
||||||
|
let recovered = 0;
|
||||||
|
for (const task of candidates) {
|
||||||
|
try {
|
||||||
|
const staleWorktree = task.worktree;
|
||||||
|
await this.store.updateTask(task.id, {
|
||||||
|
status: null,
|
||||||
|
error: null,
|
||||||
|
worktree: null,
|
||||||
|
branch: null,
|
||||||
|
sessionFile: null,
|
||||||
|
});
|
||||||
|
await this.store.logEntry(
|
||||||
|
task.id,
|
||||||
|
`Auto-recovered: retry/verification session targeted missing worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo`,
|
||||||
|
);
|
||||||
|
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||||||
|
recovered++;
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||||
|
log.error(`Failed to recover missing-worktree review failure ${task.id}: ${errorMessage}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recovered > 0) {
|
||||||
|
log.log(`Recovered ${recovered} missing-worktree review failure(s) → todo`);
|
||||||
|
}
|
||||||
|
return recovered;
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||||
|
log.error(`Missing-worktree review recovery failed: ${errorMessage}`);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Recover `in-review` tasks marked as `failed` because the agent exited
|
* Recover `in-review` tasks marked as `failed` because the agent exited
|
||||||
* without calling `task_done` *with partial step progress* (some steps done,
|
* without calling `task_done` *with partial step progress* (some steps done,
|
||||||
|
|||||||
Reference in New Issue
Block a user