root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-4651): complete Step 3 — recover no-progress worktree failures

Browse Source 
Fusion-Task-Id: FN-4651
Fusion-Task-Lineage: 14d15555-468d-4690-8efd-a09b31b5d407
This commit is contained in:
Fusion
2026-05-15 12:38:58 -07:00
committed by gsxdsm
parent 5a9de72b3f
commit ebd0be13d9
4 changed files with 158 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/engine/src/__tests__/reliability-interactions/self-healing-interactions.test.ts
View File

@@ -71,9 +71,10 @@ describe("reliability interactions: self-healing", () => {
});
it.each([
"Refusing to start coding agent in missing worktree: /tmp/wt",
"Refusing to start coding agent in incomplete worktree: /tmp/wt",
"Refusing to start coding agent in unregistered git worktree: /tmp/wt",
])("recoverMissingWorktreeReviewFailures rebounds review tasks for '%s'", async (error) => {
])("recoverMissingWorktreeReviewFailures rebounds no-progress review tasks for '%s'", async (error) => {
const taskId = "WT";
const tasks = new Map<string, Task>([[
taskId,
@@ -84,7 +85,7 @@ describe("reliability interactions: self-healing", () => {
error,
worktree: "/tmp/wt",
branch: "fusion/wt",
steps: [{ id: "s1", title: "Step", status: "done" }] as any,
steps: [{ id: "s1", title: "Step", status: "pending" }] as any,
}),
]]);
const store = makeStore(tasks);
@@ -98,8 +99,9 @@ describe("reliability interactions: self-healing", () => {
expect(tasks.get(taskId)?.branch ?? null).toBeNull();
expect(store.logEntry).toHaveBeenCalledWith(
taskId,
expect.stringContaining("Auto-recovered: retry/verification session targeted unusable worktree"),
expect.stringContaining("Auto-recovered (no-progress): session-start refused unusable worktree"),
);
expect(tasks.get(taskId)?.worktreeSessionRetryCount).toBe(1);
});
it.skipIf(!hasGit)("recoverAlreadyMergedReviewTasks can still finalize from real git state", async () => {

15
packages/engine/src/__tests__/restart-recovery-coordinator.test.ts
View File

@@ -62,9 +62,18 @@ describe("RestartRecoveryCoordinator", () => {
expect(isRecoverableMissingWorktreeReviewFailureWithProgress({ ...baseTask, error: "other" })).toBe(false);
expect(isRecoverableMissingWorktreeReviewFailureWithProgress({ ...baseTask, steps: [{ id: "s2", title: "y", status: "pending" }] as any, error: "Refusing to start coding agent in missing worktree: /tmp/wt" })).toBe(false);
const noProgressTask = { ...baseTask, steps: [{ id: "s2", title: "y", status: "pending" }] as any, error: "Refusing to start coding agent in missing worktree: /tmp/wt" };
expect(isRecoverableMissingWorktreeReviewFailureNoProgress(noProgressTask)).toBe(true);
expect(isRecoverableMissingWorktreeReviewFailure(noProgressTask)).toBe(true);
const errors = [
"Refusing to start coding agent in missing worktree: /tmp/wt",
"Refusing to start coding agent in incomplete worktree: /tmp/wt",
"Refusing to start coding agent in unregistered git worktree: /tmp/wt",
];
for (const error of errors) {
const withProgressTask = { ...baseTask, error };
const noProgressTask = { ...baseTask, steps: [{ id: "s2", title: "y", status: "pending" }] as any, error };
expect(isRecoverableMissingWorktreeReviewFailureWithProgress(withProgressTask)).toBe(true);
expect(isRecoverableMissingWorktreeReviewFailureNoProgress(noProgressTask)).toBe(true);
expect(isRecoverableMissingWorktreeReviewFailure(noProgressTask)).toBe(true);
}
});
it("requeues interrupted failed tasks with no progress, then resumes remaining orphans", async () => {

85
packages/engine/src/__tests__/self-healing.test.ts
View File

@@ -1786,6 +1786,7 @@ describe("SelfHealingManager", () => {
expect(store.updateTask).toHaveBeenCalledWith("FN-3900", {
status: null,
error: null,
worktreeSessionRetryCount: 1,
worktree: "/tmp/project/.worktrees/fn-3900-stale",
branch: "fusion/fn-3900",
sessionFile: null,
@@ -1833,6 +1834,7 @@ describe("SelfHealingManager", () => {
expect(store.updateTask).toHaveBeenCalledWith("FN-4559", {
status: null,
error: null,
worktreeSessionRetryCount: 1,
worktree: "/tmp/project/.worktrees/noble-eagle-stale",
branch: "fusion/FN-4559",
sessionFile: null,
@@ -1882,6 +1884,80 @@ describe("SelfHealingManager", () => {
managerWithRecovery.stop();
});
it("requeues zero-progress unusable-worktree failures with cleared git/session metadata", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-4651",
column: "in-review",
paused: false,
status: "failed",
worktree: "/tmp/project/.worktrees/fn-4651",
branch: "fusion/FN-4651",
sessionFile: "/tmp/project/.fusion/sessions/FN-4651.json",
error: "Refusing to start coding agent in missing worktree: /tmp/project/.worktrees/fn-4651",
steps: [{ status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();
expect(result).toBe(1);
expect(store.updateTask).toHaveBeenCalledWith("FN-4651", {
status: null,
error: null,
worktreeSessionRetryCount: 1,
worktree: null,
branch: null,
sessionFile: null,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-4651",
expect.stringContaining("Auto-recovered (no-progress): session-start refused unusable worktree"),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-4651", "todo");
managerWithRecovery.stop();
});
it("escalates when unusable-worktree retry cap is exhausted", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-4651-CAP",
column: "in-review",
paused: false,
status: "failed",
worktreeSessionRetryCount: 3,
worktree: "/tmp/project/.worktrees/fn-4651-cap",
branch: "fusion/FN-4651-CAP",
sessionFile: "/tmp/project/.fusion/sessions/FN-4651-CAP.json",
error: "Refusing to start coding agent in unregistered git worktree: /tmp/project/.worktrees/fn-4651-cap",
steps: [{ status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();
expect(result).toBe(0);
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
"FN-4651-CAP",
"Auto-recovery exhausted (3/3) for unusable-worktree session-start failure — leaving in-review for human inspection",
);
managerWithRecovery.stop();
});
it("does not requeue non-matching in-review failures", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
@@ -1906,6 +1982,15 @@ describe("SelfHealingManager", () => {
steps: [{ status: "done" }, { status: "pending" }],
log: [],
},
{
id: "FN-3903",
column: "in-review",
paused: false,
status: "queued",
error: "Refusing to start coding agent in incomplete worktree: /tmp/project/.worktrees/fn-3903",
steps: [{ status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverMissingWorktreeReviewFailures();

67
packages/engine/src/self-healing.ts
View File

@@ -21,7 +21,12 @@ import { getInReviewStallReason, getStalePausedReviewSignal, getTaskHardMergeBlo
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js";
import { getRegisteredWorktreePaths, isUsableTaskWorktree, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import { extractMissingWorktreePathFromSessionStartFailure, isMissingWorktreeSessionStartFailure, isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
import {
extractMissingWorktreePathFromSessionStartFailure,
isMissingWorktreeSessionStartFailure,
isRecoverableMissingWorktreeReviewFailureNoProgress,
isRecoverableMissingWorktreeReviewFailureWithProgress,
} from "./restart-recovery-coordinator.js";
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
import { deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
@@ -216,6 +221,7 @@ const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
* forever; when exhausted the task stays in `in-review` for human inspection.
*/
const MAX_TASK_DONE_RETRIES = 3;
const MAX_WORKTREE_SESSION_RETRIES = 3;
const MAX_AUTO_MERGE_RETRIES = 3;
const MAX_STARVATION_DROPS = 3;
const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000;
@@ -4092,7 +4098,10 @@ export class SelfHealingManager {
async recoverMissingWorktreeReviewFailures(): Promise<number> {
try {
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
const candidates = tasks.filter((task) => isRecoverableMissingWorktreeReviewFailure(task));
const candidates = tasks.filter((task) =>
isRecoverableMissingWorktreeReviewFailureWithProgress(task)
|| isRecoverableMissingWorktreeReviewFailureNoProgress(task),
);
if (candidates.length === 0) return 0;
@@ -4101,18 +4110,48 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of candidates) {
try {
const nextCount = (task.worktreeSessionRetryCount ?? 0) + 1;
if (nextCount > MAX_WORKTREE_SESSION_RETRIES) {
await this.store.logEntry(
task.id,
`Auto-recovery exhausted (${MAX_WORKTREE_SESSION_RETRIES}/${MAX_WORKTREE_SESSION_RETRIES}) for unusable-worktree session-start failure — leaving in-review for human inspection`,
);
try {
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "maintenance",
});
await auditor.database({
type: "task:auto-recover-worktree-session-exhausted",
target: task.id,
metadata: {
retries: task.worktreeSessionRetryCount ?? 0,
maxRetries: MAX_WORKTREE_SESSION_RETRIES,
},
});
} catch (auditErr: unknown) {
log.warn(`Failed to write worktree-session exhausted run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
}
continue;
}
const staleWorktree = task.worktree;
const missingWorktreePath = extractMissingWorktreePathFromSessionStartFailure(task.error);
const hasMismatchedLiveWorktree =
typeof staleWorktree === "string" && staleWorktree.length > 0 &&
typeof missingWorktreePath === "string" && missingWorktreePath.length > 0 &&
resolve(staleWorktree) !== resolve(missingWorktreePath);
typeof staleWorktree === "string" && staleWorktree.length > 0
&& typeof missingWorktreePath === "string" && missingWorktreePath.length > 0
&& resolve(staleWorktree) !== resolve(missingWorktreePath);
const noProgress = isRecoverableMissingWorktreeReviewFailureNoProgress(task);
await this.store.updateTask(task.id, {
status: null,
error: null,
worktree: hasMismatchedLiveWorktree ? staleWorktree : null,
branch: hasMismatchedLiveWorktree ? task.branch ?? null : null,
worktreeSessionRetryCount: nextCount,
worktree: noProgress ? null : (hasMismatchedLiveWorktree ? staleWorktree : null),
branch: noProgress ? null : (hasMismatchedLiveWorktree ? task.branch ?? null : null),
sessionFile: null,
});
const failureExcerpt = typeof task.error === "string"
@@ -4120,11 +4159,17 @@ export class SelfHealingManager {
: "unknown error";
await this.store.logEntry(
task.id,
hasMismatchedLiveWorktree
? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (failure: ${failureExcerpt})`
: `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (failure: ${failureExcerpt})`,
noProgress
? `Auto-recovered (no-progress): session-start refused unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
: hasMismatchedLiveWorktree
? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
: `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`,
);
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
if (noProgress) {
await this.store.moveTask(task.id, "todo");
} else {
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
}
recovered++;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);