root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(FN-4917): complete Step 5 — add session-start recovery interaction coverage

Browse Source 
Fusion-Task-Id: FN-4917
Fusion-Task-Lineage: 1422a545-f4b5-4e1f-a347-8be2a4202f45
This commit is contained in:
Fusion (runfusion.ai)
2026-05-17 11:22:35 -07:00
committed by gsxdsm
parent 1cb952fe58
commit 56f8fc3cdf
5 changed files with 361 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
packages/engine/src/__tests__/reliability-interactions/worktree-incomplete-session-start.test.ts Normal file
View File

@@ -0,0 +1,112 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import "../executor-test-helpers.js";
import { TaskExecutor } from "../../executor.js";
import { createFnAgent } from "../../pi.js";
import { createMockStore, resetExecutorMocks } from "../executor-test-helpers.js";
const mockedCreateFnAgent = vi.mocked(createFnAgent);
function makeTask(overrides: Record<string, unknown> = {}) {
return {
id: "FN-4917-T",
title: "Task",
description: "Desc",
column: "in-progress",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
...overrides,
} as any;
}
describe("reliability interactions: FN-4917 worktree incomplete session-start", () => {
beforeEach(() => {
resetExecutorMocks();
});
it.each([
["missing", "Refusing to start coding agent in missing worktree: /tmp/wt"],
["incomplete", "Refusing to start coding agent in incomplete worktree: /tmp/wt"],
["unregistered", "Refusing to start coding agent in unregistered git worktree: /tmp/wt"],
])("executor auto-recovers %s session-start failures", async (classification, errorText) => {
const store = createMockStore();
const events: any[] = [];
store.recordRunAuditEvent = vi.fn(async (event: any) => events.push(event));
let task = makeTask({ worktree: "/tmp/wt", branch: "fusion/fn-4917-t" });
store.getTask.mockImplementation(async () => task);
store.updateTask.mockImplementation(async (_id: string, updates: any) => {
task = { ...task, ...updates };
return task;
});
store.moveTask.mockImplementation(async (_id: string, column: string, _opts?: any) => {
task = { ...task, column };
});
mockedCreateFnAgent.mockRejectedValueOnce(new Error(errorText));
const executor = new TaskExecutor(store, process.cwd());
await executor.execute(task);
expect(task.column).toBe("todo");
expect(task.status).not.toBe("failed");
expect(task.worktreeSessionRetryCount).toBe(1);
expect(task.worktree).toBeNull();
expect(task.branch).toBeNull();
expect(task.sessionFile).toBeNull();
expect(events.map((e) => e.mutationType)).toEqual(expect.arrayContaining([
"worktree:incomplete-detected",
"worktree:auto-recovered",
]));
const sessionStartEvent = events.find((e) => e.mutationType === "worktree:incomplete-detected" && e.metadata?.source === "session-start");
expect(sessionStartEvent?.metadata?.classification).toBe(classification);
expect(store.logEntry).not.toHaveBeenCalledWith("FN-4917-T", expect.stringMatching(/Refusing to start coding agent/), expect.anything(), expect.anything());
});
it("preserves progress when steps already completed", async () => {
const store = createMockStore();
let task = makeTask({
worktree: "/tmp/wt",
branch: "fusion/fn-4917-t",
steps: [
{ id: "1", title: "done", status: "done" },
{ id: "2", title: "next", status: "pending" },
],
});
store.recordRunAuditEvent = vi.fn(async () => undefined);
store.getTask.mockImplementation(async () => task);
store.updateTask.mockImplementation(async (_id: string, updates: any) => {
task = { ...task, ...updates };
return task;
});
mockedCreateFnAgent.mockRejectedValueOnce(new Error("Refusing to start coding agent in incomplete worktree: /tmp/wt"));
const executor = new TaskExecutor(store, process.cwd());
await executor.execute(task);
expect(store.moveTask).toHaveBeenCalledWith("FN-4917-T", "todo", { preserveProgress: true });
});
it("does not intercept unrelated session-start failures", async () => {
const store = createMockStore();
let task = makeTask({ worktree: "/tmp/wt", branch: "fusion/fn-4917-t" });
store.recordRunAuditEvent = vi.fn(async () => undefined);
store.getTask.mockImplementation(async () => task);
store.updateTask.mockImplementation(async (_id: string, updates: any) => {
task = { ...task, ...updates };
return task;
});
mockedCreateFnAgent.mockRejectedValueOnce(new Error("model API key missing"));
const executor = new TaskExecutor(store, process.cwd());
await executor.execute(task);
expect(store.moveTask).toHaveBeenCalledWith("FN-4917-T", "in-review");
expect(store.recordRunAuditEvent).not.toHaveBeenCalledWith(expect.objectContaining({ mutationType: "worktree:auto-recovered" }));
});
});

167
packages/engine/src/executor.ts
View File

@@ -84,10 +84,12 @@ import type { AgentReflectionService } from "./agent-reflection.js";
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
import {
classifyMissingWorktreeSessionStartFailure,
extractMissingWorktreePathFromSessionStartFailure,
isMissingWorktreeSessionStartFailure,
} from "./restart-recovery-coordinator.js";
import { BranchWorktreeAutoRecoveryHandler } from "./auto-recovery-handlers/branch-worktree.js";
import { autoRecoverWorktreeSessionStartFailure, MAX_WORKTREE_SESSION_RETRIES } from "./self-healing.js";
import { ContaminationAutoRecoveryHandler } from "./auto-recovery-handlers/contamination.js";
import { createFileScopeAutoRecoveryHandler } from "./auto-recovery-handlers/file-scope.js";
import { ReadonlyViolationError, filterCustomToolsForReadonly } from "./workflow-step-tool-policy.js";
@@ -3672,40 +3674,51 @@ export class TaskExecutor {
// sessionFile must be let because it's destructured alongside session which is reassigned
// eslint-disable-next-line prefer-const
let { session, sessionFile } = await createResolvedAgentSession({
sessionPurpose: "executor",
runtimeHint: executorRuntimeHint,
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: executorSystemPromptFinal,
systemPromptLayers: executorLayers,
tools: "coding",
customTools,
onText: agentLogger.onText,
onThinking: agentLogger.onThinking,
onToolStart: agentLogger.onToolStart,
onToolEnd: agentLogger.onToolEnd,
defaultProvider: executorProvider,
defaultModelId: executorModelId,
fallbackProvider: executorFallbackProvider,
fallbackModelId: executorFallbackModelId,
defaultThinkingLevel: executorThinkingLevel,
sessionManager,
taskEnv,
// Skill selection: use assigned agent skills if available, otherwise role fallback
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
actionGateContext: this.buildActionGateContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
taskId: task.id,
taskTitle: detail.title,
onFallbackModelUsed: createFallbackModelObserver({
agent: "executor",
label: "executor",
store: this.store,
let session: AgentSession;
let sessionFile: string | null | undefined;
try {
const createdSession = await createResolvedAgentSession({
sessionPurpose: "executor",
runtimeHint: executorRuntimeHint,
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: executorSystemPromptFinal,
systemPromptLayers: executorLayers,
tools: "coding",
customTools,
onText: agentLogger.onText,
onThinking: agentLogger.onThinking,
onToolStart: agentLogger.onToolStart,
onToolEnd: agentLogger.onToolEnd,
defaultProvider: executorProvider,
defaultModelId: executorModelId,
fallbackProvider: executorFallbackProvider,
fallbackModelId: executorFallbackModelId,
defaultThinkingLevel: executorThinkingLevel,
sessionManager,
taskEnv,
// Skill selection: use assigned agent skills if available, otherwise role fallback
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
actionGateContext: this.buildActionGateContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
permanentAgentGating: this.buildPermanentAgentGatingContext(task.id, assignedAgent, settings.defaultAgentPermissionPolicy),
taskId: task.id,
taskTitle: detail.title,
}),
});
onFallbackModelUsed: createFallbackModelObserver({
agent: "executor",
label: "executor",
store: this.store,
taskId: task.id,
taskTitle: detail.title,
}),
});
session = createdSession.session;
sessionFile = createdSession.sessionFile;
} catch (sessionStartError) {
if (await this.recoverMissingWorktreeSessionStartFailure(task, worktreePath, sessionStartError, audit)) {
return;
}
throw sessionStartError;
}
const executorModelDesc = describeModel(session);
const executorModelMarker = `Executor using model: ${executorModelDesc}`;
@@ -4134,25 +4147,13 @@ export class TaskExecutor {
role: "executor",
});
} catch (retryError) {
const retryErrorText = retryError instanceof Error ? retryError.message : String(retryError);
if (!isMissingWorktreeSessionStartFailure(retryErrorText)) {
throw retryError;
}
const recoveredPath = extractMissingWorktreePathFromSessionStartFailure(retryErrorText) ?? worktreePath;
const reclaimMessage = `${task.id}: no-fn_task_done retry hit missing worktree session-start failure (${recoveredPath}) — clearing stale metadata and requeueing`;
executorLog.log(reclaimMessage);
await this.store.logEntry(task.id, reclaimMessage, undefined, this.currentRunContext);
await this.store.updateTask(task.id, {
sessionFile: null,
worktree: null,
branch: null,
baseCommitSha: null,
});
this.deleteActiveSession(task.id);
this.tokenUsageBaselines.delete(task.id);
retrySession?.dispose();
retryAbortedDueToReclaim = true;
break;
if (await this.recoverMissingWorktreeSessionStartFailure(task, worktreePath, retryError, audit)) {
return;
}
throw retryError;
}
if (!taskDone) {
@@ -8179,6 +8180,76 @@ Backward compat fallback: if JSON is unavailable, you may still begin output wit
}
}
private async recoverMissingWorktreeSessionStartFailure(
task: Task,
worktreePath: string,
error: unknown,
audit: RunAuditor,
): Promise<boolean> {
const errorText = error instanceof Error ? error.message : String(error);
if (!isMissingWorktreeSessionStartFailure(errorText)) return false;
const classification = classifyMissingWorktreeSessionStartFailure(errorText);
const staleWorktreePath = extractMissingWorktreePathFromSessionStartFailure(errorText) ?? worktreePath;
await audit.git({
type: "worktree:incomplete-detected",
target: staleWorktreePath,
metadata: { classification, reason: errorText, source: "session-start", taskId: task.id },
});
if (isInsideWorktreesDir(this.rootDir, staleWorktreePath)) {
try {
await removeWorktree({
rootDir: this.rootDir,
worktreePath: staleWorktreePath,
settings: await this.store.getSettings(),
reason: RemovalReason.PoolPrune,
taskId: task.id,
audit,
});
} catch (removeErr) {
executorLog.warn(`${task.id}: failed to remove unusable session-start worktree ${staleWorktreePath}: ${formatError(removeErr)}`);
}
}
const recovery = await autoRecoverWorktreeSessionStartFailure(this.store, task, {
failure: error,
source: "executor-session-start",
auditor: null,
});
await audit.git({
type: "worktree:auto-recovered",
target: staleWorktreePath,
metadata: {
classification: recovery.classification,
action: recovery.outcome === "escalate-exhausted" ? "escalate-exhausted" : "requeue-todo",
retries: recovery.retries,
maxRetries: MAX_WORKTREE_SESSION_RETRIES,
staleWorktree: staleWorktreePath,
taskId: task.id,
},
});
if (recovery.outcome === "escalate-exhausted") {
await this.store.logEntry(
task.id,
`Worktree session-start auto-recovery exhausted (${recovery.retries}/${MAX_WORKTREE_SESSION_RETRIES}); task left for human inspection`,
undefined,
this.currentRunContext,
);
} else {
await this.store.logEntry(
task.id,
`Worktree was ${classification} at session start; requeued to todo for clean retry (attempt ${recovery.retries}/${MAX_WORKTREE_SESSION_RETRIES})`,
undefined,
this.currentRunContext,
);
}
return true;
}
private async emitStaleLockAudit(
taskId: string,
event:

14
packages/engine/src/restart-recovery-coordinator.ts
View File

@@ -21,7 +21,7 @@ function isNoTaskDoneFailure(task: Task): boolean {
* - Refusing to start coding agent in incomplete worktree:
* - Refusing to start coding agent in unregistered git worktree:
*/
const MISSING_WORKTREE_SESSION_PREFIXES = [
export const MISSING_WORKTREE_SESSION_PREFIXES = [
"Refusing to start coding agent in missing worktree:",
"Refusing to start coding agent in incomplete worktree:",
"Refusing to start coding agent in unregistered git worktree:",
@@ -43,6 +43,18 @@ export function isMissingWorktreeSessionStartFailure(error: unknown): boolean {
return findMissingWorktreeSessionPrefix(error) !== null;
}
export function classifyMissingWorktreeSessionStartFailure(error: unknown): "missing" | "incomplete" | "unregistered" | "unknown" {
const text = typeof error === "string"
? error
: error instanceof Error
? error.message
: "";
if (text.startsWith(MISSING_WORKTREE_SESSION_PREFIXES[0])) return "missing";
if (text.startsWith(MISSING_WORKTREE_SESSION_PREFIXES[1])) return "incomplete";
if (text.startsWith(MISSING_WORKTREE_SESSION_PREFIXES[2])) return "unregistered";
return "unknown";
}
export function extractMissingWorktreePathFromSessionStartFailure(error: unknown): string | null {
if (typeof error !== "string") return null;
const prefix = findMissingWorktreeSessionPrefix(error);

25
packages/engine/src/run-audit.ts
View File

@@ -62,10 +62,35 @@ export interface EngineRunContext {
// ── Git mutation types ─────────────────────────────────────────────────────────
/**
* Additional worktree session-start recovery metadata:
*
* ```ts
* // worktree:incomplete-detected
* metadata: {
* classification: "missing" | "incomplete" | "unregistered";
* reason?: string;
* source: "pool-acquire" | "resume" | "session-start";
* taskId?: string;
* }
*
* // worktree:auto-recovered
* metadata: {
* classification: "missing" | "incomplete" | "unregistered" | "unknown";
* action: "requeue-todo" | "escalate-exhausted";
* retries: number;
* maxRetries: number;
* staleWorktree?: string;
* taskId?: string;
* }
* ```
*/
export type GitMutationType =
| "worktree:create"
| "worktree:remove"
| "worktree:reuse"
| "worktree:incomplete-detected"
| "worktree:auto-recovered"
/**
* worktrunk run-audit metadata shape:
*

154
packages/engine/src/self-healing.ts
View File

@@ -38,7 +38,7 @@ import {
} from "./restart-recovery-coordinator.js";
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
import { deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
import { createRunAuditor, generateSyntheticRunId, type RunAuditor } from "./run-audit.js";
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
import { activeSessionRegistry } from "./active-session-registry.js";
import { findAlreadyMergedTaskCommit } from "./already-merged-detector.js";
@@ -247,7 +247,7 @@ const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
* forever; when exhausted the task stays in `in-review` for human inspection.
*/
const MAX_TASK_DONE_RETRIES = 3;
const MAX_WORKTREE_SESSION_RETRIES = 3;
export const MAX_WORKTREE_SESSION_RETRIES = 3;
const MAX_AUTO_MERGE_RETRIES = 3;
const MAX_STARVATION_DROPS = 3;
const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000;
@@ -272,6 +272,84 @@ function bumpTaskPriority(priority: TaskPriority | undefined): TaskPriority {
}
}
function classifyWorktreeSessionStartFailure(error: unknown): "missing" | "incomplete" | "unregistered" | "unknown" {
const text = typeof error === "string"
? error
: error instanceof Error
? error.message
: String(error);
if (text.startsWith("Refusing to start coding agent in missing worktree:")) return "missing";
if (text.startsWith("Refusing to start coding agent in incomplete worktree:")) return "incomplete";
if (text.startsWith("Refusing to start coding agent in unregistered git worktree:")) return "unregistered";
return "unknown";
}
export async function autoRecoverWorktreeSessionStartFailure(
store: TaskStore,
task: Task,
opts: {
failure: unknown;
source: "executor-session-start" | "in-review-sweep" | "resume-guard";
auditor: RunAuditor | null;
},
): Promise<{ outcome: "requeue-todo" | "escalate-exhausted"; retries: number; classification: "missing" | "incomplete" | "unregistered" | "unknown" }> {
const classification = classifyWorktreeSessionStartFailure(opts.failure);
const nextCount = (task.worktreeSessionRetryCount ?? 0) + 1;
if (nextCount > MAX_WORKTREE_SESSION_RETRIES) {
await store.logEntry(
task.id,
`Auto-recovery exhausted (${MAX_WORKTREE_SESSION_RETRIES}/${MAX_WORKTREE_SESSION_RETRIES}) for unusable-worktree session-start failure — leaving in-review for human inspection`,
);
await opts.auditor?.database({
type: "task:auto-recover-worktree-session-exhausted",
target: task.id,
metadata: {
retries: task.worktreeSessionRetryCount ?? 0,
maxRetries: MAX_WORKTREE_SESSION_RETRIES,
source: opts.source,
},
});
return { outcome: "escalate-exhausted", retries: task.worktreeSessionRetryCount ?? 0, classification };
}
const staleWorktree = task.worktree;
const missingWorktreePath = extractMissingWorktreePathFromSessionStartFailure(opts.failure);
const hasMismatchedLiveWorktree =
typeof staleWorktree === "string" && staleWorktree.length > 0
&& typeof missingWorktreePath === "string" && missingWorktreePath.length > 0
&& resolve(staleWorktree) !== resolve(missingWorktreePath);
const noProgress = !hasStepProgress(task);
await store.updateTask(task.id, {
status: null,
error: null,
worktreeSessionRetryCount: nextCount,
worktree: noProgress ? null : (hasMismatchedLiveWorktree ? staleWorktree : null),
branch: noProgress ? null : (hasMismatchedLiveWorktree ? task.branch ?? null : null),
sessionFile: null,
});
const failureExcerpt = typeof task.error === "string"
? task.error.slice(0, 200)
: opts.failure instanceof Error
? opts.failure.message.slice(0, 200)
: String(opts.failure).slice(0, 200);
await store.logEntry(
task.id,
noProgress
? `Auto-recovered (no-progress): session-start refused unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
: hasMismatchedLiveWorktree
? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
: `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`,
);
if (noProgress) {
await store.moveTask(task.id, "todo");
} else {
await store.moveTask(task.id, "todo", { preserveProgress: true });
}
return { outcome: "requeue-todo", retries: nextCount, classification };
}
interface OrphanBranchInspection {
branch: string;
tipSha: string;
@@ -4738,67 +4816,19 @@ export class SelfHealingManager {
let recovered = 0;
for (const task of candidates) {
try {
const nextCount = (task.worktreeSessionRetryCount ?? 0) + 1;
if (nextCount > MAX_WORKTREE_SESSION_RETRIES) {
await this.store.logEntry(
task.id,
`Auto-recovery exhausted (${MAX_WORKTREE_SESSION_RETRIES}/${MAX_WORKTREE_SESSION_RETRIES}) for unusable-worktree session-start failure — leaving in-review for human inspection`,
);
try {
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "maintenance",
});
await auditor.database({
type: "task:auto-recover-worktree-session-exhausted",
target: task.id,
metadata: {
retries: task.worktreeSessionRetryCount ?? 0,
maxRetries: MAX_WORKTREE_SESSION_RETRIES,
},
});
} catch (auditErr: unknown) {
log.warn(`Failed to write worktree-session exhausted run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
}
continue;
}
const staleWorktree = task.worktree;
const missingWorktreePath = extractMissingWorktreePathFromSessionStartFailure(task.error);
const hasMismatchedLiveWorktree =
typeof staleWorktree === "string" && staleWorktree.length > 0
&& typeof missingWorktreePath === "string" && missingWorktreePath.length > 0
&& resolve(staleWorktree) !== resolve(missingWorktreePath);
const noProgress = isRecoverableMissingWorktreeReviewFailureNoProgress(task);
await this.store.updateTask(task.id, {
status: null,
error: null,
worktreeSessionRetryCount: nextCount,
worktree: noProgress ? null : (hasMismatchedLiveWorktree ? staleWorktree : null),
branch: noProgress ? null : (hasMismatchedLiveWorktree ? task.branch ?? null : null),
sessionFile: null,
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "maintenance",
});
const failureExcerpt = typeof task.error === "string"
? task.error.slice(0, 200)
: "unknown error";
await this.store.logEntry(
task.id,
noProgress
? `Auto-recovered (no-progress): session-start refused unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
: hasMismatchedLiveWorktree
? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
: `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`,
);
if (noProgress) {
await this.store.moveTask(task.id, "todo");
} else {
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
}
recovered++;
const result = await autoRecoverWorktreeSessionStartFailure(this.store, task, {
failure: task.error,
source: "in-review-sweep",
auditor,
});
if (result.outcome === "requeue-todo") recovered++;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Failed to recover unusable-worktree review failure ${task.id}: ${errorMessage}`);