feat(FN-4015): classify and recover from stale module-resolution incidents

Merges FN-4015 to add stale module-resolution incident detection and self-healing in the engine (`transient-error-detector.ts`, `self-healing.ts`) with mobile regression coverage for the shared `AgentErrorDetailsModal` layout.

Fusion-Task-Id: FN-4015
This commit is contained in:
Fusion
2026-05-11 11:59:07 -07:00
committed by gsxdsm
parent 0157e66ff9
commit 385c2dd58f
8 changed files with 301 additions and 15 deletions

View File

@@ -31,6 +31,7 @@
flex: 1 1 auto;
min-height: 0;
overflow: hidden;
display: flex;
}
.agent-error-modal__error {
@@ -52,15 +53,14 @@
.agent-error-modal {
width: 100%;
max-width: 100%;
height: 100vh;
height: 100dvh;
max-height: 100vh;
max-height: 100dvh;
height: 100%;
max-height: 100%;
flex: 1 1 auto;
min-height: 0;
}
.agent-error-modal__content {
display: flex;
flex: 1 1 auto;
}
.agent-error-modal__error {
@@ -69,5 +69,6 @@
min-height: 0;
width: 100%;
-webkit-overflow-scrolling: touch;
overscroll-behavior: contain;
}
}

View File

@@ -0,0 +1,68 @@
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import { render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { AgentErrorDetailsModal, AgentErrorIndicator } from "../AgentErrorDetailsModal";
const issueContext = {
surface: "AgentsView",
agentId: "agent-1",
agentName: "Test Agent",
agentState: "error",
runId: "run-1",
taskId: "FN-1",
timestamp: "2026-01-01T00:00:00.000Z",
};
describe("AgentErrorDetailsModal", () => {
const originalClipboard = navigator.clipboard;
const openSpy = vi.spyOn(window, "open").mockImplementation(() => null);
beforeEach(() => {
Object.defineProperty(navigator, "clipboard", {
value: { writeText: vi.fn().mockResolvedValue(undefined) },
configurable: true,
});
openSpy.mockClear();
});
afterEach(() => {
Object.defineProperty(navigator, "clipboard", { value: originalClipboard, configurable: true });
});
it("does not render when closed", () => {
const { container } = render(<AgentErrorDetailsModal open={false} onClose={vi.fn()} errorText="boom" issueContext={issueContext} />);
expect(container).toBeEmptyDOMElement();
});
it("renders long error text in scrollable error region", () => {
render(<AgentErrorDetailsModal open={true} onClose={vi.fn()} errorText={"stderr\n".repeat(200)} issueContext={issueContext} />);
const errorRegion = document.querySelector(".agent-error-modal__error");
expect(errorRegion).toBeInTheDocument();
expect(errorRegion).toHaveTextContent("stderr");
});
it("copies error text", async () => {
const user = userEvent.setup();
render(<AgentErrorDetailsModal open={true} onClose={vi.fn()} errorText="copy me" issueContext={issueContext} />);
await user.click(screen.getByRole("button", { name: "Copy error to clipboard" }));
await waitFor(() => {
expect(screen.getByRole("button", { name: "Copied error to clipboard" })).toBeInTheDocument();
});
});
it("opens github report link", async () => {
const user = userEvent.setup();
render(<AgentErrorDetailsModal open={true} onClose={vi.fn()} errorText="report me" issueContext={issueContext} />);
await user.click(screen.getByRole("link", { name: /report on github/i }));
expect(openSpy).toHaveBeenCalledTimes(1);
expect(openSpy.mock.calls[0]?.[0]).toContain("https://github.com/Runfusion/Fusion/issues/new?");
});
it("AgentErrorIndicator opens shared modal", async () => {
const user = userEvent.setup();
render(<AgentErrorIndicator errorText="indicator error" issueContext={issueContext} />);
await user.click(screen.getByRole("button", { name: "Open error details" }));
expect(screen.getByRole("dialog", { name: "Agent error details" })).toBeInTheDocument();
});
});

View File

@@ -394,14 +394,15 @@ describe("agent modal mobile CSS structure", () => {
});
describe("AgentErrorDetailsModal", () => {
it("mobile rules constrain modal to viewport height", () => {
it("mobile rules let modal fill the fullscreen container without double viewport clipping", () => {
const styles = readStyles();
const modalRuleMatch = styles.match(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal\s*\{[^}]+\}/);
const modalRuleMatch = styles.match(/@media\s*\(max-width:\s*768px\)\s*\{\s*\.agent-error-modal\s*\{[^}]+\}/);
expect(modalRuleMatch).toBeTruthy();
const modalRule = modalRuleMatch![0];
expect(modalRule).toContain("height: 100dvh");
expect(modalRule).toContain("max-height: 100dvh");
expect(modalRule).toContain("height: 100%");
expect(modalRule).toContain("max-height: 100%");
expect(modalRule).toContain("min-height: 0");
expect(modalRule).toContain("width: 100%");
expect(modalRule).toContain("max-width: 100%");
});
@@ -414,6 +415,7 @@ describe("agent modal mobile CSS structure", () => {
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*max-height:\s*none;[^}]*\}/);
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*-webkit-overflow-scrolling:\s*touch;[^}]*\}/);
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*overscroll-behavior:\s*contain;[^}]*\}/);
});
});

View File

@@ -298,22 +298,26 @@ describe("core modals mobile css coverage", () => {
expect(fullscreenBlockMatch![0]).toContain("max-height: unset");
});
it("AgentErrorDetailsModal: mobile uses viewport-sized modal with inner scrolling log region", () => {
it("AgentErrorDetailsModal: mobile fills fullscreen container and keeps inner scrolling log region", () => {
const css = loadAllAppCss();
const mobileBlock = getMainMobileBlock(css);
const modalRuleMatch = mobileBlock.match(/\.agent-error-modal\s*\{[^}]+\}/s);
expect(modalRuleMatch).not.toBeNull();
expect(modalRuleMatch![0]).toContain("height: 100dvh");
expect(modalRuleMatch![0]).toContain("max-height: 100dvh");
expect(modalRuleMatch![0]).toContain("height: 100%");
expect(modalRuleMatch![0]).toContain("max-height: 100%");
expect(modalRuleMatch![0]).toContain("min-height: 0");
const contentRuleMatch = css.match(/\.agent-error-modal__content\s*\{[^}]+\}/s);
expect(contentRuleMatch).not.toBeNull();
expect(contentRuleMatch![0]).toContain("overflow: hidden");
expect(contentRuleMatch![0]).toContain("display: flex");
const errorRuleMatch = mobileBlock.match(/\.agent-error-modal__error\s*\{[^}]+\}/s);
expect(errorRuleMatch).not.toBeNull();
expect(errorRuleMatch![0]).toContain("max-height: none");
expect(errorRuleMatch![0]).toContain("-webkit-overflow-scrolling: touch");
expect(errorRuleMatch![0]).toContain("overscroll-behavior: contain");
});
it("NewTaskModal: quick fields buttons meet 36px touch target on mobile", () => {

View File

@@ -599,6 +599,127 @@ describe("SelfHealingManager", () => {
managerWithAgents.stop();
});
it("suppresses stale worktree missing-module durable errors from transient auto-restart", async () => {
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
const warn = getSelfHealingLogger().warn;
warn.mockClear();
const now = Date.now();
const missingPath = "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js";
const agentStore = createMockAgentStore([
{
id: "agent-stale-path",
state: "error",
lastError:
`Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${missingPath}' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts`,
updatedAt: new Date(now - 120_000).toISOString(),
} as Agent,
]);
const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
const result = await managerWithAgents.recoverOrphanedAgents();
expect(result).toBe(0);
expect(agentStore.updateAgentState).not.toHaveBeenCalled();
expect(agentStore.updateAgent).toHaveBeenCalledWith(
"agent-stale-path",
expect.objectContaining({
metadata: expect.objectContaining({
durableErrorRecovery: expect.objectContaining({
lastReason: "stale-path-module-resolution",
lastMissingModulePath: missingPath,
consecutiveMissingModulePathCount: 1,
}),
}),
}),
);
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(
expect.stringContaining("Suppressed durable-agent auto-restart for agent-stale-path: stale module-resolution"),
);
managerWithAgents.stop();
});
it("emits stronger stale-process hint when same missing-module path repeats 3 times", async () => {
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
const warn = getSelfHealingLogger().warn;
warn.mockClear();
const now = Date.now();
const missingPath = "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js";
const agentStore = createMockAgentStore([
{
id: "agent-stale-repeat",
state: "error",
lastError:
`Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${missingPath}' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts`,
updatedAt: new Date(now - 120_000).toISOString(),
metadata: {
durableErrorRecovery: {
lastMissingModulePath: missingPath,
consecutiveMissingModulePathCount: 2,
},
},
} as unknown as Agent,
]);
const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
const result = await managerWithAgents.recoverOrphanedAgents();
expect(result).toBe(0);
expect(agentStore.updateAgent).toHaveBeenCalledWith(
"agent-stale-repeat",
expect.objectContaining({
metadata: expect.objectContaining({
durableErrorRecovery: expect.objectContaining({
lastMissingModulePath: missingPath,
consecutiveMissingModulePathCount: 3,
}),
}),
}),
);
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(
expect.stringContaining("FN-4013 tracks systemic prevention"),
);
managerWithAgents.stop();
});
it("resets stale missing-module consecutive count when a different path appears", async () => {
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
const now = Date.now();
const oldPath = "/Users/me/Projects/kb/.worktrees/deleted-a/node_modules/@runfusion/fusion/dist/bin.js";
const newPath = "/Users/me/Projects/kb/.worktrees/deleted-b/node_modules/@runfusion/fusion/dist/bin.js";
const agentStore = createMockAgentStore([
{
id: "agent-stale-reset",
state: "error",
lastError:
`Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${newPath}' imported from /Users/me/Projects/kb/.worktrees/deleted-b/packages/engine/src/pi.ts`,
updatedAt: new Date(now - 120_000).toISOString(),
metadata: {
durableErrorRecovery: {
lastMissingModulePath: oldPath,
consecutiveMissingModulePathCount: 2,
},
},
} as unknown as Agent,
]);
const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
const result = await managerWithAgents.recoverOrphanedAgents();
expect(result).toBe(0);
expect(agentStore.updateAgent).toHaveBeenCalledWith(
"agent-stale-reset",
expect.objectContaining({
metadata: expect.objectContaining({
durableErrorRecovery: expect.objectContaining({
lastMissingModulePath: newPath,
consecutiveMissingModulePathCount: 1,
}),
}),
}),
);
managerWithAgents.stop();
});
it("suppresses transient recovery while cooldown is active", async () => {
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
const now = Date.now();

View File

@@ -3,7 +3,9 @@ import {
isTransientError,
classifyError,
isSilentTransientError,
extractMissingModulePath,
isOperatorActionableAgentError,
isStaleWorktreeModuleResolutionError,
TRANSIENT_ERROR_PATTERNS,
} from "../transient-error-detector.js";
import { isUsageLimitError } from "../usage-limit-detector.js";
@@ -239,6 +241,34 @@ describe("Transient Error Detector", () => {
});
});
describe("isStaleWorktreeModuleResolutionError", () => {
it("returns true for cannot-find-module node_modules imported-from stale worktree signature", () => {
const message =
"Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts";
expect(isStaleWorktreeModuleResolutionError(message)).toBe(true);
});
it("returns false for other missing-module errors without stale-path signature", () => {
expect(isStaleWorktreeModuleResolutionError("Cannot find module 'vitest'")).toBe(false);
expect(isStaleWorktreeModuleResolutionError("socket hang up")).toBe(false);
});
});
describe("extractMissingModulePath", () => {
it("extracts the missing node_modules path from stale signature", () => {
const message =
"Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts";
expect(extractMissingModulePath(message)).toBe(
"/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js",
);
});
it("returns null when no stale module path is present", () => {
expect(extractMissingModulePath("Cannot find module 'vitest'")).toBeNull();
expect(extractMissingModulePath("socket hang up")).toBeNull();
});
});
describe("isOperatorActionableAgentError", () => {
it("returns true for credential/model/billing errors", () => {
expect(isOperatorActionableAgentError("invalid api key")).toBe(true);

View File

@@ -22,7 +22,7 @@ import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js";
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
import { classifyError, isOperatorActionableAgentError } from "./transient-error-detector.js";
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
const log = createLogger("self-healing");
const execAsync = promisify(exec);
@@ -2192,20 +2192,28 @@ export class SelfHealingManager {
attempts: number;
nextRetryAt?: string;
exhausted?: boolean;
lastMissingModulePath?: string;
consecutiveMissingModulePathCount: number;
} {
const metadata = agent.metadata ?? {};
const raw = metadata.durableErrorRecovery;
if (!raw || typeof raw !== "object") {
return { attempts: 0 };
return { attempts: 0, consecutiveMissingModulePathCount: 0 };
}
const record = raw as Record<string, unknown>;
const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts)
? Math.max(0, Math.floor(record.attempts))
: 0;
const consecutiveMissingModulePathCount =
typeof record.consecutiveMissingModulePathCount === "number" && Number.isFinite(record.consecutiveMissingModulePathCount)
? Math.max(0, Math.floor(record.consecutiveMissingModulePathCount))
: 0;
return {
attempts,
nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined,
exhausted: record.exhausted === true,
lastMissingModulePath: typeof record.lastMissingModulePath === "string" ? record.lastMissingModulePath : undefined,
consecutiveMissingModulePathCount,
};
}
@@ -2257,7 +2265,7 @@ export class SelfHealingManager {
if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
return false;
}
if (classifyError(agent.lastError ?? "") !== "transient") {
if (classifyError(agent.lastError ?? "") !== "transient" && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) {
return false;
}
if (isOperatorActionableAgentError(agent.lastError ?? "")) {
@@ -2291,6 +2299,35 @@ export class SelfHealingManager {
try {
if (agent.state === "error") {
const recoveryState = this.getDurableAgentRecoveryState(agent);
const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? "");
if (isStaleMissingModule) {
const missingModulePath = extractMissingModulePath(agent.lastError ?? "");
const repeatedPath =
missingModulePath && recoveryState.lastMissingModulePath === missingModulePath
? recoveryState.consecutiveMissingModulePathCount + 1
: 1;
await agentStore.updateAgent(agent.id, {
metadata: {
...(agent.metadata ?? {}),
durableErrorRecovery: {
attempts: recoveryState.attempts,
nextRetryAt: recoveryState.nextRetryAt,
exhausted: recoveryState.exhausted,
lastReason: "stale-path-module-resolution",
lastMissingModulePath: missingModulePath ?? recoveryState.lastMissingModulePath,
consecutiveMissingModulePathCount: repeatedPath,
lastObservedAt: new Date().toISOString(),
},
},
});
log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: stale module-resolution failure indicates stale host process/worktree path`);
if (missingModulePath && repeatedPath >= 3) {
log.warn(
`Durable agent ${agent.id} repeated missing-module path ${repeatedPath} times (${missingModulePath}). Hosting dashboard/engine process is likely stale (for example, zombie process from a deleted worktree); clean up stale process/worktree. FN-4013 tracks systemic prevention.`,
);
}
continue;
}
const nextAttempts = recoveryState.attempts + 1;
const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES;
const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString();
@@ -2303,6 +2340,8 @@ export class SelfHealingManager {
nextRetryAt,
exhausted,
lastReason: exhausted ? "retry-budget-exhausted" : "transient-error",
lastMissingModulePath: undefined,
consecutiveMissingModulePathCount: 0,
},
},
});

View File

@@ -170,6 +170,27 @@ export function classifyError(errorMessage: string): "transient" | "usage-limit"
return "permanent";
}
const STALE_WORKTREE_MODULE_RESOLUTION_PATTERN = /Cannot find module\s+['"][^'"]*node_modules[^'"]*['"][\s\S]*imported from\s+/i;
const STALE_WORKTREE_MODULE_PATH_PATTERN = /Cannot find module\s+['"]([^'"]*node_modules[^'"]*)['"]/i;
export function isStaleWorktreeModuleResolutionError(errorMessage: string): boolean {
if (!errorMessage || typeof errorMessage !== "string") {
return false;
}
return STALE_WORKTREE_MODULE_RESOLUTION_PATTERN.test(errorMessage);
}
export function extractMissingModulePath(errorMessage: string): string | null {
if (!errorMessage || typeof errorMessage !== "string") {
return null;
}
const match = errorMessage.match(STALE_WORKTREE_MODULE_PATH_PATTERN);
if (!match?.[1]) {
return null;
}
return match[1];
}
const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [
/invalid api key/i,
/authentication failed/i,