feat(FN-4015): classify and recover from stale module-resolution incidents
Merges FN-4015 to add stale module-resolution incident detection and self-healing in the engine (`transient-error-detector.ts`, `self-healing.ts`) with mobile regression coverage for the shared `AgentErrorDetailsModal` layout. Fusion-Task-Id: FN-4015
This commit is contained in:
@@ -31,6 +31,7 @@
|
|||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
display: flex;
|
||||||
}
|
}
|
||||||
|
|
||||||
.agent-error-modal__error {
|
.agent-error-modal__error {
|
||||||
@@ -52,15 +53,14 @@
|
|||||||
.agent-error-modal {
|
.agent-error-modal {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
max-width: 100%;
|
max-width: 100%;
|
||||||
height: 100vh;
|
height: 100%;
|
||||||
height: 100dvh;
|
max-height: 100%;
|
||||||
max-height: 100vh;
|
|
||||||
max-height: 100dvh;
|
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
|
min-height: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.agent-error-modal__content {
|
.agent-error-modal__content {
|
||||||
display: flex;
|
flex: 1 1 auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
.agent-error-modal__error {
|
.agent-error-modal__error {
|
||||||
@@ -69,5 +69,6 @@
|
|||||||
min-height: 0;
|
min-height: 0;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
-webkit-overflow-scrolling: touch;
|
-webkit-overflow-scrolling: touch;
|
||||||
|
overscroll-behavior: contain;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
|
||||||
|
import { render, screen, waitFor } from "@testing-library/react";
|
||||||
|
import userEvent from "@testing-library/user-event";
|
||||||
|
import { AgentErrorDetailsModal, AgentErrorIndicator } from "../AgentErrorDetailsModal";
|
||||||
|
|
||||||
|
const issueContext = {
|
||||||
|
surface: "AgentsView",
|
||||||
|
agentId: "agent-1",
|
||||||
|
agentName: "Test Agent",
|
||||||
|
agentState: "error",
|
||||||
|
runId: "run-1",
|
||||||
|
taskId: "FN-1",
|
||||||
|
timestamp: "2026-01-01T00:00:00.000Z",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("AgentErrorDetailsModal", () => {
|
||||||
|
const originalClipboard = navigator.clipboard;
|
||||||
|
const openSpy = vi.spyOn(window, "open").mockImplementation(() => null);
|
||||||
|
beforeEach(() => {
|
||||||
|
Object.defineProperty(navigator, "clipboard", {
|
||||||
|
value: { writeText: vi.fn().mockResolvedValue(undefined) },
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
openSpy.mockClear();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
Object.defineProperty(navigator, "clipboard", { value: originalClipboard, configurable: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not render when closed", () => {
|
||||||
|
const { container } = render(<AgentErrorDetailsModal open={false} onClose={vi.fn()} errorText="boom" issueContext={issueContext} />);
|
||||||
|
expect(container).toBeEmptyDOMElement();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders long error text in scrollable error region", () => {
|
||||||
|
render(<AgentErrorDetailsModal open={true} onClose={vi.fn()} errorText={"stderr\n".repeat(200)} issueContext={issueContext} />);
|
||||||
|
const errorRegion = document.querySelector(".agent-error-modal__error");
|
||||||
|
expect(errorRegion).toBeInTheDocument();
|
||||||
|
expect(errorRegion).toHaveTextContent("stderr");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("copies error text", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
render(<AgentErrorDetailsModal open={true} onClose={vi.fn()} errorText="copy me" issueContext={issueContext} />);
|
||||||
|
|
||||||
|
await user.click(screen.getByRole("button", { name: "Copy error to clipboard" }));
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(screen.getByRole("button", { name: "Copied error to clipboard" })).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("opens github report link", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
render(<AgentErrorDetailsModal open={true} onClose={vi.fn()} errorText="report me" issueContext={issueContext} />);
|
||||||
|
|
||||||
|
await user.click(screen.getByRole("link", { name: /report on github/i }));
|
||||||
|
expect(openSpy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(openSpy.mock.calls[0]?.[0]).toContain("https://github.com/Runfusion/Fusion/issues/new?");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("AgentErrorIndicator opens shared modal", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
render(<AgentErrorIndicator errorText="indicator error" issueContext={issueContext} />);
|
||||||
|
await user.click(screen.getByRole("button", { name: "Open error details" }));
|
||||||
|
expect(screen.getByRole("dialog", { name: "Agent error details" })).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -394,14 +394,15 @@ describe("agent modal mobile CSS structure", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("AgentErrorDetailsModal", () => {
|
describe("AgentErrorDetailsModal", () => {
|
||||||
it("mobile rules constrain modal to viewport height", () => {
|
it("mobile rules let modal fill the fullscreen container without double viewport clipping", () => {
|
||||||
const styles = readStyles();
|
const styles = readStyles();
|
||||||
const modalRuleMatch = styles.match(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal\s*\{[^}]+\}/);
|
const modalRuleMatch = styles.match(/@media\s*\(max-width:\s*768px\)\s*\{\s*\.agent-error-modal\s*\{[^}]+\}/);
|
||||||
expect(modalRuleMatch).toBeTruthy();
|
expect(modalRuleMatch).toBeTruthy();
|
||||||
const modalRule = modalRuleMatch![0];
|
const modalRule = modalRuleMatch![0];
|
||||||
|
|
||||||
expect(modalRule).toContain("height: 100dvh");
|
expect(modalRule).toContain("height: 100%");
|
||||||
expect(modalRule).toContain("max-height: 100dvh");
|
expect(modalRule).toContain("max-height: 100%");
|
||||||
|
expect(modalRule).toContain("min-height: 0");
|
||||||
expect(modalRule).toContain("width: 100%");
|
expect(modalRule).toContain("width: 100%");
|
||||||
expect(modalRule).toContain("max-width: 100%");
|
expect(modalRule).toContain("max-width: 100%");
|
||||||
});
|
});
|
||||||
@@ -414,6 +415,7 @@ describe("agent modal mobile CSS structure", () => {
|
|||||||
|
|
||||||
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*max-height:\s*none;[^}]*\}/);
|
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*max-height:\s*none;[^}]*\}/);
|
||||||
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*-webkit-overflow-scrolling:\s*touch;[^}]*\}/);
|
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*-webkit-overflow-scrolling:\s*touch;[^}]*\}/);
|
||||||
|
expect(styles).toMatch(/@media \(max-width: 768px\)[\s\S]*?\.agent-error-modal__error\s*\{[^}]*overscroll-behavior:\s*contain;[^}]*\}/);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -298,22 +298,26 @@ describe("core modals mobile css coverage", () => {
|
|||||||
expect(fullscreenBlockMatch![0]).toContain("max-height: unset");
|
expect(fullscreenBlockMatch![0]).toContain("max-height: unset");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("AgentErrorDetailsModal: mobile uses viewport-sized modal with inner scrolling log region", () => {
|
it("AgentErrorDetailsModal: mobile fills fullscreen container and keeps inner scrolling log region", () => {
|
||||||
const css = loadAllAppCss();
|
const css = loadAllAppCss();
|
||||||
const mobileBlock = getMainMobileBlock(css);
|
const mobileBlock = getMainMobileBlock(css);
|
||||||
|
|
||||||
const modalRuleMatch = mobileBlock.match(/\.agent-error-modal\s*\{[^}]+\}/s);
|
const modalRuleMatch = mobileBlock.match(/\.agent-error-modal\s*\{[^}]+\}/s);
|
||||||
expect(modalRuleMatch).not.toBeNull();
|
expect(modalRuleMatch).not.toBeNull();
|
||||||
expect(modalRuleMatch![0]).toContain("height: 100dvh");
|
expect(modalRuleMatch![0]).toContain("height: 100%");
|
||||||
expect(modalRuleMatch![0]).toContain("max-height: 100dvh");
|
expect(modalRuleMatch![0]).toContain("max-height: 100%");
|
||||||
|
expect(modalRuleMatch![0]).toContain("min-height: 0");
|
||||||
|
|
||||||
const contentRuleMatch = css.match(/\.agent-error-modal__content\s*\{[^}]+\}/s);
|
const contentRuleMatch = css.match(/\.agent-error-modal__content\s*\{[^}]+\}/s);
|
||||||
expect(contentRuleMatch).not.toBeNull();
|
expect(contentRuleMatch).not.toBeNull();
|
||||||
expect(contentRuleMatch![0]).toContain("overflow: hidden");
|
expect(contentRuleMatch![0]).toContain("overflow: hidden");
|
||||||
|
expect(contentRuleMatch![0]).toContain("display: flex");
|
||||||
|
|
||||||
const errorRuleMatch = mobileBlock.match(/\.agent-error-modal__error\s*\{[^}]+\}/s);
|
const errorRuleMatch = mobileBlock.match(/\.agent-error-modal__error\s*\{[^}]+\}/s);
|
||||||
expect(errorRuleMatch).not.toBeNull();
|
expect(errorRuleMatch).not.toBeNull();
|
||||||
|
expect(errorRuleMatch![0]).toContain("max-height: none");
|
||||||
expect(errorRuleMatch![0]).toContain("-webkit-overflow-scrolling: touch");
|
expect(errorRuleMatch![0]).toContain("-webkit-overflow-scrolling: touch");
|
||||||
|
expect(errorRuleMatch![0]).toContain("overscroll-behavior: contain");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("NewTaskModal: quick fields buttons meet 36px touch target on mobile", () => {
|
it("NewTaskModal: quick fields buttons meet 36px touch target on mobile", () => {
|
||||||
|
|||||||
@@ -599,6 +599,127 @@ describe("SelfHealingManager", () => {
|
|||||||
managerWithAgents.stop();
|
managerWithAgents.stop();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("suppresses stale worktree missing-module durable errors from transient auto-restart", async () => {
|
||||||
|
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
||||||
|
const warn = getSelfHealingLogger().warn;
|
||||||
|
warn.mockClear();
|
||||||
|
const now = Date.now();
|
||||||
|
const missingPath = "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js";
|
||||||
|
const agentStore = createMockAgentStore([
|
||||||
|
{
|
||||||
|
id: "agent-stale-path",
|
||||||
|
state: "error",
|
||||||
|
lastError:
|
||||||
|
`Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${missingPath}' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts`,
|
||||||
|
updatedAt: new Date(now - 120_000).toISOString(),
|
||||||
|
} as Agent,
|
||||||
|
]);
|
||||||
|
const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
|
||||||
|
|
||||||
|
const result = await managerWithAgents.recoverOrphanedAgents();
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(agentStore.updateAgentState).not.toHaveBeenCalled();
|
||||||
|
expect(agentStore.updateAgent).toHaveBeenCalledWith(
|
||||||
|
"agent-stale-path",
|
||||||
|
expect.objectContaining({
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
durableErrorRecovery: expect.objectContaining({
|
||||||
|
lastReason: "stale-path-module-resolution",
|
||||||
|
lastMissingModulePath: missingPath,
|
||||||
|
consecutiveMissingModulePathCount: 1,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining("Suppressed durable-agent auto-restart for agent-stale-path: stale module-resolution"),
|
||||||
|
);
|
||||||
|
managerWithAgents.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits stronger stale-process hint when same missing-module path repeats 3 times", async () => {
|
||||||
|
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
||||||
|
const warn = getSelfHealingLogger().warn;
|
||||||
|
warn.mockClear();
|
||||||
|
const now = Date.now();
|
||||||
|
const missingPath = "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js";
|
||||||
|
const agentStore = createMockAgentStore([
|
||||||
|
{
|
||||||
|
id: "agent-stale-repeat",
|
||||||
|
state: "error",
|
||||||
|
lastError:
|
||||||
|
`Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${missingPath}' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts`,
|
||||||
|
updatedAt: new Date(now - 120_000).toISOString(),
|
||||||
|
metadata: {
|
||||||
|
durableErrorRecovery: {
|
||||||
|
lastMissingModulePath: missingPath,
|
||||||
|
consecutiveMissingModulePathCount: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as unknown as Agent,
|
||||||
|
]);
|
||||||
|
const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
|
||||||
|
|
||||||
|
const result = await managerWithAgents.recoverOrphanedAgents();
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(agentStore.updateAgent).toHaveBeenCalledWith(
|
||||||
|
"agent-stale-repeat",
|
||||||
|
expect.objectContaining({
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
durableErrorRecovery: expect.objectContaining({
|
||||||
|
lastMissingModulePath: missingPath,
|
||||||
|
consecutiveMissingModulePathCount: 3,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining("FN-4013 tracks systemic prevention"),
|
||||||
|
);
|
||||||
|
managerWithAgents.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resets stale missing-module consecutive count when a different path appears", async () => {
|
||||||
|
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
||||||
|
const now = Date.now();
|
||||||
|
const oldPath = "/Users/me/Projects/kb/.worktrees/deleted-a/node_modules/@runfusion/fusion/dist/bin.js";
|
||||||
|
const newPath = "/Users/me/Projects/kb/.worktrees/deleted-b/node_modules/@runfusion/fusion/dist/bin.js";
|
||||||
|
const agentStore = createMockAgentStore([
|
||||||
|
{
|
||||||
|
id: "agent-stale-reset",
|
||||||
|
state: "error",
|
||||||
|
lastError:
|
||||||
|
`Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${newPath}' imported from /Users/me/Projects/kb/.worktrees/deleted-b/packages/engine/src/pi.ts`,
|
||||||
|
updatedAt: new Date(now - 120_000).toISOString(),
|
||||||
|
metadata: {
|
||||||
|
durableErrorRecovery: {
|
||||||
|
lastMissingModulePath: oldPath,
|
||||||
|
consecutiveMissingModulePathCount: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as unknown as Agent,
|
||||||
|
]);
|
||||||
|
const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
|
||||||
|
|
||||||
|
const result = await managerWithAgents.recoverOrphanedAgents();
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(agentStore.updateAgent).toHaveBeenCalledWith(
|
||||||
|
"agent-stale-reset",
|
||||||
|
expect.objectContaining({
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
durableErrorRecovery: expect.objectContaining({
|
||||||
|
lastMissingModulePath: newPath,
|
||||||
|
consecutiveMissingModulePathCount: 1,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
managerWithAgents.stop();
|
||||||
|
});
|
||||||
|
|
||||||
it("suppresses transient recovery while cooldown is active", async () => {
|
it("suppresses transient recovery while cooldown is active", async () => {
|
||||||
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|||||||
@@ -3,7 +3,9 @@ import {
|
|||||||
isTransientError,
|
isTransientError,
|
||||||
classifyError,
|
classifyError,
|
||||||
isSilentTransientError,
|
isSilentTransientError,
|
||||||
|
extractMissingModulePath,
|
||||||
isOperatorActionableAgentError,
|
isOperatorActionableAgentError,
|
||||||
|
isStaleWorktreeModuleResolutionError,
|
||||||
TRANSIENT_ERROR_PATTERNS,
|
TRANSIENT_ERROR_PATTERNS,
|
||||||
} from "../transient-error-detector.js";
|
} from "../transient-error-detector.js";
|
||||||
import { isUsageLimitError } from "../usage-limit-detector.js";
|
import { isUsageLimitError } from "../usage-limit-detector.js";
|
||||||
@@ -239,6 +241,34 @@ describe("Transient Error Detector", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("isStaleWorktreeModuleResolutionError", () => {
|
||||||
|
it("returns true for cannot-find-module node_modules imported-from stale worktree signature", () => {
|
||||||
|
const message =
|
||||||
|
"Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts";
|
||||||
|
expect(isStaleWorktreeModuleResolutionError(message)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns false for other missing-module errors without stale-path signature", () => {
|
||||||
|
expect(isStaleWorktreeModuleResolutionError("Cannot find module 'vitest'")).toBe(false);
|
||||||
|
expect(isStaleWorktreeModuleResolutionError("socket hang up")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("extractMissingModulePath", () => {
|
||||||
|
it("extracts the missing node_modules path from stale signature", () => {
|
||||||
|
const message =
|
||||||
|
"Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts";
|
||||||
|
expect(extractMissingModulePath(message)).toBe(
|
||||||
|
"/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null when no stale module path is present", () => {
|
||||||
|
expect(extractMissingModulePath("Cannot find module 'vitest'")).toBeNull();
|
||||||
|
expect(extractMissingModulePath("socket hang up")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("isOperatorActionableAgentError", () => {
|
describe("isOperatorActionableAgentError", () => {
|
||||||
it("returns true for credential/model/billing errors", () => {
|
it("returns true for credential/model/billing errors", () => {
|
||||||
expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
|
expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
|||||||
import { createLogger } from "./logger.js";
|
import { createLogger } from "./logger.js";
|
||||||
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||||
import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
|
import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
|
||||||
import { classifyError, isOperatorActionableAgentError } from "./transient-error-detector.js";
|
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
|
||||||
|
|
||||||
const log = createLogger("self-healing");
|
const log = createLogger("self-healing");
|
||||||
const execAsync = promisify(exec);
|
const execAsync = promisify(exec);
|
||||||
@@ -2192,20 +2192,28 @@ export class SelfHealingManager {
|
|||||||
attempts: number;
|
attempts: number;
|
||||||
nextRetryAt?: string;
|
nextRetryAt?: string;
|
||||||
exhausted?: boolean;
|
exhausted?: boolean;
|
||||||
|
lastMissingModulePath?: string;
|
||||||
|
consecutiveMissingModulePathCount: number;
|
||||||
} {
|
} {
|
||||||
const metadata = agent.metadata ?? {};
|
const metadata = agent.metadata ?? {};
|
||||||
const raw = metadata.durableErrorRecovery;
|
const raw = metadata.durableErrorRecovery;
|
||||||
if (!raw || typeof raw !== "object") {
|
if (!raw || typeof raw !== "object") {
|
||||||
return { attempts: 0 };
|
return { attempts: 0, consecutiveMissingModulePathCount: 0 };
|
||||||
}
|
}
|
||||||
const record = raw as Record<string, unknown>;
|
const record = raw as Record<string, unknown>;
|
||||||
const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts)
|
const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts)
|
||||||
? Math.max(0, Math.floor(record.attempts))
|
? Math.max(0, Math.floor(record.attempts))
|
||||||
: 0;
|
: 0;
|
||||||
|
const consecutiveMissingModulePathCount =
|
||||||
|
typeof record.consecutiveMissingModulePathCount === "number" && Number.isFinite(record.consecutiveMissingModulePathCount)
|
||||||
|
? Math.max(0, Math.floor(record.consecutiveMissingModulePathCount))
|
||||||
|
: 0;
|
||||||
return {
|
return {
|
||||||
attempts,
|
attempts,
|
||||||
nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined,
|
nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined,
|
||||||
exhausted: record.exhausted === true,
|
exhausted: record.exhausted === true,
|
||||||
|
lastMissingModulePath: typeof record.lastMissingModulePath === "string" ? record.lastMissingModulePath : undefined,
|
||||||
|
consecutiveMissingModulePathCount,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2257,7 +2265,7 @@ export class SelfHealingManager {
|
|||||||
if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
|
if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (classifyError(agent.lastError ?? "") !== "transient") {
|
if (classifyError(agent.lastError ?? "") !== "transient" && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (isOperatorActionableAgentError(agent.lastError ?? "")) {
|
if (isOperatorActionableAgentError(agent.lastError ?? "")) {
|
||||||
@@ -2291,6 +2299,35 @@ export class SelfHealingManager {
|
|||||||
try {
|
try {
|
||||||
if (agent.state === "error") {
|
if (agent.state === "error") {
|
||||||
const recoveryState = this.getDurableAgentRecoveryState(agent);
|
const recoveryState = this.getDurableAgentRecoveryState(agent);
|
||||||
|
const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? "");
|
||||||
|
if (isStaleMissingModule) {
|
||||||
|
const missingModulePath = extractMissingModulePath(agent.lastError ?? "");
|
||||||
|
const repeatedPath =
|
||||||
|
missingModulePath && recoveryState.lastMissingModulePath === missingModulePath
|
||||||
|
? recoveryState.consecutiveMissingModulePathCount + 1
|
||||||
|
: 1;
|
||||||
|
await agentStore.updateAgent(agent.id, {
|
||||||
|
metadata: {
|
||||||
|
...(agent.metadata ?? {}),
|
||||||
|
durableErrorRecovery: {
|
||||||
|
attempts: recoveryState.attempts,
|
||||||
|
nextRetryAt: recoveryState.nextRetryAt,
|
||||||
|
exhausted: recoveryState.exhausted,
|
||||||
|
lastReason: "stale-path-module-resolution",
|
||||||
|
lastMissingModulePath: missingModulePath ?? recoveryState.lastMissingModulePath,
|
||||||
|
consecutiveMissingModulePathCount: repeatedPath,
|
||||||
|
lastObservedAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: stale module-resolution failure indicates stale host process/worktree path`);
|
||||||
|
if (missingModulePath && repeatedPath >= 3) {
|
||||||
|
log.warn(
|
||||||
|
`Durable agent ${agent.id} repeated missing-module path ${repeatedPath} times (${missingModulePath}). Hosting dashboard/engine process is likely stale (for example, zombie process from a deleted worktree); clean up stale process/worktree. FN-4013 tracks systemic prevention.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const nextAttempts = recoveryState.attempts + 1;
|
const nextAttempts = recoveryState.attempts + 1;
|
||||||
const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES;
|
const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES;
|
||||||
const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString();
|
const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString();
|
||||||
@@ -2303,6 +2340,8 @@ export class SelfHealingManager {
|
|||||||
nextRetryAt,
|
nextRetryAt,
|
||||||
exhausted,
|
exhausted,
|
||||||
lastReason: exhausted ? "retry-budget-exhausted" : "transient-error",
|
lastReason: exhausted ? "retry-budget-exhausted" : "transient-error",
|
||||||
|
lastMissingModulePath: undefined,
|
||||||
|
consecutiveMissingModulePathCount: 0,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -170,6 +170,27 @@ export function classifyError(errorMessage: string): "transient" | "usage-limit"
|
|||||||
return "permanent";
|
return "permanent";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const STALE_WORKTREE_MODULE_RESOLUTION_PATTERN = /Cannot find module\s+['"][^'"]*node_modules[^'"]*['"][\s\S]*imported from\s+/i;
|
||||||
|
const STALE_WORKTREE_MODULE_PATH_PATTERN = /Cannot find module\s+['"]([^'"]*node_modules[^'"]*)['"]/i;
|
||||||
|
|
||||||
|
export function isStaleWorktreeModuleResolutionError(errorMessage: string): boolean {
|
||||||
|
if (!errorMessage || typeof errorMessage !== "string") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return STALE_WORKTREE_MODULE_RESOLUTION_PATTERN.test(errorMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractMissingModulePath(errorMessage: string): string | null {
|
||||||
|
if (!errorMessage || typeof errorMessage !== "string") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const match = errorMessage.match(STALE_WORKTREE_MODULE_PATH_PATTERN);
|
||||||
|
if (!match?.[1]) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return match[1];
|
||||||
|
}
|
||||||
|
|
||||||
const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [
|
const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [
|
||||||
/invalid api key/i,
|
/invalid api key/i,
|
||||||
/authentication failed/i,
|
/authentication failed/i,
|
||||||
|
|||||||
Reference in New Issue
Block a user