feat(FN-4015): classify and recover from stale module-resolution incidents

Merges FN-4015 to add stale module-resolution incident detection and self-healing in the engine (`transient-error-detector.ts`, `self-healing.ts`) with mobile regression coverage for the shared `AgentErrorDetailsModal` layout. Fusion-Task-Id: FN-4015
2026-05-11 11:59:07 -07:00
parent 0157e66ff9
commit 385c2dd58f
8 changed files with 301 additions and 15 deletions
--- a/packages/engine/src/tests/self-healing.test.ts
+++ b/packages/engine/src/tests/self-healing.test.ts
@@ -599,6 +599,127 @@ describe("SelfHealingManager", () => {
      managerWithAgents.stop();
    });

+    it("suppresses stale worktree missing-module durable errors from transient auto-restart", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const warn = getSelfHealingLogger().warn;
+      warn.mockClear();
+      const now = Date.now();
+      const missingPath = "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js";
+      const agentStore = createMockAgentStore([
+        {
+          id: "agent-stale-path",
+          state: "error",
+          lastError:
+            `Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${missingPath}' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts`,
+          updatedAt: new Date(now - 120_000).toISOString(),
+        } as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgentState).not.toHaveBeenCalled();
+      expect(agentStore.updateAgent).toHaveBeenCalledWith(
+        "agent-stale-path",
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            durableErrorRecovery: expect.objectContaining({
+              lastReason: "stale-path-module-resolution",
+              lastMissingModulePath: missingPath,
+              consecutiveMissingModulePathCount: 1,
+            }),
+          }),
+        }),
+      );
+      expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(
+        expect.stringContaining("Suppressed durable-agent auto-restart for agent-stale-path: stale module-resolution"),
+      );
+      managerWithAgents.stop();
+    });
+
+    it("emits stronger stale-process hint when same missing-module path repeats 3 times", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const warn = getSelfHealingLogger().warn;
+      warn.mockClear();
+      const now = Date.now();
+      const missingPath = "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js";
+      const agentStore = createMockAgentStore([
+        {
+          id: "agent-stale-repeat",
+          state: "error",
+          lastError:
+            `Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${missingPath}' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts`,
+          updatedAt: new Date(now - 120_000).toISOString(),
+          metadata: {
+            durableErrorRecovery: {
+              lastMissingModulePath: missingPath,
+              consecutiveMissingModulePathCount: 2,
+            },
+          },
+        } as unknown as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgent).toHaveBeenCalledWith(
+        "agent-stale-repeat",
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            durableErrorRecovery: expect.objectContaining({
+              lastMissingModulePath: missingPath,
+              consecutiveMissingModulePathCount: 3,
+            }),
+          }),
+        }),
+      );
+      expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(
+        expect.stringContaining("FN-4013 tracks systemic prevention"),
+      );
+      managerWithAgents.stop();
+    });
+
+    it("resets stale missing-module consecutive count when a different path appears", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const now = Date.now();
+      const oldPath = "/Users/me/Projects/kb/.worktrees/deleted-a/node_modules/@runfusion/fusion/dist/bin.js";
+      const newPath = "/Users/me/Projects/kb/.worktrees/deleted-b/node_modules/@runfusion/fusion/dist/bin.js";
+      const agentStore = createMockAgentStore([
+        {
+          id: "agent-stale-reset",
+          state: "error",
+          lastError:
+            `Error [ERR_MODULE_NOT_FOUND]: Cannot find module '${newPath}' imported from /Users/me/Projects/kb/.worktrees/deleted-b/packages/engine/src/pi.ts`,
+          updatedAt: new Date(now - 120_000).toISOString(),
+          metadata: {
+            durableErrorRecovery: {
+              lastMissingModulePath: oldPath,
+              consecutiveMissingModulePathCount: 2,
+            },
+          },
+        } as unknown as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgent).toHaveBeenCalledWith(
+        "agent-stale-reset",
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            durableErrorRecovery: expect.objectContaining({
+              lastMissingModulePath: newPath,
+              consecutiveMissingModulePathCount: 1,
+            }),
+          }),
+        }),
+      );
+      managerWithAgents.stop();
+    });
+
    it("suppresses transient recovery while cooldown is active", async () => {
      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
      const now = Date.now();
--- a/packages/engine/src/tests/transient-error-detector.test.ts
+++ b/packages/engine/src/tests/transient-error-detector.test.ts
@@ -3,7 +3,9 @@ import {
  isTransientError,
  classifyError,
  isSilentTransientError,
+  extractMissingModulePath,
  isOperatorActionableAgentError,
+  isStaleWorktreeModuleResolutionError,
  TRANSIENT_ERROR_PATTERNS,
 } from "../transient-error-detector.js";
 import { isUsageLimitError } from "../usage-limit-detector.js";
@@ -239,6 +241,34 @@ describe("Transient Error Detector", () => {
    });
  });

+  describe("isStaleWorktreeModuleResolutionError", () => {
+    it("returns true for cannot-find-module node_modules imported-from stale worktree signature", () => {
+      const message =
+        "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts";
+      expect(isStaleWorktreeModuleResolutionError(message)).toBe(true);
+    });
+
+    it("returns false for other missing-module errors without stale-path signature", () => {
+      expect(isStaleWorktreeModuleResolutionError("Cannot find module 'vitest'")).toBe(false);
+      expect(isStaleWorktreeModuleResolutionError("socket hang up")).toBe(false);
+    });
+  });
+
+  describe("extractMissingModulePath", () => {
+    it("extracts the missing node_modules path from stale signature", () => {
+      const message =
+        "Error [ERR_MODULE_NOT_FOUND]: Cannot find module '/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js' imported from /Users/me/Projects/kb/.worktrees/deleted/packages/engine/src/pi.ts";
+      expect(extractMissingModulePath(message)).toBe(
+        "/Users/me/Projects/kb/.worktrees/deleted/node_modules/@runfusion/fusion/dist/bin.js",
+      );
+    });
+
+    it("returns null when no stale module path is present", () => {
+      expect(extractMissingModulePath("Cannot find module 'vitest'")).toBeNull();
+      expect(extractMissingModulePath("socket hang up")).toBeNull();
+    });
+  });
+
  describe("isOperatorActionableAgentError", () => {
    it("returns true for credential/model/billing errors", () => {
      expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
--- a/packages/engine/src/self-healing.ts
+++ b/packages/engine/src/self-healing.ts
@@ -22,7 +22,7 @@ import type { MeshLeaseManager } from "./mesh-lease-manager.js";
 import { createLogger } from "./logger.js";
 import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
 import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
-import { classifyError, isOperatorActionableAgentError } from "./transient-error-detector.js";
+import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";

 const log = createLogger("self-healing");
 const execAsync = promisify(exec);
@@ -2192,20 +2192,28 @@ export class SelfHealingManager {
    attempts: number;
    nextRetryAt?: string;
    exhausted?: boolean;
+    lastMissingModulePath?: string;
+    consecutiveMissingModulePathCount: number;
  } {
    const metadata = agent.metadata ?? {};
    const raw = metadata.durableErrorRecovery;
    if (!raw || typeof raw !== "object") {
-      return { attempts: 0 };
+      return { attempts: 0, consecutiveMissingModulePathCount: 0 };
    }
    const record = raw as Record<string, unknown>;
    const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts)
      ? Math.max(0, Math.floor(record.attempts))
      : 0;
+    const consecutiveMissingModulePathCount =
+      typeof record.consecutiveMissingModulePathCount === "number" && Number.isFinite(record.consecutiveMissingModulePathCount)
+        ? Math.max(0, Math.floor(record.consecutiveMissingModulePathCount))
+        : 0;
    return {
      attempts,
      nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined,
      exhausted: record.exhausted === true,
+      lastMissingModulePath: typeof record.lastMissingModulePath === "string" ? record.lastMissingModulePath : undefined,
+      consecutiveMissingModulePathCount,
    };
  }

@@ -2257,7 +2265,7 @@ export class SelfHealingManager {
          if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
            return false;
          }
-          if (classifyError(agent.lastError ?? "") !== "transient") {
+          if (classifyError(agent.lastError ?? "") !== "transient" && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) {
            return false;
          }
          if (isOperatorActionableAgentError(agent.lastError ?? "")) {
@@ -2291,6 +2299,35 @@ export class SelfHealingManager {
        try {
          if (agent.state === "error") {
            const recoveryState = this.getDurableAgentRecoveryState(agent);
+            const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? "");
+            if (isStaleMissingModule) {
+              const missingModulePath = extractMissingModulePath(agent.lastError ?? "");
+              const repeatedPath =
+                missingModulePath && recoveryState.lastMissingModulePath === missingModulePath
+                  ? recoveryState.consecutiveMissingModulePathCount + 1
+                  : 1;
+              await agentStore.updateAgent(agent.id, {
+                metadata: {
+                  ...(agent.metadata ?? {}),
+                  durableErrorRecovery: {
+                    attempts: recoveryState.attempts,
+                    nextRetryAt: recoveryState.nextRetryAt,
+                    exhausted: recoveryState.exhausted,
+                    lastReason: "stale-path-module-resolution",
+                    lastMissingModulePath: missingModulePath ?? recoveryState.lastMissingModulePath,
+                    consecutiveMissingModulePathCount: repeatedPath,
+                    lastObservedAt: new Date().toISOString(),
+                  },
+                },
+              });
+              log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: stale module-resolution failure indicates stale host process/worktree path`);
+              if (missingModulePath && repeatedPath >= 3) {
+                log.warn(
+                  `Durable agent ${agent.id} repeated missing-module path ${repeatedPath} times (${missingModulePath}). Hosting dashboard/engine process is likely stale (for example, zombie process from a deleted worktree); clean up stale process/worktree. FN-4013 tracks systemic prevention.`,
+                );
+              }
+              continue;
+            }
            const nextAttempts = recoveryState.attempts + 1;
            const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES;
            const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString();
@@ -2303,6 +2340,8 @@ export class SelfHealingManager {
                  nextRetryAt,
                  exhausted,
                  lastReason: exhausted ? "retry-budget-exhausted" : "transient-error",
+                  lastMissingModulePath: undefined,
+                  consecutiveMissingModulePathCount: 0,
                },
              },
            });
--- a/packages/engine/src/transient-error-detector.ts
+++ b/packages/engine/src/transient-error-detector.ts
@@ -170,6 +170,27 @@ export function classifyError(errorMessage: string): "transient" | "usage-limit"
  return "permanent";
 }

+const STALE_WORKTREE_MODULE_RESOLUTION_PATTERN = /Cannot find module\s+['"][^'"]*node_modules[^'"]*['"][\s\S]*imported from\s+/i;
+const STALE_WORKTREE_MODULE_PATH_PATTERN = /Cannot find module\s+['"]([^'"]*node_modules[^'"]*)['"]/i;
+
+export function isStaleWorktreeModuleResolutionError(errorMessage: string): boolean {
+  if (!errorMessage || typeof errorMessage !== "string") {
+    return false;
+  }
+  return STALE_WORKTREE_MODULE_RESOLUTION_PATTERN.test(errorMessage);
+}
+
+export function extractMissingModulePath(errorMessage: string): string | null {
+  if (!errorMessage || typeof errorMessage !== "string") {
+    return null;
+  }
+  const match = errorMessage.match(STALE_WORKTREE_MODULE_PATH_PATTERN);
+  if (!match?.[1]) {
+    return null;
+  }
+  return match[1];
+}
+
 const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [
  /invalid api key/i,
  /authentication failed/i,