feat(FN-4005): harden durable transient recovery auto-restart flow

- Add bounded auto-restart handling in in-process runtime for durable agents after transient failures - Extend self-healing recovery logic and transient error detection to classify and recover retryable runtime interruptions - Add regression coverage across heartbeat executor, self-healing, and transient detector test suites - Document durable agent transient recovery behavior in docs/agents.md Fusion-Task-Id: FN-4005
2026-05-11 09:17:08 -07:00
parent ef4aeb253a
commit dfdfa722ed
7 changed files with 330 additions and 7 deletions
--- a/packages/engine/src/tests/heartbeat-executor.test.ts
+++ b/packages/engine/src/tests/heartbeat-executor.test.ts
@@ -2465,6 +2465,39 @@ describe("executeHeartbeat", () => {
      });
    });

+    it("persists automation source recovery context on run records", async () => {
+      const store = createStoreWithAgentForExec();
+      const mockSession = createMockAgentSession();
+      mockedCreateFnAgent.mockResolvedValue({
+        session: mockSession as any,
+      });
+
+      const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
+
+      const result = await monitor.executeHeartbeat({
+        agentId: "agent-001",
+        source: "automation",
+        triggerDetail: "self-healing durable-agent transient recovery",
+        contextSnapshot: {
+          selfHealing: {
+            reason: "transient-error",
+            attempt: 1,
+            source: "durable-agent-transient-error-recovery",
+          },
+        },
+      });
+
+      expect(result.contextSnapshot).toEqual(
+        expect.objectContaining({
+          selfHealing: {
+            reason: "transient-error",
+            attempt: 1,
+            source: "durable-agent-transient-error-recovery",
+          },
+        }),
+      );
+    });
+
    it("records agent logs, context taskId, and stdoutExcerpt for successful runs", async () => {
      const store = createStoreWithAgentForExec();
      const appendAgentLog = vi.fn().mockResolvedValue(undefined);
--- a/packages/engine/src/tests/self-healing.test.ts
+++ b/packages/engine/src/tests/self-healing.test.ts
@@ -529,19 +529,43 @@ describe("SelfHealingManager", () => {
      managerWithAgents.stop();
    });

-    it("recovers orphaned agent in error state", async () => {
+    it("recovers orphaned agent in transient error state", async () => {
      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
      const now = Date.now();
      const agentStore = createMockAgentStore([
-        { id: "orphan-1", state: "error", updatedAt: new Date(now - 120_000).toISOString() } as Agent,
+        {
+          id: "orphan-1",
+          state: "error",
+          lastError: "socket hang up",
+          metadata: {},
+          updatedAt: new Date(now - 120_000).toISOString(),
+        } as Agent,
      ]);
-      const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
+      const restartDurableAgentHeartbeat = vi.fn().mockResolvedValue(true);
+      const managerWithAgents = new SelfHealingManager(store, {
+        rootDir: "/tmp/test-project",
+        agentStore,
+        restartDurableAgentHeartbeat,
+      });

      const result = await managerWithAgents.recoverOrphanedAgents();

      expect(result).toBe(1);
      expect(agentStore.updateAgentState).toHaveBeenCalledWith("orphan-1", "active");
-      expect(agentStore.updateAgent).toHaveBeenCalledWith("orphan-1", { lastError: undefined });
+      expect(agentStore.updateAgent).toHaveBeenLastCalledWith("orphan-1", { lastError: undefined });
+      expect(agentStore.updateAgent).toHaveBeenCalledWith(
+        "orphan-1",
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            durableErrorRecovery: expect.objectContaining({
+              attempts: 1,
+              exhausted: false,
+              lastReason: "transient-error",
+            }),
+          }),
+        }),
+      );
+      expect(restartDurableAgentHeartbeat).toHaveBeenCalledWith("orphan-1", { reason: "transient-error", attempt: 1 });
      managerWithAgents.stop();
    });

@@ -549,7 +573,7 @@ describe("SelfHealingManager", () => {
      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
      const now = Date.now();
      const agentStore = createMockAgentStore([
-        { id: "orphan-1", state: "error", updatedAt: new Date(now - 10_000).toISOString() } as Agent,
+        { id: "orphan-1", state: "error", lastError: "socket hang up", updatedAt: new Date(now - 10_000).toISOString() } as Agent,
      ]);
      const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });

@@ -560,6 +584,99 @@ describe("SelfHealingManager", () => {
      managerWithAgents.stop();
    });

+    it("skips non-transient/operator-actionable durable errors", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const now = Date.now();
+      const agentStore = createMockAgentStore([
+        { id: "agent-perm", state: "error", lastError: "invalid api key", updatedAt: new Date(now - 120_000).toISOString() } as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgentState).not.toHaveBeenCalled();
+      managerWithAgents.stop();
+    });
+
+    it("suppresses transient recovery while cooldown is active", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const now = Date.now();
+      const agentStore = createMockAgentStore([
+        {
+          id: "agent-cooldown",
+          state: "error",
+          lastError: "socket hang up",
+          updatedAt: new Date(now - 120_000).toISOString(),
+          metadata: { durableErrorRecovery: { attempts: 2, nextRetryAt: new Date(now + 5 * 60_000).toISOString() } },
+        } as unknown as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, {
+        rootDir: "/tmp/test-project",
+        agentStore,
+      });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgent).not.toHaveBeenCalled();
+      managerWithAgents.stop();
+    });
+
+    it("suppresses transient recovery when active agent execution is present", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const now = Date.now();
+      const agentStore = createMockAgentStore([
+        { id: "agent-active", state: "error", lastError: "socket hang up", updatedAt: new Date(now - 120_000).toISOString() } as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, {
+        rootDir: "/tmp/test-project",
+        agentStore,
+        hasActiveAgentExecution: (agentId) => agentId === "agent-active",
+      });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgentState).not.toHaveBeenCalled();
+      managerWithAgents.stop();
+    });
+
+    it("suppresses transient recovery when retry budget is exhausted", async () => {
+      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
+      const now = Date.now();
+      const agentStore = createMockAgentStore([
+        {
+          id: "agent-exhausted",
+          state: "error",
+          lastError: "socket hang up",
+          updatedAt: new Date(now - 120_000).toISOString(),
+          metadata: { durableErrorRecovery: { attempts: 4 } },
+        } as unknown as Agent,
+      ]);
+      const managerWithAgents = new SelfHealingManager(store, {
+        rootDir: "/tmp/test-project",
+        agentStore,
+      });
+
+      const result = await managerWithAgents.recoverOrphanedAgents();
+
+      expect(result).toBe(0);
+      expect(agentStore.updateAgentState).not.toHaveBeenCalled();
+      expect(agentStore.updateAgent).toHaveBeenCalledWith(
+        "agent-exhausted",
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            durableErrorRecovery: expect.objectContaining({
+              exhausted: true,
+              lastReason: "retry-budget-exhausted",
+            }),
+          }),
+        }),
+      );
+      managerWithAgents.stop();
+    });
+
    it("skips ephemeral agents", async () => {
      vi.mocked(store.getSettings).mockResolvedValue({ taskStuckTimeoutMs: 60_000 } as unknown as Settings);
      const now = Date.now();
--- a/packages/engine/src/tests/transient-error-detector.test.ts
+++ b/packages/engine/src/tests/transient-error-detector.test.ts
@@ -3,6 +3,7 @@ import {
  isTransientError,
  classifyError,
  isSilentTransientError,
+  isOperatorActionableAgentError,
  TRANSIENT_ERROR_PATTERNS,
 } from "../transient-error-detector.js";
 import { isUsageLimitError } from "../usage-limit-detector.js";
@@ -238,6 +239,21 @@ describe("Transient Error Detector", () => {
    });
  });

+  describe("isOperatorActionableAgentError", () => {
+    it("returns true for credential/model/billing errors", () => {
+      expect(isOperatorActionableAgentError("invalid api key")).toBe(true);
+      expect(isOperatorActionableAgentError("Authentication failed for provider")).toBe(true);
+      expect(isOperatorActionableAgentError("model gpt-x not found")).toBe(true);
+      expect(isOperatorActionableAgentError("missing OPENAI_API_KEY")).toBe(true);
+      expect(isOperatorActionableAgentError("billing issue: quota exceeded")).toBe(true);
+    });
+
+    it("returns false for transient network errors", () => {
+      expect(isOperatorActionableAgentError("socket hang up")).toBe(false);
+      expect(isOperatorActionableAgentError("upstream connect error")).toBe(false);
+    });
+  });
+
  describe("isSilentTransientError", () => {
    it("returns true for 'request was aborted'", () => {
      expect(isSilentTransientError("request was aborted")).toBe(true);
--- a/packages/engine/src/runtimes/in-process-runtime.ts
+++ b/packages/engine/src/runtimes/in-process-runtime.ts
@@ -635,6 +635,25 @@ export class InProcessRuntime
        enqueueMerge: this.mergeEnqueuer ? (taskId: string) => this.mergeEnqueuer?.(taskId) : undefined,
        getActiveMergeTaskId: () => this.activeMergeTaskIdProvider?.() ?? null,
        leaseManager: this.leaseManager,
+        hasActiveAgentExecution: (agentId: string) => this.heartbeatMonitor?.getTrackedAgents().includes(agentId) ?? false,
+        restartDurableAgentHeartbeat: async (agentId: string, context: { reason: string; attempt: number }) => {
+          if (!this.heartbeatMonitor) {
+            return false;
+          }
+          const run = await this.heartbeatMonitor.executeHeartbeat({
+            agentId,
+            source: "automation",
+            triggerDetail: `self-healing durable-agent transient recovery (${context.reason}, attempt ${context.attempt})`,
+            contextSnapshot: {
+              selfHealing: {
+                reason: context.reason,
+                attempt: context.attempt,
+                source: "durable-agent-transient-error-recovery",
+              },
+            },
+          });
+          return !!run;
+        },
      });
      this.selfHealingManager.start();
      this.stuckTaskDetector.start();
--- a/packages/engine/src/self-healing.ts
+++ b/packages/engine/src/self-healing.ts
@@ -22,6 +22,7 @@ import type { MeshLeaseManager } from "./mesh-lease-manager.js";
 import { createLogger } from "./logger.js";
 import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
 import { isRecoverableMissingWorktreeReviewFailure } from "./restart-recovery-coordinator.js";
+import { classifyError, isOperatorActionableAgentError } from "./transient-error-detector.js";

 const log = createLogger("self-healing");
 const execAsync = promisify(exec);
@@ -91,6 +92,8 @@ export interface SelfHealingOptions {
   * Used to avoid clearing a transient merge status mid-merge.
   */
  getActiveMergeTaskId?: () => string | null;
+  hasActiveAgentExecution?: (agentId: string) => boolean;
+  restartDurableAgentHeartbeat?: (agentId: string, context: { reason: string; attempt: number }) => Promise<boolean>;
 }

 const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
@@ -124,6 +127,9 @@ const MAX_TASK_DONE_RETRIES = 3;
 const MAX_AUTO_MERGE_RETRIES = 3;
 const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000;
 const DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS = 5 * 60_000;
+const DURABLE_ERROR_RECOVERY_MAX_RETRIES = 5;
+const DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS = 30_000;
+const DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS = 15 * 60_000;

 interface LandedTaskCommit {
  sha: string;
@@ -2182,6 +2188,33 @@ export class SelfHealingManager {
    }
  }

+  private getDurableAgentRecoveryState(agent: { metadata?: Record<string, unknown> | null }): {
+    attempts: number;
+    nextRetryAt?: string;
+    exhausted?: boolean;
+  } {
+    const metadata = agent.metadata ?? {};
+    const raw = metadata.durableErrorRecovery;
+    if (!raw || typeof raw !== "object") {
+      return { attempts: 0 };
+    }
+    const record = raw as Record<string, unknown>;
+    const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts)
+      ? Math.max(0, Math.floor(record.attempts))
+      : 0;
+    return {
+      attempts,
+      nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined,
+      exhausted: record.exhausted === true,
+    };
+  }
+
+  private computeDurableAgentRecoveryCooldownMs(attempts: number): number {
+    const clampedAttempts = Math.max(1, attempts);
+    const exponential = DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS * Math.pow(2, clampedAttempts - 1);
+    return Math.min(exponential, DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS);
+  }
+
  async recoverOrphanedAgents(): Promise<number> {
    const agentStore = this.options.agentStore;
    if (!agentStore) {
@@ -2212,10 +2245,39 @@ export class SelfHealingManager {
          return false;
        }
        const updatedAt = Date.parse(agent.updatedAt ?? "");
-        if (!Number.isFinite(updatedAt)) {
+        if (!Number.isFinite(updatedAt) || now - updatedAt < recoveryTimeoutMs) {
          return false;
        }
-        return now - updatedAt >= recoveryTimeoutMs;
+
+        if (agent.state === "error") {
+          const runtimeConfig = (agent.runtimeConfig ?? {}) as Record<string, unknown>;
+          if (runtimeConfig.enabled === false) {
+            return false;
+          }
+          if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
+            return false;
+          }
+          if (classifyError(agent.lastError ?? "") !== "transient") {
+            return false;
+          }
+          if (isOperatorActionableAgentError(agent.lastError ?? "")) {
+            return false;
+          }
+
+          const recoveryState = this.getDurableAgentRecoveryState(agent);
+          if (recoveryState.exhausted) {
+            return false;
+          }
+          if (recoveryState.nextRetryAt) {
+            const nextRetryMs = Date.parse(recoveryState.nextRetryAt);
+            if (Number.isFinite(nextRetryMs) && nextRetryMs > now) {
+              log.log(`Durable agent ${agent.id} transient recovery delayed until ${recoveryState.nextRetryAt}`);
+              return false;
+            }
+          }
+        }
+
+        return true;
      });

      if (orphaned.length === 0) {
@@ -2227,10 +2289,44 @@ export class SelfHealingManager {
        const updatedAt = Date.parse(agent.updatedAt ?? "");
        const stuckForMs = Math.max(0, now - updatedAt);
        try {
+          if (agent.state === "error") {
+            const recoveryState = this.getDurableAgentRecoveryState(agent);
+            const nextAttempts = recoveryState.attempts + 1;
+            const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES;
+            const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString();
+            await agentStore.updateAgent(agent.id, {
+              metadata: {
+                ...(agent.metadata ?? {}),
+                durableErrorRecovery: {
+                  attempts: nextAttempts,
+                  lastAttemptAt: new Date().toISOString(),
+                  nextRetryAt,
+                  exhausted,
+                  lastReason: exhausted ? "retry-budget-exhausted" : "transient-error",
+                },
+              },
+            });
+            if (exhausted) {
+              log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: retry budget exhausted`);
+              continue;
+            }
+          }
+
          await agentStore.updateAgentState(agent.id, "active");
          await agentStore.updateAgent(agent.id, {
            lastError: undefined,
          });
+
+          if (agent.state === "error" && this.options.restartDurableAgentHeartbeat) {
+            const restartOk = await this.options.restartDurableAgentHeartbeat(agent.id, {
+              reason: "transient-error",
+              attempt: this.getDurableAgentRecoveryState(agent).attempts + 1,
+            });
+            if (!restartOk) {
+              log.warn(`Durable-agent transient recovery heartbeat restart skipped for ${agent.id}`);
+            }
+          }
+
          log.log(
            `Auto-recovered: orphaned agent ${agent.id} stuck in ${agent.state} for ${Math.round(stuckForMs / 1000)}s — reset to active`,
          );
--- a/packages/engine/src/transient-error-detector.ts
+++ b/packages/engine/src/transient-error-detector.ts
@@ -169,3 +169,25 @@ export function classifyError(errorMessage: string): "transient" | "usage-limit"
  // Default to permanent (mark as failed)
  return "permanent";
 }
+
+const OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS: RegExp[] = [
+  /invalid api key/i,
+  /authentication failed/i,
+  /unauthorized/i,
+  /forbidden/i,
+  /insufficient permissions?/i,
+  /model .* not found/i,
+  /unknown model/i,
+  /no such model/i,
+  /credential/i,
+  /missing .*key/i,
+  /billing/i,
+  /quota exceeded/i,
+];
+
+export function isOperatorActionableAgentError(errorMessage: string): boolean {
+  if (!errorMessage || typeof errorMessage !== "string") {
+    return false;
+  }
+  return OPERATOR_ACTIONABLE_AGENT_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage));
+}