feat(FN-3452): document mesh lease recovery semantics

Documents mesh lease recovery semantics across the agents, architecture, and multi-project reference files, adding 32 lines of clarifying documentation to explain how mesh leases are recovered in the system.

Fusion-Task-Id: FN-3452

packages/engine/src/__tests__/mesh-lease-manager.test.ts

@@ -0,0 +1,85 @@
+import { describe, expect, it, vi } from "vitest";
+import type { AgentStore, Task, TaskStore } from "@fusion/core";
+import { MeshLeaseManager } from "../mesh-lease-manager.js";
+
+function task(overrides: Partial<Task> = {}): Task {
+  return {
+    id: "FN-1",
+    description: "x",
+    column: "in-progress",
+    dependencies: [],
+    steps: [],
+    currentStep: 0,
+    log: [],
+    createdAt: "2026-05-01T00:00:00.000Z",
+    updatedAt: "2026-05-01T00:00:00.000Z",
+    checkedOutBy: "agent-1",
+    checkedOutAt: "2026-05-01T00:00:00.000Z",
+    checkoutLeaseRenewedAt: "2026-05-01T00:00:00.000Z",
+    checkoutLeaseEpoch: 1,
+    checkoutNodeId: "node-a",
+    ...overrides,
+  };
+}
+
+describe("MeshLeaseManager", () => {
+  it("prefers active local execution over stale replicated timestamps", async () => {
+    const getTask = vi.fn().mockResolvedValue(task());
+    const manager = new MeshLeaseManager({
+      taskStore: { getTask } as unknown as TaskStore,
+      getExecutingTaskIds: () => new Set(["FN-1"]),
+    });
+
+    const result = await manager.isLeaseRecoverable(task(), Date.parse("2026-05-01T00:10:00.000Z"));
+    expect(result).toEqual({ recoverable: false, reason: "active_local_execution" });
+  });
+
+  it("marks lease recoverable when owner node is offline", async () => {
+    const manager = new MeshLeaseManager({
+      taskStore: {} as TaskStore,
+      nodeHealthMonitor: { getNodeHealth: () => "offline" } as any,
+    });
+
+    const result = await manager.isLeaseRecoverable(task(), Date.parse("2026-05-01T00:01:00.000Z"));
+    expect(result).toEqual({ recoverable: true, reason: "owner_node_offline" });
+  });
+
+  it("recovers stale lease by bumping epoch and clearing owner fields", async () => {
+    const currentTask = task({ checkoutLeaseRenewedAt: "2026-05-01T00:00:00.000Z" });
+    const updateTask = vi.fn().mockResolvedValue(currentTask);
+    const moveTask = vi.fn().mockResolvedValue(currentTask);
+    const logEntry = vi.fn().mockResolvedValue(undefined);
+    const taskStore = {
+      getTask: vi.fn().mockResolvedValue(currentTask),
+      updateTask,
+      moveTask,
+      logEntry,
+    } as unknown as TaskStore;
+
+    const agentStore = {
+      getAgent: vi.fn().mockResolvedValue({
+        id: "agent-1",
+        runtimeConfig: { heartbeatTimeoutMs: 60_000 },
+        lastHeartbeatAt: "2026-05-01T00:00:00.000Z",
+      }),
+    } as unknown as AgentStore;
+
+    const manager = new MeshLeaseManager({ taskStore, agentStore });
+    const ok = await manager.recoverAbandonedLease("FN-1", "stale-heartbeat");
+
+    expect(ok).toBe(true);
+    expect(updateTask).toHaveBeenCalledWith(
+      "FN-1",
+      expect.objectContaining({
+        checkedOutBy: null,
+        checkedOutAt: null,
+        checkoutNodeId: null,
+        checkoutRunId: null,
+        checkoutLeaseRenewedAt: null,
+        checkoutLeaseEpoch: 2,
+      }),
+      undefined,
+    );
+    expect(moveTask).toHaveBeenCalledWith("FN-1", "todo", expect.any(Object));
+  });
+});

packages/engine/src/executor.ts

@@ -595,6 +595,34 @@ export class TaskExecutor {
   /** Set of ephemeral spawned agent IDs with in-flight cleanup (prevents duplicate deletion attempts). */
   private pendingEphemeralDeletions = new Set<string>();
 
+  private async renewTaskLease(
+    taskId: string,
+    agentId: string,
+    leaseEpoch: number,
+    nodeId: string,
+    runId: string | undefined,
+  ): Promise<void> {
+    const renewedAt = new Date().toISOString();
+    if (this.options.agentStore) {
+      await this.options.agentStore.checkoutTask(
+        agentId,
+        taskId,
+        {
+          nodeId,
+          runId,
+          leaseEpoch,
+          renewedAt,
+        },
+        this.currentRunContext,
+      );
+      return;
+    }
+    await this.store.updateTask(taskId, {
+      checkoutRunId: runId ?? null,
+      checkoutLeaseRenewedAt: renewedAt,
+    });
+  }
+
   private async finalizeAlreadyReviewedTask(taskId: string): Promise<"merged" | "blocked" | "missing"> {
     const latestTask = await this.store.getTask(taskId);
     if (!latestTask || latestTask.column !== "in-review") {
@@ -3136,6 +3164,17 @@ export class TaskExecutor {
           lastAssignedAgentId: detail.assignedAgentId ?? null,
         });
 
+        let leaseRenewalTimer: ReturnType<typeof setInterval> | undefined;
+        if (detail.assignedAgentId && detail.checkedOutBy === detail.assignedAgentId) {
+          const leaseEpoch = detail.checkoutLeaseEpoch ?? 0;
+          const checkoutNodeId = detail.checkoutNodeId ?? detail.effectiveNodeId ?? detail.nodeId ?? "local";
+          const runId = this.currentRunContext?.runId;
+          await this.renewTaskLease(task.id, detail.assignedAgentId, leaseEpoch, checkoutNodeId, runId).catch(() => {});
+          leaseRenewalTimer = setInterval(() => {
+            void this.renewTaskLease(task.id, detail.assignedAgentId!, leaseEpoch, checkoutNodeId, runId).catch(() => {});
+          }, 30_000);
+        }
+
         // Register with stuck task detector for heartbeat monitoring
         stuckDetector?.trackTask(task.id, session);
         executorLog.log(`${task.id}: session registered (model=${describeModel(session)}, stuckDetector=${!!stuckDetector})`);
@@ -3559,6 +3598,9 @@ export class TaskExecutor {
             }
           }
         } finally {
+          if (leaseRenewalTimer) {
+            clearInterval(leaseRenewalTimer);
+          }
           this.activeSessions.delete(task.id);
           stuckDetector?.untrackTask(task.id);
           await agentLogger.flush();

packages/engine/src/index.ts

@@ -17,6 +17,7 @@ export { TriageProcessor, type TriageProcessorOptions } from "./triage.js";
 export { TaskExecutor, type TaskExecutorOptions } from "./executor.js";
 export { collectTaskEvaluationEvidence } from "./evaluator-evidence.js";
 export { Scheduler, type SchedulerOptions } from "./scheduler.js";
+export { MeshLeaseManager, type MeshLeaseManagerOptions, type LeaseRecoveryContext } from "./mesh-lease-manager.js";
 export { MissionAutopilot, type MissionAutopilotOptions } from "./mission-autopilot.js";
 export { MissionExecutionLoop, type MissionExecutionLoopOptions, type ValidationResult, loopLog } from "./mission-execution-loop.js";
 export { aiMergeTask, type MergerOptions } from "./merger.js";

packages/engine/src/mesh-lease-manager.ts

@@ -0,0 +1,107 @@
+import type { AgentStore, RunMutationContext, Task, TaskStore } from "@fusion/core";
+import type { NodeHealthMonitor } from "./node-health-monitor.js";
+
+export interface MeshLeaseManagerOptions {
+  taskStore: TaskStore;
+  agentStore?: AgentStore;
+  nodeHealthMonitor?: NodeHealthMonitor;
+  getExecutingTaskIds?: () => Set<string>;
+}
+
+export interface LeaseRecoveryContext {
+  runContext?: RunMutationContext;
+  preserveProgress?: boolean;
+}
+
+export class MeshLeaseManager {
+  constructor(private readonly options: MeshLeaseManagerOptions) {}
+
+  private staleThresholdMs(agentHeartbeatTimeoutMs?: number): number {
+    return Math.max((agentHeartbeatTimeoutMs ?? 60_000) * 2, 120_000);
+  }
+
+  async isLeaseRecoverable(task: Task, now = Date.now()): Promise<{ recoverable: boolean; reason?: string }> {
+    if (!task.checkedOutBy) {
+      return { recoverable: false, reason: "no_lease" };
+    }
+
+    if (this.options.getExecutingTaskIds?.().has(task.id)) {
+      return { recoverable: false, reason: "active_local_execution" };
+    }
+
+    if (task.checkoutNodeId && this.options.nodeHealthMonitor) {
+      const status = this.options.nodeHealthMonitor.getNodeHealth(task.checkoutNodeId);
+      if (status === "offline" || status === "error") {
+        return { recoverable: true, reason: `owner_node_${status}` };
+      }
+    }
+
+    const renewedAtIso = task.checkoutLeaseRenewedAt ?? task.checkedOutAt;
+    if (!renewedAtIso) {
+      return { recoverable: false, reason: "lease_never_renewed" };
+    }
+
+    let heartbeatTimeoutMs = 60_000;
+    let ownerLastHeartbeatAt: string | undefined;
+    if (this.options.agentStore && task.checkedOutBy) {
+      const owner = await this.options.agentStore.getAgent(task.checkedOutBy);
+      if (owner?.runtimeConfig && typeof owner.runtimeConfig.heartbeatTimeoutMs === "number") {
+        heartbeatTimeoutMs = owner.runtimeConfig.heartbeatTimeoutMs;
+      }
+      ownerLastHeartbeatAt = owner?.lastHeartbeatAt;
+    }
+
+    const staleMs = this.staleThresholdMs(heartbeatTimeoutMs);
+    const renewedAtMs = Date.parse(renewedAtIso);
+    if (!Number.isFinite(renewedAtMs) || now - renewedAtMs < staleMs) {
+      return { recoverable: false, reason: "lease_not_stale" };
+    }
+
+    if (!ownerLastHeartbeatAt) {
+      return { recoverable: true, reason: "owner_heartbeat_missing" };
+    }
+
+    const ownerHeartbeatMs = Date.parse(ownerLastHeartbeatAt);
+    if (!Number.isFinite(ownerHeartbeatMs) || now - ownerHeartbeatMs >= staleMs) {
+      return { recoverable: true, reason: "owner_heartbeat_stale" };
+    }
+
+    return { recoverable: false, reason: "owner_heartbeat_fresh" };
+  }
+
+  async recoverAbandonedLease(taskId: string, reason: string, context: LeaseRecoveryContext = {}): Promise<boolean> {
+    const task = await this.options.taskStore.getTask(taskId);
+    if (!task) return false;
+
+    const stale = await this.isLeaseRecoverable(task);
+    if (!stale.recoverable) {
+      return false;
+    }
+
+    const nextEpoch = (task.checkoutLeaseEpoch ?? 0) + 1;
+    await this.options.taskStore.updateTask(
+      taskId,
+      {
+        checkedOutBy: null,
+        checkedOutAt: null,
+        checkoutNodeId: null,
+        checkoutRunId: null,
+        checkoutLeaseRenewedAt: null,
+        checkoutLeaseEpoch: nextEpoch,
+      },
+      context.runContext,
+    );
+    await this.options.taskStore.logEntry(
+      taskId,
+      "Recovered abandoned lease",
+      `${reason} (${stale.reason ?? "stale"}); epoch=${nextEpoch}`,
+      context.runContext,
+    );
+    if (task.column !== "todo") {
+      await this.options.taskStore.moveTask(taskId, "todo", {
+        preserveProgress: context.preserveProgress ?? (task.currentStep > 0 || task.steps.some((step) => step.status !== "pending")),
+      });
+    }
+    return true;
+  }
+}

packages/engine/src/runtimes/in-process-runtime.ts

@@ -33,6 +33,7 @@ import { runtimeLog } from "../logger.js";
 import { StuckTaskDetector } from "../stuck-task-detector.js";
 import type { UsageLimitPauser } from "../usage-limit-detector.js";
 import { SelfHealingManager } from "../self-healing.js";
+import { MeshLeaseManager } from "../mesh-lease-manager.js";
 import { PluginRunner } from "../plugin-runner.js";
 import { MissionAutopilot } from "../mission-autopilot.js";
 import { MissionExecutionLoop } from "../mission-execution-loop.js";
@@ -91,6 +92,7 @@ export class InProcessRuntime
   private stuckTaskDetector?: StuckTaskDetector;
   private usageLimitPauser?: UsageLimitPauser;
   private selfHealingManager?: SelfHealingManager;
+  private leaseManager?: MeshLeaseManager;
   private agentStore?: AgentStore;
   private heartbeatMonitor?: HeartbeatMonitor;
   private triggerScheduler?: HeartbeatTriggerScheduler;
@@ -285,6 +287,12 @@ export class InProcessRuntime
           })
         : undefined;
 
+      this.leaseManager = new MeshLeaseManager({
+        taskStore: this.taskStore,
+        agentStore: this.agentStore,
+        getExecutingTaskIds: () => this.executor?.getExecutingTaskIds() ?? new Set<string>(),
+      });
+
       this.scheduler = new Scheduler(this.taskStore, {
         maxConcurrent: this.config.maxConcurrent,
         maxWorktrees: this.config.maxWorktrees,
@@ -292,6 +300,7 @@ export class InProcessRuntime
         missionStore,
         missionAutopilot,
         missionExecutionLoop,
+        leaseManager: this.leaseManager,
         onTaskFailed: (taskId) => {
           if (missionAutopilot) {
             void missionAutopilot.handleTaskFailure(taskId);
@@ -623,6 +632,7 @@ export class InProcessRuntime
         evictStaleTriageProcessing: () => this.triageProcessor?.evictStaleProcessing() ?? new Set<string>(),
         enqueueMerge: this.mergeEnqueuer ? (taskId: string) => this.mergeEnqueuer?.(taskId) : undefined,
         getActiveMergeTaskId: () => this.activeMergeTaskIdProvider?.() ?? null,
+        leaseManager: this.leaseManager,
       });
       this.selfHealingManager.start();
       this.stuckTaskDetector.start();

packages/engine/src/scheduler.ts

@@ -20,6 +20,7 @@ import { evaluateSpecStaleness, getPromptPath } from "./spec-staleness.js";
 import { resolveEffectiveNode } from "./effective-node.js";
 import { applyUnavailableNodePolicy } from "./node-routing-policy.js";
 import type { NodeDispatchValidationResult } from "./node-dispatch-validation.js";
+import type { MeshLeaseManager } from "./mesh-lease-manager.js";
 
 /**
  * Check whether two sets of file scope paths overlap.
@@ -117,6 +118,8 @@ export interface SchedulerOptions {
   prMonitor?: PrMonitor;
   /** Optional MissionStore for slice activation and auto-advance */
   missionStore?: MissionStore;
+  /** Optional lease manager used to recover stale checkout leases before scheduling. */
+  leaseManager?: MeshLeaseManager;
   /** Optional MissionAutopilot for autonomous mission progression */
   missionAutopilot?: import("./mission-autopilot.js").MissionAutopilot;
   /**
@@ -676,6 +679,18 @@ export class Scheduler {
       for (const taskId of ordered) {
         const task = tasks.find((t) => t.id === taskId)!;
 
+        if (task.checkedOutBy && this.options.leaseManager) {
+          const recovered = await this.options.leaseManager.recoverAbandonedLease(
+            task.id,
+            "scheduler detected stale todo lease",
+            { preserveProgress: true },
+          );
+          if (!recovered) {
+            await this.store.updateTask(task.id, { status: "queued" });
+            continue;
+          }
+        }
+
         // Check all deps are satisfied (done, in-review, or archived)
         const unmetDeps = task.dependencies.filter((depId) => {
           const dep = tasks.find((t) => t.id === depId);

packages/engine/src/self-healing.ts

@@ -18,6 +18,7 @@ import { promisify } from "node:util";
 import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
 import { isAbsolute, join, relative, resolve } from "node:path";
 import { getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
+import type { MeshLeaseManager } from "./mesh-lease-manager.js";
 import { createLogger } from "./logger.js";
 import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
 
@@ -29,6 +30,8 @@ export interface SelfHealingOptions {
   rootDir: string;
   /** Optional AgentStore for agent-level self-healing checks. */
   agentStore?: AgentStore;
+  /** Canonical stale-lease recovery manager. */
+  leaseManager?: MeshLeaseManager;
   /**
    * Callback to recover a completed task that is stuck in in-progress.
    * Called by the periodic maintenance cycle when it detects a task whose
@@ -1491,6 +1494,18 @@ export class SelfHealingManager {
             ? "worktree exists but no active session"
             : "missing worktree/session";
 
+          if (this.options.leaseManager && task.checkedOutBy) {
+            const leaseRecovered = await this.options.leaseManager.recoverAbandonedLease(
+              task.id,
+              `orphaned execution: ${reason}`,
+              { preserveProgress: true },
+            );
+            if (leaseRecovered) {
+              recovered++;
+              continue;
+            }
+          }
+
           // Reset steps whose work was never committed before clearing the worktree
           await this.resetStepsIfWorkLost(task);